2024-03-04T07:14:23.170 INFO:root:teuthology version: 0.0.1.dev260+gb1dac55 2024-03-04T07:14:23.170 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679 --name leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi --owner scheduled_leonidus@teuthology --verbose --description fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/strays} -- /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/orig.config.yaml 2024-03-04T07:14:23.211 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-04T07:14:23.296 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679 branch: wip-lusov-qdb description: fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/strays} email: null first_in_suite: false job_id: '7580679' kernel: kdb: true sha1: distro last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi name: leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-lusov-qdb ceph: conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 pid file: '' rados mon op timeout: 900 rados osd op timeout: 900 mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon op complaint time: 120 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - MDS_UP_LESS_THAN_MAX - POOL_APP_NOT_ENABLED - overall HEALTH_ - Replacing daemon - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd subvolume_version: 2 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-lusov-qdb sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd owner: scheduled_leonidus@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.y - mds.a - mds.c - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - client.2 - - mon.b - mon.c - mgr.x - mds.b - mds.d - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - client.3 seed: 5072 sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd sleep_before_teardown: 0 subset: 246/10000 suite: fs suite_branch: wip-lusov-qdb suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi064.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFbHJeUCtklmTdRTeWvj7/TLDObNEjK0smR7zXProxs0CCq4kBiFvtnX0XZZZNjy0cajgLMmeYQq3V0ahWEsOmM= smithi066.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJgFwnT3fiITMHowq7yiXYhbbE9FmTZ3/ZH0dOJkclVFjVeeC6i38jqL7TbA0ccgfJevOf/HWCuKqgGfd4IXD+Y= tasks: - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - ceph: create_rbd_pool: false - ceph-fuse: null - cephfs_test_runner: modules: - tasks.cephfs.test_strays teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return false end - local function is_kdistro() return false end - local function is_fuse() return true end - local function syntax_version() return '' end teuthology_branch: main teuthology_sha1: b1dac5519c57c269ea98a22fb7729016a1d0e4d2 timestamp: 2024-03-04_07:05:03 tube: smithi user: leonidus verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-04T07:14:23.296 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa; will attempt to use it 2024-03-04T07:14:23.297 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks 2024-03-04T07:14:23.297 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-04T07:14:23.298 INFO:teuthology.task.internal:Checking packages... 2024-03-04T07:14:23.322 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd' 2024-03-04T07:14:23.323 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-04T07:14:23.323 INFO:teuthology.packaging:ref: None 2024-03-04T07:14:23.323 INFO:teuthology.packaging:tag: None 2024-03-04T07:14:23.323 INFO:teuthology.packaging:branch: wip-lusov-qdb 2024-03-04T07:14:23.323 INFO:teuthology.packaging:sha1: 1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-04T07:14:23.323 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-lusov-qdb 2024-03-04T07:14:23.468 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1739.g1bc250c7 2024-03-04T07:14:23.470 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-04T07:14:23.475 INFO:teuthology.task.internal:no buildpackages task found 2024-03-04T07:14:23.475 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-04T07:14:23.485 INFO:teuthology.task.internal:Saving configuration 2024-03-04T07:14:23.501 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-04T07:14:23.506 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-04T07:14:23.531 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-04 07:06:15.822678', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFbHJeUCtklmTdRTeWvj7/TLDObNEjK0smR7zXProxs0CCq4kBiFvtnX0XZZZNjy0cajgLMmeYQq3V0ahWEsOmM='} 2024-03-04T07:14:23.553 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi066.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-04 07:06:15.821259', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJgFwnT3fiITMHowq7yiXYhbbE9FmTZ3/ZH0dOJkclVFjVeeC6i38jqL7TbA0ccgfJevOf/HWCuKqgGfd4IXD+Y='} 2024-03-04T07:14:23.553 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-04T07:14:23.559 INFO:teuthology.task.internal:roles: ubuntu@smithi064.front.sepia.ceph.com - ['mon.a', 'mgr.y', 'mds.a', 'mds.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'client.2'] 2024-03-04T07:14:23.559 INFO:teuthology.task.internal:roles: ubuntu@smithi066.front.sepia.ceph.com - ['mon.b', 'mon.c', 'mgr.x', 'mds.b', 'mds.d', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'client.3'] 2024-03-04T07:14:23.559 INFO:teuthology.run_tasks:Running task console_log... 2024-03-04T07:14:23.628 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f28a46d4f70>, signals=[15]) 2024-03-04T07:14:23.628 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-04T07:14:23.634 INFO:teuthology.task.internal:Opening connections... 2024-03-04T07:14:23.634 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:14:23.636 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:14:23.746 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:14:23.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:14:23.848 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-04T07:14:23.853 DEBUG:teuthology.orchestra.run.smithi064:> uname -m 2024-03-04T07:14:23.871 INFO:teuthology.orchestra.run.smithi064.stdout:x86_64 2024-03-04T07:14:23.872 DEBUG:teuthology.orchestra.run.smithi064:> cat /etc/os-release 2024-03-04T07:14:23.929 INFO:teuthology.orchestra.run.smithi064.stdout:NAME="CentOS Stream" 2024-03-04T07:14:23.929 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION="9" 2024-03-04T07:14:23.929 INFO:teuthology.orchestra.run.smithi064.stdout:ID="centos" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:ID_LIKE="rhel fedora" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION_ID="9" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:PLATFORM_ID="platform:el9" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:ANSI_COLOR="0;31" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:LOGO="fedora-logo-icon" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:HOME_URL="https://centos.org/" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-04T07:14:23.930 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-04T07:14:23.931 INFO:teuthology.lock.ops:Updating smithi064.front.sepia.ceph.com on lock server 2024-03-04T07:14:23.955 DEBUG:teuthology.orchestra.run.smithi066:> uname -m 2024-03-04T07:14:23.973 INFO:teuthology.orchestra.run.smithi066.stdout:x86_64 2024-03-04T07:14:23.973 DEBUG:teuthology.orchestra.run.smithi066:> cat /etc/os-release 2024-03-04T07:14:24.029 INFO:teuthology.orchestra.run.smithi066.stdout:NAME="CentOS Stream" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:VERSION="9" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:ID="centos" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:ID_LIKE="rhel fedora" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:VERSION_ID="9" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:PLATFORM_ID="platform:el9" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:ANSI_COLOR="0;31" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:LOGO="fedora-logo-icon" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:HOME_URL="https://centos.org/" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-04T07:14:24.030 INFO:teuthology.orchestra.run.smithi066.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-04T07:14:24.031 INFO:teuthology.lock.ops:Updating smithi066.front.sepia.ceph.com on lock server 2024-03-04T07:14:24.047 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-04T07:14:24.058 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-04T07:14:24.069 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-04T07:14:24.069 DEBUG:teuthology.orchestra.run.smithi064:> test '!' -e /home/ubuntu/cephtest 2024-03-04T07:14:24.072 DEBUG:teuthology.orchestra.run.smithi066:> test '!' -e /home/ubuntu/cephtest 2024-03-04T07:14:24.088 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-04T07:14:24.093 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-04T07:14:24.094 DEBUG:teuthology.orchestra.run.smithi064:> test -z $(ls -A /var/lib/ceph) 2024-03-04T07:14:24.130 DEBUG:teuthology.orchestra.run.smithi066:> test -z $(ls -A /var/lib/ceph) 2024-03-04T07:14:24.153 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-04T07:14:24.212 INFO:teuthology.run_tasks:Running task kernel... 2024-03-04T07:14:24.226 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-04T07:14:24.226 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-04T07:14:24.226 DEBUG:teuthology.orchestra.run.smithi064:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-04T07:14:24.227 DEBUG:teuthology.orchestra.run.smithi066:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-04T07:14:24.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:14:24.243 DEBUG:teuthology.orchestra.run.smithi066:> uname -r 2024-03-04T07:14:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:14:24.244 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2024-03-04T07:14:24.299 INFO:teuthology.orchestra.run.smithi066.stdout:5.14.0-361.el9.x86_64 2024-03-04T07:14:24.299 INFO:teuthology.task.kernel:Running kernel on smithi066: 5.14.0-361.el9.x86_64 2024-03-04T07:14:24.299 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum install -y kernel 2024-03-04T07:14:24.301 INFO:teuthology.orchestra.run.smithi064.stdout:5.14.0-361.el9.x86_64 2024-03-04T07:14:24.301 INFO:teuthology.task.kernel:Running kernel on smithi064: 5.14.0-361.el9.x86_64 2024-03-04T07:14:24.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum install -y kernel 2024-03-04T07:14:26.565 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-04T07:14:26.757 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS-9 - AppStream 8.3 MB/s | 6.7 MB 00:00 2024-03-04T07:14:28.198 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS-9 - BaseOS 4.2 MB/s | 2.0 MB 00:00 2024-03-04T07:14:28.382 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS-9 - BaseOS 4.5 MB/s | 2.0 MB 00:00 2024-03-04T07:14:29.181 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.0 MB 00:00 2024-03-04T07:14:31.709 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - BaseOS 2.7 MB/s | 8.0 MB 00:02 2024-03-04T07:14:32.407 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - AppStream 9.1 MB/s | 19 MB 00:02 2024-03-04T07:14:33.732 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - AppStream 21 MB/s | 19 MB 00:00 2024-03-04T07:14:36.749 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.1 MB 00:00 2024-03-04T07:14:39.275 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - Extras packages 11 kB/s | 15 kB 00:01 2024-03-04T07:14:40.141 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - CRB 2.2 MB/s | 6.1 MB 00:02 2024-03-04T07:14:41.758 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - Extras packages 34 kB/s | 15 kB 00:00 2024-03-04T07:14:46.704 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 4.2 MB/s | 20 MB 00:04 2024-03-04T07:14:51.842 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-03-04T07:14:54.224 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-04T07:14:54.224 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-04T07:14:54.324 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout: kernel x86_64 5.14.0-427.el9 baseos 6.0 M 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-core x86_64 5.14.0-427.el9 baseos 21 M 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules x86_64 5.14.0-427.el9 baseos 39 M 2024-03-04T07:14:54.330 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-core x86_64 5.14.0-427.el9 baseos 33 M 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout:Install 4 Packages 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 100 M 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 123 M 2024-03-04T07:14:54.331 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-03-04T07:15:00.535 INFO:teuthology.orchestra.run.smithi066.stdout:Extra Packages for Enterprise Linux 993 kB/s | 20 MB 00:21 2024-03-04T07:15:04.118 INFO:teuthology.orchestra.run.smithi064.stdout:(1/4): kernel-5.14.0-427.el9.x86_64.rpm 645 kB/s | 6.0 MB 00:09 2024-03-04T07:15:05.650 INFO:teuthology.orchestra.run.smithi066.stdout:lab-extras 37 kB/s | 1.7 kB 00:00 2024-03-04T07:15:08.023 INFO:teuthology.orchestra.run.smithi066.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-04T07:15:08.023 INFO:teuthology.orchestra.run.smithi066.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-04T07:15:08.120 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2024-03-04T07:15:08.132 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2024-03-04T07:15:08.132 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2024-03-04T07:15:08.132 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2024-03-04T07:15:08.132 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2024-03-04T07:15:08.132 INFO:teuthology.orchestra.run.smithi066.stdout: kernel x86_64 5.14.0-427.el9 baseos 6.0 M 2024-03-04T07:15:08.132 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2024-03-04T07:15:08.132 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-core x86_64 5.14.0-427.el9 baseos 21 M 2024-03-04T07:15:08.133 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-modules x86_64 5.14.0-427.el9 baseos 39 M 2024-03-04T07:15:08.133 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-modules-core x86_64 5.14.0-427.el9 baseos 33 M 2024-03-04T07:15:08.133 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:15:08.133 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2024-03-04T07:15:08.133 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2024-03-04T07:15:08.133 INFO:teuthology.orchestra.run.smithi066.stdout:Install 4 Packages 2024-03-04T07:15:08.133 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:15:08.134 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 100 M 2024-03-04T07:15:08.134 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 123 M 2024-03-04T07:15:08.134 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2024-03-04T07:15:08.909 INFO:teuthology.orchestra.run.smithi066.stdout:(1/4): kernel-5.14.0-427.el9.x86_64.rpm 9.4 MB/s | 6.0 MB 00:00 2024-03-04T07:15:12.859 INFO:teuthology.orchestra.run.smithi066.stdout:(2/4): kernel-core-5.14.0-427.el9.x86_64.rpm 4.5 MB/s | 21 MB 00:04 2024-03-04T07:15:14.109 INFO:teuthology.orchestra.run.smithi066.stdout:(3/4): kernel-modules-core-5.14.0-427.el9.x86_6 6.4 MB/s | 33 MB 00:05 2024-03-04T07:15:15.318 INFO:teuthology.orchestra.run.smithi066.stdout:(4/4): kernel-modules-5.14.0-427.el9.x86_64.rpm 5.6 MB/s | 39 MB 00:07 2024-03-04T07:15:15.321 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2024-03-04T07:15:15.321 INFO:teuthology.orchestra.run.smithi066.stdout:Total 14 MB/s | 100 MB 00:07 2024-03-04T07:15:16.249 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2024-03-04T07:15:16.657 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2024-03-04T07:15:16.657 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2024-03-04T07:15:18.053 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2024-03-04T07:15:18.054 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2024-03-04T07:15:20.922 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2024-03-04T07:15:21.591 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : kernel-modules-core-5.14.0-427.el9.x86_64 1/4 2024-03-04T07:15:21.735 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-04T07:15:24.657 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-04T07:15:24.691 INFO:teuthology.orchestra.run.smithi064.stdout:(2/4): kernel-core-5.14.0-427.el9.x86_64.rpm 709 kB/s | 21 MB 00:30 2024-03-04T07:15:24.827 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-04T07:15:29.942 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-04T07:15:30.064 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : kernel-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:15:32.307 INFO:teuthology.orchestra.run.smithi064.stdout:(3/4): kernel-modules-core-5.14.0-427.el9.x86_6 1.2 MB/s | 33 MB 00:28 2024-03-04T07:15:35.197 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:15:57.722 INFO:teuthology.orchestra.run.smithi064.stdout:(4/4): kernel-modules-5.14.0-427.el9.x86_64.rpm 638 kB/s | 39 MB 01:03 2024-03-04T07:15:57.722 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-03-04T07:15:57.722 INFO:teuthology.orchestra.run.smithi064.stdout:Total 1.6 MB/s | 100 MB 01:03 2024-03-04T07:15:58.638 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-03-04T07:15:59.103 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-03-04T07:15:59.103 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-03-04T07:16:00.508 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-03-04T07:16:00.509 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-03-04T07:16:03.371 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-03-04T07:16:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-modules-core-5.14.0-427.el9.x86_64 1/4 2024-03-04T07:16:04.205 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-04T07:16:05.672 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:05.672 INFO:teuthology.orchestra.run.smithi066.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-04T07:16:05.672 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:16:05.760 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:06.213 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:06.214 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : kernel-5.14.0-427.el9.x86_64 1/4 2024-03-04T07:16:06.214 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-04T07:16:06.214 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-04T07:16:06.647 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:06.647 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:16:06.648 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2024-03-04T07:16:06.648 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-5.14.0-427.el9.x86_64 2024-03-04T07:16:06.648 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-core-5.14.0-427.el9.x86_64 2024-03-04T07:16:06.648 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-modules-5.14.0-427.el9.x86_64 2024-03-04T07:16:06.648 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-modules-core-5.14.0-427.el9.x86_64 2024-03-04T07:16:06.648 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:16:06.648 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2024-03-04T07:16:07.125 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-04T07:16:07.142 DEBUG:teuthology.orchestra.run.smithi066:> rpm -q kernel | sort -rV | head -n 1 2024-03-04T07:16:07.297 INFO:teuthology.orchestra.run.smithi066.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-04T07:16:07.298 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-04T07:16:07.298 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-427.el9.x86_64 2024-03-04T07:16:07.298 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-04T07:16:07.298 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-04T07:16:07.298 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-03-04T07:16:07.298 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi066.front.sepia.ceph.com, path=None, version=distro) 2024-03-04T07:16:07.299 DEBUG:teuthology.orchestra.run.smithi066:> rpm -q kernel | sort -rV | head -n 1 2024-03-04T07:16:07.314 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-04T07:16:07.424 INFO:teuthology.orchestra.run.smithi066.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-04T07:16:07.425 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-04T07:16:07.425 DEBUG:teuthology.orchestra.run.smithi066:> sudo rpm -qi grub2-tools 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Name : grub2-tools 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Epoch : 1 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Version : 2.06 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Release : 68.el9 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Architecture: x86_64 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Group : Unspecified 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Size : 8264254 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:License : GPLv3+ 2024-03-04T07:16:07.472 INFO:teuthology.orchestra.run.smithi066.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:Packager : builder@centos.org 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:Vendor : CentOS 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:Summary : Support tools for GRUB. 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:Description : 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:hardware devices. 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:16:07.473 INFO:teuthology.orchestra.run.smithi066.stdout:This subpackage provides tools for support of all platforms. 2024-03-04T07:16:07.474 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-04T07:16:07.474 INFO:teuthology.task.kernel:Updating grub on smithi066 to boot 5.14.0-427.el9.x86_64 2024-03-04T07:16:07.474 DEBUG:teuthology.orchestra.run.smithi066:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-04T07:16:08.107 INFO:teuthology.orchestra.run.smithi066.stderr:Generating grub configuration file ... 2024-03-04T07:16:09.477 INFO:teuthology.orchestra.run.smithi066.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-04T07:16:09.517 INFO:teuthology.orchestra.run.smithi066.stderr:done 2024-03-04T07:16:09.519 DEBUG:teuthology.orchestra.run.smithi066:> mktemp 2024-03-04T07:16:09.534 INFO:teuthology.orchestra.run.smithi066.stdout:/tmp/tmp.Pzwf7U4e4C 2024-03-04T07:16:09.535 DEBUG:teuthology.orchestra.run.smithi066:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Pzwf7U4e4C 2024-03-04T07:16:09.606 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 0666 /tmp/tmp.Pzwf7U4e4C 2024-03-04T07:16:09.742 DEBUG:teuthology.orchestra.remote:smithi066:/tmp/tmp.Pzwf7U4e4C is 6KB 2024-03-04T07:16:09.762 DEBUG:teuthology.orchestra.run.smithi066:> rm -fr /tmp/tmp.Pzwf7U4e4C 2024-03-04T07:16:09.776 DEBUG:teuthology.orchestra.run.smithi066:> sudo /bin/ls /boot/loader/entries || true 2024-03-04T07:16:09.848 INFO:teuthology.orchestra.run.smithi066.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-04T07:16:09.848 INFO:teuthology.orchestra.run.smithi066.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-04T07:16:09.849 INFO:teuthology.orchestra.run.smithi066.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-04T07:16:09.849 INFO:teuthology.orchestra.run.smithi066.stdout:15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64.conf 2024-03-04T07:16:09.850 DEBUG:teuthology.orchestra.run.smithi066:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64 2024-03-04T07:16:09.994 DEBUG:teuthology.orchestra.run.smithi066:> sudo shutdown -r now 2024-03-04T07:16:12.420 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-04T07:16:12.542 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:39.997 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-03-04T07:16:39.997 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:16:39.998 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:16:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:48.204 INFO:teuthology.orchestra.run.smithi064.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-04T07:16:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:16:48.291 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:48.759 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:48.759 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-5.14.0-427.el9.x86_64 1/4 2024-03-04T07:16:48.759 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-04T07:16:48.759 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-5.14.0-427.el9.x86_64 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-core-5.14.0-427.el9.x86_64 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-5.14.0-427.el9.x86_64 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-core-5.14.0-427.el9.x86_64 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-03-04T07:16:49.690 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2024-03-04T07:16:49.836 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-04T07:16:49.836 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-04T07:16:49.836 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-427.el9.x86_64 2024-03-04T07:16:49.836 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-04T07:16:49.836 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-04T07:16:49.836 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-03-04T07:16:49.837 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi064.front.sepia.ceph.com, path=None, version=distro) 2024-03-04T07:16:49.837 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2024-03-04T07:16:49.962 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-04T07:16:49.963 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-04T07:16:49.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo rpm -qi grub2-tools 2024-03-04T07:16:50.003 INFO:teuthology.orchestra.run.smithi064.stdout:Name : grub2-tools 2024-03-04T07:16:50.003 INFO:teuthology.orchestra.run.smithi064.stdout:Epoch : 1 2024-03-04T07:16:50.003 INFO:teuthology.orchestra.run.smithi064.stdout:Version : 2.06 2024-03-04T07:16:50.003 INFO:teuthology.orchestra.run.smithi064.stdout:Release : 68.el9 2024-03-04T07:16:50.003 INFO:teuthology.orchestra.run.smithi064.stdout:Architecture: x86_64 2024-03-04T07:16:50.003 INFO:teuthology.orchestra.run.smithi064.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-04T07:16:50.003 INFO:teuthology.orchestra.run.smithi064.stdout:Group : Unspecified 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Size : 8264254 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:License : GPLv3+ 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Packager : builder@centos.org 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Vendor : CentOS 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Summary : Support tools for GRUB. 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:Description : 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-04T07:16:50.004 INFO:teuthology.orchestra.run.smithi064.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-04T07:16:50.005 INFO:teuthology.orchestra.run.smithi064.stdout:hardware devices. 2024-03-04T07:16:50.005 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:16:50.005 INFO:teuthology.orchestra.run.smithi064.stdout:This subpackage provides tools for support of all platforms. 2024-03-04T07:16:50.005 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-04T07:16:50.006 INFO:teuthology.task.kernel:Updating grub on smithi064 to boot 5.14.0-427.el9.x86_64 2024-03-04T07:16:50.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-04T07:16:50.636 INFO:teuthology.orchestra.run.smithi064.stderr:Generating grub configuration file ... 2024-03-04T07:16:52.079 INFO:teuthology.orchestra.run.smithi064.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-04T07:16:52.117 INFO:teuthology.orchestra.run.smithi064.stderr:done 2024-03-04T07:16:52.119 DEBUG:teuthology.orchestra.run.smithi064:> mktemp 2024-03-04T07:16:52.134 INFO:teuthology.orchestra.run.smithi064.stdout:/tmp/tmp.3kfOMlX0Iq 2024-03-04T07:16:52.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.3kfOMlX0Iq 2024-03-04T07:16:52.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0666 /tmp/tmp.3kfOMlX0Iq 2024-03-04T07:16:52.352 DEBUG:teuthology.orchestra.remote:smithi064:/tmp/tmp.3kfOMlX0Iq is 6KB 2024-03-04T07:16:52.401 DEBUG:teuthology.orchestra.run.smithi064:> rm -fr /tmp/tmp.3kfOMlX0Iq 2024-03-04T07:16:52.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo /bin/ls /boot/loader/entries || true 2024-03-04T07:16:52.484 INFO:teuthology.orchestra.run.smithi064.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-04T07:16:52.484 INFO:teuthology.orchestra.run.smithi064.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-04T07:16:52.484 INFO:teuthology.orchestra.run.smithi064.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-04T07:16:52.484 INFO:teuthology.orchestra.run.smithi064.stdout:15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64.conf 2024-03-04T07:16:52.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64 2024-03-04T07:16:52.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo shutdown -r now 2024-03-04T07:16:58.556 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2024-03-04T07:17:07.564 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:17:07.565 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:17:10.620 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2024-03-04T07:17:22.632 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:17:22.632 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:17:22.634 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-03-04T07:17:22.634 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-03-04T07:17:22.635 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:17:25.692 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2024-03-04T07:17:40.704 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:17:40.705 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:17:41.056 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-03-04T07:17:43.772 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2024-03-04T07:17:50.060 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-03-04T07:17:50.061 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:17:53.120 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-03-04T07:18:01.780 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:18:01.781 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:18:05.124 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-03-04T07:18:05.124 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:18:33.652 DEBUG:teuthology.orchestra.run.smithi066:> true 2024-03-04T07:18:34.247 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:18:34.247 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-427.el9.x86_64"... 2024-03-04T07:18:34.248 DEBUG:teuthology.orchestra.run.smithi066:> uname -r 2024-03-04T07:18:34.263 INFO:teuthology.orchestra.run.smithi066.stdout:5.14.0-427.el9.x86_64 2024-03-04T07:18:34.263 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-427.el9.x86_64 vs 5.14.0-427.el9.x86_64 2024-03-04T07:18:34.263 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-04T07:18:34.263 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-04T07:18:35.265 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-03-04T07:18:35.265 DEBUG:teuthology.orchestra.run.smithi066:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-04T07:18:35.327 INFO:teuthology.orchestra.run.smithi066.stdout:ttyS1 2024-03-04T07:18:35.364 DEBUG:teuthology.parallel:result is None 2024-03-04T07:18:36.944 DEBUG:teuthology.orchestra.run.smithi064:> true 2024-03-04T07:18:37.295 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-03-04T07:18:37.296 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-427.el9.x86_64"... 2024-03-04T07:18:37.296 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2024-03-04T07:18:37.310 INFO:teuthology.orchestra.run.smithi064.stdout:5.14.0-427.el9.x86_64 2024-03-04T07:18:37.310 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-427.el9.x86_64 vs 5.14.0-427.el9.x86_64 2024-03-04T07:18:37.310 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-04T07:18:37.310 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-04T07:18:38.311 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-03-04T07:18:38.311 DEBUG:teuthology.orchestra.run.smithi064:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-04T07:18:38.376 INFO:teuthology.orchestra.run.smithi064.stdout:ttyS1 2024-03-04T07:18:38.410 DEBUG:teuthology.parallel:result is None 2024-03-04T07:18:38.411 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-04T07:18:38.417 INFO:teuthology.task.internal:Creating test directory... 2024-03-04T07:18:38.418 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-04T07:18:38.421 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-04T07:18:38.437 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-04T07:18:38.445 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-04T07:18:38.459 INFO:teuthology.task.internal:Creating archive directory... 2024-03-04T07:18:38.459 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-04T07:18:38.477 DEBUG:teuthology.orchestra.run.smithi066:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-04T07:18:38.505 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-04T07:18:38.513 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-04T07:18:38.514 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-04T07:18:38.539 DEBUG:teuthology.orchestra.run.smithi066:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-04T07:18:38.575 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-04T07:18:38.577 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-04T07:18:38.590 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-04T07:18:38.591 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-04T07:18:38.593 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-04T07:18:38.599 INFO:teuthology.task.internal:Configuring sudo... 2024-03-04T07:18:38.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-04T07:18:38.633 DEBUG:teuthology.orchestra.run.smithi066:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-04T07:18:38.669 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-04T07:18:38.678 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-04T07:18:38.678 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-04T07:18:38.700 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-04T07:18:38.727 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-04T07:18:38.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-04T07:18:38.865 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-04T07:18:38.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-04T07:18:38.976 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:18:38.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-04T07:18:39.059 DEBUG:teuthology.orchestra.run.smithi066:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-04T07:18:39.102 DEBUG:teuthology.orchestra.run.smithi066:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-04T07:18:39.173 DEBUG:teuthology.orchestra.run.smithi066:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-04T07:18:39.214 DEBUG:teuthology.orchestra.run.smithi066:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-04T07:18:39.266 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:18:39.266 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-04T07:18:39.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo service rsyslog restart 2024-03-04T07:18:39.343 DEBUG:teuthology.orchestra.run.smithi066:> sudo service rsyslog restart 2024-03-04T07:18:39.416 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-04T07:18:39.434 INFO:teuthology.orchestra.run.smithi066.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-04T07:18:39.912 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-04T07:18:39.920 INFO:teuthology.task.internal:Starting timer... 2024-03-04T07:18:39.920 INFO:teuthology.run_tasks:Running task pcp... 2024-03-04T07:18:39.942 INFO:teuthology.run_tasks:Running task selinux... 2024-03-04T07:18:39.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo service auditd rotate 2024-03-04T07:18:40.004 INFO:teuthology.orchestra.run.smithi064.stdout:Rotating logs: 2024-03-04T07:18:40.005 DEBUG:teuthology.orchestra.run.smithi066:> sudo service auditd rotate 2024-03-04T07:18:40.113 INFO:teuthology.orchestra.run.smithi066.stdout:Rotating logs: 2024-03-04T07:18:40.118 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-04T07:18:40.118 DEBUG:teuthology.orchestra.run.smithi064:> /usr/sbin/getenforce 2024-03-04T07:18:40.309 INFO:teuthology.orchestra.run.smithi064.stdout:Permissive 2024-03-04T07:18:40.309 DEBUG:teuthology.orchestra.run.smithi066:> /usr/sbin/getenforce 2024-03-04T07:18:40.342 INFO:teuthology.orchestra.run.smithi066.stdout:Permissive 2024-03-04T07:18:40.343 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi064.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi066.front.sepia.ceph.com': 'permissive'} 2024-03-04T07:18:40.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-03-04T07:18:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:18:40.373 DEBUG:teuthology.orchestra.run.smithi066:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-03-04T07:18:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:18:40.441 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-04T07:18:40.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo /usr/sbin/setenforce permissive 2024-03-04T07:18:40.477 DEBUG:teuthology.orchestra.run.smithi066:> sudo /usr/sbin/setenforce permissive 2024-03-04T07:18:40.662 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-04T07:18:40.672 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-04T07:18:40.691 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-04T07:18:40.693 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi064.front.sepia.ceph.com,smithi066.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-04T07:23:32.909 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi064.front.sepia.ceph.com'), Remote(name='ubuntu@smithi066.front.sepia.ceph.com')] 2024-03-04T07:23:32.910 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-03-04T07:23:32.911 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:23:33.003 DEBUG:teuthology.orchestra.run.smithi064:> true 2024-03-04T07:23:33.078 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-03-04T07:23:33.079 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:23:33.079 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-04T07:23:33.201 DEBUG:teuthology.orchestra.run.smithi066:> true 2024-03-04T07:23:33.281 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi066.front.sepia.ceph.com' 2024-03-04T07:23:33.281 INFO:teuthology.run_tasks:Running task clock... 2024-03-04T07:23:33.292 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-04T07:23:33.293 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-04T07:23:33.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-04T07:23:33.296 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-04T07:23:33.296 DEBUG:teuthology.orchestra.run.smithi066:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-04T07:23:33.325 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-04T07:23:33.342 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-04T07:23:33.360 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-04T07:23:33.372 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: ntpd: command not found 2024-03-04T07:23:33.381 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-04T07:23:33.385 INFO:teuthology.orchestra.run.smithi064.stdout:506 Cannot talk to daemon 2024-03-04T07:23:33.401 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-04T07:23:33.417 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-04T07:23:33.418 INFO:teuthology.orchestra.run.smithi066.stderr:sudo: ntpd: command not found 2024-03-04T07:23:33.437 INFO:teuthology.orchestra.run.smithi066.stdout:506 Cannot talk to daemon 2024-03-04T07:23:33.456 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-04T07:23:33.467 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-03-04T07:23:33.470 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-04T07:23:33.470 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-03-04T07:23:33.470 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.470 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.470 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.471 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.475 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-04T07:23:33.541 INFO:teuthology.orchestra.run.smithi066.stderr:bash: line 1: ntpq: command not found 2024-03-04T07:23:33.544 INFO:teuthology.orchestra.run.smithi066.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-04T07:23:33.544 INFO:teuthology.orchestra.run.smithi066.stdout:=============================================================================== 2024-03-04T07:23:33.544 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.545 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.545 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.545 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:23:33.545 INFO:teuthology.run_tasks:Running task install... 2024-03-04T07:23:33.553 DEBUG:teuthology.task.install:project ceph 2024-03-04T07:23:33.553 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}} 2024-03-04T07:23:33.555 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'} 2024-03-04T07:23:33.555 INFO:teuthology.task.install:Using flavor: default 2024-03-04T07:23:33.562 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-04T07:23:33.562 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']} 2024-03-04T07:23:33.563 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'tag': None, 'wait_for_package': False} 2024-03-04T07:23:33.563 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-04T07:23:33.565 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'tag': None, 'wait_for_package': False} 2024-03-04T07:23:33.565 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-04T07:23:33.712 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/ 2024-03-04T07:23:33.712 INFO:teuthology.task.install.rpm:Package version is 19.0.0-1739.g1bc250c7 2024-03-04T07:23:33.819 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/ 2024-03-04T07:23:33.819 INFO:teuthology.task.install.rpm:Package version is 19.0.0-1739.g1bc250c7 2024-03-04T07:23:33.880 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-04T07:23:33.880 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:23:33.880 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-04T07:23:33.913 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-03-04T07:23:33.913 DEBUG:teuthology.orchestra.run.smithi066:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-04T07:23:33.985 DEBUG:teuthology.orchestra.run.smithi066:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-04T07:23:34.073 DEBUG:teuthology.orchestra.run.smithi066:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-04T07:23:34.103 INFO:teuthology.orchestra.run.smithi066.stdout:check_obsoletes = 1 2024-03-04T07:23:34.105 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum clean all 2024-03-04T07:23:34.118 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-qdb/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-04T07:23:34.118 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:23:34.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-04T07:23:34.147 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-03-04T07:23:34.147 DEBUG:teuthology.orchestra.run.smithi064:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-04T07:23:34.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-04T07:23:34.298 DEBUG:teuthology.orchestra.run.smithi064:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-04T07:23:34.364 INFO:teuthology.orchestra.run.smithi064.stdout:check_obsoletes = 1 2024-03-04T07:23:34.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum clean all 2024-03-04T07:23:34.373 INFO:teuthology.orchestra.run.smithi066.stdout:74 files removed 2024-03-04T07:23:34.400 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-03-04T07:23:34.618 INFO:teuthology.orchestra.run.smithi064.stdout:74 files removed 2024-03-04T07:23:34.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-03-04T07:23:35.225 INFO:teuthology.orchestra.run.smithi066.stdout:ceph packages for x86_64 170 kB/s | 83 kB 00:00 2024-03-04T07:23:35.519 INFO:teuthology.orchestra.run.smithi064.stdout:ceph packages for x86_64 151 kB/s | 83 kB 00:00 2024-03-04T07:23:35.675 INFO:teuthology.orchestra.run.smithi066.stdout:ceph noarch packages 25 kB/s | 10 kB 00:00 2024-03-04T07:23:36.019 INFO:teuthology.orchestra.run.smithi064.stdout:ceph noarch packages 22 kB/s | 10 kB 00:00 2024-03-04T07:23:36.209 INFO:teuthology.orchestra.run.smithi066.stdout:ceph source packages 3.8 kB/s | 1.9 kB 00:00 2024-03-04T07:23:36.461 INFO:teuthology.orchestra.run.smithi064.stdout:ceph source packages 4.7 kB/s | 1.9 kB 00:00 2024-03-04T07:23:36.778 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS-9 - AppStream 13 MB/s | 6.7 MB 00:00 2024-03-04T07:23:38.082 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS-9 - AppStream 4.2 MB/s | 6.7 MB 00:01 2024-03-04T07:23:38.362 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS-9 - BaseOS 4.7 MB/s | 2.0 MB 00:00 2024-03-04T07:23:39.589 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS-9 - BaseOS 5.6 MB/s | 2.0 MB 00:00 2024-03-04T07:23:40.278 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - BaseOS 5.1 MB/s | 8.0 MB 00:01 2024-03-04T07:23:42.671 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - BaseOS 2.9 MB/s | 8.0 MB 00:02 2024-03-04T07:23:43.003 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - AppStream 12 MB/s | 19 MB 00:01 2024-03-04T07:23:47.507 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - CRB 9.3 MB/s | 6.1 MB 00:00 2024-03-04T07:23:48.991 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 9 - Extras packages 55 kB/s | 15 kB 00:00 2024-03-04T07:23:50.232 INFO:teuthology.orchestra.run.smithi066.stdout:Extra Packages for Enterprise Linux 18 MB/s | 20 MB 00:01 2024-03-04T07:23:55.316 INFO:teuthology.orchestra.run.smithi066.stdout:lab-extras 35 kB/s | 1.7 kB 00:00 2024-03-04T07:23:57.986 INFO:teuthology.orchestra.run.smithi066.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-04T07:23:57.986 INFO:teuthology.orchestra.run.smithi066.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-04T07:23:57.999 INFO:teuthology.orchestra.run.smithi066.stdout:Package NetworkManager-1:1.44.0-2.el9.x86_64 is already installed. 2024-03-04T07:23:57.999 INFO:teuthology.orchestra.run.smithi066.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-03-04T07:23:58.002 INFO:teuthology.orchestra.run.smithi066.stdout:Package util-linux-2.37.4-15.el9.x86_64 is already installed. 2024-03-04T07:23:58.002 INFO:teuthology.orchestra.run.smithi066.stdout:Package libacl-devel-2.3.1-3.el9.x86_64 is already installed. 2024-03-04T07:23:58.003 INFO:teuthology.orchestra.run.smithi066.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-03-04T07:23:58.006 INFO:teuthology.orchestra.run.smithi066.stdout:Package xfsdump-3.1.12-3.el9.x86_64 is already installed. 2024-03-04T07:23:58.007 INFO:teuthology.orchestra.run.smithi066.stdout:Package xfsprogs-5.19.0-4.el9.x86_64 is already installed. 2024-03-04T07:23:58.094 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2024-03-04T07:23:58.111 INFO:teuthology.orchestra.run.smithi066.stdout:============================================================================================ 2024-03-04T07:23:58.111 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2024-03-04T07:23:58.111 INFO:teuthology.orchestra.run.smithi066.stdout:============================================================================================ 2024-03-04T07:23:58.111 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2024-03-04T07:23:58.111 INFO:teuthology.orchestra.run.smithi066.stdout: bison x86_64 3.7.4-5.el9 CentOS-AppStream 920 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 6.6 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-base x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.3 M 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 888 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 145 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 1.5 M 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 138 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-dashboard noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 1.8 M 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 7.4 M 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-rook noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 49 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-radosgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 9.8 M 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-test x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 47 M 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-volume noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 264 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 748 k 2024-03-04T07:23:58.112 INFO:teuthology.orchestra.run.smithi066.stdout: cephfs-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 220 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: cephfs-shell x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 27 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: cephfs-top noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 18 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 CentOS-AppStream 20 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: flex x86_64 2.6.4-9.el9 CentOS-AppStream 307 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: libaio-devel x86_64 0.3.111-13.el9 CentOS-AppStream 10 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 31 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 747 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: librados-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 127 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: libtool x86_64 2.4.6-45.el9 CentOS-AppStream 578 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: libuuid-devel x86_64 2.37.4-18.el9 CentOS-AppStream 24 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: openssl-devel x86_64 1:3.0.7-27.el9 CentOS-AppStream 3.0 M 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql x86_64 13.11-1.el9 CentOS-AppStream 1.5 M 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql-contrib x86_64 13.11-1.el9 CentOS-AppStream 811 k 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql-server x86_64 13.11-1.el9 CentOS-AppStream 5.7 M 2024-03-04T07:23:58.113 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cephfs x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 162 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rados x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 321 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 298 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 99 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 85 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-nbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 171 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 CentOS-AppStream 87 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout:Upgrading: 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager x86_64 1:1.46.0-1.el9 CentOS-BaseOS 2.3 M 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager-libnm x86_64 1:1.46.0-1.el9 CentOS-BaseOS 1.8 M 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager-team x86_64 1:1.46.0-1.el9 CentOS-BaseOS 41 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager-tui x86_64 1:1.46.0-1.el9 CentOS-BaseOS 247 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: acl x86_64 2.3.1-4.el9 CentOS-BaseOS 69 k 2024-03-04T07:23:58.114 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-debuginfod-client x86_64 0.190-2.el9 CentOS-BaseOS 35 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-libelf x86_64 0.190-2.el9 CentOS-BaseOS 191 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-libs x86_64 0.190-2.el9 CentOS-BaseOS 254 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: iproute x86_64 6.7.0-2.el9 CentOS-BaseOS 802 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: iproute-tc x86_64 6.7.0-2.el9 CentOS-BaseOS 433 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: libacl x86_64 2.3.1-4.el9 CentOS-BaseOS 22 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: libacl-devel x86_64 2.3.1-4.el9 CentOS-AppStream 68 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: libblkid x86_64 2.37.4-18.el9 CentOS-BaseOS 107 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: libfdisk x86_64 2.37.4-18.el9 CentOS-BaseOS 154 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: libmount x86_64 2.37.4-18.el9 CentOS-BaseOS 135 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: librados2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.4 M 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: librbd1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: libsmartcols x86_64 2.37.4-18.el9 CentOS-BaseOS 63 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: libuuid x86_64 2.37.4-18.el9 CentOS-BaseOS 28 k 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: openssl x86_64 1:3.0.7-27.el9 CentOS-BaseOS 1.2 M 2024-03-04T07:23:58.115 INFO:teuthology.orchestra.run.smithi066.stdout: openssl-libs x86_64 1:3.0.7-27.el9 CentOS-BaseOS 2.2 M 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: util-linux x86_64 2.37.4-18.el9 CentOS-BaseOS 2.2 M 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: util-linux-core x86_64 2.37.4-18.el9 CentOS-BaseOS 438 k 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: util-linux-user x86_64 2.37.4-18.el9 CentOS-BaseOS 30 k 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: xfsdump x86_64 3.1.12-4.el9 CentOS-BaseOS 321 k 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: xfsprogs x86_64 6.3.0-1.el9 CentOS-BaseOS 1.1 M 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: automake noarch 1.16.2-8.el9 CentOS-AppStream 662 k 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: boost-program-options x86_64 1.75.0-8.el9 CentOS-AppStream 106 k 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 20 M 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-grafana-dashboards noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 31 k 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mds x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 2.3 M 2024-03-04T07:23:58.116 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-modules-core noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 248 k 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mon x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 4.7 M 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-osd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 17 M 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-prometheus-alerts noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 15 k 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-selinux x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 25 k 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-04T07:23:58.117 INFO:teuthology.orchestra.run.smithi066.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-03-04T07:23:58.125 INFO:teuthology.orchestra.run.smithi066.stdout: jq x86_64 1.6-16.el9 CentOS-BaseOS 186 k 2024-03-04T07:23:58.125 INFO:teuthology.orchestra.run.smithi066.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-03-04T07:23:58.125 INFO:teuthology.orchestra.run.smithi066.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: libcephsqlite x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 163 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: libgfortran x86_64 11.4.1-3.el9 CentOS-BaseOS 802 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: libquadmath x86_64 11.4.1-3.el9 CentOS-BaseOS 194 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: libradosstriper1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 501 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: librgw2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.1 M 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: libzstd-devel x86_64 1.5.1-2.el9 CentOS-AppStream 46 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: lua x86_64 5.4.4-4.el9 CentOS-AppStream 187 k 2024-03-04T07:23:58.126 INFO:teuthology.orchestra.run.smithi066.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-BaseOS 217 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql-private-libs x86_64 13.11-1.el9 CentOS-AppStream 134 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: protobuf x86_64 3.14.0-13.el9 CentOS-AppStream 1.0 M 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: python3-asyncssh noarch 2.13.2-4.el9 epel 548 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: python3-attrs noarch 20.3.0-7.el9 CentOS-AppStream 78 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-04T07:23:58.127 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-argparse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 45 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 131 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-04T07:23:58.128 INFO:teuthology.orchestra.run.smithi066.stdout: python3-google-auth noarch 1:2.28.0-1.el9 epel 203 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jinja2 noarch 2.11.3-5.el9 CentOS-AppStream 227 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-04T07:23:58.129 INFO:teuthology.orchestra.run.smithi066.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-packaging noarch 20.9-5.el9 CentOS-AppStream 69 k 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-04T07:23:58.130 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-protobuf noarch 3.14.0-13.el9 CentOS-AppStream 238 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-requests noarch 2.25.1-8.el9 CentOS-BaseOS 113 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-04T07:23:58.131 INFO:teuthology.orchestra.run.smithi066.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-wcwidth noarch 0.2.5-8.el9 CentOS-AppStream 41 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: qatlib x86_64 23.11.0-1.el9 CentOS-AppStream 215 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: qatzip-libs x86_64 1.1.2-2.el9 CentOS-AppStream 43 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-04T07:23:58.132 INFO:teuthology.orchestra.run.smithi066.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout: uuid x86_64 1.6.2-55.el9 CentOS-AppStream 56 k 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout: zlib-devel x86_64 1.2.11-40.el9 CentOS-AppStream 44 k 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout:Installing weak dependencies: 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout: qatlib-service x86_64 23.11.0-1.el9 CentOS-AppStream 34 k 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout:============================================================================================ 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout:Install 150 Packages 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout:Upgrade 26 Packages 2024-03-04T07:23:58.133 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:23:58.134 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 224 M 2024-03-04T07:23:58.134 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2024-03-04T07:23:58.725 INFO:teuthology.orchestra.run.smithi066.stdout:(1/176): ceph-19.0.0-1739.g1bc250c7.el9.x86_64. 23 kB/s | 6.6 kB 00:00 2024-03-04T07:23:59.487 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - AppStream 1.2 MB/s | 19 MB 00:15 2024-03-04T07:24:00.042 INFO:teuthology.orchestra.run.smithi066.stdout:(2/176): ceph-fuse-19.0.0-1739.g1bc250c7.el9.x8 675 kB/s | 888 kB 00:01 2024-03-04T07:24:00.367 INFO:teuthology.orchestra.run.smithi066.stdout:(3/176): ceph-immutable-object-cache-19.0.0-173 447 kB/s | 145 kB 00:00 2024-03-04T07:24:01.251 INFO:teuthology.orchestra.run.smithi066.stdout:(4/176): ceph-base-19.0.0-1739.g1bc250c7.el9.x8 1.9 MB/s | 5.3 MB 00:02 2024-03-04T07:24:01.876 INFO:teuthology.orchestra.run.smithi066.stdout:(5/176): ceph-mgr-19.0.0-1739.g1bc250c7.el9.x86 2.4 MB/s | 1.5 MB 00:00 2024-03-04T07:24:03.026 INFO:teuthology.orchestra.run.smithi066.stdout:(6/176): ceph-mds-19.0.0-1739.g1bc250c7.el9.x86 890 kB/s | 2.3 MB 00:02 2024-03-04T07:24:03.643 INFO:teuthology.orchestra.run.smithi066.stdout:(7/176): ceph-mon-19.0.0-1739.g1bc250c7.el9.x86 2.7 MB/s | 4.7 MB 00:01 2024-03-04T07:24:03.835 INFO:teuthology.orchestra.run.smithi066.stdout:(8/176): ceph-common-19.0.0-1739.g1bc250c7.el9. 3.8 MB/s | 20 MB 00:05 2024-03-04T07:24:03.935 INFO:teuthology.orchestra.run.smithi066.stdout:(9/176): ceph-selinux-19.0.0-1739.g1bc250c7.el9 252 kB/s | 25 kB 00:00 2024-03-04T07:24:05.122 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - CRB 3.3 MB/s | 6.1 MB 00:01 2024-03-04T07:24:06.630 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - Extras packages 51 kB/s | 15 kB 00:00 2024-03-04T07:24:12.343 INFO:teuthology.orchestra.run.smithi066.stdout:(10/176): ceph-radosgw-19.0.0-1739.g1bc250c7.el 1.1 MB/s | 9.8 MB 00:08 2024-03-04T07:24:12.660 INFO:teuthology.orchestra.run.smithi066.stdout:(11/176): cephfs-mirror-19.0.0-1739.g1bc250c7.e 697 kB/s | 220 kB 00:00 2024-03-04T07:24:12.752 INFO:teuthology.orchestra.run.smithi066.stdout:(12/176): cephfs-shell-19.0.0-1739.g1bc250c7.el 296 kB/s | 27 kB 00:00 2024-03-04T07:24:12.852 INFO:teuthology.orchestra.run.smithi066.stdout:(13/176): libcephfs-devel-19.0.0-1739.g1bc250c7 312 kB/s | 31 kB 00:00 2024-03-04T07:24:13.760 INFO:teuthology.orchestra.run.smithi066.stdout:(14/176): libcephfs2-19.0.0-1739.g1bc250c7.el9. 822 kB/s | 747 kB 00:00 2024-03-04T07:24:14.352 INFO:teuthology.orchestra.run.smithi066.stdout:(15/176): libcephsqlite-19.0.0-1739.g1bc250c7.e 276 kB/s | 163 kB 00:00 2024-03-04T07:24:14.594 INFO:teuthology.orchestra.run.smithi066.stdout:(16/176): librados-devel-19.0.0-1739.g1bc250c7. 524 kB/s | 127 kB 00:00 2024-03-04T07:24:15.455 INFO:teuthology.orchestra.run.smithi066.stdout:(17/176): libradosstriper1-19.0.0-1739.g1bc250c 584 kB/s | 501 kB 00:00 2024-03-04T07:24:17.831 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 1.9 MB/s | 20 MB 00:11 2024-03-04T07:24:20.161 INFO:teuthology.orchestra.run.smithi066.stdout:(18/176): librgw2-19.0.0-1739.g1bc250c7.el9.x86 1.1 MB/s | 5.1 MB 00:04 2024-03-04T07:24:20.261 INFO:teuthology.orchestra.run.smithi066.stdout:(19/176): python3-ceph-argparse-19.0.0-1739.g1b 452 kB/s | 45 kB 00:00 2024-03-04T07:24:20.420 INFO:teuthology.orchestra.run.smithi066.stdout:(20/176): python3-ceph-common-19.0.0-1739.g1bc2 829 kB/s | 131 kB 00:00 2024-03-04T07:24:20.570 INFO:teuthology.orchestra.run.smithi066.stdout:(21/176): python3-cephfs-19.0.0-1739.g1bc250c7. 1.1 MB/s | 162 kB 00:00 2024-03-04T07:24:20.862 INFO:teuthology.orchestra.run.smithi066.stdout:(22/176): python3-rados-19.0.0-1739.g1bc250c7.e 1.1 MB/s | 321 kB 00:00 2024-03-04T07:24:21.096 INFO:teuthology.orchestra.run.smithi066.stdout:(23/176): ceph-osd-19.0.0-1739.g1bc250c7.el9.x8 953 kB/s | 17 MB 00:18 2024-03-04T07:24:21.179 INFO:teuthology.orchestra.run.smithi066.stdout:(24/176): python3-rbd-19.0.0-1739.g1bc250c7.el9 940 kB/s | 298 kB 00:00 2024-03-04T07:24:21.255 INFO:teuthology.orchestra.run.smithi066.stdout:(25/176): python3-rgw-19.0.0-1739.g1bc250c7.el9 627 kB/s | 99 kB 00:00 2024-03-04T07:24:21.280 INFO:teuthology.orchestra.run.smithi066.stdout:(26/176): rbd-fuse-19.0.0-1739.g1bc250c7.el9.x8 852 kB/s | 85 kB 00:00 2024-03-04T07:24:21.438 INFO:teuthology.orchestra.run.smithi066.stdout:(27/176): rbd-nbd-19.0.0-1739.g1bc250c7.el9.x86 1.1 MB/s | 171 kB 00:00 2024-03-04T07:24:21.531 INFO:teuthology.orchestra.run.smithi066.stdout:(28/176): ceph-grafana-dashboards-19.0.0-1739.g 342 kB/s | 31 kB 00:00 2024-03-04T07:24:21.714 INFO:teuthology.orchestra.run.smithi066.stdout:(29/176): ceph-mgr-cephadm-19.0.0-1739.g1bc250c 754 kB/s | 138 kB 00:00 2024-03-04T07:24:22.989 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-03-04T07:24:23.347 INFO:teuthology.orchestra.run.smithi066.stdout:(30/176): ceph-mgr-dashboard-19.0.0-1739.g1bc25 1.1 MB/s | 1.8 MB 00:01 2024-03-04T07:24:24.148 INFO:teuthology.orchestra.run.smithi066.stdout:(31/176): rbd-mirror-19.0.0-1739.g1bc250c7.el9. 1.1 MB/s | 3.1 MB 00:02 2024-03-04T07:24:24.432 INFO:teuthology.orchestra.run.smithi066.stdout:(32/176): ceph-mgr-modules-core-19.0.0-1739.g1b 877 kB/s | 248 kB 00:00 2024-03-04T07:24:24.531 INFO:teuthology.orchestra.run.smithi066.stdout:(33/176): ceph-mgr-rook-19.0.0-1739.g1bc250c7.e 493 kB/s | 49 kB 00:00 2024-03-04T07:24:24.624 INFO:teuthology.orchestra.run.smithi066.stdout:(34/176): ceph-prometheus-alerts-19.0.0-1739.g1 167 kB/s | 15 kB 00:00 2024-03-04T07:24:24.932 INFO:teuthology.orchestra.run.smithi066.stdout:(35/176): ceph-volume-19.0.0-1739.g1bc250c7.el9 856 kB/s | 264 kB 00:00 2024-03-04T07:24:25.599 INFO:teuthology.orchestra.run.smithi066.stdout:(36/176): cephadm-19.0.0-1739.g1bc250c7.el9.noa 1.1 MB/s | 748 kB 00:00 2024-03-04T07:24:25.651 INFO:teuthology.orchestra.run.smithi064.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-04T07:24:25.652 INFO:teuthology.orchestra.run.smithi064.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-04T07:24:25.665 INFO:teuthology.orchestra.run.smithi064.stdout:Package NetworkManager-1:1.44.0-2.el9.x86_64 is already installed. 2024-03-04T07:24:25.665 INFO:teuthology.orchestra.run.smithi064.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-03-04T07:24:25.666 INFO:teuthology.orchestra.run.smithi064.stdout:Package util-linux-2.37.4-15.el9.x86_64 is already installed. 2024-03-04T07:24:25.667 INFO:teuthology.orchestra.run.smithi064.stdout:Package libacl-devel-2.3.1-3.el9.x86_64 is already installed. 2024-03-04T07:24:25.669 INFO:teuthology.orchestra.run.smithi064.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-03-04T07:24:25.672 INFO:teuthology.orchestra.run.smithi064.stdout:Package xfsdump-3.1.12-3.el9.x86_64 is already installed. 2024-03-04T07:24:25.672 INFO:teuthology.orchestra.run.smithi064.stdout:Package xfsprogs-5.19.0-4.el9.x86_64 is already installed. 2024-03-04T07:24:25.708 INFO:teuthology.orchestra.run.smithi066.stdout:(37/176): cephfs-top-19.0.0-1739.g1bc250c7.el9. 169 kB/s | 18 kB 00:00 2024-03-04T07:24:25.758 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: bison x86_64 3.7.4-5.el9 CentOS-AppStream 920 k 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 6.6 k 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.3 M 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 888 k 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 145 k 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 1.5 M 2024-03-04T07:24:25.782 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 138 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 1.8 M 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 7.4 M 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 49 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 9.8 M 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 47 M 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-volume noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 264 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 748 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: cephfs-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 220 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: cephfs-shell x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 27 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: cephfs-top noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 18 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 CentOS-AppStream 20 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: flex x86_64 2.6.4-9.el9 CentOS-AppStream 307 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: libaio-devel x86_64 0.3.111-13.el9 CentOS-AppStream 10 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 31 k 2024-03-04T07:24:25.783 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 747 k 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 127 k 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: libtool x86_64 2.4.6-45.el9 CentOS-AppStream 578 k 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: libuuid-devel x86_64 2.37.4-18.el9 CentOS-AppStream 24 k 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: openssl-devel x86_64 1:3.0.7-27.el9 CentOS-AppStream 3.0 M 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql x86_64 13.11-1.el9 CentOS-AppStream 1.5 M 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql-contrib x86_64 13.11-1.el9 CentOS-AppStream 811 k 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql-server x86_64 13.11-1.el9 CentOS-AppStream 5.7 M 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 162 k 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 321 k 2024-03-04T07:24:25.784 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 298 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 99 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 85 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 171 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 CentOS-AppStream 87 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrading: 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager x86_64 1:1.46.0-1.el9 CentOS-BaseOS 2.3 M 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager-libnm x86_64 1:1.46.0-1.el9 CentOS-BaseOS 1.8 M 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager-team x86_64 1:1.46.0-1.el9 CentOS-BaseOS 41 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager-tui x86_64 1:1.46.0-1.el9 CentOS-BaseOS 247 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: acl x86_64 2.3.1-4.el9 CentOS-BaseOS 69 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-debuginfod-client x86_64 0.190-2.el9 CentOS-BaseOS 35 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-libelf x86_64 0.190-2.el9 CentOS-BaseOS 191 k 2024-03-04T07:24:25.785 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-libs x86_64 0.190-2.el9 CentOS-BaseOS 254 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: iproute x86_64 6.7.0-2.el9 CentOS-BaseOS 802 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: iproute-tc x86_64 6.7.0-2.el9 CentOS-BaseOS 433 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: libacl x86_64 2.3.1-4.el9 CentOS-BaseOS 22 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: libacl-devel x86_64 2.3.1-4.el9 CentOS-AppStream 68 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: libblkid x86_64 2.37.4-18.el9 CentOS-BaseOS 107 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: libfdisk x86_64 2.37.4-18.el9 CentOS-BaseOS 154 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: libmount x86_64 2.37.4-18.el9 CentOS-BaseOS 135 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: librados2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.4 M 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 3.1 M 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: libsmartcols x86_64 2.37.4-18.el9 CentOS-BaseOS 63 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: libuuid x86_64 2.37.4-18.el9 CentOS-BaseOS 28 k 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: openssl x86_64 1:3.0.7-27.el9 CentOS-BaseOS 1.2 M 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: openssl-libs x86_64 1:3.0.7-27.el9 CentOS-BaseOS 2.2 M 2024-03-04T07:24:25.786 INFO:teuthology.orchestra.run.smithi064.stdout: util-linux x86_64 2.37.4-18.el9 CentOS-BaseOS 2.2 M 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: util-linux-core x86_64 2.37.4-18.el9 CentOS-BaseOS 438 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: util-linux-user x86_64 2.37.4-18.el9 CentOS-BaseOS 30 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: xfsdump x86_64 3.1.12-4.el9 CentOS-BaseOS 321 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: xfsprogs x86_64 6.3.0-1.el9 CentOS-BaseOS 1.1 M 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: automake noarch 1.16.2-8.el9 CentOS-AppStream 662 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: boost-program-options x86_64 1.75.0-8.el9 CentOS-AppStream 106 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 20 M 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 31 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 2.3 M 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 248 k 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 4.7 M 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 17 M 2024-03-04T07:24:25.787 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts noarch 2:19.0.0-1739.g1bc250c7.el9 ceph-noarch 15 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 25 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: jq x86_64 1.6-16.el9 CentOS-BaseOS 186 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 163 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran x86_64 11.4.1-3.el9 CentOS-BaseOS 802 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-04T07:24:25.788 INFO:teuthology.orchestra.run.smithi064.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath x86_64 11.4.1-3.el9 CentOS-BaseOS 194 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 501 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2 x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 5.1 M 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: libzstd-devel x86_64 1.5.1-2.el9 CentOS-AppStream 46 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: lua x86_64 5.4.4-4.el9 CentOS-AppStream 187 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-BaseOS 217 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-04T07:24:25.789 INFO:teuthology.orchestra.run.smithi064.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql-private-libs x86_64 13.11-1.el9 CentOS-AppStream 134 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: protobuf x86_64 3.14.0-13.el9 CentOS-AppStream 1.0 M 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh noarch 2.13.2-4.el9 epel 548 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-attrs noarch 20.3.0-7.el9 CentOS-AppStream 78 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 45 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common x86_64 2:19.0.0-1739.g1bc250c7.el9 ceph 131 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-04T07:24:25.790 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth noarch 1:2.28.0-1.el9 epel 203 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2 noarch 2.11.3-5.el9 CentOS-AppStream 227 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-04T07:24:25.791 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-packaging noarch 20.9-5.el9 CentOS-AppStream 69 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-04T07:24:25.792 INFO:teuthology.orchestra.run.smithi064.stdout: python3-protobuf noarch 3.14.0-13.el9 CentOS-AppStream 238 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests noarch 2.25.1-8.el9 CentOS-BaseOS 113 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-04T07:24:25.793 INFO:teuthology.orchestra.run.smithi064.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-wcwidth noarch 0.2.5-8.el9 CentOS-AppStream 41 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: qatlib x86_64 23.11.0-1.el9 CentOS-AppStream 215 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: qatzip-libs x86_64 1.1.2-2.el9 CentOS-AppStream 43 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: uuid x86_64 1.6.2-55.el9 CentOS-AppStream 56 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout: zlib-devel x86_64 1.2.11-40.el9 CentOS-AppStream 44 k 2024-03-04T07:24:25.794 INFO:teuthology.orchestra.run.smithi064.stdout:Installing weak dependencies: 2024-03-04T07:24:25.795 INFO:teuthology.orchestra.run.smithi064.stdout: qatlib-service x86_64 23.11.0-1.el9 CentOS-AppStream 34 k 2024-03-04T07:24:25.795 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:24:25.795 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-03-04T07:24:25.795 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2024-03-04T07:24:25.795 INFO:teuthology.orchestra.run.smithi064.stdout:Install 150 Packages 2024-03-04T07:24:25.795 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrade 26 Packages 2024-03-04T07:24:25.795 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:24:25.796 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 224 M 2024-03-04T07:24:25.796 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-03-04T07:24:25.916 INFO:teuthology.orchestra.run.smithi066.stdout:(38/176): automake-1.16.2-8.el9.noarch.rpm 3.1 MB/s | 662 kB 00:00 2024-03-04T07:24:25.992 INFO:teuthology.orchestra.run.smithi066.stdout:(39/176): bison-3.7.4-5.el9.x86_64.rpm 12 MB/s | 920 kB 00:00 2024-03-04T07:24:26.059 INFO:teuthology.orchestra.run.smithi066.stdout:(40/176): boost-program-options-1.75.0-8.el9.x8 1.6 MB/s | 106 kB 00:00 2024-03-04T07:24:26.128 INFO:teuthology.orchestra.run.smithi066.stdout:(41/176): elfutils-libelf-devel-0.190-2.el9.x86 305 kB/s | 20 kB 00:00 2024-03-04T07:24:26.176 INFO:teuthology.orchestra.run.smithi066.stdout:(42/176): flex-2.6.4-9.el9.x86_64.rpm 6.0 MB/s | 307 kB 00:00 2024-03-04T07:24:26.210 INFO:teuthology.orchestra.run.smithi066.stdout:(43/176): flexiblas-3.0.4-8.el9.x86_64.rpm 952 kB/s | 32 kB 00:00 2024-03-04T07:24:26.310 INFO:teuthology.orchestra.run.smithi066.stdout:(44/176): flexiblas-netlib-3.0.4-8.el9.x86_64.r 30 MB/s | 3.0 MB 00:00 2024-03-04T07:24:26.352 INFO:teuthology.orchestra.run.smithi066.stdout:(45/176): flexiblas-openblas-openmp-3.0.4-8.el9 411 kB/s | 17 kB 00:00 2024-03-04T07:24:26.394 INFO:teuthology.orchestra.run.smithi066.stdout:(46/176): libaio-devel-0.3.111-13.el9.x86_64.rp 242 kB/s | 10 kB 00:00 2024-03-04T07:24:26.436 INFO:teuthology.orchestra.run.smithi066.stdout:(47/176): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.7 MB/s | 159 kB 00:00 2024-03-04T07:24:26.478 INFO:teuthology.orchestra.run.smithi066.stdout:(48/176): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.0 MB/s | 44 kB 00:00 2024-03-04T07:24:26.528 INFO:teuthology.orchestra.run.smithi066.stdout:(49/176): librdkafka-1.6.1-102.el9.x86_64.rpm 13 MB/s | 661 kB 00:00 2024-03-04T07:24:26.570 INFO:teuthology.orchestra.run.smithi066.stdout:(50/176): libtool-2.4.6-45.el9.x86_64.rpm 14 MB/s | 578 kB 00:00 2024-03-04T07:24:26.620 INFO:teuthology.orchestra.run.smithi066.stdout:(51/176): libuuid-devel-2.37.4-18.el9.x86_64.rp 485 kB/s | 24 kB 00:00 2024-03-04T07:24:26.623 INFO:teuthology.orchestra.run.smithi064.stdout:(1/176): ceph-19.0.0-1739.g1bc250c7.el9.x86_64. 14 kB/s | 6.6 kB 00:00 2024-03-04T07:24:26.662 INFO:teuthology.orchestra.run.smithi066.stdout:(52/176): libzstd-devel-1.5.1-2.el9.x86_64.rpm 1.1 MB/s | 46 kB 00:00 2024-03-04T07:24:26.696 INFO:teuthology.orchestra.run.smithi066.stdout:(53/176): lttng-ust-2.12.0-6.el9.x86_64.rpm 8.2 MB/s | 282 kB 00:00 2024-03-04T07:24:26.729 INFO:teuthology.orchestra.run.smithi066.stdout:(54/176): lua-5.4.4-4.el9.x86_64.rpm 5.5 MB/s | 187 kB 00:00 2024-03-04T07:24:26.763 INFO:teuthology.orchestra.run.smithi066.stdout:(55/176): openblas-0.3.21-2.el9.x86_64.rpm 1.0 MB/s | 34 kB 00:00 2024-03-04T07:24:26.922 INFO:teuthology.orchestra.run.smithi066.stdout:(56/176): openblas-openmp-0.3.21-2.el9.x86_64.r 30 MB/s | 4.7 MB 00:00 2024-03-04T07:24:27.022 INFO:teuthology.orchestra.run.smithi066.stdout:(57/176): openssl-devel-3.0.7-27.el9.x86_64.rpm 30 MB/s | 3.0 MB 00:00 2024-03-04T07:24:27.080 INFO:teuthology.orchestra.run.smithi066.stdout:(58/176): postgresql-13.11-1.el9.x86_64.rpm 26 MB/s | 1.5 MB 00:00 2024-03-04T07:24:27.139 INFO:teuthology.orchestra.run.smithi066.stdout:(59/176): postgresql-contrib-13.11-1.el9.x86_64 14 MB/s | 811 kB 00:00 2024-03-04T07:24:27.190 INFO:teuthology.orchestra.run.smithi066.stdout:(60/176): postgresql-private-libs-13.11-1.el9.x 2.6 MB/s | 134 kB 00:00 2024-03-04T07:24:27.466 INFO:teuthology.orchestra.run.smithi064.stdout:(2/176): ceph-fuse-19.0.0-1739.g1bc250c7.el9.x8 1.0 MB/s | 888 kB 00:00 2024-03-04T07:24:27.473 INFO:teuthology.orchestra.run.smithi066.stdout:(61/176): postgresql-server-13.11-1.el9.x86_64. 20 MB/s | 5.7 MB 00:00 2024-03-04T07:24:27.557 INFO:teuthology.orchestra.run.smithi066.stdout:(62/176): protobuf-3.14.0-13.el9.x86_64.rpm 12 MB/s | 1.0 MB 00:00 2024-03-04T07:24:27.632 INFO:teuthology.orchestra.run.smithi066.stdout:(63/176): python3-attrs-20.3.0-7.el9.noarch.rpm 1.0 MB/s | 78 kB 00:00 2024-03-04T07:24:27.944 INFO:teuthology.orchestra.run.smithi066.stdout:(64/176): python3-babel-2.9.1-2.el9.noarch.rpm 19 MB/s | 5.8 MB 00:00 2024-03-04T07:24:27.982 INFO:teuthology.orchestra.run.smithi066.stdout:(65/176): python3-jinja2-2.11.3-5.el9.noarch.rp 5.3 MB/s | 227 kB 00:00 2024-03-04T07:24:28.008 INFO:teuthology.orchestra.run.smithi066.stdout:(66/176): python3-jsonpatch-1.21-16.el9.noarch. 976 kB/s | 24 kB 00:00 2024-03-04T07:24:28.033 INFO:teuthology.orchestra.run.smithi066.stdout:(67/176): python3-jsonpointer-2.0-4.el9.noarch. 721 kB/s | 18 kB 00:00 2024-03-04T07:24:28.067 INFO:teuthology.orchestra.run.smithi066.stdout:(68/176): python3-mako-1.1.4-6.el9.noarch.rpm 4.2 MB/s | 144 kB 00:00 2024-03-04T07:24:28.100 INFO:teuthology.orchestra.run.smithi066.stdout:(69/176): python3-markupsafe-1.1.1-12.el9.x86_6 938 kB/s | 32 kB 00:00 2024-03-04T07:24:28.282 INFO:teuthology.orchestra.run.smithi064.stdout:(3/176): ceph-immutable-object-cache-19.0.0-173 178 kB/s | 145 kB 00:00 2024-03-04T07:24:28.351 INFO:teuthology.orchestra.run.smithi066.stdout:(70/176): python3-numpy-1.20.1-5.el9.x86_64.rpm 19 MB/s | 4.8 MB 00:00 2024-03-04T07:24:28.366 INFO:teuthology.orchestra.run.smithi064.stdout:(4/176): ceph-base-19.0.0-1739.g1bc250c7.el9.x8 2.4 MB/s | 5.3 MB 00:02 2024-03-04T07:24:28.401 INFO:teuthology.orchestra.run.smithi066.stdout:(71/176): python3-numpy-f2py-1.20.1-5.el9.x86_6 4.3 MB/s | 217 kB 00:00 2024-03-04T07:24:28.443 INFO:teuthology.orchestra.run.smithi066.stdout:(72/176): python3-oauthlib-3.1.1-5.el9.noarch.r 4.1 MB/s | 174 kB 00:00 2024-03-04T07:24:28.485 INFO:teuthology.orchestra.run.smithi066.stdout:(73/176): python3-packaging-20.9-5.el9.noarch.r 1.6 MB/s | 69 kB 00:00 2024-03-04T07:24:28.535 INFO:teuthology.orchestra.run.smithi066.stdout:(74/176): python3-prettytable-0.7.2-27.el9.noar 822 kB/s | 41 kB 00:00 2024-03-04T07:24:28.586 INFO:teuthology.orchestra.run.smithi066.stdout:(75/176): python3-protobuf-3.14.0-13.el9.noarch 4.7 MB/s | 238 kB 00:00 2024-03-04T07:24:28.636 INFO:teuthology.orchestra.run.smithi066.stdout:(76/176): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.6 MB/s | 132 kB 00:00 2024-03-04T07:24:28.678 INFO:teuthology.orchestra.run.smithi066.stdout:(77/176): python3-pyasn1-modules-0.4.8-6.el9.no 5.0 MB/s | 211 kB 00:00 2024-03-04T07:24:28.699 INFO:teuthology.orchestra.run.smithi064.stdout:(5/176): ceph-mgr-19.0.0-1739.g1bc250c7.el9.x86 4.5 MB/s | 1.5 MB 00:00 2024-03-04T07:24:28.703 INFO:teuthology.orchestra.run.smithi066.stdout:(78/176): python3-pytz-2021.1-5.el9.noarch.rpm 1.8 MB/s | 47 kB 00:00 2024-03-04T07:24:28.837 INFO:teuthology.orchestra.run.smithi066.stdout:(79/176): python3-requests-oauthlib-1.3.0-12.el 320 kB/s | 43 kB 00:00 2024-03-04T07:24:29.358 INFO:teuthology.orchestra.run.smithi064.stdout:(6/176): ceph-mds-19.0.0-1739.g1bc250c7.el9.x86 2.1 MB/s | 2.3 MB 00:01 2024-03-04T07:24:29.495 INFO:teuthology.orchestra.run.smithi066.stdout:(80/176): python3-scipy-1.6.2-8.el9.x86_64.rpm 24 MB/s | 16 MB 00:00 2024-03-04T07:24:29.529 INFO:teuthology.orchestra.run.smithi066.stdout:(81/176): python3-toml-0.10.2-6.el9.noarch.rpm 1.1 MB/s | 39 kB 00:00 2024-03-04T07:24:29.571 INFO:teuthology.orchestra.run.smithi066.stdout:(82/176): python3-wcwidth-0.2.5-8.el9.noarch.rp 978 kB/s | 41 kB 00:00 2024-03-04T07:24:29.622 INFO:teuthology.orchestra.run.smithi066.stdout:(83/176): qatlib-23.11.0-1.el9.x86_64.rpm 4.2 MB/s | 215 kB 00:00 2024-03-04T07:24:29.691 INFO:teuthology.orchestra.run.smithi066.stdout:(84/176): qatlib-service-23.11.0-1.el9.x86_64.r 485 kB/s | 34 kB 00:00 2024-03-04T07:24:29.722 INFO:teuthology.orchestra.run.smithi066.stdout:(85/176): qatzip-libs-1.1.2-2.el9.x86_64.rpm 1.4 MB/s | 43 kB 00:00 2024-03-04T07:24:29.764 INFO:teuthology.orchestra.run.smithi066.stdout:(86/176): socat-1.7.4.1-5.el9.x86_64.rpm 7.0 MB/s | 300 kB 00:00 2024-03-04T07:24:29.792 INFO:teuthology.orchestra.run.smithi064.stdout:(7/176): ceph-common-19.0.0-1739.g1bc250c7.el9. 5.6 MB/s | 20 MB 00:03 2024-03-04T07:24:29.797 INFO:teuthology.orchestra.run.smithi066.stdout:(87/176): uuid-1.6.2-55.el9.x86_64.rpm 1.7 MB/s | 56 kB 00:00 2024-03-04T07:24:29.856 INFO:teuthology.orchestra.run.smithi066.stdout:(88/176): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 1.5 MB/s | 87 kB 00:00 2024-03-04T07:24:29.858 INFO:teuthology.orchestra.run.smithi064.stdout:(8/176): ceph-mon-19.0.0-1739.g1bc250c7.el9.x86 4.0 MB/s | 4.7 MB 00:01 2024-03-04T07:24:29.890 INFO:teuthology.orchestra.run.smithi066.stdout:(89/176): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.8 MB/s | 63 kB 00:00 2024-03-04T07:24:29.923 INFO:teuthology.orchestra.run.smithi066.stdout:(90/176): zlib-devel-1.2.11-40.el9.x86_64.rpm 1.3 MB/s | 44 kB 00:00 2024-03-04T07:24:29.965 INFO:teuthology.orchestra.run.smithi066.stdout:(91/176): fuse-2.9.9-15.el9.x86_64.rpm 1.8 MB/s | 79 kB 00:00 2024-03-04T07:24:29.975 INFO:teuthology.orchestra.run.smithi064.stdout:(9/176): ceph-selinux-19.0.0-1739.g1bc250c7.el9 216 kB/s | 25 kB 00:00 2024-03-04T07:24:30.007 INFO:teuthology.orchestra.run.smithi066.stdout:(92/176): jq-1.6-16.el9.x86_64.rpm 4.4 MB/s | 186 kB 00:00 2024-03-04T07:24:30.069 INFO:teuthology.orchestra.run.smithi066.stdout:(93/176): libgfortran-11.4.1-3.el9.x86_64.rpm 13 MB/s | 802 kB 00:00 2024-03-04T07:24:30.107 INFO:teuthology.orchestra.run.smithi066.stdout:(94/176): libquadmath-11.4.1-3.el9.x86_64.rpm 4.6 MB/s | 194 kB 00:00 2024-03-04T07:24:30.149 INFO:teuthology.orchestra.run.smithi066.stdout:(95/176): oniguruma-6.9.6-1.el9.5.x86_64.rpm 5.1 MB/s | 217 kB 00:00 2024-03-04T07:24:30.275 INFO:teuthology.orchestra.run.smithi066.stdout:(96/176): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.9 MB/s | 241 kB 00:00 2024-03-04T07:24:30.350 INFO:teuthology.orchestra.run.smithi066.stdout:(97/176): python3-cryptography-36.0.1-4.el9.x86 15 MB/s | 1.1 MB 00:00 2024-03-04T07:24:30.386 INFO:teuthology.orchestra.run.smithi066.stdout:(98/176): python3-ply-3.11-14.el9.noarch.rpm 2.8 MB/s | 103 kB 00:00 2024-03-04T07:24:30.420 INFO:teuthology.orchestra.run.smithi066.stdout:(99/176): python3-pycparser-2.20-6.el9.noarch.r 3.6 MB/s | 124 kB 00:00 2024-03-04T07:24:30.462 INFO:teuthology.orchestra.run.smithi066.stdout:(100/176): python3-requests-2.25.1-8.el9.noarch 2.7 MB/s | 113 kB 00:00 2024-03-04T07:24:30.504 INFO:teuthology.orchestra.run.smithi066.stdout:(101/176): python3-urllib3-1.26.5-5.el9.noarch. 4.4 MB/s | 187 kB 00:00 2024-03-04T07:24:30.712 INFO:teuthology.orchestra.run.smithi066.stdout:(102/176): python3-devel-3.9.17-2.el9.x86_64.rp 1.2 MB/s | 245 kB 00:00 2024-03-04T07:24:30.796 INFO:teuthology.orchestra.run.smithi066.stdout:(103/176): lua-devel-5.4.4-4.el9.x86_64.rpm 267 kB/s | 22 kB 00:00 2024-03-04T07:24:30.963 INFO:teuthology.orchestra.run.smithi066.stdout:(104/176): protobuf-compiler-3.14.0-13.el9.x86_ 5.1 MB/s | 863 kB 00:00 2024-03-04T07:24:31.225 INFO:teuthology.orchestra.run.smithi064.stdout:(10/176): ceph-radosgw-19.0.0-1739.g1bc250c7.el 6.8 MB/s | 9.8 MB 00:01 2024-03-04T07:24:31.328 INFO:teuthology.orchestra.run.smithi064.stdout:(11/176): cephfs-mirror-19.0.0-1739.g1bc250c7.e 2.1 MB/s | 220 kB 00:00 2024-03-04T07:24:31.355 INFO:teuthology.orchestra.run.smithi066.stdout:(105/176): ceph-mgr-diskprediction-local-19.0.0 948 kB/s | 7.4 MB 00:08 2024-03-04T07:24:31.388 INFO:teuthology.orchestra.run.smithi066.stdout:(106/176): abseil-cpp-20211102.0-3.el9.x86_64.r 1.3 MB/s | 548 kB 00:00 2024-03-04T07:24:31.420 INFO:teuthology.orchestra.run.smithi064.stdout:(12/176): cephfs-shell-19.0.0-1739.g1bc250c7.el 295 kB/s | 27 kB 00:00 2024-03-04T07:24:31.439 INFO:teuthology.orchestra.run.smithi066.stdout:(107/176): grpc-data-1.46.7-10.el9.noarch.rpm 389 kB/s | 19 kB 00:00 2024-03-04T07:24:31.512 INFO:teuthology.orchestra.run.smithi064.stdout:(13/176): libcephfs-devel-19.0.0-1739.g1bc250c7 342 kB/s | 31 kB 00:00 2024-03-04T07:24:31.630 INFO:teuthology.orchestra.run.smithi066.stdout:(108/176): gperftools-libs-2.9.1-2.el9.x86_64.r 1.1 MB/s | 309 kB 00:00 2024-03-04T07:24:31.687 INFO:teuthology.orchestra.run.smithi064.stdout:(14/176): libcephfs2-19.0.0-1739.g1bc250c7.el9. 4.2 MB/s | 747 kB 00:00 2024-03-04T07:24:31.689 INFO:teuthology.orchestra.run.smithi066.stdout:(109/176): libarrow-doc-9.0.0-10.el9.noarch.rpm 444 kB/s | 26 kB 00:00 2024-03-04T07:24:31.739 INFO:teuthology.orchestra.run.smithi066.stdout:(110/176): liboath-2.6.7-2.el9.x86_64.rpm 976 kB/s | 49 kB 00:00 2024-03-04T07:24:31.779 INFO:teuthology.orchestra.run.smithi064.stdout:(15/176): libcephsqlite-19.0.0-1739.g1bc250c7.e 1.7 MB/s | 163 kB 00:00 2024-03-04T07:24:31.790 INFO:teuthology.orchestra.run.smithi066.stdout:(111/176): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-03-04T07:24:31.871 INFO:teuthology.orchestra.run.smithi064.stdout:(16/176): librados-devel-19.0.0-1739.g1bc250c7. 1.4 MB/s | 127 kB 00:00 2024-03-04T07:24:31.873 INFO:teuthology.orchestra.run.smithi066.stdout:(112/176): luarocks-3.9.2-1.el9.noarch.rpm 1.8 MB/s | 151 kB 00:00 2024-03-04T07:24:31.988 INFO:teuthology.orchestra.run.smithi064.stdout:(17/176): libradosstriper1-19.0.0-1739.g1bc250c 4.2 MB/s | 501 kB 00:00 2024-03-04T07:24:32.182 INFO:teuthology.orchestra.run.smithi066.stdout:(113/176): parquet-libs-9.0.0-10.el9.x86_64.rpm 2.7 MB/s | 838 kB 00:00 2024-03-04T07:24:32.257 INFO:teuthology.orchestra.run.smithi066.stdout:(114/176): libarrow-9.0.0-10.el9.x86_64.rpm 5.4 MB/s | 4.4 MB 00:00 2024-03-04T07:24:32.332 INFO:teuthology.orchestra.run.smithi066.stdout:(115/176): python3-bcrypt-3.2.2-1.el9.x86_64.rp 580 kB/s | 43 kB 00:00 2024-03-04T07:24:32.366 INFO:teuthology.orchestra.run.smithi066.stdout:(116/176): python3-cachetools-4.2.4-1.el9.noarc 965 kB/s | 32 kB 00:00 2024-03-04T07:24:32.400 INFO:teuthology.orchestra.run.smithi066.stdout:(117/176): python3-certifi-2021.10.8-2.el9.noar 451 kB/s | 15 kB 00:00 2024-03-04T07:24:32.425 INFO:teuthology.orchestra.run.smithi066.stdout:(118/176): python3-asyncssh-2.13.2-4.el9.noarch 2.2 MB/s | 548 kB 00:00 2024-03-04T07:24:32.442 INFO:teuthology.orchestra.run.smithi066.stdout:(119/176): python3-cheroot-8.6.0-4.el9.noarch.r 4.0 MB/s | 172 kB 00:00 2024-03-04T07:24:32.559 INFO:teuthology.orchestra.run.smithi066.stdout:(120/176): python3-cmd2-2.4.2-2.el9.noarch.rpm 2.5 MB/s | 303 kB 00:00 2024-03-04T07:24:32.584 INFO:teuthology.orchestra.run.smithi066.stdout:(121/176): python3-cherrypy-18.6.1-2.el9.noarch 2.2 MB/s | 358 kB 00:00 2024-03-04T07:24:32.609 INFO:teuthology.orchestra.run.smithi066.stdout:(122/176): python3-colorama-0.4.4-12.el9.noarch 705 kB/s | 35 kB 00:00 2024-03-04T07:24:32.660 INFO:teuthology.orchestra.run.smithi066.stdout:(123/176): python3-google-auth-2.28.0-1.el9.noa 2.6 MB/s | 203 kB 00:00 2024-03-04T07:24:32.726 INFO:teuthology.orchestra.run.smithi066.stdout:(124/176): python3-grpcio-tools-1.46.7-10.el9.x 2.1 MB/s | 144 kB 00:00 2024-03-04T07:24:32.768 INFO:teuthology.orchestra.run.smithi066.stdout:(125/176): python3-jaraco-8.2.1-3.el9.noarch.rp 257 kB/s | 11 kB 00:00 2024-03-04T07:24:32.780 INFO:teuthology.orchestra.run.smithi064.stdout:(18/176): librgw2-19.0.0-1739.g1bc250c7.el9.x86 6.5 MB/s | 5.1 MB 00:00 2024-03-04T07:24:32.827 INFO:teuthology.orchestra.run.smithi066.stdout:(126/176): python3-grpcio-1.46.7-10.el9.x86_64. 9.4 MB/s | 2.0 MB 00:00 2024-03-04T07:24:32.853 INFO:teuthology.orchestra.run.smithi066.stdout:(127/176): python3-jaraco-classes-3.2.1-5.el9.n 211 kB/s | 18 kB 00:00 2024-03-04T07:24:32.878 INFO:teuthology.orchestra.run.smithi066.stdout:(128/176): python3-jaraco-collections-3.0.0-8.e 461 kB/s | 23 kB 00:00 2024-03-04T07:24:32.880 INFO:teuthology.orchestra.run.smithi064.stdout:(19/176): python3-ceph-argparse-19.0.0-1739.g1b 453 kB/s | 45 kB 00:00 2024-03-04T07:24:32.907 INFO:teuthology.orchestra.run.smithi066.stdout:(129/176): python3-jaraco-functools-3.5.0-2.el9 388 kB/s | 19 kB 00:00 2024-03-04T07:24:32.929 INFO:teuthology.orchestra.run.smithi066.stdout:(130/176): python3-jaraco-text-3.2.0-6.el9.noar 390 kB/s | 20 kB 00:00 2024-03-04T07:24:32.971 INFO:teuthology.orchestra.run.smithi066.stdout:(131/176): python3-logutils-0.3.5-21.el9.noarch 1.1 MB/s | 46 kB 00:00 2024-03-04T07:24:32.972 INFO:teuthology.orchestra.run.smithi064.stdout:(20/176): python3-ceph-common-19.0.0-1739.g1bc2 1.4 MB/s | 131 kB 00:00 2024-03-04T07:24:33.012 INFO:teuthology.orchestra.run.smithi066.stdout:(132/176): python3-more-itertools-8.12.0-2.el9. 1.8 MB/s | 79 kB 00:00 2024-03-04T07:24:33.054 INFO:teuthology.orchestra.run.smithi066.stdout:(133/176): python3-natsort-7.1.1-5.el9.noarch.r 1.4 MB/s | 58 kB 00:00 2024-03-04T07:24:33.064 INFO:teuthology.orchestra.run.smithi064.stdout:(21/176): python3-cephfs-19.0.0-1739.g1bc250c7. 1.7 MB/s | 162 kB 00:00 2024-03-04T07:24:33.105 INFO:teuthology.orchestra.run.smithi066.stdout:(134/176): python3-pecan-1.4.2-3.el9.noarch.rpm 5.3 MB/s | 272 kB 00:00 2024-03-04T07:24:33.146 INFO:teuthology.orchestra.run.smithi066.stdout:(135/176): python3-portend-3.1.0-2.el9.noarch.r 396 kB/s | 16 kB 00:00 2024-03-04T07:24:33.165 INFO:teuthology.orchestra.run.smithi064.stdout:(22/176): python3-rados-19.0.0-1739.g1bc250c7.e 3.1 MB/s | 321 kB 00:00 2024-03-04T07:24:33.197 INFO:teuthology.orchestra.run.smithi066.stdout:(136/176): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.7 MB/s | 90 kB 00:00 2024-03-04T07:24:33.247 INFO:teuthology.orchestra.run.smithi066.stdout:(137/176): python3-pyperclip-1.8.2-2.el9.noarch 574 kB/s | 28 kB 00:00 2024-03-04T07:24:33.264 INFO:teuthology.orchestra.run.smithi064.stdout:(23/176): python3-rbd-19.0.0-1739.g1bc250c7.el9 2.9 MB/s | 298 kB 00:00 2024-03-04T07:24:33.289 INFO:teuthology.orchestra.run.smithi066.stdout:(138/176): python3-repoze-lru-0.7-16.el9.noarch 738 kB/s | 31 kB 00:00 2024-03-04T07:24:33.331 INFO:teuthology.orchestra.run.smithi066.stdout:(139/176): python3-routes-2.5.1-5.el9.noarch.rp 4.4 MB/s | 188 kB 00:00 2024-03-04T07:24:33.340 INFO:teuthology.orchestra.run.smithi064.stdout:(24/176): python3-rgw-19.0.0-1739.g1bc250c7.el9 1.3 MB/s | 99 kB 00:00 2024-03-04T07:24:33.365 INFO:teuthology.orchestra.run.smithi066.stdout:(140/176): python3-kubernetes-26.1.0-2.el9.noar 2.2 MB/s | 1.0 MB 00:00 2024-03-04T07:24:33.390 INFO:teuthology.orchestra.run.smithi066.stdout:(141/176): python3-rsa-4.9-2.el9.noarch.rpm 1.0 MB/s | 59 kB 00:00 2024-03-04T07:24:33.407 INFO:teuthology.orchestra.run.smithi064.stdout:(25/176): rbd-fuse-19.0.0-1739.g1bc250c7.el9.x8 1.3 MB/s | 85 kB 00:00 2024-03-04T07:24:33.415 INFO:teuthology.orchestra.run.smithi066.stdout:(142/176): python3-tempora-5.0.0-2.el9.noarch.r 713 kB/s | 36 kB 00:00 2024-03-04T07:24:33.440 INFO:teuthology.orchestra.run.smithi066.stdout:(143/176): python3-typing-extensions-4.4.0-2.el 1.0 MB/s | 51 kB 00:00 2024-03-04T07:24:33.482 INFO:teuthology.orchestra.run.smithi066.stdout:(144/176): python3-websocket-client-1.2.3-2.el9 2.1 MB/s | 90 kB 00:00 2024-03-04T07:24:33.533 INFO:teuthology.orchestra.run.smithi066.stdout:(145/176): python3-webob-1.8.7-6.el9.noarch.rpm 1.9 MB/s | 230 kB 00:00 2024-03-04T07:24:33.558 INFO:teuthology.orchestra.run.smithi066.stdout:(146/176): python3-werkzeug-2.0.3-3.el9.1.noarc 5.6 MB/s | 427 kB 00:00 2024-03-04T07:24:33.583 INFO:teuthology.orchestra.run.smithi066.stdout:(147/176): python3-zc-lockfile-2.0-10.el9.noarc 399 kB/s | 20 kB 00:00 2024-03-04T07:24:33.609 INFO:teuthology.orchestra.run.smithi066.stdout:(148/176): re2-20211101-3.el9.x86_64.rpm 3.7 MB/s | 192 kB 00:00 2024-03-04T07:24:33.890 INFO:teuthology.orchestra.run.smithi064.stdout:(26/176): rbd-mirror-19.0.0-1739.g1bc250c7.el9. 6.4 MB/s | 3.1 MB 00:00 2024-03-04T07:24:33.974 INFO:teuthology.orchestra.run.smithi064.stdout:(27/176): rbd-nbd-19.0.0-1739.g1bc250c7.el9.x86 2.0 MB/s | 171 kB 00:00 2024-03-04T07:24:34.034 INFO:teuthology.orchestra.run.smithi066.stdout:(149/176): ceph-test-19.0.0-1739.g1bc250c7.el9. 1.6 MB/s | 47 MB 00:30 2024-03-04T07:24:34.058 INFO:teuthology.orchestra.run.smithi064.stdout:(28/176): ceph-grafana-dashboards-19.0.0-1739.g 376 kB/s | 31 kB 00:00 2024-03-04T07:24:34.133 INFO:teuthology.orchestra.run.smithi064.stdout:(29/176): ceph-mgr-cephadm-19.0.0-1739.g1bc250c 1.8 MB/s | 138 kB 00:00 2024-03-04T07:24:34.192 INFO:teuthology.orchestra.run.smithi066.stdout:(150/176): thrift-0.15.0-2.el9.x86_64.rpm 2.6 MB/s | 1.6 MB 00:00 2024-03-04T07:24:34.226 INFO:teuthology.orchestra.run.smithi066.stdout:(151/176): libacl-devel-2.3.1-4.el9.x86_64.rpm 2.0 MB/s | 68 kB 00:00 2024-03-04T07:24:34.326 INFO:teuthology.orchestra.run.smithi066.stdout:(152/176): NetworkManager-1.46.0-1.el9.x86_64.r 23 MB/s | 2.3 MB 00:00 2024-03-04T07:24:34.410 INFO:teuthology.orchestra.run.smithi066.stdout:(153/176): NetworkManager-libnm-1.46.0-1.el9.x8 22 MB/s | 1.8 MB 00:00 2024-03-04T07:24:34.433 INFO:teuthology.orchestra.run.smithi064.stdout:(30/176): ceph-mgr-dashboard-19.0.0-1739.g1bc25 5.9 MB/s | 1.8 MB 00:00 2024-03-04T07:24:34.435 INFO:teuthology.orchestra.run.smithi066.stdout:(154/176): NetworkManager-team-1.46.0-1.el9.x86 1.6 MB/s | 41 kB 00:00 2024-03-04T07:24:34.477 INFO:teuthology.orchestra.run.smithi066.stdout:(155/176): NetworkManager-tui-1.46.0-1.el9.x86_ 5.8 MB/s | 247 kB 00:00 2024-03-04T07:24:34.511 INFO:teuthology.orchestra.run.smithi066.stdout:(156/176): acl-2.3.1-4.el9.x86_64.rpm 2.0 MB/s | 69 kB 00:00 2024-03-04T07:24:34.544 INFO:teuthology.orchestra.run.smithi066.stdout:(157/176): elfutils-debuginfod-client-0.190-2.e 1.0 MB/s | 35 kB 00:00 2024-03-04T07:24:34.586 INFO:teuthology.orchestra.run.smithi066.stdout:(158/176): elfutils-libelf-0.190-2.el9.x86_64.r 4.5 MB/s | 191 kB 00:00 2024-03-04T07:24:34.620 INFO:teuthology.orchestra.run.smithi066.stdout:(159/176): elfutils-libs-0.190-2.el9.x86_64.rpm 7.4 MB/s | 254 kB 00:00 2024-03-04T07:24:34.670 INFO:teuthology.orchestra.run.smithi066.stdout:(160/176): iproute-6.7.0-2.el9.x86_64.rpm 16 MB/s | 802 kB 00:00 2024-03-04T07:24:34.712 INFO:teuthology.orchestra.run.smithi066.stdout:(161/176): iproute-tc-6.7.0-2.el9.x86_64.rpm 10 MB/s | 433 kB 00:00 2024-03-04T07:24:34.746 INFO:teuthology.orchestra.run.smithi066.stdout:(162/176): libacl-2.3.1-4.el9.x86_64.rpm 656 kB/s | 22 kB 00:00 2024-03-04T07:24:34.780 INFO:teuthology.orchestra.run.smithi066.stdout:(163/176): libblkid-2.37.4-18.el9.x86_64.rpm 3.2 MB/s | 107 kB 00:00 2024-03-04T07:24:34.805 INFO:teuthology.orchestra.run.smithi066.stdout:(164/176): libfdisk-2.37.4-18.el9.x86_64.rpm 6.0 MB/s | 154 kB 00:00 2024-03-04T07:24:34.838 INFO:teuthology.orchestra.run.smithi066.stdout:(165/176): libmount-2.37.4-18.el9.x86_64.rpm 4.0 MB/s | 135 kB 00:00 2024-03-04T07:24:34.872 INFO:teuthology.orchestra.run.smithi066.stdout:(166/176): libsmartcols-2.37.4-18.el9.x86_64.rp 1.8 MB/s | 63 kB 00:00 2024-03-04T07:24:34.905 INFO:teuthology.orchestra.run.smithi066.stdout:(167/176): libuuid-2.37.4-18.el9.x86_64.rpm 833 kB/s | 28 kB 00:00 2024-03-04T07:24:34.964 INFO:teuthology.orchestra.run.smithi066.stdout:(168/176): openssl-3.0.7-27.el9.x86_64.rpm 20 MB/s | 1.2 MB 00:00 2024-03-04T07:24:35.048 INFO:teuthology.orchestra.run.smithi066.stdout:(169/176): openssl-libs-3.0.7-27.el9.x86_64.rpm 26 MB/s | 2.2 MB 00:00 2024-03-04T07:24:35.123 INFO:teuthology.orchestra.run.smithi066.stdout:(170/176): util-linux-2.37.4-18.el9.x86_64.rpm 29 MB/s | 2.2 MB 00:00 2024-03-04T07:24:35.165 INFO:teuthology.orchestra.run.smithi066.stdout:(171/176): util-linux-core-2.37.4-18.el9.x86_64 10 MB/s | 438 kB 00:00 2024-03-04T07:24:35.198 INFO:teuthology.orchestra.run.smithi066.stdout:(172/176): util-linux-user-2.37.4-18.el9.x86_64 907 kB/s | 30 kB 00:00 2024-03-04T07:24:35.240 INFO:teuthology.orchestra.run.smithi066.stdout:(173/176): xfsdump-3.1.12-4.el9.x86_64.rpm 7.6 MB/s | 321 kB 00:00 2024-03-04T07:24:35.290 INFO:teuthology.orchestra.run.smithi066.stdout:(174/176): xfsprogs-6.3.0-1.el9.x86_64.rpm 21 MB/s | 1.1 MB 00:00 2024-03-04T07:24:35.492 INFO:teuthology.orchestra.run.smithi064.stdout:(31/176): ceph-mgr-diskprediction-local-19.0.0- 7.0 MB/s | 7.4 MB 00:01 2024-03-04T07:24:35.550 INFO:teuthology.orchestra.run.smithi064.stdout:(32/176): ceph-mgr-modules-core-19.0.0-1739.g1b 4.2 MB/s | 248 kB 00:00 2024-03-04T07:24:35.625 INFO:teuthology.orchestra.run.smithi064.stdout:(33/176): ceph-mgr-rook-19.0.0-1739.g1bc250c7.e 656 kB/s | 49 kB 00:00 2024-03-04T07:24:35.778 INFO:teuthology.orchestra.run.smithi064.stdout:(34/176): ceph-prometheus-alerts-19.0.0-1739.g1 101 kB/s | 15 kB 00:00 2024-03-04T07:24:35.868 INFO:teuthology.orchestra.run.smithi064.stdout:(35/176): ceph-volume-19.0.0-1739.g1bc250c7.el9 2.9 MB/s | 264 kB 00:00 2024-03-04T07:24:36.152 INFO:teuthology.orchestra.run.smithi064.stdout:(36/176): ceph-osd-19.0.0-1739.g1bc250c7.el9.x8 2.5 MB/s | 17 MB 00:06 2024-03-04T07:24:36.181 INFO:teuthology.orchestra.run.smithi064.stdout:(37/176): cephadm-19.0.0-1739.g1bc250c7.el9.noa 2.4 MB/s | 748 kB 00:00 2024-03-04T07:24:36.207 INFO:teuthology.orchestra.run.smithi066.stdout:(175/176): librados2-19.0.0-1739.g1bc250c7.el9. 1.3 MB/s | 3.4 MB 00:02 2024-03-04T07:24:36.235 INFO:teuthology.orchestra.run.smithi064.stdout:(38/176): cephfs-top-19.0.0-1739.g1bc250c7.el9. 218 kB/s | 18 kB 00:00 2024-03-04T07:24:36.361 INFO:teuthology.orchestra.run.smithi064.stdout:(39/176): bison-3.7.4-5.el9.x86_64.rpm 7.3 MB/s | 920 kB 00:00 2024-03-04T07:24:36.386 INFO:teuthology.orchestra.run.smithi064.stdout:(40/176): automake-1.16.2-8.el9.noarch.rpm 3.1 MB/s | 662 kB 00:00 2024-03-04T07:24:36.411 INFO:teuthology.orchestra.run.smithi064.stdout:(41/176): boost-program-options-1.75.0-8.el9.x8 2.1 MB/s | 106 kB 00:00 2024-03-04T07:24:36.428 INFO:teuthology.orchestra.run.smithi064.stdout:(42/176): elfutils-libelf-devel-0.190-2.el9.x86 490 kB/s | 20 kB 00:00 2024-03-04T07:24:36.470 INFO:teuthology.orchestra.run.smithi064.stdout:(43/176): flex-2.6.4-9.el9.x86_64.rpm 5.1 MB/s | 307 kB 00:00 2024-03-04T07:24:36.495 INFO:teuthology.orchestra.run.smithi064.stdout:(44/176): flexiblas-3.0.4-8.el9.x86_64.rpm 472 kB/s | 32 kB 00:00 2024-03-04T07:24:36.571 INFO:teuthology.orchestra.run.smithi064.stdout:(45/176): flexiblas-netlib-3.0.4-8.el9.x86_64.r 30 MB/s | 3.0 MB 00:00 2024-03-04T07:24:36.582 INFO:teuthology.orchestra.run.smithi066.stdout:(176/176): librbd1-19.0.0-1739.g1bc250c7.el9.x8 1.2 MB/s | 3.1 MB 00:02 2024-03-04T07:24:36.588 INFO:teuthology.orchestra.run.smithi064.stdout:(46/176): flexiblas-openblas-openmp-3.0.4-8.el9 187 kB/s | 17 kB 00:00 2024-03-04T07:24:36.588 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2024-03-04T07:24:36.588 INFO:teuthology.orchestra.run.smithi066.stdout:Total 5.8 MB/s | 224 MB 00:38 2024-03-04T07:24:36.604 INFO:teuthology.orchestra.run.smithi064.stdout:(47/176): libaio-devel-0.3.111-13.el9.x86_64.rp 300 kB/s | 10 kB 00:00 2024-03-04T07:24:36.630 INFO:teuthology.orchestra.run.smithi064.stdout:(48/176): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.7 MB/s | 159 kB 00:00 2024-03-04T07:24:36.633 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2024-03-04T07:24:36.647 INFO:teuthology.orchestra.run.smithi064.stdout:(49/176): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.0 MB/s | 44 kB 00:00 2024-03-04T07:24:36.680 INFO:teuthology.orchestra.run.smithi064.stdout:(50/176): librdkafka-1.6.1-102.el9.x86_64.rpm 13 MB/s | 661 kB 00:00 2024-03-04T07:24:36.706 INFO:teuthology.orchestra.run.smithi064.stdout:(51/176): libtool-2.4.6-45.el9.x86_64.rpm 9.8 MB/s | 578 kB 00:00 2024-03-04T07:24:36.731 INFO:teuthology.orchestra.run.smithi064.stdout:(52/176): libuuid-devel-2.37.4-18.el9.x86_64.rp 491 kB/s | 24 kB 00:00 2024-03-04T07:24:36.756 INFO:teuthology.orchestra.run.smithi064.stdout:(53/176): libzstd-devel-1.5.1-2.el9.x86_64.rpm 924 kB/s | 46 kB 00:00 2024-03-04T07:24:36.782 INFO:teuthology.orchestra.run.smithi064.stdout:(54/176): lttng-ust-2.12.0-6.el9.x86_64.rpm 5.5 MB/s | 282 kB 00:00 2024-03-04T07:24:36.807 INFO:teuthology.orchestra.run.smithi064.stdout:(55/176): lua-5.4.4-4.el9.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2024-03-04T07:24:36.832 INFO:teuthology.orchestra.run.smithi064.stdout:(56/176): openblas-0.3.21-2.el9.x86_64.rpm 687 kB/s | 34 kB 00:00 2024-03-04T07:24:36.933 INFO:teuthology.orchestra.run.smithi064.stdout:(57/176): openblas-openmp-0.3.21-2.el9.x86_64.r 38 MB/s | 4.7 MB 00:00 2024-03-04T07:24:36.982 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2024-03-04T07:24:36.982 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2024-03-04T07:24:36.991 INFO:teuthology.orchestra.run.smithi064.stdout:(58/176): openssl-devel-3.0.7-27.el9.x86_64.rpm 19 MB/s | 3.0 MB 00:00 2024-03-04T07:24:37.024 INFO:teuthology.orchestra.run.smithi064.stdout:(59/176): postgresql-13.11-1.el9.x86_64.rpm 17 MB/s | 1.5 MB 00:00 2024-03-04T07:24:37.058 INFO:teuthology.orchestra.run.smithi064.stdout:(60/176): postgresql-contrib-13.11-1.el9.x86_64 12 MB/s | 811 kB 00:00 2024-03-04T07:24:37.083 INFO:teuthology.orchestra.run.smithi064.stdout:(61/176): postgresql-private-libs-13.11-1.el9.x 2.3 MB/s | 134 kB 00:00 2024-03-04T07:24:37.209 INFO:teuthology.orchestra.run.smithi064.stdout:(62/176): postgresql-server-13.11-1.el9.x86_64. 38 MB/s | 5.7 MB 00:00 2024-03-04T07:24:37.242 INFO:teuthology.orchestra.run.smithi064.stdout:(63/176): protobuf-3.14.0-13.el9.x86_64.rpm 6.5 MB/s | 1.0 MB 00:00 2024-03-04T07:24:37.267 INFO:teuthology.orchestra.run.smithi064.stdout:(64/176): python3-attrs-20.3.0-7.el9.noarch.rpm 1.3 MB/s | 78 kB 00:00 2024-03-04T07:24:37.385 INFO:teuthology.orchestra.run.smithi064.stdout:(65/176): python3-babel-2.9.1-2.el9.noarch.rpm 41 MB/s | 5.8 MB 00:00 2024-03-04T07:24:37.410 INFO:teuthology.orchestra.run.smithi064.stdout:(66/176): python3-jinja2-2.11.3-5.el9.noarch.rp 1.6 MB/s | 227 kB 00:00 2024-03-04T07:24:37.435 INFO:teuthology.orchestra.run.smithi064.stdout:(67/176): python3-jsonpatch-1.21-16.el9.noarch. 490 kB/s | 24 kB 00:00 2024-03-04T07:24:37.469 INFO:teuthology.orchestra.run.smithi064.stdout:(68/176): python3-jsonpointer-2.0-4.el9.noarch. 307 kB/s | 18 kB 00:00 2024-03-04T07:24:37.494 INFO:teuthology.orchestra.run.smithi064.stdout:(69/176): python3-mako-1.1.4-6.el9.noarch.rpm 2.4 MB/s | 144 kB 00:00 2024-03-04T07:24:37.520 INFO:teuthology.orchestra.run.smithi064.stdout:(70/176): python3-markupsafe-1.1.1-12.el9.x86_6 631 kB/s | 32 kB 00:00 2024-03-04T07:24:37.620 INFO:teuthology.orchestra.run.smithi064.stdout:(71/176): python3-numpy-1.20.1-5.el9.x86_64.rpm 38 MB/s | 4.8 MB 00:00 2024-03-04T07:24:37.645 INFO:teuthology.orchestra.run.smithi064.stdout:(72/176): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.7 MB/s | 217 kB 00:00 2024-03-04T07:24:37.671 INFO:teuthology.orchestra.run.smithi064.stdout:(73/176): python3-oauthlib-3.1.1-5.el9.noarch.r 3.4 MB/s | 174 kB 00:00 2024-03-04T07:24:37.696 INFO:teuthology.orchestra.run.smithi064.stdout:(74/176): python3-packaging-20.9-5.el9.noarch.r 1.3 MB/s | 69 kB 00:00 2024-03-04T07:24:37.721 INFO:teuthology.orchestra.run.smithi064.stdout:(75/176): python3-prettytable-0.7.2-27.el9.noar 823 kB/s | 41 kB 00:00 2024-03-04T07:24:37.755 INFO:teuthology.orchestra.run.smithi064.stdout:(76/176): python3-protobuf-3.14.0-13.el9.noarch 4.0 MB/s | 238 kB 00:00 2024-03-04T07:24:37.780 INFO:teuthology.orchestra.run.smithi064.stdout:(77/176): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.2 MB/s | 132 kB 00:00 2024-03-04T07:24:37.805 INFO:teuthology.orchestra.run.smithi064.stdout:(78/176): python3-pyasn1-modules-0.4.8-6.el9.no 4.1 MB/s | 211 kB 00:00 2024-03-04T07:24:37.830 INFO:teuthology.orchestra.run.smithi064.stdout:(79/176): python3-pytz-2021.1-5.el9.noarch.rpm 949 kB/s | 47 kB 00:00 2024-03-04T07:24:37.856 INFO:teuthology.orchestra.run.smithi064.stdout:(80/176): python3-requests-oauthlib-1.3.0-12.el 855 kB/s | 43 kB 00:00 2024-03-04T07:24:38.123 INFO:teuthology.orchestra.run.smithi064.stdout:(81/176): python3-toml-0.10.2-6.el9.noarch.rpm 146 kB/s | 39 kB 00:00 2024-03-04T07:24:38.281 INFO:teuthology.orchestra.run.smithi064.stdout:(82/176): python3-scipy-1.6.2-8.el9.x86_64.rpm 35 MB/s | 16 MB 00:00 2024-03-04T07:24:38.298 INFO:teuthology.orchestra.run.smithi064.stdout:(83/176): python3-wcwidth-0.2.5-8.el9.noarch.rp 234 kB/s | 41 kB 00:00 2024-03-04T07:24:38.323 INFO:teuthology.orchestra.run.smithi064.stdout:(84/176): qatlib-23.11.0-1.el9.x86_64.rpm 5.0 MB/s | 215 kB 00:00 2024-03-04T07:24:38.340 INFO:teuthology.orchestra.run.smithi064.stdout:(85/176): qatlib-service-23.11.0-1.el9.x86_64.r 812 kB/s | 34 kB 00:00 2024-03-04T07:24:38.358 INFO:teuthology.orchestra.run.smithi064.stdout:(86/176): qatzip-libs-1.1.2-2.el9.x86_64.rpm 1.3 MB/s | 43 kB 00:00 2024-03-04T07:24:38.382 INFO:teuthology.orchestra.run.smithi064.stdout:(87/176): socat-1.7.4.1-5.el9.x86_64.rpm 7.0 MB/s | 300 kB 00:00 2024-03-04T07:24:38.408 INFO:teuthology.orchestra.run.smithi064.stdout:(88/176): uuid-1.6.2-55.el9.x86_64.rpm 1.1 MB/s | 56 kB 00:00 2024-03-04T07:24:38.433 INFO:teuthology.orchestra.run.smithi064.stdout:(89/176): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 1.7 MB/s | 87 kB 00:00 2024-03-04T07:24:38.458 INFO:teuthology.orchestra.run.smithi064.stdout:(90/176): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 63 kB 00:00 2024-03-04T07:24:38.484 INFO:teuthology.orchestra.run.smithi064.stdout:(91/176): zlib-devel-1.2.11-40.el9.x86_64.rpm 876 kB/s | 44 kB 00:00 2024-03-04T07:24:38.509 INFO:teuthology.orchestra.run.smithi064.stdout:(92/176): fuse-2.9.9-15.el9.x86_64.rpm 1.5 MB/s | 79 kB 00:00 2024-03-04T07:24:38.534 INFO:teuthology.orchestra.run.smithi064.stdout:(93/176): jq-1.6-16.el9.x86_64.rpm 3.6 MB/s | 186 kB 00:00 2024-03-04T07:24:38.568 INFO:teuthology.orchestra.run.smithi064.stdout:(94/176): libgfortran-11.4.1-3.el9.x86_64.rpm 13 MB/s | 802 kB 00:00 2024-03-04T07:24:38.593 INFO:teuthology.orchestra.run.smithi064.stdout:(95/176): libquadmath-11.4.1-3.el9.x86_64.rpm 3.3 MB/s | 194 kB 00:00 2024-03-04T07:24:38.619 INFO:teuthology.orchestra.run.smithi064.stdout:(96/176): oniguruma-6.9.6-1.el9.5.x86_64.rpm 4.2 MB/s | 217 kB 00:00 2024-03-04T07:24:38.644 INFO:teuthology.orchestra.run.smithi064.stdout:(97/176): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.7 MB/s | 241 kB 00:00 2024-03-04T07:24:38.686 INFO:teuthology.orchestra.run.smithi064.stdout:(98/176): python3-cryptography-36.0.1-4.el9.x86 17 MB/s | 1.1 MB 00:00 2024-03-04T07:24:38.711 INFO:teuthology.orchestra.run.smithi064.stdout:(99/176): python3-ply-3.11-14.el9.noarch.rpm 1.5 MB/s | 103 kB 00:00 2024-03-04T07:24:38.737 INFO:teuthology.orchestra.run.smithi064.stdout:(100/176): python3-pycparser-2.20-6.el9.noarch. 2.4 MB/s | 124 kB 00:00 2024-03-04T07:24:38.762 INFO:teuthology.orchestra.run.smithi064.stdout:(101/176): python3-requests-2.25.1-8.el9.noarch 2.2 MB/s | 113 kB 00:00 2024-03-04T07:24:38.787 INFO:teuthology.orchestra.run.smithi064.stdout:(102/176): python3-urllib3-1.26.5-5.el9.noarch. 3.7 MB/s | 187 kB 00:00 2024-03-04T07:24:38.954 INFO:teuthology.orchestra.run.smithi064.stdout:(103/176): lua-devel-5.4.4-4.el9.x86_64.rpm 134 kB/s | 22 kB 00:00 2024-03-04T07:24:39.052 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2024-03-04T07:24:39.053 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2024-03-04T07:24:39.071 INFO:teuthology.orchestra.run.smithi064.stdout:(104/176): python3-devel-3.9.17-2.el9.x86_64.rp 795 kB/s | 245 kB 00:00 2024-03-04T07:24:39.297 INFO:teuthology.orchestra.run.smithi064.stdout:(105/176): protobuf-compiler-3.14.0-13.el9.x86_ 2.5 MB/s | 863 kB 00:00 2024-03-04T07:24:39.330 INFO:teuthology.orchestra.run.smithi064.stdout:(106/176): abseil-cpp-20211102.0-3.el9.x86_64.r 2.1 MB/s | 548 kB 00:00 2024-03-04T07:24:39.405 INFO:teuthology.orchestra.run.smithi064.stdout:(107/176): grpc-data-1.46.7-10.el9.noarch.rpm 392 kB/s | 19 kB 00:00 2024-03-04T07:24:39.572 INFO:teuthology.orchestra.run.smithi064.stdout:(108/176): gperftools-libs-2.9.1-2.el9.x86_64.r 1.1 MB/s | 309 kB 00:00 2024-03-04T07:24:39.614 INFO:teuthology.orchestra.run.smithi064.stdout:(109/176): libarrow-doc-9.0.0-10.el9.noarch.rpm 623 kB/s | 26 kB 00:00 2024-03-04T07:24:39.664 INFO:teuthology.orchestra.run.smithi064.stdout:(110/176): liboath-2.6.7-2.el9.x86_64.rpm 971 kB/s | 49 kB 00:00 2024-03-04T07:24:39.706 INFO:teuthology.orchestra.run.smithi064.stdout:(111/176): libunwind-1.6.2-1.el9.x86_64.rpm 1.6 MB/s | 67 kB 00:00 2024-03-04T07:24:40.140 INFO:teuthology.orchestra.run.smithi064.stdout:(112/176): ceph-test-19.0.0-1739.g1bc250c7.el9. 4.7 MB/s | 47 MB 00:10 2024-03-04T07:24:40.165 INFO:teuthology.orchestra.run.smithi064.stdout:(113/176): luarocks-3.9.2-1.el9.noarch.rpm 329 kB/s | 151 kB 00:00 2024-03-04T07:24:40.307 INFO:teuthology.orchestra.run.smithi064.stdout:(114/176): python3-asyncssh-2.13.2-4.el9.noarch 3.8 MB/s | 548 kB 00:00 2024-03-04T07:24:40.349 INFO:teuthology.orchestra.run.smithi064.stdout:(115/176): python3-bcrypt-3.2.2-1.el9.x86_64.rp 1.0 MB/s | 43 kB 00:00 2024-03-04T07:24:40.391 INFO:teuthology.orchestra.run.smithi064.stdout:(116/176): python3-cachetools-4.2.4-1.el9.noarc 770 kB/s | 32 kB 00:00 2024-03-04T07:24:40.433 INFO:teuthology.orchestra.run.smithi064.stdout:(117/176): python3-certifi-2021.10.8-2.el9.noar 362 kB/s | 15 kB 00:00 2024-03-04T07:24:40.508 INFO:teuthology.orchestra.run.smithi064.stdout:(118/176): parquet-libs-9.0.0-10.el9.x86_64.rpm 2.2 MB/s | 838 kB 00:00 2024-03-04T07:24:40.533 INFO:teuthology.orchestra.run.smithi064.stdout:(119/176): python3-cheroot-8.6.0-4.el9.noarch.r 1.7 MB/s | 172 kB 00:00 2024-03-04T07:24:40.609 INFO:teuthology.orchestra.run.smithi064.stdout:(120/176): python3-cherrypy-18.6.1-2.el9.noarch 3.5 MB/s | 358 kB 00:00 2024-03-04T07:24:40.652 INFO:teuthology.orchestra.run.smithi064.stdout:(121/176): python3-cmd2-2.4.2-2.el9.noarch.rpm 2.5 MB/s | 303 kB 00:00 2024-03-04T07:24:40.667 INFO:teuthology.orchestra.run.smithi064.stdout:(122/176): python3-colorama-0.4.4-12.el9.noarch 603 kB/s | 35 kB 00:00 2024-03-04T07:24:40.726 INFO:teuthology.orchestra.run.smithi064.stdout:(123/176): python3-google-auth-2.28.0-1.el9.noa 2.6 MB/s | 203 kB 00:00 2024-03-04T07:24:40.785 INFO:teuthology.orchestra.run.smithi064.stdout:(124/176): python3-grpcio-tools-1.46.7-10.el9.x 2.4 MB/s | 144 kB 00:00 2024-03-04T07:24:40.893 INFO:teuthology.orchestra.run.smithi064.stdout:(125/176): libarrow-9.0.0-10.el9.x86_64.rpm 3.0 MB/s | 4.4 MB 00:01 2024-03-04T07:24:40.919 INFO:teuthology.orchestra.run.smithi064.stdout:(126/176): python3-jaraco-8.2.1-3.el9.noarch.rp 80 kB/s | 11 kB 00:00 2024-03-04T07:24:40.944 INFO:teuthology.orchestra.run.smithi064.stdout:(127/176): python3-jaraco-classes-3.2.1-5.el9.n 350 kB/s | 18 kB 00:00 2024-03-04T07:24:40.969 INFO:teuthology.orchestra.run.smithi064.stdout:(128/176): python3-jaraco-collections-3.0.0-8.e 461 kB/s | 23 kB 00:00 2024-03-04T07:24:40.995 INFO:teuthology.orchestra.run.smithi064.stdout:(129/176): python3-jaraco-functools-3.5.0-2.el9 389 kB/s | 19 kB 00:00 2024-03-04T07:24:41.020 INFO:teuthology.orchestra.run.smithi064.stdout:(130/176): python3-jaraco-text-3.2.0-6.el9.noar 392 kB/s | 20 kB 00:00 2024-03-04T07:24:41.078 INFO:teuthology.orchestra.run.smithi064.stdout:(131/176): python3-logutils-0.3.5-21.el9.noarch 797 kB/s | 46 kB 00:00 2024-03-04T07:24:41.170 INFO:teuthology.orchestra.run.smithi064.stdout:(132/176): python3-more-itertools-8.12.0-2.el9. 861 kB/s | 79 kB 00:00 2024-03-04T07:24:41.237 INFO:teuthology.orchestra.run.smithi064.stdout:(133/176): python3-grpcio-1.46.7-10.el9.x86_64. 3.6 MB/s | 2.0 MB 00:00 2024-03-04T07:24:41.279 INFO:teuthology.orchestra.run.smithi064.stdout:(134/176): python3-natsort-7.1.1-5.el9.noarch.r 531 kB/s | 58 kB 00:00 2024-03-04T07:24:41.313 INFO:teuthology.orchestra.run.smithi064.stdout:(135/176): python3-kubernetes-26.1.0-2.el9.noar 3.2 MB/s | 1.0 MB 00:00 2024-03-04T07:24:41.338 INFO:teuthology.orchestra.run.smithi064.stdout:(136/176): python3-portend-3.1.0-2.el9.noarch.r 281 kB/s | 16 kB 00:00 2024-03-04T07:24:41.363 INFO:teuthology.orchestra.run.smithi064.stdout:(137/176): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.8 MB/s | 90 kB 00:00 2024-03-04T07:24:41.389 INFO:teuthology.orchestra.run.smithi064.stdout:(138/176): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-03-04T07:24:41.422 INFO:teuthology.orchestra.run.smithi064.stdout:(139/176): python3-pyperclip-1.8.2-2.el9.noarch 337 kB/s | 28 kB 00:00 2024-03-04T07:24:41.448 INFO:teuthology.orchestra.run.smithi064.stdout:(140/176): python3-repoze-lru-0.7-16.el9.noarch 367 kB/s | 31 kB 00:00 2024-03-04T07:24:41.473 INFO:teuthology.orchestra.run.smithi064.stdout:(141/176): python3-rsa-4.9-2.el9.noarch.rpm 1.2 MB/s | 59 kB 00:00 2024-03-04T07:24:41.498 INFO:teuthology.orchestra.run.smithi064.stdout:(142/176): python3-tempora-5.0.0-2.el9.noarch.r 715 kB/s | 36 kB 00:00 2024-03-04T07:24:41.523 INFO:teuthology.orchestra.run.smithi064.stdout:(143/176): python3-typing-extensions-4.4.0-2.el 1.0 MB/s | 51 kB 00:00 2024-03-04T07:24:41.565 INFO:teuthology.orchestra.run.smithi064.stdout:(144/176): python3-websocket-client-1.2.3-2.el9 2.1 MB/s | 90 kB 00:00 2024-03-04T07:24:41.590 INFO:teuthology.orchestra.run.smithi064.stdout:(145/176): python3-routes-2.5.1-5.el9.noarch.rp 936 kB/s | 188 kB 00:00 2024-03-04T07:24:41.616 INFO:teuthology.orchestra.run.smithi064.stdout:(146/176): python3-webob-1.8.7-6.el9.noarch.rpm 1.9 MB/s | 230 kB 00:00 2024-03-04T07:24:41.658 INFO:teuthology.orchestra.run.smithi064.stdout:(147/176): python3-zc-lockfile-2.0-10.el9.noarc 482 kB/s | 20 kB 00:00 2024-03-04T07:24:41.700 INFO:teuthology.orchestra.run.smithi064.stdout:(148/176): re2-20211101-3.el9.x86_64.rpm 2.2 MB/s | 192 kB 00:00 2024-03-04T07:24:41.741 INFO:teuthology.orchestra.run.smithi064.stdout:(149/176): python3-werkzeug-2.0.3-3.el9.1.noarc 2.4 MB/s | 427 kB 00:00 2024-03-04T07:24:42.617 INFO:teuthology.orchestra.run.smithi064.stdout:(150/176): thrift-0.15.0-2.el9.x86_64.rpm 1.6 MB/s | 1.6 MB 00:00 2024-03-04T07:24:42.659 INFO:teuthology.orchestra.run.smithi064.stdout:(151/176): libacl-devel-2.3.1-4.el9.x86_64.rpm 1.6 MB/s | 68 kB 00:00 2024-03-04T07:24:42.742 INFO:teuthology.orchestra.run.smithi064.stdout:(152/176): NetworkManager-1.46.0-1.el9.x86_64.r 27 MB/s | 2.3 MB 00:00 2024-03-04T07:24:42.818 INFO:teuthology.orchestra.run.smithi064.stdout:(153/176): NetworkManager-libnm-1.46.0-1.el9.x8 24 MB/s | 1.8 MB 00:00 2024-03-04T07:24:42.851 INFO:teuthology.orchestra.run.smithi064.stdout:(154/176): NetworkManager-team-1.46.0-1.el9.x86 1.2 MB/s | 41 kB 00:00 2024-03-04T07:24:42.893 INFO:teuthology.orchestra.run.smithi064.stdout:(155/176): NetworkManager-tui-1.46.0-1.el9.x86_ 5.8 MB/s | 247 kB 00:00 2024-03-04T07:24:42.926 INFO:teuthology.orchestra.run.smithi064.stdout:(156/176): acl-2.3.1-4.el9.x86_64.rpm 2.0 MB/s | 69 kB 00:00 2024-03-04T07:24:42.960 INFO:teuthology.orchestra.run.smithi064.stdout:(157/176): elfutils-debuginfod-client-0.190-2.e 1.0 MB/s | 35 kB 00:00 2024-03-04T07:24:43.027 INFO:teuthology.orchestra.run.smithi064.stdout:(158/176): elfutils-libelf-0.190-2.el9.x86_64.r 2.8 MB/s | 191 kB 00:00 2024-03-04T07:24:43.069 INFO:teuthology.orchestra.run.smithi064.stdout:(159/176): elfutils-libs-0.190-2.el9.x86_64.rpm 5.9 MB/s | 254 kB 00:00 2024-03-04T07:24:43.120 INFO:teuthology.orchestra.run.smithi064.stdout:(160/176): iproute-6.7.0-2.el9.x86_64.rpm 16 MB/s | 802 kB 00:00 2024-03-04T07:24:43.178 INFO:teuthology.orchestra.run.smithi064.stdout:(161/176): iproute-tc-6.7.0-2.el9.x86_64.rpm 7.3 MB/s | 433 kB 00:00 2024-03-04T07:24:43.204 INFO:teuthology.orchestra.run.smithi064.stdout:(162/176): libacl-2.3.1-4.el9.x86_64.rpm 876 kB/s | 22 kB 00:00 2024-03-04T07:24:43.253 INFO:teuthology.orchestra.run.smithi064.stdout:(163/176): librados2-19.0.0-1739.g1bc250c7.el9. 2.2 MB/s | 3.4 MB 00:01 2024-03-04T07:24:43.279 INFO:teuthology.orchestra.run.smithi064.stdout:(164/176): libblkid-2.37.4-18.el9.x86_64.rpm 1.4 MB/s | 107 kB 00:00 2024-03-04T07:24:43.304 INFO:teuthology.orchestra.run.smithi064.stdout:(165/176): libfdisk-2.37.4-18.el9.x86_64.rpm 3.0 MB/s | 154 kB 00:00 2024-03-04T07:24:43.330 INFO:teuthology.orchestra.run.smithi064.stdout:(166/176): libmount-2.37.4-18.el9.x86_64.rpm 2.6 MB/s | 135 kB 00:00 2024-03-04T07:24:43.355 INFO:teuthology.orchestra.run.smithi064.stdout:(167/176): libsmartcols-2.37.4-18.el9.x86_64.rp 1.2 MB/s | 63 kB 00:00 2024-03-04T07:24:43.381 INFO:teuthology.orchestra.run.smithi064.stdout:(168/176): libuuid-2.37.4-18.el9.x86_64.rpm 553 kB/s | 28 kB 00:00 2024-03-04T07:24:43.414 INFO:teuthology.orchestra.run.smithi064.stdout:(169/176): openssl-3.0.7-27.el9.x86_64.rpm 20 MB/s | 1.2 MB 00:00 2024-03-04T07:24:43.464 INFO:teuthology.orchestra.run.smithi064.stdout:(170/176): openssl-libs-3.0.7-27.el9.x86_64.rpm 27 MB/s | 2.2 MB 00:00 2024-03-04T07:24:43.514 INFO:teuthology.orchestra.run.smithi064.stdout:(171/176): util-linux-2.37.4-18.el9.x86_64.rpm 23 MB/s | 2.2 MB 00:00 2024-03-04T07:24:43.548 INFO:teuthology.orchestra.run.smithi064.stdout:(172/176): util-linux-core-2.37.4-18.el9.x86_64 5.3 MB/s | 438 kB 00:00 2024-03-04T07:24:43.565 INFO:teuthology.orchestra.run.smithi064.stdout:(173/176): util-linux-user-2.37.4-18.el9.x86_64 611 kB/s | 30 kB 00:00 2024-03-04T07:24:43.590 INFO:teuthology.orchestra.run.smithi064.stdout:(174/176): xfsdump-3.1.12-4.el9.x86_64.rpm 7.5 MB/s | 321 kB 00:00 2024-03-04T07:24:43.640 INFO:teuthology.orchestra.run.smithi064.stdout:(175/176): librbd1-19.0.0-1739.g1bc250c7.el9.x8 1.7 MB/s | 3.1 MB 00:01 2024-03-04T07:24:43.665 INFO:teuthology.orchestra.run.smithi064.stdout:(176/176): xfsprogs-6.3.0-1.el9.x86_64.rpm 11 MB/s | 1.1 MB 00:00 2024-03-04T07:24:43.673 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-03-04T07:24:43.673 INFO:teuthology.orchestra.run.smithi064.stdout:Total 13 MB/s | 224 MB 00:17 2024-03-04T07:24:43.718 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-03-04T07:24:44.059 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-03-04T07:24:44.059 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-03-04T07:24:46.134 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-03-04T07:24:46.135 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-03-04T07:24:46.381 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2024-03-04T07:24:46.480 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : openssl-libs-1:3.0.7-27.el9.x86_64 1/202 2024-03-04T07:24:46.516 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : libuuid-2.37.4-18.el9.x86_64 2/202 2024-03-04T07:24:46.543 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-04T07:24:46.566 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-04T07:24:46.605 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : elfutils-libelf-0.190-2.el9.x86_64 4/202 2024-03-04T07:24:46.645 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 5/202 2024-03-04T07:24:46.699 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/202 2024-03-04T07:24:46.741 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : thrift-0.15.0-2.el9.x86_64 7/202 2024-03-04T07:24:46.766 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 8/202 2024-03-04T07:24:46.822 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 9/202 2024-03-04T07:24:47.162 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 10/202 2024-03-04T07:24:47.283 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-04T07:24:47.417 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-04T07:24:47.568 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-04T07:24:47.644 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-04T07:24:47.679 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-04T07:24:47.846 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : postgresql-private-libs-13.11-1.el9.x86_64 14/202 2024-03-04T07:24:47.895 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : postgresql-13.11-1.el9.x86_64 15/202 2024-03-04T07:24:47.922 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-04T07:24:48.027 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-04T07:24:48.048 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-04T07:24:48.130 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-04T07:24:48.169 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : libmount-2.37.4-18.el9.x86_64 18/202 2024-03-04T07:24:48.191 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-04T07:24:48.207 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-04T07:24:48.350 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-04T07:24:48.381 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-04T07:24:48.424 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : libsmartcols-2.37.4-18.el9.x86_64 21/202 2024-03-04T07:24:48.599 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : libacl-2.3.1-4.el9.x86_64 22/202 2024-03-04T07:24:48.706 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-04T07:24:48.774 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-04T07:24:48.881 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-04T07:24:48.942 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-04T07:24:49.159 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-04T07:24:49.276 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : re2-1:20211101-3.el9.x86_64 25/202 2024-03-04T07:24:49.389 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 26/202 2024-03-04T07:24:49.531 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 27/202 2024-03-04T07:24:49.602 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : liboath-2.6.7-2.el9.x86_64 28/202 2024-03-04T07:24:49.654 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 29/202 2024-03-04T07:24:49.684 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 30/202 2024-03-04T07:24:49.719 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 31/202 2024-03-04T07:24:49.760 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 32/202 2024-03-04T07:24:49.834 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 33/202 2024-03-04T07:24:49.875 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 34/202 2024-03-04T07:24:49.919 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 35/202 2024-03-04T07:24:49.952 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 36/202 2024-03-04T07:24:49.978 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : lua-5.4.4-4.el9.x86_64 37/202 2024-03-04T07:24:50.059 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 38/202 2024-03-04T07:24:50.092 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 39/202 2024-03-04T07:24:50.136 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 40/202 2024-03-04T07:24:50.164 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 41/202 2024-03-04T07:24:50.237 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 42/202 2024-03-04T07:24:50.287 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : luarocks-3.9.2-1.el9.noarch 43/202 2024-03-04T07:24:50.363 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 44/202 2024-03-04T07:24:50.510 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 45/202 2024-03-04T07:24:50.565 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 46/202 2024-03-04T07:24:51.127 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rsa-4.9-2.el9.noarch 47/202 2024-03-04T07:24:51.212 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 48/202 2024-03-04T07:24:51.317 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 49/202 2024-03-04T07:24:51.362 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-04T07:24:51.370 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-04T07:24:51.409 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : qatzip-libs-1.1.2-2.el9.x86_64 51/202 2024-03-04T07:24:51.448 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 52/202 2024-03-04T07:24:51.487 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 53/202 2024-03-04T07:24:51.593 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : elfutils-libs-0.190-2.el9.x86_64 54/202 2024-03-04T07:24:51.639 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : iproute-6.7.0-2.el9.x86_64 55/202 2024-03-04T07:24:51.896 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : libfdisk-2.37.4-18.el9.x86_64 56/202 2024-03-04T07:24:52.068 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : util-linux-2.37.4-18.el9.x86_64 57/202 2024-03-04T07:24:52.148 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : xfsprogs-6.3.0-1.el9.x86_64 58/202 2024-03-04T07:24:52.193 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 59/202 2024-03-04T07:24:52.328 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 60/202 2024-03-04T07:24:52.370 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : openssl-1:3.0.7-27.el9.x86_64 61/202 2024-03-04T07:24:52.427 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 62/202 2024-03-04T07:24:52.510 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 63/202 2024-03-04T07:24:52.546 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 64/202 2024-03-04T07:24:52.582 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 65/202 2024-03-04T07:24:52.629 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 66/202 2024-03-04T07:24:52.665 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 67/202 2024-03-04T07:24:52.709 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 68/202 2024-03-04T07:24:52.768 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 69/202 2024-03-04T07:24:52.912 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 70/202 2024-03-04T07:24:52.956 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 71/202 2024-03-04T07:24:52.986 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 72/202 2024-03-04T07:24:53.029 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 73/202 2024-03-04T07:24:53.126 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 74/202 2024-03-04T07:24:53.160 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-google-auth-1:2.28.0-1.el9.noarch 75/202 2024-03-04T07:24:53.201 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 76/202 2024-03-04T07:24:53.226 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 77/202 2024-03-04T07:24:53.251 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 78/202 2024-03-04T07:24:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-03-04T07:24:53.340 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 79/202 2024-03-04T07:24:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : openssl-libs-1:3.0.7-27.el9.x86_64 1/202 2024-03-04T07:24:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libuuid-2.37.4-18.el9.x86_64 2/202 2024-03-04T07:24:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-04T07:24:53.459 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libblkid-2.37.4-18.el9.x86_64 3/202 2024-03-04T07:24:53.475 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 80/202 2024-03-04T07:24:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : elfutils-libelf-0.190-2.el9.x86_64 4/202 2024-03-04T07:24:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 5/202 2024-03-04T07:24:53.589 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 81/202 2024-03-04T07:24:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/202 2024-03-04T07:24:53.634 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 82/202 2024-03-04T07:24:53.693 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-ply-3.11-14.el9.noarch 83/202 2024-03-04T07:24:53.769 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 84/202 2024-03-04T07:24:53.874 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : thrift-0.15.0-2.el9.x86_64 7/202 2024-03-04T07:24:53.906 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 8/202 2024-03-04T07:24:53.965 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 85/202 2024-03-04T07:24:53.990 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 9/202 2024-03-04T07:24:54.013 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 86/202 2024-03-04T07:24:54.158 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/202 2024-03-04T07:24:54.195 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-asyncssh-2.13.2-4.el9.noarch 88/202 2024-03-04T07:24:54.230 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 89/202 2024-03-04T07:24:54.259 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 90/202 2024-03-04T07:24:54.294 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : jq-1.6-16.el9.x86_64 91/202 2024-03-04T07:24:54.325 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : fuse-2.9.9-15.el9.x86_64 92/202 2024-03-04T07:24:54.332 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 10/202 2024-03-04T07:24:54.353 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : zlib-devel-1.2.11-40.el9.x86_64 93/202 2024-03-04T07:24:54.384 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-04T07:24:54.387 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 94/202 2024-03-04T07:24:54.427 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : uuid-1.6.2-55.el9.x86_64 95/202 2024-03-04T07:24:54.458 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 96/202 2024-03-04T07:24:54.487 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 97/202 2024-03-04T07:24:54.514 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-04T07:24:54.544 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-04T07:24:54.568 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 98/202 2024-03-04T07:24:54.607 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 99/202 2024-03-04T07:24:54.636 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-04T07:24:54.639 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 100/202 2024-03-04T07:24:54.671 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-04T07:24:54.678 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 101/202 2024-03-04T07:24:54.707 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 102/202 2024-03-04T07:24:54.801 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 103/202 2024-03-04T07:24:54.833 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : postgresql-private-libs-13.11-1.el9.x86_64 14/202 2024-03-04T07:24:54.896 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : postgresql-13.11-1.el9.x86_64 15/202 2024-03-04T07:24:54.899 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 104/202 2024-03-04T07:24:54.923 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-04T07:24:54.947 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 105/202 2024-03-04T07:24:54.978 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 106/202 2024-03-04T07:24:55.026 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 107/202 2024-03-04T07:24:55.029 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-04T07:24:55.057 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-04T07:24:55.122 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 17/202 2024-03-04T07:24:55.137 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-packaging-20.9-5.el9.noarch 108/202 2024-03-04T07:24:55.169 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libmount-2.37.4-18.el9.x86_64 18/202 2024-03-04T07:24:55.190 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 109/202 2024-03-04T07:24:55.200 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-04T07:24:55.207 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: qatlib-service-23.11.0-1.el9.x86_64 19/202 2024-03-04T07:24:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-04T07:24:55.431 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : qatlib-23.11.0-1.el9.x86_64 20/202 2024-03-04T07:24:55.463 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libsmartcols-2.37.4-18.el9.x86_64 21/202 2024-03-04T07:24:55.633 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libacl-2.3.1-4.el9.x86_64 22/202 2024-03-04T07:24:55.748 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-04T07:24:55.808 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 23/202 2024-03-04T07:24:55.926 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-04T07:24:55.972 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 110/202 2024-03-04T07:24:55.976 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-04T07:24:56.027 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 111/202 2024-03-04T07:24:56.060 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 112/202 2024-03-04T07:24:56.110 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 113/202 2024-03-04T07:24:56.201 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 114/202 2024-03-04T07:24:56.206 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: NetworkManager-1:1.46.0-1.el9.x86_64 24/202 2024-03-04T07:24:56.236 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 115/202 2024-03-04T07:24:56.309 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : re2-1:20211101-3.el9.x86_64 25/202 2024-03-04T07:24:56.404 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 26/202 2024-03-04T07:24:56.422 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : openblas-0.3.21-2.el9.x86_64 116/202 2024-03-04T07:24:56.432 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 27/202 2024-03-04T07:24:56.451 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 117/202 2024-03-04T07:24:56.502 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : liboath-2.6.7-2.el9.x86_64 28/202 2024-03-04T07:24:56.531 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 118/202 2024-03-04T07:24:56.555 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 29/202 2024-03-04T07:24:56.585 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 30/202 2024-03-04T07:24:56.620 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 31/202 2024-03-04T07:24:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 32/202 2024-03-04T07:24:56.743 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 33/202 2024-03-04T07:24:56.776 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 34/202 2024-03-04T07:24:56.820 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 35/202 2024-03-04T07:24:56.852 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 36/202 2024-03-04T07:24:56.880 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : lua-5.4.4-4.el9.x86_64 37/202 2024-03-04T07:24:56.951 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 38/202 2024-03-04T07:24:56.985 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 39/202 2024-03-04T07:24:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 40/202 2024-03-04T07:24:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 41/202 2024-03-04T07:24:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 42/202 2024-03-04T07:24:57.163 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : luarocks-3.9.2-1.el9.noarch 43/202 2024-03-04T07:24:57.212 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 119/202 2024-03-04T07:24:57.239 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 44/202 2024-03-04T07:24:57.327 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 120/202 2024-03-04T07:24:57.376 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 45/202 2024-03-04T07:24:57.425 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 46/202 2024-03-04T07:24:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rsa-4.9-2.el9.noarch 47/202 2024-03-04T07:24:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 48/202 2024-03-04T07:24:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 49/202 2024-03-04T07:24:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-04T07:24:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: util-linux-core-2.37.4-18.el9.x86_64 50/202 2024-03-04T07:24:58.236 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : qatzip-libs-1.1.2-2.el9.x86_64 51/202 2024-03-04T07:24:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 52/202 2024-03-04T07:24:58.313 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 53/202 2024-03-04T07:24:58.430 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : elfutils-libs-0.190-2.el9.x86_64 54/202 2024-03-04T07:24:58.499 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : iproute-6.7.0-2.el9.x86_64 55/202 2024-03-04T07:24:58.619 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 121/202 2024-03-04T07:24:58.695 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 122/202 2024-03-04T07:24:58.722 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 123/202 2024-03-04T07:24:58.864 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libfdisk-2.37.4-18.el9.x86_64 56/202 2024-03-04T07:24:59.034 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : util-linux-2.37.4-18.el9.x86_64 57/202 2024-03-04T07:24:59.050 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 124/202 2024-03-04T07:24:59.076 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-04T07:24:59.108 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : xfsprogs-6.3.0-1.el9.x86_64 58/202 2024-03-04T07:24:59.151 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-04T07:24:59.153 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 59/202 2024-03-04T07:24:59.181 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 126/202 2024-03-04T07:24:59.218 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 127/202 2024-03-04T07:24:59.284 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 60/202 2024-03-04T07:24:59.322 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : openssl-1:3.0.7-27.el9.x86_64 61/202 2024-03-04T07:24:59.379 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 62/202 2024-03-04T07:24:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 63/202 2024-03-04T07:24:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 64/202 2024-03-04T07:24:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 65/202 2024-03-04T07:24:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 66/202 2024-03-04T07:24:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 67/202 2024-03-04T07:24:59.646 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 68/202 2024-03-04T07:24:59.689 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 69/202 2024-03-04T07:24:59.697 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 128/202 2024-03-04T07:24:59.722 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-04T07:24:59.793 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-04T07:24:59.823 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 130/202 2024-03-04T07:24:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 70/202 2024-03-04T07:24:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 71/202 2024-03-04T07:24:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 72/202 2024-03-04T07:24:59.932 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 73/202 2024-03-04T07:25:00.029 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 74/202 2024-03-04T07:25:00.073 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-google-auth-1:2.28.0-1.el9.noarch 75/202 2024-03-04T07:25:00.114 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 76/202 2024-03-04T07:25:00.139 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 77/202 2024-03-04T07:25:00.164 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 78/202 2024-03-04T07:25:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 79/202 2024-03-04T07:25:00.367 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 80/202 2024-03-04T07:25:00.474 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 81/202 2024-03-04T07:25:00.522 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 82/202 2024-03-04T07:25:00.571 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ply-3.11-14.el9.noarch 83/202 2024-03-04T07:25:00.627 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 84/202 2024-03-04T07:25:00.815 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 85/202 2024-03-04T07:25:00.860 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 86/202 2024-03-04T07:25:01.001 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/202 2024-03-04T07:25:01.041 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-asyncssh-2.13.2-4.el9.noarch 88/202 2024-03-04T07:25:01.073 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 89/202 2024-03-04T07:25:01.105 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 90/202 2024-03-04T07:25:01.142 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : jq-1.6-16.el9.x86_64 91/202 2024-03-04T07:25:01.180 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : fuse-2.9.9-15.el9.x86_64 92/202 2024-03-04T07:25:01.208 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : zlib-devel-1.2.11-40.el9.x86_64 93/202 2024-03-04T07:25:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 94/202 2024-03-04T07:25:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : uuid-1.6.2-55.el9.x86_64 95/202 2024-03-04T07:25:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 96/202 2024-03-04T07:25:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 97/202 2024-03-04T07:25:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 98/202 2024-03-04T07:25:01.487 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 99/202 2024-03-04T07:25:01.520 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 100/202 2024-03-04T07:25:01.561 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 101/202 2024-03-04T07:25:01.596 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 102/202 2024-03-04T07:25:01.687 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 103/202 2024-03-04T07:25:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 104/202 2024-03-04T07:25:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 105/202 2024-03-04T07:25:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 106/202 2024-03-04T07:25:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 107/202 2024-03-04T07:25:02.007 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-packaging-20.9-5.el9.noarch 108/202 2024-03-04T07:25:02.060 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 109/202 2024-03-04T07:25:02.088 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-04T07:25:02.129 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-04T07:25:02.707 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-04T07:25:02.757 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-04T07:25:02.821 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 110/202 2024-03-04T07:25:02.831 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-04T07:25:02.831 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-04T07:25:02.831 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-04T07:25:02.831 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:02.858 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-04T07:25:02.874 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 111/202 2024-03-04T07:25:02.907 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 112/202 2024-03-04T07:25:02.957 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 113/202 2024-03-04T07:25:03.056 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 114/202 2024-03-04T07:25:03.092 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 115/202 2024-03-04T07:25:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-0.3.21-2.el9.x86_64 116/202 2024-03-04T07:25:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 117/202 2024-03-04T07:25:03.394 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 118/202 2024-03-04T07:25:04.065 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 119/202 2024-03-04T07:25:04.182 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 120/202 2024-03-04T07:25:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 121/202 2024-03-04T07:25:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 122/202 2024-03-04T07:25:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 123/202 2024-03-04T07:25:05.893 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 124/202 2024-03-04T07:25:05.917 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-04T07:25:06.008 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 125/202 2024-03-04T07:25:06.038 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 126/202 2024-03-04T07:25:06.075 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 127/202 2024-03-04T07:25:06.558 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 128/202 2024-03-04T07:25:06.596 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-04T07:25:06.666 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 129/202 2024-03-04T07:25:06.697 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 130/202 2024-03-04T07:25:09.005 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-04T07:25:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-04T07:25:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 131/202 2024-03-04T07:25:09.856 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-04T07:25:09.936 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 132/202 2024-03-04T07:25:09.937 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-04T07:25:09.937 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-04T07:25:09.937 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:09.966 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-04T07:25:15.229 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /sys 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /proc 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /mnt 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /var/tmp 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /home 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /root 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /tmp 2024-03-04T07:25:15.230 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:15.411 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-04T07:25:15.426 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-04T07:25:15.426 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:15.426 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-03-04T07:25:15.427 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:16.956 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-04T07:25:16.983 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-04T07:25:16.983 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:16.983 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-04T07:25:16.983 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-04T07:25:16.983 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-04T07:25:16.983 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:17.206 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-04T07:25:17.229 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-04T07:25:17.229 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:17.229 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-04T07:25:17.229 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-04T07:25:17.229 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-04T07:25:17.229 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:17.707 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-04T07:25:17.730 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-04T07:25:17.730 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:17.731 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-04T07:25:17.731 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-04T07:25:17.731 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-04T07:25:17.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:17.825 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : automake-1.16.2-8.el9.noarch 138/202 2024-03-04T07:25:18.286 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-04T07:25:18.315 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-04T07:25:18.347 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 140/202 2024-03-04T07:25:18.409 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 141/202 2024-03-04T07:25:18.438 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-04T07:25:18.678 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-04T07:25:18.705 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-04T07:25:19.722 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-04T07:25:19.747 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-04T07:25:19.853 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-04T07:25:20.017 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 145/202 2024-03-04T07:25:20.041 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-04T07:25:20.065 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-04T07:25:20.065 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:20.065 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-04T07:25:20.065 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-04T07:25:20.065 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-04T07:25:20.065 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:20.117 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-04T07:25:20.127 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-04T07:25:20.172 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 148/202 2024-03-04T07:25:21.088 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libtool-2.4.6-45.el9.x86_64 149/202 2024-03-04T07:25:21.118 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-04T07:25:21.142 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-04T07:25:21.142 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:21.142 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-04T07:25:21.142 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-04T07:25:21.142 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-04T07:25:21.142 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:21.186 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-04T07:25:21.209 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-04T07:25:21.209 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:21.209 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-04T07:25:21.209 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:21.262 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-04T07:25:21.285 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-04T07:25:21.285 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:21.285 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-03-04T07:25:21.286 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-04T07:25:21.286 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-04T07:25:21.286 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:21.621 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-04T07:25:21.645 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-04T07:25:21.645 INFO:teuthology.orchestra.run.smithi066.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:21.645 INFO:teuthology.orchestra.run.smithi066.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-04T07:25:21.645 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-04T07:25:21.645 INFO:teuthology.orchestra.run.smithi066.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-04T07:25:21.645 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 133/202 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /sys 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /proc 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /mnt 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /var/tmp 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /home 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /root 2024-03-04T07:25:21.978 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /tmp 2024-03-04T07:25:21.979 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-04T07:25:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 134/202 2024-03-04T07:25:22.169 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:22.169 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-03-04T07:25:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:23.715 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-04T07:25:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 135/202 2024-03-04T07:25:23.741 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:23.741 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-04T07:25:23.741 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-04T07:25:23.741 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-04T07:25:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-04T07:25:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 136/202 2024-03-04T07:25:23.996 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:23.996 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-04T07:25:23.997 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-04T07:25:23.997 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-04T07:25:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:24.466 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-04T07:25:24.488 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 137/202 2024-03-04T07:25:24.489 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:24.489 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-04T07:25:24.489 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-04T07:25:24.489 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-04T07:25:24.489 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:24.584 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : automake-1.16.2-8.el9.noarch 138/202 2024-03-04T07:25:25.045 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-04T07:25:25.074 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 139/202 2024-03-04T07:25:25.106 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 140/202 2024-03-04T07:25:25.168 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 141/202 2024-03-04T07:25:25.198 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-04T07:25:25.439 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 142/202 2024-03-04T07:25:25.474 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-04T07:25:26.325 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 154/202 2024-03-04T07:25:26.367 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 155/202 2024-03-04T07:25:26.397 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 156/202 2024-03-04T07:25:26.424 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 157/202 2024-03-04T07:25:26.493 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 143/202 2024-03-04T07:25:26.524 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-04T07:25:26.621 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 158/202 2024-03-04T07:25:26.628 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 144/202 2024-03-04T07:25:26.744 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : postgresql-contrib-13.11-1.el9.x86_64 159/202 2024-03-04T07:25:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 145/202 2024-03-04T07:25:26.798 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 160/202 2024-03-04T07:25:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-04T07:25:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 146/202 2024-03-04T07:25:26.849 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:26.849 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-04T07:25:26.849 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-04T07:25:26.850 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-04T07:25:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:26.859 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 161/202 2024-03-04T07:25:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-04T07:25:26.899 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : xfsdump-3.1.12-4.el9.x86_64 162/202 2024-03-04T07:25:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 147/202 2024-03-04T07:25:26.966 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : util-linux-user-2.37.4-18.el9.x86_64 163/202 2024-03-04T07:25:26.966 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 148/202 2024-03-04T07:25:27.024 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : iproute-tc-6.7.0-2.el9.x86_64 164/202 2024-03-04T07:25:27.067 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : elfutils-debuginfod-client-0.190-2.el9.x86_64 165/202 2024-03-04T07:25:27.098 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 166/202 2024-03-04T07:25:27.127 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : NetworkManager-team-1:1.46.0-1.el9.x86_64 167/202 2024-03-04T07:25:27.203 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : NetworkManager-tui-1:1.46.0-1.el9.x86_64 168/202 2024-03-04T07:25:27.267 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : libacl-devel-2.3.1-4.el9.x86_64 169/202 2024-03-04T07:25:27.300 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : acl-2.3.1-4.el9.x86_64 170/202 2024-03-04T07:25:27.890 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libtool-2.4.6-45.el9.x86_64 149/202 2024-03-04T07:25:27.909 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-04T07:25:27.920 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-04T07:25:27.942 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 150/202 2024-03-04T07:25:27.942 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:27.943 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-04T07:25:27.943 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-04T07:25:27.943 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-04T07:25:27.943 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:27.948 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-04T07:25:27.969 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-04T07:25:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-04T07:25:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 151/202 2024-03-04T07:25:28.009 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:28.009 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-04T07:25:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-04T07:25:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 152/202 2024-03-04T07:25:28.077 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:28.077 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-03-04T07:25:28.077 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-04T07:25:28.077 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-04T07:25:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:28.414 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-04T07:25:28.436 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-04T07:25:28.436 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-04T07:25:28.436 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-04T07:25:28.436 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-04T07:25:28.436 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-04T07:25:28.436 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:28.856 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 172/202 2024-03-04T07:25:28.926 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : openssl-devel-1:3.0.7-27.el9.x86_64 173/202 2024-03-04T07:25:28.969 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 174/202 2024-03-04T07:25:29.065 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : flex-2.6.4-9.el9.x86_64 175/202 2024-03-04T07:25:29.096 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : bison-3.7.4-5.el9.x86_64 176/202 2024-03-04T07:25:29.121 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : NetworkManager-tui-1:1.44.0-2.el9.x86_64 177/202 2024-03-04T07:25:29.147 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : elfutils-debuginfod-client-0.189-3.el9.x86_64 178/202 2024-03-04T07:25:29.173 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : elfutils-libs-0.189-3.el9.x86_64 179/202 2024-03-04T07:25:29.173 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-04T07:25:29.258 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-04T07:25:29.286 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : libacl-devel-2.3.1-3.el9.x86_64 181/202 2024-03-04T07:25:29.315 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : openssl-1:3.0.7-24.el9.x86_64 182/202 2024-03-04T07:25:29.316 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-04T07:25:29.417 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-04T07:25:29.444 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : xfsdump-3.1.12-3.el9.x86_64 184/202 2024-03-04T07:25:29.485 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : xfsprogs-5.19.0-4.el9.x86_64 185/202 2024-03-04T07:25:29.510 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : iproute-tc-6.2.0-5.el9.x86_64 186/202 2024-03-04T07:25:29.540 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : acl-2.3.1-3.el9.x86_64 187/202 2024-03-04T07:25:29.590 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : iproute-6.2.0-5.el9.x86_64 188/202 2024-03-04T07:25:29.635 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : util-linux-user-2.37.4-15.el9.x86_64 189/202 2024-03-04T07:25:29.676 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : util-linux-2.37.4-15.el9.x86_64 190/202 2024-03-04T07:25:29.707 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : util-linux-core-2.37.4-15.el9.x86_64 191/202 2024-03-04T07:25:29.733 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : libfdisk-2.37.4-15.el9.x86_64 192/202 2024-03-04T07:25:29.762 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : libmount-2.37.4-15.el9.x86_64 193/202 2024-03-04T07:25:29.783 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : libblkid-2.37.4-15.el9.x86_64 194/202 2024-03-04T07:25:29.809 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : NetworkManager-team-1:1.44.0-2.el9.x86_64 195/202 2024-03-04T07:25:29.815 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-04T07:25:29.815 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-04T07:25:30.003 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-04T07:25:30.004 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-04T07:25:30.086 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-04T07:25:30.119 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : libuuid-2.37.4-15.el9.x86_64 198/202 2024-03-04T07:25:30.144 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : libsmartcols-2.37.4-15.el9.x86_64 199/202 2024-03-04T07:25:30.169 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : elfutils-libelf-0.189-3.el9.x86_64 200/202 2024-03-04T07:25:30.195 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : libacl-2.3.1-3.el9.x86_64 201/202 2024-03-04T07:25:30.265 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-04T07:25:33.169 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 154/202 2024-03-04T07:25:33.227 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 155/202 2024-03-04T07:25:33.257 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 156/202 2024-03-04T07:25:33.284 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 157/202 2024-03-04T07:25:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 158/202 2024-03-04T07:25:33.613 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : postgresql-contrib-13.11-1.el9.x86_64 159/202 2024-03-04T07:25:33.658 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 160/202 2024-03-04T07:25:33.709 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 161/202 2024-03-04T07:25:33.749 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : xfsdump-3.1.12-4.el9.x86_64 162/202 2024-03-04T07:25:33.824 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : util-linux-user-2.37.4-18.el9.x86_64 163/202 2024-03-04T07:25:33.859 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : iproute-tc-6.7.0-2.el9.x86_64 164/202 2024-03-04T07:25:33.894 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : elfutils-debuginfod-client-0.190-2.el9.x86_64 165/202 2024-03-04T07:25:33.933 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 166/202 2024-03-04T07:25:33.963 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : NetworkManager-team-1:1.46.0-1.el9.x86_64 167/202 2024-03-04T07:25:34.033 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : NetworkManager-tui-1:1.46.0-1.el9.x86_64 168/202 2024-03-04T07:25:34.101 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libacl-devel-2.3.1-4.el9.x86_64 169/202 2024-03-04T07:25:34.144 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : acl-2.3.1-4.el9.x86_64 170/202 2024-03-04T07:25:34.760 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-04T07:25:34.801 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-04T07:25:34.821 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 171/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 1/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 3/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 4/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 5/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 6/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 7/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 8/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 9/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 10/202 2024-03-04T07:25:34.882 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 14/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 15/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 17/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 18/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 19/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 20/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 21/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 22/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 23/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 24/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 25/202 2024-03-04T07:25:34.883 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 26/202 2024-03-04T07:25:34.884 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 27/202 2024-03-04T07:25:34.884 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 28/202 2024-03-04T07:25:34.884 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 29/202 2024-03-04T07:25:34.906 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 30/202 2024-03-04T07:25:34.906 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 31/202 2024-03-04T07:25:34.906 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 32/202 2024-03-04T07:25:34.906 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 33/202 2024-03-04T07:25:34.906 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 34/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 35/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 36/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 37/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 38/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 39/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : automake-1.16.2-8.el9.noarch 40/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : bison-3.7.4-5.el9.x86_64 41/202 2024-03-04T07:25:34.907 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 42/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 43/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : flex-2.6.4-9.el9.x86_64 44/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 48/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 49/202 2024-03-04T07:25:34.908 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 50/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 51/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 52/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 53/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 54/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 55/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : lua-5.4.4-4.el9.x86_64 56/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 57/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 58/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openssl-devel-1:3.0.7-27.el9.x86_64 59/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : postgresql-13.11-1.el9.x86_64 60/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : postgresql-contrib-13.11-1.el9.x86_64 61/202 2024-03-04T07:25:34.909 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : postgresql-private-libs-13.11-1.el9.x86_64 62/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : postgresql-server-13.11-1.el9.x86_64 63/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 64/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 65/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 66/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 67/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 68/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 69/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 70/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 71/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 72/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 73/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 74/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 75/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 76/202 2024-03-04T07:25:34.910 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 77/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 78/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 79/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 80/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 81/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 82/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 83/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 84/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : qatlib-23.11.0-1.el9.x86_64 85/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : qatlib-service-23.11.0-1.el9.x86_64 86/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : qatzip-libs-1.1.2-2.el9.x86_64 87/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 88/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 89/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 90/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 91/202 2024-03-04T07:25:34.911 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : zlib-devel-1.2.11-40.el9.x86_64 92/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 93/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : jq-1.6-16.el9.x86_64 94/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 95/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 96/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 97/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 98/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 99/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-ply-3.11-14.el9.noarch 100/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 101/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 102/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 103/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 104/202 2024-03-04T07:25:34.912 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 105/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 106/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 107/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 108/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 109/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 110/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 111/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 112/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 113/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 114/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 115/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-asyncssh-2.13.2-4.el9.noarch 116/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 117/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 118/202 2024-03-04T07:25:34.913 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 119/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 120/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 121/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 122/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 123/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-google-auth-1:2.28.0-1.el9.noarch 124/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 125/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 126/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 127/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 128/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 129/202 2024-03-04T07:25:34.914 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 130/202 2024-03-04T07:25:34.915 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 131/202 2024-03-04T07:25:34.915 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 132/202 2024-03-04T07:25:34.915 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 133/202 2024-03-04T07:25:34.915 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 134/202 2024-03-04T07:25:34.915 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 135/202 2024-03-04T07:25:34.915 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 136/202 2024-03-04T07:25:34.916 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 137/202 2024-03-04T07:25:34.916 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 138/202 2024-03-04T07:25:34.916 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 139/202 2024-03-04T07:25:34.916 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 140/202 2024-03-04T07:25:34.916 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 141/202 2024-03-04T07:25:34.916 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 142/202 2024-03-04T07:25:34.917 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 143/202 2024-03-04T07:25:34.917 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 144/202 2024-03-04T07:25:34.917 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 145/202 2024-03-04T07:25:34.917 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 146/202 2024-03-04T07:25:34.917 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 147/202 2024-03-04T07:25:34.917 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 148/202 2024-03-04T07:25:34.917 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : re2-1:20211101-3.el9.x86_64 149/202 2024-03-04T07:25:34.918 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 150/202 2024-03-04T07:25:34.918 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 151/202 2024-03-04T07:25:34.918 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 152/202 2024-03-04T07:25:34.918 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-04T07:25:34.918 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 154/202 2024-03-04T07:25:34.918 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libacl-devel-2.3.1-4.el9.x86_64 155/202 2024-03-04T07:25:34.919 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libacl-devel-2.3.1-3.el9.x86_64 156/202 2024-03-04T07:25:34.919 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-1:1.46.0-1.el9.x86_64 157/202 2024-03-04T07:25:34.919 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-1:1.44.0-2.el9.x86_64 158/202 2024-03-04T07:25:34.919 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 159/202 2024-03-04T07:25:34.919 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 160/202 2024-03-04T07:25:34.919 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-team-1:1.46.0-1.el9.x86_64 161/202 2024-03-04T07:25:34.919 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-team-1:1.44.0-2.el9.x86_64 162/202 2024-03-04T07:25:34.920 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-tui-1:1.46.0-1.el9.x86_64 163/202 2024-03-04T07:25:34.920 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : NetworkManager-tui-1:1.44.0-2.el9.x86_64 164/202 2024-03-04T07:25:34.920 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : acl-2.3.1-4.el9.x86_64 165/202 2024-03-04T07:25:34.920 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : acl-2.3.1-3.el9.x86_64 166/202 2024-03-04T07:25:34.920 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 167/202 2024-03-04T07:25:34.920 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : elfutils-debuginfod-client-0.189-3.el9.x86_64 168/202 2024-03-04T07:25:34.921 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 169/202 2024-03-04T07:25:34.921 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : elfutils-libelf-0.189-3.el9.x86_64 170/202 2024-03-04T07:25:34.921 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 171/202 2024-03-04T07:25:34.921 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : elfutils-libs-0.189-3.el9.x86_64 172/202 2024-03-04T07:25:34.921 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : iproute-6.7.0-2.el9.x86_64 173/202 2024-03-04T07:25:34.921 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : iproute-6.2.0-5.el9.x86_64 174/202 2024-03-04T07:25:34.921 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : iproute-tc-6.7.0-2.el9.x86_64 175/202 2024-03-04T07:25:34.922 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : iproute-tc-6.2.0-5.el9.x86_64 176/202 2024-03-04T07:25:34.922 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libacl-2.3.1-4.el9.x86_64 177/202 2024-03-04T07:25:34.922 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libacl-2.3.1-3.el9.x86_64 178/202 2024-03-04T07:25:34.922 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libblkid-2.37.4-18.el9.x86_64 179/202 2024-03-04T07:25:34.922 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libblkid-2.37.4-15.el9.x86_64 180/202 2024-03-04T07:25:34.922 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libfdisk-2.37.4-18.el9.x86_64 181/202 2024-03-04T07:25:34.923 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libfdisk-2.37.4-15.el9.x86_64 182/202 2024-03-04T07:25:34.923 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libmount-2.37.4-18.el9.x86_64 183/202 2024-03-04T07:25:34.923 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libmount-2.37.4-15.el9.x86_64 184/202 2024-03-04T07:25:34.923 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libsmartcols-2.37.4-18.el9.x86_64 185/202 2024-03-04T07:25:34.923 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libsmartcols-2.37.4-15.el9.x86_64 186/202 2024-03-04T07:25:34.923 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libuuid-2.37.4-18.el9.x86_64 187/202 2024-03-04T07:25:34.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libuuid-2.37.4-15.el9.x86_64 188/202 2024-03-04T07:25:34.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 189/202 2024-03-04T07:25:34.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openssl-1:3.0.7-24.el9.x86_64 190/202 2024-03-04T07:25:34.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 191/202 2024-03-04T07:25:34.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openssl-libs-1:3.0.7-24.el9.x86_64 192/202 2024-03-04T07:25:34.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : util-linux-2.37.4-18.el9.x86_64 193/202 2024-03-04T07:25:34.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : util-linux-2.37.4-15.el9.x86_64 194/202 2024-03-04T07:25:34.925 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : util-linux-core-2.37.4-18.el9.x86_64 195/202 2024-03-04T07:25:34.925 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : util-linux-core-2.37.4-15.el9.x86_64 196/202 2024-03-04T07:25:34.925 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : util-linux-user-2.37.4-18.el9.x86_64 197/202 2024-03-04T07:25:34.925 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : util-linux-user-2.37.4-15.el9.x86_64 198/202 2024-03-04T07:25:34.925 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : xfsdump-3.1.12-4.el9.x86_64 199/202 2024-03-04T07:25:34.925 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : xfsdump-3.1.12-3.el9.x86_64 200/202 2024-03-04T07:25:34.926 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 201/202 2024-03-04T07:25:35.698 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 172/202 2024-03-04T07:25:35.771 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openssl-devel-1:3.0.7-27.el9.x86_64 173/202 2024-03-04T07:25:35.813 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 174/202 2024-03-04T07:25:35.899 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flex-2.6.4-9.el9.x86_64 175/202 2024-03-04T07:25:35.933 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : bison-3.7.4-5.el9.x86_64 176/202 2024-03-04T07:25:35.958 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : NetworkManager-tui-1:1.44.0-2.el9.x86_64 177/202 2024-03-04T07:25:35.984 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : elfutils-debuginfod-client-0.189-3.el9.x86_64 178/202 2024-03-04T07:25:36.010 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : elfutils-libs-0.189-3.el9.x86_64 179/202 2024-03-04T07:25:36.010 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-04T07:25:36.103 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 180/202 2024-03-04T07:25:36.131 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libacl-devel-2.3.1-3.el9.x86_64 181/202 2024-03-04T07:25:36.161 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : openssl-1:3.0.7-24.el9.x86_64 182/202 2024-03-04T07:25:36.161 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-04T07:25:36.288 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 183/202 2024-03-04T07:25:36.315 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : xfsdump-3.1.12-3.el9.x86_64 184/202 2024-03-04T07:25:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : xfsprogs-5.19.0-4.el9.x86_64 185/202 2024-03-04T07:25:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : iproute-tc-6.2.0-5.el9.x86_64 186/202 2024-03-04T07:25:36.417 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : acl-2.3.1-3.el9.x86_64 187/202 2024-03-04T07:25:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : iproute-6.2.0-5.el9.x86_64 188/202 2024-03-04T07:25:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : util-linux-user-2.37.4-15.el9.x86_64 189/202 2024-03-04T07:25:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : util-linux-2.37.4-15.el9.x86_64 190/202 2024-03-04T07:25:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : util-linux-core-2.37.4-15.el9.x86_64 191/202 2024-03-04T07:25:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libfdisk-2.37.4-15.el9.x86_64 192/202 2024-03-04T07:25:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libmount-2.37.4-15.el9.x86_64 193/202 2024-03-04T07:25:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libblkid-2.37.4-15.el9.x86_64 194/202 2024-03-04T07:25:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : NetworkManager-team-1:1.44.0-2.el9.x86_64 195/202 2024-03-04T07:25:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-04T07:25:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-04T07:25:36.850 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 196/202 2024-03-04T07:25:36.850 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-04T07:25:36.924 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: NetworkManager-libnm-1:1.44.0-2.el9.x86_64 197/202 2024-03-04T07:25:36.956 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libuuid-2.37.4-15.el9.x86_64 198/202 2024-03-04T07:25:36.983 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libsmartcols-2.37.4-15.el9.x86_64 199/202 2024-03-04T07:25:37.008 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : elfutils-libelf-0.189-3.el9.x86_64 200/202 2024-03-04T07:25:37.042 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libacl-2.3.1-3.el9.x86_64 201/202 2024-03-04T07:25:37.110 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-04T07:25:37.608 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : xfsprogs-5.19.0-4.el9.x86_64 202/202 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout:Upgraded: 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager-team-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: NetworkManager-tui-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: acl-2.3.1-4.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-debuginfod-client-0.190-2.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-libelf-0.190-2.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-libs-0.190-2.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: iproute-6.7.0-2.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: iproute-tc-6.7.0-2.el9.x86_64 2024-03-04T07:25:37.609 INFO:teuthology.orchestra.run.smithi066.stdout: libacl-2.3.1-4.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: libacl-devel-2.3.1-4.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: libblkid-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: libfdisk-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: libmount-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: libsmartcols-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: libuuid-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: openssl-1:3.0.7-27.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: openssl-libs-1:3.0.7-27.el9.x86_64 2024-03-04T07:25:37.610 INFO:teuthology.orchestra.run.smithi066.stdout: util-linux-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: util-linux-core-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: util-linux-user-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: xfsdump-3.1.12-4.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: xfsprogs-6.3.0-1.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: automake-1.16.2-8.el9.noarch 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: bison-3.7.4-5.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.611 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-immutable-object-cache-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.612 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: flex-2.6.4-9.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: fuse-2.9.9-15.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: grpc-data-1.46.7-10.el9.noarch 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: jq-1.6-16.el9.x86_64 2024-03-04T07:25:37.613 INFO:teuthology.orchestra.run.smithi066.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libarrow-9.0.0-10.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.614 INFO:teuthology.orchestra.run.smithi066.stdout: libtool-2.4.6-45.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: lua-5.4.4-4.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: luarocks-3.9.2-1.el9.noarch 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: openssl-devel-1:3.0.7-27.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql-13.11-1.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql-contrib-13.11-1.el9.x86_64 2024-03-04T07:25:37.615 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql-private-libs-13.11-1.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: postgresql-server-13.11-1.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: protobuf-3.14.0-13.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-asyncssh-2.13.2-4.el9.noarch 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-04T07:25:37.616 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-google-auth-1:2.28.0-1.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-04T07:25:37.617 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-packaging-20.9-5.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ply-3.11-14.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-04T07:25:37.618 INFO:teuthology.orchestra.run.smithi066.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-04T07:25:37.619 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: qatlib-23.11.0-1.el9.x86_64 2024-03-04T07:25:37.620 INFO:teuthology.orchestra.run.smithi066.stdout: qatlib-service-23.11.0-1.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: qatzip-libs-1.1.2-2.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: re2-1:20211101-3.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: uuid-1.6.2-55.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: zlib-devel-1.2.11-40.el9.x86_64 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:37.621 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2024-03-04T07:25:38.213 DEBUG:teuthology.parallel:result is None 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: openssl-libs-1:3.0.7-24.el9.x86_64 202/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 1/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 3/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 4/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-1739.g1bc25 5/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 6/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 7/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 8/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 9/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 10/202 2024-03-04T07:25:41.834 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 11/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 12/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 13/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 14/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_ 15/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 16/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 17/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_6 18/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86 19/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 20/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el 21/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9. 22/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_6 23/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 24/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 25/202 2024-03-04T07:25:41.835 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 26/202 2024-03-04T07:25:41.836 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 27/202 2024-03-04T07:25:41.836 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 28/202 2024-03-04T07:25:41.836 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 29/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7. 30/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noa 31/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.n 32/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc 33/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el 34/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 35/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.e 36/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 37/202 2024-03-04T07:25:41.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 38/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 39/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : automake-1.16.2-8.el9.noarch 40/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : bison-3.7.4-5.el9.x86_64 41/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 42/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 43/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flex-2.6.4-9.el9.x86_64 44/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 48/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 49/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 50/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 51/202 2024-03-04T07:25:41.852 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 52/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 53/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 54/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 55/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : lua-5.4.4-4.el9.x86_64 56/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 57/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 58/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openssl-devel-1:3.0.7-27.el9.x86_64 59/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : postgresql-13.11-1.el9.x86_64 60/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : postgresql-contrib-13.11-1.el9.x86_64 61/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : postgresql-private-libs-13.11-1.el9.x86_64 62/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : postgresql-server-13.11-1.el9.x86_64 63/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 64/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 65/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 66/202 2024-03-04T07:25:41.853 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 67/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 68/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 69/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 70/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 71/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 72/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 73/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 74/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 75/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 76/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 77/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 78/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 79/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 80/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 81/202 2024-03-04T07:25:41.854 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 82/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 83/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 84/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : qatlib-23.11.0-1.el9.x86_64 85/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : qatlib-service-23.11.0-1.el9.x86_64 86/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : qatzip-libs-1.1.2-2.el9.x86_64 87/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 88/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 89/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 90/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 91/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : zlib-devel-1.2.11-40.el9.x86_64 92/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 93/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : jq-1.6-16.el9.x86_64 94/202 2024-03-04T07:25:41.855 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 95/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 96/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 97/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 98/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 99/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ply-3.11-14.el9.noarch 100/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 101/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 102/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 103/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 104/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 105/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 106/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 107/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 108/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 109/202 2024-03-04T07:25:41.856 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 110/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 111/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 112/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 113/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 114/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 115/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-asyncssh-2.13.2-4.el9.noarch 116/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 117/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 118/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 119/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 120/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 121/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 122/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 123/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-google-auth-1:2.28.0-1.el9.noarch 124/202 2024-03-04T07:25:41.857 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 125/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 126/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 127/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 128/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 129/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 130/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 131/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 132/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 133/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 134/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 135/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 136/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 137/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 138/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 139/202 2024-03-04T07:25:41.858 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 140/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 141/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 142/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 143/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 144/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 145/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 146/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 147/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 148/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : re2-1:20211101-3.el9.x86_64 149/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 150/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 151/202 2024-03-04T07:25:41.859 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 152/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 153/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 154/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libacl-devel-2.3.1-4.el9.x86_64 155/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libacl-devel-2.3.1-3.el9.x86_64 156/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-1:1.46.0-1.el9.x86_64 157/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-1:1.44.0-2.el9.x86_64 158/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-libnm-1:1.46.0-1.el9.x86_64 159/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 160/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-team-1:1.46.0-1.el9.x86_64 161/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-team-1:1.44.0-2.el9.x86_64 162/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-tui-1:1.46.0-1.el9.x86_64 163/202 2024-03-04T07:25:41.860 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : NetworkManager-tui-1:1.44.0-2.el9.x86_64 164/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : acl-2.3.1-4.el9.x86_64 165/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : acl-2.3.1-3.el9.x86_64 166/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 167/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : elfutils-debuginfod-client-0.189-3.el9.x86_64 168/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 169/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : elfutils-libelf-0.189-3.el9.x86_64 170/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 171/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : elfutils-libs-0.189-3.el9.x86_64 172/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : iproute-6.7.0-2.el9.x86_64 173/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : iproute-6.2.0-5.el9.x86_64 174/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : iproute-tc-6.7.0-2.el9.x86_64 175/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : iproute-tc-6.2.0-5.el9.x86_64 176/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libacl-2.3.1-4.el9.x86_64 177/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libacl-2.3.1-3.el9.x86_64 178/202 2024-03-04T07:25:41.861 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libblkid-2.37.4-18.el9.x86_64 179/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libblkid-2.37.4-15.el9.x86_64 180/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libfdisk-2.37.4-18.el9.x86_64 181/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libfdisk-2.37.4-15.el9.x86_64 182/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libmount-2.37.4-18.el9.x86_64 183/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libmount-2.37.4-15.el9.x86_64 184/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libsmartcols-2.37.4-18.el9.x86_64 185/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libsmartcols-2.37.4-15.el9.x86_64 186/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libuuid-2.37.4-18.el9.x86_64 187/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libuuid-2.37.4-15.el9.x86_64 188/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 189/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openssl-1:3.0.7-24.el9.x86_64 190/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 191/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openssl-libs-1:3.0.7-24.el9.x86_64 192/202 2024-03-04T07:25:41.862 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : util-linux-2.37.4-18.el9.x86_64 193/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : util-linux-2.37.4-15.el9.x86_64 194/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : util-linux-core-2.37.4-18.el9.x86_64 195/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : util-linux-core-2.37.4-15.el9.x86_64 196/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : util-linux-user-2.37.4-18.el9.x86_64 197/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : util-linux-user-2.37.4-15.el9.x86_64 198/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xfsdump-3.1.12-4.el9.x86_64 199/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xfsdump-3.1.12-3.el9.x86_64 200/202 2024-03-04T07:25:41.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 201/202 2024-03-04T07:25:44.689 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xfsprogs-5.19.0-4.el9.x86_64 202/202 2024-03-04T07:25:44.689 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:44.689 INFO:teuthology.orchestra.run.smithi064.stdout:Upgraded: 2024-03-04T07:25:44.689 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager-libnm-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager-team-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: NetworkManager-tui-1:1.46.0-1.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: acl-2.3.1-4.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-debuginfod-client-0.190-2.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-libelf-0.190-2.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-libs-0.190-2.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: iproute-6.7.0-2.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: iproute-tc-6.7.0-2.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: libacl-2.3.1-4.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: libacl-devel-2.3.1-4.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: libblkid-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: libfdisk-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.690 INFO:teuthology.orchestra.run.smithi064.stdout: libmount-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: librados2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: libsmartcols-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: libuuid-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: openssl-1:3.0.7-27.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: openssl-libs-1:3.0.7-27.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: util-linux-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: util-linux-core-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: util-linux-user-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: xfsdump-3.1.12-4.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: xfsprogs-6.3.0-1.el9.x86_64 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-03-04T07:25:44.691 INFO:teuthology.orchestra.run.smithi064.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: automake-1.16.2-8.el9.noarch 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: bison-3.7.4-5.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.692 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-volume-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.693 INFO:teuthology.orchestra.run.smithi064.stdout: cephfs-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: cephfs-shell-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: cephfs-top-2:19.0.0-1739.g1bc250c7.el9.noarch 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: flex-2.6.4-9.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: fuse-2.9.9-15.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: grpc-data-1.46.7-10.el9.noarch 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: jq-1.6-16.el9.x86_64 2024-03-04T07:25:44.694 INFO:teuthology.orchestra.run.smithi064.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow-9.0.0-10.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-04T07:25:44.695 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: libtool-2.4.6-45.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: lua-5.4.4-4.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: luarocks-3.9.2-1.el9.noarch 2024-03-04T07:25:44.696 INFO:teuthology.orchestra.run.smithi064.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: openssl-devel-1:3.0.7-27.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql-13.11-1.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql-contrib-13.11-1.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql-private-libs-13.11-1.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: postgresql-server-13.11-1.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: protobuf-3.14.0-13.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh-2.13.2-4.el9.noarch 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-03-04T07:25:44.697 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-04T07:25:44.698 INFO:teuthology.orchestra.run.smithi064.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth-1:2.28.0-1.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-04T07:25:44.699 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-packaging-20.9-5.el9.noarch 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-04T07:25:44.700 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ply-3.11-14.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.701 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-04T07:25:44.702 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-04T07:25:44.702 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-04T07:25:44.702 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.702 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-04T07:25:44.703 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: qatlib-23.11.0-1.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: qatlib-service-23.11.0-1.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: qatzip-libs-1.1.2-2.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd-2:19.0.0-1739.g1bc250c7.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: re2-1:20211101-3.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: uuid-1.6.2-55.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-03-04T07:25:44.704 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-04T07:25:44.706 INFO:teuthology.orchestra.run.smithi064.stdout: zlib-devel-1.2.11-40.el9.x86_64 2024-03-04T07:25:44.707 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:44.707 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-03-04T07:25:45.328 DEBUG:teuthology.parallel:result is None 2024-03-04T07:25:45.329 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-04T07:25:45.545 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-04T07:25:45.567 INFO:teuthology.orchestra.run.smithi064.stdout:19.0.0-1739.g1bc250c7.el9 2024-03-04T07:25:45.568 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-1739.g1bc250c7.el9 2024-03-04T07:25:45.568 INFO:teuthology.task.install:The correct ceph version 19.0.0-1739.g1bc250c7 is installed. 2024-03-04T07:25:45.570 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd 2024-03-04T07:25:45.711 DEBUG:teuthology.orchestra.run.smithi066:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-04T07:25:45.739 INFO:teuthology.orchestra.run.smithi066.stdout:19.0.0-1739.g1bc250c7.el9 2024-03-04T07:25:45.739 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-1739.g1bc250c7.el9 2024-03-04T07:25:45.739 INFO:teuthology.task.install:The correct ceph version 19.0.0-1739.g1bc250c7 is installed. 2024-03-04T07:25:45.741 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-04T07:25:45.741 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:45.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-04T07:25:45.772 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:45.772 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-04T07:25:45.819 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-04T07:25:45.820 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:45.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/daemon-helper 2024-03-04T07:25:45.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-04T07:25:45.916 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:45.916 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/daemon-helper 2024-03-04T07:25:45.953 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-04T07:25:46.027 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-04T07:25:46.028 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:46.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-04T07:25:46.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-04T07:25:46.123 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:46.123 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-04T07:25:46.157 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-04T07:25:46.230 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-04T07:25:46.230 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:46.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/stdin-killer 2024-03-04T07:25:46.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-04T07:25:46.323 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:46.323 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/stdin-killer 2024-03-04T07:25:46.355 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-04T07:25:46.427 INFO:teuthology.run_tasks:Running task ceph... 2024-03-04T07:25:46.540 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-03-04T07:25:46.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 777 /var/log/ceph 2024-03-04T07:25:46.542 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 777 /var/log/ceph 2024-03-04T07:25:46.574 INFO:tasks.ceph:Disabling ceph logrotate... 2024-03-04T07:25:46.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /etc/logrotate.d/ceph 2024-03-04T07:25:46.610 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f -- /etc/logrotate.d/ceph 2024-03-04T07:25:46.644 INFO:tasks.ceph:Creating extra log directories... 2024-03-04T07:25:46.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-03-04T07:25:46.678 DEBUG:teuthology.orchestra.run.smithi066:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-03-04T07:25:46.722 INFO:tasks.ceph:Setting up log rotation with {'ceph-mds': '10G', 'ceph-osd': '10G'} 2024-03-04T07:25:46.722 INFO:tasks.ceph:writing logrotate stanza for ceph-mds 2024-03-04T07:25:46.722 INFO:tasks.ceph:writing logrotate stanza for ceph-osd 2024-03-04T07:25:46.722 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:46.723 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/home/ubuntu/cephtest/logrotate.ceph-test.conf 2024-03-04T07:25:46.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo mv /home/ubuntu/cephtest/logrotate.ceph-test.conf /etc/logrotate.d/ceph-test.conf && sudo chmod 0644 /etc/logrotate.d/ceph-test.conf && sudo chown root.root /etc/logrotate.d/ceph-test.conf 2024-03-04T07:25:46.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:etc_t:s0 /etc/logrotate.d/ceph-test.conf 2024-03-04T07:25:46.902 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:46.902 DEBUG:teuthology.orchestra.run.smithi066:> dd of=/home/ubuntu/cephtest/logrotate.ceph-test.conf 2024-03-04T07:25:46.918 DEBUG:teuthology.orchestra.run.smithi066:> sudo mv /home/ubuntu/cephtest/logrotate.ceph-test.conf /etc/logrotate.d/ceph-test.conf && sudo chmod 0644 /etc/logrotate.d/ceph-test.conf && sudo chown root.root /etc/logrotate.d/ceph-test.conf 2024-03-04T07:25:47.046 DEBUG:teuthology.orchestra.run.smithi066:> sudo chcon system_u:object_r:etc_t:s0 /etc/logrotate.d/ceph-test.conf 2024-03-04T07:25:47.088 INFO:tasks.ceph:Creating ceph cluster ceph... 2024-03-04T07:25:47.088 INFO:tasks.ceph:config {'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-03-04T07:25:47.088 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679', 'branch': 'wip-lusov-qdb', 'description': 'fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/strays}', 'email': None, 'first_in_suite': False, 'job_id': '7580679', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'log-rotate': {'ceph-mds': '10G', 'ceph-osd': '10G'}, 'machine_type': 'smithi', 'name': 'leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'openstack': [{'volumes': {'count': 4, 'size': 20}}, {'machine': {'disk': 200}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-lusov-qdb'}, 'ceph': {'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}}, 'subvolume_version': 2, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-lusov-qdb', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}}, 'owner': 'scheduled_leonidus@teuthology', 'priority': 80, 'repo': 'https://git.ceph.com/ceph-ci.git', 'roles': [['mon.a', 'mgr.y', 'mds.a', 'mds.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'client.2'], ['mon.b', 'mon.c', 'mgr.x', 'mds.b', 'mds.d', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'client.3']], 'seed': 5072, 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'sleep_before_teardown': 0, 'subset': '246/10000', 'suite': 'fs', 'suite_branch': 'wip-lusov-qdb', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph-ci.git', 'suite_sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'syslog': {'ignorelist': ['WARNING*.*check_session_state', 'WARNING*.*__ceph_remove_cap']}, 'targets': {'smithi064.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFbHJeUCtklmTdRTeWvj7/TLDObNEjK0smR7zXProxs0CCq4kBiFvtnX0XZZZNjy0cajgLMmeYQq3V0ahWEsOmM=', 'smithi066.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJgFwnT3fiITMHowq7yiXYhbbE9FmTZ3/ZH0dOJkclVFjVeeC6i38jqL7TbA0ccgfJevOf/HWCuKqgGfd4IXD+Y='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd'}}, {'ceph': {'create_rbd_pool': False, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'sha1': '1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd', 'cluster': 'ceph'}}, {'ceph-fuse': None}, {'cephfs_test_runner': {'modules': ['tasks.cephfs.test_strays']}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': ['local function is_kupstream() return false end', 'local function is_kdistro() return false end', 'local function is_fuse() return true end', "local function syntax_version() return '' end"]}, 'teuthology_branch': 'main', 'teuthology_sha1': 'b1dac5519c57c269ea98a22fb7729016a1d0e4d2', 'timestamp': '2024-03-04_07:05:03', 'tube': 'smithi', 'user': 'leonidus', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885'} 2024-03-04T07:25:47.089 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-03-04T07:25:47.092 DEBUG:teuthology.orchestra.run.smithi066:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-03-04T07:25:47.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo install -d -m0777 -- /var/run/ceph 2024-03-04T07:25:47.146 DEBUG:teuthology.orchestra.run.smithi066:> sudo install -d -m0777 -- /var/run/ceph 2024-03-04T07:25:47.204 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:47.204 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/scratch_devs of=/dev/stdout 2024-03-04T07:25:47.226 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-04T07:25:47.226 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_1 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-03-04 07:25:37.435144723 +0000 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-03-04 07:23:07.615315902 +0000 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-03-04 07:23:07.615315902 +0000 2024-03-04T07:25:47.281 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-03-04 07:23:07.615315902 +0000 2024-03-04T07:25:47.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-04T07:25:47.346 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-03-04T07:25:47.346 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-03-04T07:25:47.346 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000198854 s, 2.6 MB/s 2024-03-04T07:25:47.347 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-04T07:25:47.403 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_2 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-03-04 07:25:37.436144704 +0000 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-03-04 07:23:07.991308723 +0000 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-03-04 07:23:07.991308723 +0000 2024-03-04T07:25:47.458 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-03-04 07:23:07.991308723 +0000 2024-03-04T07:25:47.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-04T07:25:47.523 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-03-04T07:25:47.523 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-03-04T07:25:47.523 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000200887 s, 2.5 MB/s 2024-03-04T07:25:47.524 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-04T07:25:47.579 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_3 2024-03-04T07:25:47.633 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-03-04 07:25:37.436144704 +0000 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-03-04 07:23:08.362301639 +0000 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-03-04 07:23:08.362301639 +0000 2024-03-04T07:25:47.634 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-03-04 07:23:08.362301639 +0000 2024-03-04T07:25:47.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-04T07:25:47.699 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-03-04T07:25:47.699 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-03-04T07:25:47.699 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000212264 s, 2.4 MB/s 2024-03-04T07:25:47.700 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-04T07:25:47.756 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_4 2024-03-04T07:25:47.811 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-03-04 07:25:37.437144685 +0000 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-03-04 07:23:08.708295032 +0000 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-03-04 07:23:08.708295032 +0000 2024-03-04T07:25:47.812 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-03-04 07:23:08.708295032 +0000 2024-03-04T07:25:47.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-04T07:25:47.877 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-03-04T07:25:47.877 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-03-04T07:25:47.877 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000175822 s, 2.9 MB/s 2024-03-04T07:25:47.878 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-04T07:25:47.933 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-04T07:25:47.933 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:47.933 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/scratch_devs of=/dev/stdout 2024-03-04T07:25:47.951 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-04T07:25:47.951 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_1 2024-03-04T07:25:48.009 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2024-03-04 07:25:30.599213661 +0000 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2024-03-04 07:23:07.613285736 +0000 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2024-03-04 07:23:07.613285736 +0000 2024-03-04T07:25:48.010 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: 2024-03-04 07:23:07.613285736 +0000 2024-03-04T07:25:48.010 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-04T07:25:48.083 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2024-03-04T07:25:48.083 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2024-03-04T07:25:48.083 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000205049 s, 2.5 MB/s 2024-03-04T07:25:48.085 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-04T07:25:48.143 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_2 2024-03-04T07:25:48.200 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-04T07:25:48.200 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:48.200 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-03-04T07:25:48.200 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:48.200 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:48.200 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2024-03-04 07:25:30.600213655 +0000 2024-03-04T07:25:48.200 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2024-03-04 07:23:07.974283481 +0000 2024-03-04T07:25:48.201 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2024-03-04 07:23:07.974283481 +0000 2024-03-04T07:25:48.201 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: 2024-03-04 07:23:07.974283481 +0000 2024-03-04T07:25:48.201 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-04T07:25:48.273 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2024-03-04T07:25:48.273 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2024-03-04T07:25:48.273 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000212931 s, 2.4 MB/s 2024-03-04T07:25:48.274 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-04T07:25:48.331 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_3 2024-03-04T07:25:48.388 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-04T07:25:48.388 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:48.388 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-03-04T07:25:48.388 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:48.389 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:48.389 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2024-03-04 07:25:30.600213655 +0000 2024-03-04T07:25:48.389 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2024-03-04 07:23:08.332281245 +0000 2024-03-04T07:25:48.389 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2024-03-04 07:23:08.332281245 +0000 2024-03-04T07:25:48.389 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: 2024-03-04 07:23:08.332281245 +0000 2024-03-04T07:25:48.389 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-04T07:25:48.459 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2024-03-04T07:25:48.459 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2024-03-04T07:25:48.459 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000198411 s, 2.6 MB/s 2024-03-04T07:25:48.460 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-04T07:25:48.517 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_4 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2024-03-04 07:25:30.601213648 +0000 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2024-03-04 07:23:08.682279059 +0000 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2024-03-04 07:23:08.682279059 +0000 2024-03-04T07:25:48.573 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: 2024-03-04 07:23:08.682279059 +0000 2024-03-04T07:25:48.574 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-04T07:25:48.643 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2024-03-04T07:25:48.643 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2024-03-04T07:25:48.643 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000204047 s, 2.5 MB/s 2024-03-04T07:25:48.644 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-04T07:25:48.700 INFO:tasks.ceph:osd dev map: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-04T07:25:48.700 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi064.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi066.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2024-03-04T07:25:48.700 INFO:tasks.ceph:Generating config... 2024-03-04T07:25:48.702 INFO:tasks.ceph:[client] client mount timeout = 600 2024-03-04T07:25:48.702 INFO:tasks.ceph:[client] debug client = 20 2024-03-04T07:25:48.702 INFO:tasks.ceph:[client] debug ms = 1 2024-03-04T07:25:48.702 INFO:tasks.ceph:[client] pid file = 2024-03-04T07:25:48.702 INFO:tasks.ceph:[client] rados mon op timeout = 900 2024-03-04T07:25:48.702 INFO:tasks.ceph:[client] rados osd op timeout = 900 2024-03-04T07:25:48.702 INFO:tasks.ceph:[mds] debug mds = 20 2024-03-04T07:25:48.702 INFO:tasks.ceph:[mds] debug mds balancer = 20 2024-03-04T07:25:48.702 INFO:tasks.ceph:[mds] debug ms = 1 2024-03-04T07:25:48.702 INFO:tasks.ceph:[mds] mds debug frag = True 2024-03-04T07:25:48.702 INFO:tasks.ceph:[mds] mds debug scatterstat = True 2024-03-04T07:25:48.702 INFO:tasks.ceph:[mds] mds op complaint time = 180 2024-03-04T07:25:48.702 INFO:tasks.ceph:[mds] mds verify scatter = True 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mds] osd op complaint time = 180 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mds] rados mon op timeout = 900 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mds] rados osd op timeout = 900 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mgr] debug ms = 1 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mon] debug mon = 20 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mon] debug ms = 1 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mon] debug paxos = 20 2024-03-04T07:25:48.703 INFO:tasks.ceph:[mon] mon op complaint time = 120 2024-03-04T07:25:48.703 INFO:tasks.ceph:[osd] bdev async discard = True 2024-03-04T07:25:48.703 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-03-04T07:25:48.703 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2024-03-04T07:25:48.703 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-03-04T07:25:48.703 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-03-04T07:25:48.703 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] debug ms = 1 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] debug osd = 20 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-03-04T07:25:48.704 INFO:tasks.ceph:[osd] osd op complaint time = 180 2024-03-04T07:25:48.704 INFO:tasks.ceph:Setting up mon.a... 2024-03-04T07:25:48.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2024-03-04T07:25:48.754 INFO:teuthology.orchestra.run.smithi064.stdout:creating /etc/ceph/ceph.keyring 2024-03-04T07:25:48.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2024-03-04T07:25:48.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-03-04T07:25:48.828 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.64'), ('mon.b', '172.21.15.66'), ('mon.c', '[v2:172.21.15.66:3301,v1:172.21.15.66:6790]')] 2024-03-04T07:25:48.829 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]'}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false', 'mon op complaint time': 120}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'pid file': '', 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon.a': {}, 'mon.b': {}, 'mon.c': {}, 'mds.a': {}, 'mds.c': {}, 'mds.b': {}, 'mds.d': {}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}} 2024-03-04T07:25:48.829 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:48.829 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-03-04T07:25:48.883 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.64 --add b 172.21.15.66 --addv c '[v2:172.21.15.66:3301,v1:172.21.15.66:6790]' --print /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:monmaptool: generated fsid c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:setting min_mon_release = quincy 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:epoch 0 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:fsid c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:last_changed 2024-03-04T07:25:48.969265+0000 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:created 2024-03-04T07:25:48.969265+0000 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:min_mon_release 17 (quincy) 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:election_strategy: 1 2024-03-04T07:25:48.968 INFO:teuthology.orchestra.run.smithi064.stdout:0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-03-04T07:25:48.969 INFO:teuthology.orchestra.run.smithi064.stdout:1: [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] mon.b 2024-03-04T07:25:48.969 INFO:teuthology.orchestra.run.smithi064.stdout:2: [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] mon.c 2024-03-04T07:25:48.969 INFO:teuthology.orchestra.run.smithi064.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2024-03-04T07:25:48.970 DEBUG:teuthology.orchestra.run.smithi064:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-03-04T07:25:48.984 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID c3ef085c-fd40-4f87-8c92-fdb42a102a8c... 2024-03-04T07:25:48.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:25:49.027 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:25:49.074 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-03-04T07:25:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: chdir = "" 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: auth supported = cephx 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: filestore xattr use omap = true 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: mon clock drift allowed = 1.000 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: osd crush chooseleaf type = 0 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: auth debug = true 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on old message = true 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on bug = true 2024-03-04T07:25:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon pg warn max object skew = 0 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default size = 2 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow primary affinity = true 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow pg remap = true 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on no sortbitwise = false 2024-03-04T07:25:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on too few osds = false 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: mon_allow_pool_size_one = true 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: osd default data pool replay window = 5 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: mon allow pool delete = true 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: mon cluster log file level = debug 2024-03-04T07:25:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: debug asserts on shutdown = true 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: mon health detail to clog = false 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout:[osd] 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: osd journal size = 100 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub load threshold = 5.0 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub max interval = 600 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: osd recover clone overlap = true 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: osd recovery max chunk = 1048576 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug shutdown = true 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug op order = true 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug verify stray on activate = true 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug trim objects = true 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: osd open classes on start = true 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug pg log writeout = true 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: osd map max advance = 10 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: journal zero on create = true 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: filestore apply finisher threads = 3 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: bdev debug aio = true 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug misdirected ops = true 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: bdev async discard = True 2024-03-04T07:25:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: bdev enable discard = True 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore allocator = bitmap 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore block size = 96636764160 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore fsck on mount = True 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluefs = 1/20 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluestore = 1/20 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: debug osd = 20 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: debug rocksdb = 4/10 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd full ratio = 0.9 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:25:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: osd objectstore = bluestore 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr] 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: debug mgr = 20 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: mgr/telemetry/nag = false 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout:[mon] 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:25:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: debug paxos = 20 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon data avail warn = 5 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon mgr mkfs grace = 240 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd reporter subtree level = osd 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd prime pg temp = true 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:25:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: # 1m isn't quite enough 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: mon op complaint time = 120 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout:[client] 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: rgw cache enabled = true 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable ops log = true 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable usage log = true 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:25:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: client mount timeout = 600 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: debug client = 20 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = "" 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.a] 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.b] 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.c] 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:25:49.085 INFO:teuthology.orchestra.run.smithi064.stdout:[mds] 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds = 20 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds balancer = 20 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug frag = True 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug scatterstat = True 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: mds op complaint time = 180 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: mds verify scatter = True 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:25:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout:[global] 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: chdir = "" 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: auth supported = cephx 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: filestore xattr use omap = true 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: mon clock drift allowed = 1.000 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.088 INFO:teuthology.orchestra.run.smithi066.stdout: osd crush chooseleaf type = 0 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: auth debug = true 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on old message = true 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on bug = true 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: mon pg warn max object skew = 0 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default size = 2 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.089 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow primary affinity = true 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow pg remap = true 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on no sortbitwise = false 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on too few osds = false 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: mon_allow_pool_size_one = true 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: osd default data pool replay window = 5 2024-03-04T07:25:49.090 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: mon allow pool delete = true 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: mon cluster log file level = debug 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: debug asserts on shutdown = true 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: mon health detail to clog = false 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout:[osd] 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: osd journal size = 100 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub load threshold = 5.0 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub max interval = 600 2024-03-04T07:25:49.091 INFO:teuthology.orchestra.run.smithi066.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd recover clone overlap = true 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd recovery max chunk = 1048576 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug shutdown = true 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug op order = true 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug verify stray on activate = true 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug trim objects = true 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd open classes on start = true 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug pg log writeout = true 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.092 INFO:teuthology.orchestra.run.smithi066.stdout: osd map max advance = 10 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: journal zero on create = true 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: filestore apply finisher threads = 3 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: bdev debug aio = true 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug misdirected ops = true 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: bdev async discard = True 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: bdev enable discard = True 2024-03-04T07:25:49.093 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore allocator = bitmap 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore block size = 96636764160 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore fsck on mount = True 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluefs = 1/20 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluestore = 1/20 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: debug osd = 20 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: debug rocksdb = 4/10 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd full ratio = 0.9 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: osd objectstore = bluestore 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr] 2024-03-04T07:25:49.094 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: debug mgr = 20 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: mgr/telemetry/nag = false 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout:[mon] 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:25:49.095 INFO:teuthology.orchestra.run.smithi066.stdout: debug paxos = 20 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: mon data avail warn = 5 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: mon mgr mkfs grace = 240 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd reporter subtree level = osd 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd prime pg temp = true 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.096 INFO:teuthology.orchestra.run.smithi066.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: # 1m isn't quite enough 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: mon op complaint time = 120 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout:[client] 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: rgw cache enabled = true 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable ops log = true 2024-03-04T07:25:49.097 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable usage log = true 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: client mount timeout = 600 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: debug client = 20 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = "" 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.a] 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.b] 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.c] 2024-03-04T07:25:49.098 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout:[mds] 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds = 20 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds balancer = 20 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug frag = True 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug scatterstat = True 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: mds op complaint time = 180 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: mds verify scatter = True 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:25:49.099 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:25:49.105 INFO:tasks.ceph:Creating admin key on mon.a... 2024-03-04T07:25:49.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2024-03-04T07:25:49.183 INFO:tasks.ceph:Copying monmap to all nodes... 2024-03-04T07:25:49.184 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:49.184 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2024-03-04T07:25:49.199 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:49.199 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2024-03-04T07:25:49.253 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:25:49.254 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:49.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.keyring 2024-03-04T07:25:49.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-03-04T07:25:49.332 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:25:49.332 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:25:49.386 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:25:49.387 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:49.387 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/ceph/ceph.keyring 2024-03-04T07:25:49.387 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-03-04T07:25:49.433 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:25:49.433 DEBUG:teuthology.orchestra.run.smithi066:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:25:49.488 INFO:tasks.ceph:Setting up mon nodes... 2024-03-04T07:25:49.488 INFO:tasks.ceph:Setting up mgr nodes... 2024-03-04T07:25:49.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2024-03-04T07:25:49.543 INFO:teuthology.orchestra.run.smithi064.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2024-03-04T07:25:49.545 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2024-03-04T07:25:49.645 INFO:teuthology.orchestra.run.smithi066.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2024-03-04T07:25:49.647 INFO:tasks.ceph:Setting up mds nodes... 2024-03-04T07:25:49.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/mds/ceph-a && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.a /var/lib/ceph/mds/ceph-a/keyring 2024-03-04T07:25:49.703 INFO:teuthology.orchestra.run.smithi064.stdout:creating /var/lib/ceph/mds/ceph-a/keyring 2024-03-04T07:25:49.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-a 2024-03-04T07:25:49.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/mds/ceph-c && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.c /var/lib/ceph/mds/ceph-c/keyring 2024-03-04T07:25:49.826 INFO:teuthology.orchestra.run.smithi064.stdout:creating /var/lib/ceph/mds/ceph-c/keyring 2024-03-04T07:25:49.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-c 2024-03-04T07:25:49.855 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/mds/ceph-b && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.b /var/lib/ceph/mds/ceph-b/keyring 2024-03-04T07:25:49.918 INFO:teuthology.orchestra.run.smithi066.stdout:creating /var/lib/ceph/mds/ceph-b/keyring 2024-03-04T07:25:49.921 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-b 2024-03-04T07:25:49.951 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/mds/ceph-d && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mds.d /var/lib/ceph/mds/ceph-d/keyring 2024-03-04T07:25:50.055 INFO:teuthology.orchestra.run.smithi066.stdout:creating /var/lib/ceph/mds/ceph-d/keyring 2024-03-04T07:25:50.057 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/mds/ceph-d 2024-03-04T07:25:50.087 INFO:tasks.ceph_client:Setting up client nodes... 2024-03-04T07:25:50.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/ceph.client.0.keyring && sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-04T07:25:50.130 INFO:teuthology.orchestra.run.smithi064.stdout:creating /etc/ceph/ceph.client.0.keyring 2024-03-04T07:25:50.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.2 /etc/ceph/ceph.client.2.keyring && sudo chmod 0644 /etc/ceph/ceph.client.2.keyring 2024-03-04T07:25:50.228 INFO:teuthology.orchestra.run.smithi064.stdout:creating /etc/ceph/ceph.client.2.keyring 2024-03-04T07:25:50.243 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/ceph.client.1.keyring && sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-03-04T07:25:50.290 INFO:teuthology.orchestra.run.smithi066.stdout:creating /etc/ceph/ceph.client.1.keyring 2024-03-04T07:25:50.309 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.3 /etc/ceph/ceph.client.3.keyring && sudo chmod 0644 /etc/ceph/ceph.client.3.keyring 2024-03-04T07:25:50.355 INFO:teuthology.orchestra.run.smithi066.stdout:creating /etc/ceph/ceph.client.3.keyring 2024-03-04T07:25:50.373 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-03-04T07:25:50.373 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi064.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi066.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2024-03-04T07:25:50.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2024-03-04T07:25:50.402 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-04T07:25:50.402 INFO:tasks.ceph:role: osd.0 2024-03-04T07:25:50.402 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:25:50.403 DEBUG:teuthology.orchestra.run.smithi064:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-03-04T07:25:50.476 INFO:teuthology.orchestra.run.smithi064.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:25:50.477 INFO:teuthology.orchestra.run.smithi064.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:25:50.978 INFO:teuthology.orchestra.run.smithi064.stdout:Discarding blocks...Done. 2024-03-04T07:25:50.980 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi064.front.sepia.ceph.com -o noatime 2024-03-04T07:25:50.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2024-03-04T07:25:51.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2024-03-04T07:25:51.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2024-03-04T07:25:51.209 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-04T07:25:51.209 INFO:tasks.ceph:role: osd.1 2024-03-04T07:25:51.209 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:25:51.209 DEBUG:teuthology.orchestra.run.smithi064:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-03-04T07:25:51.283 INFO:teuthology.orchestra.run.smithi064.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:25:51.283 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:25:51.283 INFO:teuthology.orchestra.run.smithi064.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:25:51.283 INFO:teuthology.orchestra.run.smithi064.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:25:51.283 INFO:teuthology.orchestra.run.smithi064.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:25:51.284 INFO:teuthology.orchestra.run.smithi064.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:25:51.284 INFO:teuthology.orchestra.run.smithi064.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:25:51.284 INFO:teuthology.orchestra.run.smithi064.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:25:51.284 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:25:51.284 INFO:teuthology.orchestra.run.smithi064.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:25:51.734 INFO:teuthology.orchestra.run.smithi064.stdout:Discarding blocks...Done. 2024-03-04T07:25:51.736 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi064.front.sepia.ceph.com -o noatime 2024-03-04T07:25:51.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2024-03-04T07:25:51.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2024-03-04T07:25:51.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2024-03-04T07:25:51.926 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-04T07:25:51.927 INFO:tasks.ceph:role: osd.2 2024-03-04T07:25:51.927 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:25:51.927 DEBUG:teuthology.orchestra.run.smithi064:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:25:52.001 INFO:teuthology.orchestra.run.smithi064.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:25:52.484 INFO:teuthology.orchestra.run.smithi064.stdout:Discarding blocks...Done. 2024-03-04T07:25:52.486 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi064.front.sepia.ceph.com -o noatime 2024-03-04T07:25:52.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2024-03-04T07:25:52.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2024-03-04T07:25:52.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2024-03-04T07:25:52.711 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-03-04T07:25:52.711 INFO:tasks.ceph:role: osd.3 2024-03-04T07:25:52.711 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:25:52.711 DEBUG:teuthology.orchestra.run.smithi064:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:25:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:25:52.786 INFO:teuthology.orchestra.run.smithi064.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:25:53.308 INFO:teuthology.orchestra.run.smithi064.stdout:Discarding blocks...Done. 2024-03-04T07:25:53.309 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi064.front.sepia.ceph.com -o noatime 2024-03-04T07:25:53.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2024-03-04T07:25:53.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2024-03-04T07:25:53.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:25:53.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:53.591+0000 7f951e01e640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2024-03-04T07:25:53.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:53.591+0000 7f951e01e640 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2024-03-04T07:25:53.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:53.591+0000 7f951e01e640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-03-04T07:25:53.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:53.591+0000 7f951e01e640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-03-04T07:25:53.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:53.591+0000 7f951e01e640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-03-04T07:25:53.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:53.592+0000 7f951e01e640 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2024-03-04T07:25:56.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-04T07:25:56.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:25:56.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:56.560+0000 7efe97f9a640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2024-03-04T07:25:56.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:56.560+0000 7efe97f9a640 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2024-03-04T07:25:56.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:56.560+0000 7efe97f9a640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-03-04T07:25:56.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:56.560+0000 7efe97f9a640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-03-04T07:25:56.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:56.560+0000 7efe97f9a640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-03-04T07:25:56.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:56.560+0000 7efe97f9a640 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2024-03-04T07:25:59.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-04T07:25:59.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:25:59.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:59.533+0000 7fa377e21640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2024-03-04T07:25:59.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:59.533+0000 7fa377e21640 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2024-03-04T07:25:59.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:59.533+0000 7fa377e21640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-03-04T07:25:59.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:59.533+0000 7fa377e21640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-03-04T07:25:59.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:59.533+0000 7fa377e21640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-03-04T07:25:59.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:25:59.534+0000 7fa377e21640 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2024-03-04T07:26:02.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-04T07:26:02.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:26:02.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:02.504+0000 7f6a5ef38640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2024-03-04T07:26:02.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:02.505+0000 7f6a5ef38640 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2024-03-04T07:26:02.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:02.505+0000 7f6a5ef38640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-03-04T07:26:02.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:02.505+0000 7f6a5ef38640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-03-04T07:26:02.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:02.505+0000 7f6a5ef38640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-03-04T07:26:02.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:02.505+0000 7f6a5ef38640 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2024-03-04T07:26:05.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-04T07:26:05.381 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/osd/ceph-4 2024-03-04T07:26:05.421 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-04T07:26:05.421 INFO:tasks.ceph:role: osd.4 2024-03-04T07:26:05.421 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:26:05.421 DEBUG:teuthology.orchestra.run.smithi066:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-03-04T07:26:05.503 INFO:teuthology.orchestra.run.smithi066.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:26:05.503 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:26:05.503 INFO:teuthology.orchestra.run.smithi066.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:26:05.503 INFO:teuthology.orchestra.run.smithi066.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:26:05.503 INFO:teuthology.orchestra.run.smithi066.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:26:05.503 INFO:teuthology.orchestra.run.smithi066.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:26:05.504 INFO:teuthology.orchestra.run.smithi066.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:26:05.504 INFO:teuthology.orchestra.run.smithi066.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:26:05.504 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:26:05.504 INFO:teuthology.orchestra.run.smithi066.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:26:06.018 INFO:teuthology.orchestra.run.smithi066.stdout:Discarding blocks...Done. 2024-03-04T07:26:06.020 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi066.front.sepia.ceph.com -o noatime 2024-03-04T07:26:06.020 DEBUG:teuthology.orchestra.run.smithi066:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-4 2024-03-04T07:26:06.146 DEBUG:teuthology.orchestra.run.smithi066:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-4 2024-03-04T07:26:06.186 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/osd/ceph-5 2024-03-04T07:26:06.260 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-04T07:26:06.260 INFO:tasks.ceph:role: osd.5 2024-03-04T07:26:06.260 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:26:06.261 DEBUG:teuthology.orchestra.run.smithi066:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-03-04T07:26:06.340 INFO:teuthology.orchestra.run.smithi066.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:26:06.340 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:26:06.340 INFO:teuthology.orchestra.run.smithi066.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:26:06.340 INFO:teuthology.orchestra.run.smithi066.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:26:06.340 INFO:teuthology.orchestra.run.smithi066.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:26:06.340 INFO:teuthology.orchestra.run.smithi066.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:26:06.341 INFO:teuthology.orchestra.run.smithi066.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:26:06.341 INFO:teuthology.orchestra.run.smithi066.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:26:06.341 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:26:06.341 INFO:teuthology.orchestra.run.smithi066.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:26:06.799 INFO:teuthology.orchestra.run.smithi066.stdout:Discarding blocks...Done. 2024-03-04T07:26:06.802 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi066.front.sepia.ceph.com -o noatime 2024-03-04T07:26:06.802 DEBUG:teuthology.orchestra.run.smithi066:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-5 2024-03-04T07:26:06.900 DEBUG:teuthology.orchestra.run.smithi066:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-5 2024-03-04T07:26:06.938 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/osd/ceph-6 2024-03-04T07:26:07.013 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-04T07:26:07.013 INFO:tasks.ceph:role: osd.6 2024-03-04T07:26:07.013 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:26:07.013 DEBUG:teuthology.orchestra.run.smithi066:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:26:07.094 INFO:teuthology.orchestra.run.smithi066.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:26:07.634 INFO:teuthology.orchestra.run.smithi066.stdout:Discarding blocks...Done. 2024-03-04T07:26:07.636 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi066.front.sepia.ceph.com -o noatime 2024-03-04T07:26:07.636 DEBUG:teuthology.orchestra.run.smithi066:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-6 2024-03-04T07:26:07.778 DEBUG:teuthology.orchestra.run.smithi066:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-6 2024-03-04T07:26:07.820 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/osd/ceph-7 2024-03-04T07:26:07.895 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2024-03-04T07:26:07.895 INFO:tasks.ceph:role: osd.7 2024-03-04T07:26:07.895 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:26:07.895 DEBUG:teuthology.orchestra.run.smithi066:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-03-04T07:26:07.975 INFO:teuthology.orchestra.run.smithi066.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-03-04T07:26:07.975 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 attr=2, projid32bit=1 2024-03-04T07:26:07.975 INFO:teuthology.orchestra.run.smithi066.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-03-04T07:26:07.975 INFO:teuthology.orchestra.run.smithi066.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-03-04T07:26:07.975 INFO:teuthology.orchestra.run.smithi066.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-03-04T07:26:07.975 INFO:teuthology.orchestra.run.smithi066.stdout: = sunit=0 swidth=0 blks 2024-03-04T07:26:07.975 INFO:teuthology.orchestra.run.smithi066.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-03-04T07:26:07.976 INFO:teuthology.orchestra.run.smithi066.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-03-04T07:26:07.976 INFO:teuthology.orchestra.run.smithi066.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-03-04T07:26:07.976 INFO:teuthology.orchestra.run.smithi066.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-03-04T07:26:08.470 INFO:teuthology.orchestra.run.smithi066.stdout:Discarding blocks...Done. 2024-03-04T07:26:08.472 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi066.front.sepia.ceph.com -o noatime 2024-03-04T07:26:08.473 DEBUG:teuthology.orchestra.run.smithi066:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-7 2024-03-04T07:26:08.563 DEBUG:teuthology.orchestra.run.smithi066:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-7 2024-03-04T07:26:08.603 DEBUG:teuthology.orchestra.run.smithi066:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:26:08.711 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:08.710+0000 7f875d149640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-4/keyring: can't open /var/lib/ceph/osd/ceph-4/keyring: (2) No such file or directory 2024-03-04T07:26:08.712 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:08.710+0000 7f875d149640 -1 created new key in keyring /var/lib/ceph/osd/ceph-4/keyring 2024-03-04T07:26:08.712 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:08.711+0000 7f875d149640 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2024-03-04T07:26:08.712 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:08.711+0000 7f875d149640 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2024-03-04T07:26:08.712 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:08.711+0000 7f875d149640 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2024-03-04T07:26:08.712 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:08.711+0000 7f875d149640 -1 bluestore(/var/lib/ceph/osd/ceph-4) _read_fsid unparsable uuid 2024-03-04T07:26:11.564 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-04T07:26:11.601 DEBUG:teuthology.orchestra.run.smithi066:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:26:11.698 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:11.698+0000 7f0c6e20c640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-5/keyring: can't open /var/lib/ceph/osd/ceph-5/keyring: (2) No such file or directory 2024-03-04T07:26:11.698 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:11.698+0000 7f0c6e20c640 -1 created new key in keyring /var/lib/ceph/osd/ceph-5/keyring 2024-03-04T07:26:11.699 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:11.698+0000 7f0c6e20c640 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2024-03-04T07:26:11.699 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:11.698+0000 7f0c6e20c640 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2024-03-04T07:26:11.699 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:11.698+0000 7f0c6e20c640 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2024-03-04T07:26:11.699 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:11.699+0000 7f0c6e20c640 -1 bluestore(/var/lib/ceph/osd/ceph-5) _read_fsid unparsable uuid 2024-03-04T07:26:14.549 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-04T07:26:14.585 DEBUG:teuthology.orchestra.run.smithi066:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 6 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:26:14.681 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:14.680+0000 7f24022f7640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-6/keyring: can't open /var/lib/ceph/osd/ceph-6/keyring: (2) No such file or directory 2024-03-04T07:26:14.681 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:14.680+0000 7f24022f7640 -1 created new key in keyring /var/lib/ceph/osd/ceph-6/keyring 2024-03-04T07:26:14.681 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:14.680+0000 7f24022f7640 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2024-03-04T07:26:14.681 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:14.680+0000 7f24022f7640 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2024-03-04T07:26:14.681 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:14.680+0000 7f24022f7640 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2024-03-04T07:26:14.681 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:14.681+0000 7f24022f7640 -1 bluestore(/var/lib/ceph/osd/ceph-6) _read_fsid unparsable uuid 2024-03-04T07:26:17.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:26:17.093 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:26:17.546 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-04T07:26:17.582 DEBUG:teuthology.orchestra.run.smithi066:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 7 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:26:17.679 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:17.678+0000 7f23cc213640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-7/keyring: can't open /var/lib/ceph/osd/ceph-7/keyring: (2) No such file or directory 2024-03-04T07:26:17.680 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:17.679+0000 7f23cc213640 -1 created new key in keyring /var/lib/ceph/osd/ceph-7/keyring 2024-03-04T07:26:17.680 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:17.679+0000 7f23cc213640 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2024-03-04T07:26:17.680 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:17.679+0000 7f23cc213640 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2024-03-04T07:26:17.680 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:17.679+0000 7f23cc213640 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2024-03-04T07:26:17.680 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:17.679+0000 7f23cc213640 -1 bluestore(/var/lib/ceph/osd/ceph-7) _read_fsid unparsable uuid 2024-03-04T07:26:20.528 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-04T07:26:20.564 INFO:tasks.ceph:Reading keys from all nodes... 2024-03-04T07:26:20.564 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:20.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2024-03-04T07:26:20.596 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:20.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/mds/ceph-a/keyring of=/dev/stdout 2024-03-04T07:26:20.667 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:20.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/mds/ceph-c/keyring of=/dev/stdout 2024-03-04T07:26:20.736 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:20.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2024-03-04T07:26:20.804 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:20.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2024-03-04T07:26:20.873 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:20.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2024-03-04T07:26:20.941 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:20.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2024-03-04T07:26:21.009 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.009 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2024-03-04T07:26:21.046 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.046 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/mds/ceph-b/keyring of=/dev/stdout 2024-03-04T07:26:21.121 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.121 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/mds/ceph-d/keyring of=/dev/stdout 2024-03-04T07:26:21.195 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.195 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-4/keyring of=/dev/stdout 2024-03-04T07:26:21.269 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.269 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-5/keyring of=/dev/stdout 2024-03-04T07:26:21.342 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.342 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-6/keyring of=/dev/stdout 2024-03-04T07:26:21.414 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.414 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-7/keyring of=/dev/stdout 2024-03-04T07:26:21.485 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:21.486 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.client.0.keyring of=/dev/stdout 2024-03-04T07:26:21.502 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:21.502 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.client.2.keyring of=/dev/stdout 2024-03-04T07:26:21.558 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.558 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/etc/ceph/ceph.client.1.keyring of=/dev/stdout 2024-03-04T07:26:21.574 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:21.574 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/etc/ceph/ceph.client.3.keyring of=/dev/stdout 2024-03-04T07:26:21.631 INFO:tasks.ceph:Adding keys to all mons... 2024-03-04T07:26:21.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo tee -a /etc/ceph/ceph.keyring 2024-03-04T07:26:21.633 DEBUG:teuthology.orchestra.run.smithi066:> sudo tee -a /etc/ceph/ceph.keyring 2024-03-04T07:26:21.674 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr.y] 2024-03-04T07:26:21.674 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing== 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ== 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ== 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.0] 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A== 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.1] 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA== 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.2] 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw== 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.3] 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A== 2024-03-04T07:26:21.675 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr.x] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q== 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA== 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ== 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.4] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg== 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.5] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg== 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.6] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA== 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout:[osd.7] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg== 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:26:21.676 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:26:21.677 INFO:teuthology.orchestra.run.smithi064.stdout:[client.2] 2024-03-04T07:26:21.677 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q== 2024-03-04T07:26:21.677 INFO:teuthology.orchestra.run.smithi064.stdout:[client.1] 2024-03-04T07:26:21.677 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q== 2024-03-04T07:26:21.677 INFO:teuthology.orchestra.run.smithi064.stdout:[client.3] 2024-03-04T07:26:21.677 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A== 2024-03-04T07:26:21.701 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr.y] 2024-03-04T07:26:21.701 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing== 2024-03-04T07:26:21.701 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:26:21.701 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ== 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ== 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.0] 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A== 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.1] 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA== 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.2] 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw== 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.3] 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A== 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr.x] 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q== 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:26:21.702 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA== 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ== 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.4] 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg== 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.5] 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg== 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.6] 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA== 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout:[osd.7] 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg== 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout:[client.0] 2024-03-04T07:26:21.703 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:26:21.704 INFO:teuthology.orchestra.run.smithi066.stdout:[client.2] 2024-03-04T07:26:21.704 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q== 2024-03-04T07:26:21.704 INFO:teuthology.orchestra.run.smithi066.stdout:[client.1] 2024-03-04T07:26:21.704 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q== 2024-03-04T07:26:21.704 INFO:teuthology.orchestra.run.smithi066.stdout:[client.3] 2024-03-04T07:26:21.704 INFO:teuthology.orchestra.run.smithi066.stdout: key = AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A== 2024-03-04T07:26:21.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-04T07:26:21.721 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-04T07:26:21.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:21.805 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.a --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:21.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.c --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:21.866 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.c --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:21.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:21.929 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:21.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:21.988 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.046 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.105 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-04T07:26:22.165 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-03-04T07:26:22.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.b --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:22.223 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.b --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:22.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.d --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:22.285 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mds.d --cap mon 'allow *' --cap mgr 'allow *' --cap osd 'allow *' --cap mds allow 2024-03-04T07:26:22.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.347 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.405 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.463 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.521 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-03-04T07:26:22.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.581 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.642 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.705 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.767 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-03-04T07:26:22.827 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-03-04T07:26:22.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2024-03-04T07:26:22.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-03-04T07:26:22.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2024-03-04T07:26:22.977 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2024-03-04T07:26:23.009 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-03-04T07:26:23.111 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2024-03-04T07:26:23.142 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2024-03-04T07:26:23.213 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-03-04T07:26:23.314 DEBUG:teuthology.orchestra.run.smithi066:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2024-03-04T07:26:23.345 DEBUG:teuthology.orchestra.run.smithi064:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:26:23.348 DEBUG:teuthology.orchestra.run.smithi066:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-03-04T07:26:23.401 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2024-03-04T07:26:23.402 INFO:tasks.ceph.mon.a:Restarting daemon 2024-03-04T07:26:23.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-03-04T07:26:23.405 INFO:tasks.ceph.mon.a:Started 2024-03-04T07:26:23.405 INFO:tasks.ceph.mon.b:Restarting daemon 2024-03-04T07:26:23.405 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-03-04T07:26:23.444 INFO:tasks.ceph.mon.b:Started 2024-03-04T07:26:23.445 INFO:tasks.ceph.mon.c:Restarting daemon 2024-03-04T07:26:23.445 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-03-04T07:26:23.446 INFO:tasks.ceph.mon.c:Started 2024-03-04T07:26:23.447 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2024-03-04T07:26:23.447 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-03-04T07:26:23.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-03-04T07:26:23.449 INFO:tasks.ceph.mgr.y:Started 2024-03-04T07:26:23.449 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-03-04T07:26:23.449 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-03-04T07:26:23.451 INFO:tasks.ceph.mgr.x:Started 2024-03-04T07:26:23.451 INFO:tasks.ceph:Setting crush tunables to default 2024-03-04T07:26:23.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --cluster ceph osd crush tunables default 2024-03-04T07:26:23.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 Processor -- start 2024-03-04T07:26:23.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 -- start start 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f830806bb40 con 0x7f83081621b0 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f83080d0430 con 0x7f83080b1b20 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.554+0000 7f830d05c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f830806bd90 con 0x7f8308162590 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f8306d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f8306d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54024/0 (socket says 172.21.15.64:54024) 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 learned_addr learned my addr 172.21.15.64:0/1835972853 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 15 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-04T07:26:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.555+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 15 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-04T07:26:23.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:23.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:23.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-04T07:26:23.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:23.756+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-04T07:26:24.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:24.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-04T07:26:24.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:24.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:24.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-04T07:26:24.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:24.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.157+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-04T07:26:24.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:24.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 13 2024-03-04T07:26:24.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f8306d9d640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 14 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f82fe59c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 15 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-04T07:26:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:24.958+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-04T07:26:26.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 msgr2=0x7f83080b15e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-04T07:26:26.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8308162590 0x7f83080b15e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:26.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 msgr2=0x7f83080d08c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-04T07:26:26.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.66:3300/0 conn(0x7f83080b1b20 0x7f83080d08c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:26.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 -- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 msgr2=0x7f83080b4290 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-04T07:26:26.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 --2- 172.21.15.64:0/1835972853 >> v2:172.21.15.64:3300/0 conn(0x7f83081621b0 0x7f83080b4290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:26.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 -- 172.21.15.64:0/1835972853 --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f830806c070 con 0x7f82f4003670 2024-03-04T07:26:26.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:26.555+0000 7f830559a640 1 -- 172.21.15.64:0/1835972853 --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f82f400c120 con 0x7f82f40073a0 2024-03-04T07:26:28.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.770+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3175818846 0 0) 0x7f830806c070 con 0x7f82f4003670 2024-03-04T07:26:28.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.770+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f82e8003520 con 0x7f82f4003670 2024-03-04T07:26:28.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2877411571 0 0) 0x7f82e8003520 con 0x7f82f4003670 2024-03-04T07:26:28.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 >> v1:172.21.15.66:6789/0 conn(0x7f82f40073a0 legacy=0x7f82f4009860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:28.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82f400d360 con 0x7f82f4003670 2024-03-04T07:26:28.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f82f800f040 con 0x7f82f4003670 2024-03-04T07:26:28.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 >> v1:172.21.15.64:6789/0 conn(0x7f82f4003670 legacy=0x7f82f4005ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:28.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 reap_dead start 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8003a30 0x7f82e8003e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e8004800 0x7f82e8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e80099b0 0x7f82e800bda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82e800c390 con 0x7f82e80099b0 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82e800c510 con 0x7f82e8004800 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f82e800c710 con 0x7f82e8003a30 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e80099b0 0x7f82e800bda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e8004800 0x7f82e8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:28.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.771+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8003a30 0x7f82e8003e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:28.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.772+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8003a30 msgr2=0x7f82e8003e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:28.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.772+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8003a30 0x7f82e8003e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.772+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e8004800 msgr2=0x7f82e8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:28.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.772+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e8004800 0x7f82e8004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.772+0000 7f830759e640 1 -- 172.21.15.64:0/1835972853 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82e8003520 con 0x7f82e80099b0 2024-03-04T07:26:28.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.773+0000 7f830759e640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e80099b0 0x7f82e800bda0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f83080727f0 tx=0x7f82e8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a2bb22c7cce6d847 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:28.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.773+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f830003a070 con 0x7f82e80099b0 2024-03-04T07:26:28.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.773+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8300035410 con 0x7f82e80099b0 2024-03-04T07:26:28.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.773+0000 7f830659c640 1 -- 172.21.15.64:0/1835972853 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8300003db0 con 0x7f82e80099b0 2024-03-04T07:26:28.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 -- 172.21.15.64:0/1835972853 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e80099b0 msgr2=0x7f82e800bda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:28.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e80099b0 0x7f82e800bda0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f83080727f0 tx=0x7f82e8005830 comp rx=0 tx=0).stop 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 -- 172.21.15.64:0/1835972853 shutdown_connections 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e80099b0 0x7f82e800bda0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e8004800 0x7f82e8004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 --2- 172.21.15.64:0/1835972853 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8003a30 0x7f82e8003e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 -- 172.21.15.64:0/1835972853 >> 172.21.15.64:0/1835972853 conn(0x7f83080b3770 msgr2=0x7f83080b3b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 -- 172.21.15.64:0/1835972853 shutdown_connections 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.774+0000 7f830d05c640 1 -- 172.21.15.64:0/1835972853 wait complete. 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.776+0000 7f830d05c640 1 Processor -- start 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.777+0000 7f830d05c640 1 -- start start 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.777+0000 7f830d05c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e8003a30 0x7f83080ae9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.777+0000 7f830d05c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8004800 0x7f83080b4860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:28.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.777+0000 7f830d05c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e80099b0 0x7f83080b4da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.777+0000 7f830d05c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8308162b00 con 0x7f82e8003a30 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.777+0000 7f830d05c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83080cfd70 con 0x7f82e80099b0 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.777+0000 7f830d05c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8308162c80 con 0x7f82e8004800 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e8003a30 0x7f83080ae9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e8003a30 0x7f83080ae9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54106/0 (socket says 172.21.15.64:54106) 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 -- 172.21.15.64:0/1858826224 learned_addr learned my addr 172.21.15.64:0/1858826224 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f82fe59c640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8004800 0x7f83080b4860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f830759e640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e80099b0 0x7f83080b4da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 -- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8004800 msgr2=0x7f83080b4860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8004800 0x7f83080b4860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 -- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e80099b0 msgr2=0x7f83080b4da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e80099b0 0x7f83080b4da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 -- 172.21.15.64:0/1858826224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83081f3f70 con 0x7f82e8003a30 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.778+0000 7f8306d9d640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e8003a30 0x7f83080ae9e0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f82f8016990 tx=0x7f82f8016e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.779+0000 7f82fffff640 1 -- 172.21.15.64:0/1858826224 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f82f801c070 con 0x7f82e8003a30 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.779+0000 7f82fffff640 1 -- 172.21.15.64:0/1858826224 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f82f8014c60 con 0x7f82e8003a30 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.779+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83080aefc0 con 0x7f82e8003a30 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.779+0000 7f82fffff640 1 -- 172.21.15.64:0/1858826224 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f82f800f040 con 0x7f82e8003a30 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.779+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83080af580 con 0x7f82e8003a30 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.779+0000 7f82fffff640 1 -- 172.21.15.64:0/1858826224 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f82f80216a0 con 0x7f82e8003a30 2024-03-04T07:26:28.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.779+0000 7f82fffff640 1 -- 172.21.15.64:0/1858826224 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f82f80189c0 con 0x7f82e8003a30 2024-03-04T07:26:28.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.780+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f830809fd60 con 0x7f82e8003a30 2024-03-04T07:26:28.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.783+0000 7f82fffff640 1 -- 172.21.15.64:0/1858826224 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f82f802f020 con 0x7f82e8003a30 2024-03-04T07:26:28.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.855+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7f83080cdd00 con 0x7f82e8003a30 2024-03-04T07:26:28.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.856+0000 7f82fffff640 1 -- 172.21.15.64:0/1858826224 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v2) v1 ==== 124+0+0 (secure 0 0 0) 0x7f82f8018450 con 0x7f82e8003a30 2024-03-04T07:26:28.856 INFO:teuthology.orchestra.run.smithi064.stderr:adjusted tunables profile to default 2024-03-04T07:26:28.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e8003a30 msgr2=0x7f83080ae9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:28.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e8003a30 0x7f83080ae9e0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f82f8016990 tx=0x7f82f8016e50 comp rx=0 tx=0).stop 2024-03-04T07:26:28.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 shutdown_connections 2024-03-04T07:26:28.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82e80099b0 0x7f83080b4da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f82e8004800 0x7f83080b4860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 --2- 172.21.15.64:0/1858826224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82e8003a30 0x7f83080ae9e0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:28.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 >> 172.21.15.64:0/1858826224 conn(0x7f83080b3770 msgr2=0x7f8308152830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:28.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 shutdown_connections 2024-03-04T07:26:28.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:28.857+0000 7f830d05c640 1 -- 172.21.15.64:0/1858826224 wait complete. 2024-03-04T07:26:28.868 INFO:tasks.ceph:check_enable_crimson: False 2024-03-04T07:26:28.869 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2024-03-04T07:26:28.869 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:28.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2024-03-04T07:26:28.896 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:28.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2024-03-04T07:26:28.964 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:28.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2024-03-04T07:26:29.031 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:26:29.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2024-03-04T07:26:29.098 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:29.098 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-4/fsid of=/dev/stdout 2024-03-04T07:26:29.130 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:29.130 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-5/fsid of=/dev/stdout 2024-03-04T07:26:29.201 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:29.201 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-6/fsid of=/dev/stdout 2024-03-04T07:26:29.263 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:29.263+0000 7f75fb151040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-04T07:26:29.272 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:26:29.272 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/var/lib/ceph/osd/ceph-7/fsid of=/dev/stdout 2024-03-04T07:26:29.289 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:29.290+0000 7f605583d040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-04T07:26:29.320 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:29.320+0000 7f75fb151040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-04T07:26:29.343 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new fb72a7b9-ce40-4a01-82e5-ae0017868138 0 2024-03-04T07:26:29.345 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:29.346+0000 7f605583d040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-04T07:26:29.489 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.489+0000 7facfd242640 1 Processor -- start 2024-03-04T07:26:29.489 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.489+0000 7facfd242640 1 -- start start 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facfd242640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7facf80aef10 0x7facf8066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facfd242640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7facf815e520 con 0x7facf8067240 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facfd242640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7facf815f2a0 con 0x7facf806e1c0 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facfd242640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7facf815e300 con 0x7facf80aef10 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf759e640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7facf806e1c0 0x7facf80b3cc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.66:57162/0 (socket says 172.21.15.66:57162) 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf759e640 1 -- 172.21.15.66:0/889445610 learned_addr learned my addr 172.21.15.66:0/889445610 (peer_addr_for_me v1:172.21.15.66:0/0) 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf6d9d640 1 --2- 172.21.15.66:0/889445610 >> v2:172.21.15.64:3300/0 conn(0x7facf80aef10 0x7facf8066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3965942152 0 0) 0x7facf815f2a0 con 0x7facf806e1c0 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7face4003520 con 0x7facf806e1c0 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 <== mon.1 v1:172.21.15.66:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1380293145 0 0) 0x7face4003520 con 0x7facf806e1c0 2024-03-04T07:26:29.493 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 >> v2:172.21.15.64:3300/0 conn(0x7facf80aef10 msgr2=0x7facf8066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 --2- 172.21.15.66:0/889445610 >> v2:172.21.15.64:3300/0 conn(0x7facf80aef10 0x7facf8066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 >> v1:172.21.15.64:6789/0 conn(0x7facf8067240 legacy=0x7facf806dab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 --> v1:172.21.15.66:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facf815e7c0 con 0x7facf806e1c0 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 <== mon.1 v1:172.21.15.66:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7facf00019f0 con 0x7facf806e1c0 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 >> v1:172.21.15.66:6789/0 conn(0x7facf806e1c0 legacy=0x7facf80b3cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7face40039b0 0x7face4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face4004800 0x7face4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7face40051c0 0x7face400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf815e300 con 0x7face40039b0 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7face400c160 con 0x7face40051c0 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.490+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7face400c360 con 0x7face4004800 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf6d9d640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7face40039b0 0x7face4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf759e640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7face40051c0 0x7face400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.494 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7faceffff640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face4004800 0x7face4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf759e640 1 -- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face4004800 msgr2=0x7face4004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf759e640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face4004800 0x7face4004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf759e640 1 -- 172.21.15.66:0/889445610 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7face40039b0 msgr2=0x7face4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf759e640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7face40039b0 0x7face4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf759e640 1 -- 172.21.15.66:0/889445610 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7face4003520 con 0x7face40051c0 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf6d9d640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7face40039b0 0x7face4003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.491+0000 7facf759e640 1 -- 172.21.15.66:0/889445610 reap_dead start 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facf759e640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7face40051c0 0x7face400b9c0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7facf000a860 tx=0x7face4005830 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c1e0a5beffc3fd92 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face8009690 con 0x7face40051c0 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7facf0034630 con 0x7face40051c0 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facf659c640 1 -- 172.21.15.66:0/889445610 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facf000a480 con 0x7face40051c0 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facfd242640 1 -- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7face40051c0 msgr2=0x7face400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facfd242640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7face40051c0 0x7face400b9c0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7facf000a860 tx=0x7face4005830 comp rx=0 tx=0).stop 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facfd242640 1 -- 172.21.15.66:0/889445610 shutdown_connections 2024-03-04T07:26:29.495 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facfd242640 1 --2- 172.21.15.66:0/889445610 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7face40051c0 0x7face400b9c0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.496 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facfd242640 1 -- 172.21.15.66:0/889445610 >> 172.21.15.66:0/889445610 conn(0x7facf80d09c0 msgr2=0x7facf80d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:29.496 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facfd242640 1 -- 172.21.15.66:0/889445610 shutdown_connections 2024-03-04T07:26:29.496 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.492+0000 7facfd242640 1 -- 172.21.15.66:0/889445610 wait complete. 2024-03-04T07:26:29.498 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 Processor -- start 2024-03-04T07:26:29.498 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 -- start start 2024-03-04T07:26:29.498 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face40051c0 0x7facf80cfd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf815c0d0 0x7facf8147040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf815c550 0x7facf8147580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf806e820 con 0x7facf815c550 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf815ed80 con 0x7facf815c0d0 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.494+0000 7facfd242640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facf80af2f0 con 0x7face40051c0 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf815c0d0 0x7facf8147040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf815c0d0 0x7facf8147040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:49060/0 (socket says 172.21.15.66:49060) 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 -- 172.21.15.66:0/582995114 learned_addr learned my addr 172.21.15.66:0/582995114 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7facf6d9d640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face40051c0 0x7facf80cfd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7facf759e640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf815c550 0x7facf8147580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 -- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face40051c0 msgr2=0x7facf80cfd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face40051c0 0x7facf80cfd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 -- 172.21.15.66:0/582995114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf815c550 msgr2=0x7facf8147580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.499 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf815c550 0x7facf8147580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 -- 172.21.15.66:0/582995114 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facf80cec70 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7faceffff640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf815c0d0 0x7facf8147040 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7face000f5f0 tx=0x7face000f9f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7facef7fe640 1 -- 172.21.15.66:0/582995114 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face00052e0 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7facef7fe640 1 -- 172.21.15.66:0/582995114 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7face0018070 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7facef7fe640 1 -- 172.21.15.66:0/582995114 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face0003810 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7facf80cef50 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.495+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7facf80cf510 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.496+0000 7facef7fe640 1 -- 172.21.15.66:0/582995114 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7face00039b0 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.496+0000 7facef7fe640 1 -- 172.21.15.66:0/582995114 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 725+0+0 (secure 0 0 0) 0x7face001dd60 con 0x7facf815c0d0 2024-03-04T07:26:29.500 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.496+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7facf8073380 con 0x7facf815c0d0 2024-03-04T07:26:29.501 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.498+0000 7facef7fe640 1 -- 172.21.15.66:0/582995114 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7face001d460 con 0x7facf815c0d0 2024-03-04T07:26:29.573 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.572+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd new", "uuid": "fb72a7b9-ce40-4a01-82e5-ae0017868138", "id": 0} v 0) v1 -- 0x7facf804f410 con 0x7facf815c0d0 2024-03-04T07:26:29.575 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.575+0000 7facef7fe640 1 -- 172.21.15.66:0/582995114 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "fb72a7b9-ce40-4a01-82e5-ae0017868138", "id": 0}]=0 v3) v1 ==== 112+0+2 (secure 0 0 0) 0x7face001d640 con 0x7facf815c0d0 2024-03-04T07:26:29.576 INFO:teuthology.orchestra.run.smithi066.stdout:0 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf815c0d0 msgr2=0x7facf8147040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf815c0d0 0x7facf8147040 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7face000f5f0 tx=0x7face000f9f0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 shutdown_connections 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf815c550 0x7facf8147580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf815c0d0 0x7facf8147040 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 --2- 172.21.15.66:0/582995114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7face40051c0 0x7facf80cfd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 >> 172.21.15.66:0/582995114 conn(0x7facf80d09c0 msgr2=0x7facf80ad8b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 shutdown_connections 2024-03-04T07:26:29.577 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.576+0000 7facfd242640 1 -- 172.21.15.66:0/582995114 wait complete. 2024-03-04T07:26:29.588 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new 181008cb-9136-4647-bf89-704c76688299 1 2024-03-04T07:26:29.693 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.693+0000 7f4bac8df640 1 Processor -- start 2024-03-04T07:26:29.694 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.693+0000 7f4bac8df640 1 -- start start 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.694+0000 7f4bac8df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba80cf720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.694+0000 7f4bac8df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ba80cf4f0 con 0x7f4ba815be60 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.694+0000 7f4bac8df640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ba80cef30 con 0x7f4ba80b4260 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.694+0000 7f4bac8df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba80cc660 con 0x7f4ba80d04f0 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.694+0000 7f4ba6d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba80cf720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.694+0000 7f4ba5d9b640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f4ba80b4260 0x7f4ba80cfde0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.66:57170/0 (socket says 172.21.15.66:57170) 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.694+0000 7f4ba5d9b640 1 -- 172.21.15.66:0/1305422982 learned_addr learned my addr 172.21.15.66:0/1305422982 (peer_addr_for_me v1:172.21.15.66:0/0) 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba6d9d640 1 -- 172.21.15.66:0/1305422982 >> v1:172.21.15.66:6789/0 conn(0x7f4ba80b4260 legacy=0x7f4ba80cfde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba559a640 1 -- 172.21.15.66:0/1305422982 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2871514794 0 0) 0x7f4ba80cef30 con 0x7f4ba80b4260 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba6d9d640 1 -- 172.21.15.66:0/1305422982 >> v1:172.21.15.64:6789/0 conn(0x7f4ba815be60 legacy=0x7f4ba80cd370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.695 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba6d9d640 1 -- 172.21.15.66:0/1305422982 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ba80cf210 con 0x7f4ba80d04f0 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba6d9d640 1 --2- 172.21.15.66:0/1305422982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba80cf720 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f4ba0009dc0 tx=0x7f4ba002bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=57ff3443878506df server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba559a640 1 -- 172.21.15.66:0/1305422982 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ba0039070 con 0x7f4ba80d04f0 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba559a640 1 -- 172.21.15.66:0/1305422982 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ba002ccd0 con 0x7f4ba80d04f0 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.695+0000 7f4ba559a640 1 -- 172.21.15.66:0/1305422982 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ba0034660 con 0x7f4ba80d04f0 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.696+0000 7f4bac8df640 1 -- 172.21.15.66:0/1305422982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 msgr2=0x7f4ba80cf720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.696+0000 7f4bac8df640 1 --2- 172.21.15.66:0/1305422982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba80cf720 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f4ba0009dc0 tx=0x7f4ba002bf20 comp rx=0 tx=0).stop 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.696+0000 7f4bac8df640 1 -- 172.21.15.66:0/1305422982 shutdown_connections 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.696+0000 7f4bac8df640 1 --2- 172.21.15.66:0/1305422982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba80cf720 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.696+0000 7f4bac8df640 1 -- 172.21.15.66:0/1305422982 >> 172.21.15.66:0/1305422982 conn(0x7f4ba80b3660 msgr2=0x7f4ba80b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.696+0000 7f4bac8df640 1 -- 172.21.15.66:0/1305422982 shutdown_connections 2024-03-04T07:26:29.697 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.696+0000 7f4bac8df640 1 -- 172.21.15.66:0/1305422982 wait complete. 2024-03-04T07:26:29.700 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.697+0000 7f4bac8df640 1 Processor -- start 2024-03-04T07:26:29.700 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.697+0000 7f4bac8df640 1 -- start start 2024-03-04T07:26:29.700 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4bac8df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4260 0x7f4ba80afb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.700 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4bac8df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba8148a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.700 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4bac8df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba815be60 0x7f4ba8148f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4bac8df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba80ccb70 con 0x7f4ba80b4260 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4bac8df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba80cda80 con 0x7f4ba815be60 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4bac8df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba80cce50 con 0x7f4ba80d04f0 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4ba5d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba8148a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4ba6d9d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba815be60 0x7f4ba8148f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4ba5d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba8148a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49756/0 (socket says 172.21.15.66:49756) 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4ba5d9b640 1 -- 172.21.15.66:0/2869975640 learned_addr learned my addr 172.21.15.66:0/2869975640 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.698+0000 7f4ba659c640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4260 0x7f4ba80afb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4ba6d9d640 1 -- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 msgr2=0x7f4ba8148a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4ba6d9d640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba8148a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4ba6d9d640 1 -- 172.21.15.66:0/2869975640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4260 msgr2=0x7f4ba80afb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4ba6d9d640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4260 0x7f4ba80afb90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.701 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4ba6d9d640 1 -- 172.21.15.66:0/2869975640 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ba80c8e70 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4ba5d9b640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba8148a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4ba6d9d640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba815be60 0x7f4ba8148f70 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4ba0033000 tx=0x7f4ba0004a40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4b96ffd640 1 -- 172.21.15.66:0/2869975640 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ba0039050 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4b96ffd640 1 -- 172.21.15.66:0/2869975640 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ba003a040 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4b96ffd640 1 -- 172.21.15.66:0/2869975640 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ba003e610 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ba80c9150 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ba80c9710 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4b96ffd640 1 -- 172.21.15.66:0/2869975640 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f4ba0047020 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.699+0000 7f4b96ffd640 1 -- 172.21.15.66:0/2869975640 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 840+0+0 (secure 0 0 0) 0x7f4ba0040aa0 con 0x7f4ba815be60 2024-03-04T07:26:29.702 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.700+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ba809fc50 con 0x7f4ba815be60 2024-03-04T07:26:29.703 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.702+0000 7f4b96ffd640 1 -- 172.21.15.66:0/2869975640 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f4ba0040cf0 con 0x7f4ba815be60 2024-03-04T07:26:29.777 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.777+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd new", "uuid": "181008cb-9136-4647-bf89-704c76688299", "id": 1} v 0) v1 -- 0x7f4ba804f410 con 0x7f4ba815be60 2024-03-04T07:26:29.782 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.782+0000 7f4b96ffd640 1 -- 172.21.15.66:0/2869975640 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "181008cb-9136-4647-bf89-704c76688299", "id": 1}]=0 v4) v1 ==== 112+0+2 (secure 0 0 0) 0x7f4ba003e7b0 con 0x7f4ba815be60 2024-03-04T07:26:29.782 INFO:teuthology.orchestra.run.smithi066.stdout:1 2024-03-04T07:26:29.783 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba815be60 msgr2=0x7f4ba8148f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.783 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba815be60 0x7f4ba8148f70 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4ba0033000 tx=0x7f4ba0004a40 comp rx=0 tx=0).stop 2024-03-04T07:26:29.783 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 shutdown_connections 2024-03-04T07:26:29.783 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba815be60 0x7f4ba8148f70 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.784 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80d04f0 0x7f4ba8148a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.784 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 --2- 172.21.15.66:0/2869975640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4260 0x7f4ba80afb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.784 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 >> 172.21.15.66:0/2869975640 conn(0x7f4ba80b3660 msgr2=0x7f4ba80cf6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:29.784 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 shutdown_connections 2024-03-04T07:26:29.784 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.783+0000 7f4bac8df640 1 -- 172.21.15.66:0/2869975640 wait complete. 2024-03-04T07:26:29.794 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new a2ada673-135a-4368-9204-33df19ddaf38 2 2024-03-04T07:26:29.890 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:29.890+0000 7f75fb151040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-04T07:26:29.900 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.900+0000 7f1f999e9640 1 Processor -- start 2024-03-04T07:26:29.900 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.900+0000 7f1f999e9640 1 -- start start 2024-03-04T07:26:29.901 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.900+0000 7f1f999e9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1f9415dd90 0x7f1f940ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.901 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.900+0000 7f1f999e9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1f940d3740 con 0x7f1f940b6d10 2024-03-04T07:26:29.901 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.900+0000 7f1f999e9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1f940d3120 con 0x7f1f940d29d0 2024-03-04T07:26:29.901 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.900+0000 7f1f999e9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1f940d3400 con 0x7f1f9415dd90 2024-03-04T07:26:29.901 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f927fc640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f1f940d29d0 0x7f1f940d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.66:57174/0 (socket says 172.21.15.66:57174) 2024-03-04T07:26:29.901 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f927fc640 1 -- 172.21.15.66:0/2662632235 learned_addr learned my addr 172.21.15.66:0/2662632235 (peer_addr_for_me v1:172.21.15.66:0/0) 2024-03-04T07:26:29.902 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f92ffd640 1 --2- 172.21.15.66:0/2662632235 >> v2:172.21.15.64:3300/0 conn(0x7f1f9415dd90 0x7f1f940ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.902 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f91ffb640 1 -- 172.21.15.66:0/2662632235 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3531631106 0 0) 0x7f1f940d3120 con 0x7f1f940d29d0 2024-03-04T07:26:29.902 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f91ffb640 1 -- 172.21.15.66:0/2662632235 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1f7c003520 con 0x7f1f940d29d0 2024-03-04T07:26:29.902 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f92ffd640 1 -- 172.21.15.66:0/2662632235 >> v1:172.21.15.66:6789/0 conn(0x7f1f940d29d0 legacy=0x7f1f940d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.902 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f92ffd640 1 -- 172.21.15.66:0/2662632235 >> v1:172.21.15.64:6789/0 conn(0x7f1f940b6d10 legacy=0x7f1f940d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.902 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.901+0000 7f1f92ffd640 1 -- 172.21.15.66:0/2662632235 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f940d39e0 con 0x7f1f9415dd90 2024-03-04T07:26:29.903 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.902+0000 7f1f92ffd640 1 --2- 172.21.15.66:0/2662632235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9415dd90 0x7f1f940ceea0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f1f840097a0 tx=0x7f1f8402b9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2b9c602120655c92 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:29.903 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.902+0000 7f1f91ffb640 1 -- 172.21.15.66:0/2662632235 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8402f810 con 0x7f1f9415dd90 2024-03-04T07:26:29.903 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.902+0000 7f1f91ffb640 1 -- 172.21.15.66:0/2662632235 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f8402f9b0 con 0x7f1f9415dd90 2024-03-04T07:26:29.903 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.902+0000 7f1f91ffb640 1 -- 172.21.15.66:0/2662632235 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8402fd60 con 0x7f1f9415dd90 2024-03-04T07:26:29.903 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.903+0000 7f1f999e9640 1 -- 172.21.15.66:0/2662632235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9415dd90 msgr2=0x7f1f940ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.903+0000 7f1f999e9640 1 --2- 172.21.15.66:0/2662632235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9415dd90 0x7f1f940ceea0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f1f840097a0 tx=0x7f1f8402b9f0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.903+0000 7f1f999e9640 1 -- 172.21.15.66:0/2662632235 shutdown_connections 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.903+0000 7f1f999e9640 1 --2- 172.21.15.66:0/2662632235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9415dd90 0x7f1f940ceea0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.903+0000 7f1f999e9640 1 -- 172.21.15.66:0/2662632235 >> 172.21.15.66:0/2662632235 conn(0x7f1f94071960 msgr2=0x7f1f94071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.903+0000 7f1f999e9640 1 -- 172.21.15.66:0/2662632235 shutdown_connections 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.903+0000 7f1f999e9640 1 -- 172.21.15.66:0/2662632235 wait complete. 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.904+0000 7f1f999e9640 1 Processor -- start 2024-03-04T07:26:29.904 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.904+0000 7f1f999e9640 1 -- start start 2024-03-04T07:26:29.906 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f999e9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f940b6d10 0x7f1f941f74b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.906 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f999e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f940d29d0 0x7f1f941f4950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.906 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f999e9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9415dd90 0x7f1f9414b020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:29.906 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f999e9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f940d2df0 con 0x7f1f940b6d10 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f999e9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f9415e170 con 0x7f1f9415dd90 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f999e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f940d48f0 con 0x7f1f940d29d0 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f927fc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f940d29d0 0x7f1f941f4950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f927fc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f940d29d0 0x7f1f941f4950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49766/0 (socket says 172.21.15.66:49766) 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f927fc640 1 -- 172.21.15.66:0/3226374889 learned_addr learned my addr 172.21.15.66:0/3226374889 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f937fe640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9415dd90 0x7f1f9414b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f92ffd640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f940b6d10 0x7f1f941f74b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f937fe640 1 -- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f940d29d0 msgr2=0x7f1f941f4950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f937fe640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f940d29d0 0x7f1f941f4950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f937fe640 1 -- 172.21.15.66:0/3226374889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f940b6d10 msgr2=0x7f1f941f74b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f937fe640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f940b6d10 0x7f1f941f74b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f937fe640 1 -- 172.21.15.66:0/3226374889 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f941f7be0 con 0x7f1f9415dd90 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f927fc640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f940d29d0 0x7f1f941f4950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.905+0000 7f1f937fe640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9415dd90 0x7f1f9414b020 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f1f8c00c6c0 tx=0x7f1f8c00cb80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.906+0000 7f1f7bfff640 1 -- 172.21.15.66:0/3226374889 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8c003b70 con 0x7f1f9415dd90 2024-03-04T07:26:29.907 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.906+0000 7f1f7bfff640 1 -- 172.21.15.66:0/3226374889 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f8c003d10 con 0x7f1f9415dd90 2024-03-04T07:26:29.908 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.906+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1f941f7ec0 con 0x7f1f9415dd90 2024-03-04T07:26:29.908 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.906+0000 7f1f7bfff640 1 -- 172.21.15.66:0/3226374889 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8c0052f0 con 0x7f1f9415dd90 2024-03-04T07:26:29.908 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.906+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1f940c7a00 con 0x7f1f9415dd90 2024-03-04T07:26:29.908 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.906+0000 7f1f7bfff640 1 -- 172.21.15.66:0/3226374889 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f1f8c005490 con 0x7f1f9415dd90 2024-03-04T07:26:29.908 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.906+0000 7f1f7bfff640 1 -- 172.21.15.66:0/3226374889 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 955+0+0 (secure 0 0 0) 0x7f1f8c012890 con 0x7f1f9415dd90 2024-03-04T07:26:29.908 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:29.908+0000 7f605583d040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-04T07:26:29.909 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.907+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1f94072a30 con 0x7f1f9415dd90 2024-03-04T07:26:29.909 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.909+0000 7f1f7bfff640 1 -- 172.21.15.66:0/3226374889 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f1f8c012470 con 0x7f1f9415dd90 2024-03-04T07:26:29.949 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:29.949+0000 7f75fb151040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-04T07:26:29.964 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:29.965+0000 7f605583d040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-04T07:26:29.982 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.982+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd new", "uuid": "a2ada673-135a-4368-9204-33df19ddaf38", "id": 2} v 0) v1 -- 0x7f1f940c7bd0 con 0x7f1f9415dd90 2024-03-04T07:26:29.987 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.987+0000 7f1f7bfff640 1 -- 172.21.15.66:0/3226374889 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "a2ada673-135a-4368-9204-33df19ddaf38", "id": 2}]=0 v5) v1 ==== 112+0+2 (secure 0 0 0) 0x7f1f8c012b70 con 0x7f1f9415dd90 2024-03-04T07:26:29.987 INFO:teuthology.orchestra.run.smithi066.stdout:2 2024-03-04T07:26:29.988 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9415dd90 msgr2=0x7f1f9414b020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9415dd90 0x7f1f9414b020 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f1f8c00c6c0 tx=0x7f1f8c00cb80 comp rx=0 tx=0).stop 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 shutdown_connections 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9415dd90 0x7f1f9414b020 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f940d29d0 0x7f1f941f4950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 --2- 172.21.15.66:0/3226374889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f940b6d10 0x7f1f941f74b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 >> 172.21.15.66:0/3226374889 conn(0x7f1f94071960 msgr2=0x7f1f9415c1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 shutdown_connections 2024-03-04T07:26:29.989 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:29.988+0000 7f1f999e9640 1 -- 172.21.15.66:0/3226374889 wait complete. 2024-03-04T07:26:29.997 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:29.997+0000 7f75fb151040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-04T07:26:29.999 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new 4cdda1a3-e9e6-4169-849b-92384852886d 3 2024-03-04T07:26:30.016 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.012+0000 7f605583d040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-04T07:26:30.055 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.055+0000 7f75fb151040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-04T07:26:30.068 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.068+0000 7f605583d040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-04T07:26:30.101 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.100+0000 7f75fb151040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-04T07:26:30.105 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.105+0000 7f047a7d7640 1 Processor -- start 2024-03-04T07:26:30.105 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.105+0000 7f047a7d7640 1 -- start start 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f047a7d7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f04741600a0 0x7f04740cd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f047a7d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0474072540 0x7f04740cff90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f047a7d7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04740729f0 con 0x7f04741457c0 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f047a7d7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0474145ba0 con 0x7f04741600a0 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f047a7d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04740cf1d0 con 0x7f0474072540 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f0478d74640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0474072540 0x7f04740cff90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f04737fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f04741600a0 0x7f04740cd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f0478d74640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0474072540 0x7f04740cff90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49772/0 (socket says 172.21.15.66:49772) 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f0478d74640 1 -- 172.21.15.66:0/1075408909 learned_addr learned my addr 172.21.15.66:0/1075408909 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f04737fe640 1 -- 172.21.15.66:0/1075408909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0474072540 msgr2=0x7f04740cff90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f04737fe640 1 --2- 172.21.15.66:0/1075408909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0474072540 0x7f04740cff90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.107 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f04737fe640 1 -- 172.21.15.66:0/1075408909 >> v1:172.21.15.64:6789/0 conn(0x7f04741457c0 legacy=0x7f047415f990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.108 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f04737fe640 1 -- 172.21.15.66:0/1075408909 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04740cf4b0 con 0x7f04741600a0 2024-03-04T07:26:30.108 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f0472ffd640 1 -- 172.21.15.66:0/1075408909 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2505118727 0 0) 0x7f0464001110 con 0x7f04741457c0 2024-03-04T07:26:30.108 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.106+0000 7f0478d74640 1 --2- 172.21.15.66:0/1075408909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0474072540 0x7f04740cff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:30.112 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.107+0000 7f04737fe640 1 --2- 172.21.15.66:0/1075408909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04741600a0 0x7f04740cd3a0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f04680097d0 tx=0x7f046802b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc9ceffff430c25f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.112 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.107+0000 7f0472ffd640 1 -- 172.21.15.66:0/1075408909 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0468035020 con 0x7f04741600a0 2024-03-04T07:26:30.112 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.107+0000 7f0472ffd640 1 -- 172.21.15.66:0/1075408909 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0468031800 con 0x7f04741600a0 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.107+0000 7f0472ffd640 1 -- 172.21.15.66:0/1075408909 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0468031bd0 con 0x7f04741600a0 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 -- 172.21.15.66:0/1075408909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04741600a0 msgr2=0x7f04740cd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 --2- 172.21.15.66:0/1075408909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04741600a0 0x7f04740cd3a0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f04680097d0 tx=0x7f046802b8a0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 -- 172.21.15.66:0/1075408909 shutdown_connections 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 --2- 172.21.15.66:0/1075408909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0474072540 0x7f04740cff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 --2- 172.21.15.66:0/1075408909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04741600a0 0x7f04740cd3a0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 -- 172.21.15.66:0/1075408909 >> 172.21.15.66:0/1075408909 conn(0x7f04740b2790 msgr2=0x7f04740b2ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 -- 172.21.15.66:0/1075408909 shutdown_connections 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.108+0000 7f047a7d7640 1 -- 172.21.15.66:0/1075408909 wait complete. 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.109+0000 7f047a7d7640 1 Processor -- start 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.109+0000 7f047a7d7640 1 -- start start 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f047a7d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0474072540 0x7f0474155450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f047a7d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04741457c0 0x7f047415c490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.113 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f047a7d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04741600a0 0x7f04741f55d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f047a7d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04740cdaa0 con 0x7f0474072540 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f047a7d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04740cfa70 con 0x7f04741600a0 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f047a7d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04740cdc20 con 0x7f04741457c0 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f04737fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04741457c0 0x7f047415c490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f0478d74640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04741600a0 0x7f04741f55d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f0473fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0474072540 0x7f0474155450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f04737fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04741457c0 0x7f047415c490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49776/0 (socket says 172.21.15.66:49776) 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f04737fe640 1 -- 172.21.15.66:0/3002885566 learned_addr learned my addr 172.21.15.66:0/3002885566 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f0473fff640 1 -- 172.21.15.66:0/3002885566 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04741457c0 msgr2=0x7f047415c490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f0473fff640 1 --2- 172.21.15.66:0/3002885566 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04741457c0 0x7f047415c490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f0473fff640 1 -- 172.21.15.66:0/3002885566 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04741600a0 msgr2=0x7f04741f55d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f0473fff640 1 --2- 172.21.15.66:0/3002885566 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04741600a0 0x7f04741f55d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.110+0000 7f0473fff640 1 -- 172.21.15.66:0/3002885566 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0474155990 con 0x7f0474072540 2024-03-04T07:26:30.114 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f0473fff640 1 --2- 172.21.15.66:0/3002885566 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0474072540 0x7f0474155450 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f046400c750 tx=0x7f046400cc10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.115 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f0470ff9640 1 -- 172.21.15.66:0/3002885566 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0464017070 con 0x7f0474072540 2024-03-04T07:26:30.115 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f0470ff9640 1 -- 172.21.15.66:0/3002885566 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0464005070 con 0x7f0474072540 2024-03-04T07:26:30.115 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f0470ff9640 1 -- 172.21.15.66:0/3002885566 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f046400a640 con 0x7f0474072540 2024-03-04T07:26:30.115 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f047415c9d0 con 0x7f0474072540 2024-03-04T07:26:30.116 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f047413ad30 con 0x7f0474072540 2024-03-04T07:26:30.116 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f0470ff9640 1 -- 172.21.15.66:0/3002885566 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f046401a070 con 0x7f0474072540 2024-03-04T07:26:30.116 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.111+0000 7f0470ff9640 1 -- 172.21.15.66:0/3002885566 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1070+0+0 (secure 0 0 0) 0x7f0464003da0 con 0x7f0474072540 2024-03-04T07:26:30.116 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.112+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f047415bce0 con 0x7f0474072540 2024-03-04T07:26:30.117 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.113+0000 7f605583d040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-04T07:26:30.117 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.114+0000 7f0470ff9640 1 -- 172.21.15.66:0/3002885566 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f0464021020 con 0x7f0474072540 2024-03-04T07:26:30.189 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.188+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd new", "uuid": "4cdda1a3-e9e6-4169-849b-92384852886d", "id": 3} v 0) v1 -- 0x7f047404f410 con 0x7f0474072540 2024-03-04T07:26:30.192 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.191+0000 7f0470ff9640 1 -- 172.21.15.66:0/3002885566 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "4cdda1a3-e9e6-4169-849b-92384852886d", "id": 3}]=0 v6) v1 ==== 112+0+2 (secure 0 0 0) 0x7f04740729f0 con 0x7f0474072540 2024-03-04T07:26:30.192 INFO:teuthology.orchestra.run.smithi066.stdout:3 2024-03-04T07:26:30.193 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.192+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0474072540 msgr2=0x7f0474155450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.192+0000 7f047a7d7640 1 --2- 172.21.15.66:0/3002885566 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0474072540 0x7f0474155450 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f046400c750 tx=0x7f046400cc10 comp rx=0 tx=0).stop 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.192+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 shutdown_connections 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.192+0000 7f047a7d7640 1 --2- 172.21.15.66:0/3002885566 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04741600a0 0x7f04741f55d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.192+0000 7f047a7d7640 1 --2- 172.21.15.66:0/3002885566 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04741457c0 0x7f047415c490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.192+0000 7f047a7d7640 1 --2- 172.21.15.66:0/3002885566 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0474072540 0x7f0474155450 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.192+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 >> 172.21.15.66:0/3002885566 conn(0x7f04740b2790 msgr2=0x7f0474142db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.193+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 shutdown_connections 2024-03-04T07:26:30.194 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.193+0000 7f047a7d7640 1 -- 172.21.15.66:0/3002885566 wait complete. 2024-03-04T07:26:30.204 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new f982f669-8e67-4b99-8b43-7b070a797819 4 2024-03-04T07:26:30.226 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.225+0000 7f75fb151040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-04T07:26:30.234 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.234+0000 7f605583d040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-04T07:26:30.301 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.301+0000 7f75fb151040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-04T07:26:30.307 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.307+0000 7f605583d040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-04T07:26:30.310 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.310+0000 7f30c6bbc640 1 Processor -- start 2024-03-04T07:26:30.310 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.310+0000 7f30c6bbc640 1 -- start start 2024-03-04T07:26:30.311 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.310+0000 7f30c6bbc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f30c015e160 0x7f30c00730d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.311 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.310+0000 7f30c6bbc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f30c00b5210 0x7f30c00adbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.311 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.310+0000 7f30c6bbc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f30c00b1a00 con 0x7f30c0161b10 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.310+0000 7f30c6bbc640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f30c00b1ce0 con 0x7f30c00b5210 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.310+0000 7f30c6bbc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f30c00b3300 con 0x7f30c015e160 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c4958640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f30c015e160 0x7f30c00730d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c4958640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f30c015e160 0x7f30c00730d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:49110/0 (socket says 172.21.15.66:49110) 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c4958640 1 -- 172.21.15.66:0/1925549147 learned_addr learned my addr 172.21.15.66:0/1925549147 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c5159640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f30c0161b10 0x7f30c0161f60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.66:57178/0 (socket says 172.21.15.66:57178) 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30bffff640 1 --2- 172.21.15.66:0/1925549147 >> v2:172.21.15.64:3300/0 conn(0x7f30c00b5210 0x7f30c00adbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c4958640 1 -- 172.21.15.66:0/1925549147 >> v2:172.21.15.64:3300/0 conn(0x7f30c00b5210 msgr2=0x7f30c00adbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c4958640 1 --2- 172.21.15.66:0/1925549147 >> v2:172.21.15.64:3300/0 conn(0x7f30c00b5210 0x7f30c00adbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c4958640 1 -- 172.21.15.66:0/1925549147 >> v1:172.21.15.66:6789/0 conn(0x7f30c0161b10 legacy=0x7f30c0161f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.312 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.311+0000 7f30c4958640 1 -- 172.21.15.66:0/1925549147 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30c00b35e0 con 0x7f30c015e160 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30c4958640 1 --2- 172.21.15.66:0/1925549147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c00730d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f30c00b1ea0 tx=0x7f30b402b910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f488481f6c50c492 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30bf7fe640 1 -- 172.21.15.66:0/1925549147 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30b4035020 con 0x7f30c015e160 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30bf7fe640 1 -- 172.21.15.66:0/1925549147 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f30b4031870 con 0x7f30c015e160 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30bf7fe640 1 -- 172.21.15.66:0/1925549147 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30b4031c40 con 0x7f30c015e160 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30c6bbc640 1 -- 172.21.15.66:0/1925549147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 msgr2=0x7f30c00730d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30c6bbc640 1 --2- 172.21.15.66:0/1925549147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c00730d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f30c00b1ea0 tx=0x7f30b402b910 comp rx=0 tx=0).stop 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30c6bbc640 1 -- 172.21.15.66:0/1925549147 shutdown_connections 2024-03-04T07:26:30.314 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30c6bbc640 1 --2- 172.21.15.66:0/1925549147 >> v2:172.21.15.64:3300/0 conn(0x7f30c00b5210 0x7f30c00adbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.315 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30c6bbc640 1 --2- 172.21.15.66:0/1925549147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c00730d0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.315 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.312+0000 7f30c6bbc640 1 -- 172.21.15.66:0/1925549147 >> 172.21.15.66:0/1925549147 conn(0x7f30c00725b0 msgr2=0x7f30c00729c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.315 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.313+0000 7f30c6bbc640 1 -- 172.21.15.66:0/1925549147 shutdown_connections 2024-03-04T07:26:30.315 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.313+0000 7f30c6bbc640 1 -- 172.21.15.66:0/1925549147 wait complete. 2024-03-04T07:26:30.315 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.314+0000 7f30c6bbc640 1 Processor -- start 2024-03-04T07:26:30.315 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.314+0000 7f30c6bbc640 1 -- start start 2024-03-04T07:26:30.317 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30c00b5210 0x7f30c01f7700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.317 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c014cdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30c0161b10 0x7f30c00cb720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30c01627f0 con 0x7f30c00b5210 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30c0162670 con 0x7f30c015e160 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f30c0162970 con 0x7f30c0161b10 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c014cdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c014cdf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:49118/0 (socket says 172.21.15.66:49118) 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 -- 172.21.15.66:0/2908746068 learned_addr learned my addr 172.21.15.66:0/2908746068 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c5159640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30c0161b10 0x7f30c00cb720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c4958640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30c00b5210 0x7f30c01f7700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 -- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30c0161b10 msgr2=0x7f30c00cb720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30c0161b10 0x7f30c00cb720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 -- 172.21.15.66:0/2908746068 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30c00b5210 msgr2=0x7f30c01f7700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30c00b5210 0x7f30c01f7700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.318 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 -- 172.21.15.66:0/2908746068 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30c015a360 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c5159640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30c0161b10 0x7f30c00cb720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c4958640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30c00b5210 0x7f30c01f7700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bffff640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c014cdf0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f30ac00e9b0 tx=0x7f30ac00ee70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bd7fa640 1 -- 172.21.15.66:0/2908746068 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30ac017070 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bd7fa640 1 -- 172.21.15.66:0/2908746068 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f30ac00c9a0 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f30c01f7f50 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30bd7fa640 1 -- 172.21.15.66:0/2908746068 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30ac00cd90 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.315+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f30c01f8510 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.316+0000 7f30bd7fa640 1 -- 172.21.15.66:0/2908746068 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f30ac01a070 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.316+0000 7f30bd7fa640 1 -- 172.21.15.66:0/2908746068 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1185+0+0 (secure 0 0 0) 0x7f30ac006660 con 0x7f30c015e160 2024-03-04T07:26:30.319 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.316+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f30c009e530 con 0x7f30c015e160 2024-03-04T07:26:30.320 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.318+0000 7f30bd7fa640 1 -- 172.21.15.66:0/2908746068 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f30ac006020 con 0x7f30c015e160 2024-03-04T07:26:30.351 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.351+0000 7f75fb151040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-04T07:26:30.355 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.356+0000 7f605583d040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-04T07:26:30.392 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.392+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd new", "uuid": "f982f669-8e67-4b99-8b43-7b070a797819", "id": 4} v 0) v1 -- 0x7f30c01612c0 con 0x7f30c015e160 2024-03-04T07:26:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.397+0000 7f30bd7fa640 1 -- 172.21.15.66:0/2908746068 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "f982f669-8e67-4b99-8b43-7b070a797819", "id": 4}]=0 v7) v1 ==== 112+0+2 (secure 0 0 0) 0x7f30ac005990 con 0x7f30c015e160 2024-03-04T07:26:30.397 INFO:teuthology.orchestra.run.smithi066.stdout:4 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 msgr2=0x7f30c014cdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c014cdf0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f30ac00e9b0 tx=0x7f30ac00ee70 comp rx=0 tx=0).stop 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 shutdown_connections 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30c0161b10 0x7f30c00cb720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30c015e160 0x7f30c014cdf0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 --2- 172.21.15.66:0/2908746068 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30c00b5210 0x7f30c01f7700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 >> 172.21.15.66:0/2908746068 conn(0x7f30c00725b0 msgr2=0x7f30c006e7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 shutdown_connections 2024-03-04T07:26:30.399 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.398+0000 7f30c6bbc640 1 -- 172.21.15.66:0/2908746068 wait complete. 2024-03-04T07:26:30.407 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.407+0000 7f75fb151040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-04T07:26:30.409 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new e4ebc912-75e2-40c7-9eb2-760859196dff 5 2024-03-04T07:26:30.410 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.411+0000 7f605583d040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-04T07:26:30.457 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.457+0000 7f75fb151040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-04T07:26:30.460 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.460+0000 7f605583d040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-04T07:26:30.555 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.555+0000 7f160c1ac640 1 Processor -- start 2024-03-04T07:26:30.556 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.555+0000 7f160c1ac640 1 -- start start 2024-03-04T07:26:30.557 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f160c1ac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1604143e60 0x7f16040b3bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.557 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f160c1ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f16040ce040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.557 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f160c1ac640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1604072b50 con 0x7f16040726e0 2024-03-04T07:26:30.557 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f160c1ac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f160406ecf0 con 0x7f1604143e60 2024-03-04T07:26:30.557 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f160c1ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f160406efd0 con 0x7f16040720b0 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f1609f48640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1604143e60 0x7f16040b3bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f160a749640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f16040ce040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f1609f48640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1604143e60 0x7f16040b3bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:49126/0 (socket says 172.21.15.66:49126) 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.556+0000 7f1609f48640 1 -- 172.21.15.66:0/2792157214 learned_addr learned my addr 172.21.15.66:0/2792157214 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.557+0000 7f1609f48640 1 -- 172.21.15.66:0/2792157214 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 msgr2=0x7f16040ce040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.557+0000 7f1609f48640 1 --2- 172.21.15.66:0/2792157214 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f16040ce040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.557+0000 7f1609f48640 1 -- 172.21.15.66:0/2792157214 >> v1:172.21.15.64:6789/0 conn(0x7f16040726e0 legacy=0x7f16040719a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.558 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.557+0000 7f1609f48640 1 -- 172.21.15.66:0/2792157214 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f160406f2b0 con 0x7f1604143e60 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.557+0000 7f1609f48640 1 --2- 172.21.15.66:0/2792157214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 0x7f16040b3bb0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f15f400bb10 tx=0x7f15f402e150 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ea1976e63337d2bf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f1608f46640 1 -- 172.21.15.66:0/2792157214 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f403b070 con 0x7f1604143e60 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f1608f46640 1 -- 172.21.15.66:0/2792157214 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f15f4036440 con 0x7f1604143e60 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f1608f46640 1 -- 172.21.15.66:0/2792157214 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f4036810 con 0x7f1604143e60 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 -- 172.21.15.66:0/2792157214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 msgr2=0x7f16040b3bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 --2- 172.21.15.66:0/2792157214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 0x7f16040b3bb0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f15f400bb10 tx=0x7f15f402e150 comp rx=0 tx=0).stop 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 -- 172.21.15.66:0/2792157214 shutdown_connections 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 --2- 172.21.15.66:0/2792157214 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f16040ce040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 --2- 172.21.15.66:0/2792157214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 0x7f16040b3bb0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 -- 172.21.15.66:0/2792157214 >> 172.21.15.66:0/2792157214 conn(0x7f16040b2790 msgr2=0x7f16040b2ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 -- 172.21.15.66:0/2792157214 shutdown_connections 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.558+0000 7f160c1ac640 1 -- 172.21.15.66:0/2792157214 wait complete. 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.559+0000 7f160c1ac640 1 Processor -- start 2024-03-04T07:26:30.560 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160c1ac640 1 -- start start 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160c1ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f1604138710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160c1ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16040726e0 0x7f160413e9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160c1ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 0x7f1604142a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160c1ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1604144240 con 0x7f16040726e0 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160c1ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f160406ea10 con 0x7f1604143e60 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160c1ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16041443c0 con 0x7f16040720b0 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f1609f48640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f1604138710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f1609f48640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f1604138710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49798/0 (socket says 172.21.15.66:49798) 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f1609f48640 1 -- 172.21.15.66:0/2082149708 learned_addr learned my addr 172.21.15.66:0/2082149708 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f160a749640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 0x7f1604142a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.560+0000 7f1609747640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16040726e0 0x7f160413e9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.563 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f1609f48640 1 -- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 msgr2=0x7f1604142a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f1609f48640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 0x7f1604142a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f1609f48640 1 -- 172.21.15.66:0/2082149708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16040726e0 msgr2=0x7f160413e9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f1609f48640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16040726e0 0x7f160413e9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f1609f48640 1 -- 172.21.15.66:0/2082149708 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f160413f340 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f1609f48640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f1604138710 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f15f4000c00 tx=0x7f15f40046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f15fa7fc640 1 -- 172.21.15.66:0/2082149708 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f403b040 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f15fa7fc640 1 -- 172.21.15.66:0/2082149708 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f15f40344c0 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f15fa7fc640 1 -- 172.21.15.66:0/2082149708 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f40348b0 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f160413f620 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f160413fbe0 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f15fa7fc640 1 -- 172.21.15.66:0/2082149708 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f15f4048030 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.561+0000 7f15fa7fc640 1 -- 172.21.15.66:0/2082149708 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(7..7 src has 1..7) v4 ==== 1300+0+0 (secure 0 0 0) 0x7f15f403fbb0 con 0x7f16040720b0 2024-03-04T07:26:30.564 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.562+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16041395d0 con 0x7f16040720b0 2024-03-04T07:26:30.565 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.564+0000 7f15fa7fc640 1 -- 172.21.15.66:0/2082149708 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f15f403fe50 con 0x7f16040720b0 2024-03-04T07:26:30.638 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.638+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "osd new", "uuid": "e4ebc912-75e2-40c7-9eb2-760859196dff", "id": 5} v 0) v1 -- 0x7f160404f410 con 0x7f16040720b0 2024-03-04T07:26:30.643 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.643+0000 7f15fa7fc640 1 -- 172.21.15.66:0/2082149708 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "e4ebc912-75e2-40c7-9eb2-760859196dff", "id": 5}]=0 v8) v1 ==== 112+0+2 (secure 0 0 0) 0x7f15f403f430 con 0x7f16040720b0 2024-03-04T07:26:30.643 INFO:teuthology.orchestra.run.smithi066.stdout:5 2024-03-04T07:26:30.644 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 msgr2=0x7f1604138710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.644 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f1604138710 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f15f4000c00 tx=0x7f15f40046c0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.645 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 shutdown_connections 2024-03-04T07:26:30.645 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1604143e60 0x7f1604142a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.645 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16040726e0 0x7f160413e9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.645 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 --2- 172.21.15.66:0/2082149708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16040720b0 0x7f1604138710 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.645 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 >> 172.21.15.66:0/2082149708 conn(0x7f16040b2790 msgr2=0x7f16040b0700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.645 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 shutdown_connections 2024-03-04T07:26:30.645 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.644+0000 7f160c1ac640 1 -- 172.21.15.66:0/2082149708 wait complete. 2024-03-04T07:26:30.654 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new 79429dec-fad1-4f75-8847-f4faebaef17d 6 2024-03-04T07:26:30.803 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 Processor -- start 2024-03-04T07:26:30.803 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 -- start start 2024-03-04T07:26:30.804 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f09a01457c0 0x7f09a00cd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f09a0072540 0x7f09a00cff90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a015f990 0x7f09a015fe20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f09a0145ba0 con 0x7f09a01457c0 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f09a00729b0 con 0x7f09a0072540 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.803+0000 7f09a6193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09a00cf1d0 con 0x7f09a015f990 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f09a0072540 0x7f09a00cff90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a015f990 0x7f09a015fe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f09a0072540 0x7f09a00cff90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:49140/0 (socket says 172.21.15.66:49140) 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 -- 172.21.15.66:0/1762254868 learned_addr learned my addr 172.21.15.66:0/1762254868 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099f7fe640 1 --2- 172.21.15.66:0/1762254868 >> v2:172.21.15.64:3300/0 conn(0x7f09a01457c0 0x7f09a00cd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 -- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a015f990 msgr2=0x7f09a015fe20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 --2- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a015f990 0x7f09a015fe20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 -- 172.21.15.66:0/1762254868 >> v2:172.21.15.64:3300/0 conn(0x7f09a01457c0 msgr2=0x7f09a00cd3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 --2- 172.21.15.66:0/1762254868 >> v2:172.21.15.64:3300/0 conn(0x7f09a01457c0 0x7f09a00cd3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.805 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099effd640 1 -- 172.21.15.66:0/1762254868 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09a00cf4b0 con 0x7f09a0072540 2024-03-04T07:26:30.806 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.804+0000 7f099ffff640 1 --2- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a015f990 0x7f09a015fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.805+0000 7f099effd640 1 --2- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a0072540 0x7f09a00cff90 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f0994009870 tx=0x7f099402b8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=10a752236f663b7f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.805+0000 7f099e7fc640 1 -- 172.21.15.66:0/1762254868 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0994035020 con 0x7f09a0072540 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.805+0000 7f099e7fc640 1 -- 172.21.15.66:0/1762254868 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0994031640 con 0x7f09a0072540 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.805+0000 7f099e7fc640 1 -- 172.21.15.66:0/1762254868 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f099402d040 con 0x7f09a0072540 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.805+0000 7f09a6193640 1 -- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a0072540 msgr2=0x7f09a00cff90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.805+0000 7f09a6193640 1 --2- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a0072540 0x7f09a00cff90 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f0994009870 tx=0x7f099402b8f0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.806+0000 7f09a6193640 1 -- 172.21.15.66:0/1762254868 shutdown_connections 2024-03-04T07:26:30.807 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.806+0000 7f09a6193640 1 --2- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a015f990 0x7f09a015fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.808 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.806+0000 7f09a6193640 1 --2- 172.21.15.66:0/1762254868 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a0072540 0x7f09a00cff90 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.808 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.806+0000 7f09a6193640 1 --2- 172.21.15.66:0/1762254868 >> v2:172.21.15.64:3300/0 conn(0x7f09a01457c0 0x7f09a00cd3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.808 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.806+0000 7f09a6193640 1 -- 172.21.15.66:0/1762254868 >> 172.21.15.66:0/1762254868 conn(0x7f09a00b2790 msgr2=0x7f09a00b2ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.808 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.806+0000 7f09a6193640 1 -- 172.21.15.66:0/1762254868 shutdown_connections 2024-03-04T07:26:30.808 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.806+0000 7f09a6193640 1 -- 172.21.15.66:0/1762254868 wait complete. 2024-03-04T07:26:30.808 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.807+0000 7f09a6193640 1 Processor -- start 2024-03-04T07:26:30.808 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.807+0000 7f09a6193640 1 -- start start 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f09a6193640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09a0072540 0x7f09a013e790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f09a6193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a01457c0 0x7f09a015e5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f09a6193640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a015f990 0x7f09a015eb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f09a6193640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09a00cdaa0 con 0x7f09a0072540 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f09a6193640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09a00cfa70 con 0x7f09a015f990 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f09a6193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09a00cdc20 con 0x7f09a01457c0 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099effd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a01457c0 0x7f09a015e5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099ffff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a015f990 0x7f09a015eb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099effd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a01457c0 0x7f09a015e5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49820/0 (socket says 172.21.15.66:49820) 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09a0072540 0x7f09a013e790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099effd640 1 -- 172.21.15.66:0/124746873 learned_addr learned my addr 172.21.15.66:0/124746873 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:30.810 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099ffff640 1 -- 172.21.15.66:0/124746873 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a01457c0 msgr2=0x7f09a015e5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099ffff640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a01457c0 0x7f09a015e5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099ffff640 1 -- 172.21.15.66:0/124746873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09a0072540 msgr2=0x7f09a013e790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099ffff640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09a0072540 0x7f09a013e790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099ffff640 1 -- 172.21.15.66:0/124746873 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09a00b3de0 con 0x7f09a015f990 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099f7fe640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09a0072540 0x7f09a013e790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f099ffff640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a015f990 0x7f09a015eb30 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f099800efc0 tx=0x7f099800c490 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.808+0000 7f0987fff640 1 -- 172.21.15.66:0/124746873 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0998018030 con 0x7f09a015f990 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.809+0000 7f0987fff640 1 -- 172.21.15.66:0/124746873 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f099800cd30 con 0x7f09a015f990 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.809+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f09a00b40c0 con 0x7f09a015f990 2024-03-04T07:26:30.811 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.809+0000 7f0987fff640 1 -- 172.21.15.66:0/124746873 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0998003bb0 con 0x7f09a015f990 2024-03-04T07:26:30.812 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.809+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f09a00b4680 con 0x7f09a015f990 2024-03-04T07:26:30.812 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.809+0000 7f0987fff640 1 -- 172.21.15.66:0/124746873 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f099801b070 con 0x7f09a015f990 2024-03-04T07:26:30.812 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.809+0000 7f0987fff640 1 -- 172.21.15.66:0/124746873 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(8..8 src has 1..8) v4 ==== 1415+0+0 (secure 0 0 0) 0x7f099801da00 con 0x7f09a015f990 2024-03-04T07:26:30.812 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.810+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f09a004f410 con 0x7f09a015f990 2024-03-04T07:26:30.812 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.812+0000 7f0987fff640 1 -- 172.21.15.66:0/124746873 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f0998010040 con 0x7f09a015f990 2024-03-04T07:26:30.858 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:30.859+0000 7f605583d040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-04T07:26:30.867 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:30.867+0000 7f75fb151040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-04T07:26:30.886 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.885+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd new", "uuid": "79429dec-fad1-4f75-8847-f4faebaef17d", "id": 6} v 0) v1 -- 0x7f09a0072bd0 con 0x7f09a015f990 2024-03-04T07:26:30.890 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.890+0000 7f0987fff640 1 -- 172.21.15.66:0/124746873 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "79429dec-fad1-4f75-8847-f4faebaef17d", "id": 6}]=0 v9) v1 ==== 112+0+2 (secure 0 0 0) 0x7f099801dd70 con 0x7f09a015f990 2024-03-04T07:26:30.890 INFO:teuthology.orchestra.run.smithi066.stdout:6 2024-03-04T07:26:30.891 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a015f990 msgr2=0x7f09a015eb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:30.891 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a015f990 0x7f09a015eb30 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f099800efc0 tx=0x7f099800c490 comp rx=0 tx=0).stop 2024-03-04T07:26:30.891 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 shutdown_connections 2024-03-04T07:26:30.892 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09a015f990 0x7f09a015eb30 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.892 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09a01457c0 0x7f09a015e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.892 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 --2- 172.21.15.66:0/124746873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09a0072540 0x7f09a013e790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:30.892 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 >> 172.21.15.66:0/124746873 conn(0x7f09a00b2790 msgr2=0x7f09a013ff50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:30.892 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 shutdown_connections 2024-03-04T07:26:30.892 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:30.891+0000 7f09a6193640 1 -- 172.21.15.66:0/124746873 wait complete. 2024-03-04T07:26:30.901 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --cluster ceph osd new 59d05be1-af84-47ee-8132-20c66e6aa6be 7 2024-03-04T07:26:31.001 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.001+0000 7f605583d040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-04T07:26:31.008 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.008+0000 7f833b002640 1 Processor -- start 2024-03-04T07:26:31.008 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.008+0000 7f833b002640 1 -- start start 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833b002640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f833406b430 0x7f833406b810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833b002640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f83340b2f60 0x7f833406bd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833b002640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f833415dec0 0x7f833415e350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833b002640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f83340cc5a0 con 0x7f83340b2f60 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833b002640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f83340d11c0 con 0x7f833415dec0 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833b002640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f833415e890 con 0x7f833406b430 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f8338d9e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f833406b430 0x7f833406b810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833959f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f833415dec0 0x7f833415e350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f8338d9e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f833406b430 0x7f833406b810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49828/0 (socket says 172.21.15.66:49828) 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833959f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f833415dec0 0x7f833415e350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:49150/0 (socket says 172.21.15.66:49150) 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f8338d9e640 1 -- 172.21.15.66:0/1310526970 learned_addr learned my addr 172.21.15.66:0/1310526970 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f8333fff640 1 --2- 172.21.15.66:0/1310526970 >> v2:172.21.15.64:3300/0 conn(0x7f83340b2f60 0x7f833406bd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.010 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833959f640 1 -- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f833406b430 msgr2=0x7f833406b810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:31.011 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833959f640 1 --2- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f833406b430 0x7f833406b810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.011 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833959f640 1 -- 172.21.15.66:0/1310526970 >> v2:172.21.15.64:3300/0 conn(0x7f83340b2f60 msgr2=0x7f833406bd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:31.011 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833959f640 1 --2- 172.21.15.66:0/1310526970 >> v2:172.21.15.64:3300/0 conn(0x7f83340b2f60 0x7f833406bd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.011 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f833959f640 1 -- 172.21.15.66:0/1310526970 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83340d0920 con 0x7f833415dec0 2024-03-04T07:26:31.011 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.009+0000 7f8338d9e640 1 --2- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f833406b430 0x7f833406b810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:31.012 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.010+0000 7f833959f640 1 --2- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f833415e350 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f83340cc120 tx=0x7f832c02beb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=23b621cd05f25430 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:31.012 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.010+0000 7f83337fe640 1 -- 172.21.15.66:0/1310526970 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f832c039070 con 0x7f833415dec0 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.010+0000 7f83337fe640 1 -- 172.21.15.66:0/1310526970 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f832c02cc60 con 0x7f833415dec0 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.010+0000 7f83337fe640 1 -- 172.21.15.66:0/1310526970 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f832c034660 con 0x7f833415dec0 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 -- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 msgr2=0x7f833415e350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 --2- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f833415e350 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f83340cc120 tx=0x7f832c02beb0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 -- 172.21.15.66:0/1310526970 shutdown_connections 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 --2- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f833415e350 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 --2- 172.21.15.66:0/1310526970 >> v2:172.21.15.64:3300/0 conn(0x7f83340b2f60 0x7f833406bd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 --2- 172.21.15.66:0/1310526970 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f833406b430 0x7f833406b810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 -- 172.21.15.66:0/1310526970 >> 172.21.15.66:0/1310526970 conn(0x7f83340b36d0 msgr2=0x7f83340b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 -- 172.21.15.66:0/1310526970 shutdown_connections 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.011+0000 7f833b002640 1 -- 172.21.15.66:0/1310526970 wait complete. 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.012+0000 7f833b002640 1 Processor -- start 2024-03-04T07:26:31.013 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.012+0000 7f833b002640 1 -- start start 2024-03-04T07:26:31.014 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f75fb151040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-04T07:26:31.016 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833b002640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f833406b430 0x7f8334153080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.016 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833b002640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f83340b2f60 0x7f8334161590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833b002640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f83340cade0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833b002640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83340cc230 con 0x7f833406b430 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833b002640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83340d0ee0 con 0x7f833415dec0 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833b002640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f83340ce3c0 con 0x7f83340b2f60 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8333fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f83340b2f60 0x7f8334161590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833959f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f83340cade0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833959f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f83340cade0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:49158/0 (socket says 172.21.15.66:49158) 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833959f640 1 -- 172.21.15.66:0/1409877399 learned_addr learned my addr 172.21.15.66:0/1409877399 (peer_addr_for_me v2:172.21.15.66:0/0) 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8333fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f83340b2f60 0x7f8334161590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.66:49840/0 (socket says 172.21.15.66:49840) 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8338d9e640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f833406b430 0x7f8334153080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8338d9e640 1 -- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f83340b2f60 msgr2=0x7f8334161590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8338d9e640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f83340b2f60 0x7f8334161590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.017 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8338d9e640 1 -- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 msgr2=0x7f83340cade0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8338d9e640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f83340cade0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8338d9e640 1 -- 172.21.15.66:0/1409877399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83340b6900 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f833959f640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f83340cade0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.013+0000 7f8333fff640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f83340b2f60 0x7f8334161590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f8338d9e640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f833406b430 0x7f8334153080 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f832800e890 tx=0x7f832800ed50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f83317fa640 1 -- 172.21.15.66:0/1409877399 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8328017070 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f83317fa640 1 -- 172.21.15.66:0/1409877399 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f832800c940 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83341573c0 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f83317fa640 1 -- 172.21.15.66:0/1409877399 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f832800ccf0 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8334157980 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f83317fa640 1 -- 172.21.15.66:0/1409877399 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f832801a070 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.014+0000 7f83317fa640 1 -- 172.21.15.66:0/1409877399 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(9..9 src has 1..9) v4 ==== 1530+0+0 (secure 0 0 0) 0x7f8328013ad0 con 0x7f833406b430 2024-03-04T07:26:31.018 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.015+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f833406d000 con 0x7f833406b430 2024-03-04T07:26:31.019 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.017+0000 7f83317fa640 1 -- 172.21.15.66:0/1409877399 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f833406d000 con 0x7f833406b430 2024-03-04T07:26:31.091 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.091+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd new", "uuid": "59d05be1-af84-47ee-8132-20c66e6aa6be", "id": 7} v 0) v1 -- 0x7f833415d970 con 0x7f833406b430 2024-03-04T07:26:31.096 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.096+0000 7f83317fa640 1 -- 172.21.15.66:0/1409877399 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "59d05be1-af84-47ee-8132-20c66e6aa6be", "id": 7}]=0 v10) v1 ==== 112+0+2 (secure 0 0 0) 0x7f833415d970 con 0x7f833406b430 2024-03-04T07:26:31.096 INFO:teuthology.orchestra.run.smithi066.stdout:7 2024-03-04T07:26:31.097 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f833406b430 msgr2=0x7f8334153080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.097 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f833406b430 0x7f8334153080 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f832800e890 tx=0x7f832800ed50 comp rx=0 tx=0).stop 2024-03-04T07:26:31.097 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 shutdown_connections 2024-03-04T07:26:31.097 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f833415dec0 0x7f83340cade0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.097 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f83340b2f60 0x7f8334161590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.098 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 --2- 172.21.15.66:0/1409877399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f833406b430 0x7f8334153080 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.098 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 >> 172.21.15.66:0/1409877399 conn(0x7f83340b36d0 msgr2=0x7f833406f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:31.098 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 shutdown_connections 2024-03-04T07:26:31.098 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:26:31.097+0000 7f833b002640 1 -- 172.21.15.66:0/1409877399 wait complete. 2024-03-04T07:26:31.108 INFO:tasks.ceph.osd.0:Restarting daemon 2024-03-04T07:26:31.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-03-04T07:26:31.110 INFO:tasks.ceph.osd.0:Started 2024-03-04T07:26:31.110 INFO:tasks.ceph.osd.1:Restarting daemon 2024-03-04T07:26:31.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-03-04T07:26:31.112 INFO:tasks.ceph.osd.1:Started 2024-03-04T07:26:31.112 INFO:tasks.ceph.osd.2:Restarting daemon 2024-03-04T07:26:31.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-03-04T07:26:31.114 INFO:tasks.ceph.osd.2:Started 2024-03-04T07:26:31.114 INFO:tasks.ceph.osd.3:Restarting daemon 2024-03-04T07:26:31.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-03-04T07:26:31.116 INFO:tasks.ceph.osd.3:Started 2024-03-04T07:26:31.117 INFO:tasks.ceph.osd.4:Restarting daemon 2024-03-04T07:26:31.117 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 4 2024-03-04T07:26:31.119 INFO:tasks.ceph.osd.4:Started 2024-03-04T07:26:31.119 INFO:tasks.ceph.osd.5:Restarting daemon 2024-03-04T07:26:31.119 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 5 2024-03-04T07:26:31.121 INFO:tasks.ceph.osd.5:Started 2024-03-04T07:26:31.121 INFO:tasks.ceph.osd.6:Restarting daemon 2024-03-04T07:26:31.121 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 6 2024-03-04T07:26:31.123 INFO:tasks.ceph.osd.6:Started 2024-03-04T07:26:31.123 INFO:tasks.ceph.osd.7:Restarting daemon 2024-03-04T07:26:31.123 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 7 2024-03-04T07:26:31.125 INFO:tasks.ceph.osd.7:Started 2024-03-04T07:26:31.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:31.130 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.130+0000 7f75fb151040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-04T07:26:31.135 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.136+0000 7f605583d040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-04T07:26:31.210 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.211+0000 7f605583d040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-04T07:26:31.211 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.211+0000 7f75fb151040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-04T07:26:31.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.251+0000 7f2cd2955640 1 Processor -- start 2024-03-04T07:26:31.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.251+0000 7f2cd2955640 1 -- start start 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2cd2955640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2ccc15df00 0x7f2ccc0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2cd2955640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ccc0b95a0 0x7f2ccc0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2cd2955640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2ccc0cea30 con 0x7f2ccc0b9a30 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2cd2955640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2ccc0d2af0 con 0x7f2ccc0b95a0 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2cd2955640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2ccc0cf3c0 con 0x7f2ccc15df00 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2cd0ef2640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2ccc0b9a30 0x7f2ccc0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:37666/0 (socket says 172.21.15.64:37666) 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2cd0ef2640 1 -- 172.21.15.64:0/3608545810 learned_addr learned my addr 172.21.15.64:0/3608545810 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccb7fe640 1 --2- 172.21.15.64:0/3608545810 >> v2:172.21.15.64:3300/0 conn(0x7f2ccc0b95a0 0x7f2ccc0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccbfff640 1 --2- 172.21.15.64:0/3608545810 >> v2:172.21.15.66:3300/0 conn(0x7f2ccc15df00 0x7f2ccc0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccb7fe640 1 -- 172.21.15.64:0/3608545810 >> v2:172.21.15.66:3300/0 conn(0x7f2ccc15df00 msgr2=0x7f2ccc0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccb7fe640 1 --2- 172.21.15.64:0/3608545810 >> v2:172.21.15.66:3300/0 conn(0x7f2ccc15df00 0x7f2ccc0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccb7fe640 1 -- 172.21.15.64:0/3608545810 >> v1:172.21.15.64:6789/0 conn(0x7f2ccc0b9a30 legacy=0x7f2ccc0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccaffd640 1 -- 172.21.15.64:0/3608545810 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 488729161 0 0) 0x7f2ccc0cea30 con 0x7f2ccc0b9a30 2024-03-04T07:26:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccb7fe640 1 -- 172.21.15.64:0/3608545810 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ccc0cf0e0 con 0x7f2ccc0b95a0 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.252+0000 7f2ccbfff640 1 --2- 172.21.15.64:0/3608545810 >> v2:172.21.15.66:3300/0 conn(0x7f2ccc15df00 0x7f2ccc0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.253+0000 7f2ccb7fe640 1 --2- 172.21.15.64:0/3608545810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc0b5700 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f2ccc0d11b0 tx=0x7f2cc002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf950075c051a7e1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.253+0000 7f2ccaffd640 1 -- 172.21.15.64:0/3608545810 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cc0035020 con 0x7f2ccc0b95a0 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.253+0000 7f2ccaffd640 1 -- 172.21.15.64:0/3608545810 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2cc0031830 con 0x7f2ccc0b95a0 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.253+0000 7f2ccaffd640 1 -- 172.21.15.64:0/3608545810 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cc0031c00 con 0x7f2ccc0b95a0 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 -- 172.21.15.64:0/3608545810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 msgr2=0x7f2ccc0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 --2- 172.21.15.64:0/3608545810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc0b5700 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f2ccc0d11b0 tx=0x7f2cc002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 -- 172.21.15.64:0/3608545810 shutdown_connections 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 --2- 172.21.15.64:0/3608545810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc0b5700 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 --2- 172.21.15.64:0/3608545810 >> v2:172.21.15.66:3300/0 conn(0x7f2ccc15df00 0x7f2ccc0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 -- 172.21.15.64:0/3608545810 >> 172.21.15.64:0/3608545810 conn(0x7f2ccc071960 msgr2=0x7f2ccc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 -- 172.21.15.64:0/3608545810 shutdown_connections 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.254+0000 7f2cd2955640 1 -- 172.21.15.64:0/3608545810 wait complete. 2024-03-04T07:26:31.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.255+0000 7f2cd2955640 1 Processor -- start 2024-03-04T07:26:31.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.255+0000 7f2cd2955640 1 -- start start 2024-03-04T07:26:31.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2cd2955640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc15e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2cd2955640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ccc15df00 0x7f2ccc1f73c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2cd2955640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ccc1f7900 0x7f2ccc0b3de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2cd2955640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ccc0cec70 con 0x7f2ccc0b95a0 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2cd2955640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ccc0ce750 con 0x7f2ccc15df00 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2cd2955640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ccc0cedf0 con 0x7f2ccc1f7900 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc15e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc15e2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54164/0 (socket says 172.21.15.64:54164) 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccbfff640 1 -- 172.21.15.64:0/2906239635 learned_addr learned my addr 172.21.15.64:0/2906239635 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccb7fe640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ccc15df00 0x7f2ccc1f73c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2cd0ef2640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ccc1f7900 0x7f2ccc0b3de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccb7fe640 1 -- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ccc1f7900 msgr2=0x7f2ccc0b3de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccb7fe640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ccc1f7900 0x7f2ccc0b3de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccb7fe640 1 -- 172.21.15.64:0/2906239635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 msgr2=0x7f2ccc15e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccb7fe640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc15e2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccb7fe640 1 -- 172.21.15.64:0/2906239635 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ccc0c8d50 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.256+0000 7f2ccb7fe640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ccc15df00 0x7f2ccc1f73c0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f2cc0033000 tx=0x7f2cc00046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.257+0000 7f2cc8ff9640 1 -- 172.21.15.64:0/2906239635 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cc0041070 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.257+0000 7f2cc8ff9640 1 -- 172.21.15.64:0/2906239635 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2cc003b920 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.257+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ccc0c9030 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.257+0000 7f2cc8ff9640 1 -- 172.21.15.64:0/2906239635 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cc0035040 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.257+0000 7f2cc8ff9640 1 -- 172.21.15.64:0/2906239635 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f2cc004c060 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.257+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ccc0c95f0 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.258+0000 7f2cc8ff9640 1 -- 172.21.15.64:0/2906239635 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(10..10 src has 1..10) v4 ==== 1645+0+0 (secure 0 0 0) 0x7f2cc003bd10 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.258+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ccc067ff0 con 0x7f2ccc15df00 2024-03-04T07:26:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.260+0000 7f2cc8ff9640 1 -- 172.21.15.64:0/2906239635 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f2cc0031800 con 0x7f2ccc15df00 2024-03-04T07:26:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.322+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2ccc0d26e0 con 0x7f2ccc15df00 2024-03-04T07:26:31.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.323+0000 7f2cc8ff9640 1 -- 172.21.15.64:0/2906239635 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v10) v1 ==== 74+0+7058 (secure 0 0 0) 0x7f2cc003a5d0 con 0x7f2ccc15df00 2024-03-04T07:26:31.323 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:31.323 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":10,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:31.093625+0000","last_up_change":"0.000000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ccc15df00 msgr2=0x7f2ccc1f73c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ccc15df00 0x7f2ccc1f73c0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f2cc0033000 tx=0x7f2cc00046c0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 shutdown_connections 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ccc1f7900 0x7f2ccc0b3de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ccc15df00 0x7f2ccc1f73c0 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 --2- 172.21.15.64:0/2906239635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ccc0b95a0 0x7f2ccc15e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 >> 172.21.15.64:0/2906239635 conn(0x7f2ccc071960 msgr2=0x7f2ccc0d3f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 shutdown_connections 2024-03-04T07:26:31.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:31.324+0000 7f2cd2955640 1 -- 172.21.15.64:0/2906239635 wait complete. 2024-03-04T07:26:31.331 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.331+0000 7f605583d040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-04T07:26:31.331 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.330+0000 7f75fb151040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-04T07:26:31.333 INFO:tasks.ceph.ceph_manager.ceph:[] 2024-03-04T07:26:31.334 INFO:tasks.ceph:Waiting for OSDs to come up 2024-03-04T07:26:31.431 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.430+0000 7f75fb151040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-04T07:26:31.431 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.431+0000 7f605583d040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-04T07:26:31.592 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.592+0000 7f605583d040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-04T07:26:31.592 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.592+0000 7f75fb151040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-04T07:26:31.636 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.637+0000 7f605583d040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-04T07:26:31.637 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.636+0000 7f75fb151040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-04T07:26:31.722 INFO:tasks.ceph.osd.0.smithi064.stderr:2024-03-04T07:26:31.722+0000 7f3f68a48640 -1 Falling back to public interface 2024-03-04T07:26:31.722 INFO:tasks.ceph.osd.1.smithi064.stderr:2024-03-04T07:26:31.722+0000 7ff957a06640 -1 Falling back to public interface 2024-03-04T07:26:31.722 INFO:tasks.ceph.osd.0.smithi064.stderr:2024-03-04T07:26:31.722+0000 7f3f679f7640 -1 PosixStack listen unable to listen on v1:0.0.0.0:6801/0: (98) Address already in use 2024-03-04T07:26:31.723 INFO:tasks.ceph.osd.0.smithi064.stderr:2024-03-04T07:26:31.723+0000 7f3f679f7640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6804/0: (98) Address already in use 2024-03-04T07:26:31.724 INFO:tasks.ceph.osd.5.smithi066.stderr:2024-03-04T07:26:31.724+0000 7f1c384f7640 -1 Falling back to public interface 2024-03-04T07:26:31.725 INFO:tasks.ceph.osd.7.smithi066.stderr:2024-03-04T07:26:31.724+0000 7f9d711e1640 -1 Falling back to public interface 2024-03-04T07:26:31.725 INFO:tasks.ceph.osd.5.smithi066.stderr:2024-03-04T07:26:31.725+0000 7f1c374a6640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6802/0: (98) Address already in use 2024-03-04T07:26:31.729 INFO:tasks.ceph.osd.6.smithi066.stderr:2024-03-04T07:26:31.729+0000 7f1d87a1d640 -1 Falling back to public interface 2024-03-04T07:26:31.730 INFO:tasks.ceph.osd.3.smithi064.stderr:2024-03-04T07:26:31.730+0000 7fe30f81b640 -1 Falling back to public interface 2024-03-04T07:26:31.737 INFO:tasks.ceph.osd.2.smithi064.stderr:2024-03-04T07:26:31.738+0000 7fb8ad60d640 -1 Falling back to public interface 2024-03-04T07:26:31.748 INFO:tasks.ceph.osd.4.smithi066.stderr:2024-03-04T07:26:31.748+0000 7fa0e9017640 -1 Falling back to public interface 2024-03-04T07:26:31.800 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:31.801+0000 7f605583d040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-04T07:26:31.805 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:31.805+0000 7f75fb151040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-04T07:26:32.139 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:32.230 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.230+0000 7f239f6c2640 1 Processor -- start 2024-03-04T07:26:32.230 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.231+0000 7f239f6c2640 1 -- start start 2024-03-04T07:26:32.233 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.231+0000 7f239f6c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398143eb0 0x7f23980cdef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.231+0000 7f239f6c2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2398144560 0x7f23980b3ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.231+0000 7f239f6c2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f239806db30 con 0x7f2398071a30 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.231+0000 7f239f6c2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2398072b50 con 0x7f2398144560 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.231+0000 7f239f6c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f23980b4000 con 0x7f2398143eb0 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239cc5d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2398144560 0x7f23980b3ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239cc5d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2398144560 0x7f23980b3ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54178/0 (socket says 172.21.15.64:54178) 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239cc5d640 1 -- 172.21.15.64:0/1776032416 learned_addr learned my addr 172.21.15.64:0/1776032416 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239d45e640 1 --2- 172.21.15.64:0/1776032416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398143eb0 0x7f23980cdef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239cc5d640 1 -- 172.21.15.64:0/1776032416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398143eb0 msgr2=0x7f23980cdef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239cc5d640 1 --2- 172.21.15.64:0/1776032416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398143eb0 0x7f23980cdef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f238ffff640 1 -- 172.21.15.64:0/1776032416 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 537037568 0 0) 0x7f239806db30 con 0x7f2398071a30 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239cc5d640 1 -- 172.21.15.64:0/1776032416 >> v1:172.21.15.66:6789/0 conn(0x7f2398071a30 legacy=0x7f2398071e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:32.234 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239cc5d640 1 -- 172.21.15.64:0/1776032416 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f239806de10 con 0x7f2398144560 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.232+0000 7f239d45e640 1 --2- 172.21.15.64:0/1776032416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398143eb0 0x7f23980cdef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.233+0000 7f239cc5d640 1 --2- 172.21.15.64:0/1776032416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980b3ac0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f2388009870 tx=0x7f238802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=207d532d05cab6a2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.233+0000 7f238ffff640 1 -- 172.21.15.64:0/1776032416 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2388034070 con 0x7f2398144560 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.233+0000 7f238ffff640 1 -- 172.21.15.64:0/1776032416 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2388030840 con 0x7f2398144560 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.233+0000 7f238ffff640 1 -- 172.21.15.64:0/1776032416 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2388030c10 con 0x7f2398144560 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 -- 172.21.15.64:0/1776032416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 msgr2=0x7f23980b3ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 --2- 172.21.15.64:0/1776032416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980b3ac0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f2388009870 tx=0x7f238802b980 comp rx=0 tx=0).stop 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 -- 172.21.15.64:0/1776032416 shutdown_connections 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 --2- 172.21.15.64:0/1776032416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980b3ac0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 --2- 172.21.15.64:0/1776032416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398143eb0 0x7f23980cdef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 -- 172.21.15.64:0/1776032416 >> 172.21.15.64:0/1776032416 conn(0x7f23980b27c0 msgr2=0x7f23980b2bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 -- 172.21.15.64:0/1776032416 shutdown_connections 2024-03-04T07:26:32.235 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.234+0000 7f239f6c2640 1 -- 172.21.15.64:0/1776032416 wait complete. 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.235+0000 7f239f6c2640 1 Processor -- start 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.235+0000 7f239f6c2640 1 -- start start 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239f6c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398071a30 0x7f23980c5a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239f6c2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2398143eb0 0x7f2398142200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239f6c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980c69a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239f6c2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f239806e6b0 con 0x7f2398144560 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239f6c2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f239806e3d0 con 0x7f2398143eb0 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239f6c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2398144290 con 0x7f2398071a30 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980c69a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980c69a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54186/0 (socket says 172.21.15.64:54186) 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 -- 172.21.15.64:0/2930134210 learned_addr learned my addr 172.21.15.64:0/2930134210 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:32.238 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239d45e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398071a30 0x7f23980c5a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239cc5d640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2398143eb0 0x7f2398142200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 -- 172.21.15.64:0/2930134210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398071a30 msgr2=0x7f23980c5a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398071a30 0x7f23980c5a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 -- 172.21.15.64:0/2930134210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2398143eb0 msgr2=0x7f2398142200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2398143eb0 0x7f2398142200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 -- 172.21.15.64:0/2930134210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2398157060 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.236+0000 7f239dc5f640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980c69a0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f239000c790 tx=0x7f239000cc50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.237+0000 7f238dffb640 1 -- 172.21.15.64:0/2930134210 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2390003da0 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.237+0000 7f238dffb640 1 -- 172.21.15.64:0/2930134210 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2390005070 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.237+0000 7f238dffb640 1 -- 172.21.15.64:0/2930134210 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2390005460 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.237+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2398157340 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.237+0000 7f238dffb640 1 -- 172.21.15.64:0/2930134210 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 1) v1 ==== 807+0+0 (secure 0 0 0) 0x7f2390005600 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.237+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2398157900 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.237+0000 7f238dffb640 1 -- 172.21.15.64:0/2930134210 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(10..10 src has 1..10) v4 ==== 1645+0+0 (secure 0 0 0) 0x7f239001b590 con 0x7f2398144560 2024-03-04T07:26:32.239 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.238+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f239804f330 con 0x7f2398144560 2024-03-04T07:26:32.241 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.241+0000 7f238dffb640 1 -- 172.21.15.64:0/2930134210 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73141 (secure 0 0 0) 0x7f239001b790 con 0x7f2398144560 2024-03-04T07:26:32.303 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.304+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2398072630 con 0x7f2398144560 2024-03-04T07:26:32.304 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f238dffb640 1 -- 172.21.15.64:0/2930134210 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v10) v1 ==== 74+0+7058 (secure 0 0 0) 0x7f239000a640 con 0x7f2398144560 2024-03-04T07:26:32.304 INFO:teuthology.misc.health.smithi064.stdout: 2024-03-04T07:26:32.304 INFO:teuthology.misc.health.smithi064.stdout:{"epoch":10,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:31.093625+0000","last_up_change":"0.000000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 msgr2=0x7f23980c69a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f239f6c2640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980c69a0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f239000c790 tx=0x7f239000cc50 comp rx=0 tx=0).stop 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 shutdown_connections 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f239f6c2640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2398144560 0x7f23980c69a0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f239f6c2640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2398143eb0 0x7f2398142200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f239f6c2640 1 --2- 172.21.15.64:0/2930134210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2398071a30 0x7f23980c5a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.305+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 >> 172.21.15.64:0/2930134210 conn(0x7f23980b27c0 msgr2=0x7f23980621d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.306+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 shutdown_connections 2024-03-04T07:26:32.306 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:32.306+0000 7f239f6c2640 1 -- 172.21.15.64:0/2930134210 wait complete. 2024-03-04T07:26:32.314 DEBUG:teuthology.misc:0 of 8 OSDs are up 2024-03-04T07:26:32.654 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:32.654+0000 7f605583d040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-04T07:26:32.658 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:32.658+0000 7f75fb151040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-04T07:26:32.944 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:32.944+0000 7f605583d040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-04T07:26:32.950 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:32.950+0000 7f75fb151040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-04T07:26:32.997 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:32.998+0000 7f605583d040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-04T07:26:33.004 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:33.003+0000 7f75fb151040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-04T07:26:33.117 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:33.118+0000 7f605583d040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-04T07:26:33.124 INFO:tasks.ceph.mgr.x.smithi066.stderr:2024-03-04T07:26:33.124+0000 7f75fb151040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-04T07:26:34.144 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:34.144+0000 7f5ff64dd640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-04T07:26:34.145 INFO:tasks.ceph.mgr.y.smithi064.stderr:2024-03-04T07:26:34.146+0000 7f5ff64dd640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-03-04T07:26:35.213 INFO:tasks.ceph.osd.3.smithi064.stderr:2024-03-04T07:26:35.213+0000 7fe30f81b640 -1 osd.3 0 log_to_monitors true 2024-03-04T07:26:35.220 INFO:tasks.ceph.osd.5.smithi066.stderr:2024-03-04T07:26:35.220+0000 7f1c384f7640 -1 osd.5 0 log_to_monitors true 2024-03-04T07:26:35.237 INFO:tasks.ceph.osd.2.smithi064.stderr:2024-03-04T07:26:35.238+0000 7fb8ad60d640 -1 osd.2 0 log_to_monitors true 2024-03-04T07:26:35.245 INFO:tasks.ceph.osd.0.smithi064.stderr:2024-03-04T07:26:35.246+0000 7f3f68a48640 -1 osd.0 0 log_to_monitors true 2024-03-04T07:26:35.254 INFO:tasks.ceph.osd.1.smithi064.stderr:2024-03-04T07:26:35.255+0000 7ff957a06640 -1 osd.1 0 log_to_monitors true 2024-03-04T07:26:35.265 INFO:tasks.ceph.osd.7.smithi066.stderr:2024-03-04T07:26:35.264+0000 7f9d711e1640 -1 osd.7 0 log_to_monitors true 2024-03-04T07:26:35.272 INFO:tasks.ceph.osd.4.smithi066.stderr:2024-03-04T07:26:35.272+0000 7fa0e9017640 -1 osd.4 0 log_to_monitors true 2024-03-04T07:26:35.274 INFO:tasks.ceph.osd.6.smithi066.stderr:2024-03-04T07:26:35.274+0000 7f1d87a1d640 -1 osd.6 0 log_to_monitors true 2024-03-04T07:26:39.121 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:39.204 INFO:tasks.ceph.osd.7.smithi066.stderr:2024-03-04T07:26:39.204+0000 7f9d6e18c640 -1 osd.7 0 waiting for initial osdmap 2024-03-04T07:26:39.223 INFO:tasks.ceph.osd.7.smithi066.stderr:2024-03-04T07:26:39.223+0000 7f9d64f86640 -1 osd.7 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.240 INFO:tasks.ceph.osd.1.smithi064.stderr:2024-03-04T07:26:39.239+0000 7ff953179640 -1 osd.1 0 waiting for initial osdmap 2024-03-04T07:26:39.240 INFO:tasks.ceph.osd.2.smithi064.stderr:2024-03-04T07:26:39.240+0000 7fb8a9ee9640 -1 osd.2 0 waiting for initial osdmap 2024-03-04T07:26:39.254 INFO:tasks.ceph.osd.2.smithi064.stderr:2024-03-04T07:26:39.254+0000 7fb8a0ce3640 -1 osd.2 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.255 INFO:tasks.ceph.osd.1.smithi064.stderr:2024-03-04T07:26:39.255+0000 7ff94b7ab640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.255 INFO:tasks.ceph.osd.0.smithi064.stderr:2024-03-04T07:26:39.255+0000 7f3f641bb640 -1 osd.0 0 waiting for initial osdmap 2024-03-04T07:26:39.260 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.260+0000 7fc93b565640 1 Processor -- start 2024-03-04T07:26:39.260 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.260+0000 7fc93b565640 1 -- start start 2024-03-04T07:26:39.262 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc93b565640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340cfed0 0x7fc9340d1ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:39.262 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc93b565640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9340707a0 con 0x7fc9340702a0 2024-03-04T07:26:39.262 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc93b565640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc934070a80 con 0x7fc9340cdb50 2024-03-04T07:26:39.262 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc93b565640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9340d1630 con 0x7fc9340cfed0 2024-03-04T07:26:39.262 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc938b00640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc9340702a0 0x7fc93415cc00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54208/0 (socket says 172.21.15.64:54208) 2024-03-04T07:26:39.262 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc938b00640 1 -- 172.21.15.64:0/690198386 learned_addr learned my addr 172.21.15.64:0/690198386 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:39.263 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc939b02640 1 --2- 172.21.15.64:0/690198386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340cfed0 0x7fc9340d1ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:39.263 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc92bfff640 1 -- 172.21.15.64:0/690198386 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1308627828 0 0) 0x7fc9340707a0 con 0x7fc9340702a0 2024-03-04T07:26:39.263 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.261+0000 7fc92bfff640 1 -- 172.21.15.64:0/690198386 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc920003520 con 0x7fc9340702a0 2024-03-04T07:26:39.263 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.262+0000 7fc939b02640 1 -- 172.21.15.64:0/690198386 >> v1:172.21.15.66:6789/0 conn(0x7fc9340cdb50 legacy=0x7fc9340cdf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:39.263 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.262+0000 7fc939b02640 1 -- 172.21.15.64:0/690198386 >> v1:172.21.15.64:6789/0 conn(0x7fc9340702a0 legacy=0x7fc93415cc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:39.263 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.262+0000 7fc939b02640 1 -- 172.21.15.64:0/690198386 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc934070d60 con 0x7fc9340cfed0 2024-03-04T07:26:39.263 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.263+0000 7fc939b02640 1 --2- 172.21.15.64:0/690198386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340cfed0 0x7fc9340d1ac0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fc92c009dc0 tx=0x7fc92c02be90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6286fe2e8f18f240 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:39.264 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.264+0000 7fc92bfff640 1 -- 172.21.15.64:0/690198386 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc92c039070 con 0x7fc9340cfed0 2024-03-04T07:26:39.264 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.264+0000 7fc92bfff640 1 -- 172.21.15.64:0/690198386 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc92c02cc40 con 0x7fc9340cfed0 2024-03-04T07:26:39.265 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.264+0000 7fc92bfff640 1 -- 172.21.15.64:0/690198386 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc92c034440 con 0x7fc9340cfed0 2024-03-04T07:26:39.265 INFO:tasks.ceph.osd.6.smithi066.stderr:2024-03-04T07:26:39.265+0000 7f1d82adb640 -1 osd.6 0 waiting for initial osdmap 2024-03-04T07:26:39.266 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.265+0000 7fc93b565640 1 -- 172.21.15.64:0/690198386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340cfed0 msgr2=0x7fc9340d1ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:39.266 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.265+0000 7fc93b565640 1 --2- 172.21.15.64:0/690198386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340cfed0 0x7fc9340d1ac0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fc92c009dc0 tx=0x7fc92c02be90 comp rx=0 tx=0).stop 2024-03-04T07:26:39.266 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.265+0000 7fc93b565640 1 -- 172.21.15.64:0/690198386 shutdown_connections 2024-03-04T07:26:39.266 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.265+0000 7fc93b565640 1 --2- 172.21.15.64:0/690198386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340cfed0 0x7fc9340d1ac0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:39.266 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.265+0000 7fc93b565640 1 -- 172.21.15.64:0/690198386 >> 172.21.15.64:0/690198386 conn(0x7fc9340b3750 msgr2=0x7fc9340b3b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:39.266 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.265+0000 7fc93b565640 1 -- 172.21.15.64:0/690198386 shutdown_connections 2024-03-04T07:26:39.266 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.265+0000 7fc93b565640 1 -- 172.21.15.64:0/690198386 wait complete. 2024-03-04T07:26:39.267 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.266+0000 7fc93b565640 1 Processor -- start 2024-03-04T07:26:39.267 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.266+0000 7fc93b565640 1 -- start start 2024-03-04T07:26:39.267 INFO:tasks.ceph.osd.0.smithi064.stderr:2024-03-04T07:26:39.267+0000 7f3f5c7ed640 -1 osd.0 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.270 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc93b565640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340702a0 0x7fc93415aff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:39.270 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc93b565640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9340cdb50 0x7fc9341384e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:39.270 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc93b565640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9340cfed0 0x7fc934138a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:39.270 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc93b565640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc934145920 con 0x7fc9340cfed0 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc93b565640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9340ce9d0 con 0x7fc9340cdb50 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc93b565640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc934145aa0 con 0x7fc9340702a0 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc939b02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9340cfed0 0x7fc934138a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc938b00640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9340cdb50 0x7fc9341384e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc939b02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9340cfed0 0x7fc934138a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39622/0 (socket says 172.21.15.64:39622) 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc939b02640 1 -- 172.21.15.64:0/19655330 learned_addr learned my addr 172.21.15.64:0/19655330 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc939301640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340702a0 0x7fc93415aff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:39.271 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc939b02640 1 -- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340702a0 msgr2=0x7fc93415aff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc939b02640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340702a0 0x7fc93415aff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.267+0000 7fc939b02640 1 -- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9340cdb50 msgr2=0x7fc9341384e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc939b02640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9340cdb50 0x7fc9341384e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc939b02640 1 -- 172.21.15.64:0/19655330 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc93413ba60 con 0x7fc9340cfed0 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc939301640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340702a0 0x7fc93415aff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc938b00640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9340cdb50 0x7fc9341384e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc939b02640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9340cfed0 0x7fc934138a20 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fc92c037040 tx=0x7fc92c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc929ffb640 1 -- 172.21.15.64:0/19655330 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc92c039050 con 0x7fc9340cfed0 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc929ffb640 1 -- 172.21.15.64:0/19655330 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc92c045070 con 0x7fc9340cfed0 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc93413bd40 con 0x7fc9340cfed0 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc929ffb640 1 -- 172.21.15.64:0/19655330 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc92c040690 con 0x7fc9340cfed0 2024-03-04T07:26:39.272 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.268+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc93413c300 con 0x7fc9340cfed0 2024-03-04T07:26:39.273 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.270+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc93404f330 con 0x7fc9340cfed0 2024-03-04T07:26:39.273 INFO:tasks.ceph.osd.4.smithi066.stderr:2024-03-04T07:26:39.269+0000 7fa0e5a92640 -1 osd.4 0 waiting for initial osdmap 2024-03-04T07:26:39.274 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.271+0000 7fc929ffb640 1 -- 172.21.15.64:0/19655330 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 4) v1 ==== 89724+0+0 (secure 0 0 0) 0x7fc92c040830 con 0x7fc9340cfed0 2024-03-04T07:26:39.274 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.271+0000 7fc929ffb640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc91806b470 0x7fc91806d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:39.274 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.271+0000 7fc929ffb640 1 -- 172.21.15.64:0/19655330 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(12..12 src has 1..12) v4 ==== 2231+0+0 (secure 0 0 0) 0x7fc92c0abd00 con 0x7fc9340cfed0 2024-03-04T07:26:39.274 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.271+0000 7fc939301640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc91806b470 0x7fc91806d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:39.275 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.271+0000 7fc939301640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc91806b470 0x7fc91806d930 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fc930000f60 tx=0x7fc930009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:39.280 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.280+0000 7fc929ffb640 1 -- 172.21.15.64:0/19655330 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc92c07b300 con 0x7fc9340cfed0 2024-03-04T07:26:39.283 INFO:tasks.ceph.osd.4.smithi066.stderr:2024-03-04T07:26:39.283+0000 7fa0dc88c640 -1 osd.4 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.283 INFO:tasks.ceph.osd.6.smithi066.stderr:2024-03-04T07:26:39.283+0000 7f1d7b10d640 -1 osd.6 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.291 INFO:tasks.ceph.osd.3.smithi064.stderr:2024-03-04T07:26:39.291+0000 7fe30a8ca640 -1 osd.3 0 waiting for initial osdmap 2024-03-04T07:26:39.304 INFO:tasks.ceph.osd.3.smithi064.stderr:2024-03-04T07:26:39.304+0000 7fe302efc640 -1 osd.3 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.309 INFO:tasks.ceph.osd.5.smithi066.stderr:2024-03-04T07:26:39.309+0000 7f1c33c6a640 -1 osd.5 0 waiting for initial osdmap 2024-03-04T07:26:39.327 INFO:tasks.ceph.osd.5.smithi066.stderr:2024-03-04T07:26:39.327+0000 7f1c2c29c640 -1 osd.5 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-04T07:26:39.464 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.463+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc9340b6a20 con 0x7fc9340cfed0 2024-03-04T07:26:39.494 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.464+0000 7fc929ffb640 1 -- 172.21.15.64:0/19655330 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v12) v1 ==== 74+0+7058 (secure 0 0 0) 0x7fc92c07f3a0 con 0x7fc9340cfed0 2024-03-04T07:26:39.494 INFO:teuthology.misc.health.smithi064.stdout: 2024-03-04T07:26:39.495 INFO:teuthology.misc.health.smithi064.stdout:{"epoch":12,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:37.146074+0000","last_up_change":"0.000000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:39.496 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc91806b470 msgr2=0x7fc91806d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:39.496 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc91806b470 0x7fc91806d930 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fc930000f60 tx=0x7fc930009040 comp rx=0 tx=0).stop 2024-03-04T07:26:39.496 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9340cfed0 msgr2=0x7fc934138a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:39.496 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9340cfed0 0x7fc934138a20 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fc92c037040 tx=0x7fc92c004a40 comp rx=0 tx=0).stop 2024-03-04T07:26:39.496 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 shutdown_connections 2024-03-04T07:26:39.496 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc91806b470 0x7fc91806d930 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:39.497 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9340cfed0 0x7fc934138a20 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:39.497 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9340cdb50 0x7fc9341384e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:39.497 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 --2- 172.21.15.64:0/19655330 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9340702a0 0x7fc93415aff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:39.497 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 >> 172.21.15.64:0/19655330 conn(0x7fc9340b3750 msgr2=0x7fc93415eb60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:39.497 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 shutdown_connections 2024-03-04T07:26:39.497 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:39.466+0000 7fc93b565640 1 -- 172.21.15.64:0/19655330 wait complete. 2024-03-04T07:26:39.497 DEBUG:teuthology.misc:0 of 8 OSDs are up 2024-03-04T07:26:46.305 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:46.407 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.407+0000 7fd91cac9640 1 Processor -- start 2024-03-04T07:26:46.408 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.407+0000 7fd91cac9640 1 -- start start 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd91cac9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd9180b3100 0x7fd9180b96b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd91cac9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd9180d2d30 0x7fd9180d22e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd91cac9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd9180d39e0 con 0x7fd9180d2820 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd91cac9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd9180b9bf0 con 0x7fd9180d2d30 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd91cac9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd9180d3420 con 0x7fd9180b3100 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd90ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd9180d2d30 0x7fd9180d22e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd916d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd9180d2820 0x7fd9180b6e30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36152/0 (socket says 172.21.15.64:36152) 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd916d9d640 1 -- 172.21.15.64:0/1554726563 learned_addr learned my addr 172.21.15.64:0/1554726563 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd91659c640 1 --2- 172.21.15.64:0/1554726563 >> v2:172.21.15.66:3300/0 conn(0x7fd9180b3100 0x7fd9180b96b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd90ffff640 1 -- 172.21.15.64:0/1554726563 >> v2:172.21.15.66:3300/0 conn(0x7fd9180b3100 msgr2=0x7fd9180b96b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:46.409 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd90ffff640 1 --2- 172.21.15.64:0/1554726563 >> v2:172.21.15.66:3300/0 conn(0x7fd9180b3100 0x7fd9180b96b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.410 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd90ffff640 1 -- 172.21.15.64:0/1554726563 >> v1:172.21.15.64:6789/0 conn(0x7fd9180d2820 legacy=0x7fd9180b6e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.410 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.408+0000 7fd90ffff640 1 -- 172.21.15.64:0/1554726563 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9180d3700 con 0x7fd9180d2d30 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.409+0000 7fd90ffff640 1 --2- 172.21.15.64:0/1554726563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd9180d22e0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fd9180b6870 tx=0x7fd90002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f58dcdd98b579b62 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.409+0000 7fd915d9b640 1 -- 172.21.15.64:0/1554726563 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9000315a0 con 0x7fd9180d2d30 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.409+0000 7fd915d9b640 1 -- 172.21.15.64:0/1554726563 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd900037070 con 0x7fd9180d2d30 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.409+0000 7fd915d9b640 1 -- 172.21.15.64:0/1554726563 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd900031960 con 0x7fd9180d2d30 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.409+0000 7fd91cac9640 1 -- 172.21.15.64:0/1554726563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 msgr2=0x7fd9180d22e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.409+0000 7fd91cac9640 1 --2- 172.21.15.64:0/1554726563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd9180d22e0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fd9180b6870 tx=0x7fd90002b840 comp rx=0 tx=0).stop 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.410+0000 7fd91cac9640 1 -- 172.21.15.64:0/1554726563 shutdown_connections 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.410+0000 7fd91cac9640 1 --2- 172.21.15.64:0/1554726563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd9180d22e0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.410+0000 7fd91cac9640 1 --2- 172.21.15.64:0/1554726563 >> v2:172.21.15.66:3300/0 conn(0x7fd9180b3100 0x7fd9180b96b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.410+0000 7fd91cac9640 1 -- 172.21.15.64:0/1554726563 >> 172.21.15.64:0/1554726563 conn(0x7fd9180d3d00 msgr2=0x7fd9180d4110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.410+0000 7fd91cac9640 1 -- 172.21.15.64:0/1554726563 shutdown_connections 2024-03-04T07:26:46.411 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.410+0000 7fd91cac9640 1 -- 172.21.15.64:0/1554726563 wait complete. 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.411+0000 7fd91cac9640 1 Processor -- start 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.411+0000 7fd91cac9640 1 -- start start 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd91cac9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9180b3100 0x7fd9180c9aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd91cac9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9180d2820 0x7fd918155690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd91cac9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd918155bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd91cac9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9180b7540 con 0x7fd9180d2d30 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd91cac9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9180cefe0 con 0x7fd9180d2820 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd91cac9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd9180b76c0 con 0x7fd9180b3100 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd918155bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd918155bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40956/0 (socket says 172.21.15.64:40956) 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 -- 172.21.15.64:0/3828458725 learned_addr learned my addr 172.21.15.64:0/3828458725 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd90ffff640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9180d2820 0x7fd918155690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd91659c640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9180b3100 0x7fd9180c9aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.414 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 -- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9180b3100 msgr2=0x7fd9180c9aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9180b3100 0x7fd9180c9aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 -- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9180d2820 msgr2=0x7fd918155690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9180d2820 0x7fd918155690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 -- 172.21.15.64:0/3828458725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9180c8560 con 0x7fd9180d2d30 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.412+0000 7fd916d9d640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd918155bd0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fd91000cab0 tx=0x7fd91000cf70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.413+0000 7fd90effd640 1 -- 172.21.15.64:0/3828458725 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd910017070 con 0x7fd9180d2d30 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.413+0000 7fd90effd640 1 -- 172.21.15.64:0/3828458725 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd91000ac30 con 0x7fd9180d2d30 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.413+0000 7fd90effd640 1 -- 172.21.15.64:0/3828458725 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd910012400 con 0x7fd9180d2d30 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.413+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd9180c8840 con 0x7fd9180d2d30 2024-03-04T07:26:46.415 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.413+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd9180c8e00 con 0x7fd9180d2d30 2024-03-04T07:26:46.416 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.414+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd9180b2cb0 con 0x7fd9180d2d30 2024-03-04T07:26:46.416 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.414+0000 7fd90effd640 1 -- 172.21.15.64:0/3828458725 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd91001a070 con 0x7fd9180d2d30 2024-03-04T07:26:46.416 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.414+0000 7fd90effd640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd8f806b4d0 0x7fd8f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.416 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.415+0000 7fd91659c640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd8f806b4d0 0x7fd8f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.416 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.415+0000 7fd90effd640 1 -- 172.21.15.64:0/3828458725 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fd910089b30 con 0x7fd9180d2d30 2024-03-04T07:26:46.416 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.415+0000 7fd91659c640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd8f806b4d0 0x7fd8f806d990 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fd91815d4b0 tx=0x7fd908008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:46.419 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.419+0000 7fd90effd640 1 -- 172.21.15.64:0/3828458725 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd9180b2cb0 con 0x7fd9180d2d30 2024-03-04T07:26:46.604 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.603+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fd91804f330 con 0x7fd9180d2d30 2024-03-04T07:26:46.604 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.604+0000 7fd90effd640 1 -- 172.21.15.64:0/3828458725 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v16) v1 ==== 74+0+12836 (secure 0 0 0) 0x7fd910058630 con 0x7fd9180d2d30 2024-03-04T07:26:46.604 INFO:teuthology.misc.health.smithi064.stdout: 2024-03-04T07:26:46.605 INFO:teuthology.misc.health.smithi064.stdout:{"epoch":16,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:43.215051+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd8f806b4d0 msgr2=0x7fd8f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd8f806b4d0 0x7fd8f806d990 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fd91815d4b0 tx=0x7fd908008040 comp rx=0 tx=0).stop 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 msgr2=0x7fd918155bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd918155bd0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fd91000cab0 tx=0x7fd91000cf70 comp rx=0 tx=0).stop 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 shutdown_connections 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd8f806b4d0 0x7fd8f806d990 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9180d2d30 0x7fd918155bd0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9180d2820 0x7fd918155690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 --2- 172.21.15.64:0/3828458725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9180b3100 0x7fd9180c9aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.607 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 >> 172.21.15.64:0/3828458725 conn(0x7fd9180d3d00 msgr2=0x7fd9180d42c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:46.608 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 shutdown_connections 2024-03-04T07:26:46.608 INFO:teuthology.misc.health.smithi064.stderr:2024-03-04T07:26:46.606+0000 7fd91cac9640 1 -- 172.21.15.64:0/3828458725 wait complete. 2024-03-04T07:26:46.616 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-03-04T07:26:46.616 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2024-03-04T07:26:46.617 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:26:46.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:26:46.619 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:26:46.619 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:26:46.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:26:46.621 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:26:46.621 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:26:46.621 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:26:46.624 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:26:46.624 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:26:46.625 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:26:46.627 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:26:46.627 INFO:tasks.ceph:Setting up CephFS filesystem(s)... 2024-03-04T07:26:46.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:46.695 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:26:46.695 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:26:46.717 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:26:46.717 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:26:46.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.737+0000 7f4cee18a640 1 Processor -- start 2024-03-04T07:26:46.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.737+0000 7f4cee18a640 1 -- start start 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.737+0000 7f4cee18a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce8072660 0x7f4ce815ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.737+0000 7f4cee18a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4ce80b55d0 0x7f4ce8071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.737+0000 7f4cee18a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ce80b4160 con 0x7f4ce806e700 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4cee18a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4ce80b5b40 con 0x7f4ce80b55d0 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4cee18a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ce80b3ba0 con 0x7f4ce8072660 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce77fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f4ce806e700 0x7f4ce806d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49364/0 (socket says 172.21.15.64:49364) 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce6ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce8072660 0x7f4ce815ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce77fe640 1 -- 172.21.15.64:0/2589464006 learned_addr learned my addr 172.21.15.64:0/2589464006 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce7fff640 1 --2- 172.21.15.64:0/2589464006 >> v2:172.21.15.66:3300/0 conn(0x7f4ce80b55d0 0x7f4ce8071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce67fc640 1 -- 172.21.15.64:0/2589464006 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4017930365 0 0) 0x7f4ce80b4160 con 0x7f4ce806e700 2024-03-04T07:26:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce67fc640 1 -- 172.21.15.64:0/2589464006 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4cd0003520 con 0x7f4ce806e700 2024-03-04T07:26:46.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce6ffd640 1 -- 172.21.15.64:0/2589464006 >> v2:172.21.15.66:3300/0 conn(0x7f4ce80b55d0 msgr2=0x7f4ce8071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:46.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce6ffd640 1 --2- 172.21.15.64:0/2589464006 >> v2:172.21.15.66:3300/0 conn(0x7f4ce80b55d0 0x7f4ce8071980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce6ffd640 1 -- 172.21.15.64:0/2589464006 >> v1:172.21.15.66:6789/0 conn(0x7f4ce806e700 legacy=0x7f4ce806d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.738+0000 7f4ce6ffd640 1 -- 172.21.15.64:0/2589464006 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ce80b3e80 con 0x7f4ce8072660 2024-03-04T07:26:46.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.739+0000 7f4ce6ffd640 1 --2- 172.21.15.64:0/2589464006 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce8072660 0x7f4ce815ef80 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f4cdc0097a0 tx=0x7f4cdc02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=581ff594d3e99d10 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:46.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.739+0000 7f4ce67fc640 1 -- 172.21.15.64:0/2589464006 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cdc035020 con 0x7f4ce8072660 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.739+0000 7f4ce67fc640 1 -- 172.21.15.64:0/2589464006 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cdc031830 con 0x7f4ce8072660 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.739+0000 7f4ce67fc640 1 -- 172.21.15.64:0/2589464006 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cdc031c00 con 0x7f4ce8072660 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 -- 172.21.15.64:0/2589464006 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce8072660 msgr2=0x7f4ce815ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2589464006 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce8072660 0x7f4ce815ef80 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f4cdc0097a0 tx=0x7f4cdc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 -- 172.21.15.64:0/2589464006 shutdown_connections 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2589464006 >> v2:172.21.15.66:3300/0 conn(0x7f4ce80b55d0 0x7f4ce8071980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2589464006 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce8072660 0x7f4ce815ef80 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 -- 172.21.15.64:0/2589464006 >> 172.21.15.64:0/2589464006 conn(0x7f4ce80b2720 msgr2=0x7f4ce80b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:46.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 -- 172.21.15.64:0/2589464006 shutdown_connections 2024-03-04T07:26:46.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.740+0000 7f4cee18a640 1 -- 172.21.15.64:0/2589464006 wait complete. 2024-03-04T07:26:46.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.741+0000 7f4cee18a640 1 Processor -- start 2024-03-04T07:26:46.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.741+0000 7f4cee18a640 1 -- start start 2024-03-04T07:26:46.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce806e700 0x7f4ce8154990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ce8072660 0x7f4ce815b3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ce80b55d0 0x7f4ce8160710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ce8072040 con 0x7f4ce8072660 2024-03-04T07:26:46.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ce8071ec0 con 0x7f4ce80b55d0 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ce80721c0 con 0x7f4ce806e700 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ce8072660 0x7f4ce815b3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ce8072660 0x7f4ce815b3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40990/0 (socket says 172.21.15.64:40990) 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 -- 172.21.15.64:0/2741980563 learned_addr learned my addr 172.21.15.64:0/2741980563 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce77fe640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce806e700 0x7f4ce8154990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce7fff640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ce80b55d0 0x7f4ce8160710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 -- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce806e700 msgr2=0x7f4ce8154990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce806e700 0x7f4ce8154990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 -- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ce80b55d0 msgr2=0x7f4ce8160710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ce80b55d0 0x7f4ce8160710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 -- 172.21.15.64:0/2741980563 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ce81f41e0 con 0x7f4ce8072660 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce77fe640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce806e700 0x7f4ce8154990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ce6ffd640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ce8072660 0x7f4ce815b3a0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f4cdc02bee0 tx=0x7f4cdc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:46.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ccffff640 1 -- 172.21.15.64:0/2741980563 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cdc035040 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ccffff640 1 -- 172.21.15.64:0/2741980563 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cdc03b920 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ce81f44c0 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4ccffff640 1 -- 172.21.15.64:0/2741980563 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cdc03bd10 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.742+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ce81f4a80 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.744+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ce8067ff0 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.744+0000 7f4ccffff640 1 -- 172.21.15.64:0/2741980563 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4cdc038020 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.744+0000 7f4ccffff640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cd406b4d0 0x7f4cd406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.744+0000 7f4ccffff640 1 -- 172.21.15.64:0/2741980563 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f4cdc0abf20 con 0x7f4ce8072660 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.744+0000 7f4ce77fe640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cd406b4d0 0x7f4cd406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:46.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.745+0000 7f4ce77fe640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cd406b4d0 0x7f4cd406d990 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f4cd8007fd0 tx=0x7f4cd8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:46.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.749+0000 7f4ccffff640 1 -- 172.21.15.64:0/2741980563 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4cdc07b0f0 con 0x7f4ce8072660 2024-03-04T07:26:46.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.931+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4ce806ef40 con 0x7f4ce8072660 2024-03-04T07:26:46.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.932+0000 7f4ccffff640 1 -- 172.21.15.64:0/2741980563 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v16) v1 ==== 74+0+12836 (secure 0 0 0) 0x7f4cdc07a9f0 con 0x7f4ce8072660 2024-03-04T07:26:46.932 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:46.932 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":16,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:43.215051+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cd406b4d0 msgr2=0x7f4cd406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cd406b4d0 0x7f4cd406d990 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f4cd8007fd0 tx=0x7f4cd8009040 comp rx=0 tx=0).stop 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ce8072660 msgr2=0x7f4ce815b3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ce8072660 0x7f4ce815b3a0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f4cdc02bee0 tx=0x7f4cdc0046c0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 shutdown_connections 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cd406b4d0 0x7f4cd406d990 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ce80b55d0 0x7f4ce8160710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ce8072660 0x7f4ce815b3a0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 --2- 172.21.15.64:0/2741980563 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ce806e700 0x7f4ce8154990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 >> 172.21.15.64:0/2741980563 conn(0x7f4ce80b2720 msgr2=0x7f4ce80b1db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 shutdown_connections 2024-03-04T07:26:46.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:46.934+0000 7f4cee18a640 1 -- 172.21.15.64:0/2741980563 wait complete. 2024-03-04T07:26:46.945 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:26:46.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:26:47.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.049+0000 7f837e4c2640 1 Processor -- start 2024-03-04T07:26:47.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.049+0000 7f837e4c2640 1 -- start start 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f837e4c2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f837815dbd0 0x7f837815ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f837e4c2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f83780b5840 0x7f83780b39b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f837e4c2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f837815f3b0 con 0x7f837815dfb0 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f837e4c2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f837806f070 con 0x7f837815dbd0 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f837e4c2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8378071940 con 0x7f83780b5840 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f8377fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f837815dbd0 0x7f837815ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f837ca5f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f837815dfb0 0x7f8378161a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36174/0 (socket says 172.21.15.64:36174) 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f837ca5f640 1 -- 172.21.15.64:0/1574578286 learned_addr learned my addr 172.21.15.64:0/1574578286 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f83777fe640 1 --2- 172.21.15.64:0/1574578286 >> v2:172.21.15.66:3300/0 conn(0x7f83780b5840 0x7f83780b39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f8377fff640 1 -- 172.21.15.64:0/1574578286 >> v2:172.21.15.66:3300/0 conn(0x7f83780b5840 msgr2=0x7f83780b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:47.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f8377fff640 1 --2- 172.21.15.64:0/1574578286 >> v2:172.21.15.66:3300/0 conn(0x7f83780b5840 0x7f83780b39b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f8377fff640 1 -- 172.21.15.64:0/1574578286 >> v1:172.21.15.64:6789/0 conn(0x7f837815dfb0 legacy=0x7f8378161a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f8377fff640 1 -- 172.21.15.64:0/1574578286 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83780b5db0 con 0x7f837815dbd0 2024-03-04T07:26:47.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.050+0000 7f8376ffd640 1 -- 172.21.15.64:0/1574578286 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1377880939 0 0) 0x7f837815f3b0 con 0x7f837815dfb0 2024-03-04T07:26:47.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.051+0000 7f8377fff640 1 --2- 172.21.15.64:0/1574578286 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dbd0 0x7f837815ee70 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f8378073350 tx=0x7f836c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fbf8884e839b66c9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:47.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.051+0000 7f8376ffd640 1 -- 172.21.15.64:0/1574578286 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f836c035020 con 0x7f837815dbd0 2024-03-04T07:26:47.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.051+0000 7f8376ffd640 1 -- 172.21.15.64:0/1574578286 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f836c031830 con 0x7f837815dbd0 2024-03-04T07:26:47.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.051+0000 7f8376ffd640 1 -- 172.21.15.64:0/1574578286 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f836c031c00 con 0x7f837815dbd0 2024-03-04T07:26:47.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 -- 172.21.15.64:0/1574578286 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dbd0 msgr2=0x7f837815ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 --2- 172.21.15.64:0/1574578286 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dbd0 0x7f837815ee70 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f8378073350 tx=0x7f836c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 -- 172.21.15.64:0/1574578286 shutdown_connections 2024-03-04T07:26:47.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 --2- 172.21.15.64:0/1574578286 >> v2:172.21.15.66:3300/0 conn(0x7f83780b5840 0x7f83780b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 --2- 172.21.15.64:0/1574578286 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dbd0 0x7f837815ee70 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 -- 172.21.15.64:0/1574578286 >> 172.21.15.64:0/1574578286 conn(0x7f83780b26b0 msgr2=0x7f83780b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:47.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 -- 172.21.15.64:0/1574578286 shutdown_connections 2024-03-04T07:26:47.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.052+0000 7f837e4c2640 1 -- 172.21.15.64:0/1574578286 wait complete. 2024-03-04T07:26:47.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.053+0000 7f837e4c2640 1 Processor -- start 2024-03-04T07:26:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.053+0000 7f837e4c2640 1 -- start start 2024-03-04T07:26:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837e4c2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83780b5840 0x7f837814b580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837e4c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f837815dbd0 0x7f83780b1ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837e4c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dfb0 0x7f837814be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837e4c2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83780721c0 con 0x7f837815dfb0 2024-03-04T07:26:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837e4c2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8378071ee0 con 0x7f83780b5840 2024-03-04T07:26:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837e4c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f83780726e0 con 0x7f837815dbd0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dfb0 0x7f837814be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dfb0 0x7f837814be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41020/0 (socket says 172.21.15.64:41020) 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 -- 172.21.15.64:0/3919363648 learned_addr learned my addr 172.21.15.64:0/3919363648 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f8377fff640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83780b5840 0x7f837814b580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f83777fe640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f837815dbd0 0x7f83780b1ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 -- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f837815dbd0 msgr2=0x7f83780b1ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f837815dbd0 0x7f83780b1ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 -- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83780b5840 msgr2=0x7f837814b580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83780b5840 0x7f837814b580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 -- 172.21.15.64:0/3919363648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f837814ecd0 con 0x7f837815dfb0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f83777fe640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f837815dbd0 0x7f83780b1ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.054+0000 7f837ca5f640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dfb0 0x7f837814be20 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f837000c7a0 tx=0x7f837000cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.055+0000 7f8374ff9640 1 -- 172.21.15.64:0/3919363648 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8370018070 con 0x7f837815dfb0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.055+0000 7f8374ff9640 1 -- 172.21.15.64:0/3919363648 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f837000a640 con 0x7f837815dfb0 2024-03-04T07:26:47.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.055+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83780c5b00 con 0x7f837815dfb0 2024-03-04T07:26:47.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.055+0000 7f8374ff9640 1 -- 172.21.15.64:0/3919363648 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f837000aa70 con 0x7f837815dfb0 2024-03-04T07:26:47.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.055+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83780c60c0 con 0x7f837815dfb0 2024-03-04T07:26:47.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.056+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8378073380 con 0x7f837815dfb0 2024-03-04T07:26:47.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.056+0000 7f8374ff9640 1 -- 172.21.15.64:0/3919363648 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f837001b070 con 0x7f837815dfb0 2024-03-04T07:26:47.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.056+0000 7f8374ff9640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f836406b4d0 0x7f836406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.056+0000 7f8374ff9640 1 -- 172.21.15.64:0/3919363648 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f8370089440 con 0x7f837815dfb0 2024-03-04T07:26:47.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.056+0000 7f8377fff640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f836406b4d0 0x7f836406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.057+0000 7f8377fff640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f836406b4d0 0x7f836406d990 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f836c036000 tx=0x7f836c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:47.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.061+0000 7f8374ff9640 1 -- 172.21.15.64:0/3919363648 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8370057f30 con 0x7f837815dfb0 2024-03-04T07:26:47.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:26:47.201 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:26:47.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.242+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f837804f410 con 0x7f837815dfb0 2024-03-04T07:26:47.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.243+0000 7f8374ff9640 1 -- 172.21.15.64:0/3919363648 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v16) v1 ==== 93+0+10 (secure 0 0 0) 0x7f837005bfd0 con 0x7f837815dfb0 2024-03-04T07:26:47.243 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:26:47.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.244+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f836406b4d0 msgr2=0x7f836406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.244+0000 7f837e4c2640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f836406b4d0 0x7f836406d990 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f836c036000 tx=0x7f836c008030 comp rx=0 tx=0).stop 2024-03-04T07:26:47.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.244+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dfb0 msgr2=0x7f837814be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.244+0000 7f837e4c2640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dfb0 0x7f837814be20 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f837000c7a0 tx=0x7f837000cc60 comp rx=0 tx=0).stop 2024-03-04T07:26:47.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.244+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 shutdown_connections 2024-03-04T07:26:47.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.244+0000 7f837e4c2640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f836406b4d0 0x7f836406d990 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.245+0000 7f837e4c2640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f837815dfb0 0x7f837814be20 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.245+0000 7f837e4c2640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f837815dbd0 0x7f83780b1ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.245+0000 7f837e4c2640 1 --2- 172.21.15.64:0/3919363648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83780b5840 0x7f837814b580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.245+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 >> 172.21.15.64:0/3919363648 conn(0x7f83780b26b0 msgr2=0x7f837815b6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:47.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.245+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 shutdown_connections 2024-03-04T07:26:47.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.245+0000 7f837e4c2640 1 -- 172.21.15.64:0/3919363648 wait complete. 2024-03-04T07:26:47.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:26:47.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.397+0000 7f7ef59e9640 1 Processor -- start 2024-03-04T07:26:47.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.397+0000 7f7ef59e9640 1 -- start start 2024-03-04T07:26:47.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7ef59e9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ef015dca0 0x7f7ef006fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7ef59e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b00c0 0x7f7ef00b3ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7ef59e9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ef0161c20 con 0x7f7ef015e080 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7ef59e9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7ef01619d0 con 0x7f7ef015dca0 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7ef59e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef0161f00 con 0x7f7ef00b00c0 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7eeeffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ef015dca0 0x7f7ef006fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7eeeffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ef015dca0 0x7f7ef006fd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41034/0 (socket says 172.21.15.64:41034) 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.398+0000 7f7eeeffd640 1 -- 172.21.15.64:0/1015821652 learned_addr learned my addr 172.21.15.64:0/1015821652 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.399+0000 7f7eef7fe640 1 --2- 172.21.15.64:0/1015821652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b00c0 0x7f7ef00b3ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.399+0000 7f7eeeffd640 1 -- 172.21.15.64:0/1015821652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b00c0 msgr2=0x7f7ef00b3ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.399+0000 7f7eeeffd640 1 --2- 172.21.15.64:0/1015821652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b00c0 0x7f7ef00b3ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.399+0000 7f7eeeffd640 1 -- 172.21.15.64:0/1015821652 >> v1:172.21.15.66:6789/0 conn(0x7f7ef015e080 legacy=0x7f7ef0066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.399+0000 7f7eeeffd640 1 -- 172.21.15.64:0/1015821652 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ef01621e0 con 0x7f7ef015dca0 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.399+0000 7f7eeeffd640 1 --2- 172.21.15.64:0/1015821652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef015dca0 0x7f7ef006fd10 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f7ee8009870 tx=0x7f7ee802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a6996bfbaa9da147 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7eedffb640 1 -- 172.21.15.64:0/1015821652 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee8034070 con 0x7f7ef015dca0 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7eedffb640 1 -- 172.21.15.64:0/1015821652 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ee8030840 con 0x7f7ef015dca0 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7eedffb640 1 -- 172.21.15.64:0/1015821652 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee8030c10 con 0x7f7ef015dca0 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 -- 172.21.15.64:0/1015821652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef015dca0 msgr2=0x7f7ef006fd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/1015821652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef015dca0 0x7f7ef006fd10 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f7ee8009870 tx=0x7f7ee802b980 comp rx=0 tx=0).stop 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 -- 172.21.15.64:0/1015821652 shutdown_connections 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/1015821652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b00c0 0x7f7ef00b3ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/1015821652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef015dca0 0x7f7ef006fd10 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 -- 172.21.15.64:0/1015821652 >> 172.21.15.64:0/1015821652 conn(0x7f7ef00d0990 msgr2=0x7f7ef00d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:47.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 -- 172.21.15.64:0/1015821652 shutdown_connections 2024-03-04T07:26:47.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.400+0000 7f7ef59e9640 1 -- 172.21.15.64:0/1015821652 wait complete. 2024-03-04T07:26:47.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.401+0000 7f7ef59e9640 1 Processor -- start 2024-03-04T07:26:47.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.401+0000 7f7ef59e9640 1 -- start start 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7ef59e9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef00b00c0 0x7f7ef00b3f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7ef59e9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ef015dca0 0x7f7ef00cfe50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7ef59e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b44e0 0x7f7ef015a800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7ef59e9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef00703d0 con 0x7f7ef00b00c0 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7ef59e9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef0070250 con 0x7f7ef015dca0 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7ef59e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ef0067410 con 0x7f7ef00b44e0 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7eeeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef00b00c0 0x7f7ef00b3f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7eeeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef00b00c0 0x7f7ef00b3f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41040/0 (socket says 172.21.15.64:41040) 2024-03-04T07:26:47.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7eeeffd640 1 -- 172.21.15.64:0/3383725044 learned_addr learned my addr 172.21.15.64:0/3383725044 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7eef7fe640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b44e0 0x7f7ef015a800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.402+0000 7f7eee7fc640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ef015dca0 0x7f7ef00cfe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7eeeffd640 1 -- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b44e0 msgr2=0x7f7ef015a800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7eeeffd640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b44e0 0x7f7ef015a800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7eeeffd640 1 -- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ef015dca0 msgr2=0x7f7ef00cfe50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7eeeffd640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ef015dca0 0x7f7ef00cfe50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7eeeffd640 1 -- 172.21.15.64:0/3383725044 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ef00b4970 con 0x7f7ef00b00c0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7eeeffd640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef00b00c0 0x7f7ef00b3f00 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f7ee8000c00 tx=0x7f7ee8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7ee7fff640 1 -- 172.21.15.64:0/3383725044 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee8048070 con 0x7f7ef00b00c0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7ee7fff640 1 -- 172.21.15.64:0/3383725044 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ee8030640 con 0x7f7ef00b00c0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7ef00cd540 con 0x7f7ef00b00c0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7ee7fff640 1 -- 172.21.15.64:0/3383725044 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ee8034040 con 0x7f7ef00b00c0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.403+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ef00cdb00 con 0x7f7ef00b00c0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.404+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7ef0068700 con 0x7f7ef00b00c0 2024-03-04T07:26:47.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.404+0000 7f7ee7fff640 1 -- 172.21.15.64:0/3383725044 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7ee804b070 con 0x7f7ef00b00c0 2024-03-04T07:26:47.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.404+0000 7f7ee7fff640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ed006b4d0 0x7f7ed006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:47.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.405+0000 7f7ee7fff640 1 -- 172.21.15.64:0/3383725044 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f7ee80abcf0 con 0x7f7ef00b00c0 2024-03-04T07:26:47.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.405+0000 7f7eee7fc640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ed006b4d0 0x7f7ed006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:47.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.405+0000 7f7eee7fc640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ed006b4d0 0x7f7ed006d990 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f7ef0072730 tx=0x7f7ed8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:47.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.409+0000 7f7ee7fff640 1 -- 172.21.15.64:0/3383725044 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7ee80b0070 con 0x7f7ef00b00c0 2024-03-04T07:26:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.626+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7ef00cfbb0 con 0x7f7ef00b00c0 2024-03-04T07:26:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.627+0000 7f7ee7fff640 1 -- 172.21.15.64:0/3383725044 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) v1 ==== 93+0+1321 (secure 0 0 0) 0x7f7ee8030810 con 0x7f7ef00b00c0 2024-03-04T07:26:47.628 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:47.628 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2}],"filesystems":[]} 2024-03-04T07:26:47.628 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 2 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ed006b4d0 msgr2=0x7f7ed006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ed006b4d0 0x7f7ed006d990 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f7ef0072730 tx=0x7f7ed8009040 comp rx=0 tx=0).stop 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef00b00c0 msgr2=0x7f7ef00b3f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef00b00c0 0x7f7ef00b3f00 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f7ee8000c00 tx=0x7f7ee8004730 comp rx=0 tx=0).stop 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 shutdown_connections 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ed006b4d0 0x7f7ed006d990 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ef00b44e0 0x7f7ef015a800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ef015dca0 0x7f7ef00cfe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 --2- 172.21.15.64:0/3383725044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ef00b00c0 0x7f7ef00b3f00 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 >> 172.21.15.64:0/3383725044 conn(0x7f7ef00d0990 msgr2=0x7f7ef00ae940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 shutdown_connections 2024-03-04T07:26:47.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:47.629+0000 7f7ef59e9640 1 -- 172.21.15.64:0/3383725044 wait complete. 2024-03-04T07:26:49.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:26:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.754+0000 7f21ab8ed640 1 Processor -- start 2024-03-04T07:26:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.754+0000 7f21ab8ed640 1 -- start start 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21ab8ed640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f21a4157b80 0x7f21a415b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21ab8ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a415b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21ab8ed640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21a406c7b0 con 0x7f21a40cbab0 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21ab8ed640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21a406ca90 con 0x7f21a4157b80 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21ab8ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21a406cd70 con 0x7f21a4157f60 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21a9689640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f21a4157b80 0x7f21a415b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21a9689640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f21a4157b80 0x7f21a415b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41044/0 (socket says 172.21.15.64:41044) 2024-03-04T07:26:49.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21a9689640 1 -- 172.21.15.64:0/4218857118 learned_addr learned my addr 172.21.15.64:0/4218857118 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.755+0000 7f21a8e88640 1 --2- 172.21.15.64:0/4218857118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a415b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.756+0000 7f21a9689640 1 -- 172.21.15.64:0/4218857118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 msgr2=0x7f21a415b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.756+0000 7f21a9689640 1 --2- 172.21.15.64:0/4218857118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a415b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.756+0000 7f21a9689640 1 -- 172.21.15.64:0/4218857118 >> v1:172.21.15.66:6789/0 conn(0x7f21a40cbab0 legacy=0x7f21a4070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.756+0000 7f21a9689640 1 -- 172.21.15.64:0/4218857118 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21a406d050 con 0x7f21a4157b80 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.756+0000 7f21a8e88640 1 --2- 172.21.15.64:0/4218857118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a415b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.756+0000 7f21a9689640 1 --2- 172.21.15.64:0/4218857118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415b260 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f219c009870 tx=0x7f219c02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=969932b76d3ef590 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21a3fff640 1 -- 172.21.15.64:0/4218857118 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f219c034070 con 0x7f21a4157b80 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21a3fff640 1 -- 172.21.15.64:0/4218857118 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f219c030840 con 0x7f21a4157b80 2024-03-04T07:26:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21a3fff640 1 -- 172.21.15.64:0/4218857118 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f219c030c10 con 0x7f21a4157b80 2024-03-04T07:26:49.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 -- 172.21.15.64:0/4218857118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 msgr2=0x7f21a415b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/4218857118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415b260 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f219c009870 tx=0x7f219c02b980 comp rx=0 tx=0).stop 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 -- 172.21.15.64:0/4218857118 shutdown_connections 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/4218857118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a415b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/4218857118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415b260 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 -- 172.21.15.64:0/4218857118 >> 172.21.15.64:0/4218857118 conn(0x7f21a4066d00 msgr2=0x7f21a40670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 -- 172.21.15.64:0/4218857118 shutdown_connections 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.757+0000 7f21ab8ed640 1 -- 172.21.15.64:0/4218857118 wait complete. 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.758+0000 7f21ab8ed640 1 Processor -- start 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.758+0000 7f21ab8ed640 1 -- start start 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21ab8ed640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40cbab0 0x7f21a40b3c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21ab8ed640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415aca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21ab8ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a40b2610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21ab8ed640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21a40cc1d0 con 0x7f21a4157b80 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21ab8ed640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21a40cc050 con 0x7f21a40cbab0 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21ab8ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21a4071250 con 0x7f21a4157f60 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21a8e88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415aca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21a8e88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415aca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41054/0 (socket says 172.21.15.64:41054) 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.759+0000 7f21a8e88640 1 -- 172.21.15.64:0/2285256235 learned_addr learned my addr 172.21.15.64:0/2285256235 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a8e88640 1 -- 172.21.15.64:0/2285256235 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 msgr2=0x7f21a40b2610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a8e88640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a40b2610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a8e88640 1 -- 172.21.15.64:0/2285256235 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40cbab0 msgr2=0x7f21a40b3c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a8e88640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40cbab0 0x7f21a40b3c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a8e88640 1 -- 172.21.15.64:0/2285256235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21a40b4520 con 0x7f21a4157b80 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a8e88640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415aca0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f219800c820 tx=0x7f219800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a1ffb640 1 -- 172.21.15.64:0/2285256235 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2198015070 con 0x7f21a4157b80 2024-03-04T07:26:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a1ffb640 1 -- 172.21.15.64:0/2285256235 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2198007a30 con 0x7f21a4157b80 2024-03-04T07:26:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21a40b4240 con 0x7f21a4157b80 2024-03-04T07:26:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21a1ffb640 1 -- 172.21.15.64:0/2285256235 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2198007de0 con 0x7f21a4157b80 2024-03-04T07:26:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.760+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21a4142a80 con 0x7f21a4157b80 2024-03-04T07:26:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.761+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f21a4067ff0 con 0x7f21a4157b80 2024-03-04T07:26:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.762+0000 7f21a1ffb640 1 -- 172.21.15.64:0/2285256235 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2198018070 con 0x7f21a4157b80 2024-03-04T07:26:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.762+0000 7f21a1ffb640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.762+0000 7f21a1ffb640 1 -- 172.21.15.64:0/2285256235 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f2198089d50 con 0x7f21a4157b80 2024-03-04T07:26:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.762+0000 7f21a9689640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.763+0000 7f21a9689640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f219c036000 tx=0x7f219c03b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:49.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.767+0000 7f21a1ffb640 1 -- 172.21.15.64:0/2285256235 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f219808d030 con 0x7f21a4157b80 2024-03-04T07:26:49.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.986+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f21a4159fb0 con 0x7f21a4157b80 2024-03-04T07:26:49.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.987+0000 7f21a1ffb640 1 -- 172.21.15.64:0/2285256235 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) v1 ==== 93+0+3691 (secure 0 0 0) 0x7f2198058fb0 con 0x7f21a4157b80 2024-03-04T07:26:49.988 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:49.988 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4293,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[]} 2024-03-04T07:26:49.988 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 3 2024-03-04T07:26:49.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 msgr2=0x7f219006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:49.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f219c036000 tx=0x7f219c03b040 comp rx=0 tx=0).stop 2024-03-04T07:26:49.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 msgr2=0x7f21a415aca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:49.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415aca0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f219800c820 tx=0x7f219800cce0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 shutdown_connections 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4157f60 0x7f21a40b2610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a4157b80 0x7f21a415aca0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 --2- 172.21.15.64:0/2285256235 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40cbab0 0x7f21a40b3c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 >> 172.21.15.64:0/2285256235 conn(0x7f21a4066d00 msgr2=0x7f21a4158dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 shutdown_connections 2024-03-04T07:26:49.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:49.989+0000 7f21ab8ed640 1 -- 172.21.15.64:0/2285256235 wait complete. 2024-03-04T07:26:50.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:50.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.101+0000 7fe551bbc640 1 Processor -- start 2024-03-04T07:26:50.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.101+0000 7fe551bbc640 1 -- start start 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.101+0000 7fe551bbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 0x7fe54c06af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.101+0000 7fe551bbc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe54c06ed10 0x7fe54c0cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.101+0000 7fe551bbc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe54c0cc0e0 con 0x7fe54c0cd6c0 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe551bbc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe54c0d28b0 con 0x7fe54c06ed10 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe551bbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe54c0cf320 con 0x7fe54c0b1020 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe54c0cd6c0 0x7fe54c15d700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36184/0 (socket says 172.21.15.64:36184) 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54bfff640 1 -- 172.21.15.64:0/4263592891 learned_addr learned my addr 172.21.15.64:0/4263592891 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54affd640 1 --2- 172.21.15.64:0/4263592891 >> v2:172.21.15.66:3300/0 conn(0x7fe54c06ed10 0x7fe54c0cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54b7fe640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 0x7fe54c06af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 418157341 0 0) 0x7fe54c0cc0e0 con 0x7fe54c0cd6c0 2024-03-04T07:26:50.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe534003520 con 0x7fe54c0cd6c0 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3977468136 0 0) 0x7fe534003520 con 0x7fe54c0cd6c0 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 msgr2=0x7fe54c06af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 0x7fe54c06af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 >> v2:172.21.15.66:3300/0 conn(0x7fe54c06ed10 msgr2=0x7fe54c0cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 --2- 172.21.15.64:0/4263592891 >> v2:172.21.15.66:3300/0 conn(0x7fe54c06ed10 0x7fe54c0cd180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe54c0cc3c0 con 0x7fe54c0cd6c0 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fe5440019f0 con 0x7fe54c0cd6c0 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 >> v1:172.21.15.64:6789/0 conn(0x7fe54c0cd6c0 legacy=0x7fe54c15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 0x7fe5340039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534003f30 0x7fe534004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe534004d40 0x7fe5340051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe54c0d28b0 con 0x7fe534004d40 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe54c0cf320 con 0x7fe534003f30 2024-03-04T07:26:50.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe53400b9c0 con 0x7fe54c0b1020 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54bfff640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe534004d40 0x7fe5340051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54b7fe640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 0x7fe5340039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.102+0000 7fe54affd640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534003f30 0x7fe534004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.103+0000 7fe54bfff640 1 -- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 msgr2=0x7fe5340039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.103+0000 7fe54bfff640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0b1020 0x7fe5340039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.103+0000 7fe54bfff640 1 -- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534003f30 msgr2=0x7fe534004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.103+0000 7fe54bfff640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534003f30 0x7fe534004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.103+0000 7fe54bfff640 1 -- 172.21.15.64:0/4263592891 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe534003520 con 0x7fe534004d40 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.103+0000 7fe54bfff640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe534004d40 0x7fe5340051c0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fe5440013d0 tx=0x7fe534005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e4634677d1c54172 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe54400a4e0 con 0x7fe534004d40 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe544034650 con 0x7fe534004d40 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe54a7fc640 1 -- 172.21.15.64:0/4263592891 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe544034a40 con 0x7fe534004d40 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe551bbc640 1 -- 172.21.15.64:0/4263592891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe534004d40 msgr2=0x7fe5340051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe551bbc640 1 --2- 172.21.15.64:0/4263592891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe534004d40 0x7fe5340051c0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fe5440013d0 tx=0x7fe534005830 comp rx=0 tx=0).stop 2024-03-04T07:26:50.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe54bfff640 1 -- 172.21.15.64:0/4263592891 reap_dead start 2024-03-04T07:26:50.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe551bbc640 1 -- 172.21.15.64:0/4263592891 shutdown_connections 2024-03-04T07:26:50.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe551bbc640 1 -- 172.21.15.64:0/4263592891 >> 172.21.15.64:0/4263592891 conn(0x7fe54c0d1a90 msgr2=0x7fe54c0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:50.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe551bbc640 1 -- 172.21.15.64:0/4263592891 shutdown_connections 2024-03-04T07:26:50.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.104+0000 7fe551bbc640 1 -- 172.21.15.64:0/4263592891 wait complete. 2024-03-04T07:26:50.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.105+0000 7fe551bbc640 1 Processor -- start 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.105+0000 7fe551bbc640 1 -- start start 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe551bbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534004d40 0x7fe54c158700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe551bbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe54c1594c0 0x7fe54c15f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe551bbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0cbb50 0x7fe54c15f680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe551bbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe54c0ccc60 con 0x7fe54c1594c0 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe551bbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe54c0cc980 con 0x7fe534004d40 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe551bbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe54c06af20 con 0x7fe54c0cbb50 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe54c1594c0 0x7fe54c15f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe54c1594c0 0x7fe54c15f140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41066/0 (socket says 172.21.15.64:41066) 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54affd640 1 -- 172.21.15.64:0/1719811514 learned_addr learned my addr 172.21.15.64:0/1719811514 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54affd640 1 -- 172.21.15.64:0/1719811514 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0cbb50 msgr2=0x7fe54c15f680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54affd640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0cbb50 0x7fe54c15f680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54affd640 1 -- 172.21.15.64:0/1719811514 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534004d40 msgr2=0x7fe54c158700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54b7fe640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534004d40 0x7fe54c158700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.106+0000 7fe54affd640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534004d40 0x7fe54c158700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.107+0000 7fe54affd640 1 -- 172.21.15.64:0/1719811514 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe54c0c8900 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.107+0000 7fe54affd640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe54c1594c0 0x7fe54c15f140 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fe53c0118e0 tx=0x7fe53c011da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.107+0000 7fe533fff640 1 -- 172.21.15.64:0/1719811514 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe53c00faf0 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.107+0000 7fe533fff640 1 -- 172.21.15.64:0/1719811514 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe53c00fc90 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.107+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe54c158c40 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.107+0000 7fe533fff640 1 -- 172.21.15.64:0/1719811514 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe53c014440 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.107+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe54c0c9a30 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.108+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe54c067ff0 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.108+0000 7fe533fff640 1 -- 172.21.15.64:0/1719811514 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe53c012040 con 0x7fe54c1594c0 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.108+0000 7fe533fff640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe53806b500 0x7fe53806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.109+0000 7fe533fff640 1 -- 172.21.15.64:0/1719811514 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fe53c08a590 con 0x7fe54c1594c0 2024-03-04T07:26:50.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.109+0000 7fe54b7fe640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe53806b500 0x7fe53806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.109+0000 7fe54b7fe640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe53806b500 0x7fe53806d9c0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fe5400099e0 tx=0x7fe540008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.113+0000 7fe533fff640 1 -- 172.21.15.64:0/1719811514 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe53c019030 con 0x7fe54c1594c0 2024-03-04T07:26:50.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.296+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe54c15e5f0 con 0x7fe54c1594c0 2024-03-04T07:26:50.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.296+0000 7fe533fff640 1 -- 172.21.15.64:0/1719811514 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v16) v1 ==== 74+0+12836 (secure 0 0 0) 0x7fe53c059110 con 0x7fe54c1594c0 2024-03-04T07:26:50.297 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:50.297 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":16,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:43.215051+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:50.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe53806b500 msgr2=0x7fe53806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe53806b500 0x7fe53806d9c0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fe5400099e0 tx=0x7fe540008040 comp rx=0 tx=0).stop 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe54c1594c0 msgr2=0x7fe54c15f140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe54c1594c0 0x7fe54c15f140 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fe53c0118e0 tx=0x7fe53c011da0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 shutdown_connections 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe53806b500 0x7fe53806d9c0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe54c0cbb50 0x7fe54c15f680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe54c1594c0 0x7fe54c15f140 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 --2- 172.21.15.64:0/1719811514 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe534004d40 0x7fe54c158700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 >> 172.21.15.64:0/1719811514 conn(0x7fe54c0d1a90 msgr2=0x7fe54c15e830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 shutdown_connections 2024-03-04T07:26:50.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.299+0000 7fe551bbc640 1 -- 172.21.15.64:0/1719811514 wait complete. 2024-03-04T07:26:50.310 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:26:50.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:26:50.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.414+0000 7f4d6a9b5640 1 Processor -- start 2024-03-04T07:26:50.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.414+0000 7f4d6a9b5640 1 -- start start 2024-03-04T07:26:50.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d6a9b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4d640717b0 0x7f4d64071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d6a9b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640b3060 0x7f4d640d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d6a9b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4d640b2b10 0x7f4d640b6e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d6a9b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4d640d2e10 con 0x7f4d640b2b10 2024-03-04T07:26:50.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d6a9b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4d6415e4b0 con 0x7f4d640717b0 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d6a9b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d640b97e0 con 0x7f4d640b3060 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4d640b2b10 0x7f4d640b6e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4d640b2b10 0x7f4d640b6e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41082/0 (socket says 172.21.15.64:41082) 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 -- 172.21.15.64:0/2083421364 learned_addr learned my addr 172.21.15.64:0/2083421364 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d63fff640 1 --2- 172.21.15.64:0/2083421364 >> v2:172.21.15.66:3300/0 conn(0x7f4d640717b0 0x7f4d64071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d637fe640 1 --2- 172.21.15.64:0/2083421364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640b3060 0x7f4d640d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 -- 172.21.15.64:0/2083421364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640b3060 msgr2=0x7f4d640d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 --2- 172.21.15.64:0/2083421364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640b3060 0x7f4d640d32a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 -- 172.21.15.64:0/2083421364 >> v2:172.21.15.66:3300/0 conn(0x7f4d640717b0 msgr2=0x7f4d64071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 --2- 172.21.15.64:0/2083421364 >> v2:172.21.15.66:3300/0 conn(0x7f4d640717b0 0x7f4d64071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d68f52640 1 -- 172.21.15.64:0/2083421364 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d640b9a80 con 0x7f4d640b2b10 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d63fff640 1 --2- 172.21.15.64:0/2083421364 >> v2:172.21.15.66:3300/0 conn(0x7f4d640717b0 0x7f4d64071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.415+0000 7f4d637fe640 1 --2- 172.21.15.64:0/2083421364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640b3060 0x7f4d640d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:50.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.416+0000 7f4d68f52640 1 --2- 172.21.15.64:0/2083421364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b2b10 0x7f4d640b6e50 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f4d6404fd20 tx=0x7f4d5c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dae4217c8317d481 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.416+0000 7f4d62ffd640 1 -- 172.21.15.64:0/2083421364 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d5c02ca70 con 0x7f4d640b2b10 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.416+0000 7f4d62ffd640 1 -- 172.21.15.64:0/2083421364 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d5c02cc10 con 0x7f4d640b2b10 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.416+0000 7f4d62ffd640 1 -- 172.21.15.64:0/2083421364 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d5c0343f0 con 0x7f4d640b2b10 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/2083421364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b2b10 msgr2=0x7f4d640b6e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/2083421364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b2b10 0x7f4d640b6e50 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f4d6404fd20 tx=0x7f4d5c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/2083421364 shutdown_connections 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/2083421364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b2b10 0x7f4d640b6e50 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/2083421364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640b3060 0x7f4d640d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/2083421364 >> v2:172.21.15.66:3300/0 conn(0x7f4d640717b0 0x7f4d64071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/2083421364 >> 172.21.15.64:0/2083421364 conn(0x7f4d640d3d20 msgr2=0x7f4d640d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/2083421364 shutdown_connections 2024-03-04T07:26:50.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.417+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/2083421364 wait complete. 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.418+0000 7f4d6a9b5640 1 Processor -- start 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.418+0000 7f4d6a9b5640 1 -- start start 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640717b0 0x7f4d640c9790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d640b2b10 0x7f4d641e6c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b3060 0x7f4d641e71a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d640b7390 con 0x7f4d640b3060 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d640d4bf0 con 0x7f4d640b2b10 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d640b7510 con 0x7f4d640717b0 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b3060 0x7f4d641e71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b3060 0x7f4d641e71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41092/0 (socket says 172.21.15.64:41092) 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 -- 172.21.15.64:0/3750407608 learned_addr learned my addr 172.21.15.64:0/3750407608 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 -- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640717b0 msgr2=0x7f4d640c9790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d63fff640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640717b0 0x7f4d640c9790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d637fe640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d640b2b10 0x7f4d641e6c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640717b0 0x7f4d640c9790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 -- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d640b2b10 msgr2=0x7f4d641e6c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d640b2b10 0x7f4d641e6c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 -- 172.21.15.64:0/3750407608 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d641e4d00 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d68f52640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b3060 0x7f4d641e71a0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f4d5c02c4e0 tx=0x7f4d5c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d60ff9640 1 -- 172.21.15.64:0/3750407608 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d5c043020 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d60ff9640 1 -- 172.21.15.64:0/3750407608 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d5c03e450 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d640cd770 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d60ff9640 1 -- 172.21.15.64:0/3750407608 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d5c03e840 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.419+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4d641e5280 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.420+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4d6406ce40 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.421+0000 7f4d60ff9640 1 -- 172.21.15.64:0/3750407608 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4d5c046020 con 0x7f4d640b3060 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.421+0000 7f4d60ff9640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5006b4d0 0x7f4d5006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.421+0000 7f4d63fff640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5006b4d0 0x7f4d5006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.421+0000 7f4d60ff9640 1 -- 172.21.15.64:0/3750407608 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f4d5c0ac870 con 0x7f4d640b3060 2024-03-04T07:26:50.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.422+0000 7f4d63fff640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5006b4d0 0x7f4d5006d990 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f4d54006fd0 tx=0x7f4d54008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.426+0000 7f4d60ff9640 1 -- 172.21.15.64:0/3750407608 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4d5c04d030 con 0x7f4d640b3060 2024-03-04T07:26:50.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.606+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f4d6404f410 con 0x7f4d640b3060 2024-03-04T07:26:50.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.606+0000 7f4d60ff9640 1 -- 172.21.15.64:0/3750407608 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v16) v1 ==== 93+0+10 (secure 0 0 0) 0x7f4d5c07ba40 con 0x7f4d640b3060 2024-03-04T07:26:50.607 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5006b4d0 msgr2=0x7f4d5006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5006b4d0 0x7f4d5006d990 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f4d54006fd0 tx=0x7f4d54008040 comp rx=0 tx=0).stop 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b3060 msgr2=0x7f4d641e71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b3060 0x7f4d641e71a0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f4d5c02c4e0 tx=0x7f4d5c004a40 comp rx=0 tx=0).stop 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 shutdown_connections 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5006b4d0 0x7f4d5006d990 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d640b3060 0x7f4d641e71a0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d640b2b10 0x7f4d641e6c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 --2- 172.21.15.64:0/3750407608 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d640717b0 0x7f4d640c9790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 >> 172.21.15.64:0/3750407608 conn(0x7f4d640d3d20 msgr2=0x7f4d640d30f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:50.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 shutdown_connections 2024-03-04T07:26:50.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.608+0000 7f4d6a9b5640 1 -- 172.21.15.64:0/3750407608 wait complete. 2024-03-04T07:26:50.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:26:50.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.720+0000 7f1a31dc9640 1 Processor -- start 2024-03-04T07:26:50.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.720+0000 7f1a31dc9640 1 -- start start 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a31dc9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1a2c0b95a0 0x7f1a2c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a31dc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c0b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a31dc9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1a2c0cea30 con 0x7f1a2c15df00 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a31dc9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1a2c0cf3c0 con 0x7f1a2c0b95a0 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a31dc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a2c0cf0e0 con 0x7f1a2c0b9a10 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2b7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1a2c15df00 0x7f1a2c0d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36194/0 (socket says 172.21.15.64:36194) 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2b7fe640 1 -- 172.21.15.64:0/1914314398 learned_addr learned my addr 172.21.15.64:0/1914314398 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2bfff640 1 --2- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c0b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2affd640 1 --2- 172.21.15.64:0/1914314398 >> v2:172.21.15.66:3300/0 conn(0x7f1a2c0b95a0 0x7f1a2c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2a7fc640 1 -- 172.21.15.64:0/1914314398 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3568327051 0 0) 0x7f1a2c0cea30 con 0x7f1a2c15df00 2024-03-04T07:26:50.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2a7fc640 1 -- 172.21.15.64:0/1914314398 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1a14003520 con 0x7f1a2c15df00 2024-03-04T07:26:50.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2affd640 1 -- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 msgr2=0x7f1a2c0b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:50.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2affd640 1 --2- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c0b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2affd640 1 -- 172.21.15.64:0/1914314398 >> v1:172.21.15.64:6789/0 conn(0x7f1a2c15df00 legacy=0x7f1a2c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2affd640 1 -- 172.21.15.64:0/1914314398 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a2c0ce470 con 0x7f1a2c0b95a0 2024-03-04T07:26:50.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.721+0000 7f1a2bfff640 1 --2- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c0b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.722+0000 7f1a2affd640 1 --2- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 0x7f1a2c0b5700 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f1a200097a0 tx=0x7f1a2002b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5081df473acd38b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.722+0000 7f1a2a7fc640 1 -- 172.21.15.64:0/1914314398 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a20035020 con 0x7f1a2c0b95a0 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.722+0000 7f1a2a7fc640 1 -- 172.21.15.64:0/1914314398 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a20031810 con 0x7f1a2c0b95a0 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.722+0000 7f1a2a7fc640 1 -- 172.21.15.64:0/1914314398 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a20031bc0 con 0x7f1a2c0b95a0 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 -- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 msgr2=0x7f1a2c0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 0x7f1a2c0b5700 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f1a200097a0 tx=0x7f1a2002b8b0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 -- 172.21.15.64:0/1914314398 shutdown_connections 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c0b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/1914314398 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 0x7f1a2c0b5700 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 -- 172.21.15.64:0/1914314398 >> 172.21.15.64:0/1914314398 conn(0x7f1a2c071960 msgr2=0x7f1a2c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 -- 172.21.15.64:0/1914314398 shutdown_connections 2024-03-04T07:26:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.723+0000 7f1a31dc9640 1 -- 172.21.15.64:0/1914314398 wait complete. 2024-03-04T07:26:50.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.724+0000 7f1a31dc9640 1 Processor -- start 2024-03-04T07:26:50.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.724+0000 7f1a31dc9640 1 -- start start 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a31dc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 0x7f1a2c1f4560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a31dc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c15ee60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a31dc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a2c15df00 0x7f1a2c159420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a31dc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a2c0cedf0 con 0x7f1a2c15df00 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a31dc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a2c0cec70 con 0x7f1a2c0b95a0 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a31dc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a2c0d3220 con 0x7f1a2c0b9a10 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a2c15df00 0x7f1a2c159420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a2c15df00 0x7f1a2c159420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41108/0 (socket says 172.21.15.64:41108) 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 -- 172.21.15.64:0/4138905046 learned_addr learned my addr 172.21.15.64:0/4138905046 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2affd640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c15ee60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2b7fe640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 0x7f1a2c1f4560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 -- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 msgr2=0x7f1a2c15ee60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c15ee60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 -- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 msgr2=0x7f1a2c1f4560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 0x7f1a2c1f4560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.725+0000 7f1a2bfff640 1 -- 172.21.15.64:0/4138905046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a2c0cc9b0 con 0x7f1a2c15df00 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.726+0000 7f1a2bfff640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a2c15df00 0x7f1a2c159420 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f1a24009f00 tx=0x7f1a2400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.726+0000 7f1a13fff640 1 -- 172.21.15.64:0/4138905046 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a24018030 con 0x7f1a2c15df00 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.726+0000 7f1a13fff640 1 -- 172.21.15.64:0/4138905046 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a2400ce50 con 0x7f1a2c15df00 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.726+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1a2c0ccc90 con 0x7f1a2c15df00 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.726+0000 7f1a13fff640 1 -- 172.21.15.64:0/4138905046 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a24003bb0 con 0x7f1a2c15df00 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.726+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a2c15e660 con 0x7f1a2c15df00 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.727+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1a2c067ff0 con 0x7f1a2c15df00 2024-03-04T07:26:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.727+0000 7f1a13fff640 1 -- 172.21.15.64:0/4138905046 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1a2401b070 con 0x7f1a2c15df00 2024-03-04T07:26:50.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.728+0000 7f1a13fff640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a1806b500 0x7f1a1806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:50.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.728+0000 7f1a13fff640 1 -- 172.21.15.64:0/4138905046 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f1a2408a780 con 0x7f1a2c15df00 2024-03-04T07:26:50.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.728+0000 7f1a2b7fe640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a1806b500 0x7f1a1806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:50.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.728+0000 7f1a2b7fe640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a1806b500 0x7f1a1806d9c0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f1a1c005880 tx=0x7f1a1c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:50.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.732+0000 7f1a13fff640 1 -- 172.21.15.64:0/4138905046 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1a24010040 con 0x7f1a2c15df00 2024-03-04T07:26:50.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.913+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f1a2c0b8bb0 con 0x7f1a2c15df00 2024-03-04T07:26:50.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.914+0000 7f1a13fff640 1 -- 172.21.15.64:0/4138905046 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v16) v1 ==== 84+0+65 (secure 0 0 0) 0x7f1a24059300 con 0x7f1a2c15df00 2024-03-04T07:26:50.915 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:50.915 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:26:50.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:26:50.915 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:26:50.915 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:26:50.915 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:26:50.915 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a1806b500 msgr2=0x7f1a1806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a1806b500 0x7f1a1806d9c0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f1a1c005880 tx=0x7f1a1c009040 comp rx=0 tx=0).stop 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a2c15df00 msgr2=0x7f1a2c159420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a2c15df00 0x7f1a2c159420 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f1a24009f00 tx=0x7f1a2400c490 comp rx=0 tx=0).stop 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 shutdown_connections 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a1806b500 0x7f1a1806d9c0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a2c15df00 0x7f1a2c159420 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a2c0b9a10 0x7f1a2c15ee60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 --2- 172.21.15.64:0/4138905046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a2c0b95a0 0x7f1a2c1f4560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 >> 172.21.15.64:0/4138905046 conn(0x7f1a2c071960 msgr2=0x7f1a2c0b3d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 shutdown_connections 2024-03-04T07:26:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:50.916+0000 7f1a31dc9640 1 -- 172.21.15.64:0/4138905046 wait complete. 2024-03-04T07:26:50.927 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:26:50.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:26:51.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.031+0000 7f6ce7888640 1 Processor -- start 2024-03-04T07:26:51.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.031+0000 7f6ce7888640 1 -- start start 2024-03-04T07:26:51.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce7888640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6ce015bca0 0x7f6ce00d01b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce7888640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd440 0x7f6ce00b4a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce7888640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ce00cd880 0x7f6ce00b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce7888640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ce00d0b70 con 0x7f6ce00cd880 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce7888640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ce00ccfb0 con 0x7f6ce015bca0 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce7888640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce00cc6e0 con 0x7f6ce00cd440 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce5e25640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ce00cd880 0x7f6ce00b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce5e25640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ce00cd880 0x7f6ce00b4f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41120/0 (socket says 172.21.15.64:41120) 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.032+0000 7f6ce5e25640 1 -- 172.21.15.64:0/3466491368 learned_addr learned my addr 172.21.15.64:0/3466491368 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce4e23640 1 --2- 172.21.15.64:0/3466491368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd440 0x7f6ce00b4a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce5624640 1 --2- 172.21.15.64:0/3466491368 >> v2:172.21.15.66:3300/0 conn(0x7f6ce015bca0 0x7f6ce00d01b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce5e25640 1 -- 172.21.15.64:0/3466491368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd440 msgr2=0x7f6ce00b4a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce5e25640 1 --2- 172.21.15.64:0/3466491368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd440 0x7f6ce00b4a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce5e25640 1 -- 172.21.15.64:0/3466491368 >> v2:172.21.15.66:3300/0 conn(0x7f6ce015bca0 msgr2=0x7f6ce00d01b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce5e25640 1 --2- 172.21.15.64:0/3466491368 >> v2:172.21.15.66:3300/0 conn(0x7f6ce015bca0 0x7f6ce00d01b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce5e25640 1 -- 172.21.15.64:0/3466491368 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ce00b54a0 con 0x7f6ce00cd880 2024-03-04T07:26:51.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6ce5e25640 1 --2- 172.21.15.64:0/3466491368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce00cd880 0x7f6ce00b4f60 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f6ce00726f0 tx=0x7f6cd802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d37b1d35a18c3b61 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:51.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6cdffff640 1 -- 172.21.15.64:0/3466491368 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cd8035020 con 0x7f6ce00cd880 2024-03-04T07:26:51.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6cdffff640 1 -- 172.21.15.64:0/3466491368 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cd8031870 con 0x7f6ce00cd880 2024-03-04T07:26:51.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.033+0000 7f6cdffff640 1 -- 172.21.15.64:0/3466491368 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cd8031c40 con 0x7f6ce00cd880 2024-03-04T07:26:51.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 -- 172.21.15.64:0/3466491368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce00cd880 msgr2=0x7f6ce00b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 --2- 172.21.15.64:0/3466491368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce00cd880 0x7f6ce00b4f60 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f6ce00726f0 tx=0x7f6cd802b910 comp rx=0 tx=0).stop 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 -- 172.21.15.64:0/3466491368 shutdown_connections 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 --2- 172.21.15.64:0/3466491368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce00cd880 0x7f6ce00b4f60 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 --2- 172.21.15.64:0/3466491368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd440 0x7f6ce00b4a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 --2- 172.21.15.64:0/3466491368 >> v2:172.21.15.66:3300/0 conn(0x7f6ce015bca0 0x7f6ce00d01b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 -- 172.21.15.64:0/3466491368 >> 172.21.15.64:0/3466491368 conn(0x7f6ce00d1bd0 msgr2=0x7f6ce00d1fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 -- 172.21.15.64:0/3466491368 shutdown_connections 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.034+0000 7f6ce7888640 1 -- 172.21.15.64:0/3466491368 wait complete. 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.035+0000 7f6ce7888640 1 Processor -- start 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.035+0000 7f6ce7888640 1 -- start start 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce7888640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ce00cd440 0x7f6ce00b18e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce7888640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd880 0x7f6ce00af290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce7888640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce015bca0 0x7f6ce01f3970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce7888640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce00ccc40 con 0x7f6ce015bca0 2024-03-04T07:26:51.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce7888640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce00cc960 con 0x7f6ce00cd440 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce7888640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce015c080 con 0x7f6ce00cd880 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce015bca0 0x7f6ce01f3970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce015bca0 0x7f6ce01f3970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41126/0 (socket says 172.21.15.64:41126) 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 -- 172.21.15.64:0/961832747 learned_addr learned my addr 172.21.15.64:0/961832747 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce4e23640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd880 0x7f6ce00af290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5624640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ce00cd440 0x7f6ce00b18e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 -- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd880 msgr2=0x7f6ce00af290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd880 0x7f6ce00af290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 -- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ce00cd440 msgr2=0x7f6ce00b18e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ce00cd440 0x7f6ce00b18e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.036+0000 7f6ce5e25640 1 -- 172.21.15.64:0/961832747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ce00b0440 con 0x7f6ce015bca0 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.037+0000 7f6ce5e25640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce015bca0 0x7f6ce01f3970 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f6cd802b950 tx=0x7f6cd80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.037+0000 7f6cddffb640 1 -- 172.21.15.64:0/961832747 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cd8040070 con 0x7f6ce015bca0 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.037+0000 7f6cddffb640 1 -- 172.21.15.64:0/961832747 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cd803a420 con 0x7f6ce015bca0 2024-03-04T07:26:51.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.037+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ce00afba0 con 0x7f6ce015bca0 2024-03-04T07:26:51.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.037+0000 7f6cddffb640 1 -- 172.21.15.64:0/961832747 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cd8035040 con 0x7f6ce015bca0 2024-03-04T07:26:51.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.037+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ce00b0160 con 0x7f6ce015bca0 2024-03-04T07:26:51.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.038+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ce00cf0a0 con 0x7f6ce015bca0 2024-03-04T07:26:51.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.038+0000 7f6cddffb640 1 -- 172.21.15.64:0/961832747 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6cd8038020 con 0x7f6ce015bca0 2024-03-04T07:26:51.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.038+0000 7f6cddffb640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ccc06b4d0 0x7f6ccc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.039+0000 7f6cddffb640 1 -- 172.21.15.64:0/961832747 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(16..16 src has 1..16) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f6cd80abe10 con 0x7f6ce015bca0 2024-03-04T07:26:51.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.039+0000 7f6ce5624640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ccc06b4d0 0x7f6ccc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.039+0000 7f6ce5624640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ccc06b4d0 0x7f6ccc06d990 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f6cd0008e50 tx=0x7f6cd00083a0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:51.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.043+0000 7f6cddffb640 1 -- 172.21.15.64:0/961832747 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6ce00b54a0 con 0x7f6ce015bca0 2024-03-04T07:26:51.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.224+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f6ce004f410 con 0x7f6ce015bca0 2024-03-04T07:26:51.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.295+0000 7f6cddffb640 1 -- 172.21.15.64:0/961832747 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v17) v1 ==== 138+0+0 (secure 0 0 0) 0x7f6cd807b070 con 0x7f6ce015bca0 2024-03-04T07:26:51.295 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:26:51.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ccc06b4d0 msgr2=0x7f6ccc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ccc06b4d0 0x7f6ccc06d990 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f6cd0008e50 tx=0x7f6cd00083a0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce015bca0 msgr2=0x7f6ce01f3970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce015bca0 0x7f6ce01f3970 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f6cd802b950 tx=0x7f6cd80046c0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 shutdown_connections 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ccc06b4d0 0x7f6ccc06d990 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ce015bca0 0x7f6ce01f3970 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ce00cd880 0x7f6ce00af290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 --2- 172.21.15.64:0/961832747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ce00cd440 0x7f6ce00b18e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 >> 172.21.15.64:0/961832747 conn(0x7f6ce00d1bd0 msgr2=0x7f6ce00cb170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 shutdown_connections 2024-03-04T07:26:51.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.297+0000 7f6ce7888640 1 -- 172.21.15.64:0/961832747 wait complete. 2024-03-04T07:26:51.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:26:51.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.418+0000 7fa66f481640 1 Processor -- start 2024-03-04T07:26:51.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.419+0000 7fa66f481640 1 -- start start 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.419+0000 7fa66f481640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa66815e000 0x7fa6680d1e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.419+0000 7fa66f481640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6680d37a0 0x7fa6680ceae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.419+0000 7fa66f481640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa6680d2700 con 0x7fa6680b9710 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.419+0000 7fa66f481640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6680b7ec0 con 0x7fa6680d37a0 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.419+0000 7fa66f481640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6680d29e0 con 0x7fa66815e000 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66ca1c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6680d37a0 0x7fa6680ceae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66ca1c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6680d37a0 0x7fa6680ceae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41136/0 (socket says 172.21.15.64:41136) 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66da1e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa6680b9710 0x7fa6680b5790 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36210/0 (socket says 172.21.15.64:36210) 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66ca1c640 1 -- 172.21.15.64:0/2648523743 learned_addr learned my addr 172.21.15.64:0/2648523743 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66d21d640 1 --2- 172.21.15.64:0/2648523743 >> v2:172.21.15.66:3300/0 conn(0x7fa66815e000 0x7fa6680d1e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66ca1c640 1 -- 172.21.15.64:0/2648523743 >> v2:172.21.15.66:3300/0 conn(0x7fa66815e000 msgr2=0x7fa6680d1e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66ca1c640 1 --2- 172.21.15.64:0/2648523743 >> v2:172.21.15.66:3300/0 conn(0x7fa66815e000 0x7fa6680d1e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66ca1c640 1 -- 172.21.15.64:0/2648523743 >> v1:172.21.15.64:6789/0 conn(0x7fa6680b9710 legacy=0x7fa6680b5790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa65ffff640 1 -- 172.21.15.64:0/2648523743 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1683801556 0 0) 0x7fa6680d2700 con 0x7fa6680b9710 2024-03-04T07:26:51.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.420+0000 7fa66ca1c640 1 -- 172.21.15.64:0/2648523743 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6680d2cc0 con 0x7fa6680d37a0 2024-03-04T07:26:51.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66ca1c640 1 --2- 172.21.15.64:0/2648523743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6680d37a0 0x7fa6680ceae0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fa6680d4e10 tx=0x7fa65802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9b7848e37e146aa server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:51.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa65ffff640 1 -- 172.21.15.64:0/2648523743 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa658034070 con 0x7fa6680d37a0 2024-03-04T07:26:51.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa65ffff640 1 -- 172.21.15.64:0/2648523743 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa658030860 con 0x7fa6680d37a0 2024-03-04T07:26:51.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa65ffff640 1 -- 172.21.15.64:0/2648523743 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa658030c30 con 0x7fa6680d37a0 2024-03-04T07:26:51.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 -- 172.21.15.64:0/2648523743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6680d37a0 msgr2=0x7fa6680ceae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 --2- 172.21.15.64:0/2648523743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6680d37a0 0x7fa6680ceae0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fa6680d4e10 tx=0x7fa65802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 -- 172.21.15.64:0/2648523743 shutdown_connections 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 --2- 172.21.15.64:0/2648523743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6680d37a0 0x7fa6680ceae0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 --2- 172.21.15.64:0/2648523743 >> v2:172.21.15.66:3300/0 conn(0x7fa66815e000 0x7fa6680d1e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 -- 172.21.15.64:0/2648523743 >> 172.21.15.64:0/2648523743 conn(0x7fa668066d00 msgr2=0x7fa668067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 -- 172.21.15.64:0/2648523743 shutdown_connections 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.421+0000 7fa66f481640 1 -- 172.21.15.64:0/2648523743 wait complete. 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 Processor -- start 2024-03-04T07:26:51.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 -- start start 2024-03-04T07:26:51.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6680b9710 0x7fa6680c94f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6680d37a0 0x7fa6680caf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa66815e000 0x7fa66815ea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa66815e3e0 con 0x7fa66815e000 2024-03-04T07:26:51.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6680d3280 con 0x7fa6680d37a0 2024-03-04T07:26:51.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66f481640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6680d2340 con 0x7fa6680b9710 2024-03-04T07:26:51.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66da1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa66815e000 0x7fa66815ea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66da1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa66815e000 0x7fa66815ea00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41140/0 (socket says 172.21.15.64:41140) 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66da1e640 1 -- 172.21.15.64:0/1319966208 learned_addr learned my addr 172.21.15.64:0/1319966208 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.423+0000 7fa66ca1c640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6680d37a0 0x7fa6680caf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66d21d640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6680b9710 0x7fa6680c94f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66da1e640 1 -- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6680b9710 msgr2=0x7fa6680c94f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66da1e640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6680b9710 0x7fa6680c94f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66da1e640 1 -- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6680d37a0 msgr2=0x7fa6680caf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66da1e640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6680d37a0 0x7fa6680caf00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66da1e640 1 -- 172.21.15.64:0/1319966208 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa668155a00 con 0x7fa66815e000 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66da1e640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa66815e000 0x7fa66815ea00 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa66000c780 tx=0x7fa66000cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa65dffb640 1 -- 172.21.15.64:0/1319966208 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa660018070 con 0x7fa66815e000 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa65dffb640 1 -- 172.21.15.64:0/1319966208 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa66000a640 con 0x7fa66815e000 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa668155ce0 con 0x7fa66815e000 2024-03-04T07:26:51.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa65dffb640 1 -- 172.21.15.64:0/1319966208 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa66000aab0 con 0x7fa66815e000 2024-03-04T07:26:51.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.424+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6680cb8f0 con 0x7fa66815e000 2024-03-04T07:26:51.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.425+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa668072a30 con 0x7fa66815e000 2024-03-04T07:26:51.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.426+0000 7fa65dffb640 1 -- 172.21.15.64:0/1319966208 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa66001b070 con 0x7fa66815e000 2024-03-04T07:26:51.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.426+0000 7fa65dffb640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa64c06b4d0 0x7fa64c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:51.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.426+0000 7fa65dffb640 1 -- 172.21.15.64:0/1319966208 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 5227+0+0 (secure 0 0 0) 0x7fa660089500 con 0x7fa66815e000 2024-03-04T07:26:51.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.426+0000 7fa66d21d640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa64c06b4d0 0x7fa64c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:51.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.426+0000 7fa66d21d640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa64c06b4d0 0x7fa64c06d990 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fa66815c250 tx=0x7fa664008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:51.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.431+0000 7fa65dffb640 1 -- 172.21.15.64:0/1319966208 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa660057fa0 con 0x7fa66815e000 2024-03-04T07:26:51.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:51.610+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fa668161590 con 0x7fa66815e000 2024-03-04T07:26:52.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.298+0000 7fa65dffb640 1 -- 172.21.15.64:0/1319966208 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v18) v1 ==== 170+0+0 (secure 0 0 0) 0x7fa66005c040 con 0x7fa66815e000 2024-03-04T07:26:52.299 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa64c06b4d0 msgr2=0x7fa64c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa64c06b4d0 0x7fa64c06d990 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fa66815c250 tx=0x7fa664008040 comp rx=0 tx=0).stop 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa66815e000 msgr2=0x7fa66815ea00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa66815e000 0x7fa66815ea00 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa66000c780 tx=0x7fa66000cc40 comp rx=0 tx=0).stop 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 shutdown_connections 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa64c06b4d0 0x7fa64c06d990 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa66815e000 0x7fa66815ea00 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6680d37a0 0x7fa6680caf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 --2- 172.21.15.64:0/1319966208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6680b9710 0x7fa6680c94f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 >> 172.21.15.64:0/1319966208 conn(0x7fa668066d00 msgr2=0x7fa66815c720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:52.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 shutdown_connections 2024-03-04T07:26:52.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.301+0000 7fa66f481640 1 -- 172.21.15.64:0/1319966208 wait complete. 2024-03-04T07:26:52.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:26:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.428+0000 7f6dc03f6640 1 Processor -- start 2024-03-04T07:26:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.429+0000 7f6dc03f6640 1 -- start start 2024-03-04T07:26:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.429+0000 7f6dc03f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 0x7f6db8072070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.429+0000 7f6dc03f6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6db80ce900 0x7f6db80ced60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.429+0000 7f6dc03f6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6db80b3d30 con 0x7f6db815b150 2024-03-04T07:26:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.429+0000 7f6dc03f6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6db80cf2a0 con 0x7f6db80ce900 2024-03-04T07:26:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.429+0000 7f6dc03f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6db806dbb0 con 0x7f6db80b4700 2024-03-04T07:26:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd991640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6db815b150 0x7f6db80b3620 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36220/0 (socket says 172.21.15.64:36220) 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd991640 1 -- 172.21.15.64:0/3219270790 learned_addr learned my addr 172.21.15.64:0/3219270790 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbe192640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 0x7f6db8072070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbe993640 1 --2- 172.21.15.64:0/3219270790 >> v2:172.21.15.66:3300/0 conn(0x7f6db80ce900 0x7f6db80ced60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1605087073 0 0) 0x7f6da8001110 con 0x7f6db815b150 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6da4003520 con 0x7f6db815b150 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2575308793 0 0) 0x7f6da4003520 con 0x7f6db815b150 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 msgr2=0x7f6db8072070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 0x7f6db8072070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 >> v2:172.21.15.66:3300/0 conn(0x7f6db80ce900 msgr2=0x7f6db80ced60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 --2- 172.21.15.64:0/3219270790 >> v2:172.21.15.66:3300/0 conn(0x7f6db80ce900 0x7f6db80ced60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6db815b980 con 0x7f6db815b150 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f6da8001840 con 0x7f6db815b150 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 >> v1:172.21.15.64:6789/0 conn(0x7f6db815b150 legacy=0x7f6db80b3620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 0x7f6da40039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6da4003f30 0x7f6da4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4004d40 0x7f6da40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6db806dbb0 con 0x7f6da4003f30 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6da400b9c0 con 0x7f6da4004d40 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6da400bb40 con 0x7f6db80b4700 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbd991640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6da4003f30 0x7f6da4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbe192640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 0x7f6da40039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.430+0000 7f6dbe993640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4004d40 0x7f6da40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd991640 1 -- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 msgr2=0x7f6da40039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd991640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db80b4700 0x7f6da40039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd991640 1 -- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4004d40 msgr2=0x7f6da40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd991640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4004d40 0x7f6da40051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd991640 1 -- 172.21.15.64:0/3219270790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6da4003520 con 0x7f6da4003f30 2024-03-04T07:26:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbe993640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4004d40 0x7f6da40051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd991640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6da4003f30 0x7f6da4004800 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f6da800c3b0 tx=0x7f6da4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e92d0b9b73f9aec4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6da800c0e0 con 0x7f6da4003f30 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6da8030070 con 0x7f6da4003f30 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.431+0000 7f6dbd190640 1 -- 172.21.15.64:0/3219270790 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6da80058d0 con 0x7f6da4003f30 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.432+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3219270790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6da4003f30 msgr2=0x7f6da4004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.432+0000 7f6dc03f6640 1 --2- 172.21.15.64:0/3219270790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6da4003f30 0x7f6da4004800 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f6da800c3b0 tx=0x7f6da4005830 comp rx=0 tx=0).stop 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.432+0000 7f6dbe993640 1 -- 172.21.15.64:0/3219270790 reap_dead start 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.432+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3219270790 shutdown_connections 2024-03-04T07:26:52.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.432+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3219270790 >> 172.21.15.64:0/3219270790 conn(0x7f6db8066d00 msgr2=0x7f6db8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:52.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.432+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3219270790 shutdown_connections 2024-03-04T07:26:52.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.432+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3219270790 wait complete. 2024-03-04T07:26:52.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.433+0000 7f6dc03f6640 1 Processor -- start 2024-03-04T07:26:52.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.433+0000 7f6dc03f6640 1 -- start start 2024-03-04T07:26:52.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dc03f6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4003f30 0x7f6db8159f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dc03f6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6db81391e0 0x7f6db8157960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dc03f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db815c670 0x7f6db815caf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dc03f6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6db815b5d0 con 0x7f6db81391e0 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dc03f6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6db815bf40 con 0x7f6da4003f30 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dc03f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6db815a510 con 0x7f6db815c670 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6db81391e0 0x7f6db8157960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6db81391e0 0x7f6db8157960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41172/0 (socket says 172.21.15.64:41172) 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 -- 172.21.15.64:0/3308458200 learned_addr learned my addr 172.21.15.64:0/3308458200 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 -- 172.21.15.64:0/3308458200 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db815c670 msgr2=0x7f6db815caf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:52.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db815c670 0x7f6db815caf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 -- 172.21.15.64:0/3308458200 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4003f30 msgr2=0x7f6db8159f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4003f30 0x7f6db8159f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.434+0000 7f6dbd991640 1 -- 172.21.15.64:0/3308458200 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6db815a7f0 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.435+0000 7f6dbd991640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6db81391e0 0x7f6db8157960 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6da800c4d0 tx=0x7f6db81557f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.435+0000 7f6daeffd640 1 -- 172.21.15.64:0/3308458200 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6da80049e0 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.435+0000 7f6daeffd640 1 -- 172.21.15.64:0/3308458200 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6da8004b80 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.435+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6db815aad0 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.435+0000 7f6daeffd640 1 -- 172.21.15.64:0/3308458200 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6da8007070 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.435+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6db80cd7f0 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.436+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6db80ad8b0 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.436+0000 7f6daeffd640 1 -- 172.21.15.64:0/3308458200 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6da8007210 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.437+0000 7f6daeffd640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d9c06b4d0 0x7f6d9c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.437+0000 7f6daeffd640 1 -- 172.21.15.64:0/3308458200 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f6da80ab810 con 0x7f6db81391e0 2024-03-04T07:26:52.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.437+0000 7f6dbe192640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d9c06b4d0 0x7f6d9c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:52.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.437+0000 7f6dbe192640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d9c06b4d0 0x7f6d9c06d990 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f6db40099e0 tx=0x7f6db4006c30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:52.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.441+0000 7f6daeffd640 1 -- 172.21.15.64:0/3308458200 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6db80b3d30 con 0x7f6db81391e0 2024-03-04T07:26:52.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:52.659+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f6db80733c0 con 0x7f6db81391e0 2024-03-04T07:26:53.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.315+0000 7f6daeffd640 1 -- 172.21.15.64:0/3308458200 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v4) v1 ==== 172+0+0 (secure 0 0 0) 0x7f6da807a6f0 con 0x7f6db81391e0 2024-03-04T07:26:53.318 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 2 and data pool 3 2024-03-04T07:26:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.324+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d9c06b4d0 msgr2=0x7f6d9c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.324+0000 7f6dc03f6640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d9c06b4d0 0x7f6d9c06d990 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f6db40099e0 tx=0x7f6db4006c30 comp rx=0 tx=0).stop 2024-03-04T07:26:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.324+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6db81391e0 msgr2=0x7f6db8157960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.324+0000 7f6dc03f6640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6db81391e0 0x7f6db8157960 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6da800c4d0 tx=0x7f6db81557f0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.327+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 shutdown_connections 2024-03-04T07:26:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.327+0000 7f6dc03f6640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d9c06b4d0 0x7f6d9c06d990 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.327+0000 7f6dc03f6640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6db815c670 0x7f6db815caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.327+0000 7f6dc03f6640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6db81391e0 0x7f6db8157960 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.327+0000 7f6dc03f6640 1 --2- 172.21.15.64:0/3308458200 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6da4003f30 0x7f6db8159f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.327+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 >> 172.21.15.64:0/3308458200 conn(0x7f6db8066d00 msgr2=0x7f6db815e610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:53.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.331+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 shutdown_connections 2024-03-04T07:26:53.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.332+0000 7f6dc03f6640 1 -- 172.21.15.64:0/3308458200 wait complete. 2024-03-04T07:26:53.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.453+0000 7fc83626b640 1 Processor -- start 2024-03-04T07:26:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.453+0000 7fc83626b640 1 -- start start 2024-03-04T07:26:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc83626b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc8300b95a0 0x7fc8300b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc83626b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc8300b9a10 0x7fc8300b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc83626b640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc8300cea30 con 0x7fc83015df00 2024-03-04T07:26:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc83626b640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc8300cf3c0 con 0x7fc8300b95a0 2024-03-04T07:26:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc83626b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc8300cf0e0 con 0x7fc8300b9a10 2024-03-04T07:26:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc8300b95a0 0x7fc8300b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc8300b95a0 0x7fc8300b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40628/0 (socket says 172.21.15.64:40628) 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82f7fe640 1 -- 172.21.15.64:0/1478299192 learned_addr learned my addr 172.21.15.64:0/1478299192 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82f7fe640 1 -- 172.21.15.64:0/1478299192 >> v2:172.21.15.66:3300/0 conn(0x7fc8300b9a10 msgr2=0x7fc8300b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc834808640 1 --2- 172.21.15.64:0/1478299192 >> v2:172.21.15.66:3300/0 conn(0x7fc8300b9a10 0x7fc8300b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82f7fe640 1 --2- 172.21.15.64:0/1478299192 >> v2:172.21.15.66:3300/0 conn(0x7fc8300b9a10 0x7fc8300b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82effd640 1 -- 172.21.15.64:0/1478299192 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 49496917 0 0) 0x7fc8300cea30 con 0x7fc83015df00 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82f7fe640 1 -- 172.21.15.64:0/1478299192 >> v1:172.21.15.64:6789/0 conn(0x7fc83015df00 legacy=0x7fc8300d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc82f7fe640 1 -- 172.21.15.64:0/1478299192 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc8300ce470 con 0x7fc8300b95a0 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.454+0000 7fc834808640 1 --2- 172.21.15.64:0/1478299192 >> v2:172.21.15.66:3300/0 conn(0x7fc8300b9a10 0x7fc8300b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc82f7fe640 1 --2- 172.21.15.64:0/1478299192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8300b95a0 0x7fc8300b5700 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fc8300d4d20 tx=0x7fc82002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fe2d87b7459ff867 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc82effd640 1 -- 172.21.15.64:0/1478299192 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc820034070 con 0x7fc8300b95a0 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc82effd640 1 -- 172.21.15.64:0/1478299192 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc820030860 con 0x7fc8300b95a0 2024-03-04T07:26:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc82effd640 1 -- 172.21.15.64:0/1478299192 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc820030c30 con 0x7fc8300b95a0 2024-03-04T07:26:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc83626b640 1 -- 172.21.15.64:0/1478299192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8300b95a0 msgr2=0x7fc8300b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc83626b640 1 --2- 172.21.15.64:0/1478299192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8300b95a0 0x7fc8300b5700 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fc8300d4d20 tx=0x7fc82002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc83626b640 1 -- 172.21.15.64:0/1478299192 shutdown_connections 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc83626b640 1 --2- 172.21.15.64:0/1478299192 >> v2:172.21.15.66:3300/0 conn(0x7fc8300b9a10 0x7fc8300b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc83626b640 1 --2- 172.21.15.64:0/1478299192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8300b95a0 0x7fc8300b5700 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.455+0000 7fc83626b640 1 -- 172.21.15.64:0/1478299192 >> 172.21.15.64:0/1478299192 conn(0x7fc830071960 msgr2=0x7fc830071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.456+0000 7fc83626b640 1 -- 172.21.15.64:0/1478299192 shutdown_connections 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.456+0000 7fc83626b640 1 -- 172.21.15.64:0/1478299192 wait complete. 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 Processor -- start 2024-03-04T07:26:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 -- start start 2024-03-04T07:26:53.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8300b95a0 0x7fc8301600e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8300b9a10 0x7fc8301e8320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc83015df00 0x7fc83015b100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8300cedf0 con 0x7fc83015df00 2024-03-04T07:26:53.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8300cec70 con 0x7fc8300b9a10 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.457+0000 7fc83626b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc8300d3220 con 0x7fc8300b95a0 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc83015df00 0x7fc83015b100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc83015df00 0x7fc83015b100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40632/0 (socket says 172.21.15.64:40632) 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 -- 172.21.15.64:0/3159741712 learned_addr learned my addr 172.21.15.64:0/3159741712 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc82f7fe640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8300b9a10 0x7fc8301e8320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc82ffff640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8300b95a0 0x7fc8301600e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 -- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8300b95a0 msgr2=0x7fc8301600e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8300b95a0 0x7fc8301600e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 -- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8300b9a10 msgr2=0x7fc8301e8320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8300b9a10 0x7fc8301e8320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 -- 172.21.15.64:0/3159741712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc830160740 con 0x7fc83015df00 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc82ffff640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8300b95a0 0x7fc8301600e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc834808640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc83015df00 0x7fc83015b100 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fc82800cef0 tx=0x7fc82800a710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc82cff9640 1 -- 172.21.15.64:0/3159741712 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc82801a070 con 0x7fc83015df00 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc82cff9640 1 -- 172.21.15.64:0/3159741712 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc82800d860 con 0x7fc83015df00 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc830160a20 con 0x7fc83015df00 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc82cff9640 1 -- 172.21.15.64:0/3159741712 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc82800dc50 con 0x7fc83015df00 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.458+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc830160fe0 con 0x7fc83015df00 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.459+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc830161220 con 0x7fc83015df00 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.460+0000 7fc82cff9640 1 -- 172.21.15.64:0/3159741712 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc82801d070 con 0x7fc83015df00 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.460+0000 7fc82cff9640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc81c06b4d0 0x7fc81c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.460+0000 7fc82ffff640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc81c06b4d0 0x7fc81c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.460+0000 7fc82cff9640 1 -- 172.21.15.64:0/3159741712 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(19..19 src has 1..19) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc82808aa40 con 0x7fc83015df00 2024-03-04T07:26:53.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.461+0000 7fc82ffff640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc81c06b4d0 0x7fc81c06d990 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fc824000f60 tx=0x7fc824009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:53.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.465+0000 7fc82cff9640 1 -- 172.21.15.64:0/3159741712 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc830161220 con 0x7fc83015df00 2024-03-04T07:26:53.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.647+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc8300c7830 con 0x7fc83015df00 2024-03-04T07:26:53.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.648+0000 7fc82cff9640 1 -- 172.21.15.64:0/3159741712 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v19) v1 ==== 74+0+16570 (secure 0 0 0) 0x7fc82808ae20 con 0x7fc83015df00 2024-03-04T07:26:53.648 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:53.648 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":19,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:52.660447+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc81c06b4d0 msgr2=0x7fc81c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc81c06b4d0 0x7fc81c06d990 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fc824000f60 tx=0x7fc824009040 comp rx=0 tx=0).stop 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc83015df00 msgr2=0x7fc83015b100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc83015df00 0x7fc83015b100 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fc82800cef0 tx=0x7fc82800a710 comp rx=0 tx=0).stop 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 shutdown_connections 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc81c06b4d0 0x7fc81c06d990 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc83015df00 0x7fc83015b100 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8300b9a10 0x7fc8301e8320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 --2- 172.21.15.64:0/3159741712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8300b95a0 0x7fc8301600e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 >> 172.21.15.64:0/3159741712 conn(0x7fc830071960 msgr2=0x7fc830071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:53.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 shutdown_connections 2024-03-04T07:26:53.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.650+0000 7fc83626b640 1 -- 172.21.15.64:0/3159741712 wait complete. 2024-03-04T07:26:53.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:53.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.763+0000 7f1ca2d93640 1 Processor -- start 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.763+0000 7f1ca2d93640 1 -- start start 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.763+0000 7f1ca2d93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c9c0b1020 0x7f1c9c06af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.763+0000 7f1ca2d93640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1c9c06ed10 0x7f1c9c0cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.763+0000 7f1ca2d93640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1c9c0cc0e0 con 0x7f1c9c0cd6c0 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.763+0000 7f1ca2d93640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1c9c0d28b0 con 0x7f1c9c0b1020 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.763+0000 7f1ca2d93640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1c9c0cf320 con 0x7f1c9c06ed10 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca0b2f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c9c0b1020 0x7f1c9c06af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca1330640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1c9c0cd6c0 0x7f1c9c15d700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42774/0 (socket says 172.21.15.64:42774) 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca0b2f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c9c0b1020 0x7f1c9c06af20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40648/0 (socket says 172.21.15.64:40648) 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca1330640 1 -- 172.21.15.64:0/1904431714 learned_addr learned my addr 172.21.15.64:0/1904431714 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1c9bfff640 1 --2- 172.21.15.64:0/1904431714 >> v2:172.21.15.66:3300/0 conn(0x7f1c9c06ed10 0x7f1c9c0cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca0b2f640 1 -- 172.21.15.64:0/1904431714 >> v2:172.21.15.66:3300/0 conn(0x7f1c9c06ed10 msgr2=0x7f1c9c0cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca0b2f640 1 --2- 172.21.15.64:0/1904431714 >> v2:172.21.15.66:3300/0 conn(0x7f1c9c06ed10 0x7f1c9c0cd180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca0b2f640 1 -- 172.21.15.64:0/1904431714 >> v1:172.21.15.64:6789/0 conn(0x7f1c9c0cd6c0 legacy=0x7f1c9c15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.764+0000 7f1ca0b2f640 1 -- 172.21.15.64:0/1904431714 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c9c0cc3c0 con 0x7f1c9c0b1020 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca0b2f640 1 --2- 172.21.15.64:0/1904431714 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c06af20 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f1c9c0723a0 tx=0x7f1c9002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9fc637581da0b884 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1c9b7fe640 1 -- 172.21.15.64:0/1904431714 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c90035020 con 0x7f1c9c0b1020 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1c9b7fe640 1 -- 172.21.15.64:0/1904431714 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1c90031830 con 0x7f1c9c0b1020 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1c9b7fe640 1 -- 172.21.15.64:0/1904431714 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c90031c00 con 0x7f1c9c0b1020 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 -- 172.21.15.64:0/1904431714 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 msgr2=0x7f1c9c06af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/1904431714 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c06af20 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f1c9c0723a0 tx=0x7f1c9002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 -- 172.21.15.64:0/1904431714 shutdown_connections 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/1904431714 >> v2:172.21.15.66:3300/0 conn(0x7f1c9c06ed10 0x7f1c9c0cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/1904431714 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c06af20 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 -- 172.21.15.64:0/1904431714 >> 172.21.15.64:0/1904431714 conn(0x7f1c9c0d1a90 msgr2=0x7f1c9c0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:53.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 -- 172.21.15.64:0/1904431714 shutdown_connections 2024-03-04T07:26:53.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.765+0000 7f1ca2d93640 1 -- 172.21.15.64:0/1904431714 wait complete. 2024-03-04T07:26:53.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.766+0000 7f1ca2d93640 1 Processor -- start 2024-03-04T07:26:53.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1ca2d93640 1 -- start start 2024-03-04T07:26:53.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1ca2d93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c9c06ed10 0x7f1c9c0b48e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1ca2d93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c0c7530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1ca2d93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c9c0cd6c0 0x7f1c9c0b6760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1ca2d93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c9c0ccc60 con 0x7f1c9c0b1020 2024-03-04T07:26:53.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1ca2d93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c9c0cc980 con 0x7f1c9c06ed10 2024-03-04T07:26:53.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1ca2d93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1c9c0cdad0 con 0x7f1c9c0cd6c0 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1c9bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c0c7530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1c9bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c0c7530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40654/0 (socket says 172.21.15.64:40654) 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.767+0000 7f1c9bfff640 1 -- 172.21.15.64:0/812890630 learned_addr learned my addr 172.21.15.64:0/812890630 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1ca0b2f640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c9c06ed10 0x7f1c9c0b48e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1ca1330640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c9c0cd6c0 0x7f1c9c0b6760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c9bfff640 1 -- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c9c0cd6c0 msgr2=0x7f1c9c0b6760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c9bfff640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c9c0cd6c0 0x7f1c9c0b6760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c9bfff640 1 -- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c9c06ed10 msgr2=0x7f1c9c0b48e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c9bfff640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c9c06ed10 0x7f1c9c0b48e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c9bfff640 1 -- 172.21.15.64:0/812890630 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c9c0c68b0 con 0x7f1c9c0b1020 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c9bfff640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c0c7530 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f1c8800e9b0 tx=0x7f1c8800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c997fa640 1 -- 172.21.15.64:0/812890630 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c88017070 con 0x7f1c9c0b1020 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c997fa640 1 -- 172.21.15.64:0/812890630 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1c8800c9a0 con 0x7f1c9c0b1020 2024-03-04T07:26:53.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1c997fa640 1 -- 172.21.15.64:0/812890630 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c8800cd90 con 0x7f1c9c0b1020 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1c9c1f5130 con 0x7f1c9c0b1020 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.768+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c9c1f59d0 con 0x7f1c9c0b1020 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.769+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1c9c067ff0 con 0x7f1c9c0b1020 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.769+0000 7f1c997fa640 1 -- 172.21.15.64:0/812890630 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1c8801a070 con 0x7f1c9c0b1020 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.770+0000 7f1c997fa640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c8006b4d0 0x7f1c8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.770+0000 7f1c997fa640 1 -- 172.21.15.64:0/812890630 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(19..19 src has 1..19) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1c8808a530 con 0x7f1c9c0b1020 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.770+0000 7f1ca0b2f640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c8006b4d0 0x7f1c8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:53.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.770+0000 7f1ca0b2f640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c8006b4d0 0x7f1c8006d990 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f1c90036000 tx=0x7f1c90008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:53.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.774+0000 7f1c997fa640 1 -- 172.21.15.64:0/812890630 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1c88058d60 con 0x7f1c9c0b1020 2024-03-04T07:26:53.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.955+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1c9c0c8820 con 0x7f1c9c0b1020 2024-03-04T07:26:53.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.956+0000 7f1c997fa640 1 -- 172.21.15.64:0/812890630 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v19) v1 ==== 74+0+16570 (secure 0 0 0) 0x7f1c8805ce00 con 0x7f1c9c0b1020 2024-03-04T07:26:53.956 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:53.956 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":19,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:52.660447+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c8006b4d0 msgr2=0x7f1c8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c8006b4d0 0x7f1c8006d990 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f1c90036000 tx=0x7f1c90008030 comp rx=0 tx=0).stop 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 msgr2=0x7f1c9c0c7530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c0c7530 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f1c8800e9b0 tx=0x7f1c8800ee70 comp rx=0 tx=0).stop 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 shutdown_connections 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c8006b4d0 0x7f1c8006d990 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c9c0cd6c0 0x7f1c9c0b6760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c9c0b1020 0x7f1c9c0c7530 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 --2- 172.21.15.64:0/812890630 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c9c06ed10 0x7f1c9c0b48e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:53.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 >> 172.21.15.64:0/812890630 conn(0x7f1c9c0d1a90 msgr2=0x7f1c9c06b480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:53.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 shutdown_connections 2024-03-04T07:26:53.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:53.958+0000 7f1ca2d93640 1 -- 172.21.15.64:0/812890630 wait complete. 2024-03-04T07:26:53.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:26:54.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.074+0000 7f8a17815640 1 Processor -- start 2024-03-04T07:26:54.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.074+0000 7f8a17815640 1 -- start start 2024-03-04T07:26:54.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a17815640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a10071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a17815640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8a100b2b10 0x7f8a100d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a17815640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8a1015e4b0 con 0x7f8a100b3060 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a17815640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8a100d2e10 con 0x7f8a100b2b10 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a17815640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8a100b97e0 con 0x7f8a100717b0 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a155b1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a10071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a155b1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a10071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:47956/0 (socket says 172.21.15.64:47956) 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a155b1640 1 -- 172.21.15.64:0/2127237456 learned_addr learned my addr 172.21.15.64:0/2127237456 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a15db2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8a100b2b10 0x7f8a100d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a155b1640 1 -- 172.21.15.64:0/2127237456 >> v2:172.21.15.66:3300/0 conn(0x7f8a100b2b10 msgr2=0x7f8a100d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a155b1640 1 --2- 172.21.15.64:0/2127237456 >> v2:172.21.15.66:3300/0 conn(0x7f8a100b2b10 0x7f8a100d32a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a155b1640 1 -- 172.21.15.64:0/2127237456 >> v1:172.21.15.66:6789/0 conn(0x7f8a100b3060 legacy=0x7f8a100b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:54.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.075+0000 7f8a155b1640 1 -- 172.21.15.64:0/2127237456 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a100b9a80 con 0x7f8a100717b0 2024-03-04T07:26:54.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a155b1640 1 --2- 172.21.15.64:0/2127237456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a10071b90 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c009870 tx=0x7f8a0c02b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8f0c14f4083948ce server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:54.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a07fff640 1 -- 172.21.15.64:0/2127237456 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c034070 con 0x7f8a100717b0 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a07fff640 1 -- 172.21.15.64:0/2127237456 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8a0c030840 con 0x7f8a100717b0 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a07fff640 1 -- 172.21.15.64:0/2127237456 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c030c10 con 0x7f8a100717b0 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a17815640 1 -- 172.21.15.64:0/2127237456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 msgr2=0x7f8a10071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a17815640 1 --2- 172.21.15.64:0/2127237456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a10071b90 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c009870 tx=0x7f8a0c02b980 comp rx=0 tx=0).stop 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a17815640 1 -- 172.21.15.64:0/2127237456 shutdown_connections 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a17815640 1 --2- 172.21.15.64:0/2127237456 >> v2:172.21.15.66:3300/0 conn(0x7f8a100b2b10 0x7f8a100d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a17815640 1 --2- 172.21.15.64:0/2127237456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a10071b90 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.076+0000 7f8a17815640 1 -- 172.21.15.64:0/2127237456 >> 172.21.15.64:0/2127237456 conn(0x7f8a100d3d20 msgr2=0x7f8a100d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.077+0000 7f8a17815640 1 -- 172.21.15.64:0/2127237456 shutdown_connections 2024-03-04T07:26:54.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.077+0000 7f8a17815640 1 -- 172.21.15.64:0/2127237456 wait complete. 2024-03-04T07:26:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 Processor -- start 2024-03-04T07:26:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 -- start start 2024-03-04T07:26:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a101577f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8a100b2b10 0x7f8a10155670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8a100b3060 0x7f8a101588e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a100b7560 con 0x7f8a100b3060 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a100d4bf0 con 0x7f8a100b2b10 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a17815640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8a100b76e0 con 0x7f8a100717b0 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.078+0000 7f8a155b1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a101577f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a14db0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8a100b2b10 0x7f8a10155670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a101577f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:47966/0 (socket says 172.21.15.64:47966) 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 -- 172.21.15.64:0/79425904 learned_addr learned my addr 172.21.15.64:0/79425904 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a15db2640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8a100b3060 0x7f8a101588e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 -- 172.21.15.64:0/79425904 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8a100b2b10 msgr2=0x7f8a10155670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8a100b2b10 0x7f8a10155670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 -- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8a100b3060 msgr2=0x7f8a101588e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8a100b3060 0x7f8a101588e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 -- 172.21.15.64:0/79425904 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a10156d50 con 0x7f8a100717b0 2024-03-04T07:26:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a155b1640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a101577f0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c000c00 tx=0x7f8a0c004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:54.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a05ffb640 1 -- 172.21.15.64:0/79425904 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c048070 con 0x7f8a100717b0 2024-03-04T07:26:54.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a05ffb640 1 -- 172.21.15.64:0/79425904 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8a0c030640 con 0x7f8a100717b0 2024-03-04T07:26:54.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8a10159250 con 0x7f8a100717b0 2024-03-04T07:26:54.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a05ffb640 1 -- 172.21.15.64:0/79425904 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c034040 con 0x7f8a100717b0 2024-03-04T07:26:54.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.079+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a10159530 con 0x7f8a100717b0 2024-03-04T07:26:54.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.080+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8a1006ce40 con 0x7f8a100717b0 2024-03-04T07:26:54.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.081+0000 7f8a05ffb640 1 -- 172.21.15.64:0/79425904 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8a0c04b070 con 0x7f8a100717b0 2024-03-04T07:26:54.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.081+0000 7f8a05ffb640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89f406b4d0 0x7f89f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:54.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.081+0000 7f8a05ffb640 1 -- 172.21.15.64:0/79425904 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(19..19 src has 1..19) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8a0c0abf90 con 0x7f8a100717b0 2024-03-04T07:26:54.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.081+0000 7f8a14db0640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89f406b4d0 0x7f89f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:54.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.082+0000 7f8a14db0640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89f406b4d0 0x7f89f406d990 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f8a10158110 tx=0x7f8a00009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:54.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.086+0000 7f8a05ffb640 1 -- 172.21.15.64:0/79425904 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8a0c07a970 con 0x7f8a100717b0 2024-03-04T07:26:54.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:54.305+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f8a100b5d20 con 0x7f8a100717b0 2024-03-04T07:26:55.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.305+0000 7f8a05ffb640 1 -- 172.21.15.64:0/79425904 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v6) v1 ==== 118+0+0 (secure 0 0 0) 0x7f8a0c07ea10 con 0x7f8a100717b0 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89f406b4d0 msgr2=0x7f89f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89f406b4d0 0x7f89f406d990 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f8a10158110 tx=0x7f8a00009040 comp rx=0 tx=0).stop 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 msgr2=0x7f8a101577f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a101577f0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c000c00 tx=0x7f8a0c004730 comp rx=0 tx=0).stop 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 shutdown_connections 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89f406b4d0 0x7f89f406d990 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8a100b3060 0x7f8a101588e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8a100b2b10 0x7f8a10155670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 --2- 172.21.15.64:0/79425904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8a100717b0 0x7f8a101577f0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 >> 172.21.15.64:0/79425904 conn(0x7f8a100d3d20 msgr2=0x7f8a1015c1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:55.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 shutdown_connections 2024-03-04T07:26:55.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.308+0000 7f8a17815640 1 -- 172.21.15.64:0/79425904 wait complete. 2024-03-04T07:26:55.322 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:26:55.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:26:55.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.434+0000 7f8591d4b640 1 Processor -- start 2024-03-04T07:26:55.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.434+0000 7f8591d4b640 1 -- start start 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f8591d4b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f858c0cf0f0 0x7f858c071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f8591d4b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0724b0 0x7f858c0d2760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f8591d4b640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f858c072cf0 con 0x7f858c15dd50 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f8591d4b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f858c072fd0 con 0x7f858c0cf0f0 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f8591d4b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f858c0d1f60 con 0x7f858c0724b0 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858bfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0724b0 0x7f858c0d2760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858affd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f858c0cf0f0 0x7f858c071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:55.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858b7fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f858c15dd50 0x7f858c0ce9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43692/0 (socket says 172.21.15.64:43692) 2024-03-04T07:26:55.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858b7fe640 1 -- 172.21.15.64:0/3128297957 learned_addr learned my addr 172.21.15.64:0/3128297957 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:55.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858affd640 1 -- 172.21.15.64:0/3128297957 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0724b0 msgr2=0x7f858c0d2760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:55.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858affd640 1 --2- 172.21.15.64:0/3128297957 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0724b0 0x7f858c0d2760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858affd640 1 -- 172.21.15.64:0/3128297957 >> v1:172.21.15.66:6789/0 conn(0x7f858c15dd50 legacy=0x7f858c0ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:55.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858affd640 1 -- 172.21.15.64:0/3128297957 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f858c0d2240 con 0x7f858c0cf0f0 2024-03-04T07:26:55.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.435+0000 7f858a7fc640 1 -- 172.21.15.64:0/3128297957 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3736114552 0 0) 0x7f858c072cf0 con 0x7f858c15dd50 2024-03-04T07:26:55.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.436+0000 7f858affd640 1 --2- 172.21.15.64:0/3128297957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0cf0f0 0x7f858c071f70 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f85800097a0 tx=0x7f858002b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e9b27855612e4b11 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:55.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.436+0000 7f858a7fc640 1 -- 172.21.15.64:0/3128297957 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f85800315a0 con 0x7f858c0cf0f0 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.436+0000 7f858a7fc640 1 -- 172.21.15.64:0/3128297957 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8580037070 con 0x7f858c0cf0f0 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.436+0000 7f858a7fc640 1 -- 172.21.15.64:0/3128297957 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8580031960 con 0x7f858c0cf0f0 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 -- 172.21.15.64:0/3128297957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0cf0f0 msgr2=0x7f858c071f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3128297957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0cf0f0 0x7f858c071f70 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f85800097a0 tx=0x7f858002b840 comp rx=0 tx=0).stop 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 -- 172.21.15.64:0/3128297957 shutdown_connections 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3128297957 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0724b0 0x7f858c0d2760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3128297957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0cf0f0 0x7f858c071f70 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 -- 172.21.15.64:0/3128297957 >> 172.21.15.64:0/3128297957 conn(0x7f858c0d3ef0 msgr2=0x7f858c0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 -- 172.21.15.64:0/3128297957 shutdown_connections 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.437+0000 7f8591d4b640 1 -- 172.21.15.64:0/3128297957 wait complete. 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.438+0000 7f8591d4b640 1 Processor -- start 2024-03-04T07:26:55.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.438+0000 7f8591d4b640 1 -- start start 2024-03-04T07:26:55.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f8591d4b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0724b0 0x7f858c1e7570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:55.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f8591d4b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0cf0f0 0x7f858c0b5b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:55.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f8591d4b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f858c15dd50 0x7f858c0b6050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f8591d4b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f858c15e2b0 con 0x7f858c15dd50 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f8591d4b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f858c15e130 con 0x7f858c0724b0 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f8591d4b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f858c0d2ca0 con 0x7f858c0cf0f0 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f858c15dd50 0x7f858c0b6050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858b7fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0724b0 0x7f858c1e7570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858affd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0cf0f0 0x7f858c0b5b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f858c15dd50 0x7f858c0b6050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40680/0 (socket says 172.21.15.64:40680) 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 -- 172.21.15.64:0/3122942091 learned_addr learned my addr 172.21.15.64:0/3122942091 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 -- 172.21.15.64:0/3122942091 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0cf0f0 msgr2=0x7f858c0b5b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0cf0f0 0x7f858c0b5b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 -- 172.21.15.64:0/3122942091 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0724b0 msgr2=0x7f858c1e7570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0724b0 0x7f858c1e7570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:55.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 -- 172.21.15.64:0/3122942091 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f858c0b6680 con 0x7f858c15dd50 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858b7fe640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0724b0 0x7f858c1e7570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.439+0000 7f858bfff640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f858c15dd50 0x7f858c0b6050 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f858400efc0 tx=0x7f858400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.440+0000 7f8573fff640 1 -- 172.21.15.64:0/3122942091 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8584018020 con 0x7f858c15dd50 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.440+0000 7f8573fff640 1 -- 172.21.15.64:0/3122942091 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8584014480 con 0x7f858c15dd50 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.440+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f858c1e67b0 con 0x7f858c15dd50 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.440+0000 7f8573fff640 1 -- 172.21.15.64:0/3122942091 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8584014870 con 0x7f858c15dd50 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.440+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f858c1e6d70 con 0x7f858c15dd50 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.441+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f858c06cdd0 con 0x7f858c15dd50 2024-03-04T07:26:55.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.441+0000 7f8573fff640 1 -- 172.21.15.64:0/3122942091 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8584005d10 con 0x7f858c15dd50 2024-03-04T07:26:55.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.441+0000 7f8573fff640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f857806b4d0 0x7f857806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:55.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.442+0000 7f8573fff640 1 -- 172.21.15.64:0/3122942091 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f858408aec0 con 0x7f858c15dd50 2024-03-04T07:26:55.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.442+0000 7f858b7fe640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f857806b4d0 0x7f857806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:55.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.442+0000 7f858b7fe640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f857806b4d0 0x7f857806d990 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f857c007fd0 tx=0x7f857c007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:55.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.446+0000 7f8573fff640 1 -- 172.21.15.64:0/3122942091 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8584022030 con 0x7f858c15dd50 2024-03-04T07:26:55.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:55.665+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f858c04f410 con 0x7f858c15dd50 2024-03-04T07:26:56.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.317+0000 7f8573fff640 1 -- 172.21.15.64:0/3122942091 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v7) v1 ==== 122+0+0 (secure 0 0 0) 0x7f8584010040 con 0x7f858c15dd50 2024-03-04T07:26:56.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.319+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f857806b4d0 msgr2=0x7f857806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.319+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f857806b4d0 0x7f857806d990 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f857c007fd0 tx=0x7f857c007de0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.319+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f858c15dd50 msgr2=0x7f858c0b6050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.319+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f858c15dd50 0x7f858c0b6050 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f858400efc0 tx=0x7f858400c490 comp rx=0 tx=0).stop 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 shutdown_connections 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f857806b4d0 0x7f857806d990 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f858c15dd50 0x7f858c0b6050 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f858c0cf0f0 0x7f858c0b5b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 --2- 172.21.15.64:0/3122942091 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f858c0724b0 0x7f858c1e7570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 >> 172.21.15.64:0/3122942091 conn(0x7f858c0d3ef0 msgr2=0x7f858c0b87f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 shutdown_connections 2024-03-04T07:26:56.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.320+0000 7f8591d4b640 1 -- 172.21.15.64:0/3122942091 wait complete. 2024-03-04T07:26:56.332 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:26:56.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:26:56.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.441+0000 7f9b261a2640 1 Processor -- start 2024-03-04T07:26:56.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.441+0000 7f9b261a2640 1 -- start start 2024-03-04T07:26:56.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b261a2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b2015dd20 0x7f9b200d1bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b261a2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9b200d2100 0x7f9b200d0a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b261a2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9b200d0f70 0x7f9b200d3a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b261a2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9b200d3190 con 0x7f9b200d0f70 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b261a2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9b200d1390 con 0x7f9b200d2100 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b261a2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b2015e100 con 0x7f9b2015dd20 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b259a1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9b200d0f70 0x7f9b200d3a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b251a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b2015dd20 0x7f9b200d1bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b2499f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9b200d2100 0x7f9b200d0a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b251a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b2015dd20 0x7f9b200d1bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:47994/0 (socket says 172.21.15.64:47994) 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b251a0640 1 -- 172.21.15.64:0/4141845085 learned_addr learned my addr 172.21.15.64:0/4141845085 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b259a1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9b200d0f70 0x7f9b200d3a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40692/0 (socket says 172.21.15.64:40692) 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b259a1640 1 -- 172.21.15.64:0/4141845085 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b2015dd20 msgr2=0x7f9b200d1bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:56.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b259a1640 1 --2- 172.21.15.64:0/4141845085 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b2015dd20 0x7f9b200d1bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b259a1640 1 -- 172.21.15.64:0/4141845085 >> v2:172.21.15.66:3300/0 conn(0x7f9b200d2100 msgr2=0x7f9b200d0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b259a1640 1 --2- 172.21.15.64:0/4141845085 >> v2:172.21.15.66:3300/0 conn(0x7f9b200d2100 0x7f9b200d0a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b259a1640 1 -- 172.21.15.64:0/4141845085 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b200d2580 con 0x7f9b200d0f70 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.442+0000 7f9b2499f640 1 --2- 172.21.15.64:0/4141845085 >> v2:172.21.15.66:3300/0 conn(0x7f9b200d2100 0x7f9b200d0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.443+0000 7f9b259a1640 1 --2- 172.21.15.64:0/4141845085 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b200d3a00 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f9b200d4b70 tx=0x7f9b1802be20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c5877215e88b55d1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.443+0000 7f9b17fff640 1 -- 172.21.15.64:0/4141845085 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b18039070 con 0x7f9b200d0f70 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.443+0000 7f9b17fff640 1 -- 172.21.15.64:0/4141845085 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b1802cbd0 con 0x7f9b200d0f70 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.443+0000 7f9b17fff640 1 -- 172.21.15.64:0/4141845085 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b18034440 con 0x7f9b200d0f70 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 -- 172.21.15.64:0/4141845085 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 msgr2=0x7f9b200d3a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 --2- 172.21.15.64:0/4141845085 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b200d3a00 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f9b200d4b70 tx=0x7f9b1802be20 comp rx=0 tx=0).stop 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 -- 172.21.15.64:0/4141845085 shutdown_connections 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 --2- 172.21.15.64:0/4141845085 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b200d3a00 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 --2- 172.21.15.64:0/4141845085 >> v2:172.21.15.66:3300/0 conn(0x7f9b200d2100 0x7f9b200d0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 --2- 172.21.15.64:0/4141845085 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b2015dd20 0x7f9b200d1bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 -- 172.21.15.64:0/4141845085 >> 172.21.15.64:0/4141845085 conn(0x7f9b20071960 msgr2=0x7f9b20071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:56.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 -- 172.21.15.64:0/4141845085 shutdown_connections 2024-03-04T07:26:56.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.444+0000 7f9b261a2640 1 -- 172.21.15.64:0/4141845085 wait complete. 2024-03-04T07:26:56.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.445+0000 7f9b261a2640 1 Processor -- start 2024-03-04T07:26:56.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.445+0000 7f9b261a2640 1 -- start start 2024-03-04T07:26:56.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b261a2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b201f7010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b261a2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b200d2100 0x7f9b200cb580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b261a2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b2015dd20 0x7f9b201604e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b261a2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b200d2e20 con 0x7f9b200d0f70 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b261a2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b200d2b40 con 0x7f9b2015dd20 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b261a2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b200b9580 con 0x7f9b200d2100 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b201f7010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b201f7010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40700/0 (socket says 172.21.15.64:40700) 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 -- 172.21.15.64:0/1383199648 learned_addr learned my addr 172.21.15.64:0/1383199648 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b259a1640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b2015dd20 0x7f9b201604e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b2499f640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b200d2100 0x7f9b200cb580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 -- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b200d2100 msgr2=0x7f9b200cb580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b200d2100 0x7f9b200cb580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 -- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b2015dd20 msgr2=0x7f9b201604e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b2015dd20 0x7f9b201604e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 -- 172.21.15.64:0/1383199648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b200cc040 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b251a0640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b201f7010 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f9b1c00e870 tx=0x7f9b1c00ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b15ffb640 1 -- 172.21.15.64:0/1383199648 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b1c017070 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b15ffb640 1 -- 172.21.15.64:0/1383199648 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b1c00c8e0 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b15ffb640 1 -- 172.21.15.64:0/1383199648 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b1c00ccd0 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.446+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b200cc320 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.447+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b200cc8e0 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.448+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b2004f330 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.448+0000 7f9b15ffb640 1 -- 172.21.15.64:0/1383199648 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9b1c01a070 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.448+0000 7f9b15ffb640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b0406b4d0 0x7f9b0406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.448+0000 7f9b15ffb640 1 -- 172.21.15.64:0/1383199648 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9b1c08a330 con 0x7f9b200d0f70 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.448+0000 7f9b2499f640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b0406b4d0 0x7f9b0406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.449+0000 7f9b2499f640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b0406b4d0 0x7f9b0406d990 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f9b10009560 tx=0x7f9b10009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:56.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.453+0000 7f9b15ffb640 1 -- 172.21.15.64:0/1383199648 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9b1c058c20 con 0x7f9b200d0f70 2024-03-04T07:26:56.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.671+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9b200b2db0 con 0x7f9b200d0f70 2024-03-04T07:26:56.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.671+0000 7f9b15ffb640 1 -- 172.21.15.64:0/1383199648 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f9b1c05ccc0 con 0x7f9b200d0f70 2024-03-04T07:26:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:56.671 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:26:56.671 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b0406b4d0 msgr2=0x7f9b0406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b0406b4d0 0x7f9b0406d990 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f9b10009560 tx=0x7f9b10009190 comp rx=0 tx=0).stop 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 msgr2=0x7f9b201f7010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b201f7010 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f9b1c00e870 tx=0x7f9b1c00ed30 comp rx=0 tx=0).stop 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 shutdown_connections 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b0406b4d0 0x7f9b0406d990 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b2015dd20 0x7f9b201604e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b200d2100 0x7f9b200cb580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 --2- 172.21.15.64:0/1383199648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b200d0f70 0x7f9b201f7010 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 >> 172.21.15.64:0/1383199648 conn(0x7f9b20071960 msgr2=0x7f9b200b8620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 shutdown_connections 2024-03-04T07:26:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.674+0000 7f9b261a2640 1 -- 172.21.15.64:0/1383199648 wait complete. 2024-03-04T07:26:56.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:56.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.786+0000 7f56a6bd7640 1 Processor -- start 2024-03-04T07:26:56.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.786+0000 7f56a6bd7640 1 -- start start 2024-03-04T07:26:56.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f56a6bd7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00b9650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f56a6bd7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f56a00b5700 0x7f56a00b5b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f56a6bd7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f56a00b83c0 con 0x7f56a00d1e50 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f56a6bd7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f56a00b8fd0 con 0x7f56a00b5700 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f56a6bd7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f56a00b6080 con 0x7f56a015dd70 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f56a00b5700 0x7f56a00b5b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f56a4973640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00b9650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f56a00b5700 0x7f56a00b5b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40714/0 (socket says 172.21.15.64:40714) 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569ffff640 1 -- 172.21.15.64:0/2357357896 learned_addr learned my addr 172.21.15.64:0/2357357896 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f56a5174640 1 --1- 172.21.15.64:0/2357357896 >> v1:172.21.15.64:6789/0 conn(0x7f56a00d1e50 0x7f56a00d22e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42784/0 (socket says 172.21.15.64:42784) 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569ffff640 1 -- 172.21.15.64:0/2357357896 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 msgr2=0x7f56a00b9650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569ffff640 1 --2- 172.21.15.64:0/2357357896 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00b9650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569ffff640 1 -- 172.21.15.64:0/2357357896 >> v1:172.21.15.64:6789/0 conn(0x7f56a00d1e50 legacy=0x7f56a00d22e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569ffff640 1 -- 172.21.15.64:0/2357357896 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56a00b86a0 con 0x7f56a00b5700 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.787+0000 7f569f7fe640 1 -- 172.21.15.64:0/2357357896 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4043289322 0 0) 0x7f56a00b83c0 con 0x7f56a00d1e50 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.788+0000 7f569ffff640 1 --2- 172.21.15.64:0/2357357896 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00b5700 0x7f56a00b5b40 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f568c009870 tx=0x7f568c02ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=146b3d66c76bcb5f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:56.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.788+0000 7f569f7fe640 1 -- 172.21.15.64:0/2357357896 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f568c02f840 con 0x7f56a00b5700 2024-03-04T07:26:56.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.788+0000 7f569f7fe640 1 -- 172.21.15.64:0/2357357896 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f568c02f9e0 con 0x7f56a00b5700 2024-03-04T07:26:56.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.788+0000 7f569f7fe640 1 -- 172.21.15.64:0/2357357896 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f568c02fdb0 con 0x7f56a00b5700 2024-03-04T07:26:56.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.788+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2357357896 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00b5700 msgr2=0x7f56a00b5b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.788+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2357357896 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00b5700 0x7f56a00b5b40 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f568c009870 tx=0x7f568c02ba20 comp rx=0 tx=0).stop 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.789+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2357357896 shutdown_connections 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.789+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2357357896 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00b5700 0x7f56a00b5b40 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.789+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2357357896 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00b9650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.789+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2357357896 >> 172.21.15.64:0/2357357896 conn(0x7f56a0071960 msgr2=0x7f56a0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.789+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2357357896 shutdown_connections 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.789+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2357357896 wait complete. 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.790+0000 7f56a6bd7640 1 Processor -- start 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.790+0000 7f56a6bd7640 1 -- start start 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56a00b5700 0x7f56a0160f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00d1e50 0x7f56a014df90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00c9fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56a00b9b90 con 0x7f56a00d1e50 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56a00b8c60 con 0x7f56a00b5700 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f56a015e150 con 0x7f56a015dd70 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00d1e50 0x7f56a014df90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00d1e50 0x7f56a014df90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40728/0 (socket says 172.21.15.64:40728) 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 -- 172.21.15.64:0/2056454221 learned_addr learned my addr 172.21.15.64:0/2056454221 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a4973640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56a00b5700 0x7f56a0160f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a5174640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00c9fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 -- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 msgr2=0x7f56a00c9fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00c9fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 -- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56a00b5700 msgr2=0x7f56a0160f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56a00b5700 0x7f56a0160f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 -- 172.21.15.64:0/2056454221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56a00c8f00 con 0x7f56a00d1e50 2024-03-04T07:26:56.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569ffff640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00d1e50 0x7f56a014df90 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f568c000c00 tx=0x7f568c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:56.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569d7fa640 1 -- 172.21.15.64:0/2056454221 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f568c037bc0 con 0x7f56a00d1e50 2024-03-04T07:26:56.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569d7fa640 1 -- 172.21.15.64:0/2056454221 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f568c046070 con 0x7f56a00d1e50 2024-03-04T07:26:56.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56a00c91e0 con 0x7f56a00d1e50 2024-03-04T07:26:56.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f569d7fa640 1 -- 172.21.15.64:0/2056454221 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f568c041400 con 0x7f56a00d1e50 2024-03-04T07:26:56.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.791+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56a00c9a80 con 0x7f56a00d1e50 2024-03-04T07:26:56.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.792+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56a00c95d0 con 0x7f56a00d1e50 2024-03-04T07:26:56.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.793+0000 7f569d7fa640 1 -- 172.21.15.64:0/2056454221 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f568c040070 con 0x7f56a00d1e50 2024-03-04T07:26:56.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.793+0000 7f569d7fa640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f568406b4d0 0x7f568406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:56.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.793+0000 7f56a4973640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f568406b4d0 0x7f568406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:56.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.793+0000 7f569d7fa640 1 -- 172.21.15.64:0/2056454221 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f568c0ac7b0 con 0x7f56a00d1e50 2024-03-04T07:26:56.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.794+0000 7f56a4973640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f568406b4d0 0x7f568406d990 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f5694009aa0 tx=0x7f5694009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:56.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.798+0000 7f569d7fa640 1 -- 172.21.15.64:0/2056454221 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f568c033030 con 0x7f56a00d1e50 2024-03-04T07:26:56.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.983+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f56a0067ff0 con 0x7f56a00d1e50 2024-03-04T07:26:56.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.984+0000 7f569d7fa640 1 -- 172.21.15.64:0/2056454221 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f568c07b100 con 0x7f56a00d1e50 2024-03-04T07:26:56.984 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:56.984 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:56.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f568406b4d0 msgr2=0x7f568406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f568406b4d0 0x7f568406d990 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f5694009aa0 tx=0x7f5694009190 comp rx=0 tx=0).stop 2024-03-04T07:26:56.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00d1e50 msgr2=0x7f56a014df90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00d1e50 0x7f56a014df90 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f568c000c00 tx=0x7f568c004730 comp rx=0 tx=0).stop 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 shutdown_connections 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f568406b4d0 0x7f568406d990 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56a015dd70 0x7f56a00c9fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56a00d1e50 0x7f56a014df90 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 --2- 172.21.15.64:0/2056454221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56a00b5700 0x7f56a0160f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 >> 172.21.15.64:0/2056454221 conn(0x7f56a0071960 msgr2=0x7f56a00d2580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 shutdown_connections 2024-03-04T07:26:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:56.986+0000 7f56a6bd7640 1 -- 172.21.15.64:0/2056454221 wait complete. 2024-03-04T07:26:56.996 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:26:56.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:26:57.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.098+0000 7f49f7464640 1 Processor -- start 2024-03-04T07:26:57.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.098+0000 7f49f7464640 1 -- start start 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f7464640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f49f006e700 0x7f49f015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f7464640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f49f00b55d0 0x7f49f0071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f7464640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f49f00b4160 con 0x7f49f0072660 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f7464640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f49f00b5b40 con 0x7f49f006e700 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f7464640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f49f00b3ba0 con 0x7f49f00b55d0 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5200640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f49f006e700 0x7f49f015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5200640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f49f006e700 0x7f49f015ef80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40736/0 (socket says 172.21.15.64:40736) 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5200640 1 -- 172.21.15.64:0/3274877435 learned_addr learned my addr 172.21.15.64:0/3274877435 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5a01640 1 --2- 172.21.15.64:0/3274877435 >> v2:172.21.15.66:3300/0 conn(0x7f49f00b55d0 0x7f49f0071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5200640 1 -- 172.21.15.64:0/3274877435 >> v2:172.21.15.66:3300/0 conn(0x7f49f00b55d0 msgr2=0x7f49f0071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5200640 1 --2- 172.21.15.64:0/3274877435 >> v2:172.21.15.66:3300/0 conn(0x7f49f00b55d0 0x7f49f0071980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5200640 1 -- 172.21.15.64:0/3274877435 >> v1:172.21.15.66:6789/0 conn(0x7f49f0072660 legacy=0x7f49f006d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49f5200640 1 -- 172.21.15.64:0/3274877435 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f49f00b3e80 con 0x7f49f006e700 2024-03-04T07:26:57.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.099+0000 7f49e7fff640 1 -- 172.21.15.64:0/3274877435 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2369001588 0 0) 0x7f49f00b4160 con 0x7f49f0072660 2024-03-04T07:26:57.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.100+0000 7f49f5200640 1 --2- 172.21.15.64:0/3274877435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f006e700 0x7f49f015ef80 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f49f0071580 tx=0x7f49ec02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e75cc6775a6402da server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.100+0000 7f49e7fff640 1 -- 172.21.15.64:0/3274877435 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f49ec034070 con 0x7f49f006e700 2024-03-04T07:26:57.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.100+0000 7f49e7fff640 1 -- 172.21.15.64:0/3274877435 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f49ec030860 con 0x7f49f006e700 2024-03-04T07:26:57.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.100+0000 7f49e7fff640 1 -- 172.21.15.64:0/3274877435 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f49ec030c30 con 0x7f49f006e700 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 -- 172.21.15.64:0/3274877435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f006e700 msgr2=0x7f49f015ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 --2- 172.21.15.64:0/3274877435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f006e700 0x7f49f015ef80 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f49f0071580 tx=0x7f49ec02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 -- 172.21.15.64:0/3274877435 shutdown_connections 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 --2- 172.21.15.64:0/3274877435 >> v2:172.21.15.66:3300/0 conn(0x7f49f00b55d0 0x7f49f0071980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 --2- 172.21.15.64:0/3274877435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f006e700 0x7f49f015ef80 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 -- 172.21.15.64:0/3274877435 >> 172.21.15.64:0/3274877435 conn(0x7f49f00b2720 msgr2=0x7f49f00b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 -- 172.21.15.64:0/3274877435 shutdown_connections 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.101+0000 7f49f7464640 1 -- 172.21.15.64:0/3274877435 wait complete. 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.102+0000 7f49f7464640 1 Processor -- start 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.102+0000 7f49f7464640 1 -- start start 2024-03-04T07:26:57.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f7464640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f49f006e700 0x7f49f00cc4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f7464640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f0072660 0x7f49f01475a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f7464640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f49f00b55d0 0x7f49f015b490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f7464640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49f0072040 con 0x7f49f0072660 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f7464640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49f0071ec0 con 0x7f49f00b55d0 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f7464640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f49f00721c0 con 0x7f49f006e700 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f0072660 0x7f49f01475a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f5a01640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f49f00b55d0 0x7f49f015b490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f0072660 0x7f49f01475a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40744/0 (socket says 172.21.15.64:40744) 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 -- 172.21.15.64:0/2250479472 learned_addr learned my addr 172.21.15.64:0/2250479472 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f5200640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f49f006e700 0x7f49f00cc4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 -- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f49f006e700 msgr2=0x7f49f00cc4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f49f006e700 0x7f49f00cc4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 -- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f49f00b55d0 msgr2=0x7f49f015b490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f49f00b55d0 0x7f49f015b490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 -- 172.21.15.64:0/2250479472 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f49f00adb00 con 0x7f49f0072660 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f5a01640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f49f00b55d0 0x7f49f015b490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.103+0000 7f49f49ff640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f0072660 0x7f49f01475a0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f49e000c550 tx=0x7f49e000ca10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.104+0000 7f49e5ffb640 1 -- 172.21.15.64:0/2250479472 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f49e0017070 con 0x7f49f0072660 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.104+0000 7f49e5ffb640 1 -- 172.21.15.64:0/2250479472 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f49e00035f0 con 0x7f49f0072660 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.104+0000 7f49e5ffb640 1 -- 172.21.15.64:0/2250479472 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f49e00039e0 con 0x7f49f0072660 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.104+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f49f00adde0 con 0x7f49f0072660 2024-03-04T07:26:57.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.104+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f49f00ae3a0 con 0x7f49f0072660 2024-03-04T07:26:57.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.105+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f49f0067ff0 con 0x7f49f0072660 2024-03-04T07:26:57.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.105+0000 7f49e5ffb640 1 -- 172.21.15.64:0/2250479472 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f49e001a070 con 0x7f49f0072660 2024-03-04T07:26:57.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.105+0000 7f49e5ffb640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f49d406b500 0x7f49d406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.105+0000 7f49e5ffb640 1 -- 172.21.15.64:0/2250479472 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f49e00894e0 con 0x7f49f0072660 2024-03-04T07:26:57.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.105+0000 7f49f5200640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f49d406b500 0x7f49d406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.106+0000 7f49f5200640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f49d406b500 0x7f49d406d9c0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f49ec036000 tx=0x7f49ec008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.110+0000 7f49e5ffb640 1 -- 172.21.15.64:0/2250479472 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f49e0057da0 con 0x7f49f0072660 2024-03-04T07:26:57.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.286+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f49f00ad8b0 con 0x7f49d406b500 2024-03-04T07:26:57.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.293+0000 7f49e5ffb640 1 -- 172.21.15.64:0/2250479472 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251948 (secure 0 0 0) 0x7f49f00ad8b0 con 0x7f49d406b500 2024-03-04T07:26:57.295 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:57.296 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f49d406b500 msgr2=0x7f49d406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f49d406b500 0x7f49d406d9c0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f49ec036000 tx=0x7f49ec008030 comp rx=0 tx=0).stop 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f0072660 msgr2=0x7f49f01475a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f0072660 0x7f49f01475a0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f49e000c550 tx=0x7f49e000ca10 comp rx=0 tx=0).stop 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 shutdown_connections 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f49d406b500 0x7f49d406d9c0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f49f00b55d0 0x7f49f015b490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.295+0000 7f49f7464640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f49f0072660 0x7f49f01475a0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.296+0000 7f49f7464640 1 --2- 172.21.15.64:0/2250479472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f49f006e700 0x7f49f00cc4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.296+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 >> 172.21.15.64:0/2250479472 conn(0x7f49f00b2720 msgr2=0x7f49f015d730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:57.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.296+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 shutdown_connections 2024-03-04T07:26:57.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.296+0000 7f49f7464640 1 -- 172.21.15.64:0/2250479472 wait complete. 2024-03-04T07:26:57.306 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-03-04T07:26:57.142622+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":181,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219008,"kb_used_data":2824,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755712,"statfs":{"total":773094113280,"available":772869849088,"internally_reserved":0,"allocated":2891776,"data_stored":1753028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":23,"apply_latency_ms":23,"commit_latency_ns":23000000,"apply_latency_ns":23000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.845538"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389967+0000","last_change":"2024-03-04T07:26:53.331782+0000","last_active":"2024-03-04T07:26:54.389967+0000","last_peered":"2024-03-04T07:26:54.389967+0000","last_clean":"2024-03-04T07:26:54.389967+0000","last_became_active":"2024-03-04T07:26:53.331449+0000","last_became_peered":"2024-03-04T07:26:53.331449+0000","last_unstale":"2024-03-04T07:26:54.389967+0000","last_undegraded":"2024-03-04T07:26:54.389967+0000","last_fullsized":"2024-03-04T07:26:54.389967+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305647+0000","last_change":"2024-03-04T07:26:53.321240+0000","last_active":"2024-03-04T07:26:54.305647+0000","last_peered":"2024-03-04T07:26:54.305647+0000","last_clean":"2024-03-04T07:26:54.305647+0000","last_became_active":"2024-03-04T07:26:53.320945+0000","last_became_peered":"2024-03-04T07:26:53.320945+0000","last_unstale":"2024-03-04T07:26:54.305647+0000","last_undegraded":"2024-03-04T07:26:54.305647+0000","last_fullsized":"2024-03-04T07:26:54.305647+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390253+0000","last_change":"2024-03-04T07:26:53.317216+0000","last_active":"2024-03-04T07:26:54.390253+0000","last_peered":"2024-03-04T07:26:54.390253+0000","last_clean":"2024-03-04T07:26:54.390253+0000","last_became_active":"2024-03-04T07:26:53.316875+0000","last_became_peered":"2024-03-04T07:26:53.316875+0000","last_unstale":"2024-03-04T07:26:54.390253+0000","last_undegraded":"2024-03-04T07:26:54.390253+0000","last_fullsized":"2024-03-04T07:26:54.390253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306389+0000","last_change":"2024-03-04T07:26:53.326427+0000","last_active":"2024-03-04T07:26:54.306389+0000","last_peered":"2024-03-04T07:26:54.306389+0000","last_clean":"2024-03-04T07:26:54.306389+0000","last_became_active":"2024-03-04T07:26:53.326097+0000","last_became_peered":"2024-03-04T07:26:53.326097+0000","last_unstale":"2024-03-04T07:26:54.306389+0000","last_undegraded":"2024-03-04T07:26:54.306389+0000","last_fullsized":"2024-03-04T07:26:54.306389+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389857+0000","last_change":"2024-03-04T07:26:53.334207+0000","last_active":"2024-03-04T07:26:54.389857+0000","last_peered":"2024-03-04T07:26:54.389857+0000","last_clean":"2024-03-04T07:26:54.389857+0000","last_became_active":"2024-03-04T07:26:53.333629+0000","last_became_peered":"2024-03-04T07:26:53.333629+0000","last_unstale":"2024-03-04T07:26:54.389857+0000","last_undegraded":"2024-03-04T07:26:54.389857+0000","last_fullsized":"2024-03-04T07:26:54.389857+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305566+0000","last_change":"2024-03-04T07:26:53.322181+0000","last_active":"2024-03-04T07:26:54.305566+0000","last_peered":"2024-03-04T07:26:54.305566+0000","last_clean":"2024-03-04T07:26:54.305566+0000","last_became_active":"2024-03-04T07:26:53.321619+0000","last_became_peered":"2024-03-04T07:26:53.321619+0000","last_unstale":"2024-03-04T07:26:54.305566+0000","last_undegraded":"2024-03-04T07:26:54.305566+0000","last_fullsized":"2024-03-04T07:26:54.305566+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389617+0000","last_change":"2024-03-04T07:26:53.329485+0000","last_active":"2024-03-04T07:26:54.389617+0000","last_peered":"2024-03-04T07:26:54.389617+0000","last_clean":"2024-03-04T07:26:54.389617+0000","last_became_active":"2024-03-04T07:26:53.329174+0000","last_became_peered":"2024-03-04T07:26:53.329174+0000","last_unstale":"2024-03-04T07:26:54.389617+0000","last_undegraded":"2024-03-04T07:26:54.389617+0000","last_fullsized":"2024-03-04T07:26:54.389617+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389430+0000","last_change":"2024-03-04T07:26:53.327402+0000","last_active":"2024-03-04T07:26:54.389430+0000","last_peered":"2024-03-04T07:26:54.389430+0000","last_clean":"2024-03-04T07:26:54.389430+0000","last_became_active":"2024-03-04T07:26:53.326893+0000","last_became_peered":"2024-03-04T07:26:53.326893+0000","last_unstale":"2024-03-04T07:26:54.389430+0000","last_undegraded":"2024-03-04T07:26:54.389430+0000","last_fullsized":"2024-03-04T07:26:54.389430+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.341109+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.340834+0000","last_became_peered":"2024-03-04T07:26:53.340834+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324909+0000","last_change":"2024-03-04T07:26:53.325503+0000","last_active":"2024-03-04T07:26:54.324909+0000","last_peered":"2024-03-04T07:26:54.324909+0000","last_clean":"2024-03-04T07:26:54.324909+0000","last_became_active":"2024-03-04T07:26:53.325191+0000","last_became_peered":"2024-03-04T07:26:53.325191+0000","last_unstale":"2024-03-04T07:26:54.324909+0000","last_undegraded":"2024-03-04T07:26:54.324909+0000","last_fullsized":"2024-03-04T07:26:54.324909+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324908+0000","last_change":"2024-03-04T07:26:53.327284+0000","last_active":"2024-03-04T07:26:54.324908+0000","last_peered":"2024-03-04T07:26:54.324908+0000","last_clean":"2024-03-04T07:26:54.324908+0000","last_became_active":"2024-03-04T07:26:53.327011+0000","last_became_peered":"2024-03-04T07:26:53.327011+0000","last_unstale":"2024-03-04T07:26:54.324908+0000","last_undegraded":"2024-03-04T07:26:54.324908+0000","last_fullsized":"2024-03-04T07:26:54.324908+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306083+0000","last_change":"2024-03-04T07:26:53.316248+0000","last_active":"2024-03-04T07:26:54.306083+0000","last_peered":"2024-03-04T07:26:54.306083+0000","last_clean":"2024-03-04T07:26:54.306083+0000","last_became_active":"2024-03-04T07:26:53.315903+0000","last_became_peered":"2024-03-04T07:26:53.315903+0000","last_unstale":"2024-03-04T07:26:54.306083+0000","last_undegraded":"2024-03-04T07:26:54.306083+0000","last_fullsized":"2024-03-04T07:26:54.306083+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389868+0000","last_change":"2024-03-04T07:26:53.319814+0000","last_active":"2024-03-04T07:26:54.389868+0000","last_peered":"2024-03-04T07:26:54.389868+0000","last_clean":"2024-03-04T07:26:54.389868+0000","last_became_active":"2024-03-04T07:26:53.319503+0000","last_became_peered":"2024-03-04T07:26:53.319503+0000","last_unstale":"2024-03-04T07:26:54.389868+0000","last_undegraded":"2024-03-04T07:26:54.389868+0000","last_fullsized":"2024-03-04T07:26:54.389868+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389681+0000","last_change":"2024-03-04T07:26:53.318624+0000","last_active":"2024-03-04T07:26:54.389681+0000","last_peered":"2024-03-04T07:26:54.389681+0000","last_clean":"2024-03-04T07:26:54.389681+0000","last_became_active":"2024-03-04T07:26:53.318299+0000","last_became_peered":"2024-03-04T07:26:53.318299+0000","last_unstale":"2024-03-04T07:26:54.389681+0000","last_undegraded":"2024-03-04T07:26:54.389681+0000","last_fullsized":"2024-03-04T07:26:54.389681+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:13:39.358840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305782+0000","last_change":"2024-03-04T07:26:53.319312+0000","last_active":"2024-03-04T07:26:54.305782+0000","last_peered":"2024-03-04T07:26:54.305782+0000","last_clean":"2024-03-04T07:26:54.305782+0000","last_became_active":"2024-03-04T07:26:53.319021+0000","last_became_peered":"2024-03-04T07:26:53.319021+0000","last_unstale":"2024-03-04T07:26:54.305782+0000","last_undegraded":"2024-03-04T07:26:54.305782+0000","last_fullsized":"2024-03-04T07:26:54.305782+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305585+0000","last_change":"2024-03-04T07:26:53.313675+0000","last_active":"2024-03-04T07:26:54.305585+0000","last_peered":"2024-03-04T07:26:54.305585+0000","last_clean":"2024-03-04T07:26:54.305585+0000","last_became_active":"2024-03-04T07:26:53.313325+0000","last_became_peered":"2024-03-04T07:26:53.313325+0000","last_unstale":"2024-03-04T07:26:54.305585+0000","last_undegraded":"2024-03-04T07:26:54.305585+0000","last_fullsized":"2024-03-04T07:26:54.305585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306601+0000","last_change":"2024-03-04T07:26:53.335375+0000","last_active":"2024-03-04T07:26:54.306601+0000","last_peered":"2024-03-04T07:26:54.306601+0000","last_clean":"2024-03-04T07:26:54.306601+0000","last_became_active":"2024-03-04T07:26:53.335101+0000","last_became_peered":"2024-03-04T07:26:53.335101+0000","last_unstale":"2024-03-04T07:26:54.306601+0000","last_undegraded":"2024-03-04T07:26:54.306601+0000","last_fullsized":"2024-03-04T07:26:54.306601+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390303+0000","last_change":"2024-03-04T07:26:53.320188+0000","last_active":"2024-03-04T07:26:54.390303+0000","last_peered":"2024-03-04T07:26:54.390303+0000","last_clean":"2024-03-04T07:26:54.390303+0000","last_became_active":"2024-03-04T07:26:53.319902+0000","last_became_peered":"2024-03-04T07:26:53.319902+0000","last_unstale":"2024-03-04T07:26:54.390303+0000","last_undegraded":"2024-03-04T07:26:54.390303+0000","last_fullsized":"2024-03-04T07:26:54.390303+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390194+0000","last_change":"2024-03-04T07:26:53.334274+0000","last_active":"2024-03-04T07:26:54.390194+0000","last_peered":"2024-03-04T07:26:54.390194+0000","last_clean":"2024-03-04T07:26:54.390194+0000","last_became_active":"2024-03-04T07:26:53.333828+0000","last_became_peered":"2024-03-04T07:26:53.333828+0000","last_unstale":"2024-03-04T07:26:54.390194+0000","last_undegraded":"2024-03-04T07:26:54.390194+0000","last_fullsized":"2024-03-04T07:26:54.390194+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:08:42.738284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325119+0000","last_change":"2024-03-04T07:26:53.322810+0000","last_active":"2024-03-04T07:26:54.325119+0000","last_peered":"2024-03-04T07:26:54.325119+0000","last_clean":"2024-03-04T07:26:54.325119+0000","last_became_active":"2024-03-04T07:26:53.322527+0000","last_became_peered":"2024-03-04T07:26:53.322527+0000","last_unstale":"2024-03-04T07:26:54.325119+0000","last_undegraded":"2024-03-04T07:26:54.325119+0000","last_fullsized":"2024-03-04T07:26:54.325119+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390380+0000","last_change":"2024-03-04T07:26:53.326816+0000","last_active":"2024-03-04T07:26:54.390380+0000","last_peered":"2024-03-04T07:26:54.390380+0000","last_clean":"2024-03-04T07:26:54.390380+0000","last_became_active":"2024-03-04T07:26:53.326028+0000","last_became_peered":"2024-03-04T07:26:53.326028+0000","last_unstale":"2024-03-04T07:26:54.390380+0000","last_undegraded":"2024-03-04T07:26:54.390380+0000","last_fullsized":"2024-03-04T07:26:54.390380+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306215+0000","last_change":"2024-03-04T07:26:53.328614+0000","last_active":"2024-03-04T07:26:54.306215+0000","last_peered":"2024-03-04T07:26:54.306215+0000","last_clean":"2024-03-04T07:26:54.306215+0000","last_became_active":"2024-03-04T07:26:53.328262+0000","last_became_peered":"2024-03-04T07:26:53.328262+0000","last_unstale":"2024-03-04T07:26:54.306215+0000","last_undegraded":"2024-03-04T07:26:54.306215+0000","last_fullsized":"2024-03-04T07:26:54.306215+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390277+0000","last_change":"2024-03-04T07:26:53.322798+0000","last_active":"2024-03-04T07:26:54.390277+0000","last_peered":"2024-03-04T07:26:54.390277+0000","last_clean":"2024-03-04T07:26:54.390277+0000","last_became_active":"2024-03-04T07:26:53.321992+0000","last_became_peered":"2024-03-04T07:26:53.321992+0000","last_unstale":"2024-03-04T07:26:54.390277+0000","last_undegraded":"2024-03-04T07:26:54.390277+0000","last_fullsized":"2024-03-04T07:26:54.390277+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305730+0000","last_change":"2024-03-04T07:26:53.323813+0000","last_active":"2024-03-04T07:26:54.305730+0000","last_peered":"2024-03-04T07:26:54.305730+0000","last_clean":"2024-03-04T07:26:54.305730+0000","last_became_active":"2024-03-04T07:26:53.323530+0000","last_became_peered":"2024-03-04T07:26:53.323530+0000","last_unstale":"2024-03-04T07:26:54.305730+0000","last_undegraded":"2024-03-04T07:26:54.305730+0000","last_fullsized":"2024-03-04T07:26:54.305730+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390125+0000","last_change":"2024-03-04T07:26:53.322537+0000","last_active":"2024-03-04T07:26:54.390125+0000","last_peered":"2024-03-04T07:26:54.390125+0000","last_clean":"2024-03-04T07:26:54.390125+0000","last_became_active":"2024-03-04T07:26:53.322254+0000","last_became_peered":"2024-03-04T07:26:53.322254+0000","last_unstale":"2024-03-04T07:26:54.390125+0000","last_undegraded":"2024-03-04T07:26:54.390125+0000","last_fullsized":"2024-03-04T07:26:54.390125+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325071+0000","last_change":"2024-03-04T07:26:53.332147+0000","last_active":"2024-03-04T07:26:54.325071+0000","last_peered":"2024-03-04T07:26:54.325071+0000","last_clean":"2024-03-04T07:26:54.325071+0000","last_became_active":"2024-03-04T07:26:53.331346+0000","last_became_peered":"2024-03-04T07:26:53.331346+0000","last_unstale":"2024-03-04T07:26:54.325071+0000","last_undegraded":"2024-03-04T07:26:54.325071+0000","last_fullsized":"2024-03-04T07:26:54.325071+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305687+0000","last_change":"2024-03-04T07:26:53.325767+0000","last_active":"2024-03-04T07:26:54.305687+0000","last_peered":"2024-03-04T07:26:54.305687+0000","last_clean":"2024-03-04T07:26:54.305687+0000","last_became_active":"2024-03-04T07:26:53.325486+0000","last_became_peered":"2024-03-04T07:26:53.325486+0000","last_unstale":"2024-03-04T07:26:54.305687+0000","last_undegraded":"2024-03-04T07:26:54.305687+0000","last_fullsized":"2024-03-04T07:26:54.305687+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389819+0000","last_change":"2024-03-04T07:26:53.323242+0000","last_active":"2024-03-04T07:26:54.389819+0000","last_peered":"2024-03-04T07:26:54.389819+0000","last_clean":"2024-03-04T07:26:54.389819+0000","last_became_active":"2024-03-04T07:26:53.322945+0000","last_became_peered":"2024-03-04T07:26:53.322945+0000","last_unstale":"2024-03-04T07:26:54.389819+0000","last_undegraded":"2024-03-04T07:26:54.389819+0000","last_fullsized":"2024-03-04T07:26:54.389819+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390452+0000","last_change":"2024-03-04T07:26:53.321011+0000","last_active":"2024-03-04T07:26:54.390452+0000","last_peered":"2024-03-04T07:26:54.390452+0000","last_clean":"2024-03-04T07:26:54.390452+0000","last_became_active":"2024-03-04T07:26:53.320663+0000","last_became_peered":"2024-03-04T07:26:53.320663+0000","last_unstale":"2024-03-04T07:26:54.390452+0000","last_undegraded":"2024-03-04T07:26:54.390452+0000","last_fullsized":"2024-03-04T07:26:54.390452+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306658+0000","last_change":"2024-03-04T07:26:53.321889+0000","last_active":"2024-03-04T07:26:54.306658+0000","last_peered":"2024-03-04T07:26:54.306658+0000","last_clean":"2024-03-04T07:26:54.306658+0000","last_became_active":"2024-03-04T07:26:53.321304+0000","last_became_peered":"2024-03-04T07:26:53.321304+0000","last_unstale":"2024-03-04T07:26:54.306658+0000","last_undegraded":"2024-03-04T07:26:54.306658+0000","last_fullsized":"2024-03-04T07:26:54.306658+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390060+0000","last_change":"2024-03-04T07:26:53.323893+0000","last_active":"2024-03-04T07:26:54.390060+0000","last_peered":"2024-03-04T07:26:54.390060+0000","last_clean":"2024-03-04T07:26:54.390060+0000","last_became_active":"2024-03-04T07:26:53.323608+0000","last_became_peered":"2024-03-04T07:26:53.323608+0000","last_unstale":"2024-03-04T07:26:54.390060+0000","last_undegraded":"2024-03-04T07:26:54.390060+0000","last_fullsized":"2024-03-04T07:26:54.390060+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324678+0000","last_change":"2024-03-04T07:26:53.318669+0000","last_active":"2024-03-04T07:26:54.324678+0000","last_peered":"2024-03-04T07:26:54.324678+0000","last_clean":"2024-03-04T07:26:54.324678+0000","last_became_active":"2024-03-04T07:26:53.318412+0000","last_became_peered":"2024-03-04T07:26:53.318412+0000","last_unstale":"2024-03-04T07:26:54.324678+0000","last_undegraded":"2024-03-04T07:26:54.324678+0000","last_fullsized":"2024-03-04T07:26:54.324678+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:53.317343+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:53.317041+0000","last_became_peered":"2024-03-04T07:26:53.317041+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:05.230078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390061+0000","last_change":"2024-03-04T07:26:52.326593+0000","last_active":"2024-03-04T07:26:54.390061+0000","last_peered":"2024-03-04T07:26:54.390061+0000","last_clean":"2024-03-04T07:26:54.390061+0000","last_became_active":"2024-03-04T07:26:52.326321+0000","last_became_peered":"2024-03-04T07:26:52.326321+0000","last_unstale":"2024-03-04T07:26:54.390061+0000","last_undegraded":"2024-03-04T07:26:54.390061+0000","last_fullsized":"2024-03-04T07:26:54.390061+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390516+0000","last_change":"2024-03-04T07:26:53.317929+0000","last_active":"2024-03-04T07:26:54.390516+0000","last_peered":"2024-03-04T07:26:54.390516+0000","last_clean":"2024-03-04T07:26:54.390516+0000","last_became_active":"2024-03-04T07:26:53.317631+0000","last_became_peered":"2024-03-04T07:26:53.317631+0000","last_unstale":"2024-03-04T07:26:54.390516+0000","last_undegraded":"2024-03-04T07:26:54.390516+0000","last_fullsized":"2024-03-04T07:26:54.390516+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324662+0000","last_change":"2024-03-04T07:26:52.321659+0000","last_active":"2024-03-04T07:26:54.324662+0000","last_peered":"2024-03-04T07:26:54.324662+0000","last_clean":"2024-03-04T07:26:54.324662+0000","last_became_active":"2024-03-04T07:26:52.321247+0000","last_became_peered":"2024-03-04T07:26:52.321247+0000","last_unstale":"2024-03-04T07:26:54.324662+0000","last_undegraded":"2024-03-04T07:26:54.324662+0000","last_fullsized":"2024-03-04T07:26:54.324662+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390279+0000","last_change":"2024-03-04T07:26:52.316474+0000","last_active":"2024-03-04T07:26:54.390279+0000","last_peered":"2024-03-04T07:26:54.390279+0000","last_clean":"2024-03-04T07:26:54.390279+0000","last_became_active":"2024-03-04T07:26:52.316148+0000","last_became_peered":"2024-03-04T07:26:52.316148+0000","last_unstale":"2024-03-04T07:26:54.390279+0000","last_undegraded":"2024-03-04T07:26:54.390279+0000","last_fullsized":"2024-03-04T07:26:54.390279+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389444+0000","last_change":"2024-03-04T07:26:53.325862+0000","last_active":"2024-03-04T07:26:54.389444+0000","last_peered":"2024-03-04T07:26:54.389444+0000","last_clean":"2024-03-04T07:26:54.389444+0000","last_became_active":"2024-03-04T07:26:53.325450+0000","last_became_peered":"2024-03-04T07:26:53.325450+0000","last_unstale":"2024-03-04T07:26:54.389444+0000","last_undegraded":"2024-03-04T07:26:54.389444+0000","last_fullsized":"2024-03-04T07:26:54.389444+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305831+0000","last_change":"2024-03-04T07:26:52.312838+0000","last_active":"2024-03-04T07:26:54.305831+0000","last_peered":"2024-03-04T07:26:54.305831+0000","last_clean":"2024-03-04T07:26:54.305831+0000","last_became_active":"2024-03-04T07:26:52.312335+0000","last_became_peered":"2024-03-04T07:26:52.312335+0000","last_unstale":"2024-03-04T07:26:54.305831+0000","last_undegraded":"2024-03-04T07:26:54.305831+0000","last_fullsized":"2024-03-04T07:26:54.305831+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305454+0000","last_change":"2024-03-04T07:26:53.324974+0000","last_active":"2024-03-04T07:26:54.305454+0000","last_peered":"2024-03-04T07:26:54.305454+0000","last_clean":"2024-03-04T07:26:54.305454+0000","last_became_active":"2024-03-04T07:26:53.324662+0000","last_became_peered":"2024-03-04T07:26:53.324662+0000","last_unstale":"2024-03-04T07:26:54.305454+0000","last_undegraded":"2024-03-04T07:26:54.305454+0000","last_fullsized":"2024-03-04T07:26:54.305454+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324840+0000","last_change":"2024-03-04T07:26:52.317328+0000","last_active":"2024-03-04T07:26:54.324840+0000","last_peered":"2024-03-04T07:26:54.324840+0000","last_clean":"2024-03-04T07:26:54.324840+0000","last_became_active":"2024-03-04T07:26:52.317039+0000","last_became_peered":"2024-03-04T07:26:52.317039+0000","last_unstale":"2024-03-04T07:26:54.324840+0000","last_undegraded":"2024-03-04T07:26:54.324840+0000","last_fullsized":"2024-03-04T07:26:54.324840+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306365+0000","last_change":"2024-03-04T07:26:53.329677+0000","last_active":"2024-03-04T07:26:54.306365+0000","last_peered":"2024-03-04T07:26:54.306365+0000","last_clean":"2024-03-04T07:26:54.306365+0000","last_became_active":"2024-03-04T07:26:53.327273+0000","last_became_peered":"2024-03-04T07:26:53.327273+0000","last_unstale":"2024-03-04T07:26:54.306365+0000","last_undegraded":"2024-03-04T07:26:54.306365+0000","last_fullsized":"2024-03-04T07:26:54.306365+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306307+0000","last_change":"2024-03-04T07:26:52.317388+0000","last_active":"2024-03-04T07:26:54.306307+0000","last_peered":"2024-03-04T07:26:54.306307+0000","last_clean":"2024-03-04T07:26:54.306307+0000","last_became_active":"2024-03-04T07:26:52.316946+0000","last_became_peered":"2024-03-04T07:26:52.316946+0000","last_unstale":"2024-03-04T07:26:54.306307+0000","last_undegraded":"2024-03-04T07:26:54.306307+0000","last_fullsized":"2024-03-04T07:26:54.306307+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390169+0000","last_change":"2024-03-04T07:26:53.324201+0000","last_active":"2024-03-04T07:26:54.390169+0000","last_peered":"2024-03-04T07:26:54.390169+0000","last_clean":"2024-03-04T07:26:54.390169+0000","last_became_active":"2024-03-04T07:26:53.323923+0000","last_became_peered":"2024-03-04T07:26:53.323923+0000","last_unstale":"2024-03-04T07:26:54.390169+0000","last_undegraded":"2024-03-04T07:26:54.390169+0000","last_fullsized":"2024-03-04T07:26:54.390169+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:50.898693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"19'3","reported_seq":17,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.317701+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.317411+0000","last_became_peered":"2024-03-04T07:26:52.317411+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306245+0000","last_change":"2024-03-04T07:26:53.334757+0000","last_active":"2024-03-04T07:26:54.306245+0000","last_peered":"2024-03-04T07:26:54.306245+0000","last_clean":"2024-03-04T07:26:54.306245+0000","last_became_active":"2024-03-04T07:26:53.331793+0000","last_became_peered":"2024-03-04T07:26:53.331793+0000","last_unstale":"2024-03-04T07:26:54.306245+0000","last_undegraded":"2024-03-04T07:26:54.306245+0000","last_fullsized":"2024-03-04T07:26:54.306245+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:23.419725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"19'6","reported_seq":20,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325093+0000","last_change":"2024-03-04T07:26:52.318084+0000","last_active":"2024-03-04T07:26:54.325093+0000","last_peered":"2024-03-04T07:26:54.325093+0000","last_clean":"2024-03-04T07:26:54.325093+0000","last_became_active":"2024-03-04T07:26:52.317701+0000","last_became_peered":"2024-03-04T07:26:52.317701+0000","last_unstale":"2024-03-04T07:26:54.325093+0000","last_undegraded":"2024-03-04T07:26:54.325093+0000","last_fullsized":"2024-03-04T07:26:54.325093+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390023+0000","last_change":"2024-03-04T07:26:53.331720+0000","last_active":"2024-03-04T07:26:54.390023+0000","last_peered":"2024-03-04T07:26:54.390023+0000","last_clean":"2024-03-04T07:26:54.390023+0000","last_became_active":"2024-03-04T07:26:53.331397+0000","last_became_peered":"2024-03-04T07:26:53.331397+0000","last_unstale":"2024-03-04T07:26:54.390023+0000","last_undegraded":"2024-03-04T07:26:54.390023+0000","last_fullsized":"2024-03-04T07:26:54.390023+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390546+0000","last_change":"2024-03-04T07:26:52.319865+0000","last_active":"2024-03-04T07:26:54.390546+0000","last_peered":"2024-03-04T07:26:54.390546+0000","last_clean":"2024-03-04T07:26:54.390546+0000","last_became_active":"2024-03-04T07:26:52.319514+0000","last_became_peered":"2024-03-04T07:26:52.319514+0000","last_unstale":"2024-03-04T07:26:54.390546+0000","last_undegraded":"2024-03-04T07:26:54.390546+0000","last_fullsized":"2024-03-04T07:26:54.390546+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389831+0000","last_change":"2024-03-04T07:26:53.327475+0000","last_active":"2024-03-04T07:26:54.389831+0000","last_peered":"2024-03-04T07:26:54.389831+0000","last_clean":"2024-03-04T07:26:54.389831+0000","last_became_active":"2024-03-04T07:26:53.325978+0000","last_became_peered":"2024-03-04T07:26:53.325978+0000","last_unstale":"2024-03-04T07:26:54.389831+0000","last_undegraded":"2024-03-04T07:26:54.389831+0000","last_fullsized":"2024-03-04T07:26:54.389831+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:14:35.134882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306345+0000","last_change":"2024-03-04T07:26:52.322481+0000","last_active":"2024-03-04T07:26:54.306345+0000","last_peered":"2024-03-04T07:26:54.306345+0000","last_clean":"2024-03-04T07:26:54.306345+0000","last_became_active":"2024-03-04T07:26:52.322193+0000","last_became_peered":"2024-03-04T07:26:52.322193+0000","last_unstale":"2024-03-04T07:26:54.306345+0000","last_undegraded":"2024-03-04T07:26:54.306345+0000","last_fullsized":"2024-03-04T07:26:54.306345+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389953+0000","last_change":"2024-03-04T07:26:53.339564+0000","last_active":"2024-03-04T07:26:54.389953+0000","last_peered":"2024-03-04T07:26:54.389953+0000","last_clean":"2024-03-04T07:26:54.389953+0000","last_became_active":"2024-03-04T07:26:53.338911+0000","last_became_peered":"2024-03-04T07:26:53.338911+0000","last_unstale":"2024-03-04T07:26:54.389953+0000","last_undegraded":"2024-03-04T07:26:54.389953+0000","last_fullsized":"2024-03-04T07:26:54.389953+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324657+0000","last_change":"2024-03-04T07:26:52.321570+0000","last_active":"2024-03-04T07:26:54.324657+0000","last_peered":"2024-03-04T07:26:54.324657+0000","last_clean":"2024-03-04T07:26:54.324657+0000","last_became_active":"2024-03-04T07:26:52.321193+0000","last_became_peered":"2024-03-04T07:26:52.321193+0000","last_unstale":"2024-03-04T07:26:54.324657+0000","last_undegraded":"2024-03-04T07:26:54.324657+0000","last_fullsized":"2024-03-04T07:26:54.324657+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324683+0000","last_change":"2024-03-04T07:26:53.318654+0000","last_active":"2024-03-04T07:26:54.324683+0000","last_peered":"2024-03-04T07:26:54.324683+0000","last_clean":"2024-03-04T07:26:54.324683+0000","last_became_active":"2024-03-04T07:26:53.318357+0000","last_became_peered":"2024-03-04T07:26:53.318357+0000","last_unstale":"2024-03-04T07:26:54.324683+0000","last_undegraded":"2024-03-04T07:26:54.324683+0000","last_fullsized":"2024-03-04T07:26:54.324683+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"19'8","reported_seq":22,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390444+0000","last_change":"2024-03-04T07:26:52.317772+0000","last_active":"2024-03-04T07:26:54.390444+0000","last_peered":"2024-03-04T07:26:54.390444+0000","last_clean":"2024-03-04T07:26:54.390444+0000","last_became_active":"2024-03-04T07:26:52.316794+0000","last_became_peered":"2024-03-04T07:26:52.316794+0000","last_unstale":"2024-03-04T07:26:54.390444+0000","last_undegraded":"2024-03-04T07:26:54.390444+0000","last_fullsized":"2024-03-04T07:26:54.390444+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389575+0000","last_change":"2024-03-04T07:26:53.340074+0000","last_active":"2024-03-04T07:26:54.389575+0000","last_peered":"2024-03-04T07:26:54.389575+0000","last_clean":"2024-03-04T07:26:54.389575+0000","last_became_active":"2024-03-04T07:26:53.339265+0000","last_became_peered":"2024-03-04T07:26:53.339265+0000","last_unstale":"2024-03-04T07:26:54.389575+0000","last_undegraded":"2024-03-04T07:26:54.389575+0000","last_fullsized":"2024-03-04T07:26:54.389575+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306140+0000","last_change":"2024-03-04T07:26:52.313758+0000","last_active":"2024-03-04T07:26:54.306140+0000","last_peered":"2024-03-04T07:26:54.306140+0000","last_clean":"2024-03-04T07:26:54.306140+0000","last_became_active":"2024-03-04T07:26:52.313362+0000","last_became_peered":"2024-03-04T07:26:52.313362+0000","last_unstale":"2024-03-04T07:26:54.306140+0000","last_undegraded":"2024-03-04T07:26:54.306140+0000","last_fullsized":"2024-03-04T07:26:54.306140+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389920+0000","last_change":"2024-03-04T07:26:53.337849+0000","last_active":"2024-03-04T07:26:54.389920+0000","last_peered":"2024-03-04T07:26:54.389920+0000","last_clean":"2024-03-04T07:26:54.389920+0000","last_became_active":"2024-03-04T07:26:53.337529+0000","last_became_peered":"2024-03-04T07:26:53.337529+0000","last_unstale":"2024-03-04T07:26:54.389920+0000","last_undegraded":"2024-03-04T07:26:54.389920+0000","last_fullsized":"2024-03-04T07:26:54.389920+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390527+0000","last_change":"2024-03-04T07:26:52.318490+0000","last_active":"2024-03-04T07:26:54.390527+0000","last_peered":"2024-03-04T07:26:54.390527+0000","last_clean":"2024-03-04T07:26:54.390527+0000","last_became_active":"2024-03-04T07:26:52.318217+0000","last_became_peered":"2024-03-04T07:26:52.318217+0000","last_unstale":"2024-03-04T07:26:54.390527+0000","last_undegraded":"2024-03-04T07:26:54.390527+0000","last_fullsized":"2024-03-04T07:26:54.390527+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390128+0000","last_change":"2024-03-04T07:26:53.338814+0000","last_active":"2024-03-04T07:26:54.390128+0000","last_peered":"2024-03-04T07:26:54.390128+0000","last_clean":"2024-03-04T07:26:54.390128+0000","last_became_active":"2024-03-04T07:26:53.338090+0000","last_became_peered":"2024-03-04T07:26:53.338090+0000","last_unstale":"2024-03-04T07:26:54.390128+0000","last_undegraded":"2024-03-04T07:26:54.390128+0000","last_fullsized":"2024-03-04T07:26:54.390128+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:33.083393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305758+0000","last_change":"2024-03-04T07:26:52.320092+0000","last_active":"2024-03-04T07:26:54.305758+0000","last_peered":"2024-03-04T07:26:54.305758+0000","last_clean":"2024-03-04T07:26:54.305758+0000","last_became_active":"2024-03-04T07:26:52.319835+0000","last_became_peered":"2024-03-04T07:26:52.319835+0000","last_unstale":"2024-03-04T07:26:54.305758+0000","last_undegraded":"2024-03-04T07:26:54.305758+0000","last_fullsized":"2024-03-04T07:26:54.305758+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306585+0000","last_change":"2024-03-04T07:26:53.325995+0000","last_active":"2024-03-04T07:26:54.306585+0000","last_peered":"2024-03-04T07:26:54.306585+0000","last_clean":"2024-03-04T07:26:54.306585+0000","last_became_active":"2024-03-04T07:26:53.325550+0000","last_became_peered":"2024-03-04T07:26:53.325550+0000","last_unstale":"2024-03-04T07:26:54.306585+0000","last_undegraded":"2024-03-04T07:26:54.306585+0000","last_fullsized":"2024-03-04T07:26:54.306585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389741+0000","last_change":"2024-03-04T07:26:52.318023+0000","last_active":"2024-03-04T07:26:54.389741+0000","last_peered":"2024-03-04T07:26:54.389741+0000","last_clean":"2024-03-04T07:26:54.389741+0000","last_became_active":"2024-03-04T07:26:52.317127+0000","last_became_peered":"2024-03-04T07:26:52.317127+0000","last_unstale":"2024-03-04T07:26:54.389741+0000","last_undegraded":"2024-03-04T07:26:54.389741+0000","last_fullsized":"2024-03-04T07:26:54.389741+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:10.212684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390046+0000","last_change":"2024-03-04T07:26:53.337973+0000","last_active":"2024-03-04T07:26:54.390046+0000","last_peered":"2024-03-04T07:26:54.390046+0000","last_clean":"2024-03-04T07:26:54.390046+0000","last_became_active":"2024-03-04T07:26:53.337551+0000","last_became_peered":"2024-03-04T07:26:53.337551+0000","last_unstale":"2024-03-04T07:26:54.390046+0000","last_undegraded":"2024-03-04T07:26:54.390046+0000","last_fullsized":"2024-03-04T07:26:54.390046+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":170,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389506+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:26:54.389506+0000","last_peered":"2024-03-04T07:26:54.389506+0000","last_clean":"2024-03-04T07:26:54.389506+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:26:54.389506+0000","last_undegraded":"2024-03-04T07:26:54.389506+0000","last_fullsized":"2024-03-04T07:26:54.389506+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306129+0000","last_change":"2024-03-04T07:26:52.317678+0000","last_active":"2024-03-04T07:26:54.306129+0000","last_peered":"2024-03-04T07:26:54.306129+0000","last_clean":"2024-03-04T07:26:54.306129+0000","last_became_active":"2024-03-04T07:26:52.317008+0000","last_became_peered":"2024-03-04T07:26:52.317008+0000","last_unstale":"2024-03-04T07:26:54.306129+0000","last_undegraded":"2024-03-04T07:26:54.306129+0000","last_fullsized":"2024-03-04T07:26:54.306129+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389685+0000","last_change":"2024-03-04T07:26:53.327257+0000","last_active":"2024-03-04T07:26:54.389685+0000","last_peered":"2024-03-04T07:26:54.389685+0000","last_clean":"2024-03-04T07:26:54.389685+0000","last_became_active":"2024-03-04T07:26:53.326960+0000","last_became_peered":"2024-03-04T07:26:53.326960+0000","last_unstale":"2024-03-04T07:26:54.389685+0000","last_undegraded":"2024-03-04T07:26:54.389685+0000","last_fullsized":"2024-03-04T07:26:54.389685+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389451+0000","last_change":"2024-03-04T07:26:52.325578+0000","last_active":"2024-03-04T07:26:54.389451+0000","last_peered":"2024-03-04T07:26:54.389451+0000","last_clean":"2024-03-04T07:26:54.389451+0000","last_became_active":"2024-03-04T07:26:52.325291+0000","last_became_peered":"2024-03-04T07:26:52.325291+0000","last_unstale":"2024-03-04T07:26:54.389451+0000","last_undegraded":"2024-03-04T07:26:54.389451+0000","last_fullsized":"2024-03-04T07:26:54.389451+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:58.480243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389676+0000","last_change":"2024-03-04T07:26:53.309927+0000","last_active":"2024-03-04T07:26:54.389676+0000","last_peered":"2024-03-04T07:26:54.389676+0000","last_clean":"2024-03-04T07:26:54.389676+0000","last_became_active":"2024-03-04T07:26:53.309679+0000","last_became_peered":"2024-03-04T07:26:53.309679+0000","last_unstale":"2024-03-04T07:26:54.389676+0000","last_undegraded":"2024-03-04T07:26:54.389676+0000","last_fullsized":"2024-03-04T07:26:54.389676+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390148+0000","last_change":"2024-03-04T07:26:52.319140+0000","last_active":"2024-03-04T07:26:54.390148+0000","last_peered":"2024-03-04T07:26:54.390148+0000","last_clean":"2024-03-04T07:26:54.390148+0000","last_became_active":"2024-03-04T07:26:52.318861+0000","last_became_peered":"2024-03-04T07:26:52.318861+0000","last_unstale":"2024-03-04T07:26:54.390148+0000","last_undegraded":"2024-03-04T07:26:54.390148+0000","last_fullsized":"2024-03-04T07:26:54.390148+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:03.812984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305804+0000","last_change":"2024-03-04T07:26:53.317717+0000","last_active":"2024-03-04T07:26:54.305804+0000","last_peered":"2024-03-04T07:26:54.305804+0000","last_clean":"2024-03-04T07:26:54.305804+0000","last_became_active":"2024-03-04T07:26:53.317374+0000","last_became_peered":"2024-03-04T07:26:53.317374+0000","last_unstale":"2024-03-04T07:26:54.305804+0000","last_undegraded":"2024-03-04T07:26:54.305804+0000","last_fullsized":"2024-03-04T07:26:54.305804+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306441+0000","last_change":"2024-03-04T07:26:52.318464+0000","last_active":"2024-03-04T07:26:54.306441+0000","last_peered":"2024-03-04T07:26:54.306441+0000","last_clean":"2024-03-04T07:26:54.306441+0000","last_became_active":"2024-03-04T07:26:52.318141+0000","last_became_peered":"2024-03-04T07:26:52.318141+0000","last_unstale":"2024-03-04T07:26:54.306441+0000","last_undegraded":"2024-03-04T07:26:54.306441+0000","last_fullsized":"2024-03-04T07:26:54.306441+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306253+0000","last_change":"2024-03-04T07:26:53.313219+0000","last_active":"2024-03-04T07:26:54.306253+0000","last_peered":"2024-03-04T07:26:54.306253+0000","last_clean":"2024-03-04T07:26:54.306253+0000","last_became_active":"2024-03-04T07:26:53.312894+0000","last_became_peered":"2024-03-04T07:26:53.312894+0000","last_unstale":"2024-03-04T07:26:54.306253+0000","last_undegraded":"2024-03-04T07:26:54.306253+0000","last_fullsized":"2024-03-04T07:26:54.306253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:52.315301+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:52.315023+0000","last_became_peered":"2024-03-04T07:26:52.315023+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306519+0000","last_change":"2024-03-04T07:26:53.324968+0000","last_active":"2024-03-04T07:26:54.306519+0000","last_peered":"2024-03-04T07:26:54.306519+0000","last_clean":"2024-03-04T07:26:54.306519+0000","last_became_active":"2024-03-04T07:26:53.324658+0000","last_became_peered":"2024-03-04T07:26:53.324658+0000","last_unstale":"2024-03-04T07:26:54.306519+0000","last_undegraded":"2024-03-04T07:26:54.306519+0000","last_fullsized":"2024-03-04T07:26:54.306519+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390203+0000","last_change":"2024-03-04T07:26:52.325400+0000","last_active":"2024-03-04T07:26:54.390203+0000","last_peered":"2024-03-04T07:26:54.390203+0000","last_clean":"2024-03-04T07:26:54.390203+0000","last_became_active":"2024-03-04T07:26:52.325067+0000","last_became_peered":"2024-03-04T07:26:52.325067+0000","last_unstale":"2024-03-04T07:26:54.390203+0000","last_undegraded":"2024-03-04T07:26:54.390203+0000","last_fullsized":"2024-03-04T07:26:54.390203+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305544+0000","last_change":"2024-03-04T07:26:53.317693+0000","last_active":"2024-03-04T07:26:54.305544+0000","last_peered":"2024-03-04T07:26:54.305544+0000","last_clean":"2024-03-04T07:26:54.305544+0000","last_became_active":"2024-03-04T07:26:53.317394+0000","last_became_peered":"2024-03-04T07:26:53.317394+0000","last_unstale":"2024-03-04T07:26:54.305544+0000","last_undegraded":"2024-03-04T07:26:54.305544+0000","last_fullsized":"2024-03-04T07:26:54.305544+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"20'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324149+0000","last_change":"2024-03-04T07:26:52.316749+0000","last_active":"2024-03-04T07:26:54.324149+0000","last_peered":"2024-03-04T07:26:54.324149+0000","last_clean":"2024-03-04T07:26:54.324149+0000","last_became_active":"2024-03-04T07:26:52.316450+0000","last_became_peered":"2024-03-04T07:26:52.316450+0000","last_unstale":"2024-03-04T07:26:54.324149+0000","last_undegraded":"2024-03-04T07:26:54.324149+0000","last_fullsized":"2024-03-04T07:26:54.324149+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.331415+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.331128+0000","last_became_peered":"2024-03-04T07:26:53.331128+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324940+0000","last_change":"2024-03-04T07:26:52.325268+0000","last_active":"2024-03-04T07:26:54.324940+0000","last_peered":"2024-03-04T07:26:54.324940+0000","last_clean":"2024-03-04T07:26:54.324940+0000","last_became_active":"2024-03-04T07:26:52.324890+0000","last_became_peered":"2024-03-04T07:26:52.324890+0000","last_unstale":"2024-03-04T07:26:54.324940+0000","last_undegraded":"2024-03-04T07:26:54.324940+0000","last_fullsized":"2024-03-04T07:26:54.324940+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390532+0000","last_change":"2024-03-04T07:26:53.323671+0000","last_active":"2024-03-04T07:26:54.390532+0000","last_peered":"2024-03-04T07:26:54.390532+0000","last_clean":"2024-03-04T07:26:54.390532+0000","last_became_active":"2024-03-04T07:26:53.323120+0000","last_became_peered":"2024-03-04T07:26:53.323120+0000","last_unstale":"2024-03-04T07:26:54.390532+0000","last_undegraded":"2024-03-04T07:26:54.390532+0000","last_fullsized":"2024-03-04T07:26:54.390532+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389452+0000","last_change":"2024-03-04T07:26:52.324959+0000","last_active":"2024-03-04T07:26:54.389452+0000","last_peered":"2024-03-04T07:26:54.389452+0000","last_clean":"2024-03-04T07:26:54.389452+0000","last_became_active":"2024-03-04T07:26:52.324641+0000","last_became_peered":"2024-03-04T07:26:52.324641+0000","last_unstale":"2024-03-04T07:26:54.389452+0000","last_undegraded":"2024-03-04T07:26:54.389452+0000","last_fullsized":"2024-03-04T07:26:54.389452+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:22.417961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305752+0000","last_change":"2024-03-04T07:26:53.318712+0000","last_active":"2024-03-04T07:26:54.305752+0000","last_peered":"2024-03-04T07:26:54.305752+0000","last_clean":"2024-03-04T07:26:54.305752+0000","last_became_active":"2024-03-04T07:26:53.317676+0000","last_became_peered":"2024-03-04T07:26:53.317676+0000","last_unstale":"2024-03-04T07:26:54.305752+0000","last_undegraded":"2024-03-04T07:26:54.305752+0000","last_fullsized":"2024-03-04T07:26:54.305752+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305560+0000","last_change":"2024-03-04T07:26:52.324864+0000","last_active":"2024-03-04T07:26:54.305560+0000","last_peered":"2024-03-04T07:26:54.305560+0000","last_clean":"2024-03-04T07:26:54.305560+0000","last_became_active":"2024-03-04T07:26:52.324553+0000","last_became_peered":"2024-03-04T07:26:52.324553+0000","last_unstale":"2024-03-04T07:26:54.305560+0000","last_undegraded":"2024-03-04T07:26:54.305560+0000","last_fullsized":"2024-03-04T07:26:54.305560+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390011+0000","last_change":"2024-03-04T07:26:53.320516+0000","last_active":"2024-03-04T07:26:54.390011+0000","last_peered":"2024-03-04T07:26:54.390011+0000","last_clean":"2024-03-04T07:26:54.390011+0000","last_became_active":"2024-03-04T07:26:53.320216+0000","last_became_peered":"2024-03-04T07:26:53.320216+0000","last_unstale":"2024-03-04T07:26:54.390011+0000","last_undegraded":"2024-03-04T07:26:54.390011+0000","last_fullsized":"2024-03-04T07:26:54.390011+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.318968+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.318681+0000","last_became_peered":"2024-03-04T07:26:52.318681+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390005+0000","last_change":"2024-03-04T07:26:53.322137+0000","last_active":"2024-03-04T07:26:54.390005+0000","last_peered":"2024-03-04T07:26:54.390005+0000","last_clean":"2024-03-04T07:26:54.390005+0000","last_became_active":"2024-03-04T07:26:53.321716+0000","last_became_peered":"2024-03-04T07:26:53.321716+0000","last_unstale":"2024-03-04T07:26:54.390005+0000","last_undegraded":"2024-03-04T07:26:54.390005+0000","last_fullsized":"2024-03-04T07:26:54.390005+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389972+0000","last_change":"2024-03-04T07:26:52.325680+0000","last_active":"2024-03-04T07:26:54.389972+0000","last_peered":"2024-03-04T07:26:54.389972+0000","last_clean":"2024-03-04T07:26:54.389972+0000","last_became_active":"2024-03-04T07:26:52.324823+0000","last_became_peered":"2024-03-04T07:26:52.324823+0000","last_unstale":"2024-03-04T07:26:54.389972+0000","last_undegraded":"2024-03-04T07:26:54.389972+0000","last_fullsized":"2024-03-04T07:26:54.389972+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324811+0000","last_change":"2024-03-04T07:26:53.322865+0000","last_active":"2024-03-04T07:26:54.324811+0000","last_peered":"2024-03-04T07:26:54.324811+0000","last_clean":"2024-03-04T07:26:54.324811+0000","last_became_active":"2024-03-04T07:26:53.322579+0000","last_became_peered":"2024-03-04T07:26:53.322579+0000","last_unstale":"2024-03-04T07:26:54.324811+0000","last_undegraded":"2024-03-04T07:26:54.324811+0000","last_fullsized":"2024-03-04T07:26:54.324811+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306571+0000","last_change":"2024-03-04T07:26:53.322261+0000","last_active":"2024-03-04T07:26:54.306571+0000","last_peered":"2024-03-04T07:26:54.306571+0000","last_clean":"2024-03-04T07:26:54.306571+0000","last_became_active":"2024-03-04T07:26:53.321959+0000","last_became_peered":"2024-03-04T07:26:53.321959+0000","last_unstale":"2024-03-04T07:26:54.306571+0000","last_undegraded":"2024-03-04T07:26:54.306571+0000","last_fullsized":"2024-03-04T07:26:54.306571+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389738+0000","last_change":"2024-03-04T07:26:52.330362+0000","last_active":"2024-03-04T07:26:54.389738+0000","last_peered":"2024-03-04T07:26:54.389738+0000","last_clean":"2024-03-04T07:26:54.389738+0000","last_became_active":"2024-03-04T07:26:52.329955+0000","last_became_peered":"2024-03-04T07:26:52.329955+0000","last_unstale":"2024-03-04T07:26:54.389738+0000","last_undegraded":"2024-03-04T07:26:54.389738+0000","last_fullsized":"2024-03-04T07:26:54.389738+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306535+0000","last_change":"2024-03-04T07:26:53.321372+0000","last_active":"2024-03-04T07:26:54.306535+0000","last_peered":"2024-03-04T07:26:54.306535+0000","last_clean":"2024-03-04T07:26:54.306535+0000","last_became_active":"2024-03-04T07:26:53.321099+0000","last_became_peered":"2024-03-04T07:26:53.321099+0000","last_unstale":"2024-03-04T07:26:54.306535+0000","last_undegraded":"2024-03-04T07:26:54.306535+0000","last_fullsized":"2024-03-04T07:26:54.306535+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390131+0000","last_change":"2024-03-04T07:26:52.327378+0000","last_active":"2024-03-04T07:26:54.390131+0000","last_peered":"2024-03-04T07:26:54.390131+0000","last_clean":"2024-03-04T07:26:54.390131+0000","last_became_active":"2024-03-04T07:26:52.327042+0000","last_became_peered":"2024-03-04T07:26:52.327042+0000","last_unstale":"2024-03-04T07:26:54.390131+0000","last_undegraded":"2024-03-04T07:26:54.390131+0000","last_fullsized":"2024-03-04T07:26:54.390131+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390026+0000","last_change":"2024-03-04T07:26:53.325350+0000","last_active":"2024-03-04T07:26:54.390026+0000","last_peered":"2024-03-04T07:26:54.390026+0000","last_clean":"2024-03-04T07:26:54.390026+0000","last_became_active":"2024-03-04T07:26:53.325079+0000","last_became_peered":"2024-03-04T07:26:53.325079+0000","last_unstale":"2024-03-04T07:26:54.390026+0000","last_undegraded":"2024-03-04T07:26:54.390026+0000","last_fullsized":"2024-03-04T07:26:54.390026+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:30:21.821563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389813+0000","last_change":"2024-03-04T07:26:52.322256+0000","last_active":"2024-03-04T07:26:54.389813+0000","last_peered":"2024-03-04T07:26:54.389813+0000","last_clean":"2024-03-04T07:26:54.389813+0000","last_became_active":"2024-03-04T07:26:52.321972+0000","last_became_peered":"2024-03-04T07:26:52.321972+0000","last_unstale":"2024-03-04T07:26:54.389813+0000","last_undegraded":"2024-03-04T07:26:54.389813+0000","last_fullsized":"2024-03-04T07:26:54.389813+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389568+0000","last_change":"2024-03-04T07:26:53.324396+0000","last_active":"2024-03-04T07:26:54.389568+0000","last_peered":"2024-03-04T07:26:54.389568+0000","last_clean":"2024-03-04T07:26:54.389568+0000","last_became_active":"2024-03-04T07:26:53.320387+0000","last_became_peered":"2024-03-04T07:26:53.320387+0000","last_unstale":"2024-03-04T07:26:54.389568+0000","last_undegraded":"2024-03-04T07:26:54.389568+0000","last_fullsized":"2024-03-04T07:26:54.389568+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"19'4","reported_seq":18,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390337+0000","last_change":"2024-03-04T07:26:52.324964+0000","last_active":"2024-03-04T07:26:54.390337+0000","last_peered":"2024-03-04T07:26:54.390337+0000","last_clean":"2024-03-04T07:26:54.390337+0000","last_became_active":"2024-03-04T07:26:52.324642+0000","last_became_peered":"2024-03-04T07:26:52.324642+0000","last_unstale":"2024-03-04T07:26:54.390337+0000","last_undegraded":"2024-03-04T07:26:54.390337+0000","last_fullsized":"2024-03-04T07:26:54.390337+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:49.281484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574855,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":784,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":802816,"data_stored":661884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574854,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574854,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27228,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344612,"statfs":{"total":96636764160,"available":96608882688,"internally_reserved":0,"allocated":208896,"data_stored":71024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574855,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":71060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574854,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27236,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344604,"statfs":{"total":96636764160,"available":96608874496,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574854,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":204800,"data_stored":70934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574854,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":212992,"data_stored":71094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574854,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":811008,"data_stored":661940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:26:57.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:26:57.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.448+0000 7f904d920640 1 Processor -- start 2024-03-04T07:26:57.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.448+0000 7f904d920640 1 -- start start 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f904d920640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f904815e000 0x7f9048071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f904d920640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9048162ad0 con 0x7f90480d35e0 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f904d920640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f90480ce970 con 0x7f90480b2c50 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f904d920640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f90480cece0 con 0x7f904815e000 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f90477fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f90480d35e0 0x7f90481623c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42788/0 (socket says 172.21.15.64:42788) 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f90477fe640 1 -- 172.21.15.64:0/3676542185 learned_addr learned my addr 172.21.15.64:0/3676542185 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f9046ffd640 1 --2- 172.21.15.64:0/3676542185 >> v2:172.21.15.66:3300/0 conn(0x7f904815e000 0x7f9048071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f9045ffb640 1 -- 172.21.15.64:0/3676542185 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2301270858 0 0) 0x7f9048162ad0 con 0x7f90480d35e0 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f9045ffb640 1 -- 172.21.15.64:0/3676542185 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9030003520 con 0x7f90480d35e0 2024-03-04T07:26:57.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f9046ffd640 1 -- 172.21.15.64:0/3676542185 >> v1:172.21.15.66:6789/0 conn(0x7f90480b2c50 legacy=0x7f9048161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f9045ffb640 1 -- 172.21.15.64:0/3676542185 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2153914064 0 0) 0x7f90480ce970 con 0x7f90480b2c50 2024-03-04T07:26:57.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f9046ffd640 1 -- 172.21.15.64:0/3676542185 >> v1:172.21.15.64:6789/0 conn(0x7f90480d35e0 legacy=0x7f90481623c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.449+0000 7f9046ffd640 1 -- 172.21.15.64:0/3676542185 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90480cef30 con 0x7f904815e000 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.450+0000 7f9046ffd640 1 --2- 172.21.15.64:0/3676542185 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 0x7f9048071540 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f90400097a0 tx=0x7f904002b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c99c83f2dec6dc89 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.450+0000 7f9045ffb640 1 -- 172.21.15.64:0/3676542185 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9040035020 con 0x7f904815e000 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.450+0000 7f9045ffb640 1 -- 172.21.15.64:0/3676542185 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9040031830 con 0x7f904815e000 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.450+0000 7f9045ffb640 1 -- 172.21.15.64:0/3676542185 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9040031be0 con 0x7f904815e000 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.451+0000 7f904d920640 1 -- 172.21.15.64:0/3676542185 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 msgr2=0x7f9048071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.451+0000 7f904d920640 1 --2- 172.21.15.64:0/3676542185 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 0x7f9048071540 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f90400097a0 tx=0x7f904002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.451+0000 7f904d920640 1 -- 172.21.15.64:0/3676542185 shutdown_connections 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.451+0000 7f904d920640 1 --2- 172.21.15.64:0/3676542185 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 0x7f9048071540 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.451+0000 7f904d920640 1 -- 172.21.15.64:0/3676542185 >> 172.21.15.64:0/3676542185 conn(0x7f90480d3c80 msgr2=0x7f90480d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:57.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.451+0000 7f904d920640 1 -- 172.21.15.64:0/3676542185 shutdown_connections 2024-03-04T07:26:57.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.451+0000 7f904d920640 1 -- 172.21.15.64:0/3676542185 wait complete. 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.452+0000 7f904d920640 1 Processor -- start 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.452+0000 7f904d920640 1 -- start start 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f904d920640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90480b2c50 0x7f9048151640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f904d920640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90480d35e0 0x7f9048160430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f904d920640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 0x7f9048160970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f904d920640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90480b31a0 con 0x7f90480d35e0 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f904d920640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90480ce600 con 0x7f904815e000 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f904d920640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f90480b3320 con 0x7f90480b2c50 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90480d35e0 0x7f9048160430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90480d35e0 0x7f9048160430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40760/0 (socket says 172.21.15.64:40760) 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 -- 172.21.15.64:0/2514426976 learned_addr learned my addr 172.21.15.64:0/2514426976 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f9046ffd640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90480b2c50 0x7f9048151640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90477fe640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 0x7f9048160970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 -- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90480b2c50 msgr2=0x7f9048151640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90480b2c50 0x7f9048151640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 -- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 msgr2=0x7f9048160970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 0x7f9048160970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 -- 172.21.15.64:0/2514426976 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90480cadf0 con 0x7f90480d35e0 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f90467fc640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90480d35e0 0x7f9048160430 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f903c00c3c0 tx=0x7f903c00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f902ffff640 1 -- 172.21.15.64:0/2514426976 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f903c017070 con 0x7f90480d35e0 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f902ffff640 1 -- 172.21.15.64:0/2514426976 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f903c0035f0 con 0x7f90480d35e0 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f902ffff640 1 -- 172.21.15.64:0/2514426976 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f903c0039e0 con 0x7f90480d35e0 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.453+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90480cb0d0 con 0x7f90480d35e0 2024-03-04T07:26:57.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.454+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90480cb690 con 0x7f90480d35e0 2024-03-04T07:26:57.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.455+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9048067ff0 con 0x7f90480d35e0 2024-03-04T07:26:57.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.455+0000 7f902ffff640 1 -- 172.21.15.64:0/2514426976 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f903c01a070 con 0x7f90480d35e0 2024-03-04T07:26:57.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.455+0000 7f902ffff640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f903406b4d0 0x7f903406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.455+0000 7f902ffff640 1 -- 172.21.15.64:0/2514426976 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f903c089440 con 0x7f90480d35e0 2024-03-04T07:26:57.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.455+0000 7f9046ffd640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f903406b4d0 0x7f903406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.456+0000 7f9046ffd640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f903406b4d0 0x7f903406d990 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f9040036000 tx=0x7f904003d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.460+0000 7f902ffff640 1 -- 172.21.15.64:0/2514426976 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f903c057d00 con 0x7f90480d35e0 2024-03-04T07:26:57.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.638+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f90480b72a0 con 0x7f903406b4d0 2024-03-04T07:26:57.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.644+0000 7f902ffff640 1 -- 172.21.15.64:0/2514426976 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251948 (secure 0 0 0) 0x7f90480b72a0 con 0x7f903406b4d0 2024-03-04T07:26:57.645 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:57.647 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:26:57.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.646+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f903406b4d0 msgr2=0x7f903406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.646+0000 7f904d920640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f903406b4d0 0x7f903406d990 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f9040036000 tx=0x7f904003d040 comp rx=0 tx=0).stop 2024-03-04T07:26:57.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.646+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90480d35e0 msgr2=0x7f9048160430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.646+0000 7f904d920640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90480d35e0 0x7f9048160430 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f903c00c3c0 tx=0x7f903c00c880 comp rx=0 tx=0).stop 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 shutdown_connections 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f903406b4d0 0x7f903406d990 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f904815e000 0x7f9048160970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90480d35e0 0x7f9048160430 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 --2- 172.21.15.64:0/2514426976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90480b2c50 0x7f9048151640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 >> 172.21.15.64:0/2514426976 conn(0x7f90480d3c80 msgr2=0x7f90480d1790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 shutdown_connections 2024-03-04T07:26:57.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.647+0000 7f904d920640 1 -- 172.21.15.64:0/2514426976 wait complete. 2024-03-04T07:26:57.657 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-03-04T07:26:57.142622+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":181,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219008,"kb_used_data":2824,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755712,"statfs":{"total":773094113280,"available":772869849088,"internally_reserved":0,"allocated":2891776,"data_stored":1753028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":23,"apply_latency_ms":23,"commit_latency_ns":23000000,"apply_latency_ns":23000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.845538"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389967+0000","last_change":"2024-03-04T07:26:53.331782+0000","last_active":"2024-03-04T07:26:54.389967+0000","last_peered":"2024-03-04T07:26:54.389967+0000","last_clean":"2024-03-04T07:26:54.389967+0000","last_became_active":"2024-03-04T07:26:53.331449+0000","last_became_peered":"2024-03-04T07:26:53.331449+0000","last_unstale":"2024-03-04T07:26:54.389967+0000","last_undegraded":"2024-03-04T07:26:54.389967+0000","last_fullsized":"2024-03-04T07:26:54.389967+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305647+0000","last_change":"2024-03-04T07:26:53.321240+0000","last_active":"2024-03-04T07:26:54.305647+0000","last_peered":"2024-03-04T07:26:54.305647+0000","last_clean":"2024-03-04T07:26:54.305647+0000","last_became_active":"2024-03-04T07:26:53.320945+0000","last_became_peered":"2024-03-04T07:26:53.320945+0000","last_unstale":"2024-03-04T07:26:54.305647+0000","last_undegraded":"2024-03-04T07:26:54.305647+0000","last_fullsized":"2024-03-04T07:26:54.305647+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390253+0000","last_change":"2024-03-04T07:26:53.317216+0000","last_active":"2024-03-04T07:26:54.390253+0000","last_peered":"2024-03-04T07:26:54.390253+0000","last_clean":"2024-03-04T07:26:54.390253+0000","last_became_active":"2024-03-04T07:26:53.316875+0000","last_became_peered":"2024-03-04T07:26:53.316875+0000","last_unstale":"2024-03-04T07:26:54.390253+0000","last_undegraded":"2024-03-04T07:26:54.390253+0000","last_fullsized":"2024-03-04T07:26:54.390253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306389+0000","last_change":"2024-03-04T07:26:53.326427+0000","last_active":"2024-03-04T07:26:54.306389+0000","last_peered":"2024-03-04T07:26:54.306389+0000","last_clean":"2024-03-04T07:26:54.306389+0000","last_became_active":"2024-03-04T07:26:53.326097+0000","last_became_peered":"2024-03-04T07:26:53.326097+0000","last_unstale":"2024-03-04T07:26:54.306389+0000","last_undegraded":"2024-03-04T07:26:54.306389+0000","last_fullsized":"2024-03-04T07:26:54.306389+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389857+0000","last_change":"2024-03-04T07:26:53.334207+0000","last_active":"2024-03-04T07:26:54.389857+0000","last_peered":"2024-03-04T07:26:54.389857+0000","last_clean":"2024-03-04T07:26:54.389857+0000","last_became_active":"2024-03-04T07:26:53.333629+0000","last_became_peered":"2024-03-04T07:26:53.333629+0000","last_unstale":"2024-03-04T07:26:54.389857+0000","last_undegraded":"2024-03-04T07:26:54.389857+0000","last_fullsized":"2024-03-04T07:26:54.389857+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305566+0000","last_change":"2024-03-04T07:26:53.322181+0000","last_active":"2024-03-04T07:26:54.305566+0000","last_peered":"2024-03-04T07:26:54.305566+0000","last_clean":"2024-03-04T07:26:54.305566+0000","last_became_active":"2024-03-04T07:26:53.321619+0000","last_became_peered":"2024-03-04T07:26:53.321619+0000","last_unstale":"2024-03-04T07:26:54.305566+0000","last_undegraded":"2024-03-04T07:26:54.305566+0000","last_fullsized":"2024-03-04T07:26:54.305566+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389617+0000","last_change":"2024-03-04T07:26:53.329485+0000","last_active":"2024-03-04T07:26:54.389617+0000","last_peered":"2024-03-04T07:26:54.389617+0000","last_clean":"2024-03-04T07:26:54.389617+0000","last_became_active":"2024-03-04T07:26:53.329174+0000","last_became_peered":"2024-03-04T07:26:53.329174+0000","last_unstale":"2024-03-04T07:26:54.389617+0000","last_undegraded":"2024-03-04T07:26:54.389617+0000","last_fullsized":"2024-03-04T07:26:54.389617+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389430+0000","last_change":"2024-03-04T07:26:53.327402+0000","last_active":"2024-03-04T07:26:54.389430+0000","last_peered":"2024-03-04T07:26:54.389430+0000","last_clean":"2024-03-04T07:26:54.389430+0000","last_became_active":"2024-03-04T07:26:53.326893+0000","last_became_peered":"2024-03-04T07:26:53.326893+0000","last_unstale":"2024-03-04T07:26:54.389430+0000","last_undegraded":"2024-03-04T07:26:54.389430+0000","last_fullsized":"2024-03-04T07:26:54.389430+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.341109+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.340834+0000","last_became_peered":"2024-03-04T07:26:53.340834+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324909+0000","last_change":"2024-03-04T07:26:53.325503+0000","last_active":"2024-03-04T07:26:54.324909+0000","last_peered":"2024-03-04T07:26:54.324909+0000","last_clean":"2024-03-04T07:26:54.324909+0000","last_became_active":"2024-03-04T07:26:53.325191+0000","last_became_peered":"2024-03-04T07:26:53.325191+0000","last_unstale":"2024-03-04T07:26:54.324909+0000","last_undegraded":"2024-03-04T07:26:54.324909+0000","last_fullsized":"2024-03-04T07:26:54.324909+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324908+0000","last_change":"2024-03-04T07:26:53.327284+0000","last_active":"2024-03-04T07:26:54.324908+0000","last_peered":"2024-03-04T07:26:54.324908+0000","last_clean":"2024-03-04T07:26:54.324908+0000","last_became_active":"2024-03-04T07:26:53.327011+0000","last_became_peered":"2024-03-04T07:26:53.327011+0000","last_unstale":"2024-03-04T07:26:54.324908+0000","last_undegraded":"2024-03-04T07:26:54.324908+0000","last_fullsized":"2024-03-04T07:26:54.324908+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306083+0000","last_change":"2024-03-04T07:26:53.316248+0000","last_active":"2024-03-04T07:26:54.306083+0000","last_peered":"2024-03-04T07:26:54.306083+0000","last_clean":"2024-03-04T07:26:54.306083+0000","last_became_active":"2024-03-04T07:26:53.315903+0000","last_became_peered":"2024-03-04T07:26:53.315903+0000","last_unstale":"2024-03-04T07:26:54.306083+0000","last_undegraded":"2024-03-04T07:26:54.306083+0000","last_fullsized":"2024-03-04T07:26:54.306083+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389868+0000","last_change":"2024-03-04T07:26:53.319814+0000","last_active":"2024-03-04T07:26:54.389868+0000","last_peered":"2024-03-04T07:26:54.389868+0000","last_clean":"2024-03-04T07:26:54.389868+0000","last_became_active":"2024-03-04T07:26:53.319503+0000","last_became_peered":"2024-03-04T07:26:53.319503+0000","last_unstale":"2024-03-04T07:26:54.389868+0000","last_undegraded":"2024-03-04T07:26:54.389868+0000","last_fullsized":"2024-03-04T07:26:54.389868+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389681+0000","last_change":"2024-03-04T07:26:53.318624+0000","last_active":"2024-03-04T07:26:54.389681+0000","last_peered":"2024-03-04T07:26:54.389681+0000","last_clean":"2024-03-04T07:26:54.389681+0000","last_became_active":"2024-03-04T07:26:53.318299+0000","last_became_peered":"2024-03-04T07:26:53.318299+0000","last_unstale":"2024-03-04T07:26:54.389681+0000","last_undegraded":"2024-03-04T07:26:54.389681+0000","last_fullsized":"2024-03-04T07:26:54.389681+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:13:39.358840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305782+0000","last_change":"2024-03-04T07:26:53.319312+0000","last_active":"2024-03-04T07:26:54.305782+0000","last_peered":"2024-03-04T07:26:54.305782+0000","last_clean":"2024-03-04T07:26:54.305782+0000","last_became_active":"2024-03-04T07:26:53.319021+0000","last_became_peered":"2024-03-04T07:26:53.319021+0000","last_unstale":"2024-03-04T07:26:54.305782+0000","last_undegraded":"2024-03-04T07:26:54.305782+0000","last_fullsized":"2024-03-04T07:26:54.305782+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305585+0000","last_change":"2024-03-04T07:26:53.313675+0000","last_active":"2024-03-04T07:26:54.305585+0000","last_peered":"2024-03-04T07:26:54.305585+0000","last_clean":"2024-03-04T07:26:54.305585+0000","last_became_active":"2024-03-04T07:26:53.313325+0000","last_became_peered":"2024-03-04T07:26:53.313325+0000","last_unstale":"2024-03-04T07:26:54.305585+0000","last_undegraded":"2024-03-04T07:26:54.305585+0000","last_fullsized":"2024-03-04T07:26:54.305585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306601+0000","last_change":"2024-03-04T07:26:53.335375+0000","last_active":"2024-03-04T07:26:54.306601+0000","last_peered":"2024-03-04T07:26:54.306601+0000","last_clean":"2024-03-04T07:26:54.306601+0000","last_became_active":"2024-03-04T07:26:53.335101+0000","last_became_peered":"2024-03-04T07:26:53.335101+0000","last_unstale":"2024-03-04T07:26:54.306601+0000","last_undegraded":"2024-03-04T07:26:54.306601+0000","last_fullsized":"2024-03-04T07:26:54.306601+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390303+0000","last_change":"2024-03-04T07:26:53.320188+0000","last_active":"2024-03-04T07:26:54.390303+0000","last_peered":"2024-03-04T07:26:54.390303+0000","last_clean":"2024-03-04T07:26:54.390303+0000","last_became_active":"2024-03-04T07:26:53.319902+0000","last_became_peered":"2024-03-04T07:26:53.319902+0000","last_unstale":"2024-03-04T07:26:54.390303+0000","last_undegraded":"2024-03-04T07:26:54.390303+0000","last_fullsized":"2024-03-04T07:26:54.390303+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390194+0000","last_change":"2024-03-04T07:26:53.334274+0000","last_active":"2024-03-04T07:26:54.390194+0000","last_peered":"2024-03-04T07:26:54.390194+0000","last_clean":"2024-03-04T07:26:54.390194+0000","last_became_active":"2024-03-04T07:26:53.333828+0000","last_became_peered":"2024-03-04T07:26:53.333828+0000","last_unstale":"2024-03-04T07:26:54.390194+0000","last_undegraded":"2024-03-04T07:26:54.390194+0000","last_fullsized":"2024-03-04T07:26:54.390194+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:08:42.738284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325119+0000","last_change":"2024-03-04T07:26:53.322810+0000","last_active":"2024-03-04T07:26:54.325119+0000","last_peered":"2024-03-04T07:26:54.325119+0000","last_clean":"2024-03-04T07:26:54.325119+0000","last_became_active":"2024-03-04T07:26:53.322527+0000","last_became_peered":"2024-03-04T07:26:53.322527+0000","last_unstale":"2024-03-04T07:26:54.325119+0000","last_undegraded":"2024-03-04T07:26:54.325119+0000","last_fullsized":"2024-03-04T07:26:54.325119+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390380+0000","last_change":"2024-03-04T07:26:53.326816+0000","last_active":"2024-03-04T07:26:54.390380+0000","last_peered":"2024-03-04T07:26:54.390380+0000","last_clean":"2024-03-04T07:26:54.390380+0000","last_became_active":"2024-03-04T07:26:53.326028+0000","last_became_peered":"2024-03-04T07:26:53.326028+0000","last_unstale":"2024-03-04T07:26:54.390380+0000","last_undegraded":"2024-03-04T07:26:54.390380+0000","last_fullsized":"2024-03-04T07:26:54.390380+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306215+0000","last_change":"2024-03-04T07:26:53.328614+0000","last_active":"2024-03-04T07:26:54.306215+0000","last_peered":"2024-03-04T07:26:54.306215+0000","last_clean":"2024-03-04T07:26:54.306215+0000","last_became_active":"2024-03-04T07:26:53.328262+0000","last_became_peered":"2024-03-04T07:26:53.328262+0000","last_unstale":"2024-03-04T07:26:54.306215+0000","last_undegraded":"2024-03-04T07:26:54.306215+0000","last_fullsized":"2024-03-04T07:26:54.306215+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390277+0000","last_change":"2024-03-04T07:26:53.322798+0000","last_active":"2024-03-04T07:26:54.390277+0000","last_peered":"2024-03-04T07:26:54.390277+0000","last_clean":"2024-03-04T07:26:54.390277+0000","last_became_active":"2024-03-04T07:26:53.321992+0000","last_became_peered":"2024-03-04T07:26:53.321992+0000","last_unstale":"2024-03-04T07:26:54.390277+0000","last_undegraded":"2024-03-04T07:26:54.390277+0000","last_fullsized":"2024-03-04T07:26:54.390277+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305730+0000","last_change":"2024-03-04T07:26:53.323813+0000","last_active":"2024-03-04T07:26:54.305730+0000","last_peered":"2024-03-04T07:26:54.305730+0000","last_clean":"2024-03-04T07:26:54.305730+0000","last_became_active":"2024-03-04T07:26:53.323530+0000","last_became_peered":"2024-03-04T07:26:53.323530+0000","last_unstale":"2024-03-04T07:26:54.305730+0000","last_undegraded":"2024-03-04T07:26:54.305730+0000","last_fullsized":"2024-03-04T07:26:54.305730+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390125+0000","last_change":"2024-03-04T07:26:53.322537+0000","last_active":"2024-03-04T07:26:54.390125+0000","last_peered":"2024-03-04T07:26:54.390125+0000","last_clean":"2024-03-04T07:26:54.390125+0000","last_became_active":"2024-03-04T07:26:53.322254+0000","last_became_peered":"2024-03-04T07:26:53.322254+0000","last_unstale":"2024-03-04T07:26:54.390125+0000","last_undegraded":"2024-03-04T07:26:54.390125+0000","last_fullsized":"2024-03-04T07:26:54.390125+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325071+0000","last_change":"2024-03-04T07:26:53.332147+0000","last_active":"2024-03-04T07:26:54.325071+0000","last_peered":"2024-03-04T07:26:54.325071+0000","last_clean":"2024-03-04T07:26:54.325071+0000","last_became_active":"2024-03-04T07:26:53.331346+0000","last_became_peered":"2024-03-04T07:26:53.331346+0000","last_unstale":"2024-03-04T07:26:54.325071+0000","last_undegraded":"2024-03-04T07:26:54.325071+0000","last_fullsized":"2024-03-04T07:26:54.325071+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305687+0000","last_change":"2024-03-04T07:26:53.325767+0000","last_active":"2024-03-04T07:26:54.305687+0000","last_peered":"2024-03-04T07:26:54.305687+0000","last_clean":"2024-03-04T07:26:54.305687+0000","last_became_active":"2024-03-04T07:26:53.325486+0000","last_became_peered":"2024-03-04T07:26:53.325486+0000","last_unstale":"2024-03-04T07:26:54.305687+0000","last_undegraded":"2024-03-04T07:26:54.305687+0000","last_fullsized":"2024-03-04T07:26:54.305687+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389819+0000","last_change":"2024-03-04T07:26:53.323242+0000","last_active":"2024-03-04T07:26:54.389819+0000","last_peered":"2024-03-04T07:26:54.389819+0000","last_clean":"2024-03-04T07:26:54.389819+0000","last_became_active":"2024-03-04T07:26:53.322945+0000","last_became_peered":"2024-03-04T07:26:53.322945+0000","last_unstale":"2024-03-04T07:26:54.389819+0000","last_undegraded":"2024-03-04T07:26:54.389819+0000","last_fullsized":"2024-03-04T07:26:54.389819+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390452+0000","last_change":"2024-03-04T07:26:53.321011+0000","last_active":"2024-03-04T07:26:54.390452+0000","last_peered":"2024-03-04T07:26:54.390452+0000","last_clean":"2024-03-04T07:26:54.390452+0000","last_became_active":"2024-03-04T07:26:53.320663+0000","last_became_peered":"2024-03-04T07:26:53.320663+0000","last_unstale":"2024-03-04T07:26:54.390452+0000","last_undegraded":"2024-03-04T07:26:54.390452+0000","last_fullsized":"2024-03-04T07:26:54.390452+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306658+0000","last_change":"2024-03-04T07:26:53.321889+0000","last_active":"2024-03-04T07:26:54.306658+0000","last_peered":"2024-03-04T07:26:54.306658+0000","last_clean":"2024-03-04T07:26:54.306658+0000","last_became_active":"2024-03-04T07:26:53.321304+0000","last_became_peered":"2024-03-04T07:26:53.321304+0000","last_unstale":"2024-03-04T07:26:54.306658+0000","last_undegraded":"2024-03-04T07:26:54.306658+0000","last_fullsized":"2024-03-04T07:26:54.306658+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390060+0000","last_change":"2024-03-04T07:26:53.323893+0000","last_active":"2024-03-04T07:26:54.390060+0000","last_peered":"2024-03-04T07:26:54.390060+0000","last_clean":"2024-03-04T07:26:54.390060+0000","last_became_active":"2024-03-04T07:26:53.323608+0000","last_became_peered":"2024-03-04T07:26:53.323608+0000","last_unstale":"2024-03-04T07:26:54.390060+0000","last_undegraded":"2024-03-04T07:26:54.390060+0000","last_fullsized":"2024-03-04T07:26:54.390060+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324678+0000","last_change":"2024-03-04T07:26:53.318669+0000","last_active":"2024-03-04T07:26:54.324678+0000","last_peered":"2024-03-04T07:26:54.324678+0000","last_clean":"2024-03-04T07:26:54.324678+0000","last_became_active":"2024-03-04T07:26:53.318412+0000","last_became_peered":"2024-03-04T07:26:53.318412+0000","last_unstale":"2024-03-04T07:26:54.324678+0000","last_undegraded":"2024-03-04T07:26:54.324678+0000","last_fullsized":"2024-03-04T07:26:54.324678+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:53.317343+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:53.317041+0000","last_became_peered":"2024-03-04T07:26:53.317041+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:05.230078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390061+0000","last_change":"2024-03-04T07:26:52.326593+0000","last_active":"2024-03-04T07:26:54.390061+0000","last_peered":"2024-03-04T07:26:54.390061+0000","last_clean":"2024-03-04T07:26:54.390061+0000","last_became_active":"2024-03-04T07:26:52.326321+0000","last_became_peered":"2024-03-04T07:26:52.326321+0000","last_unstale":"2024-03-04T07:26:54.390061+0000","last_undegraded":"2024-03-04T07:26:54.390061+0000","last_fullsized":"2024-03-04T07:26:54.390061+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390516+0000","last_change":"2024-03-04T07:26:53.317929+0000","last_active":"2024-03-04T07:26:54.390516+0000","last_peered":"2024-03-04T07:26:54.390516+0000","last_clean":"2024-03-04T07:26:54.390516+0000","last_became_active":"2024-03-04T07:26:53.317631+0000","last_became_peered":"2024-03-04T07:26:53.317631+0000","last_unstale":"2024-03-04T07:26:54.390516+0000","last_undegraded":"2024-03-04T07:26:54.390516+0000","last_fullsized":"2024-03-04T07:26:54.390516+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324662+0000","last_change":"2024-03-04T07:26:52.321659+0000","last_active":"2024-03-04T07:26:54.324662+0000","last_peered":"2024-03-04T07:26:54.324662+0000","last_clean":"2024-03-04T07:26:54.324662+0000","last_became_active":"2024-03-04T07:26:52.321247+0000","last_became_peered":"2024-03-04T07:26:52.321247+0000","last_unstale":"2024-03-04T07:26:54.324662+0000","last_undegraded":"2024-03-04T07:26:54.324662+0000","last_fullsized":"2024-03-04T07:26:54.324662+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390279+0000","last_change":"2024-03-04T07:26:52.316474+0000","last_active":"2024-03-04T07:26:54.390279+0000","last_peered":"2024-03-04T07:26:54.390279+0000","last_clean":"2024-03-04T07:26:54.390279+0000","last_became_active":"2024-03-04T07:26:52.316148+0000","last_became_peered":"2024-03-04T07:26:52.316148+0000","last_unstale":"2024-03-04T07:26:54.390279+0000","last_undegraded":"2024-03-04T07:26:54.390279+0000","last_fullsized":"2024-03-04T07:26:54.390279+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389444+0000","last_change":"2024-03-04T07:26:53.325862+0000","last_active":"2024-03-04T07:26:54.389444+0000","last_peered":"2024-03-04T07:26:54.389444+0000","last_clean":"2024-03-04T07:26:54.389444+0000","last_became_active":"2024-03-04T07:26:53.325450+0000","last_became_peered":"2024-03-04T07:26:53.325450+0000","last_unstale":"2024-03-04T07:26:54.389444+0000","last_undegraded":"2024-03-04T07:26:54.389444+0000","last_fullsized":"2024-03-04T07:26:54.389444+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305831+0000","last_change":"2024-03-04T07:26:52.312838+0000","last_active":"2024-03-04T07:26:54.305831+0000","last_peered":"2024-03-04T07:26:54.305831+0000","last_clean":"2024-03-04T07:26:54.305831+0000","last_became_active":"2024-03-04T07:26:52.312335+0000","last_became_peered":"2024-03-04T07:26:52.312335+0000","last_unstale":"2024-03-04T07:26:54.305831+0000","last_undegraded":"2024-03-04T07:26:54.305831+0000","last_fullsized":"2024-03-04T07:26:54.305831+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305454+0000","last_change":"2024-03-04T07:26:53.324974+0000","last_active":"2024-03-04T07:26:54.305454+0000","last_peered":"2024-03-04T07:26:54.305454+0000","last_clean":"2024-03-04T07:26:54.305454+0000","last_became_active":"2024-03-04T07:26:53.324662+0000","last_became_peered":"2024-03-04T07:26:53.324662+0000","last_unstale":"2024-03-04T07:26:54.305454+0000","last_undegraded":"2024-03-04T07:26:54.305454+0000","last_fullsized":"2024-03-04T07:26:54.305454+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324840+0000","last_change":"2024-03-04T07:26:52.317328+0000","last_active":"2024-03-04T07:26:54.324840+0000","last_peered":"2024-03-04T07:26:54.324840+0000","last_clean":"2024-03-04T07:26:54.324840+0000","last_became_active":"2024-03-04T07:26:52.317039+0000","last_became_peered":"2024-03-04T07:26:52.317039+0000","last_unstale":"2024-03-04T07:26:54.324840+0000","last_undegraded":"2024-03-04T07:26:54.324840+0000","last_fullsized":"2024-03-04T07:26:54.324840+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306365+0000","last_change":"2024-03-04T07:26:53.329677+0000","last_active":"2024-03-04T07:26:54.306365+0000","last_peered":"2024-03-04T07:26:54.306365+0000","last_clean":"2024-03-04T07:26:54.306365+0000","last_became_active":"2024-03-04T07:26:53.327273+0000","last_became_peered":"2024-03-04T07:26:53.327273+0000","last_unstale":"2024-03-04T07:26:54.306365+0000","last_undegraded":"2024-03-04T07:26:54.306365+0000","last_fullsized":"2024-03-04T07:26:54.306365+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306307+0000","last_change":"2024-03-04T07:26:52.317388+0000","last_active":"2024-03-04T07:26:54.306307+0000","last_peered":"2024-03-04T07:26:54.306307+0000","last_clean":"2024-03-04T07:26:54.306307+0000","last_became_active":"2024-03-04T07:26:52.316946+0000","last_became_peered":"2024-03-04T07:26:52.316946+0000","last_unstale":"2024-03-04T07:26:54.306307+0000","last_undegraded":"2024-03-04T07:26:54.306307+0000","last_fullsized":"2024-03-04T07:26:54.306307+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390169+0000","last_change":"2024-03-04T07:26:53.324201+0000","last_active":"2024-03-04T07:26:54.390169+0000","last_peered":"2024-03-04T07:26:54.390169+0000","last_clean":"2024-03-04T07:26:54.390169+0000","last_became_active":"2024-03-04T07:26:53.323923+0000","last_became_peered":"2024-03-04T07:26:53.323923+0000","last_unstale":"2024-03-04T07:26:54.390169+0000","last_undegraded":"2024-03-04T07:26:54.390169+0000","last_fullsized":"2024-03-04T07:26:54.390169+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:50.898693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"19'3","reported_seq":17,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.317701+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.317411+0000","last_became_peered":"2024-03-04T07:26:52.317411+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306245+0000","last_change":"2024-03-04T07:26:53.334757+0000","last_active":"2024-03-04T07:26:54.306245+0000","last_peered":"2024-03-04T07:26:54.306245+0000","last_clean":"2024-03-04T07:26:54.306245+0000","last_became_active":"2024-03-04T07:26:53.331793+0000","last_became_peered":"2024-03-04T07:26:53.331793+0000","last_unstale":"2024-03-04T07:26:54.306245+0000","last_undegraded":"2024-03-04T07:26:54.306245+0000","last_fullsized":"2024-03-04T07:26:54.306245+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:23.419725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"19'6","reported_seq":20,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325093+0000","last_change":"2024-03-04T07:26:52.318084+0000","last_active":"2024-03-04T07:26:54.325093+0000","last_peered":"2024-03-04T07:26:54.325093+0000","last_clean":"2024-03-04T07:26:54.325093+0000","last_became_active":"2024-03-04T07:26:52.317701+0000","last_became_peered":"2024-03-04T07:26:52.317701+0000","last_unstale":"2024-03-04T07:26:54.325093+0000","last_undegraded":"2024-03-04T07:26:54.325093+0000","last_fullsized":"2024-03-04T07:26:54.325093+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390023+0000","last_change":"2024-03-04T07:26:53.331720+0000","last_active":"2024-03-04T07:26:54.390023+0000","last_peered":"2024-03-04T07:26:54.390023+0000","last_clean":"2024-03-04T07:26:54.390023+0000","last_became_active":"2024-03-04T07:26:53.331397+0000","last_became_peered":"2024-03-04T07:26:53.331397+0000","last_unstale":"2024-03-04T07:26:54.390023+0000","last_undegraded":"2024-03-04T07:26:54.390023+0000","last_fullsized":"2024-03-04T07:26:54.390023+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390546+0000","last_change":"2024-03-04T07:26:52.319865+0000","last_active":"2024-03-04T07:26:54.390546+0000","last_peered":"2024-03-04T07:26:54.390546+0000","last_clean":"2024-03-04T07:26:54.390546+0000","last_became_active":"2024-03-04T07:26:52.319514+0000","last_became_peered":"2024-03-04T07:26:52.319514+0000","last_unstale":"2024-03-04T07:26:54.390546+0000","last_undegraded":"2024-03-04T07:26:54.390546+0000","last_fullsized":"2024-03-04T07:26:54.390546+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389831+0000","last_change":"2024-03-04T07:26:53.327475+0000","last_active":"2024-03-04T07:26:54.389831+0000","last_peered":"2024-03-04T07:26:54.389831+0000","last_clean":"2024-03-04T07:26:54.389831+0000","last_became_active":"2024-03-04T07:26:53.325978+0000","last_became_peered":"2024-03-04T07:26:53.325978+0000","last_unstale":"2024-03-04T07:26:54.389831+0000","last_undegraded":"2024-03-04T07:26:54.389831+0000","last_fullsized":"2024-03-04T07:26:54.389831+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:14:35.134882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306345+0000","last_change":"2024-03-04T07:26:52.322481+0000","last_active":"2024-03-04T07:26:54.306345+0000","last_peered":"2024-03-04T07:26:54.306345+0000","last_clean":"2024-03-04T07:26:54.306345+0000","last_became_active":"2024-03-04T07:26:52.322193+0000","last_became_peered":"2024-03-04T07:26:52.322193+0000","last_unstale":"2024-03-04T07:26:54.306345+0000","last_undegraded":"2024-03-04T07:26:54.306345+0000","last_fullsized":"2024-03-04T07:26:54.306345+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389953+0000","last_change":"2024-03-04T07:26:53.339564+0000","last_active":"2024-03-04T07:26:54.389953+0000","last_peered":"2024-03-04T07:26:54.389953+0000","last_clean":"2024-03-04T07:26:54.389953+0000","last_became_active":"2024-03-04T07:26:53.338911+0000","last_became_peered":"2024-03-04T07:26:53.338911+0000","last_unstale":"2024-03-04T07:26:54.389953+0000","last_undegraded":"2024-03-04T07:26:54.389953+0000","last_fullsized":"2024-03-04T07:26:54.389953+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324657+0000","last_change":"2024-03-04T07:26:52.321570+0000","last_active":"2024-03-04T07:26:54.324657+0000","last_peered":"2024-03-04T07:26:54.324657+0000","last_clean":"2024-03-04T07:26:54.324657+0000","last_became_active":"2024-03-04T07:26:52.321193+0000","last_became_peered":"2024-03-04T07:26:52.321193+0000","last_unstale":"2024-03-04T07:26:54.324657+0000","last_undegraded":"2024-03-04T07:26:54.324657+0000","last_fullsized":"2024-03-04T07:26:54.324657+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324683+0000","last_change":"2024-03-04T07:26:53.318654+0000","last_active":"2024-03-04T07:26:54.324683+0000","last_peered":"2024-03-04T07:26:54.324683+0000","last_clean":"2024-03-04T07:26:54.324683+0000","last_became_active":"2024-03-04T07:26:53.318357+0000","last_became_peered":"2024-03-04T07:26:53.318357+0000","last_unstale":"2024-03-04T07:26:54.324683+0000","last_undegraded":"2024-03-04T07:26:54.324683+0000","last_fullsized":"2024-03-04T07:26:54.324683+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"19'8","reported_seq":22,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390444+0000","last_change":"2024-03-04T07:26:52.317772+0000","last_active":"2024-03-04T07:26:54.390444+0000","last_peered":"2024-03-04T07:26:54.390444+0000","last_clean":"2024-03-04T07:26:54.390444+0000","last_became_active":"2024-03-04T07:26:52.316794+0000","last_became_peered":"2024-03-04T07:26:52.316794+0000","last_unstale":"2024-03-04T07:26:54.390444+0000","last_undegraded":"2024-03-04T07:26:54.390444+0000","last_fullsized":"2024-03-04T07:26:54.390444+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389575+0000","last_change":"2024-03-04T07:26:53.340074+0000","last_active":"2024-03-04T07:26:54.389575+0000","last_peered":"2024-03-04T07:26:54.389575+0000","last_clean":"2024-03-04T07:26:54.389575+0000","last_became_active":"2024-03-04T07:26:53.339265+0000","last_became_peered":"2024-03-04T07:26:53.339265+0000","last_unstale":"2024-03-04T07:26:54.389575+0000","last_undegraded":"2024-03-04T07:26:54.389575+0000","last_fullsized":"2024-03-04T07:26:54.389575+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306140+0000","last_change":"2024-03-04T07:26:52.313758+0000","last_active":"2024-03-04T07:26:54.306140+0000","last_peered":"2024-03-04T07:26:54.306140+0000","last_clean":"2024-03-04T07:26:54.306140+0000","last_became_active":"2024-03-04T07:26:52.313362+0000","last_became_peered":"2024-03-04T07:26:52.313362+0000","last_unstale":"2024-03-04T07:26:54.306140+0000","last_undegraded":"2024-03-04T07:26:54.306140+0000","last_fullsized":"2024-03-04T07:26:54.306140+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389920+0000","last_change":"2024-03-04T07:26:53.337849+0000","last_active":"2024-03-04T07:26:54.389920+0000","last_peered":"2024-03-04T07:26:54.389920+0000","last_clean":"2024-03-04T07:26:54.389920+0000","last_became_active":"2024-03-04T07:26:53.337529+0000","last_became_peered":"2024-03-04T07:26:53.337529+0000","last_unstale":"2024-03-04T07:26:54.389920+0000","last_undegraded":"2024-03-04T07:26:54.389920+0000","last_fullsized":"2024-03-04T07:26:54.389920+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390527+0000","last_change":"2024-03-04T07:26:52.318490+0000","last_active":"2024-03-04T07:26:54.390527+0000","last_peered":"2024-03-04T07:26:54.390527+0000","last_clean":"2024-03-04T07:26:54.390527+0000","last_became_active":"2024-03-04T07:26:52.318217+0000","last_became_peered":"2024-03-04T07:26:52.318217+0000","last_unstale":"2024-03-04T07:26:54.390527+0000","last_undegraded":"2024-03-04T07:26:54.390527+0000","last_fullsized":"2024-03-04T07:26:54.390527+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390128+0000","last_change":"2024-03-04T07:26:53.338814+0000","last_active":"2024-03-04T07:26:54.390128+0000","last_peered":"2024-03-04T07:26:54.390128+0000","last_clean":"2024-03-04T07:26:54.390128+0000","last_became_active":"2024-03-04T07:26:53.338090+0000","last_became_peered":"2024-03-04T07:26:53.338090+0000","last_unstale":"2024-03-04T07:26:54.390128+0000","last_undegraded":"2024-03-04T07:26:54.390128+0000","last_fullsized":"2024-03-04T07:26:54.390128+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:33.083393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305758+0000","last_change":"2024-03-04T07:26:52.320092+0000","last_active":"2024-03-04T07:26:54.305758+0000","last_peered":"2024-03-04T07:26:54.305758+0000","last_clean":"2024-03-04T07:26:54.305758+0000","last_became_active":"2024-03-04T07:26:52.319835+0000","last_became_peered":"2024-03-04T07:26:52.319835+0000","last_unstale":"2024-03-04T07:26:54.305758+0000","last_undegraded":"2024-03-04T07:26:54.305758+0000","last_fullsized":"2024-03-04T07:26:54.305758+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306585+0000","last_change":"2024-03-04T07:26:53.325995+0000","last_active":"2024-03-04T07:26:54.306585+0000","last_peered":"2024-03-04T07:26:54.306585+0000","last_clean":"2024-03-04T07:26:54.306585+0000","last_became_active":"2024-03-04T07:26:53.325550+0000","last_became_peered":"2024-03-04T07:26:53.325550+0000","last_unstale":"2024-03-04T07:26:54.306585+0000","last_undegraded":"2024-03-04T07:26:54.306585+0000","last_fullsized":"2024-03-04T07:26:54.306585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389741+0000","last_change":"2024-03-04T07:26:52.318023+0000","last_active":"2024-03-04T07:26:54.389741+0000","last_peered":"2024-03-04T07:26:54.389741+0000","last_clean":"2024-03-04T07:26:54.389741+0000","last_became_active":"2024-03-04T07:26:52.317127+0000","last_became_peered":"2024-03-04T07:26:52.317127+0000","last_unstale":"2024-03-04T07:26:54.389741+0000","last_undegraded":"2024-03-04T07:26:54.389741+0000","last_fullsized":"2024-03-04T07:26:54.389741+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:10.212684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390046+0000","last_change":"2024-03-04T07:26:53.337973+0000","last_active":"2024-03-04T07:26:54.390046+0000","last_peered":"2024-03-04T07:26:54.390046+0000","last_clean":"2024-03-04T07:26:54.390046+0000","last_became_active":"2024-03-04T07:26:53.337551+0000","last_became_peered":"2024-03-04T07:26:53.337551+0000","last_unstale":"2024-03-04T07:26:54.390046+0000","last_undegraded":"2024-03-04T07:26:54.390046+0000","last_fullsized":"2024-03-04T07:26:54.390046+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":170,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389506+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:26:54.389506+0000","last_peered":"2024-03-04T07:26:54.389506+0000","last_clean":"2024-03-04T07:26:54.389506+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:26:54.389506+0000","last_undegraded":"2024-03-04T07:26:54.389506+0000","last_fullsized":"2024-03-04T07:26:54.389506+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306129+0000","last_change":"2024-03-04T07:26:52.317678+0000","last_active":"2024-03-04T07:26:54.306129+0000","last_peered":"2024-03-04T07:26:54.306129+0000","last_clean":"2024-03-04T07:26:54.306129+0000","last_became_active":"2024-03-04T07:26:52.317008+0000","last_became_peered":"2024-03-04T07:26:52.317008+0000","last_unstale":"2024-03-04T07:26:54.306129+0000","last_undegraded":"2024-03-04T07:26:54.306129+0000","last_fullsized":"2024-03-04T07:26:54.306129+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389685+0000","last_change":"2024-03-04T07:26:53.327257+0000","last_active":"2024-03-04T07:26:54.389685+0000","last_peered":"2024-03-04T07:26:54.389685+0000","last_clean":"2024-03-04T07:26:54.389685+0000","last_became_active":"2024-03-04T07:26:53.326960+0000","last_became_peered":"2024-03-04T07:26:53.326960+0000","last_unstale":"2024-03-04T07:26:54.389685+0000","last_undegraded":"2024-03-04T07:26:54.389685+0000","last_fullsized":"2024-03-04T07:26:54.389685+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389451+0000","last_change":"2024-03-04T07:26:52.325578+0000","last_active":"2024-03-04T07:26:54.389451+0000","last_peered":"2024-03-04T07:26:54.389451+0000","last_clean":"2024-03-04T07:26:54.389451+0000","last_became_active":"2024-03-04T07:26:52.325291+0000","last_became_peered":"2024-03-04T07:26:52.325291+0000","last_unstale":"2024-03-04T07:26:54.389451+0000","last_undegraded":"2024-03-04T07:26:54.389451+0000","last_fullsized":"2024-03-04T07:26:54.389451+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:58.480243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389676+0000","last_change":"2024-03-04T07:26:53.309927+0000","last_active":"2024-03-04T07:26:54.389676+0000","last_peered":"2024-03-04T07:26:54.389676+0000","last_clean":"2024-03-04T07:26:54.389676+0000","last_became_active":"2024-03-04T07:26:53.309679+0000","last_became_peered":"2024-03-04T07:26:53.309679+0000","last_unstale":"2024-03-04T07:26:54.389676+0000","last_undegraded":"2024-03-04T07:26:54.389676+0000","last_fullsized":"2024-03-04T07:26:54.389676+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390148+0000","last_change":"2024-03-04T07:26:52.319140+0000","last_active":"2024-03-04T07:26:54.390148+0000","last_peered":"2024-03-04T07:26:54.390148+0000","last_clean":"2024-03-04T07:26:54.390148+0000","last_became_active":"2024-03-04T07:26:52.318861+0000","last_became_peered":"2024-03-04T07:26:52.318861+0000","last_unstale":"2024-03-04T07:26:54.390148+0000","last_undegraded":"2024-03-04T07:26:54.390148+0000","last_fullsized":"2024-03-04T07:26:54.390148+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:03.812984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305804+0000","last_change":"2024-03-04T07:26:53.317717+0000","last_active":"2024-03-04T07:26:54.305804+0000","last_peered":"2024-03-04T07:26:54.305804+0000","last_clean":"2024-03-04T07:26:54.305804+0000","last_became_active":"2024-03-04T07:26:53.317374+0000","last_became_peered":"2024-03-04T07:26:53.317374+0000","last_unstale":"2024-03-04T07:26:54.305804+0000","last_undegraded":"2024-03-04T07:26:54.305804+0000","last_fullsized":"2024-03-04T07:26:54.305804+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306441+0000","last_change":"2024-03-04T07:26:52.318464+0000","last_active":"2024-03-04T07:26:54.306441+0000","last_peered":"2024-03-04T07:26:54.306441+0000","last_clean":"2024-03-04T07:26:54.306441+0000","last_became_active":"2024-03-04T07:26:52.318141+0000","last_became_peered":"2024-03-04T07:26:52.318141+0000","last_unstale":"2024-03-04T07:26:54.306441+0000","last_undegraded":"2024-03-04T07:26:54.306441+0000","last_fullsized":"2024-03-04T07:26:54.306441+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306253+0000","last_change":"2024-03-04T07:26:53.313219+0000","last_active":"2024-03-04T07:26:54.306253+0000","last_peered":"2024-03-04T07:26:54.306253+0000","last_clean":"2024-03-04T07:26:54.306253+0000","last_became_active":"2024-03-04T07:26:53.312894+0000","last_became_peered":"2024-03-04T07:26:53.312894+0000","last_unstale":"2024-03-04T07:26:54.306253+0000","last_undegraded":"2024-03-04T07:26:54.306253+0000","last_fullsized":"2024-03-04T07:26:54.306253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:52.315301+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:52.315023+0000","last_became_peered":"2024-03-04T07:26:52.315023+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306519+0000","last_change":"2024-03-04T07:26:53.324968+0000","last_active":"2024-03-04T07:26:54.306519+0000","last_peered":"2024-03-04T07:26:54.306519+0000","last_clean":"2024-03-04T07:26:54.306519+0000","last_became_active":"2024-03-04T07:26:53.324658+0000","last_became_peered":"2024-03-04T07:26:53.324658+0000","last_unstale":"2024-03-04T07:26:54.306519+0000","last_undegraded":"2024-03-04T07:26:54.306519+0000","last_fullsized":"2024-03-04T07:26:54.306519+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390203+0000","last_change":"2024-03-04T07:26:52.325400+0000","last_active":"2024-03-04T07:26:54.390203+0000","last_peered":"2024-03-04T07:26:54.390203+0000","last_clean":"2024-03-04T07:26:54.390203+0000","last_became_active":"2024-03-04T07:26:52.325067+0000","last_became_peered":"2024-03-04T07:26:52.325067+0000","last_unstale":"2024-03-04T07:26:54.390203+0000","last_undegraded":"2024-03-04T07:26:54.390203+0000","last_fullsized":"2024-03-04T07:26:54.390203+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305544+0000","last_change":"2024-03-04T07:26:53.317693+0000","last_active":"2024-03-04T07:26:54.305544+0000","last_peered":"2024-03-04T07:26:54.305544+0000","last_clean":"2024-03-04T07:26:54.305544+0000","last_became_active":"2024-03-04T07:26:53.317394+0000","last_became_peered":"2024-03-04T07:26:53.317394+0000","last_unstale":"2024-03-04T07:26:54.305544+0000","last_undegraded":"2024-03-04T07:26:54.305544+0000","last_fullsized":"2024-03-04T07:26:54.305544+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"20'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324149+0000","last_change":"2024-03-04T07:26:52.316749+0000","last_active":"2024-03-04T07:26:54.324149+0000","last_peered":"2024-03-04T07:26:54.324149+0000","last_clean":"2024-03-04T07:26:54.324149+0000","last_became_active":"2024-03-04T07:26:52.316450+0000","last_became_peered":"2024-03-04T07:26:52.316450+0000","last_unstale":"2024-03-04T07:26:54.324149+0000","last_undegraded":"2024-03-04T07:26:54.324149+0000","last_fullsized":"2024-03-04T07:26:54.324149+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.331415+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.331128+0000","last_became_peered":"2024-03-04T07:26:53.331128+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324940+0000","last_change":"2024-03-04T07:26:52.325268+0000","last_active":"2024-03-04T07:26:54.324940+0000","last_peered":"2024-03-04T07:26:54.324940+0000","last_clean":"2024-03-04T07:26:54.324940+0000","last_became_active":"2024-03-04T07:26:52.324890+0000","last_became_peered":"2024-03-04T07:26:52.324890+0000","last_unstale":"2024-03-04T07:26:54.324940+0000","last_undegraded":"2024-03-04T07:26:54.324940+0000","last_fullsized":"2024-03-04T07:26:54.324940+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390532+0000","last_change":"2024-03-04T07:26:53.323671+0000","last_active":"2024-03-04T07:26:54.390532+0000","last_peered":"2024-03-04T07:26:54.390532+0000","last_clean":"2024-03-04T07:26:54.390532+0000","last_became_active":"2024-03-04T07:26:53.323120+0000","last_became_peered":"2024-03-04T07:26:53.323120+0000","last_unstale":"2024-03-04T07:26:54.390532+0000","last_undegraded":"2024-03-04T07:26:54.390532+0000","last_fullsized":"2024-03-04T07:26:54.390532+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389452+0000","last_change":"2024-03-04T07:26:52.324959+0000","last_active":"2024-03-04T07:26:54.389452+0000","last_peered":"2024-03-04T07:26:54.389452+0000","last_clean":"2024-03-04T07:26:54.389452+0000","last_became_active":"2024-03-04T07:26:52.324641+0000","last_became_peered":"2024-03-04T07:26:52.324641+0000","last_unstale":"2024-03-04T07:26:54.389452+0000","last_undegraded":"2024-03-04T07:26:54.389452+0000","last_fullsized":"2024-03-04T07:26:54.389452+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:22.417961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305752+0000","last_change":"2024-03-04T07:26:53.318712+0000","last_active":"2024-03-04T07:26:54.305752+0000","last_peered":"2024-03-04T07:26:54.305752+0000","last_clean":"2024-03-04T07:26:54.305752+0000","last_became_active":"2024-03-04T07:26:53.317676+0000","last_became_peered":"2024-03-04T07:26:53.317676+0000","last_unstale":"2024-03-04T07:26:54.305752+0000","last_undegraded":"2024-03-04T07:26:54.305752+0000","last_fullsized":"2024-03-04T07:26:54.305752+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305560+0000","last_change":"2024-03-04T07:26:52.324864+0000","last_active":"2024-03-04T07:26:54.305560+0000","last_peered":"2024-03-04T07:26:54.305560+0000","last_clean":"2024-03-04T07:26:54.305560+0000","last_became_active":"2024-03-04T07:26:52.324553+0000","last_became_peered":"2024-03-04T07:26:52.324553+0000","last_unstale":"2024-03-04T07:26:54.305560+0000","last_undegraded":"2024-03-04T07:26:54.305560+0000","last_fullsized":"2024-03-04T07:26:54.305560+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390011+0000","last_change":"2024-03-04T07:26:53.320516+0000","last_active":"2024-03-04T07:26:54.390011+0000","last_peered":"2024-03-04T07:26:54.390011+0000","last_clean":"2024-03-04T07:26:54.390011+0000","last_became_active":"2024-03-04T07:26:53.320216+0000","last_became_peered":"2024-03-04T07:26:53.320216+0000","last_unstale":"2024-03-04T07:26:54.390011+0000","last_undegraded":"2024-03-04T07:26:54.390011+0000","last_fullsized":"2024-03-04T07:26:54.390011+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.318968+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.318681+0000","last_became_peered":"2024-03-04T07:26:52.318681+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390005+0000","last_change":"2024-03-04T07:26:53.322137+0000","last_active":"2024-03-04T07:26:54.390005+0000","last_peered":"2024-03-04T07:26:54.390005+0000","last_clean":"2024-03-04T07:26:54.390005+0000","last_became_active":"2024-03-04T07:26:53.321716+0000","last_became_peered":"2024-03-04T07:26:53.321716+0000","last_unstale":"2024-03-04T07:26:54.390005+0000","last_undegraded":"2024-03-04T07:26:54.390005+0000","last_fullsized":"2024-03-04T07:26:54.390005+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389972+0000","last_change":"2024-03-04T07:26:52.325680+0000","last_active":"2024-03-04T07:26:54.389972+0000","last_peered":"2024-03-04T07:26:54.389972+0000","last_clean":"2024-03-04T07:26:54.389972+0000","last_became_active":"2024-03-04T07:26:52.324823+0000","last_became_peered":"2024-03-04T07:26:52.324823+0000","last_unstale":"2024-03-04T07:26:54.389972+0000","last_undegraded":"2024-03-04T07:26:54.389972+0000","last_fullsized":"2024-03-04T07:26:54.389972+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324811+0000","last_change":"2024-03-04T07:26:53.322865+0000","last_active":"2024-03-04T07:26:54.324811+0000","last_peered":"2024-03-04T07:26:54.324811+0000","last_clean":"2024-03-04T07:26:54.324811+0000","last_became_active":"2024-03-04T07:26:53.322579+0000","last_became_peered":"2024-03-04T07:26:53.322579+0000","last_unstale":"2024-03-04T07:26:54.324811+0000","last_undegraded":"2024-03-04T07:26:54.324811+0000","last_fullsized":"2024-03-04T07:26:54.324811+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306571+0000","last_change":"2024-03-04T07:26:53.322261+0000","last_active":"2024-03-04T07:26:54.306571+0000","last_peered":"2024-03-04T07:26:54.306571+0000","last_clean":"2024-03-04T07:26:54.306571+0000","last_became_active":"2024-03-04T07:26:53.321959+0000","last_became_peered":"2024-03-04T07:26:53.321959+0000","last_unstale":"2024-03-04T07:26:54.306571+0000","last_undegraded":"2024-03-04T07:26:54.306571+0000","last_fullsized":"2024-03-04T07:26:54.306571+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389738+0000","last_change":"2024-03-04T07:26:52.330362+0000","last_active":"2024-03-04T07:26:54.389738+0000","last_peered":"2024-03-04T07:26:54.389738+0000","last_clean":"2024-03-04T07:26:54.389738+0000","last_became_active":"2024-03-04T07:26:52.329955+0000","last_became_peered":"2024-03-04T07:26:52.329955+0000","last_unstale":"2024-03-04T07:26:54.389738+0000","last_undegraded":"2024-03-04T07:26:54.389738+0000","last_fullsized":"2024-03-04T07:26:54.389738+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306535+0000","last_change":"2024-03-04T07:26:53.321372+0000","last_active":"2024-03-04T07:26:54.306535+0000","last_peered":"2024-03-04T07:26:54.306535+0000","last_clean":"2024-03-04T07:26:54.306535+0000","last_became_active":"2024-03-04T07:26:53.321099+0000","last_became_peered":"2024-03-04T07:26:53.321099+0000","last_unstale":"2024-03-04T07:26:54.306535+0000","last_undegraded":"2024-03-04T07:26:54.306535+0000","last_fullsized":"2024-03-04T07:26:54.306535+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390131+0000","last_change":"2024-03-04T07:26:52.327378+0000","last_active":"2024-03-04T07:26:54.390131+0000","last_peered":"2024-03-04T07:26:54.390131+0000","last_clean":"2024-03-04T07:26:54.390131+0000","last_became_active":"2024-03-04T07:26:52.327042+0000","last_became_peered":"2024-03-04T07:26:52.327042+0000","last_unstale":"2024-03-04T07:26:54.390131+0000","last_undegraded":"2024-03-04T07:26:54.390131+0000","last_fullsized":"2024-03-04T07:26:54.390131+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390026+0000","last_change":"2024-03-04T07:26:53.325350+0000","last_active":"2024-03-04T07:26:54.390026+0000","last_peered":"2024-03-04T07:26:54.390026+0000","last_clean":"2024-03-04T07:26:54.390026+0000","last_became_active":"2024-03-04T07:26:53.325079+0000","last_became_peered":"2024-03-04T07:26:53.325079+0000","last_unstale":"2024-03-04T07:26:54.390026+0000","last_undegraded":"2024-03-04T07:26:54.390026+0000","last_fullsized":"2024-03-04T07:26:54.390026+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:30:21.821563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389813+0000","last_change":"2024-03-04T07:26:52.322256+0000","last_active":"2024-03-04T07:26:54.389813+0000","last_peered":"2024-03-04T07:26:54.389813+0000","last_clean":"2024-03-04T07:26:54.389813+0000","last_became_active":"2024-03-04T07:26:52.321972+0000","last_became_peered":"2024-03-04T07:26:52.321972+0000","last_unstale":"2024-03-04T07:26:54.389813+0000","last_undegraded":"2024-03-04T07:26:54.389813+0000","last_fullsized":"2024-03-04T07:26:54.389813+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389568+0000","last_change":"2024-03-04T07:26:53.324396+0000","last_active":"2024-03-04T07:26:54.389568+0000","last_peered":"2024-03-04T07:26:54.389568+0000","last_clean":"2024-03-04T07:26:54.389568+0000","last_became_active":"2024-03-04T07:26:53.320387+0000","last_became_peered":"2024-03-04T07:26:53.320387+0000","last_unstale":"2024-03-04T07:26:54.389568+0000","last_undegraded":"2024-03-04T07:26:54.389568+0000","last_fullsized":"2024-03-04T07:26:54.389568+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"19'4","reported_seq":18,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390337+0000","last_change":"2024-03-04T07:26:52.324964+0000","last_active":"2024-03-04T07:26:54.390337+0000","last_peered":"2024-03-04T07:26:54.390337+0000","last_clean":"2024-03-04T07:26:54.390337+0000","last_became_active":"2024-03-04T07:26:52.324642+0000","last_became_peered":"2024-03-04T07:26:52.324642+0000","last_unstale":"2024-03-04T07:26:54.390337+0000","last_undegraded":"2024-03-04T07:26:54.390337+0000","last_fullsized":"2024-03-04T07:26:54.390337+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:49.281484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574855,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":784,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":802816,"data_stored":661884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574854,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574854,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27228,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344612,"statfs":{"total":96636764160,"available":96608882688,"internally_reserved":0,"allocated":208896,"data_stored":71024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574855,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":71060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574854,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27236,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344604,"statfs":{"total":96636764160,"available":96608874496,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574854,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":204800,"data_stored":70934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574854,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":212992,"data_stored":71094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574854,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":811008,"data_stored":661940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:26:57.661 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:26:57.661 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-04T07:26:57.661 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-03-04T07:26:57.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-03-04T07:26:57.662 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-03-04T07:26:57.720 INFO:tasks.ceph.mds.b.smithi066.stderr:2024-03-04T07:26:57.720+0000 7f1297c9d640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:26:57.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbe2120640 1 Processor -- start 2024-03-04T07:26:57.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbe2120640 1 -- start start 2024-03-04T07:26:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbe2120640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc0b2c50 0x7fcbdc0d2ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbe2120640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcbdc0ce970 con 0x7fcbdc15e000 2024-03-04T07:26:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbe2120640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcbdc0cef30 con 0x7fcbdc071c50 2024-03-04T07:26:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbe2120640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc0cece0 con 0x7fcbdc0b2c50 2024-03-04T07:26:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbdb7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcbdc15e000 0x7fcbdc071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42790/0 (socket says 172.21.15.64:42790) 2024-03-04T07:26:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.800+0000 7fcbdb7fe640 1 -- 172.21.15.64:0/1186415722 learned_addr learned my addr 172.21.15.64:0/1186415722 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbdbfff640 1 --2- 172.21.15.64:0/1186415722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc0b2c50 0x7fcbdc0d2ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbda7fc640 1 -- 172.21.15.64:0/1186415722 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1374496164 0 0) 0x7fcbdc0cef30 con 0x7fcbdc071c50 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbda7fc640 1 -- 172.21.15.64:0/1186415722 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcbc8003520 con 0x7fcbdc071c50 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbda7fc640 1 -- 172.21.15.64:0/1186415722 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1000853578 0 0) 0x7fcbdc0ce970 con 0x7fcbdc15e000 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbda7fc640 1 -- 172.21.15.64:0/1186415722 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcbdc0cef30 con 0x7fcbdc15e000 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbdbfff640 1 -- 172.21.15.64:0/1186415722 >> v1:172.21.15.66:6789/0 conn(0x7fcbdc071c50 legacy=0x7fcbdc0d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbdbfff640 1 -- 172.21.15.64:0/1186415722 >> v1:172.21.15.64:6789/0 conn(0x7fcbdc15e000 legacy=0x7fcbdc071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.801+0000 7fcbdbfff640 1 -- 172.21.15.64:0/1186415722 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbdc0cf210 con 0x7fcbdc0b2c50 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbdbfff640 1 --2- 172.21.15.64:0/1186415722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc0b2c50 0x7fcbdc0d2ce0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fcbd4009dc0 tx=0x7fcbd402be90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=698e81ecefaf8a07 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbda7fc640 1 -- 172.21.15.64:0/1186415722 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd4039070 con 0x7fcbdc0b2c50 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbda7fc640 1 -- 172.21.15.64:0/1186415722 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcbd402cc40 con 0x7fcbdc0b2c50 2024-03-04T07:26:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbda7fc640 1 -- 172.21.15.64:0/1186415722 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd4034440 con 0x7fcbdc0b2c50 2024-03-04T07:26:57.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbe2120640 1 -- 172.21.15.64:0/1186415722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc0b2c50 msgr2=0x7fcbdc0d2ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbe2120640 1 --2- 172.21.15.64:0/1186415722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc0b2c50 0x7fcbdc0d2ce0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fcbd4009dc0 tx=0x7fcbd402be90 comp rx=0 tx=0).stop 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbe2120640 1 -- 172.21.15.64:0/1186415722 shutdown_connections 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbe2120640 1 --2- 172.21.15.64:0/1186415722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc0b2c50 0x7fcbdc0d2ce0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbe2120640 1 -- 172.21.15.64:0/1186415722 >> 172.21.15.64:0/1186415722 conn(0x7fcbdc0d3c80 msgr2=0x7fcbdc0d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbe2120640 1 -- 172.21.15.64:0/1186415722 shutdown_connections 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.802+0000 7fcbe2120640 1 -- 172.21.15.64:0/1186415722 wait complete. 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.803+0000 7fcbe2120640 1 Processor -- start 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbe2120640 1 -- start start 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbe2120640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc071c50 0x7fcbdc1f7aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbe2120640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbdc0b2c50 0x7fcbdc15a000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbe2120640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbdc15e000 0x7fcbdc1f3b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbe2120640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc0b3320 con 0x7fcbdc0b2c50 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbe2120640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc0b31a0 con 0x7fcbdc15e000 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbe2120640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc15e3e0 con 0x7fcbdc071c50 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbdaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbdc0b2c50 0x7fcbdc15a000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbdaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbdc0b2c50 0x7fcbdc15a000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40772/0 (socket says 172.21.15.64:40772) 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbdaffd640 1 -- 172.21.15.64:0/3414124734 learned_addr learned my addr 172.21.15.64:0/3414124734 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbdb7fe640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc071c50 0x7fcbdc1f7aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.804+0000 7fcbdbfff640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbdc15e000 0x7fcbdc1f3b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdb7fe640 1 -- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbdc15e000 msgr2=0x7fcbdc1f3b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdb7fe640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbdc15e000 0x7fcbdc1f3b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdb7fe640 1 -- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbdc0b2c50 msgr2=0x7fcbdc15a000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdb7fe640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbdc0b2c50 0x7fcbdc15a000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:57.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdb7fe640 1 -- 172.21.15.64:0/3414124734 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbdc1f7fe0 con 0x7fcbdc071c50 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdaffd640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbdc0b2c50 0x7fcbdc15a000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdbfff640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbdc15e000 0x7fcbdc1f3b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbdb7fe640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc071c50 0x7fcbdc1f7aa0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fcbc400c400 tx=0x7fcbc400c8c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbd3fff640 1 -- 172.21.15.64:0/3414124734 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbc4017070 con 0x7fcbdc071c50 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbd3fff640 1 -- 172.21.15.64:0/3414124734 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcbc4005ce0 con 0x7fcbdc071c50 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbdc1f4580 con 0x7fcbdc071c50 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbd3fff640 1 -- 172.21.15.64:0/3414124734 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbc40072f0 con 0x7fcbdc071c50 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.805+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbdc1f72a0 con 0x7fcbdc071c50 2024-03-04T07:26:57.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.806+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbdc067ff0 con 0x7fcbdc071c50 2024-03-04T07:26:57.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.807+0000 7fcbd3fff640 1 -- 172.21.15.64:0/3414124734 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcbc401a070 con 0x7fcbdc071c50 2024-03-04T07:26:57.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.807+0000 7fcbd3fff640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbbc06b4d0 0x7fcbbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:57.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.807+0000 7fcbd3fff640 1 -- 172.21.15.64:0/3414124734 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcbc4089470 con 0x7fcbdc071c50 2024-03-04T07:26:57.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.807+0000 7fcbdaffd640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbbc06b4d0 0x7fcbbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:57.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.807+0000 7fcbdaffd640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbbc06b4d0 0x7fcbbc06d990 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fcbcc001010 tx=0x7fcbcc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:57.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:57.811+0000 7fcbd3fff640 1 -- 172.21.15.64:0/3414124734 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcbc4057d30 con 0x7fcbdc071c50 2024-03-04T07:26:58.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.040+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) v1 -- 0x7fcbdc0d35d0 con 0x7fcbdc071c50 2024-03-04T07:26:58.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.043+0000 7fcbd3fff640 1 -- 172.21.15.64:0/3414124734 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) v1 ==== 74+0+171782 (secure 0 0 0) 0x7fcbc405e080 con 0x7fcbdc071c50 2024-03-04T07:26:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:58.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbbc06b4d0 msgr2=0x7fcbbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbbc06b4d0 0x7fcbbc06d990 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fcbcc001010 tx=0x7fcbcc009040 comp rx=0 tx=0).stop 2024-03-04T07:26:58.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc071c50 msgr2=0x7fcbdc1f7aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc071c50 0x7fcbdc1f7aa0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fcbc400c400 tx=0x7fcbc400c8c0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 shutdown_connections 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbbc06b4d0 0x7fcbbc06d990 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbdc15e000 0x7fcbdc1f3b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbdc0b2c50 0x7fcbdc15a000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 --2- 172.21.15.64:0/3414124734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbdc071c50 0x7fcbdc1f7aa0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 >> 172.21.15.64:0/3414124734 conn(0x7fcbdc0d3c80 msgr2=0x7fcbdc0b8c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 shutdown_connections 2024-03-04T07:26:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.045+0000 7fcbe2120640 1 -- 172.21.15.64:0/3414124734 wait complete. 2024-03-04T07:26:58.056 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":5,"flags":0,"active_gid":4103,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":129800233},{"type":"v1","addr":"172.21.15.64:6833","nonce":129800233}]},"active_addr":"172.21.15.64:6833/129800233","active_change":"2024-03-04T07:26:33.132467+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":4108,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1660137535}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":2542443167}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1286383136}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":2912387556}]}]} 2024-03-04T07:26:58.059 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-03-04T07:26:58.059 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-03-04T07:26:58.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.161+0000 7f2e1f8df640 1 Processor -- start 2024-03-04T07:26:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.161+0000 7f2e1f8df640 1 -- start start 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1f8df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1815df70 0x7f2e180d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1f8df640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e180d18c0 0x7f2e18071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1f8df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2e180b6f20 con 0x7f2e18071a80 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1f8df640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2e180d3030 con 0x7f2e180d18c0 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1f8df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e180d3310 con 0x7f2e1815df70 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1ce7a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e180d18c0 0x7f2e18071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1ce7a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e180d18c0 0x7f2e18071540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40776/0 (socket says 172.21.15.64:40776) 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1de7c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2e18071a80 0x7f2e180b2700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42806/0 (socket says 172.21.15.64:42806) 2024-03-04T07:26:58.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1ce7a640 1 -- 172.21.15.64:0/3645232225 learned_addr learned my addr 172.21.15.64:0/3645232225 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1d67b640 1 --2- 172.21.15.64:0/3645232225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1815df70 0x7f2e180d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1ce7a640 1 -- 172.21.15.64:0/3645232225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1815df70 msgr2=0x7f2e180d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1ce7a640 1 --2- 172.21.15.64:0/3645232225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1815df70 0x7f2e180d27b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1ce7a640 1 -- 172.21.15.64:0/3645232225 >> v1:172.21.15.64:6789/0 conn(0x7f2e18071a80 legacy=0x7f2e180b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1ce7a640 1 -- 172.21.15.64:0/3645232225 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e180d35f0 con 0x7f2e180d18c0 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e0ffff640 1 -- 172.21.15.64:0/3645232225 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2962436233 0 0) 0x7f2e180b6f20 con 0x7f2e18071a80 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.162+0000 7f2e1d67b640 1 --2- 172.21.15.64:0/3645232225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1815df70 0x7f2e180d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:58.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e1ce7a640 1 --2- 172.21.15.64:0/3645232225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e180d18c0 0x7f2e18071540 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f2e08009870 tx=0x7f2e0802ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2b6283b373df9c30 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e0ffff640 1 -- 172.21.15.64:0/3645232225 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e0802f840 con 0x7f2e180d18c0 2024-03-04T07:26:58.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e0ffff640 1 -- 172.21.15.64:0/3645232225 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e0802f9e0 con 0x7f2e180d18c0 2024-03-04T07:26:58.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e0ffff640 1 -- 172.21.15.64:0/3645232225 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e0802fdb0 con 0x7f2e180d18c0 2024-03-04T07:26:58.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3645232225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e180d18c0 msgr2=0x7f2e18071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3645232225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e180d18c0 0x7f2e18071540 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f2e08009870 tx=0x7f2e0802ba20 comp rx=0 tx=0).stop 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3645232225 shutdown_connections 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3645232225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e180d18c0 0x7f2e18071540 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3645232225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1815df70 0x7f2e180d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.163+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3645232225 >> 172.21.15.64:0/3645232225 conn(0x7f2e180d3cf0 msgr2=0x7f2e180d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.164+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3645232225 shutdown_connections 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.164+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3645232225 wait complete. 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 Processor -- start 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 -- start start 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e18071a80 0x7f2e1814c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e180d18c0 0x7f2e180cdc40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1815df70 0x7f2e180ce180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e1815e350 con 0x7f2e18071a80 2024-03-04T07:26:58.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e180d2cf0 con 0x7f2e1815df70 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1f8df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e1815e4d0 con 0x7f2e180d18c0 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1d67b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e18071a80 0x7f2e1814c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.165+0000 7f2e1d67b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e18071a80 0x7f2e1814c430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40778/0 (socket says 172.21.15.64:40778) 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1d67b640 1 -- 172.21.15.64:0/3296783216 learned_addr learned my addr 172.21.15.64:0/3296783216 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1ce7a640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e180d18c0 0x7f2e180cdc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1de7c640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1815df70 0x7f2e180ce180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1d67b640 1 -- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e180d18c0 msgr2=0x7f2e180cdc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1d67b640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e180d18c0 0x7f2e180cdc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1d67b640 1 -- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1815df70 msgr2=0x7f2e180ce180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1d67b640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1815df70 0x7f2e180ce180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1d67b640 1 -- 172.21.15.64:0/3296783216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e1814cd20 con 0x7f2e18071a80 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1d67b640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e18071a80 0x7f2e1814c430 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f2e1400e960 tx=0x7f2e1400ee20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e0dffb640 1 -- 172.21.15.64:0/3296783216 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e14017070 con 0x7f2e18071a80 2024-03-04T07:26:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e0dffb640 1 -- 172.21.15.64:0/3296783216 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e1400ca00 con 0x7f2e18071a80 2024-03-04T07:26:58.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2e1814d000 con 0x7f2e18071a80 2024-03-04T07:26:58.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e0dffb640 1 -- 172.21.15.64:0/3296783216 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1400cdd0 con 0x7f2e18071a80 2024-03-04T07:26:58.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.166+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2e180cb2b0 con 0x7f2e18071a80 2024-03-04T07:26:58.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.167+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2e18067ff0 con 0x7f2e18071a80 2024-03-04T07:26:58.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.168+0000 7f2e0dffb640 1 -- 172.21.15.64:0/3296783216 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2e1401a070 con 0x7f2e18071a80 2024-03-04T07:26:58.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.168+0000 7f2e0dffb640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2dfc06b4d0 0x7f2dfc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.168+0000 7f2e1ce7a640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2dfc06b4d0 0x7f2dfc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.168+0000 7f2e0dffb640 1 -- 172.21.15.64:0/3296783216 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2e1408a390 con 0x7f2e18071a80 2024-03-04T07:26:58.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.168+0000 7f2e1ce7a640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2dfc06b4d0 0x7f2dfc06d990 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f2e08036000 tx=0x7f2e08008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.172+0000 7f2e0dffb640 1 -- 172.21.15.64:0/3296783216 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2e14058bc0 con 0x7f2e18071a80 2024-03-04T07:26:58.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.356+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2e1815d370 con 0x7f2e18071a80 2024-03-04T07:26:58.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.357+0000 7f2e0dffb640 1 -- 172.21.15.64:0/3296783216 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f2e1405cc60 con 0x7f2e18071a80 2024-03-04T07:26:58.357 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:58.357 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:58.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2dfc06b4d0 msgr2=0x7f2dfc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2dfc06b4d0 0x7f2dfc06d990 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f2e08036000 tx=0x7f2e08008030 comp rx=0 tx=0).stop 2024-03-04T07:26:58.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e18071a80 msgr2=0x7f2e1814c430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e18071a80 0x7f2e1814c430 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f2e1400e960 tx=0x7f2e1400ee20 comp rx=0 tx=0).stop 2024-03-04T07:26:58.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 shutdown_connections 2024-03-04T07:26:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2dfc06b4d0 0x7f2dfc06d990 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1815df70 0x7f2e180ce180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e180d18c0 0x7f2e180cdc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 --2- 172.21.15.64:0/3296783216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e18071a80 0x7f2e1814c430 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 >> 172.21.15.64:0/3296783216 conn(0x7f2e180d3cf0 msgr2=0x7f2e180b5e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 shutdown_connections 2024-03-04T07:26:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.359+0000 7f2e1f8df640 1 -- 172.21.15.64:0/3296783216 wait complete. 2024-03-04T07:26:58.370 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-03-04T07:26:58.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:26:58.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.471+0000 7ff00c9f7640 1 Processor -- start 2024-03-04T07:26:58.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.471+0000 7ff00c9f7640 1 -- start start 2024-03-04T07:26:58.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7ff00c9f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff00815df00 0x7ff0080d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7ff00c9f7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0080b95a0 0x7ff0080b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7ff00c9f7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0080cea30 con 0x7ff0080b9a30 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7ff00c9f7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0080d2af0 con 0x7ff0080b95a0 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7ff00c9f7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0080cf3c0 con 0x7ff00815df00 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7feffffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0080b95a0 0x7ff0080b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7feffffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0080b95a0 0x7ff0080b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40786/0 (socket says 172.21.15.64:40786) 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7feffffff640 1 -- 172.21.15.64:0/1740678945 learned_addr learned my addr 172.21.15.64:0/1740678945 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.472+0000 7ff00659c640 1 --2- 172.21.15.64:0/1740678945 >> v2:172.21.15.66:3300/0 conn(0x7ff00815df00 0x7ff0080d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.473+0000 7feffffff640 1 -- 172.21.15.64:0/1740678945 >> v2:172.21.15.66:3300/0 conn(0x7ff00815df00 msgr2=0x7ff0080d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.473+0000 7feffffff640 1 --2- 172.21.15.64:0/1740678945 >> v2:172.21.15.66:3300/0 conn(0x7ff00815df00 0x7ff0080d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.473+0000 7feffffff640 1 -- 172.21.15.64:0/1740678945 >> v1:172.21.15.66:6789/0 conn(0x7ff0080b9a30 legacy=0x7ff0080d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.473+0000 7ff005d9b640 1 -- 172.21.15.64:0/1740678945 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1805600460 0 0) 0x7ff0080cea30 con 0x7ff0080b9a30 2024-03-04T07:26:58.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.473+0000 7feffffff640 1 -- 172.21.15.64:0/1740678945 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0080cf0e0 con 0x7ff0080b95a0 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.473+0000 7feffffff640 1 --2- 172.21.15.64:0/1740678945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff0080b5700 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7ff0080d11b0 tx=0x7feff002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3566ff12a44cecec server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff005d9b640 1 -- 172.21.15.64:0/1740678945 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feff0034070 con 0x7ff0080b95a0 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff005d9b640 1 -- 172.21.15.64:0/1740678945 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feff0030860 con 0x7ff0080b95a0 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff005d9b640 1 -- 172.21.15.64:0/1740678945 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feff0030c30 con 0x7ff0080b95a0 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 -- 172.21.15.64:0/1740678945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 msgr2=0x7ff0080b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/1740678945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff0080b5700 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7ff0080d11b0 tx=0x7feff002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 -- 172.21.15.64:0/1740678945 shutdown_connections 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/1740678945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff0080b5700 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/1740678945 >> v2:172.21.15.66:3300/0 conn(0x7ff00815df00 0x7ff0080d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 -- 172.21.15.64:0/1740678945 >> 172.21.15.64:0/1740678945 conn(0x7ff008071960 msgr2=0x7ff008071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 -- 172.21.15.64:0/1740678945 shutdown_connections 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.474+0000 7ff00c9f7640 1 -- 172.21.15.64:0/1740678945 wait complete. 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.475+0000 7ff00c9f7640 1 Processor -- start 2024-03-04T07:26:58.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.475+0000 7ff00c9f7640 1 -- start start 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00c9f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff00815a860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00c9f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff00815df00 0x7ff0081f78d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00c9f7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff00815ae40 0x7ff008161060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00c9f7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0080cec70 con 0x7ff0080b95a0 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00c9f7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0080ce750 con 0x7ff00815ae40 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00c9f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0080cedf0 con 0x7ff00815df00 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff00815a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff00815a860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40798/0 (socket says 172.21.15.64:40798) 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff00659c640 1 -- 172.21.15.64:0/685541636 learned_addr learned my addr 172.21.15.64:0/685541636 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7feffffff640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff00815df00 0x7ff0081f78d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.476+0000 7ff006d9d640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff00815ae40 0x7ff008161060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00659c640 1 -- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff00815df00 msgr2=0x7ff0081f78d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00659c640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff00815df00 0x7ff0081f78d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00659c640 1 -- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff00815ae40 msgr2=0x7ff008161060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00659c640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff00815ae40 0x7ff008161060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00659c640 1 -- 172.21.15.64:0/685541636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0081f6e30 con 0x7ff0080b95a0 2024-03-04T07:26:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00659c640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff00815a860 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7feff800e870 tx=0x7feff800ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7feffeffd640 1 -- 172.21.15.64:0/685541636 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feff8017070 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7feffeffd640 1 -- 172.21.15.64:0/685541636 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feff800c8e0 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7feffeffd640 1 -- 172.21.15.64:0/685541636 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feff800ccd0 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff0081e8db0 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.477+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff0081f73b0 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.478+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff008067ff0 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.478+0000 7feffeffd640 1 -- 172.21.15.64:0/685541636 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feff801a070 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.479+0000 7feffeffd640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fefe806b4d0 0x7fefe806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.479+0000 7feffeffd640 1 -- 172.21.15.64:0/685541636 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7feff808a330 con 0x7ff0080b95a0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.479+0000 7feffffff640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fefe806b4d0 0x7fefe806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.479+0000 7feffffff640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fefe806b4d0 0x7fefe806d990 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7feff0036000 tx=0x7feff0008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.483+0000 7feffeffd640 1 -- 172.21.15.64:0/685541636 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feff8058b60 con 0x7ff0080b95a0 2024-03-04T07:26:58.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.664+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7ff0080ce0a0 con 0x7ff0080b95a0 2024-03-04T07:26:58.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.665+0000 7feffeffd640 1 -- 172.21.15.64:0/685541636 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7feff805cc00 con 0x7ff0080b95a0 2024-03-04T07:26:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:26:58.666 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:26:58.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fefe806b4d0 msgr2=0x7fefe806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fefe806b4d0 0x7fefe806d990 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7feff0036000 tx=0x7feff0008030 comp rx=0 tx=0).stop 2024-03-04T07:26:58.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 msgr2=0x7ff00815a860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff00815a860 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7feff800e870 tx=0x7feff800ed30 comp rx=0 tx=0).stop 2024-03-04T07:26:58.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 shutdown_connections 2024-03-04T07:26:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fefe806b4d0 0x7fefe806d990 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff00815ae40 0x7ff008161060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff00815df00 0x7ff0081f78d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 --2- 172.21.15.64:0/685541636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0080b95a0 0x7ff00815a860 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 >> 172.21.15.64:0/685541636 conn(0x7ff008071960 msgr2=0x7ff0080b8730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 shutdown_connections 2024-03-04T07:26:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.667+0000 7ff00c9f7640 1 -- 172.21.15.64:0/685541636 wait complete. 2024-03-04T07:26:58.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-03-04T07:26:58.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-03-04T07:26:58.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-03-04T07:26:58.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-03-04T07:26:58.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.4 flush_pg_stats 2024-03-04T07:26:58.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.5 flush_pg_stats 2024-03-04T07:26:58.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.6 flush_pg_stats 2024-03-04T07:26:58.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.7 flush_pg_stats 2024-03-04T07:26:58.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.853+0000 7f3265aa4640 1 Processor -- start 2024-03-04T07:26:58.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.852+0000 7f04a759e640 1 Processor -- start 2024-03-04T07:26:58.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.852+0000 7f04a759e640 1 -- start start 2024-03-04T07:26:58.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.853+0000 7f1b7a640640 1 Processor -- start 2024-03-04T07:26:58.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.853+0000 7f1b7a640640 1 -- start start 2024-03-04T07:26:58.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a759e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f04a003ada0 0x7f04a00ec210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a759e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04a0055690 con 0x7f04a0051ce0 2024-03-04T07:26:58.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a759e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04a003cc20 con 0x7f04a00567e0 2024-03-04T07:26:58.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a759e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04a003c940 con 0x7f04a003ada0 2024-03-04T07:26:58.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a659c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f04a0051ce0 0x7f04a0054870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42818/0 (socket says 172.21.15.64:42818) 2024-03-04T07:26:58.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a659c640 1 -- 172.21.15.64:0/193559337 learned_addr learned my addr 172.21.15.64:0/193559337 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:58.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.853+0000 7f3265aa4640 1 -- start start 2024-03-04T07:26:58.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a6d9d640 1 --2- 172.21.15.64:0/193559337 >> v2:172.21.15.66:3300/0 conn(0x7f04a003ada0 0x7f04a00ec210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f3265aa4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32580ead00 0x7f325804fa90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f3265aa4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f325804ffd0 0x7f3258037bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f3265aa4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3258038110 0x7f32580385a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f3265aa4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3258054330 con 0x7f3258038110 2024-03-04T07:26:58.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f3265aa4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3258053720 con 0x7f325804ffd0 2024-03-04T07:26:58.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f3265aa4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3258053a00 con 0x7f32580ead00 2024-03-04T07:26:58.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 33262252 0 0) 0x7f04a0055690 con 0x7f04a0051ce0 2024-03-04T07:26:58.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f325f7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32580ead00 0x7f325804fa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325f7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32580ead00 0x7f325804fa90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48088/0 (socket says 172.21.15.64:48088) 2024-03-04T07:26:58.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f1b7a640640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1b741585a0 0x7f1b740ce820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f1b7a640640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1b740ced60 0x7f1b740ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f1b7a640640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b7406d8d0 con 0x7f1b740aec40 2024-03-04T07:26:58.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f1b7a640640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1b740cd1e0 con 0x7f1b741585a0 2024-03-04T07:26:58.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f1b7a640640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1b740cc200 con 0x7f1b740ced60 2024-03-04T07:26:58.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.854+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0490003520 con 0x7f04a0051ce0 2024-03-04T07:26:58.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325f7fe640 1 -- 172.21.15.64:0/4006080712 learned_addr learned my addr 172.21.15.64:0/4006080712 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325effd640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.66:3300/0 conn(0x7f325804ffd0 0x7f3258037bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325ffff640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.64:3300/0 conn(0x7f3258038110 0x7f32580385a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f76d6660640 1 Processor -- start 2024-03-04T07:26:58.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325f7fe640 1 -- 172.21.15.64:0/4006080712 >> v2:172.21.15.66:3300/0 conn(0x7f325804ffd0 msgr2=0x7f3258037bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f1b737fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1b740ced60 0x7f1b740ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f1b73fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1b741585a0 0x7f1b740ce820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f1b737fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1b740ced60 0x7f1b740ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35650/0 (socket says 172.21.15.64:35650) 2024-03-04T07:26:58.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f1b737fe640 1 -- 172.21.15.64:0/1294459515 learned_addr learned my addr 172.21.15.64:0/1294459515 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f1b73fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1b741585a0 0x7f1b740ce820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40806/0 (socket says 172.21.15.64:40806) 2024-03-04T07:26:58.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1355157934 0 0) 0x7f04a003cc20 con 0x7f04a00567e0 2024-03-04T07:26:58.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f04a0055690 con 0x7f04a00567e0 2024-03-04T07:26:58.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325f7fe640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.66:3300/0 conn(0x7f325804ffd0 0x7f3258037bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325f7fe640 1 -- 172.21.15.64:0/4006080712 >> v2:172.21.15.64:3300/0 conn(0x7f3258038110 msgr2=0x7f32580385a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1170556350 0 0) 0x7f0490003520 con 0x7f04a0051ce0 2024-03-04T07:26:58.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f04a6d9d640 1 -- 172.21.15.64:0/193559337 >> v1:172.21.15.66:6789/0 conn(0x7f04a00567e0 legacy=0x7f04a0054f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f04a6d9d640 1 -- 172.21.15.64:0/193559337 >> v1:172.21.15.64:6789/0 conn(0x7f04a0051ce0 legacy=0x7f04a0054870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325f7fe640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.64:3300/0 conn(0x7f3258038110 0x7f32580385a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f04a6d9d640 1 -- 172.21.15.64:0/193559337 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04a0055920 con 0x7f04a003ada0 2024-03-04T07:26:58.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325f7fe640 1 -- 172.21.15.64:0/4006080712 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3258053ce0 con 0x7f32580ead00 2024-03-04T07:26:58.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325ffff640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.64:3300/0 conn(0x7f3258038110 0x7f32580385a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:26:58.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f325effd640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.66:3300/0 conn(0x7f325804ffd0 0x7f3258037bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:58.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f81be602640 1 Processor -- start 2024-03-04T07:26:58.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.855+0000 7f76d6660640 1 -- start start 2024-03-04T07:26:58.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f04a6d9d640 1 --2- 172.21.15.64:0/193559337 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00ec210 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f04a806a910 tx=0x7f04a806ad20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5f2c28f71bbf3bc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f1b72ffd640 1 -- 172.21.15.64:0/1294459515 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2488832352 0 0) 0x7f1b6c001490 con 0x7f1b740aec40 2024-03-04T07:26:58.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f1b72ffd640 1 -- 172.21.15.64:0/1294459515 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1b5c003520 con 0x7f1b740aec40 2024-03-04T07:26:58.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f1b73fff640 1 -- 172.21.15.64:0/1294459515 >> v2:172.21.15.66:3300/0 conn(0x7f1b740ced60 msgr2=0x7f1b740ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f1b73fff640 1 --2- 172.21.15.64:0/1294459515 >> v2:172.21.15.66:3300/0 conn(0x7f1b740ced60 0x7f1b740ae700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f325f7fe640 1 --2- 172.21.15.64:0/4006080712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32580ead00 0x7f325804fa90 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f32580524d0 tx=0x7f325402b910 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a5f99e1eb61407c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f76d6660640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c80e3fc0 0x7f76c80dfee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f1b73fff640 1 -- 172.21.15.64:0/1294459515 >> v1:172.21.15.64:6789/0 conn(0x7f1b740aec40 legacy=0x7f1b7415e500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f1b73fff640 1 -- 172.21.15.64:0/1294459515 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b740721e0 con 0x7f1b741585a0 2024-03-04T07:26:58.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04a80b3070 con 0x7f04a003ada0 2024-03-04T07:26:58.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f04a80ac5e0 con 0x7f04a003ada0 2024-03-04T07:26:58.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f04a559a640 1 -- 172.21.15.64:0/193559337 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f049c0015f0 con 0x7f04a003ada0 2024-03-04T07:26:58.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f76d565e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f76c8050bf0 0x7f76c80e38b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42830/0 (socket says 172.21.15.64:42830) 2024-03-04T07:26:58.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f76d6660640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f76c80e4420 0x7f76c80e0420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f325e7fc640 1 -- 172.21.15.64:0/4006080712 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3254035020 con 0x7f32580ead00 2024-03-04T07:26:58.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f325e7fc640 1 -- 172.21.15.64:0/4006080712 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3254031870 con 0x7f32580ead00 2024-03-04T07:26:58.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f97f3fff640 1 Processor -- start 2024-03-04T07:26:58.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f9b77fff640 1 Processor -- start 2024-03-04T07:26:58.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f1b73fff640 1 --2- 172.21.15.64:0/1294459515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 0x7f1b740ce820 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f1b740cd390 tx=0x7f1b6802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5f960f95eb5bac01 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f04a759e640 1 -- 172.21.15.64:0/193559337 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 msgr2=0x7f04a00ec210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f04a759e640 1 --2- 172.21.15.64:0/193559337 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00ec210 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f04a806a910 tx=0x7f04a806ad20 comp rx=0 tx=0).stop 2024-03-04T07:26:58.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f325e7fc640 1 -- 172.21.15.64:0/4006080712 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3254035020 con 0x7f32580ead00 2024-03-04T07:26:58.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f76d4e5d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c80e3fc0 0x7f76c80dfee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f76d4e5d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c80e3fc0 0x7f76c80dfee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48104/0 (socket says 172.21.15.64:48104) 2024-03-04T07:26:58.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d6660640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f76c8009550 con 0x7f76c8050bf0 2024-03-04T07:26:58.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d6660640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f76c80512f0 con 0x7f76c80e4420 2024-03-04T07:26:58.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d6660640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76c80090c0 con 0x7f76c80e3fc0 2024-03-04T07:26:58.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d565e640 1 -- 172.21.15.64:0/1900729451 learned_addr learned my addr 172.21.15.64:0/1900729451 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:58.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f04a759e640 1 -- 172.21.15.64:0/193559337 shutdown_connections 2024-03-04T07:26:58.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f04a759e640 1 --2- 172.21.15.64:0/193559337 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00ec210 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f04a759e640 1 -- 172.21.15.64:0/193559337 >> 172.21.15.64:0/193559337 conn(0x7f04a0053720 msgr2=0x7f04a0053b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.856+0000 7f81be602640 1 -- start start 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f81be602640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81b8165c90 0x7f81b806cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f81be602640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81b81626a0 con 0x7f81b815fb80 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f81be602640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81b8166170 con 0x7f81b80ae700 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f81be602640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f81b8072940 con 0x7f81b8165c90 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f81b7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f81b815fb80 0x7f81b806e530 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42838/0 (socket says 172.21.15.64:42838) 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f81b7fff640 1 -- 172.21.15.64:0/1664548014 learned_addr learned my addr 172.21.15.64:0/1664548014 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f81b77fe640 1 --2- 172.21.15.64:0/1664548014 >> v2:172.21.15.64:3300/0 conn(0x7f81b8165c90 0x7f81b806cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d5e5f640 1 --2- 172.21.15.64:0/1900729451 >> v2:172.21.15.66:3300/0 conn(0x7f76c80e4420 0x7f76c80e0420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f3265aa4640 1 -- 172.21.15.64:0/4006080712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32580ead00 msgr2=0x7f325804fa90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f3265aa4640 1 --2- 172.21.15.64:0/4006080712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32580ead00 0x7f325804fa90 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f32580524d0 tx=0x7f325402b910 comp rx=0 tx=0).stop 2024-03-04T07:26:58.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f3265aa4640 1 -- 172.21.15.64:0/4006080712 shutdown_connections 2024-03-04T07:26:58.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f1b72ffd640 1 -- 172.21.15.64:0/1294459515 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b68035020 con 0x7f1b741585a0 2024-03-04T07:26:58.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f3265aa4640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.64:3300/0 conn(0x7f3258038110 0x7f32580385a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d4e5d640 1 -- 172.21.15.64:0/1900729451 >> v2:172.21.15.66:3300/0 conn(0x7f76c80e4420 msgr2=0x7f76c80e0420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d4e5d640 1 --2- 172.21.15.64:0/1900729451 >> v2:172.21.15.66:3300/0 conn(0x7f76c80e4420 0x7f76c80e0420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d4e5d640 1 -- 172.21.15.64:0/1900729451 >> v1:172.21.15.64:6789/0 conn(0x7f76c8050bf0 legacy=0x7f76c80e38b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f76d4e5d640 1 -- 172.21.15.64:0/1900729451 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76c8008790 con 0x7f76c80e3fc0 2024-03-04T07:26:58.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f1b72ffd640 1 -- 172.21.15.64:0/1294459515 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b68031830 con 0x7f1b741585a0 2024-03-04T07:26:58.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f3265aa4640 1 --2- 172.21.15.64:0/4006080712 >> v2:172.21.15.66:3300/0 conn(0x7f325804ffd0 0x7f3258037bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f3265aa4640 1 --2- 172.21.15.64:0/4006080712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32580ead00 0x7f325804fa90 secure :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f32580524d0 tx=0x7f325402b910 comp rx=0 tx=0).stop 2024-03-04T07:26:58.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f3265aa4640 1 -- 172.21.15.64:0/4006080712 >> 172.21.15.64:0/4006080712 conn(0x7f3258009fb0 msgr2=0x7f325800a390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f04a759e640 1 -- 172.21.15.64:0/193559337 shutdown_connections 2024-03-04T07:26:58.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f97f3fff640 1 -- start start 2024-03-04T07:26:58.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.857+0000 7f9b77fff640 1 -- start start 2024-03-04T07:26:58.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f9b77fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b70006670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f3265aa4640 1 -- 172.21.15.64:0/4006080712 shutdown_connections 2024-03-04T07:26:58.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f97f3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f97f40cfb90 0x7f97f40d2740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f97f3fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 0x7f97f40d08f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f97f3fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f97f40d0e30 0x7f97f40b55c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f97f3fff640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97f40d12a0 con 0x7f97f40cfb90 2024-03-04T07:26:58.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f97f3fff640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97f40b7a30 con 0x7f97f40d0e30 2024-03-04T07:26:58.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f3fff640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97f40d38a0 con 0x7f97f4166690 2024-03-04T07:26:58.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f97f40d0e30 0x7f97f40b55c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f27fc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 0x7f97f40d08f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7efeff21c640 1 Processor -- start 2024-03-04T07:26:58.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f97f40d0e30 0x7f97f40b55c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35668/0 (socket says 172.21.15.64:35668) 2024-03-04T07:26:58.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 -- 172.21.15.64:0/2823720726 learned_addr learned my addr 172.21.15.64:0/2823720726 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f9b76ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9b700e3300 0x7f9b700572f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42842/0 (socket says 172.21.15.64:42842) 2024-03-04T07:26:58.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f9b76ffd640 1 -- 172.21.15.64:0/2974789215 learned_addr learned my addr 172.21.15.64:0/2974789215 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:58.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b700e36e0 con 0x7f9b700e3300 2024-03-04T07:26:58.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b70007000 con 0x7f9b70006bb0 2024-03-04T07:26:58.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b70057b40 con 0x7f9b700061f0 2024-03-04T07:26:58.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 -- 172.21.15.64:0/2823720726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 msgr2=0x7f97f40d08f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f81b77fe640 1 -- 172.21.15.64:0/1664548014 >> v1:172.21.15.66:6789/0 conn(0x7f81b80ae700 legacy=0x7f81b8162b60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-03-04T07:26:58.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f81b77fe640 1 -- 172.21.15.64:0/1664548014 >> v1:172.21.15.64:6789/0 conn(0x7f81b815fb80 legacy=0x7f81b806e530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.858+0000 7f81b77fe640 1 -- 172.21.15.64:0/1664548014 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81b8166450 con 0x7f81b8165c90 2024-03-04T07:26:58.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7efeff21c640 1 -- start start 2024-03-04T07:26:58.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 --2- 172.21.15.64:0/2823720726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 0x7f97f40d08f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 -- 172.21.15.64:0/2823720726 >> v2:172.21.15.64:3300/0 conn(0x7f97f40cfb90 msgr2=0x7f97f40d2740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 --2- 172.21.15.64:0/2823720726 >> v2:172.21.15.64:3300/0 conn(0x7f97f40cfb90 0x7f97f40d2740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f97f37fe640 1 -- 172.21.15.64:0/2823720726 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97f40ce310 con 0x7f97f40d0e30 2024-03-04T07:26:58.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f3265aa4640 1 -- 172.21.15.64:0/4006080712 wait complete. 2024-03-04T07:26:58.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f04a759e640 1 -- 172.21.15.64:0/193559337 wait complete. 2024-03-04T07:26:58.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f9b767fc640 1 --2- 172.21.15.64:0/2974789215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b70006670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.859+0000 7f1b72ffd640 1 -- 172.21.15.64:0/1294459515 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b68031c00 con 0x7f1b741585a0 2024-03-04T07:26:58.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f1b7a640640 1 -- 172.21.15.64:0/1294459515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 msgr2=0x7f1b740ce820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f1b7a640640 1 --2- 172.21.15.64:0/1294459515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 0x7f1b740ce820 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f1b740cd390 tx=0x7f1b6802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f9b75ffb640 1 -- 172.21.15.64:0/2974789215 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2484873786 0 0) 0x7f9b70007000 con 0x7f9b70006bb0 2024-03-04T07:26:58.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f9b75ffb640 1 -- 172.21.15.64:0/2974789215 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9b60003520 con 0x7f9b70006bb0 2024-03-04T07:26:58.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f9b75ffb640 1 -- 172.21.15.64:0/2974789215 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3391171202 0 0) 0x7f9b700e36e0 con 0x7f9b700e3300 2024-03-04T07:26:58.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f9b75ffb640 1 -- 172.21.15.64:0/2974789215 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9b600048e0 con 0x7f9b700e3300 2024-03-04T07:26:58.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f76d4e5d640 1 --2- 172.21.15.64:0/1900729451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c80e3fc0 0x7f76c80dfee0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f76c00097a0 tx=0x7f76c002b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=832ddfd264cd7acd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f81b77fe640 1 --2- 172.21.15.64:0/1664548014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b806cc60 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f81ac0097a0 tx=0x7f81ac02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=24da979a82c48b43 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f9b767fc640 1 -- 172.21.15.64:0/2974789215 >> v1:172.21.15.66:6789/0 conn(0x7f9b70006bb0 legacy=0x7f9b7003c120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f9b767fc640 1 -- 172.21.15.64:0/2974789215 >> v1:172.21.15.64:6789/0 conn(0x7f9b700e3300 legacy=0x7f9b700572f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f3265aa4640 1 Processor -- start 2024-03-04T07:26:58.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f3265aa4640 1 -- start start 2024-03-04T07:26:58.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f1b7a640640 1 -- 172.21.15.64:0/1294459515 shutdown_connections 2024-03-04T07:26:58.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f1b7a640640 1 --2- 172.21.15.64:0/1294459515 >> v2:172.21.15.66:3300/0 conn(0x7f1b740ced60 0x7f1b740ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f1b7a640640 1 --2- 172.21.15.64:0/1294459515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 0x7f1b740ce820 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f1b7a640640 1 -- 172.21.15.64:0/1294459515 >> 172.21.15.64:0/1294459515 conn(0x7f1b74066d00 msgr2=0x7f1b74067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f1b7a640640 1 -- 172.21.15.64:0/1294459515 shutdown_connections 2024-03-04T07:26:58.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f1b7a640640 1 -- 172.21.15.64:0/1294459515 wait complete. 2024-03-04T07:26:58.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f81b6ffd640 1 -- 172.21.15.64:0/1664548014 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81ac0315a0 con 0x7f81b8165c90 2024-03-04T07:26:58.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f81b6ffd640 1 -- 172.21.15.64:0/1664548014 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81ac037070 con 0x7f81b8165c90 2024-03-04T07:26:58.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f81b6ffd640 1 -- 172.21.15.64:0/1664548014 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81ac031960 con 0x7f81b8165c90 2024-03-04T07:26:58.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7efeff21c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efef815fc90 0x7efef8162610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7efeff21c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efef8066d00 0x7efef8067130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7efeff21c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efef80b4520 con 0x7efef806fc70 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7efeff21c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efef8071940 con 0x7efef8066d00 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7efeff21c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efef8160070 con 0x7efef815fc90 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7efefcfb8640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efef815fc90 0x7efef8162610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7efefcfb8640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efef815fc90 0x7efef8162610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35682/0 (socket says 172.21.15.64:35682) 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7efefcfb8640 1 -- 172.21.15.64:0/941944405 learned_addr learned my addr 172.21.15.64:0/941944405 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f9b767fc640 1 -- 172.21.15.64:0/2974789215 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b70057e20 con 0x7f9b700061f0 2024-03-04T07:26:58.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f9b767fc640 1 --2- 172.21.15.64:0/2974789215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b70006670 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f9b700071d0 tx=0x7f9b6c02b8a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d6a86d76bcb6f613 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f76c7fff640 1 -- 172.21.15.64:0/1900729451 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76c0039070 con 0x7f76c80e3fc0 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f76c7fff640 1 -- 172.21.15.64:0/1900729451 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76c00316f0 con 0x7f76c80e3fc0 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f76c7fff640 1 -- 172.21.15.64:0/1900729451 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76c0031ac0 con 0x7f76c80e3fc0 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f76d6660640 1 -- 172.21.15.64:0/1900729451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c80e3fc0 msgr2=0x7f76c80dfee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f76d6660640 1 --2- 172.21.15.64:0/1900729451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c80e3fc0 0x7f76c80dfee0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f76c00097a0 tx=0x7f76c002b790 comp rx=0 tx=0).stop 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f76d6660640 1 -- 172.21.15.64:0/1900729451 shutdown_connections 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f76d6660640 1 --2- 172.21.15.64:0/1900729451 >> v2:172.21.15.66:3300/0 conn(0x7f76c80e4420 0x7f76c80e0420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f76d6660640 1 --2- 172.21.15.64:0/1900729451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c80e3fc0 0x7f76c80dfee0 secure :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f76c00097a0 tx=0x7f76c002b790 comp rx=0 tx=0).stop 2024-03-04T07:26:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f76d6660640 1 -- 172.21.15.64:0/1900729451 >> 172.21.15.64:0/1900729451 conn(0x7f76c8055bd0 msgr2=0x7f76c8055fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.860+0000 7f97f37fe640 1 --2- 172.21.15.64:0/2823720726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97f40d0e30 0x7f97f40b55c0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f97f408cdc0 tx=0x7f97ec02bdc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8a758f272e3eafd7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f97f1ffb640 1 -- 172.21.15.64:0/2823720726 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97ec02c960 con 0x7f97f40d0e30 2024-03-04T07:26:58.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f97f1ffb640 1 -- 172.21.15.64:0/2823720726 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f97ec02cb00 con 0x7f97f40d0e30 2024-03-04T07:26:58.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f97f1ffb640 1 -- 172.21.15.64:0/2823720726 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97ec02c960 con 0x7f97f40d0e30 2024-03-04T07:26:58.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f04a759e640 1 Processor -- start 2024-03-04T07:26:58.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f04a759e640 1 -- start start 2024-03-04T07:26:58.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f9b75ffb640 1 -- 172.21.15.64:0/2974789215 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b6c035020 con 0x7f9b700061f0 2024-03-04T07:26:58.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f9b75ffb640 1 -- 172.21.15.64:0/2974789215 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b6c031800 con 0x7f9b700061f0 2024-03-04T07:26:58.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f9b75ffb640 1 -- 172.21.15.64:0/2974789215 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b6c035020 con 0x7f9b700061f0 2024-03-04T07:26:58.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7efefcfb8640 1 -- 172.21.15.64:0/941944405 >> v2:172.21.15.64:3300/0 conn(0x7efef8066d00 msgr2=0x7efef8067130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:58.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7efefcfb8640 1 --2- 172.21.15.64:0/941944405 >> v2:172.21.15.64:3300/0 conn(0x7efef8066d00 0x7efef8067130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7efefcfb8640 1 -- 172.21.15.64:0/941944405 >> v1:172.21.15.66:6789/0 conn(0x7efef806fc70 legacy=0x7efef80b3e10 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-03-04T07:26:58.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7efefcfb8640 1 -- 172.21.15.64:0/941944405 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efef8162cd0 con 0x7efef815fc90 2024-03-04T07:26:58.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f3265aa4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3258038110 0x7f32580e9e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f3265aa4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f325804ffd0 0x7f32580456b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f3265aa4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3258045bf0 0x7f325817e4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f3265aa4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3258054940 con 0x7f3258045bf0 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f3265aa4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32580547c0 con 0x7f325804ffd0 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f3265aa4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f325804f550 con 0x7f3258038110 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f81be602640 1 -- 172.21.15.64:0/1664548014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 msgr2=0x7f81b806cc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.861+0000 7f81be602640 1 --2- 172.21.15.64:0/1664548014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b806cc60 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f81ac0097a0 tx=0x7f81ac02b840 comp rx=0 tx=0).stop 2024-03-04T07:26:58.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f81be602640 1 -- 172.21.15.64:0/1664548014 shutdown_connections 2024-03-04T07:26:58.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f81be602640 1 --2- 172.21.15.64:0/1664548014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b806cc60 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f81be602640 1 -- 172.21.15.64:0/1664548014 >> 172.21.15.64:0/1664548014 conn(0x7f81b8066d00 msgr2=0x7f81b8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f81be602640 1 -- 172.21.15.64:0/1664548014 shutdown_connections 2024-03-04T07:26:58.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f81be602640 1 -- 172.21.15.64:0/1664548014 wait complete. 2024-03-04T07:26:58.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f04a759e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00cfaa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f325ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3258045bf0 0x7f325817e4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f325ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3258045bf0 0x7f325817e4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40856/0 (socket says 172.21.15.64:40856) 2024-03-04T07:26:58.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f325ffff640 1 -- 172.21.15.64:0/3856855466 learned_addr learned my addr 172.21.15.64:0/3856855466 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f04a759e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04a0051ce0 0x7f04a00d71f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f04a759e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04a00567e0 0x7f04a00d1360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f04a759e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04a003c610 con 0x7f04a00567e0 2024-03-04T07:26:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f04a759e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04a0055ee0 con 0x7f04a003ada0 2024-03-04T07:26:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.862+0000 7f04a759e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04a0056400 con 0x7f04a0051ce0 2024-03-04T07:26:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 msgr2=0x7f9b70006670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f9b77fff640 1 --2- 172.21.15.64:0/2974789215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b70006670 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f9b700071d0 tx=0x7f9b6c02b8a0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f1b7a640640 1 Processor -- start 2024-03-04T07:26:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f325f7fe640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3258038110 0x7f32580e9e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 shutdown_connections 2024-03-04T07:26:58.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f9b77fff640 1 --2- 172.21.15.64:0/2974789215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b70006670 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 >> 172.21.15.64:0/2974789215 conn(0x7f9b70058ef0 msgr2=0x7f9b70059300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00cfaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00cfaa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35690/0 (socket says 172.21.15.64:35690) 2024-03-04T07:26:58.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 -- 172.21.15.64:0/1474314900 learned_addr learned my addr 172.21.15.64:0/1474314900 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a6d9d640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04a00567e0 0x7f04a00d1360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f325ffff640 1 -- 172.21.15.64:0/3856855466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3258038110 msgr2=0x7f32580e9e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 shutdown_connections 2024-03-04T07:26:58.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f76d6660640 1 -- 172.21.15.64:0/1900729451 shutdown_connections 2024-03-04T07:26:58.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f76d6660640 1 -- 172.21.15.64:0/1900729451 wait complete. 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f97f3fff640 1 -- 172.21.15.64:0/2823720726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97f40d0e30 msgr2=0x7f97f40b55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f97f3fff640 1 --2- 172.21.15.64:0/2823720726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97f40d0e30 0x7f97f40b55c0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f97f408cdc0 tx=0x7f97ec02bdc0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f97f3fff640 1 -- 172.21.15.64:0/2823720726 shutdown_connections 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f97f3fff640 1 --2- 172.21.15.64:0/2823720726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97f40d0e30 0x7f97f40b55c0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f97f3fff640 1 --2- 172.21.15.64:0/2823720726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 0x7f97f40d08f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f97f3fff640 1 --2- 172.21.15.64:0/2823720726 >> v2:172.21.15.64:3300/0 conn(0x7f97f40cfb90 0x7f97f40d2740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f97f3fff640 1 -- 172.21.15.64:0/2823720726 >> 172.21.15.64:0/2823720726 conn(0x7f97f4066ee0 msgr2=0x7f97f40672f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f1b7a640640 1 -- start start 2024-03-04T07:26:58.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7efefcfb8640 1 --2- 172.21.15.64:0/941944405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 0x7efef8162610 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7efef8071ac0 tx=0x7efee402b910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c384c143f34902b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f81be602640 1 Processor -- start 2024-03-04T07:26:58.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7efef77fe640 1 -- 172.21.15.64:0/941944405 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efee4035020 con 0x7efef815fc90 2024-03-04T07:26:58.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 -- 172.21.15.64:0/1474314900 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04a0051ce0 msgr2=0x7f04a00d71f0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-04T07:26:58.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f97f3fff640 1 -- 172.21.15.64:0/2823720726 shutdown_connections 2024-03-04T07:26:58.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04a0051ce0 0x7f04a00d71f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 -- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04a00567e0 msgr2=0x7f04a00d1360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04a00567e0 0x7f04a00d1360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f04a659c640 1 -- 172.21.15.64:0/1474314900 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04a004fdd0 con 0x7f04a003ada0 2024-03-04T07:26:58.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f81be602640 1 -- start start 2024-03-04T07:26:58.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7efef77fe640 1 -- 172.21.15.64:0/941944405 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efee4031870 con 0x7efef815fc90 2024-03-04T07:26:58.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7efef77fe640 1 -- 172.21.15.64:0/941944405 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efee4031c40 con 0x7efef815fc90 2024-03-04T07:26:58.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f1b7a640640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b740aec40 0x7f1b74200860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f1b7a640640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b740ced60 0x7f1b74159990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f1b73fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b740aec40 0x7f1b74200860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7efeff21c640 1 -- 172.21.15.64:0/941944405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 msgr2=0x7efef8162610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7efeff21c640 1 --2- 172.21.15.64:0/941944405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 0x7efef8162610 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7efef8071ac0 tx=0x7efee402b910 comp rx=0 tx=0).stop 2024-03-04T07:26:58.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f1b73fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b740aec40 0x7f1b74200860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48152/0 (socket says 172.21.15.64:48152) 2024-03-04T07:26:58.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b73fff640 1 -- 172.21.15.64:0/738623189 learned_addr learned my addr 172.21.15.64:0/738623189 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b737fe640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b740ced60 0x7f1b74159990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b7a640640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 0x7f1b740af700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b7a640640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b740cc700 con 0x7f1b741585a0 2024-03-04T07:26:58.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f9b77fff640 1 -- 172.21.15.64:0/2974789215 wait complete. 2024-03-04T07:26:58.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b7a640640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b74158980 con 0x7f1b740ced60 2024-03-04T07:26:58.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b7a640640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1b740cc9e0 con 0x7f1b740aec40 2024-03-04T07:26:58.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f9b77fff640 1 Processor -- start 2024-03-04T07:26:58.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7efeff21c640 1 -- 172.21.15.64:0/941944405 shutdown_connections 2024-03-04T07:26:58.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7efeff21c640 1 --2- 172.21.15.64:0/941944405 >> v2:172.21.15.64:3300/0 conn(0x7efef8066d00 0x7efef8067130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7efeff21c640 1 --2- 172.21.15.64:0/941944405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 0x7efef8162610 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7efeff21c640 1 -- 172.21.15.64:0/941944405 >> 172.21.15.64:0/941944405 conn(0x7efef80d07f0 msgr2=0x7efef80d0c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f76d6660640 1 Processor -- start 2024-03-04T07:26:58.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81be602640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81b80ae700 0x7f81b8158ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81be602640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81b815fb80 0x7f81b80b3050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81be602640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b814cf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81b7fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81b80ae700 0x7f81b8158ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81b7fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81b80ae700 0x7f81b8158ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35710/0 (socket says 172.21.15.64:35710) 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f97f3fff640 1 -- 172.21.15.64:0/2823720726 wait complete. 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f97f3fff640 1 Processor -- start 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b737fe640 1 -- 172.21.15.64:0/738623189 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b740aec40 msgr2=0x7f1b74200860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81bcb9f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b814cf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81be602640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81b806d1a0 con 0x7f81b8165c90 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81bcb9f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b814cf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40864/0 (socket says 172.21.15.64:40864) 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81be602640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81b8166a10 con 0x7f81b80ae700 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81bcb9f640 1 -- 172.21.15.64:0/1823424433 learned_addr learned my addr 172.21.15.64:0/1823424433 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81b806d320 con 0x7f81b815fb80 2024-03-04T07:26:58.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7efeff21c640 1 -- 172.21.15.64:0/941944405 shutdown_connections 2024-03-04T07:26:58.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f04a659c640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00cfaa0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f049c0011e0 tx=0x7f049c00cbc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f325ffff640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3258038110 0x7f32580e9e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f325ffff640 1 -- 172.21.15.64:0/3856855466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f325804ffd0 msgr2=0x7f32580456b0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-04T07:26:58.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f325ffff640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f325804ffd0 0x7f32580456b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f325ffff640 1 -- 172.21.15.64:0/3856855466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3258044610 con 0x7f3258045bf0 2024-03-04T07:26:58.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.863+0000 7f325f7fe640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3258038110 0x7f32580e9e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:58.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f9b77fff640 1 -- start start 2024-03-04T07:26:58.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f325ffff640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3258045bf0 0x7f325817e4d0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f326007add0 tx=0x7f32600735a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3260094470 con 0x7f3258045bf0 2024-03-04T07:26:58.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f326008f070 con 0x7f3258045bf0 2024-03-04T07:26:58.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3260094890 con 0x7f3258045bf0 2024-03-04T07:26:58.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32580448f0 con 0x7f3258045bf0 2024-03-04T07:26:58.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.864+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3258044eb0 con 0x7f3258045bf0 2024-03-04T07:26:58.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f3249ffb640 1 -- 172.21.15.64:0/3856855466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f32580052c0 con 0x7f3258045bf0 2024-03-04T07:26:58.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81b77fe640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81b815fb80 0x7f81b80b3050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b737fe640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b740aec40 0x7f1b74200860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7efeff21c640 1 -- 172.21.15.64:0/941944405 wait complete. 2024-03-04T07:26:58.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f049c017070 con 0x7f04a003ada0 2024-03-04T07:26:58.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.865+0000 7f1b737fe640 1 -- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 msgr2=0x7f1b740af700 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-04T07:26:58.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04a00500b0 con 0x7f04a003ada0 2024-03-04T07:26:58.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f1b78bdd640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 0x7f1b740af700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04a00cbe20 con 0x7f04a003ada0 2024-03-04T07:26:58.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81b7fff640 1 -- 172.21.15.64:0/1823424433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81b815fb80 msgr2=0x7f81b80b3050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f1b737fe640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 0x7f1b740af700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f1b737fe640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b74204f70 con 0x7f1b740ced60 2024-03-04T07:26:58.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f1b78bdd640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b741585a0 0x7f1b740af700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:58.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f97f3fff640 1 -- start start 2024-03-04T07:26:58.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81b7fff640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81b815fb80 0x7f81b80b3050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81b7fff640 1 -- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 msgr2=0x7f81b814cf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81b7fff640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b814cf60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81b7fff640 1 -- 172.21.15.64:0/1823424433 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81b8155590 con 0x7f81b80ae700 2024-03-04T07:26:58.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f9b77fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b7004edb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81bcb9f640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81b8165c90 0x7f81b814cf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.866+0000 7f81b7fff640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81b80ae700 0x7f81b8158ea0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f81b001daf0 tx=0x7f81b001b060 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f9b77fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b70006bb0 0x7f9b700e52c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f9b77fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b700e3300 0x7f9b7004ffc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f76d6660640 1 -- start start 2024-03-04T07:26:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f3fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97f40cfb90 0x7f97f4070bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97f40d0e30 0x7f97f40c9f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f3fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 0x7f97f40ca440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f3fff640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97f40d32e0 con 0x7f97f40d0e30 2024-03-04T07:26:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f3fff640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97f40d3000 con 0x7f97f40cfb90 2024-03-04T07:26:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f3fff640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97f40d35c0 con 0x7f97f4166690 2024-03-04T07:26:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81b001bca0 con 0x7f81b80ae700 2024-03-04T07:26:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f27fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97f40d0e30 0x7f97f40c9f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f27fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97f40d0e30 0x7f97f40c9f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40878/0 (socket says 172.21.15.64:40878) 2024-03-04T07:26:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f27fc640 1 -- 172.21.15.64:0/3535541050 learned_addr learned my addr 172.21.15.64:0/3535541050 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f1b737fe640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b740ced60 0x7f1b74159990 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1b6400bbd0 tx=0x7f1b6400f480 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f049c005ad0 con 0x7f04a003ada0 2024-03-04T07:26:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f049c007070 con 0x7f04a003ada0 2024-03-04T07:26:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f9b76ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b7004edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f9b76ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b7004edb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48178/0 (socket says 172.21.15.64:48178) 2024-03-04T07:26:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f9b767fc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b70006bb0 0x7f9b700e52c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3260094a30 con 0x7f3258045bf0 2024-03-04T07:26:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b6401b070 con 0x7f1b740ced60 2024-03-04T07:26:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b6400d280 con 0x7f1b740ced60 2024-03-04T07:26:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b6400fdb0 con 0x7f1b740ced60 2024-03-04T07:26:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f1b7a640640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b74158bd0 con 0x7f1b740ced60 2024-03-04T07:26:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f1b7a640640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b74159190 con 0x7f1b740ced60 2024-03-04T07:26:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f0494ff9640 1 -- 172.21.15.64:0/1474314900 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f0480001030 con 0x7f04a003ada0 2024-03-04T07:26:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f049c01a070 con 0x7f04a003ada0 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81b8155870 con 0x7f81b80ae700 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81b8155e30 con 0x7f81b80ae700 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81b0023400 con 0x7f81b80ae700 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81b001be60 con 0x7f81b80ae700 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f8198001030 con 0x7f81b80ae700 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7efeff21c640 1 Processor -- start 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7efeff21c640 1 -- start start 2024-03-04T07:26:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efeff21c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efef8066d00 0x7efef815d690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b76ffd640 1 -- 172.21.15.64:0/2566503035 learned_addr learned my addr 172.21.15.64:0/2566503035 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b70058be0 con 0x7f9b700e3300 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b700583e0 con 0x7f9b70006bb0 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b700e5d80 con 0x7f9b700061f0 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b76ffd640 1 -- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b70006bb0 msgr2=0x7f9b700e52c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b76ffd640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b70006bb0 0x7f9b700e52c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b76ffd640 1 -- 172.21.15.64:0/2566503035 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b700e3300 msgr2=0x7f9b7004ffc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b76ffd640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b700e3300 0x7f9b7004ffc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f9b76ffd640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b7004dff0 con 0x7f9b700061f0 2024-03-04T07:26:58.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f9b767fc640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b70006bb0 0x7f9b700e52c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f76d6660640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c8050bf0 0x7f76c804f2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f76d6660640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f76c80e4420 0x7f76c817d170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f76d6660640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76c80dd2c0 0x7f76c8035630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f76d6660640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76c8050fd0 con 0x7f76c80dd2c0 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f76d6660640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76c8008d50 con 0x7f76c80e4420 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f76d6660640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76c80e0960 con 0x7f76c8050bf0 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f27fc640 1 -- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 msgr2=0x7f97f40ca440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.867+0000 7f97f27fc640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97f4166690 0x7f97f40ca440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f97f27fc640 1 -- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97f40cfb90 msgr2=0x7f97f4070bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f97f27fc640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97f40cfb90 0x7f97f4070bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f97f27fc640 1 -- 172.21.15.64:0/3535541050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97f4161220 con 0x7f97f40d0e30 2024-03-04T07:26:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f97f27fc640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97f40d0e30 0x7f97f40c9f00 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f97e800e9b0 tx=0x7f97e800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97e8017070 con 0x7f97f40d0e30 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f97e800c9a0 con 0x7f97f40d0e30 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.868+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97e800cd90 con 0x7f97f40d0e30 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97f41606a0 con 0x7f97f40d0e30 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f97f4160c60 con 0x7f97f40d0e30 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efeff21c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efef806fc70 0x7efef80b1460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efeff21c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 0x7efef80b19a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efefcfb8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efef8066d00 0x7efef815d690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efefcfb8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efef8066d00 0x7efef815d690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48204/0 (socket says 172.21.15.64:48204) 2024-03-04T07:26:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f0496ffd640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f048806b5d0 0x7f048806da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efefcfb8640 1 -- 172.21.15.64:0/1781092804 learned_addr learned my addr 172.21.15.64:0/1781092804 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efeff21c640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efef8163290 con 0x7efef806fc70 2024-03-04T07:26:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efeff21c640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efef8067670 con 0x7efef815fc90 2024-03-04T07:26:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efefd7b9640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 0x7efef80b19a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7efeff21c640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efef8163570 con 0x7efef8066d00 2024-03-04T07:26:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f049c089560 con 0x7f04a003ada0 2024-03-04T07:26:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f81b0023870 con 0x7f81b80ae700 2024-03-04T07:26:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f0496ffd640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f04880737f0 0x7f0488075c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f97f404f330 con 0x7f97f40d0e30 2024-03-04T07:26:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f81b4ff9640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81a406b5d0 0x7f81a406da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f324bfff640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324406b5a0 0x7f324406da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3260105440 con 0x7f3258045bf0 2024-03-04T07:26:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f1b54001030 con 0x7f1b740ced60 2024-03-04T07:26:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f76c80e4420 0x7f76c817d170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f76c80e4420 0x7f76c817d170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35748/0 (socket says 172.21.15.64:35748) 2024-03-04T07:26:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 -- 172.21.15.64:0/2316957370 learned_addr learned my addr 172.21.15.64:0/2316957370 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d565e640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c8050bf0 0x7f76c804f2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f324bfff640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f32440737c0 0x7f3244075be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f81b009b870 con 0x7f81b80ae700 2024-03-04T07:26:58.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f81b4ff9640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f81a4073770 0x7f81a4075b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f81b0022e50 con 0x7f81a4073770 2024-03-04T07:26:58.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f81b77fe640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81a406b5d0 0x7f81a406da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d5e5f640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76c80dd2c0 0x7f76c8035630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f326009ee50 con 0x7f32440737c0 2024-03-04T07:26:58.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7f81b00660a0 con 0x7f81b80ae700 2024-03-04T07:26:58.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.869+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3260105830 con 0x7f3258045bf0 2024-03-04T07:26:58.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f325effd640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f32440737c0 0x7f3244075be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f04a5d9b640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f048806b5d0 0x7f048806da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 -- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c8050bf0 msgr2=0x7f76c804f2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76c8050bf0 0x7f76c804f2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 -- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76c80dd2c0 msgr2=0x7f76c8035630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76c80dd2c0 0x7f76c8035630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f76d4e5d640 1 -- 172.21.15.64:0/2316957370 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76c804faa0 con 0x7f76c80e4420 2024-03-04T07:26:58.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f04a6d9d640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f04880737f0 0x7f0488075c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f325effd640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f32440737c0 0x7f3244075be0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f049c015e70 con 0x7f04880737f0 2024-03-04T07:26:58.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f76d4e5d640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f76c80e4420 0x7f76c817d170 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f76c002bd20 tx=0x7f76c00046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7efefd7b9640 1 -- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efef8066d00 msgr2=0x7efef815d690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7efefd7b9640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efef8066d00 0x7efef815d690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7efefd7b9640 1 -- 172.21.15.64:0/1781092804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efef806fc70 msgr2=0x7efef80b1460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7efefd7b9640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efef806fc70 0x7efef80b1460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7efefd7b9640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efef81557c0 con 0x7efef815fc90 2024-03-04T07:26:58.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7efefd7b9640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 0x7efef80b19a0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7efef000be20 tx=0x7efef000bf10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f81b77fe640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81a406b5d0 0x7f81a406da90 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f81ac036000 tx=0x7f81ac03d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76c0043070 con 0x7f76c80e4420 2024-03-04T07:26:58.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efef0019070 con 0x7efef815fc90 2024-03-04T07:26:58.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f04a5d9b640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f048806b5d0 0x7f048806da90 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f04a004c820 tx=0x7f0498007990 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f76c0034680 con 0x7f76c80e4420 2024-03-04T07:26:58.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7efeff21c640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efef8155a00 con 0x7efef815fc90 2024-03-04T07:26:58.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7efeff21c640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efef8155fc0 con 0x7efef815fc90 2024-03-04T07:26:58.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efef0014440 con 0x7efef815fc90 2024-03-04T07:26:58.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f04a6d9d640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f04880737f0 0x7f0488075c10 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.870+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7f049c090050 con 0x7f04a003ada0 2024-03-04T07:26:58.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f325f7fe640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324406b5a0 0x7f324406da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1b6401e070 con 0x7f1b740ced60 2024-03-04T07:26:58.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f9b76ffd640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b7004edb0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f9b6800c530 tx=0x7f9b6800c9f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f326009ee50 con 0x7f32440737c0 2024-03-04T07:26:58.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efef0014830 con 0x7efef815fc90 2024-03-04T07:26:58.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b68017070 con 0x7f9b700061f0 2024-03-04T07:26:58.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f1b70ff9640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b6006b5a0 0x7f1b6006da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1b64089af0 con 0x7f1b740ced60 2024-03-04T07:26:58.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b68005ce0 con 0x7f9b700061f0 2024-03-04T07:26:58.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f76d6660640 1 -- 172.21.15.64:0/2316957370 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f76c804f7f0 con 0x7f76c80e4420 2024-03-04T07:26:58.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f1b70ff9640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f1b600737c0 0x7f1b60075be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7efeff21c640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7efed4001030 con 0x7efef815fc90 2024-03-04T07:26:58.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b680072b0 con 0x7f9b700061f0 2024-03-04T07:26:58.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b7004e2d0 con 0x7f9b700061f0 2024-03-04T07:26:58.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b7004e890 con 0x7f9b700061f0 2024-03-04T07:26:58.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76c0039040 con 0x7f76c80e4420 2024-03-04T07:26:58.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== osd.2 v2:172.21.15.64:6824/2610379167 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f049c015e70 con 0x7f04880737f0 2024-03-04T07:26:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.871+0000 7f76d6660640 1 -- 172.21.15.64:0/2316957370 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f76c80e6010 con 0x7f76c80e4420 2024-03-04T07:26:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f81bcb9f640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f81a4073770 0x7f81a4075b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.873+0000 7f325f7fe640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324406b5a0 0x7f324406da60 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f3254036000 tx=0x7f3254008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f1b6401aec0 con 0x7f1b600737c0 2024-03-04T07:26:58.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.872+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1b64089ee0 con 0x7f1b740ced60 2024-03-04T07:26:58.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.873+0000 7f1b78bdd640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f1b600737c0 0x7f1b60075be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.873+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f9b70005710 con 0x7f9b700061f0 2024-03-04T07:26:58.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.873+0000 7f1b73fff640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b6006b5a0 0x7f1b6006da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.873+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f76c8005710 con 0x7f76c80e4420 2024-03-04T07:26:58.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.873+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f97e801a070 con 0x7f97f40d0e30 2024-03-04T07:26:58.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f81bcb9f640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f81a4073770 0x7f81a4075b90 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f1b78bdd640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f1b600737c0 0x7f1b60075be0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efef001c070 con 0x7efef815fc90 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f97dbfff640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f97e006b520 0x7f97e006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f97e808a520 con 0x7f97f40d0e30 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f97dbfff640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7f97e0073740 0x7f97e0075b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f97e8022e50 con 0x7f97e0073740 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7f97e808a8d0 con 0x7f97f40d0e30 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.874+0000 7f97f2ffd640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f97e006b520 0x7f97e006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f97f37fe640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7f97e0073740 0x7f97e0075b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f1b73fff640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b6006b5a0 0x7f1b6006da60 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f1b68009560 tx=0x7f1b6803c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7efef57fa640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efee006b500 0x7efee006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7efef003c050 con 0x7efef815fc90 2024-03-04T07:26:58.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f76c003c070 con 0x7f76c80e4420 2024-03-04T07:26:58.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== osd.3 v2:172.21.15.64:6816/1402142346 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f81b0022e50 con 0x7f81a4073770 2024-03-04T07:26:58.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7efef57fa640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7efee00736a0 0x7efee0075ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7efef0021e50 con 0x7efee00736a0 2024-03-04T07:26:58.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7efef008a8c0 con 0x7efef815fc90 2024-03-04T07:26:58.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f76c5ffb640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f76b406b5a0 0x7f76b406da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f97f2ffd640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f97e006b520 0x7f97e006d9e0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f97e4009670 tx=0x7f97e4006bf0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f76c00abb20 con 0x7f76c80e4420 2024-03-04T07:26:58.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f76d565e640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f76b406b5a0 0x7f76b406da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f76c5ffb640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f76b4073740 0x7f76b4075b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.875+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f76c0008eb0 con 0x7f76b4073740 2024-03-04T07:26:58.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7f76c0076640 con 0x7f76c80e4420 2024-03-04T07:26:58.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9b6801a070 con 0x7f9b700061f0 2024-03-04T07:26:58.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== osd.1 v2:172.21.15.64:6801/1190712954 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f1b6401aec0 con 0x7f1b600737c0 2024-03-04T07:26:58.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7efef7fff640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7efee00736a0 0x7efee0075ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7efefcfb8640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efee006b500 0x7efee006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f76d5e5f640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f76b4073740 0x7f76b4075b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f9b5f7fe640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b6406b500 0x7f9b6406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9b68089ca0 con 0x7f9b700061f0 2024-03-04T07:26:58.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f97f37fe640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7f97e0073740 0x7f97e0075b60 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f9b767fc640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b6406b500 0x7f9b6406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f9b5f7fe640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f9b64073700 0x7f9b64075b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:58.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f9b640761d0 con 0x7f9b64073700 2024-03-04T07:26:58.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.876+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_get_version_reply(handle=1 version=20) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9b6808a050 con 0x7f9b700061f0 2024-03-04T07:26:58.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7f9b777fe640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f9b64073700 0x7f9b64075b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:58.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7efefcfb8640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efee006b500 0x7efee006d9c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7efee40090b0 tx=0x7efee403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== osd.7 v2:172.21.15.66:6802/3006518207 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f97e8022e50 con 0x7f97e0073740 2024-03-04T07:26:58.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7f76d5e5f640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f76b4073740 0x7f76b4075b60 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7f76d565e640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f76b406b5a0 0x7f76b406da60 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f76cc000f60 tx=0x7f76cc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7f9b777fe640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f9b64073700 0x7f9b64075b20 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7efef7fff640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7efee00736a0 0x7efee0075ac0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.877+0000 7f9b767fc640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b6406b500 0x7f9b6406d9c0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9b6c036000 tx=0x7f9b6c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:58.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.878+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== osd.6 v2:172.21.15.66:6816/177421408 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f76c0008eb0 con 0x7f76b4073740 2024-03-04T07:26:58.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.878+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== osd.5 v2:172.21.15.66:6800/1583207443 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f9b640761d0 con 0x7f9b64073700 2024-03-04T07:26:58.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.881+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== osd.4 v2:172.21.15.66:6824/1927515367 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7efef0021e50 con 0x7efee00736a0 2024-03-04T07:26:58.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.900+0000 7f0494ff9640 1 -- 172.21.15.64:0/1474314900 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f0480002dd0 con 0x7f04880737f0 2024-03-04T07:26:58.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.901+0000 7f0496ffd640 1 -- 172.21.15.64:0/1474314900 <== osd.2 v2:172.21.15.64:6824/2610379167 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f0480002dd0 con 0x7f04880737f0 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.901+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f04880737f0 msgr2=0x7f0488075c10 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.901+0000 7f04a759e640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f04880737f0 0x7f0488075c10 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f048806b5d0 msgr2=0x7f048806da90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f048806b5d0 0x7f048806da90 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f04a004c820 tx=0x7f0498007990 comp rx=0 tx=0).stop 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 msgr2=0x7f04a00cfaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 --2- 172.21.15.64:0/1474314900 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04a003ada0 0x7f04a00cfaa0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f049c0011e0 tx=0x7f049c00cbc0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a6d9d640 1 -- 172.21.15.64:0/1474314900 reap_dead start 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 shutdown_connections 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 >> 172.21.15.64:0/1474314900 conn(0x7f04a0053720 msgr2=0x7f04a0054df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 shutdown_connections 2024-03-04T07:26:58.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.902+0000 7f04a759e640 1 -- 172.21.15.64:0/1474314900 wait complete. 2024-03-04T07:26:58.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.908+0000 7f3249ffb640 1 -- 172.21.15.64:0/3856855466 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f32580052c0 con 0x7f32440737c0 2024-03-04T07:26:58.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.909+0000 7f324bfff640 1 -- 172.21.15.64:0/3856855466 <== osd.0 v2:172.21.15.64:6800/2111658274 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f32580052c0 con 0x7f32440737c0 2024-03-04T07:26:58.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.909+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f8198002dd0 con 0x7f81a4073770 2024-03-04T07:26:58.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.909+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f32440737c0 msgr2=0x7f3244075be0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.909+0000 7f3265aa4640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f32440737c0 0x7f3244075be0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.909+0000 7f81b4ff9640 1 -- 172.21.15.64:0/1823424433 <== osd.3 v2:172.21.15.64:6816/1402142346 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f8198002dd0 con 0x7f81a4073770 2024-03-04T07:26:58.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324406b5a0 msgr2=0x7f324406da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324406b5a0 0x7f324406da60 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f3254036000 tx=0x7f3254008030 comp rx=0 tx=0).stop 2024-03-04T07:26:58.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3258045bf0 msgr2=0x7f325817e4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 --2- 172.21.15.64:0/3856855466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3258045bf0 0x7f325817e4d0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f326007add0 tx=0x7f32600735a0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f325ffff640 1 -- 172.21.15.64:0/3856855466 reap_dead start 2024-03-04T07:26:58.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f1b54002dd0 con 0x7f1b600737c0 2024-03-04T07:26:58.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 shutdown_connections 2024-03-04T07:26:58.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 >> 172.21.15.64:0/3856855466 conn(0x7f3258009fb0 msgr2=0x7f3258052c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f97f40d2520 con 0x7f97e0073740 2024-03-04T07:26:58.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f81a4073770 msgr2=0x7f81a4075b90 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f81be602640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f81a4073770 0x7f81a4075b90 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f1b70ff9640 1 -- 172.21.15.64:0/738623189 <== osd.1 v2:172.21.15.64:6801/1190712954 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f1b54002dd0 con 0x7f1b600737c0 2024-03-04T07:26:58.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 shutdown_connections 2024-03-04T07:26:58.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f3265aa4640 1 -- 172.21.15.64:0/3856855466 wait complete. 2024-03-04T07:26:58.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f97dbfff640 1 -- 172.21.15.64:0/3535541050 <== osd.7 v2:172.21.15.66:6802/3006518207 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f97f40d2520 con 0x7f97e0073740 2024-03-04T07:26:58.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f1b600737c0 msgr2=0x7f1b60075be0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f1b600737c0 0x7f1b60075be0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81a406b5d0 msgr2=0x7f81a406da90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f81be602640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81a406b5d0 0x7f81a406da90 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f81ac036000 tx=0x7f81ac03d040 comp rx=0 tx=0).stop 2024-03-04T07:26:58.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81b80ae700 msgr2=0x7f81b8158ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f81be602640 1 --2- 172.21.15.64:0/1823424433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81b80ae700 0x7f81b8158ea0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f81b001daf0 tx=0x7f81b001b060 comp rx=0 tx=0).stop 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.910+0000 7f81bcb9f640 1 -- 172.21.15.64:0/1823424433 reap_dead start 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 shutdown_connections 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 >> 172.21.15.64:0/1823424433 conn(0x7f81b8066d00 msgr2=0x7f81b80cd710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 shutdown_connections 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f81be602640 1 -- 172.21.15.64:0/1823424433 wait complete. 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7f97e0073740 msgr2=0x7f97e0075b60 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7f97e0073740 0x7f97e0075b60 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b6006b5a0 msgr2=0x7f1b6006da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b6006b5a0 0x7f1b6006da60 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f1b68009560 tx=0x7f1b6803c040 comp rx=0 tx=0).stop 2024-03-04T07:26:58.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b740ced60 msgr2=0x7f1b74159990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 --2- 172.21.15.64:0/738623189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b740ced60 0x7f1b74159990 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1b6400bbd0 tx=0x7f1b6400f480 comp rx=0 tx=0).stop 2024-03-04T07:26:58.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b78bdd640 1 -- 172.21.15.64:0/738623189 reap_dead start 2024-03-04T07:26:58.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f97e006b520 msgr2=0x7f97e006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f97e006b520 0x7f97e006d9e0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f97e4009670 tx=0x7f97e4006bf0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97f40d0e30 msgr2=0x7f97f40c9f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 --2- 172.21.15.64:0/3535541050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97f40d0e30 0x7f97f40c9f00 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f97e800e9b0 tx=0x7f97e800ee70 comp rx=0 tx=0).stop 2024-03-04T07:26:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f37fe640 1 -- 172.21.15.64:0/3535541050 reap_dead start 2024-03-04T07:26:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 shutdown_connections 2024-03-04T07:26:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 >> 172.21.15.64:0/3535541050 conn(0x7f97f4066ee0 msgr2=0x7f97f40b7da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f9b700572f0 con 0x7f9b64073700 2024-03-04T07:26:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 shutdown_connections 2024-03-04T07:26:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.911+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 >> 172.21.15.64:0/738623189 conn(0x7f1b74066d00 msgr2=0x7f1b7406dbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f76c80538e0 con 0x7f76b4073740 2024-03-04T07:26:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 shutdown_connections 2024-03-04T07:26:58.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f9b5f7fe640 1 -- 172.21.15.64:0/2566503035 <== osd.5 v2:172.21.15.66:6800/1583207443 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f9b700572f0 con 0x7f9b64073700 2024-03-04T07:26:58.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f1b5a7fc640 1 -- 172.21.15.64:0/738623189 wait complete. 2024-03-04T07:26:58.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 shutdown_connections 2024-03-04T07:26:58.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f97f3fff640 1 -- 172.21.15.64:0/3535541050 wait complete. 2024-03-04T07:26:58.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f76c5ffb640 1 -- 172.21.15.64:0/2316957370 <== osd.6 v2:172.21.15.66:6816/177421408 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f76c80538e0 con 0x7f76b4073740 2024-03-04T07:26:58.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f9b64073700 msgr2=0x7f9b64075b20 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f9b77fff640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f9b64073700 0x7f9b64075b20 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b6406b500 msgr2=0x7f9b6406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f9b77fff640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b6406b500 0x7f9b6406d9c0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9b6c036000 tx=0x7f9b6c03c040 comp rx=0 tx=0).stop 2024-03-04T07:26:58.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 msgr2=0x7f9b7004edb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.912+0000 7f9b77fff640 1 --2- 172.21.15.64:0/2566503035 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b700061f0 0x7f9b7004edb0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f9b6800c530 tx=0x7f9b6800c9f0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f9b777fe640 1 -- 172.21.15.64:0/2566503035 reap_dead start 2024-03-04T07:26:58.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 shutdown_connections 2024-03-04T07:26:58.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f76b4073740 msgr2=0x7f76b4075b60 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f76b4073740 0x7f76b4075b60 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 >> 172.21.15.64:0/2566503035 conn(0x7f9b70058ef0 msgr2=0x7f9b70039ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 shutdown_connections 2024-03-04T07:26:58.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f9b77fff640 1 -- 172.21.15.64:0/2566503035 wait complete. 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f76b406b5a0 msgr2=0x7f76b406da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f76b406b5a0 0x7f76b406da60 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f76cc000f60 tx=0x7f76cc009040 comp rx=0 tx=0).stop 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f76c80e4420 msgr2=0x7f76c817d170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 --2- 172.21.15.64:0/2316957370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f76c80e4420 0x7f76c817d170 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f76c002bd20 tx=0x7f76c00046c0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76d5e5f640 1 -- 172.21.15.64:0/2316957370 reap_dead start 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 shutdown_connections 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.913+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 >> 172.21.15.64:0/2316957370 conn(0x7f76c8055bd0 msgr2=0x7f76c80dee60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.914+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 shutdown_connections 2024-03-04T07:26:58.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.914+0000 7f76bb7fe640 1 -- 172.21.15.64:0/2316957370 wait complete. 2024-03-04T07:26:58.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.915+0000 7efeff21c640 1 -- 172.21.15.64:0/1781092804 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7efed4002dd0 con 0x7efee00736a0 2024-03-04T07:26:58.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.920+0000 7efef57fa640 1 -- 172.21.15.64:0/1781092804 <== osd.4 v2:172.21.15.66:6824/1927515367 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7efed4002dd0 con 0x7efee00736a0 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.921+0000 7efeeeffd640 1 -- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7efee00736a0 msgr2=0x7efee0075ac0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.921+0000 7efeeeffd640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7efee00736a0 0x7efee0075ac0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.921+0000 7efeeeffd640 1 -- 172.21.15.64:0/1781092804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efee006b500 msgr2=0x7efee006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.921+0000 7efeeeffd640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efee006b500 0x7efee006d9c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7efee40090b0 tx=0x7efee403c040 comp rx=0 tx=0).stop 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.921+0000 7efeeeffd640 1 -- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 msgr2=0x7efef80b19a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.921+0000 7efeeeffd640 1 --2- 172.21.15.64:0/1781092804 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efef815fc90 0x7efef80b19a0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7efef000be20 tx=0x7efef000bf10 comp rx=0 tx=0).stop 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.922+0000 7efefd7b9640 1 -- 172.21.15.64:0/1781092804 reap_dead start 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.922+0000 7efeeeffd640 1 -- 172.21.15.64:0/1781092804 shutdown_connections 2024-03-04T07:26:58.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.922+0000 7efeeeffd640 1 -- 172.21.15.64:0/1781092804 >> 172.21.15.64:0/1781092804 conn(0x7efef80d07f0 msgr2=0x7efef815eae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:58.970 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:58.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-03-04T07:26:58.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.923+0000 7efeeeffd640 1 -- 172.21.15.64:0/1781092804 shutdown_connections 2024-03-04T07:26:58.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:58.923+0000 7efeeeffd640 1 -- 172.21.15.64:0/1781092804 wait complete. 2024-03-04T07:26:58.971 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:58.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-03-04T07:26:58.972 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:58.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-03-04T07:26:58.973 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:58.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-03-04T07:26:58.974 INFO:teuthology.orchestra.run.smithi064.stdout:55834574856 2024-03-04T07:26:58.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.7 2024-03-04T07:26:58.975 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:58.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.5 2024-03-04T07:26:58.976 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:58.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.6 2024-03-04T07:26:58.978 INFO:teuthology.orchestra.run.smithi064.stdout:55834574856 2024-03-04T07:26:58.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.4 2024-03-04T07:26:59.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.184+0000 7f41682df640 1 Processor -- start 2024-03-04T07:26:59.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.184+0000 7f41682df640 1 -- start start 2024-03-04T07:26:59.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.184+0000 7f7cc6c14640 1 Processor -- start 2024-03-04T07:26:59.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.184+0000 7f7cc6c14640 1 -- start start 2024-03-04T07:26:59.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f2b10dd7640 1 Processor -- start 2024-03-04T07:26:59.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f2b10dd7640 1 -- start start 2024-03-04T07:26:59.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f41682df640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f41601624e0 0x7f41600cc040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f2e33102640 1 Processor -- start 2024-03-04T07:26:59.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f2e33102640 1 -- start start 2024-03-04T07:26:59.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f41682df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cc580 0x7f41600c49c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f7cc6c14640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7cc00cee00 0x7f7cc00b4550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f7cc6c14640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc00ce570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f7cc6c14640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7cc00cd7b0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f7cc6c14640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7cc00b2ec0 con 0x7f7cc00cee00 2024-03-04T07:26:59.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f7cc6c14640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7cc00cda90 con 0x7f7cc00b2ab0 2024-03-04T07:26:59.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.185+0000 7f41682df640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f41600cdb60 0x7f41600c4f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416687c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f41600cdb60 0x7f41600c4f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc5411640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7cc00cee00 0x7f7cc00b4550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc5c12640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f7cc00b20e0 0x7f7cc006ca00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43778/0 (socket says 172.21.15.64:43778) 2024-03-04T07:26:59.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc5c12640 1 -- 172.21.15.64:0/2988521979 learned_addr learned my addr 172.21.15.64:0/2988521979 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:59.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416687c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f41600cdb60 0x7f41600c4f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40918/0 (socket says 172.21.15.64:40918) 2024-03-04T07:26:59.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc6413640 1 --2- 172.21.15.64:0/2988521979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc00ce570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416607b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f41601624e0 0x7f41600cc040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416687c640 1 -- 172.21.15.64:0/2946662522 learned_addr learned my addr 172.21.15.64:0/2946662522 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f41600cd6d0 con 0x7f41600cdb60 2024-03-04T07:26:59.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416587a640 1 --2- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cc580 0x7f41600c49c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f2b10dd7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2b04005ed0 0x7f2b04006310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f21e26d3640 1 Processor -- start 2024-03-04T07:26:59.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f41600ccac0 con 0x7f41601624e0 2024-03-04T07:26:59.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f41600ccda0 con 0x7f41600cc580 2024-03-04T07:26:59.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f2b10dd7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2b04006850 0x7f2b04006cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f2b0b7fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f2b040eacf0 0x7f2b0404fa80 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43782/0 (socket says 172.21.15.64:43782) 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f2b0b7fe640 1 -- 172.21.15.64:0/236506450 learned_addr learned my addr 172.21.15.64:0/236506450 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2b04053710 con 0x7f2b040eacf0 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2b04054320 con 0x7f2b04005ed0 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2b040539f0 con 0x7f2b04006850 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc5411640 1 -- 172.21.15.64:0/2988521979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 msgr2=0x7f7cc00ce570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc5411640 1 --2- 172.21.15.64:0/2988521979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc00ce570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc5411640 1 -- 172.21.15.64:0/2988521979 >> v1:172.21.15.66:6789/0 conn(0x7f7cc00b20e0 legacy=0x7f7cc006ca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f7cc5411640 1 -- 172.21.15.64:0/2988521979 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7cc00cdd70 con 0x7f7cc00cee00 2024-03-04T07:26:59.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1656186325 0 0) 0x7f2b04053710 con 0x7f2b040eacf0 2024-03-04T07:26:59.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2afc003520 con 0x7f2b040eacf0 2024-03-04T07:26:59.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0affd640 1 --2- 172.21.15.64:0/236506450 >> v2:172.21.15.64:3300/0 conn(0x7f2b04005ed0 0x7f2b04006310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416607b640 1 -- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cc580 msgr2=0x7f41600c49c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416607b640 1 --2- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cc580 0x7f41600c49c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416607b640 1 -- 172.21.15.64:0/2946662522 >> v2:172.21.15.64:3300/0 conn(0x7f41600cdb60 msgr2=0x7f41600c4f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416607b640 1 --2- 172.21.15.64:0/2946662522 >> v2:172.21.15.64:3300/0 conn(0x7f41600cdb60 0x7f41600c4f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0bfff640 1 --2- 172.21.15.64:0/236506450 >> v2:172.21.15.66:3300/0 conn(0x7f2b04006850 0x7f2b04006cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f416607b640 1 -- 172.21.15.64:0/2946662522 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f41600cd080 con 0x7f41601624e0 2024-03-04T07:26:59.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 <== mon.1 v1:172.21.15.66:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1173897855 0 0) 0x7f2afc003520 con 0x7f2b040eacf0 2024-03-04T07:26:59.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 >> v2:172.21.15.66:3300/0 conn(0x7f2b04006850 msgr2=0x7f2b04006cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 --2- 172.21.15.64:0/236506450 >> v2:172.21.15.66:3300/0 conn(0x7f2b04006850 0x7f2b04006cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 >> v2:172.21.15.64:3300/0 conn(0x7f2b04005ed0 msgr2=0x7f2b04006310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 --2- 172.21.15.64:0/236506450 >> v2:172.21.15.64:3300/0 conn(0x7f2b04005ed0 0x7f2b04006310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 --> v1:172.21.15.66:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b04053cd0 con 0x7f2b040eacf0 2024-03-04T07:26:59.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f7cc5411640 1 --2- 172.21.15.64:0/2988521979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00cee00 0x7f7cc00b4550 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f7cc00b3040 tx=0x7f7cb802be90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f44d464d295385fc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 <== mon.1 v1:172.21.15.66:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2b00001670 con 0x7f2b040eacf0 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 >> v1:172.21.15.66:6789/0 conn(0x7f2b040eacf0 legacy=0x7f2b0404fa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2afc0039b0 0x7f2afc003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2afc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2afc0051c0 0x7f2afc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b04054320 con 0x7f2afc0039b0 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b040539f0 con 0x7f2afc0051c0 2024-03-04T07:26:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2afc00c000 con 0x7f2afc004800 2024-03-04T07:26:59.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.187+0000 7f2b0affd640 1 --2- 172.21.15.64:0/236506450 >> v2:172.21.15.64:3300/0 conn(0x7f2b04005ed0 0x7f2b04006310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:59.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.186+0000 7f21e26d3640 1 -- start start 2024-03-04T07:26:59.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e26d3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f21dc0b3540 0x7f21dc0b2ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e33102640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e2c154110 0x7f2e2c0725d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e33102640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c1544f0 0x7f2e2c15a230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e33102640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2e2c0cd710 0x7f2e2c0ae990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e33102640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2e2c15b210 con 0x7f2e2c154110 2024-03-04T07:26:59.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e16d1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f21dc0b3540 0x7f21dc0b2ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e16d1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f21dc0b3540 0x7f21dc0b2ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40958/0 (socket says 172.21.15.64:40958) 2024-03-04T07:26:59.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e16d1640 1 -- 172.21.15.64:0/3364035056 learned_addr learned my addr 172.21.15.64:0/3364035056 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f5da6edc640 1 Processor -- start 2024-03-04T07:26:59.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3364035056 >> v2:172.21.15.66:3300/0 conn(0x7f21dc0b20e0 0x7f21dc06ca00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21dc0cd7b0 con 0x7f21dc0ce570 2024-03-04T07:26:59.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21dc0cda90 con 0x7f21dc0b3540 2024-03-04T07:26:59.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21dc0cdd70 con 0x7f21dc0b20e0 2024-03-04T07:26:59.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f416607b640 1 --2- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41601624e0 0x7f41600cc040 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f41600737d0 tx=0x7f415802bec0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8327bb6b3e67050 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f5da6edc640 1 -- start start 2024-03-04T07:26:59.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e3169f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2e2c0cd710 0x7f2e2c0ae990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e30e9e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e2c154110 0x7f2e2c0725d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e3169f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2e2c0cd710 0x7f2e2c0ae990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35782/0 (socket says 172.21.15.64:35782) 2024-03-04T07:26:59.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e3169f640 1 -- 172.21.15.64:0/2361235823 learned_addr learned my addr 172.21.15.64:0/2361235823 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e2bfff640 1 --2- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c1544f0 0x7f2e2c15a230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f7cc4c10640 1 -- 172.21.15.64:0/2988521979 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cb8039070 con 0x7f7cc00cee00 2024-03-04T07:26:59.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e33102640 1 -- 172.21.15.64:0/2361235823 --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2e2c15a970 con 0x7f2e2c0cd710 2024-03-04T07:26:59.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2b0affd640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2afc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2b0b7fe640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2afc0039b0 0x7f2afc003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2e33102640 1 -- 172.21.15.64:0/2361235823 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c15ac50 con 0x7f2e2c1544f0 2024-03-04T07:26:59.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2b0affd640 1 -- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2afc0051c0 msgr2=0x7f2afc00b9c0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-03-04T07:26:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2b0affd640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2afc0051c0 0x7f2afc00b9c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2b0affd640 1 -- 172.21.15.64:0/236506450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2afc0039b0 msgr2=0x7f2afc003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2b0affd640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2afc0039b0 0x7f2afc003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f4165079640 1 -- 172.21.15.64:0/2946662522 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4158039070 con 0x7f41601624e0 2024-03-04T07:26:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f2b0affd640 1 -- 172.21.15.64:0/236506450 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b04053cd0 con 0x7f2afc004800 2024-03-04T07:26:59.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f4165079640 1 -- 172.21.15.64:0/2946662522 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f415802cc70 con 0x7f41601624e0 2024-03-04T07:26:59.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f21e0ed0640 1 --2- 172.21.15.64:0/3364035056 >> v2:172.21.15.66:3300/0 conn(0x7f21dc0b20e0 0x7f21dc06ca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f7cc4c10640 1 -- 172.21.15.64:0/2988521979 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7cb802cc40 con 0x7f7cc00cee00 2024-03-04T07:26:59.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.188+0000 7f7cc4c10640 1 -- 172.21.15.64:0/2988521979 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cb8034440 con 0x7f7cc00cee00 2024-03-04T07:26:59.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f4165079640 1 -- 172.21.15.64:0/2946662522 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4158034660 con 0x7f41601624e0 2024-03-04T07:26:59.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f21e16d1640 1 -- 172.21.15.64:0/3364035056 >> v2:172.21.15.66:3300/0 conn(0x7f21dc0b20e0 msgr2=0x7f21dc06ca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f21e16d1640 1 --2- 172.21.15.64:0/3364035056 >> v2:172.21.15.66:3300/0 conn(0x7f21dc0b20e0 0x7f21dc06ca00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f21e16d1640 1 -- 172.21.15.64:0/3364035056 >> v1:172.21.15.66:6789/0 conn(0x7f21dc0ce570 legacy=0x7f21dc0ce950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f21e16d1640 1 -- 172.21.15.64:0/3364035056 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21dc0ce050 con 0x7f21dc0b3540 2024-03-04T07:26:59.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7fea09a1c640 1 Processor -- start 2024-03-04T07:26:59.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f2e3169f640 1 -- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c1544f0 msgr2=0x7f2e2c15a230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f2e3169f640 1 --2- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c1544f0 0x7f2e2c15a230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f2e3169f640 1 -- 172.21.15.64:0/2361235823 >> v2:172.21.15.64:3300/0 conn(0x7f2e2c154110 msgr2=0x7f2e2c0725d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f2e3169f640 1 --2- 172.21.15.64:0/2361235823 >> v2:172.21.15.64:3300/0 conn(0x7f2e2c154110 0x7f2e2c0725d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f2e3169f640 1 -- 172.21.15.64:0/2361235823 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e2c15af30 con 0x7f2e2c0cd710 2024-03-04T07:26:59.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f2e2bfff640 1 --2- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c1544f0 0x7f2e2c15a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:26:59.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f5da6edc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5da01620f0 0x7f5da00b4830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f5da6edc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da00b3920 0x7f5da006e430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f5da6edc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5da00ce500 con 0x7f5da00cd710 2024-03-04T07:26:59.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f5da6edc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5da01624d0 con 0x7f5da01620f0 2024-03-04T07:26:59.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f5da6edc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5da00b3490 con 0x7f5da00b3920 2024-03-04T07:26:59.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f7cc6c14640 1 -- 172.21.15.64:0/2988521979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00cee00 msgr2=0x7f7cc00b4550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f7cc6c14640 1 --2- 172.21.15.64:0/2988521979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00cee00 0x7f7cc00b4550 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f7cc00b3040 tx=0x7f7cb802be90 comp rx=0 tx=0).stop 2024-03-04T07:26:59.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.189+0000 7f2b0bfff640 1 -- 172.21.15.64:0/236506450 reap_dead start 2024-03-04T07:26:59.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f2b0affd640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2afc004c80 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f2af8009c90 tx=0x7f2afc005830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dcbd2d369cd5a9d0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f7cc6c14640 1 -- 172.21.15.64:0/2988521979 shutdown_connections 2024-03-04T07:26:59.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f7cc6c14640 1 --2- 172.21.15.64:0/2988521979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc00ce570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f7cc6c14640 1 --2- 172.21.15.64:0/2988521979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00cee00 0x7f7cc00b4550 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f7cc6c14640 1 -- 172.21.15.64:0/2988521979 >> 172.21.15.64:0/2988521979 conn(0x7f7cc00b15c0 msgr2=0x7f7cc00b19d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5da5eda640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5da01620f0 0x7f5da00b4830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5da5eda640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5da01620f0 0x7f5da00b4830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35796/0 (socket says 172.21.15.64:35796) 2024-03-04T07:26:59.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5da5eda640 1 -- 172.21.15.64:0/841557240 learned_addr learned my addr 172.21.15.64:0/841557240 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f7cc6c14640 1 -- 172.21.15.64:0/2988521979 shutdown_connections 2024-03-04T07:26:59.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41601624e0 msgr2=0x7f41600cc040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f41682df640 1 --2- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41601624e0 0x7f41600cc040 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f41600737d0 tx=0x7f415802bec0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2af8039070 con 0x7f2afc004800 2024-03-04T07:26:59.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2af802c840 con 0x7f2afc004800 2024-03-04T07:26:59.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f2b0a7fc640 1 -- 172.21.15.64:0/236506450 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2af8009860 con 0x7f2afc004800 2024-03-04T07:26:59.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5f7b59e640 1 Processor -- start 2024-03-04T07:26:59.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5f7b59e640 1 -- start start 2024-03-04T07:26:59.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f21e16d1640 1 --2- 172.21.15.64:0/3364035056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 0x7f21dc0b2ab0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f21dc06bbf0 tx=0x7f21d402bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a8b6d9785649e5c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5da5eda640 1 -- 172.21.15.64:0/841557240 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da00b3920 msgr2=0x7f5da006e430 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-03-04T07:26:59.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5da5eda640 1 --2- 172.21.15.64:0/841557240 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da00b3920 0x7f5da006e430 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5da5eda640 1 -- 172.21.15.64:0/841557240 >> v1:172.21.15.64:6789/0 conn(0x7f5da00cd710 legacy=0x7f5da00707a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f5da5eda640 1 -- 172.21.15.64:0/841557240 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5da006c9a0 con 0x7f5da01620f0 2024-03-04T07:26:59.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7fea09a1c640 1 -- start start 2024-03-04T07:26:59.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f2e3169f640 1 --2- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c0cd710 0x7f2e2c0ae990 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f2e2c070950 tx=0x7f2e1c02fd20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cfd39227d6f61d3b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 shutdown_connections 2024-03-04T07:26:59.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f41682df640 1 --2- 172.21.15.64:0/2946662522 >> v2:172.21.15.64:3300/0 conn(0x7f41600cdb60 0x7f41600c4f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f41682df640 1 --2- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cc580 0x7f41600c49c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f41682df640 1 --2- 172.21.15.64:0/2946662522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41601624e0 0x7f41600cc040 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 >> 172.21.15.64:0/2946662522 conn(0x7f41600d19f0 msgr2=0x7f41600d1e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f21d3fff640 1 -- 172.21.15.64:0/3364035056 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21d402c880 con 0x7f21dc0b3540 2024-03-04T07:26:59.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.190+0000 7f7cc6c14640 1 -- 172.21.15.64:0/2988521979 wait complete. 2024-03-04T07:26:59.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 shutdown_connections 2024-03-04T07:26:59.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f41682df640 1 -- 172.21.15.64:0/2946662522 wait complete. 2024-03-04T07:26:59.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f21d3fff640 1 -- 172.21.15.64:0/3364035056 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f21d402ca20 con 0x7f21dc0b3540 2024-03-04T07:26:59.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 msgr2=0x7f2afc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2afc004c80 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f2af8009c90 tx=0x7f2afc005830 comp rx=0 tx=0).stop 2024-03-04T07:26:59.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f21d3fff640 1 -- 172.21.15.64:0/3364035056 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21d402cdf0 con 0x7f21dc0b3540 2024-03-04T07:26:59.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f2e2b7fe640 1 -- 172.21.15.64:0/2361235823 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1c008df0 con 0x7f2e2c0cd710 2024-03-04T07:26:59.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f2e2b7fe640 1 -- 172.21.15.64:0/2361235823 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e1c00a0d0 con 0x7f2e2c0cd710 2024-03-04T07:26:59.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f2e2b7fe640 1 -- 172.21.15.64:0/2361235823 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1c00a4a0 con 0x7f2e2c0cd710 2024-03-04T07:26:59.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f5da5eda640 1 --2- 172.21.15.64:0/841557240 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da01620f0 0x7f5da00b4830 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f5d9c009870 tx=0x7f5d9c02b910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4f278aa99d642949 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f5da4ed8640 1 -- 172.21.15.64:0/841557240 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d9c035020 con 0x7f5da01620f0 2024-03-04T07:26:59.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7fea09a1c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fea041623d0 0x7fea041627b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7fea09a1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea040b3050 0x7fea040b1120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7fea09a1c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fea04163030 con 0x7fea040b34b0 2024-03-04T07:26:59.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7fea09a1c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fea04163310 con 0x7fea041623d0 2024-03-04T07:26:59.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7fea09a1c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fea041635f0 con 0x7fea040b3050 2024-03-04T07:26:59.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f7b59e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5f7c0b2ab0 0x7f5f7c0ce570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f7b59e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5f7c0cd7b0 con 0x7f5f7c0ceab0 2024-03-04T07:26:59.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f7b59e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5f7c0cda90 con 0x7f5f7c0b3540 2024-03-04T07:26:59.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f7b59e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5f7c0cdd70 con 0x7f5f7c0b2ab0 2024-03-04T07:26:59.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f7cc6c14640 1 Processor -- start 2024-03-04T07:26:59.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f7a59c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f5f7c0b3540 0x7f5f7c06ca00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43818/0 (socket says 172.21.15.64:43818) 2024-03-04T07:26:59.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea02ffd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fea041623d0 0x7fea041627b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f7a59c640 1 -- 172.21.15.64:0/1905076059 learned_addr learned my addr 172.21.15.64:0/1905076059 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:59.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f7ad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5f7c0ceab0 0x7f5f7c163940 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42858/0 (socket says 172.21.15.64:42858) 2024-03-04T07:26:59.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea02ffd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fea041623d0 0x7fea041627b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35812/0 (socket says 172.21.15.64:35812) 2024-03-04T07:26:59.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea02ffd640 1 -- 172.21.15.64:0/630013752 learned_addr learned my addr 172.21.15.64:0/630013752 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea027fc640 1 --2- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea040b3050 0x7fea040b1120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f5da4ed8640 1 -- 172.21.15.64:0/841557240 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5d9c031870 con 0x7f5da01620f0 2024-03-04T07:26:59.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2e33102640 1 -- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c0cd710 msgr2=0x7f2e2c0ae990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2e33102640 1 --2- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c0cd710 0x7f2e2c0ae990 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f2e2c070950 tx=0x7f2e1c02fd20 comp rx=0 tx=0).stop 2024-03-04T07:26:59.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2e33102640 1 -- 172.21.15.64:0/2361235823 shutdown_connections 2024-03-04T07:26:59.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.191+0000 7f5da4ed8640 1 -- 172.21.15.64:0/841557240 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d9c031c40 con 0x7f5da01620f0 2024-03-04T07:26:59.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f71d9b640 1 --2- 172.21.15.64:0/1905076059 >> v2:172.21.15.64:3300/0 conn(0x7f5f7c0b2ab0 0x7f5f7c0ce570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3806282693 0 0) 0x7f5f7c0cda90 con 0x7f5f7c0b3540 2024-03-04T07:26:59.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5f68003520 con 0x7f5f7c0b3540 2024-03-04T07:26:59.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1068013284 0 0) 0x7f5f7c0cd7b0 con 0x7f5f7c0ceab0 2024-03-04T07:26:59.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5f7c0cda90 con 0x7f5f7c0ceab0 2024-03-04T07:26:59.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 shutdown_connections 2024-03-04T07:26:59.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/236506450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2afc004c80 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 >> 172.21.15.64:0/236506450 conn(0x7f2b04009fa0 msgr2=0x7f2b0400a380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 msgr2=0x7f21dc0b2ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3364035056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 0x7f21dc0b2ab0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f21dc06bbf0 tx=0x7f21d402bee0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5da6edc640 1 -- 172.21.15.64:0/841557240 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da01620f0 msgr2=0x7f5da00b4830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f5da6edc640 1 --2- 172.21.15.64:0/841557240 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da01620f0 0x7f5da00b4830 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f5d9c009870 tx=0x7f5d9c02b910 comp rx=0 tx=0).stop 2024-03-04T07:26:59.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2e33102640 1 --2- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c0cd710 0x7f2e2c0ae990 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2e33102640 1 --2- 172.21.15.64:0/2361235823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c1544f0 0x7f2e2c15a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2e33102640 1 --2- 172.21.15.64:0/2361235823 >> v2:172.21.15.64:3300/0 conn(0x7f2e2c154110 0x7f2e2c0725d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f2e33102640 1 -- 172.21.15.64:0/2361235823 >> 172.21.15.64:0/2361235823 conn(0x7f2e2c067120 msgr2=0x7f2e2c067530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 shutdown_connections 2024-03-04T07:26:59.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7f7cc6c14640 1 -- start start 2024-03-04T07:26:59.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5da6edc640 1 -- 172.21.15.64:0/841557240 shutdown_connections 2024-03-04T07:26:59.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5da6edc640 1 --2- 172.21.15.64:0/841557240 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da00b3920 0x7f5da006e430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5da6edc640 1 --2- 172.21.15.64:0/841557240 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da01620f0 0x7f5da00b4830 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5da6edc640 1 -- 172.21.15.64:0/841557240 >> 172.21.15.64:0/841557240 conn(0x7f5da0066d00 msgr2=0x7f5da0067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f2e33102640 1 -- 172.21.15.64:0/2361235823 shutdown_connections 2024-03-04T07:26:59.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f2e33102640 1 -- 172.21.15.64:0/2361235823 wait complete. 2024-03-04T07:26:59.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 shutdown_connections 2024-03-04T07:26:59.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3364035056 >> v2:172.21.15.66:3300/0 conn(0x7f21dc0b20e0 0x7f21dc06ca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3364035056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 0x7f21dc0b2ab0 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 >> 172.21.15.64:0/3364035056 conn(0x7f21dc0b15c0 msgr2=0x7f21dc0b19d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5f71d9b640 1 -- 172.21.15.64:0/1905076059 >> v1:172.21.15.66:6789/0 conn(0x7f5f7c0b3540 legacy=0x7f5f7c06ca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5f71d9b640 1 -- 172.21.15.64:0/1905076059 >> v1:172.21.15.64:6789/0 conn(0x7f5f7c0ceab0 legacy=0x7f5f7c163940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 <== mon.1 v1:172.21.15.66:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1742035392 0 0) 0x7f5f68003520 con 0x7f5f7c0b3540 2024-03-04T07:26:59.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f5f71d9b640 1 -- 172.21.15.64:0/1905076059 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f7c0ce050 con 0x7f5f7c0b2ab0 2024-03-04T07:26:59.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea02ffd640 1 -- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea040b3050 msgr2=0x7fea040b1120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea02ffd640 1 --2- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea040b3050 0x7fea040b1120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea02ffd640 1 -- 172.21.15.64:0/630013752 >> v1:172.21.15.66:6789/0 conn(0x7fea040b34b0 legacy=0x7fea04166030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.192+0000 7fea02ffd640 1 -- 172.21.15.64:0/630013752 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea041638d0 con 0x7fea041623d0 2024-03-04T07:26:59.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f41682df640 1 Processor -- start 2024-03-04T07:26:59.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f2b10dd7640 1 -- 172.21.15.64:0/236506450 wait complete. 2024-03-04T07:26:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f41682df640 1 -- start start 2024-03-04T07:26:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc6c14640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00b20e0 0x7f7cc0146560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc6c14640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc015c470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc6c14640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7cc00cee00 0x7f7cc015c9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc6c14640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7cc00ceab0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc6c14640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7cc00b4b30 con 0x7f7cc00cee00 2024-03-04T07:26:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc6c14640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7cc00cec30 con 0x7f7cc00b2ab0 2024-03-04T07:26:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc5c12640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00b20e0 0x7f7cc0146560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7fea02ffd640 1 --2- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea041623d0 0x7fea041627b0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fe9fc009e50 tx=0x7fe9fc02be40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e3e28684ca546bb0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 shutdown_connections 2024-03-04T07:26:59.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f21e26d3640 1 -- 172.21.15.64:0/3364035056 wait complete. 2024-03-04T07:26:59.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc5411640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc015c470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc5c12640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00b20e0 0x7f7cc0146560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40986/0 (socket says 172.21.15.64:40986) 2024-03-04T07:26:59.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc5411640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc015c470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48276/0 (socket says 172.21.15.64:48276) 2024-03-04T07:26:59.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.193+0000 7f7cc5c12640 1 -- 172.21.15.64:0/1588167612 learned_addr learned my addr 172.21.15.64:0/1588167612 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cc5c12640 1 -- 172.21.15.64:0/1588167612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 msgr2=0x7f7cc015c470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f5f71d9b640 1 --2- 172.21.15.64:0/1905076059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b2ab0 0x7f5f7c0ce570 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f5f64009720 tx=0x7f5f6402b7c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a01ef84121ffb739 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cc5c12640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc015c470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cc5c12640 1 -- 172.21.15.64:0/1588167612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7cc00cee00 msgr2=0x7f7cc015c9b0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-04T07:26:59.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cc5c12640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7cc00cee00 0x7f7cc015c9b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cc5c12640 1 -- 172.21.15.64:0/1588167612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7cc015d0c0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f64038070 con 0x7f5f7c0b2ab0 2024-03-04T07:26:59.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cc5c12640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00b20e0 0x7f7cc0146560 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f7cbc00c400 tx=0x7f7cbc00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f5da6edc640 1 -- 172.21.15.64:0/841557240 shutdown_connections 2024-03-04T07:26:59.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f64031520 con 0x7f5f7c0b2ab0 2024-03-04T07:26:59.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f5f79d9b640 1 -- 172.21.15.64:0/1905076059 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f6c001670 con 0x7f5f7c0b2ab0 2024-03-04T07:26:59.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7fea01ffb640 1 -- 172.21.15.64:0/630013752 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe9fc02c9e0 con 0x7fea041623d0 2024-03-04T07:26:59.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f5da6edc640 1 -- 172.21.15.64:0/841557240 wait complete. 2024-03-04T07:26:59.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7fea01ffb640 1 -- 172.21.15.64:0/630013752 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe9fc02cb80 con 0x7fea041623d0 2024-03-04T07:26:59.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f2b10dd7640 1 Processor -- start 2024-03-04T07:26:59.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f41682df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41600cc580 0x7f41600b67e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f41682df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cdb60 0x7f4160155ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f2e33102640 1 Processor -- start 2024-03-04T07:26:59.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f416607b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41600cc580 0x7f41600b67e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f416607b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41600cc580 0x7f41600b67e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35818/0 (socket says 172.21.15.64:35818) 2024-03-04T07:26:59.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f416607b640 1 -- 172.21.15.64:0/384038976 learned_addr learned my addr 172.21.15.64:0/384038976 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f41682df640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41601624e0 0x7f41600b02c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f2e33102640 1 -- start start 2024-03-04T07:26:59.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cb67fc640 1 -- 172.21.15.64:0/1588167612 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cbc017070 con 0x7f7cc00b20e0 2024-03-04T07:26:59.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cb67fc640 1 -- 172.21.15.64:0/1588167612 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7cbc005ce0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.194+0000 7f7cb67fc640 1 -- 172.21.15.64:0/1588167612 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cbc0072f0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f416687c640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41601624e0 0x7f41600b02c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f416587a640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cdb60 0x7f4160155ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41600b12c0 con 0x7f41601624e0 2024-03-04T07:26:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41600d27b0 con 0x7f41600cc580 2024-03-04T07:26:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7fea09a1c640 1 -- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea041623d0 msgr2=0x7fea041627b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7fea09a1c640 1 --2- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea041623d0 0x7fea041627b0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fe9fc009e50 tx=0x7fe9fc02be40 comp rx=0 tx=0).stop 2024-03-04T07:26:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f5f7b59e640 1 -- 172.21.15.64:0/1905076059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b2ab0 msgr2=0x7f5f7c0ce570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f5f7b59e640 1 --2- 172.21.15.64:0/1905076059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b2ab0 0x7f5f7c0ce570 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f5f64009720 tx=0x7f5f6402b7c0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f41600b1440 con 0x7f41600cdb60 2024-03-04T07:26:59.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f7cc6c14640 1 -- 172.21.15.64:0/1588167612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7cc015afa0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f7cc6c14640 1 -- 172.21.15.64:0/1588167612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7cc015b560 con 0x7f7cc00b20e0 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f5f7b59e640 1 -- 172.21.15.64:0/1905076059 shutdown_connections 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f5f7b59e640 1 --2- 172.21.15.64:0/1905076059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b2ab0 0x7f5f7c0ce570 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f5f7b59e640 1 -- 172.21.15.64:0/1905076059 >> 172.21.15.64:0/1905076059 conn(0x7f5f7c0b15c0 msgr2=0x7f5f7c0b19d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.195+0000 7f2b10dd7640 1 -- start start 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b10dd7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2b040d9920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b10dd7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b040464c0 0x7f2b04046940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b10dd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b0404d8a0 0x7f2b0404dd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b10dd7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b04054930 con 0x7f2b0404d8a0 2024-03-04T07:26:59.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f21e26d3640 1 Processor -- start 2024-03-04T07:26:59.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7fea09a1c640 1 -- 172.21.15.64:0/630013752 shutdown_connections 2024-03-04T07:26:59.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7fea09a1c640 1 --2- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea040b3050 0x7fea040b1120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7fea09a1c640 1 --2- 172.21.15.64:0/630013752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea041623d0 0x7fea041627b0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7fea09a1c640 1 -- 172.21.15.64:0/630013752 >> 172.21.15.64:0/630013752 conn(0x7fea04071650 msgr2=0x7fea04072500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b10dd7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b040547b0 con 0x7f2b040464c0 2024-03-04T07:26:59.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b10dd7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2b0404f540 con 0x7f2afc004800 2024-03-04T07:26:59.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f5f7b59e640 1 -- 172.21.15.64:0/1905076059 shutdown_connections 2024-03-04T07:26:59.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f5f7b59e640 1 -- 172.21.15.64:0/1905076059 wait complete. 2024-03-04T07:26:59.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b0b7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2b040d9920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b0bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b0404d8a0 0x7f2b0404dd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b0b7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2b040d9920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48298/0 (socket says 172.21.15.64:48298) 2024-03-04T07:26:59.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f2b0b7fe640 1 -- 172.21.15.64:0/3908037101 learned_addr learned my addr 172.21.15.64:0/3908037101 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f21e26d3640 1 -- start start 2024-03-04T07:26:59.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7fea09a1c640 1 -- 172.21.15.64:0/630013752 shutdown_connections 2024-03-04T07:26:59.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f5da6edc640 1 Processor -- start 2024-03-04T07:26:59.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f5da6edc640 1 -- start start 2024-03-04T07:26:59.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f2b0bfff640 1 -- 172.21.15.64:0/3908037101 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 msgr2=0x7f2b040d9920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f2b0bfff640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2b040d9920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f2b0bfff640 1 -- 172.21.15.64:0/3908037101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b040464c0 msgr2=0x7f2b04046940 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-04T07:26:59.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f2b0bfff640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b040464c0 0x7f2b04046940 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f2b0bfff640 1 -- 172.21.15.64:0/3908037101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b0404e690 con 0x7f2b0404d8a0 2024-03-04T07:26:59.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f7cc6c14640 1 -- 172.21.15.64:0/1588167612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7cc004f330 con 0x7f7cc00b20e0 2024-03-04T07:26:59.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f416607b640 1 -- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cdb60 msgr2=0x7f4160155ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f416607b640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cdb60 0x7f4160155ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f416607b640 1 -- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41601624e0 msgr2=0x7f41600b02c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f416607b640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41601624e0 0x7f41600b02c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7f416607b640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f41600b6d20 con 0x7f41600cc580 2024-03-04T07:26:59.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f5da6edc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da00b3920 0x7f5da01492a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f21e26d3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21dc0b20e0 0x7f21dc0aae80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f21e26d3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 0x7f21dc0a94a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f2b0bfff640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b0404d8a0 0x7f2b0404dd20 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f2b0c0686a0 tx=0x7f2b0c0735a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f21e26d3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21dc0ce570 0x7f21dc0a99e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f21e26d3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21dc0cf060 con 0x7f21dc0b3540 2024-03-04T07:26:59.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f21e26d3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21dc06cf40 con 0x7f21dc0b20e0 2024-03-04T07:26:59.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.197+0000 7f21e26d3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21dc0b3920 con 0x7f21dc0ce570 2024-03-04T07:26:59.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da6edc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5da00cd710 0x7f5da0156910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.196+0000 7fea09a1c640 1 -- 172.21.15.64:0/630013752 wait complete. 2024-03-04T07:26:59.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7fea09a1c640 1 Processor -- start 2024-03-04T07:26:59.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f416607b640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41600cc580 0x7f41600b67e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f4158009e80 tx=0x7f4158004a70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da6edc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da01620f0 0x7f5da015b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da6edc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5da00cdba0 con 0x7f5da00cd710 2024-03-04T07:26:59.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da6edc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5da006cf60 con 0x7f5da00b3920 2024-03-04T07:26:59.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5f7b59e640 1 Processor -- start 2024-03-04T07:26:59.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2af7fff640 1 -- 172.21.15.64:0/3908037101 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b0c094470 con 0x7f2b0404d8a0 2024-03-04T07:26:59.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da6edc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5da00cdd20 con 0x7f5da01620f0 2024-03-04T07:26:59.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f4156ffd640 1 -- 172.21.15.64:0/384038976 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4158044020 con 0x7f41600cc580 2024-03-04T07:26:59.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f4156ffd640 1 -- 172.21.15.64:0/384038976 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f415803a040 con 0x7f41600cc580 2024-03-04T07:26:59.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f4156ffd640 1 -- 172.21.15.64:0/384038976 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4158039070 con 0x7f41600cc580 2024-03-04T07:26:59.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e16d1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21dc0b20e0 0x7f21dc0aae80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e1ed2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21dc0ce570 0x7f21dc0a99e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e1ed2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21dc0ce570 0x7f21dc0a99e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48304/0 (socket says 172.21.15.64:48304) 2024-03-04T07:26:59.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e1ed2640 1 -- 172.21.15.64:0/3979407470 learned_addr learned my addr 172.21.15.64:0/3979407470 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5f7b59e640 1 -- start start 2024-03-04T07:26:59.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da00b3920 0x7f5da01492a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da00b3920 0x7f5da01492a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35840/0 (socket says 172.21.15.64:35840) 2024-03-04T07:26:59.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2b040da330 con 0x7f2b0404d8a0 2024-03-04T07:26:59.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2b040d9e60 con 0x7f2b0404d8a0 2024-03-04T07:26:59.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4160156410 con 0x7f41600cc580 2024-03-04T07:26:59.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f41600b5160 con 0x7f41600cc580 2024-03-04T07:26:59.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 -- 172.21.15.64:0/1926912154 learned_addr learned my addr 172.21.15.64:0/1926912154 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da66db640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da01620f0 0x7f5da015b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e0ed0640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 0x7f21dc0a94a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 -- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da01620f0 msgr2=0x7f5da015b750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da01620f0 0x7f5da015b750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 -- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5da00cd710 msgr2=0x7f5da0156910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5da00cd710 0x7f5da0156910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f5da5eda640 1 -- 172.21.15.64:0/1926912154 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5da01497e0 con 0x7f5da00b3920 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e16d1640 1 -- 172.21.15.64:0/3979407470 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21dc0ce570 msgr2=0x7f21dc0a99e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e16d1640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21dc0ce570 0x7f21dc0a99e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e16d1640 1 -- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 msgr2=0x7f21dc0a94a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e16d1640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 0x7f21dc0a94a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f21e16d1640 1 -- 172.21.15.64:0/3979407470 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21dc1442e0 con 0x7f21dc0b20e0 2024-03-04T07:26:59.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f7cb67fc640 1 -- 172.21.15.64:0/1588167612 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7cbc01a070 con 0x7f7cc00b20e0 2024-03-04T07:26:59.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f2af7fff640 1 -- 172.21.15.64:0/3908037101 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2b0c08f070 con 0x7f2b0404d8a0 2024-03-04T07:26:59.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f2af7fff640 1 -- 172.21.15.64:0/3908037101 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b0c0948d0 con 0x7f2b0404d8a0 2024-03-04T07:26:59.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f21e16d1640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21dc0b20e0 0x7f21dc0aae80 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f21d4032000 tx=0x7f21d4004a40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f5da5eda640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da00b3920 0x7f5da01492a0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f5d9c009870 tx=0x7f5d9c0046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f21d1ffb640 1 -- 172.21.15.64:0/3979407470 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21d4043020 con 0x7f21dc0b20e0 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7fea09a1c640 1 -- start start 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7fea09a1c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea040b3050 0x7fea0415e370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7fea09a1c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea040b34b0 0x7fea040c5990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7fea09a1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea041623d0 0x7fea04072500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7fea09a1c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea04165260 con 0x7fea040b3050 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7fea09a1c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea041650e0 con 0x7fea040b34b0 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7fea09a1c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fea041653e0 con 0x7fea041623d0 2024-03-04T07:26:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f5f7b59e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f7c0b2ab0 0x7f5f7c0a94a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7fea027fc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea040b34b0 0x7fea040c5990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f5f7b59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b3540 0x7f5f7c1f7220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5f7b59e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f7c0ceab0 0x7f5f7c0ab5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5f7b59e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f7c0b2260 con 0x7f5f7c0b3540 2024-03-04T07:26:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5f7b59e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f7c0b20e0 con 0x7f5f7c0ceab0 2024-03-04T07:26:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5f7b59e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f7c0cee90 con 0x7f5f7c0b2ab0 2024-03-04T07:26:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f5d967fc640 1 -- 172.21.15.64:0/1926912154 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d9c0443f0 con 0x7f5da00b3920 2024-03-04T07:26:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2e33102640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c0cd710 0x7f2e2c0addd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5d967fc640 1 -- 172.21.15.64:0/1926912154 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5d9c044590 con 0x7f5da00b3920 2024-03-04T07:26:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5d967fc640 1 -- 172.21.15.64:0/1926912154 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d9c044980 con 0x7f5da00b3920 2024-03-04T07:26:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5da01fb980 con 0x7f5da00b3920 2024-03-04T07:26:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2e33102640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e2c154110 0x7f2e2c1f2910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2e33102640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c1544f0 0x7f2e2c1f2e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2e33102640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c0cc310 con 0x7f2e2c154110 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2e33102640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c0cdba0 con 0x7f2e2c1544f0 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.198+0000 7f2e33102640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e2c0cc490 con 0x7f2e2c0cd710 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f2e2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e2c154110 0x7f2e2c1f2910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f2e2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e2c154110 0x7f2e2c1f2910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41022/0 (socket says 172.21.15.64:41022) 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f2e2bfff640 1 -- 172.21.15.64:0/1132007767 learned_addr learned my addr 172.21.15.64:0/1132007767 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2e30e9e640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c0cd710 0x7f2e2c0addd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5f71d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b3540 0x7f5f7c1f7220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5f71d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b3540 0x7f5f7c1f7220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41046/0 (socket says 172.21.15.64:41046) 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5f71d9b640 1 -- 172.21.15.64:0/3840269037 learned_addr learned my addr 172.21.15.64:0/3840269037 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2aec005290 con 0x7f2b0404d8a0 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7fea027fc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea040b34b0 0x7fea040c5990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:35852/0 (socket says 172.21.15.64:35852) 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7fea027fc640 1 -- 172.21.15.64:0/1718944252 learned_addr learned my addr 172.21.15.64:0/1718944252 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7fea027fc640 1 -- 172.21.15.64:0/1718944252 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea041623d0 msgr2=0x7fea04072500 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7fea027fc640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea041623d0 0x7fea04072500 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7fea027fc640 1 -- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea040b3050 msgr2=0x7fea0415e370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7fea027fc640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea040b3050 0x7fea0415e370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7fea027fc640 1 -- 172.21.15.64:0/1718944252 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea040b14c0 con 0x7fea040b34b0 2024-03-04T07:26:59.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5da01fbf40 con 0x7f5da00b3920 2024-03-04T07:26:59.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2e3169f640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c1544f0 0x7f2e2c1f2e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2e2bfff640 1 -- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c0cd710 msgr2=0x7f2e2c0addd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2e2bfff640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c0cd710 0x7f2e2c0addd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2e2bfff640 1 -- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c1544f0 msgr2=0x7f2e2c1f2e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2e2bfff640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c1544f0 0x7f2e2c1f2e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f2e2bfff640 1 -- 172.21.15.64:0/1132007767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e2c0af5d0 con 0x7f2e2c154110 2024-03-04T07:26:59.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.199+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21dc0ad100 con 0x7f21dc0b20e0 2024-03-04T07:26:59.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21dc0ad6c0 con 0x7f21dc0b20e0 2024-03-04T07:26:59.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.200+0000 7f21d1ffb640 1 -- 172.21.15.64:0/3979407470 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f21d4004040 con 0x7f21dc0b20e0 2024-03-04T07:26:59.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f21d1ffb640 1 -- 172.21.15.64:0/3979407470 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21d4004450 con 0x7f21dc0b20e0 2024-03-04T07:26:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f5f71d9b640 1 -- 172.21.15.64:0/3840269037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f7c0b2ab0 msgr2=0x7f5f7c0a94a0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-04T07:26:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f5f71d9b640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f7c0b2ab0 0x7f5f7c0a94a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f5f71d9b640 1 -- 172.21.15.64:0/3840269037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f7c0ceab0 msgr2=0x7f5f7c0ab5e0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-03-04T07:26:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f7cb67fc640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ca406b4d0 0x7f7ca406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f5f71d9b640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f7c0ceab0 0x7f5f7c0ab5e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f5f71d9b640 1 -- 172.21.15.64:0/3840269037 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f7c1eaef0 con 0x7f5f7c0b3540 2024-03-04T07:26:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f21bc005290 con 0x7f21dc0b20e0 2024-03-04T07:26:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f7cb67fc640 1 -- 172.21.15.64:0/1588167612 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7cbc0896e0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f5f71d9b640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b3540 0x7f5f7c1f7220 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f5f6402bdd0 tx=0x7f5f640046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7f2e2bfff640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e2c154110 0x7f2e2c1f2910 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f2e2000c7f0 tx=0x7f2e2000ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7f7cc5411640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ca406b4d0 0x7f7ca406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5da004f330 con 0x7f5da00b3920 2024-03-04T07:26:59.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7fea027fc640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea040b34b0 0x7fea040c5990 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fe9f800e870 tx=0x7fe9f800ed30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.201+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f416004f330 con 0x7f41600cc580 2024-03-04T07:26:59.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7fe9ebfff640 1 -- 172.21.15.64:0/1718944252 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe9f8017070 con 0x7fea040b34b0 2024-03-04T07:26:59.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea040b17a0 con 0x7fea040b34b0 2024-03-04T07:26:59.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f21d1ffb640 1 -- 172.21.15.64:0/3979407470 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f21d4046020 con 0x7f21dc0b20e0 2024-03-04T07:26:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea040b1d60 con 0x7fea040b34b0 2024-03-04T07:26:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7f7cc5411640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ca406b4d0 0x7f7ca406d990 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7cb8036000 tx=0x7f7cb8030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7f5d967fc640 1 -- 172.21.15.64:0/1926912154 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5d9c04b060 con 0x7f5da00b3920 2024-03-04T07:26:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.202+0000 7fe9ebfff640 1 -- 172.21.15.64:0/1718944252 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe9f800ee60 con 0x7fea040b34b0 2024-03-04T07:26:59.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f5d967fc640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d8406b5a0 0x7f5d8406da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f21d1ffb640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f21c006b520 0x7f21c006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f5d967fc640 1 -- 172.21.15.64:0/1926912154 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5d9c0ac270 con 0x7f5da00b3920 2024-03-04T07:26:59.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f5da56d9640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d8406b5a0 0x7f5d8406da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f4156ffd640 1 -- 172.21.15.64:0/384038976 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4158047020 con 0x7f41600cc580 2024-03-04T07:26:59.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f21d1ffb640 1 -- 172.21.15.64:0/3979407470 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f21d40ac820 con 0x7f21dc0b20e0 2024-03-04T07:26:59.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7fe9ebfff640 1 -- 172.21.15.64:0/1718944252 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe9f800cb80 con 0x7fea040b34b0 2024-03-04T07:26:59.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f2af7fff640 1 -- 172.21.15.64:0/3908037101 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2b0c094a70 con 0x7f2b0404d8a0 2024-03-04T07:26:59.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7f21e0ed0640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f21c006b520 0x7f21c006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f4156ffd640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f414406b5a0 0x7f414406da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f5da56d9640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d8406b5a0 0x7f5d8406da60 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f5da0153750 tx=0x7f5d90006c40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f416587a640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f414406b5a0 0x7f414406da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f21e0ed0640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f21c006b520 0x7f21c006d9e0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f21dc14f680 tx=0x7f21d8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f4156ffd640 1 -- 172.21.15.64:0/384038976 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f41580acb80 con 0x7f41600cc580 2024-03-04T07:26:59.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.203+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe9e4005290 con 0x7fea040b34b0 2024-03-04T07:26:59.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f2af7fff640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2af006b5d0 0x7f2af006da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f2b0b7fe640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2af006b5d0 0x7f2af006da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f416587a640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f414406b5a0 0x7f414406da60 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f41600b0f30 tx=0x7f4150008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.204+0000 7f2af7fff640 1 -- 172.21.15.64:0/3908037101 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2b0c104ea0 con 0x7f2b0404d8a0 2024-03-04T07:26:59.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.205+0000 7f2b0b7fe640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2af006b5d0 0x7f2af006da90 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f2b00007fd0 tx=0x7f2b00001e70 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.207+0000 7f5f737fe640 1 -- 172.21.15.64:0/3840269037 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f64041020 con 0x7f5f7c0b3540 2024-03-04T07:26:59.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.207+0000 7f5f7b59e640 1 -- 172.21.15.64:0/3840269037 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5f7c0a99e0 con 0x7f5f7c0b3540 2024-03-04T07:26:59.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.207+0000 7f5f7b59e640 1 -- 172.21.15.64:0/3840269037 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5f7c1eb440 con 0x7f5f7c0b3540 2024-03-04T07:26:59.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.207+0000 7f7cb67fc640 1 -- 172.21.15.64:0/1588167612 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7cbc0584c0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.208+0000 7f5f7b59e640 1 -- 172.21.15.64:0/3840269037 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5f7c1eb870 con 0x7f5f7c0b3540 2024-03-04T07:26:59.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f2af7fff640 1 -- 172.21.15.64:0/3908037101 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2b0c099060 con 0x7f2b0404d8a0 2024-03-04T07:26:59.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f2e297fa640 1 -- 172.21.15.64:0/1132007767 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e20015070 con 0x7f2e2c154110 2024-03-04T07:26:59.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f2e297fa640 1 -- 172.21.15.64:0/1132007767 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e20007820 con 0x7f2e2c154110 2024-03-04T07:26:59.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f2e297fa640 1 -- 172.21.15.64:0/1132007767 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e20007bd0 con 0x7f2e2c154110 2024-03-04T07:26:59.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2e2c0af8b0 con 0x7f2e2c154110 2024-03-04T07:26:59.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f5f737fe640 1 -- 172.21.15.64:0/3840269037 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f64031520 con 0x7f5f7c0b3540 2024-03-04T07:26:59.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f5f737fe640 1 -- 172.21.15.64:0/3840269037 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f64038040 con 0x7f5f7c0b3540 2024-03-04T07:26:59.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.211+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2e2c0afe70 con 0x7f2e2c154110 2024-03-04T07:26:59.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.213+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2e2c04f330 con 0x7f2e2c154110 2024-03-04T07:26:59.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.213+0000 7f5d967fc640 1 -- 172.21.15.64:0/1926912154 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5d9c03f070 con 0x7f5da00b3920 2024-03-04T07:26:59.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.213+0000 7f5f737fe640 1 -- 172.21.15.64:0/3840269037 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5f64044020 con 0x7f5f7c0b3540 2024-03-04T07:26:59.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.213+0000 7f5f737fe640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f5c06b4d0 0x7f5f5c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.213+0000 7f5f7a59c640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f5c06b4d0 0x7f5f5c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.214+0000 7f5f737fe640 1 -- 172.21.15.64:0/3840269037 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5f640ab940 con 0x7f5f7c0b3540 2024-03-04T07:26:59.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.214+0000 7f5f7a59c640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f5c06b4d0 0x7f5f5c06d990 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f5f6c0011e0 tx=0x7f5f6c001110 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.214+0000 7fe9ebfff640 1 -- 172.21.15.64:0/1718944252 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe9f801a070 con 0x7fea040b34b0 2024-03-04T07:26:59.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.214+0000 7f2e297fa640 1 -- 172.21.15.64:0/1132007767 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2e20018070 con 0x7f2e2c154110 2024-03-04T07:26:59.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.215+0000 7fe9ebfff640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe9f006b520 0x7fe9f006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.215+0000 7f2e297fa640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e1806b520 0x7f2e1806d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.215+0000 7fea02ffd640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe9f006b520 0x7fe9f006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.215+0000 7f2e30e9e640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e1806b520 0x7f2e1806d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.216+0000 7f2e30e9e640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e1806b520 0x7f2e1806d9e0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f2e24009fd0 tx=0x7f2e2400b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.216+0000 7fea02ffd640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe9f006b520 0x7fe9f006d9e0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fe9fc035000 tx=0x7fe9fc02e000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.222+0000 7f4156ffd640 1 -- 172.21.15.64:0/384038976 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f415804e030 con 0x7f41600cc580 2024-03-04T07:26:59.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.222+0000 7f2e297fa640 1 -- 172.21.15.64:0/1132007767 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2e20056ef0 con 0x7f2e2c154110 2024-03-04T07:26:59.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.222+0000 7f5f737fe640 1 -- 172.21.15.64:0/3840269037 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5f6404b070 con 0x7f5f7c0b3540 2024-03-04T07:26:59.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.231+0000 7fe9ebfff640 1 -- 172.21.15.64:0/1718944252 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe9f8057110 con 0x7fea040b34b0 2024-03-04T07:26:59.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.231+0000 7f21d1ffb640 1 -- 172.21.15.64:0/3979407470 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f21d404d030 con 0x7f21dc0b20e0 2024-03-04T07:26:59.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.232+0000 7f2e297fa640 1 -- 172.21.15.64:0/1132007767 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2e20059550 con 0x7f2e2c154110 2024-03-04T07:26:59.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.237+0000 7fe9ebfff640 1 -- 172.21.15.64:0/1718944252 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe9f8059770 con 0x7fea040b34b0 2024-03-04T07:26:59.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.630+0000 7f7cc6c14640 1 -- 172.21.15.64:0/1588167612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 7} v 0) v1 -- 0x7f7cc00ae4f0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.631+0000 7f7cb67fc640 1 -- 172.21.15.64:0/1588167612 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 7}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f7cbc057dc0 con 0x7f7cc00b20e0 2024-03-04T07:26:59.630 INFO:teuthology.orchestra.run.smithi064.stdout:55834574856 2024-03-04T07:26:59.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.633+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f2aec005580 con 0x7f2b0404d8a0 2024-03-04T07:26:59.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.633+0000 7f2af7fff640 1 -- 172.21.15.64:0/3908037101 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f2b0c0d3d90 con 0x7f2b0404d8a0 2024-03-04T07:26:59.633 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:59.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.634+0000 7f7ca3fff640 1 -- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ca406b4d0 msgr2=0x7f7ca406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.634+0000 7f7ca3fff640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ca406b4d0 0x7f7ca406d990 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7cb8036000 tx=0x7f7cb8030000 comp rx=0 tx=0).stop 2024-03-04T07:26:59.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.634+0000 7f7ca3fff640 1 -- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00b20e0 msgr2=0x7f7cc0146560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.634+0000 7f7ca3fff640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00b20e0 0x7f7cc0146560 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f7cbc00c400 tx=0x7f7cbc00c8c0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.635+0000 7f7ca3fff640 1 -- 172.21.15.64:0/1588167612 shutdown_connections 2024-03-04T07:26:59.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.635+0000 7f7ca3fff640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7ca406b4d0 0x7f7ca406d990 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.635+0000 7f7ca3fff640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7cc00cee00 0x7f7cc015c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.636+0000 7f7ca3fff640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7cc00b2ab0 0x7f7cc015c470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.636+0000 7f7ca3fff640 1 --2- 172.21.15.64:0/1588167612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7cc00b20e0 0x7f7cc0146560 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.635+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) v1 -- 0x7f5da015d590 con 0x7f5da00b3920 2024-03-04T07:26:59.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.636+0000 7f7ca3fff640 1 -- 172.21.15.64:0/1588167612 >> 172.21.15.64:0/1588167612 conn(0x7f7cc00b15c0 msgr2=0x7f7cc00b19a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.636+0000 7f7ca3fff640 1 -- 172.21.15.64:0/1588167612 shutdown_connections 2024-03-04T07:26:59.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.636+0000 7f5d967fc640 1 -- 172.21.15.64:0/1926912154 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f5d9c02f320 con 0x7f5da00b3920 2024-03-04T07:26:59.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.636+0000 7f7ca3fff640 1 -- 172.21.15.64:0/1588167612 wait complete. 2024-03-04T07:26:59.637 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.637+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2af006b5d0 msgr2=0x7f2af006da90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2af006b5d0 0x7f2af006da90 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f2b00007fd0 tx=0x7f2b00001e70 comp rx=0 tx=0).stop 2024-03-04T07:26:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b0404d8a0 msgr2=0x7f2b0404dd20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b0404d8a0 0x7f2b0404dd20 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f2b0c0686a0 tx=0x7f2b0c0735a0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 shutdown_connections 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2af006b5d0 0x7f2af006da90 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b0404d8a0 0x7f2b0404dd20 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b040464c0 0x7f2b04046940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 --2- 172.21.15.64:0/3908037101 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2afc004800 0x7f2b040d9920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 >> 172.21.15.64:0/3908037101 conn(0x7f2b04009fa0 msgr2=0x7f2b04036420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 shutdown_connections 2024-03-04T07:26:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.638+0000 7f2b10dd7640 1 -- 172.21.15.64:0/3908037101 wait complete. 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d8406b5a0 msgr2=0x7f5d8406da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d8406b5a0 0x7f5d8406da60 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f5da0153750 tx=0x7f5d90006c40 comp rx=0 tx=0).stop 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da00b3920 msgr2=0x7f5da01492a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da00b3920 0x7f5da01492a0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f5d9c009870 tx=0x7f5d9c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 shutdown_connections 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d8406b5a0 0x7f5d8406da60 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5da01620f0 0x7f5da015b750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5da00cd710 0x7f5da0156910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 --2- 172.21.15.64:0/1926912154 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5da00b3920 0x7f5da01492a0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.639+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 >> 172.21.15.64:0/1926912154 conn(0x7f5da0066d00 msgr2=0x7f5da006d0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.640+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 shutdown_connections 2024-03-04T07:26:59.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.640+0000 7f5da6edc640 1 -- 172.21.15.64:0/1926912154 wait complete. 2024-03-04T07:26:59.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.650+0000 7f5f7b59e640 1 -- 172.21.15.64:0/3840269037 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) v1 -- 0x7f5f7c0aefb0 con 0x7f5f7c0b3540 2024-03-04T07:26:59.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.651+0000 7f5f737fe640 1 -- 172.21.15.64:0/3840269037 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f5f6407a7a0 con 0x7f5f7c0b3540 2024-03-04T07:26:59.650 INFO:teuthology.orchestra.run.smithi064.stdout:55834574856 2024-03-04T07:26:59.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.654+0000 7f5f70d99640 1 -- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f5c06b4d0 msgr2=0x7f5f5c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.654+0000 7f5f70d99640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f5c06b4d0 0x7f5f5c06d990 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f5f6c0011e0 tx=0x7f5f6c001110 comp rx=0 tx=0).stop 2024-03-04T07:26:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.654+0000 7f5f70d99640 1 -- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b3540 msgr2=0x7f5f7c1f7220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.654+0000 7f5f70d99640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b3540 0x7f5f7c1f7220 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f5f6402bdd0 tx=0x7f5f640046c0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 -- 172.21.15.64:0/3840269037 shutdown_connections 2024-03-04T07:26:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f5c06b4d0 0x7f5f5c06d990 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f7c0ceab0 0x7f5f7c0ab5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f7c0b3540 0x7f5f7c1f7220 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 --2- 172.21.15.64:0/3840269037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f7c0b2ab0 0x7f5f7c0a94a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 -- 172.21.15.64:0/3840269037 >> 172.21.15.64:0/3840269037 conn(0x7f5f7c0b15c0 msgr2=0x7f5f7c0b19a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f21bc005580 con 0x7f21dc0b20e0 2024-03-04T07:26:59.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 -- 172.21.15.64:0/3840269037 shutdown_connections 2024-03-04T07:26:59.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f5f70d99640 1 -- 172.21.15.64:0/3840269037 wait complete. 2024-03-04T07:26:59.656 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574856 got 55834574856 for osd.7 2024-03-04T07:26:59.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.655+0000 7f21d1ffb640 1 -- 172.21.15.64:0/3979407470 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f21d407b7a0 con 0x7f21dc0b20e0 2024-03-04T07:26:59.657 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:59.657 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.656+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 6} v 0) v1 -- 0x7f2e2c0b3a70 con 0x7f2e2c154110 2024-03-04T07:26:59.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.657+0000 7f2e297fa640 1 -- 172.21.15.64:0/1132007767 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 6}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f2e20058e50 con 0x7f2e2c154110 2024-03-04T07:26:59.658 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:59.658 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574855 got 55834574855 for osd.5 2024-03-04T07:26:59.659 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.659 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574855 got 55834574855 for osd.1 2024-03-04T07:26:59.660 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f21c006b520 msgr2=0x7f21c006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f21c006b520 0x7f21c006d9e0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f21dc14f680 tx=0x7f21d8008040 comp rx=0 tx=0).stop 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21dc0b20e0 msgr2=0x7f21dc0aae80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21dc0b20e0 0x7f21dc0aae80 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f21d4032000 tx=0x7f21d4004a40 comp rx=0 tx=0).stop 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 shutdown_connections 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f21c006b520 0x7f21c006d9e0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21dc0ce570 0x7f21dc0a99e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21dc0b3540 0x7f21dc0a94a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.658+0000 7f21e26d3640 1 --2- 172.21.15.64:0/3979407470 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21dc0b20e0 0x7f21dc0aae80 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.659+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 >> 172.21.15.64:0/3979407470 conn(0x7f21dc0b15c0 msgr2=0x7f21dc0b0d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.659+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 shutdown_connections 2024-03-04T07:26:59.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.659+0000 7f21e26d3640 1 -- 172.21.15.64:0/3979407470 wait complete. 2024-03-04T07:26:59.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.659+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7f41600cf060 con 0x7f41600cc580 2024-03-04T07:26:59.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.659+0000 7f4156ffd640 1 -- 172.21.15.64:0/384038976 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f4160156410 con 0x7f41600cc580 2024-03-04T07:26:59.662 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:59.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e1806b520 msgr2=0x7f2e1806d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e1806b520 0x7f2e1806d9e0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f2e24009fd0 tx=0x7f2e2400b040 comp rx=0 tx=0).stop 2024-03-04T07:26:59.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e2c154110 msgr2=0x7f2e2c1f2910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e2c154110 0x7f2e2c1f2910 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f2e2000c7f0 tx=0x7f2e2000ccb0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 shutdown_connections 2024-03-04T07:26:59.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e1806b520 0x7f2e1806d9e0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e2c1544f0 0x7f2e2c1f2e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e2c154110 0x7f2e2c1f2910 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 --2- 172.21.15.64:0/1132007767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e2c0cd710 0x7f2e2c0addd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 >> 172.21.15.64:0/1132007767 conn(0x7f2e2c067120 msgr2=0x7f2e2c06cef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 shutdown_connections 2024-03-04T07:26:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7f2e33102640 1 -- 172.21.15.64:0/1132007767 wait complete. 2024-03-04T07:26:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.660+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7fe9e4005850 con 0x7fea040b34b0 2024-03-04T07:26:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.661+0000 7fe9ebfff640 1 -- 172.21.15.64:0/1718944252 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fe9f8059070 con 0x7fea040b34b0 2024-03-04T07:26:59.665 INFO:teuthology.orchestra.run.smithi064.stdout:55834574855 2024-03-04T07:26:59.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f414406b5a0 msgr2=0x7f414406da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f414406b5a0 0x7f414406da60 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f41600b0f30 tx=0x7f4150008040 comp rx=0 tx=0).stop 2024-03-04T07:26:59.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41600cc580 msgr2=0x7f41600b67e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41600cc580 0x7f41600b67e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f4158009e80 tx=0x7f4158004a70 comp rx=0 tx=0).stop 2024-03-04T07:26:59.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 shutdown_connections 2024-03-04T07:26:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f414406b5a0 0x7f414406da60 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41601624e0 0x7f41600b02c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41600cdb60 0x7f4160155ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 --2- 172.21.15.64:0/384038976 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41600cc580 0x7f41600b67e0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 >> 172.21.15.64:0/384038976 conn(0x7f41600d19f0 msgr2=0x7f41600cdf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 shutdown_connections 2024-03-04T07:26:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7f41682df640 1 -- 172.21.15.64:0/384038976 wait complete. 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe9f006b520 msgr2=0x7fe9f006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe9f006b520 0x7fe9f006d9e0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fe9fc035000 tx=0x7fe9fc02e000 comp rx=0 tx=0).stop 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea040b34b0 msgr2=0x7fea040c5990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea040b34b0 0x7fea040c5990 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fe9f800e870 tx=0x7fe9f800ed30 comp rx=0 tx=0).stop 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 shutdown_connections 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe9f006b520 0x7fe9f006d9e0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea041623d0 0x7fea04072500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea040b34b0 0x7fea040c5990 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 --2- 172.21.15.64:0/1718944252 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea040b3050 0x7fea0415e370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.663+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 >> 172.21.15.64:0/1718944252 conn(0x7fea04071650 msgr2=0x7fea0409c130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.664+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 shutdown_connections 2024-03-04T07:26:59.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.664+0000 7fea09a1c640 1 -- 172.21.15.64:0/1718944252 wait complete. 2024-03-04T07:26:59.675 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574855 got 55834574855 for osd.6 2024-03-04T07:26:59.675 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.676 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574856 got 55834574856 for osd.4 2024-03-04T07:26:59.676 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.677 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574855 got 55834574855 for osd.2 2024-03-04T07:26:59.677 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.680 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574855 got 55834574855 for osd.0 2024-03-04T07:26:59.680 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.681 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574855 got 55834574855 for osd.3 2024-03-04T07:26:59.681 DEBUG:teuthology.parallel:result is None 2024-03-04T07:26:59.681 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-03-04T07:26:59.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:26:59.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.822+0000 7ffa6ad0c640 1 Processor -- start 2024-03-04T07:26:59.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.822+0000 7ffa6ad0c640 1 -- start start 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.822+0000 7ffa6ad0c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ffa6415e000 0x7ffa64071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.822+0000 7ffa6ad0c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640d35e0 0x7ffa64071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.822+0000 7ffa6ad0c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffa640ce970 con 0x7ffa640b2c50 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.822+0000 7ffa6ad0c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffa640cece0 con 0x7ffa6415e000 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.822+0000 7ffa6ad0c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffa640cef30 con 0x7ffa640d35e0 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa68aa8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ffa6415e000 0x7ffa64071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa63fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ffa640b2c50 0x7ffa64161cb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42862/0 (socket says 172.21.15.64:42862) 2024-03-04T07:26:59.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa68aa8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ffa6415e000 0x7ffa64071540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41058/0 (socket says 172.21.15.64:41058) 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa63fff640 1 -- 172.21.15.64:0/1213216875 learned_addr learned my addr 172.21.15.64:0/1213216875 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa692a9640 1 --2- 172.21.15.64:0/1213216875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640d35e0 0x7ffa64071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa68aa8640 1 -- 172.21.15.64:0/1213216875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640d35e0 msgr2=0x7ffa64071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa68aa8640 1 --2- 172.21.15.64:0/1213216875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640d35e0 0x7ffa64071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa68aa8640 1 -- 172.21.15.64:0/1213216875 >> v1:172.21.15.64:6789/0 conn(0x7ffa640b2c50 legacy=0x7ffa64161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa68aa8640 1 -- 172.21.15.64:0/1213216875 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffa640cf210 con 0x7ffa6415e000 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.823+0000 7ffa637fe640 1 -- 172.21.15.64:0/1213216875 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2596124255 0 0) 0x7ffa640ce970 con 0x7ffa640b2c50 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa68aa8640 1 --2- 172.21.15.64:0/1213216875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa64071540 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7ffa5c0097a0 tx=0x7ffa5c02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=58d03e923c386f8c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa637fe640 1 -- 172.21.15.64:0/1213216875 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa5c0315a0 con 0x7ffa6415e000 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa637fe640 1 -- 172.21.15.64:0/1213216875 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffa5c037070 con 0x7ffa6415e000 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa637fe640 1 -- 172.21.15.64:0/1213216875 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa5c031960 con 0x7ffa6415e000 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/1213216875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 msgr2=0x7ffa64071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/1213216875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa64071540 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7ffa5c0097a0 tx=0x7ffa5c02b840 comp rx=0 tx=0).stop 2024-03-04T07:26:59.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/1213216875 shutdown_connections 2024-03-04T07:26:59.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/1213216875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640d35e0 0x7ffa64071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/1213216875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa64071540 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/1213216875 >> 172.21.15.64:0/1213216875 conn(0x7ffa640d3c80 msgr2=0x7ffa640d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:26:59.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/1213216875 shutdown_connections 2024-03-04T07:26:59.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.824+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/1213216875 wait complete. 2024-03-04T07:26:59.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.825+0000 7ffa6ad0c640 1 Processor -- start 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa6ad0c640 1 -- start start 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa6ad0c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640b2c50 0x7ffa6414e130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa6ad0c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa640d35e0 0x7ffa640cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa6ad0c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa640cc760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa6ad0c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffa640b3320 con 0x7ffa6415e000 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa6ad0c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffa640b31a0 con 0x7ffa640d35e0 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa6ad0c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffa6415e3e0 con 0x7ffa640b2c50 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa692a9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa640cc760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa692a9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa640cc760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41072/0 (socket says 172.21.15.64:41072) 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa692a9640 1 -- 172.21.15.64:0/4233164938 learned_addr learned my addr 172.21.15.64:0/4233164938 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa63fff640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa640d35e0 0x7ffa640cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.826+0000 7ffa68aa8640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640b2c50 0x7ffa6414e130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa692a9640 1 -- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640b2c50 msgr2=0x7ffa6414e130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa692a9640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640b2c50 0x7ffa6414e130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa692a9640 1 -- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa640d35e0 msgr2=0x7ffa640cad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa692a9640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa640d35e0 0x7ffa640cad50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa692a9640 1 -- 172.21.15.64:0/4233164938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffa640cb6c0 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa63fff640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa640d35e0 0x7ffa640cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa692a9640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa640cc760 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7ffa4c00e7f0 tx=0x7ffa4c00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa617fa640 1 -- 172.21.15.64:0/4233164938 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa4c017070 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa617fa640 1 -- 172.21.15.64:0/4233164938 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffa4c00c730 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa617fa640 1 -- 172.21.15.64:0/4233164938 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa4c00cb20 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffa640cb9a0 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.827+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffa640cbf60 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.828+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffa64067ff0 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.829+0000 7ffa617fa640 1 -- 172.21.15.64:0/4233164938 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffa4c01a070 con 0x7ffa6415e000 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.829+0000 7ffa617fa640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa4406b4d0 0x7ffa4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:26:59.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.829+0000 7ffa68aa8640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa4406b4d0 0x7ffa4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:26:59.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.829+0000 7ffa617fa640 1 -- 172.21.15.64:0/4233164938 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ffa4c08a390 con 0x7ffa6415e000 2024-03-04T07:26:59.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.830+0000 7ffa68aa8640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa4406b4d0 0x7ffa4406d990 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7ffa5c036000 tx=0x7ffa5c03d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:26:59.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:26:59.834+0000 7ffa617fa640 1 -- 172.21.15.64:0/4233164938 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffa4c058bc0 con 0x7ffa6415e000 2024-03-04T07:27:00.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.012+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7ffa6415c700 con 0x7ffa4406b4d0 2024-03-04T07:27:00.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.018+0000 7ffa617fa640 1 -- 172.21.15.64:0/4233164938 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251836 (secure 0 0 0) 0x7ffa6415c700 con 0x7ffa4406b4d0 2024-03-04T07:27:00.021 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:00.022 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa4406b4d0 msgr2=0x7ffa4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa4406b4d0 0x7ffa4406d990 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7ffa5c036000 tx=0x7ffa5c03d040 comp rx=0 tx=0).stop 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 msgr2=0x7ffa640cc760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa640cc760 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7ffa4c00e7f0 tx=0x7ffa4c00ecb0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 shutdown_connections 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa4406b4d0 0x7ffa4406d990 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa6415e000 0x7ffa640cc760 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa640d35e0 0x7ffa640cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 --2- 172.21.15.64:0/4233164938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa640b2c50 0x7ffa6414e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 >> 172.21.15.64:0/4233164938 conn(0x7ffa640d3c80 msgr2=0x7ffa640cf750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:00.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 shutdown_connections 2024-03-04T07:27:00.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.021+0000 7ffa6ad0c640 1 -- 172.21.15.64:0/4233164938 wait complete. 2024-03-04T07:27:00.031 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":22,"stamp":"2024-03-04T07:26:59.143040+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":190,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219008,"kb_used_data":2824,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755712,"statfs":{"total":773094113280,"available":772869849088,"internally_reserved":0,"allocated":2891776,"data_stored":1753028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.839614"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389967+0000","last_change":"2024-03-04T07:26:53.331782+0000","last_active":"2024-03-04T07:26:54.389967+0000","last_peered":"2024-03-04T07:26:54.389967+0000","last_clean":"2024-03-04T07:26:54.389967+0000","last_became_active":"2024-03-04T07:26:53.331449+0000","last_became_peered":"2024-03-04T07:26:53.331449+0000","last_unstale":"2024-03-04T07:26:54.389967+0000","last_undegraded":"2024-03-04T07:26:54.389967+0000","last_fullsized":"2024-03-04T07:26:54.389967+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305647+0000","last_change":"2024-03-04T07:26:53.321240+0000","last_active":"2024-03-04T07:26:54.305647+0000","last_peered":"2024-03-04T07:26:54.305647+0000","last_clean":"2024-03-04T07:26:54.305647+0000","last_became_active":"2024-03-04T07:26:53.320945+0000","last_became_peered":"2024-03-04T07:26:53.320945+0000","last_unstale":"2024-03-04T07:26:54.305647+0000","last_undegraded":"2024-03-04T07:26:54.305647+0000","last_fullsized":"2024-03-04T07:26:54.305647+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390253+0000","last_change":"2024-03-04T07:26:53.317216+0000","last_active":"2024-03-04T07:26:54.390253+0000","last_peered":"2024-03-04T07:26:54.390253+0000","last_clean":"2024-03-04T07:26:54.390253+0000","last_became_active":"2024-03-04T07:26:53.316875+0000","last_became_peered":"2024-03-04T07:26:53.316875+0000","last_unstale":"2024-03-04T07:26:54.390253+0000","last_undegraded":"2024-03-04T07:26:54.390253+0000","last_fullsized":"2024-03-04T07:26:54.390253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306389+0000","last_change":"2024-03-04T07:26:53.326427+0000","last_active":"2024-03-04T07:26:54.306389+0000","last_peered":"2024-03-04T07:26:54.306389+0000","last_clean":"2024-03-04T07:26:54.306389+0000","last_became_active":"2024-03-04T07:26:53.326097+0000","last_became_peered":"2024-03-04T07:26:53.326097+0000","last_unstale":"2024-03-04T07:26:54.306389+0000","last_undegraded":"2024-03-04T07:26:54.306389+0000","last_fullsized":"2024-03-04T07:26:54.306389+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389857+0000","last_change":"2024-03-04T07:26:53.334207+0000","last_active":"2024-03-04T07:26:54.389857+0000","last_peered":"2024-03-04T07:26:54.389857+0000","last_clean":"2024-03-04T07:26:54.389857+0000","last_became_active":"2024-03-04T07:26:53.333629+0000","last_became_peered":"2024-03-04T07:26:53.333629+0000","last_unstale":"2024-03-04T07:26:54.389857+0000","last_undegraded":"2024-03-04T07:26:54.389857+0000","last_fullsized":"2024-03-04T07:26:54.389857+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305566+0000","last_change":"2024-03-04T07:26:53.322181+0000","last_active":"2024-03-04T07:26:54.305566+0000","last_peered":"2024-03-04T07:26:54.305566+0000","last_clean":"2024-03-04T07:26:54.305566+0000","last_became_active":"2024-03-04T07:26:53.321619+0000","last_became_peered":"2024-03-04T07:26:53.321619+0000","last_unstale":"2024-03-04T07:26:54.305566+0000","last_undegraded":"2024-03-04T07:26:54.305566+0000","last_fullsized":"2024-03-04T07:26:54.305566+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389617+0000","last_change":"2024-03-04T07:26:53.329485+0000","last_active":"2024-03-04T07:26:54.389617+0000","last_peered":"2024-03-04T07:26:54.389617+0000","last_clean":"2024-03-04T07:26:54.389617+0000","last_became_active":"2024-03-04T07:26:53.329174+0000","last_became_peered":"2024-03-04T07:26:53.329174+0000","last_unstale":"2024-03-04T07:26:54.389617+0000","last_undegraded":"2024-03-04T07:26:54.389617+0000","last_fullsized":"2024-03-04T07:26:54.389617+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389430+0000","last_change":"2024-03-04T07:26:53.327402+0000","last_active":"2024-03-04T07:26:54.389430+0000","last_peered":"2024-03-04T07:26:54.389430+0000","last_clean":"2024-03-04T07:26:54.389430+0000","last_became_active":"2024-03-04T07:26:53.326893+0000","last_became_peered":"2024-03-04T07:26:53.326893+0000","last_unstale":"2024-03-04T07:26:54.389430+0000","last_undegraded":"2024-03-04T07:26:54.389430+0000","last_fullsized":"2024-03-04T07:26:54.389430+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.341109+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.340834+0000","last_became_peered":"2024-03-04T07:26:53.340834+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324909+0000","last_change":"2024-03-04T07:26:53.325503+0000","last_active":"2024-03-04T07:26:54.324909+0000","last_peered":"2024-03-04T07:26:54.324909+0000","last_clean":"2024-03-04T07:26:54.324909+0000","last_became_active":"2024-03-04T07:26:53.325191+0000","last_became_peered":"2024-03-04T07:26:53.325191+0000","last_unstale":"2024-03-04T07:26:54.324909+0000","last_undegraded":"2024-03-04T07:26:54.324909+0000","last_fullsized":"2024-03-04T07:26:54.324909+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324908+0000","last_change":"2024-03-04T07:26:53.327284+0000","last_active":"2024-03-04T07:26:54.324908+0000","last_peered":"2024-03-04T07:26:54.324908+0000","last_clean":"2024-03-04T07:26:54.324908+0000","last_became_active":"2024-03-04T07:26:53.327011+0000","last_became_peered":"2024-03-04T07:26:53.327011+0000","last_unstale":"2024-03-04T07:26:54.324908+0000","last_undegraded":"2024-03-04T07:26:54.324908+0000","last_fullsized":"2024-03-04T07:26:54.324908+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306083+0000","last_change":"2024-03-04T07:26:53.316248+0000","last_active":"2024-03-04T07:26:54.306083+0000","last_peered":"2024-03-04T07:26:54.306083+0000","last_clean":"2024-03-04T07:26:54.306083+0000","last_became_active":"2024-03-04T07:26:53.315903+0000","last_became_peered":"2024-03-04T07:26:53.315903+0000","last_unstale":"2024-03-04T07:26:54.306083+0000","last_undegraded":"2024-03-04T07:26:54.306083+0000","last_fullsized":"2024-03-04T07:26:54.306083+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389868+0000","last_change":"2024-03-04T07:26:53.319814+0000","last_active":"2024-03-04T07:26:54.389868+0000","last_peered":"2024-03-04T07:26:54.389868+0000","last_clean":"2024-03-04T07:26:54.389868+0000","last_became_active":"2024-03-04T07:26:53.319503+0000","last_became_peered":"2024-03-04T07:26:53.319503+0000","last_unstale":"2024-03-04T07:26:54.389868+0000","last_undegraded":"2024-03-04T07:26:54.389868+0000","last_fullsized":"2024-03-04T07:26:54.389868+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389681+0000","last_change":"2024-03-04T07:26:53.318624+0000","last_active":"2024-03-04T07:26:54.389681+0000","last_peered":"2024-03-04T07:26:54.389681+0000","last_clean":"2024-03-04T07:26:54.389681+0000","last_became_active":"2024-03-04T07:26:53.318299+0000","last_became_peered":"2024-03-04T07:26:53.318299+0000","last_unstale":"2024-03-04T07:26:54.389681+0000","last_undegraded":"2024-03-04T07:26:54.389681+0000","last_fullsized":"2024-03-04T07:26:54.389681+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:13:39.358840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305782+0000","last_change":"2024-03-04T07:26:53.319312+0000","last_active":"2024-03-04T07:26:54.305782+0000","last_peered":"2024-03-04T07:26:54.305782+0000","last_clean":"2024-03-04T07:26:54.305782+0000","last_became_active":"2024-03-04T07:26:53.319021+0000","last_became_peered":"2024-03-04T07:26:53.319021+0000","last_unstale":"2024-03-04T07:26:54.305782+0000","last_undegraded":"2024-03-04T07:26:54.305782+0000","last_fullsized":"2024-03-04T07:26:54.305782+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305585+0000","last_change":"2024-03-04T07:26:53.313675+0000","last_active":"2024-03-04T07:26:54.305585+0000","last_peered":"2024-03-04T07:26:54.305585+0000","last_clean":"2024-03-04T07:26:54.305585+0000","last_became_active":"2024-03-04T07:26:53.313325+0000","last_became_peered":"2024-03-04T07:26:53.313325+0000","last_unstale":"2024-03-04T07:26:54.305585+0000","last_undegraded":"2024-03-04T07:26:54.305585+0000","last_fullsized":"2024-03-04T07:26:54.305585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306601+0000","last_change":"2024-03-04T07:26:53.335375+0000","last_active":"2024-03-04T07:26:54.306601+0000","last_peered":"2024-03-04T07:26:54.306601+0000","last_clean":"2024-03-04T07:26:54.306601+0000","last_became_active":"2024-03-04T07:26:53.335101+0000","last_became_peered":"2024-03-04T07:26:53.335101+0000","last_unstale":"2024-03-04T07:26:54.306601+0000","last_undegraded":"2024-03-04T07:26:54.306601+0000","last_fullsized":"2024-03-04T07:26:54.306601+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390303+0000","last_change":"2024-03-04T07:26:53.320188+0000","last_active":"2024-03-04T07:26:54.390303+0000","last_peered":"2024-03-04T07:26:54.390303+0000","last_clean":"2024-03-04T07:26:54.390303+0000","last_became_active":"2024-03-04T07:26:53.319902+0000","last_became_peered":"2024-03-04T07:26:53.319902+0000","last_unstale":"2024-03-04T07:26:54.390303+0000","last_undegraded":"2024-03-04T07:26:54.390303+0000","last_fullsized":"2024-03-04T07:26:54.390303+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390194+0000","last_change":"2024-03-04T07:26:53.334274+0000","last_active":"2024-03-04T07:26:54.390194+0000","last_peered":"2024-03-04T07:26:54.390194+0000","last_clean":"2024-03-04T07:26:54.390194+0000","last_became_active":"2024-03-04T07:26:53.333828+0000","last_became_peered":"2024-03-04T07:26:53.333828+0000","last_unstale":"2024-03-04T07:26:54.390194+0000","last_undegraded":"2024-03-04T07:26:54.390194+0000","last_fullsized":"2024-03-04T07:26:54.390194+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:08:42.738284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325119+0000","last_change":"2024-03-04T07:26:53.322810+0000","last_active":"2024-03-04T07:26:54.325119+0000","last_peered":"2024-03-04T07:26:54.325119+0000","last_clean":"2024-03-04T07:26:54.325119+0000","last_became_active":"2024-03-04T07:26:53.322527+0000","last_became_peered":"2024-03-04T07:26:53.322527+0000","last_unstale":"2024-03-04T07:26:54.325119+0000","last_undegraded":"2024-03-04T07:26:54.325119+0000","last_fullsized":"2024-03-04T07:26:54.325119+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390380+0000","last_change":"2024-03-04T07:26:53.326816+0000","last_active":"2024-03-04T07:26:54.390380+0000","last_peered":"2024-03-04T07:26:54.390380+0000","last_clean":"2024-03-04T07:26:54.390380+0000","last_became_active":"2024-03-04T07:26:53.326028+0000","last_became_peered":"2024-03-04T07:26:53.326028+0000","last_unstale":"2024-03-04T07:26:54.390380+0000","last_undegraded":"2024-03-04T07:26:54.390380+0000","last_fullsized":"2024-03-04T07:26:54.390380+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306215+0000","last_change":"2024-03-04T07:26:53.328614+0000","last_active":"2024-03-04T07:26:54.306215+0000","last_peered":"2024-03-04T07:26:54.306215+0000","last_clean":"2024-03-04T07:26:54.306215+0000","last_became_active":"2024-03-04T07:26:53.328262+0000","last_became_peered":"2024-03-04T07:26:53.328262+0000","last_unstale":"2024-03-04T07:26:54.306215+0000","last_undegraded":"2024-03-04T07:26:54.306215+0000","last_fullsized":"2024-03-04T07:26:54.306215+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390277+0000","last_change":"2024-03-04T07:26:53.322798+0000","last_active":"2024-03-04T07:26:54.390277+0000","last_peered":"2024-03-04T07:26:54.390277+0000","last_clean":"2024-03-04T07:26:54.390277+0000","last_became_active":"2024-03-04T07:26:53.321992+0000","last_became_peered":"2024-03-04T07:26:53.321992+0000","last_unstale":"2024-03-04T07:26:54.390277+0000","last_undegraded":"2024-03-04T07:26:54.390277+0000","last_fullsized":"2024-03-04T07:26:54.390277+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305730+0000","last_change":"2024-03-04T07:26:53.323813+0000","last_active":"2024-03-04T07:26:54.305730+0000","last_peered":"2024-03-04T07:26:54.305730+0000","last_clean":"2024-03-04T07:26:54.305730+0000","last_became_active":"2024-03-04T07:26:53.323530+0000","last_became_peered":"2024-03-04T07:26:53.323530+0000","last_unstale":"2024-03-04T07:26:54.305730+0000","last_undegraded":"2024-03-04T07:26:54.305730+0000","last_fullsized":"2024-03-04T07:26:54.305730+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390125+0000","last_change":"2024-03-04T07:26:53.322537+0000","last_active":"2024-03-04T07:26:54.390125+0000","last_peered":"2024-03-04T07:26:54.390125+0000","last_clean":"2024-03-04T07:26:54.390125+0000","last_became_active":"2024-03-04T07:26:53.322254+0000","last_became_peered":"2024-03-04T07:26:53.322254+0000","last_unstale":"2024-03-04T07:26:54.390125+0000","last_undegraded":"2024-03-04T07:26:54.390125+0000","last_fullsized":"2024-03-04T07:26:54.390125+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325071+0000","last_change":"2024-03-04T07:26:53.332147+0000","last_active":"2024-03-04T07:26:54.325071+0000","last_peered":"2024-03-04T07:26:54.325071+0000","last_clean":"2024-03-04T07:26:54.325071+0000","last_became_active":"2024-03-04T07:26:53.331346+0000","last_became_peered":"2024-03-04T07:26:53.331346+0000","last_unstale":"2024-03-04T07:26:54.325071+0000","last_undegraded":"2024-03-04T07:26:54.325071+0000","last_fullsized":"2024-03-04T07:26:54.325071+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305687+0000","last_change":"2024-03-04T07:26:53.325767+0000","last_active":"2024-03-04T07:26:54.305687+0000","last_peered":"2024-03-04T07:26:54.305687+0000","last_clean":"2024-03-04T07:26:54.305687+0000","last_became_active":"2024-03-04T07:26:53.325486+0000","last_became_peered":"2024-03-04T07:26:53.325486+0000","last_unstale":"2024-03-04T07:26:54.305687+0000","last_undegraded":"2024-03-04T07:26:54.305687+0000","last_fullsized":"2024-03-04T07:26:54.305687+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389819+0000","last_change":"2024-03-04T07:26:53.323242+0000","last_active":"2024-03-04T07:26:54.389819+0000","last_peered":"2024-03-04T07:26:54.389819+0000","last_clean":"2024-03-04T07:26:54.389819+0000","last_became_active":"2024-03-04T07:26:53.322945+0000","last_became_peered":"2024-03-04T07:26:53.322945+0000","last_unstale":"2024-03-04T07:26:54.389819+0000","last_undegraded":"2024-03-04T07:26:54.389819+0000","last_fullsized":"2024-03-04T07:26:54.389819+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390452+0000","last_change":"2024-03-04T07:26:53.321011+0000","last_active":"2024-03-04T07:26:54.390452+0000","last_peered":"2024-03-04T07:26:54.390452+0000","last_clean":"2024-03-04T07:26:54.390452+0000","last_became_active":"2024-03-04T07:26:53.320663+0000","last_became_peered":"2024-03-04T07:26:53.320663+0000","last_unstale":"2024-03-04T07:26:54.390452+0000","last_undegraded":"2024-03-04T07:26:54.390452+0000","last_fullsized":"2024-03-04T07:26:54.390452+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306658+0000","last_change":"2024-03-04T07:26:53.321889+0000","last_active":"2024-03-04T07:26:54.306658+0000","last_peered":"2024-03-04T07:26:54.306658+0000","last_clean":"2024-03-04T07:26:54.306658+0000","last_became_active":"2024-03-04T07:26:53.321304+0000","last_became_peered":"2024-03-04T07:26:53.321304+0000","last_unstale":"2024-03-04T07:26:54.306658+0000","last_undegraded":"2024-03-04T07:26:54.306658+0000","last_fullsized":"2024-03-04T07:26:54.306658+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390060+0000","last_change":"2024-03-04T07:26:53.323893+0000","last_active":"2024-03-04T07:26:54.390060+0000","last_peered":"2024-03-04T07:26:54.390060+0000","last_clean":"2024-03-04T07:26:54.390060+0000","last_became_active":"2024-03-04T07:26:53.323608+0000","last_became_peered":"2024-03-04T07:26:53.323608+0000","last_unstale":"2024-03-04T07:26:54.390060+0000","last_undegraded":"2024-03-04T07:26:54.390060+0000","last_fullsized":"2024-03-04T07:26:54.390060+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324678+0000","last_change":"2024-03-04T07:26:53.318669+0000","last_active":"2024-03-04T07:26:54.324678+0000","last_peered":"2024-03-04T07:26:54.324678+0000","last_clean":"2024-03-04T07:26:54.324678+0000","last_became_active":"2024-03-04T07:26:53.318412+0000","last_became_peered":"2024-03-04T07:26:53.318412+0000","last_unstale":"2024-03-04T07:26:54.324678+0000","last_undegraded":"2024-03-04T07:26:54.324678+0000","last_fullsized":"2024-03-04T07:26:54.324678+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:53.317343+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:53.317041+0000","last_became_peered":"2024-03-04T07:26:53.317041+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:05.230078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390061+0000","last_change":"2024-03-04T07:26:52.326593+0000","last_active":"2024-03-04T07:26:54.390061+0000","last_peered":"2024-03-04T07:26:54.390061+0000","last_clean":"2024-03-04T07:26:54.390061+0000","last_became_active":"2024-03-04T07:26:52.326321+0000","last_became_peered":"2024-03-04T07:26:52.326321+0000","last_unstale":"2024-03-04T07:26:54.390061+0000","last_undegraded":"2024-03-04T07:26:54.390061+0000","last_fullsized":"2024-03-04T07:26:54.390061+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390516+0000","last_change":"2024-03-04T07:26:53.317929+0000","last_active":"2024-03-04T07:26:54.390516+0000","last_peered":"2024-03-04T07:26:54.390516+0000","last_clean":"2024-03-04T07:26:54.390516+0000","last_became_active":"2024-03-04T07:26:53.317631+0000","last_became_peered":"2024-03-04T07:26:53.317631+0000","last_unstale":"2024-03-04T07:26:54.390516+0000","last_undegraded":"2024-03-04T07:26:54.390516+0000","last_fullsized":"2024-03-04T07:26:54.390516+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324662+0000","last_change":"2024-03-04T07:26:52.321659+0000","last_active":"2024-03-04T07:26:54.324662+0000","last_peered":"2024-03-04T07:26:54.324662+0000","last_clean":"2024-03-04T07:26:54.324662+0000","last_became_active":"2024-03-04T07:26:52.321247+0000","last_became_peered":"2024-03-04T07:26:52.321247+0000","last_unstale":"2024-03-04T07:26:54.324662+0000","last_undegraded":"2024-03-04T07:26:54.324662+0000","last_fullsized":"2024-03-04T07:26:54.324662+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390279+0000","last_change":"2024-03-04T07:26:52.316474+0000","last_active":"2024-03-04T07:26:54.390279+0000","last_peered":"2024-03-04T07:26:54.390279+0000","last_clean":"2024-03-04T07:26:54.390279+0000","last_became_active":"2024-03-04T07:26:52.316148+0000","last_became_peered":"2024-03-04T07:26:52.316148+0000","last_unstale":"2024-03-04T07:26:54.390279+0000","last_undegraded":"2024-03-04T07:26:54.390279+0000","last_fullsized":"2024-03-04T07:26:54.390279+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389444+0000","last_change":"2024-03-04T07:26:53.325862+0000","last_active":"2024-03-04T07:26:54.389444+0000","last_peered":"2024-03-04T07:26:54.389444+0000","last_clean":"2024-03-04T07:26:54.389444+0000","last_became_active":"2024-03-04T07:26:53.325450+0000","last_became_peered":"2024-03-04T07:26:53.325450+0000","last_unstale":"2024-03-04T07:26:54.389444+0000","last_undegraded":"2024-03-04T07:26:54.389444+0000","last_fullsized":"2024-03-04T07:26:54.389444+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305831+0000","last_change":"2024-03-04T07:26:52.312838+0000","last_active":"2024-03-04T07:26:54.305831+0000","last_peered":"2024-03-04T07:26:54.305831+0000","last_clean":"2024-03-04T07:26:54.305831+0000","last_became_active":"2024-03-04T07:26:52.312335+0000","last_became_peered":"2024-03-04T07:26:52.312335+0000","last_unstale":"2024-03-04T07:26:54.305831+0000","last_undegraded":"2024-03-04T07:26:54.305831+0000","last_fullsized":"2024-03-04T07:26:54.305831+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305454+0000","last_change":"2024-03-04T07:26:53.324974+0000","last_active":"2024-03-04T07:26:54.305454+0000","last_peered":"2024-03-04T07:26:54.305454+0000","last_clean":"2024-03-04T07:26:54.305454+0000","last_became_active":"2024-03-04T07:26:53.324662+0000","last_became_peered":"2024-03-04T07:26:53.324662+0000","last_unstale":"2024-03-04T07:26:54.305454+0000","last_undegraded":"2024-03-04T07:26:54.305454+0000","last_fullsized":"2024-03-04T07:26:54.305454+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324840+0000","last_change":"2024-03-04T07:26:52.317328+0000","last_active":"2024-03-04T07:26:54.324840+0000","last_peered":"2024-03-04T07:26:54.324840+0000","last_clean":"2024-03-04T07:26:54.324840+0000","last_became_active":"2024-03-04T07:26:52.317039+0000","last_became_peered":"2024-03-04T07:26:52.317039+0000","last_unstale":"2024-03-04T07:26:54.324840+0000","last_undegraded":"2024-03-04T07:26:54.324840+0000","last_fullsized":"2024-03-04T07:26:54.324840+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306365+0000","last_change":"2024-03-04T07:26:53.329677+0000","last_active":"2024-03-04T07:26:54.306365+0000","last_peered":"2024-03-04T07:26:54.306365+0000","last_clean":"2024-03-04T07:26:54.306365+0000","last_became_active":"2024-03-04T07:26:53.327273+0000","last_became_peered":"2024-03-04T07:26:53.327273+0000","last_unstale":"2024-03-04T07:26:54.306365+0000","last_undegraded":"2024-03-04T07:26:54.306365+0000","last_fullsized":"2024-03-04T07:26:54.306365+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306307+0000","last_change":"2024-03-04T07:26:52.317388+0000","last_active":"2024-03-04T07:26:54.306307+0000","last_peered":"2024-03-04T07:26:54.306307+0000","last_clean":"2024-03-04T07:26:54.306307+0000","last_became_active":"2024-03-04T07:26:52.316946+0000","last_became_peered":"2024-03-04T07:26:52.316946+0000","last_unstale":"2024-03-04T07:26:54.306307+0000","last_undegraded":"2024-03-04T07:26:54.306307+0000","last_fullsized":"2024-03-04T07:26:54.306307+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390169+0000","last_change":"2024-03-04T07:26:53.324201+0000","last_active":"2024-03-04T07:26:54.390169+0000","last_peered":"2024-03-04T07:26:54.390169+0000","last_clean":"2024-03-04T07:26:54.390169+0000","last_became_active":"2024-03-04T07:26:53.323923+0000","last_became_peered":"2024-03-04T07:26:53.323923+0000","last_unstale":"2024-03-04T07:26:54.390169+0000","last_undegraded":"2024-03-04T07:26:54.390169+0000","last_fullsized":"2024-03-04T07:26:54.390169+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:50.898693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"19'3","reported_seq":17,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.317701+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.317411+0000","last_became_peered":"2024-03-04T07:26:52.317411+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306245+0000","last_change":"2024-03-04T07:26:53.334757+0000","last_active":"2024-03-04T07:26:54.306245+0000","last_peered":"2024-03-04T07:26:54.306245+0000","last_clean":"2024-03-04T07:26:54.306245+0000","last_became_active":"2024-03-04T07:26:53.331793+0000","last_became_peered":"2024-03-04T07:26:53.331793+0000","last_unstale":"2024-03-04T07:26:54.306245+0000","last_undegraded":"2024-03-04T07:26:54.306245+0000","last_fullsized":"2024-03-04T07:26:54.306245+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:23.419725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"19'6","reported_seq":20,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325093+0000","last_change":"2024-03-04T07:26:52.318084+0000","last_active":"2024-03-04T07:26:54.325093+0000","last_peered":"2024-03-04T07:26:54.325093+0000","last_clean":"2024-03-04T07:26:54.325093+0000","last_became_active":"2024-03-04T07:26:52.317701+0000","last_became_peered":"2024-03-04T07:26:52.317701+0000","last_unstale":"2024-03-04T07:26:54.325093+0000","last_undegraded":"2024-03-04T07:26:54.325093+0000","last_fullsized":"2024-03-04T07:26:54.325093+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390023+0000","last_change":"2024-03-04T07:26:53.331720+0000","last_active":"2024-03-04T07:26:54.390023+0000","last_peered":"2024-03-04T07:26:54.390023+0000","last_clean":"2024-03-04T07:26:54.390023+0000","last_became_active":"2024-03-04T07:26:53.331397+0000","last_became_peered":"2024-03-04T07:26:53.331397+0000","last_unstale":"2024-03-04T07:26:54.390023+0000","last_undegraded":"2024-03-04T07:26:54.390023+0000","last_fullsized":"2024-03-04T07:26:54.390023+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390546+0000","last_change":"2024-03-04T07:26:52.319865+0000","last_active":"2024-03-04T07:26:54.390546+0000","last_peered":"2024-03-04T07:26:54.390546+0000","last_clean":"2024-03-04T07:26:54.390546+0000","last_became_active":"2024-03-04T07:26:52.319514+0000","last_became_peered":"2024-03-04T07:26:52.319514+0000","last_unstale":"2024-03-04T07:26:54.390546+0000","last_undegraded":"2024-03-04T07:26:54.390546+0000","last_fullsized":"2024-03-04T07:26:54.390546+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389831+0000","last_change":"2024-03-04T07:26:53.327475+0000","last_active":"2024-03-04T07:26:54.389831+0000","last_peered":"2024-03-04T07:26:54.389831+0000","last_clean":"2024-03-04T07:26:54.389831+0000","last_became_active":"2024-03-04T07:26:53.325978+0000","last_became_peered":"2024-03-04T07:26:53.325978+0000","last_unstale":"2024-03-04T07:26:54.389831+0000","last_undegraded":"2024-03-04T07:26:54.389831+0000","last_fullsized":"2024-03-04T07:26:54.389831+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:14:35.134882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306345+0000","last_change":"2024-03-04T07:26:52.322481+0000","last_active":"2024-03-04T07:26:54.306345+0000","last_peered":"2024-03-04T07:26:54.306345+0000","last_clean":"2024-03-04T07:26:54.306345+0000","last_became_active":"2024-03-04T07:26:52.322193+0000","last_became_peered":"2024-03-04T07:26:52.322193+0000","last_unstale":"2024-03-04T07:26:54.306345+0000","last_undegraded":"2024-03-04T07:26:54.306345+0000","last_fullsized":"2024-03-04T07:26:54.306345+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389953+0000","last_change":"2024-03-04T07:26:53.339564+0000","last_active":"2024-03-04T07:26:54.389953+0000","last_peered":"2024-03-04T07:26:54.389953+0000","last_clean":"2024-03-04T07:26:54.389953+0000","last_became_active":"2024-03-04T07:26:53.338911+0000","last_became_peered":"2024-03-04T07:26:53.338911+0000","last_unstale":"2024-03-04T07:26:54.389953+0000","last_undegraded":"2024-03-04T07:26:54.389953+0000","last_fullsized":"2024-03-04T07:26:54.389953+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324657+0000","last_change":"2024-03-04T07:26:52.321570+0000","last_active":"2024-03-04T07:26:54.324657+0000","last_peered":"2024-03-04T07:26:54.324657+0000","last_clean":"2024-03-04T07:26:54.324657+0000","last_became_active":"2024-03-04T07:26:52.321193+0000","last_became_peered":"2024-03-04T07:26:52.321193+0000","last_unstale":"2024-03-04T07:26:54.324657+0000","last_undegraded":"2024-03-04T07:26:54.324657+0000","last_fullsized":"2024-03-04T07:26:54.324657+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324683+0000","last_change":"2024-03-04T07:26:53.318654+0000","last_active":"2024-03-04T07:26:54.324683+0000","last_peered":"2024-03-04T07:26:54.324683+0000","last_clean":"2024-03-04T07:26:54.324683+0000","last_became_active":"2024-03-04T07:26:53.318357+0000","last_became_peered":"2024-03-04T07:26:53.318357+0000","last_unstale":"2024-03-04T07:26:54.324683+0000","last_undegraded":"2024-03-04T07:26:54.324683+0000","last_fullsized":"2024-03-04T07:26:54.324683+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"19'8","reported_seq":22,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390444+0000","last_change":"2024-03-04T07:26:52.317772+0000","last_active":"2024-03-04T07:26:54.390444+0000","last_peered":"2024-03-04T07:26:54.390444+0000","last_clean":"2024-03-04T07:26:54.390444+0000","last_became_active":"2024-03-04T07:26:52.316794+0000","last_became_peered":"2024-03-04T07:26:52.316794+0000","last_unstale":"2024-03-04T07:26:54.390444+0000","last_undegraded":"2024-03-04T07:26:54.390444+0000","last_fullsized":"2024-03-04T07:26:54.390444+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389575+0000","last_change":"2024-03-04T07:26:53.340074+0000","last_active":"2024-03-04T07:26:54.389575+0000","last_peered":"2024-03-04T07:26:54.389575+0000","last_clean":"2024-03-04T07:26:54.389575+0000","last_became_active":"2024-03-04T07:26:53.339265+0000","last_became_peered":"2024-03-04T07:26:53.339265+0000","last_unstale":"2024-03-04T07:26:54.389575+0000","last_undegraded":"2024-03-04T07:26:54.389575+0000","last_fullsized":"2024-03-04T07:26:54.389575+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306140+0000","last_change":"2024-03-04T07:26:52.313758+0000","last_active":"2024-03-04T07:26:54.306140+0000","last_peered":"2024-03-04T07:26:54.306140+0000","last_clean":"2024-03-04T07:26:54.306140+0000","last_became_active":"2024-03-04T07:26:52.313362+0000","last_became_peered":"2024-03-04T07:26:52.313362+0000","last_unstale":"2024-03-04T07:26:54.306140+0000","last_undegraded":"2024-03-04T07:26:54.306140+0000","last_fullsized":"2024-03-04T07:26:54.306140+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389920+0000","last_change":"2024-03-04T07:26:53.337849+0000","last_active":"2024-03-04T07:26:54.389920+0000","last_peered":"2024-03-04T07:26:54.389920+0000","last_clean":"2024-03-04T07:26:54.389920+0000","last_became_active":"2024-03-04T07:26:53.337529+0000","last_became_peered":"2024-03-04T07:26:53.337529+0000","last_unstale":"2024-03-04T07:26:54.389920+0000","last_undegraded":"2024-03-04T07:26:54.389920+0000","last_fullsized":"2024-03-04T07:26:54.389920+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390527+0000","last_change":"2024-03-04T07:26:52.318490+0000","last_active":"2024-03-04T07:26:54.390527+0000","last_peered":"2024-03-04T07:26:54.390527+0000","last_clean":"2024-03-04T07:26:54.390527+0000","last_became_active":"2024-03-04T07:26:52.318217+0000","last_became_peered":"2024-03-04T07:26:52.318217+0000","last_unstale":"2024-03-04T07:26:54.390527+0000","last_undegraded":"2024-03-04T07:26:54.390527+0000","last_fullsized":"2024-03-04T07:26:54.390527+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390128+0000","last_change":"2024-03-04T07:26:53.338814+0000","last_active":"2024-03-04T07:26:54.390128+0000","last_peered":"2024-03-04T07:26:54.390128+0000","last_clean":"2024-03-04T07:26:54.390128+0000","last_became_active":"2024-03-04T07:26:53.338090+0000","last_became_peered":"2024-03-04T07:26:53.338090+0000","last_unstale":"2024-03-04T07:26:54.390128+0000","last_undegraded":"2024-03-04T07:26:54.390128+0000","last_fullsized":"2024-03-04T07:26:54.390128+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:33.083393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305758+0000","last_change":"2024-03-04T07:26:52.320092+0000","last_active":"2024-03-04T07:26:54.305758+0000","last_peered":"2024-03-04T07:26:54.305758+0000","last_clean":"2024-03-04T07:26:54.305758+0000","last_became_active":"2024-03-04T07:26:52.319835+0000","last_became_peered":"2024-03-04T07:26:52.319835+0000","last_unstale":"2024-03-04T07:26:54.305758+0000","last_undegraded":"2024-03-04T07:26:54.305758+0000","last_fullsized":"2024-03-04T07:26:54.305758+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306585+0000","last_change":"2024-03-04T07:26:53.325995+0000","last_active":"2024-03-04T07:26:54.306585+0000","last_peered":"2024-03-04T07:26:54.306585+0000","last_clean":"2024-03-04T07:26:54.306585+0000","last_became_active":"2024-03-04T07:26:53.325550+0000","last_became_peered":"2024-03-04T07:26:53.325550+0000","last_unstale":"2024-03-04T07:26:54.306585+0000","last_undegraded":"2024-03-04T07:26:54.306585+0000","last_fullsized":"2024-03-04T07:26:54.306585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389741+0000","last_change":"2024-03-04T07:26:52.318023+0000","last_active":"2024-03-04T07:26:54.389741+0000","last_peered":"2024-03-04T07:26:54.389741+0000","last_clean":"2024-03-04T07:26:54.389741+0000","last_became_active":"2024-03-04T07:26:52.317127+0000","last_became_peered":"2024-03-04T07:26:52.317127+0000","last_unstale":"2024-03-04T07:26:54.389741+0000","last_undegraded":"2024-03-04T07:26:54.389741+0000","last_fullsized":"2024-03-04T07:26:54.389741+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:10.212684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390046+0000","last_change":"2024-03-04T07:26:53.337973+0000","last_active":"2024-03-04T07:26:54.390046+0000","last_peered":"2024-03-04T07:26:54.390046+0000","last_clean":"2024-03-04T07:26:54.390046+0000","last_became_active":"2024-03-04T07:26:53.337551+0000","last_became_peered":"2024-03-04T07:26:53.337551+0000","last_unstale":"2024-03-04T07:26:54.390046+0000","last_undegraded":"2024-03-04T07:26:54.390046+0000","last_fullsized":"2024-03-04T07:26:54.390046+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":170,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389506+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:26:54.389506+0000","last_peered":"2024-03-04T07:26:54.389506+0000","last_clean":"2024-03-04T07:26:54.389506+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:26:54.389506+0000","last_undegraded":"2024-03-04T07:26:54.389506+0000","last_fullsized":"2024-03-04T07:26:54.389506+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306129+0000","last_change":"2024-03-04T07:26:52.317678+0000","last_active":"2024-03-04T07:26:54.306129+0000","last_peered":"2024-03-04T07:26:54.306129+0000","last_clean":"2024-03-04T07:26:54.306129+0000","last_became_active":"2024-03-04T07:26:52.317008+0000","last_became_peered":"2024-03-04T07:26:52.317008+0000","last_unstale":"2024-03-04T07:26:54.306129+0000","last_undegraded":"2024-03-04T07:26:54.306129+0000","last_fullsized":"2024-03-04T07:26:54.306129+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389685+0000","last_change":"2024-03-04T07:26:53.327257+0000","last_active":"2024-03-04T07:26:54.389685+0000","last_peered":"2024-03-04T07:26:54.389685+0000","last_clean":"2024-03-04T07:26:54.389685+0000","last_became_active":"2024-03-04T07:26:53.326960+0000","last_became_peered":"2024-03-04T07:26:53.326960+0000","last_unstale":"2024-03-04T07:26:54.389685+0000","last_undegraded":"2024-03-04T07:26:54.389685+0000","last_fullsized":"2024-03-04T07:26:54.389685+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389451+0000","last_change":"2024-03-04T07:26:52.325578+0000","last_active":"2024-03-04T07:26:54.389451+0000","last_peered":"2024-03-04T07:26:54.389451+0000","last_clean":"2024-03-04T07:26:54.389451+0000","last_became_active":"2024-03-04T07:26:52.325291+0000","last_became_peered":"2024-03-04T07:26:52.325291+0000","last_unstale":"2024-03-04T07:26:54.389451+0000","last_undegraded":"2024-03-04T07:26:54.389451+0000","last_fullsized":"2024-03-04T07:26:54.389451+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:58.480243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389676+0000","last_change":"2024-03-04T07:26:53.309927+0000","last_active":"2024-03-04T07:26:54.389676+0000","last_peered":"2024-03-04T07:26:54.389676+0000","last_clean":"2024-03-04T07:26:54.389676+0000","last_became_active":"2024-03-04T07:26:53.309679+0000","last_became_peered":"2024-03-04T07:26:53.309679+0000","last_unstale":"2024-03-04T07:26:54.389676+0000","last_undegraded":"2024-03-04T07:26:54.389676+0000","last_fullsized":"2024-03-04T07:26:54.389676+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390148+0000","last_change":"2024-03-04T07:26:52.319140+0000","last_active":"2024-03-04T07:26:54.390148+0000","last_peered":"2024-03-04T07:26:54.390148+0000","last_clean":"2024-03-04T07:26:54.390148+0000","last_became_active":"2024-03-04T07:26:52.318861+0000","last_became_peered":"2024-03-04T07:26:52.318861+0000","last_unstale":"2024-03-04T07:26:54.390148+0000","last_undegraded":"2024-03-04T07:26:54.390148+0000","last_fullsized":"2024-03-04T07:26:54.390148+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:03.812984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305804+0000","last_change":"2024-03-04T07:26:53.317717+0000","last_active":"2024-03-04T07:26:54.305804+0000","last_peered":"2024-03-04T07:26:54.305804+0000","last_clean":"2024-03-04T07:26:54.305804+0000","last_became_active":"2024-03-04T07:26:53.317374+0000","last_became_peered":"2024-03-04T07:26:53.317374+0000","last_unstale":"2024-03-04T07:26:54.305804+0000","last_undegraded":"2024-03-04T07:26:54.305804+0000","last_fullsized":"2024-03-04T07:26:54.305804+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306441+0000","last_change":"2024-03-04T07:26:52.318464+0000","last_active":"2024-03-04T07:26:54.306441+0000","last_peered":"2024-03-04T07:26:54.306441+0000","last_clean":"2024-03-04T07:26:54.306441+0000","last_became_active":"2024-03-04T07:26:52.318141+0000","last_became_peered":"2024-03-04T07:26:52.318141+0000","last_unstale":"2024-03-04T07:26:54.306441+0000","last_undegraded":"2024-03-04T07:26:54.306441+0000","last_fullsized":"2024-03-04T07:26:54.306441+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306253+0000","last_change":"2024-03-04T07:26:53.313219+0000","last_active":"2024-03-04T07:26:54.306253+0000","last_peered":"2024-03-04T07:26:54.306253+0000","last_clean":"2024-03-04T07:26:54.306253+0000","last_became_active":"2024-03-04T07:26:53.312894+0000","last_became_peered":"2024-03-04T07:26:53.312894+0000","last_unstale":"2024-03-04T07:26:54.306253+0000","last_undegraded":"2024-03-04T07:26:54.306253+0000","last_fullsized":"2024-03-04T07:26:54.306253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:52.315301+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:52.315023+0000","last_became_peered":"2024-03-04T07:26:52.315023+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306519+0000","last_change":"2024-03-04T07:26:53.324968+0000","last_active":"2024-03-04T07:26:54.306519+0000","last_peered":"2024-03-04T07:26:54.306519+0000","last_clean":"2024-03-04T07:26:54.306519+0000","last_became_active":"2024-03-04T07:26:53.324658+0000","last_became_peered":"2024-03-04T07:26:53.324658+0000","last_unstale":"2024-03-04T07:26:54.306519+0000","last_undegraded":"2024-03-04T07:26:54.306519+0000","last_fullsized":"2024-03-04T07:26:54.306519+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390203+0000","last_change":"2024-03-04T07:26:52.325400+0000","last_active":"2024-03-04T07:26:54.390203+0000","last_peered":"2024-03-04T07:26:54.390203+0000","last_clean":"2024-03-04T07:26:54.390203+0000","last_became_active":"2024-03-04T07:26:52.325067+0000","last_became_peered":"2024-03-04T07:26:52.325067+0000","last_unstale":"2024-03-04T07:26:54.390203+0000","last_undegraded":"2024-03-04T07:26:54.390203+0000","last_fullsized":"2024-03-04T07:26:54.390203+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305544+0000","last_change":"2024-03-04T07:26:53.317693+0000","last_active":"2024-03-04T07:26:54.305544+0000","last_peered":"2024-03-04T07:26:54.305544+0000","last_clean":"2024-03-04T07:26:54.305544+0000","last_became_active":"2024-03-04T07:26:53.317394+0000","last_became_peered":"2024-03-04T07:26:53.317394+0000","last_unstale":"2024-03-04T07:26:54.305544+0000","last_undegraded":"2024-03-04T07:26:54.305544+0000","last_fullsized":"2024-03-04T07:26:54.305544+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"20'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324149+0000","last_change":"2024-03-04T07:26:52.316749+0000","last_active":"2024-03-04T07:26:54.324149+0000","last_peered":"2024-03-04T07:26:54.324149+0000","last_clean":"2024-03-04T07:26:54.324149+0000","last_became_active":"2024-03-04T07:26:52.316450+0000","last_became_peered":"2024-03-04T07:26:52.316450+0000","last_unstale":"2024-03-04T07:26:54.324149+0000","last_undegraded":"2024-03-04T07:26:54.324149+0000","last_fullsized":"2024-03-04T07:26:54.324149+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.331415+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.331128+0000","last_became_peered":"2024-03-04T07:26:53.331128+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324940+0000","last_change":"2024-03-04T07:26:52.325268+0000","last_active":"2024-03-04T07:26:54.324940+0000","last_peered":"2024-03-04T07:26:54.324940+0000","last_clean":"2024-03-04T07:26:54.324940+0000","last_became_active":"2024-03-04T07:26:52.324890+0000","last_became_peered":"2024-03-04T07:26:52.324890+0000","last_unstale":"2024-03-04T07:26:54.324940+0000","last_undegraded":"2024-03-04T07:26:54.324940+0000","last_fullsized":"2024-03-04T07:26:54.324940+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390532+0000","last_change":"2024-03-04T07:26:53.323671+0000","last_active":"2024-03-04T07:26:54.390532+0000","last_peered":"2024-03-04T07:26:54.390532+0000","last_clean":"2024-03-04T07:26:54.390532+0000","last_became_active":"2024-03-04T07:26:53.323120+0000","last_became_peered":"2024-03-04T07:26:53.323120+0000","last_unstale":"2024-03-04T07:26:54.390532+0000","last_undegraded":"2024-03-04T07:26:54.390532+0000","last_fullsized":"2024-03-04T07:26:54.390532+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389452+0000","last_change":"2024-03-04T07:26:52.324959+0000","last_active":"2024-03-04T07:26:54.389452+0000","last_peered":"2024-03-04T07:26:54.389452+0000","last_clean":"2024-03-04T07:26:54.389452+0000","last_became_active":"2024-03-04T07:26:52.324641+0000","last_became_peered":"2024-03-04T07:26:52.324641+0000","last_unstale":"2024-03-04T07:26:54.389452+0000","last_undegraded":"2024-03-04T07:26:54.389452+0000","last_fullsized":"2024-03-04T07:26:54.389452+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:22.417961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305752+0000","last_change":"2024-03-04T07:26:53.318712+0000","last_active":"2024-03-04T07:26:54.305752+0000","last_peered":"2024-03-04T07:26:54.305752+0000","last_clean":"2024-03-04T07:26:54.305752+0000","last_became_active":"2024-03-04T07:26:53.317676+0000","last_became_peered":"2024-03-04T07:26:53.317676+0000","last_unstale":"2024-03-04T07:26:54.305752+0000","last_undegraded":"2024-03-04T07:26:54.305752+0000","last_fullsized":"2024-03-04T07:26:54.305752+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305560+0000","last_change":"2024-03-04T07:26:52.324864+0000","last_active":"2024-03-04T07:26:54.305560+0000","last_peered":"2024-03-04T07:26:54.305560+0000","last_clean":"2024-03-04T07:26:54.305560+0000","last_became_active":"2024-03-04T07:26:52.324553+0000","last_became_peered":"2024-03-04T07:26:52.324553+0000","last_unstale":"2024-03-04T07:26:54.305560+0000","last_undegraded":"2024-03-04T07:26:54.305560+0000","last_fullsized":"2024-03-04T07:26:54.305560+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390011+0000","last_change":"2024-03-04T07:26:53.320516+0000","last_active":"2024-03-04T07:26:54.390011+0000","last_peered":"2024-03-04T07:26:54.390011+0000","last_clean":"2024-03-04T07:26:54.390011+0000","last_became_active":"2024-03-04T07:26:53.320216+0000","last_became_peered":"2024-03-04T07:26:53.320216+0000","last_unstale":"2024-03-04T07:26:54.390011+0000","last_undegraded":"2024-03-04T07:26:54.390011+0000","last_fullsized":"2024-03-04T07:26:54.390011+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.318968+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.318681+0000","last_became_peered":"2024-03-04T07:26:52.318681+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390005+0000","last_change":"2024-03-04T07:26:53.322137+0000","last_active":"2024-03-04T07:26:54.390005+0000","last_peered":"2024-03-04T07:26:54.390005+0000","last_clean":"2024-03-04T07:26:54.390005+0000","last_became_active":"2024-03-04T07:26:53.321716+0000","last_became_peered":"2024-03-04T07:26:53.321716+0000","last_unstale":"2024-03-04T07:26:54.390005+0000","last_undegraded":"2024-03-04T07:26:54.390005+0000","last_fullsized":"2024-03-04T07:26:54.390005+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389972+0000","last_change":"2024-03-04T07:26:52.325680+0000","last_active":"2024-03-04T07:26:54.389972+0000","last_peered":"2024-03-04T07:26:54.389972+0000","last_clean":"2024-03-04T07:26:54.389972+0000","last_became_active":"2024-03-04T07:26:52.324823+0000","last_became_peered":"2024-03-04T07:26:52.324823+0000","last_unstale":"2024-03-04T07:26:54.389972+0000","last_undegraded":"2024-03-04T07:26:54.389972+0000","last_fullsized":"2024-03-04T07:26:54.389972+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324811+0000","last_change":"2024-03-04T07:26:53.322865+0000","last_active":"2024-03-04T07:26:54.324811+0000","last_peered":"2024-03-04T07:26:54.324811+0000","last_clean":"2024-03-04T07:26:54.324811+0000","last_became_active":"2024-03-04T07:26:53.322579+0000","last_became_peered":"2024-03-04T07:26:53.322579+0000","last_unstale":"2024-03-04T07:26:54.324811+0000","last_undegraded":"2024-03-04T07:26:54.324811+0000","last_fullsized":"2024-03-04T07:26:54.324811+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306571+0000","last_change":"2024-03-04T07:26:53.322261+0000","last_active":"2024-03-04T07:26:54.306571+0000","last_peered":"2024-03-04T07:26:54.306571+0000","last_clean":"2024-03-04T07:26:54.306571+0000","last_became_active":"2024-03-04T07:26:53.321959+0000","last_became_peered":"2024-03-04T07:26:53.321959+0000","last_unstale":"2024-03-04T07:26:54.306571+0000","last_undegraded":"2024-03-04T07:26:54.306571+0000","last_fullsized":"2024-03-04T07:26:54.306571+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389738+0000","last_change":"2024-03-04T07:26:52.330362+0000","last_active":"2024-03-04T07:26:54.389738+0000","last_peered":"2024-03-04T07:26:54.389738+0000","last_clean":"2024-03-04T07:26:54.389738+0000","last_became_active":"2024-03-04T07:26:52.329955+0000","last_became_peered":"2024-03-04T07:26:52.329955+0000","last_unstale":"2024-03-04T07:26:54.389738+0000","last_undegraded":"2024-03-04T07:26:54.389738+0000","last_fullsized":"2024-03-04T07:26:54.389738+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306535+0000","last_change":"2024-03-04T07:26:53.321372+0000","last_active":"2024-03-04T07:26:54.306535+0000","last_peered":"2024-03-04T07:26:54.306535+0000","last_clean":"2024-03-04T07:26:54.306535+0000","last_became_active":"2024-03-04T07:26:53.321099+0000","last_became_peered":"2024-03-04T07:26:53.321099+0000","last_unstale":"2024-03-04T07:26:54.306535+0000","last_undegraded":"2024-03-04T07:26:54.306535+0000","last_fullsized":"2024-03-04T07:26:54.306535+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390131+0000","last_change":"2024-03-04T07:26:52.327378+0000","last_active":"2024-03-04T07:26:54.390131+0000","last_peered":"2024-03-04T07:26:54.390131+0000","last_clean":"2024-03-04T07:26:54.390131+0000","last_became_active":"2024-03-04T07:26:52.327042+0000","last_became_peered":"2024-03-04T07:26:52.327042+0000","last_unstale":"2024-03-04T07:26:54.390131+0000","last_undegraded":"2024-03-04T07:26:54.390131+0000","last_fullsized":"2024-03-04T07:26:54.390131+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390026+0000","last_change":"2024-03-04T07:26:53.325350+0000","last_active":"2024-03-04T07:26:54.390026+0000","last_peered":"2024-03-04T07:26:54.390026+0000","last_clean":"2024-03-04T07:26:54.390026+0000","last_became_active":"2024-03-04T07:26:53.325079+0000","last_became_peered":"2024-03-04T07:26:53.325079+0000","last_unstale":"2024-03-04T07:26:54.390026+0000","last_undegraded":"2024-03-04T07:26:54.390026+0000","last_fullsized":"2024-03-04T07:26:54.390026+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:30:21.821563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389813+0000","last_change":"2024-03-04T07:26:52.322256+0000","last_active":"2024-03-04T07:26:54.389813+0000","last_peered":"2024-03-04T07:26:54.389813+0000","last_clean":"2024-03-04T07:26:54.389813+0000","last_became_active":"2024-03-04T07:26:52.321972+0000","last_became_peered":"2024-03-04T07:26:52.321972+0000","last_unstale":"2024-03-04T07:26:54.389813+0000","last_undegraded":"2024-03-04T07:26:54.389813+0000","last_fullsized":"2024-03-04T07:26:54.389813+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389568+0000","last_change":"2024-03-04T07:26:53.324396+0000","last_active":"2024-03-04T07:26:54.389568+0000","last_peered":"2024-03-04T07:26:54.389568+0000","last_clean":"2024-03-04T07:26:54.389568+0000","last_became_active":"2024-03-04T07:26:53.320387+0000","last_became_peered":"2024-03-04T07:26:53.320387+0000","last_unstale":"2024-03-04T07:26:54.389568+0000","last_undegraded":"2024-03-04T07:26:54.389568+0000","last_fullsized":"2024-03-04T07:26:54.389568+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"19'4","reported_seq":18,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390337+0000","last_change":"2024-03-04T07:26:52.324964+0000","last_active":"2024-03-04T07:26:54.390337+0000","last_peered":"2024-03-04T07:26:54.390337+0000","last_clean":"2024-03-04T07:26:54.390337+0000","last_became_active":"2024-03-04T07:26:52.324642+0000","last_became_peered":"2024-03-04T07:26:52.324642+0000","last_unstale":"2024-03-04T07:26:54.390337+0000","last_undegraded":"2024-03-04T07:26:54.390337+0000","last_fullsized":"2024-03-04T07:26:54.390337+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:49.281484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574856,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":784,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":802816,"data_stored":661884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574855,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574855,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27228,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344612,"statfs":{"total":96636764160,"available":96608882688,"internally_reserved":0,"allocated":208896,"data_stored":71024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574856,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":71060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574855,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27236,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344604,"statfs":{"total":96636764160,"available":96608874496,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574855,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":204800,"data_stored":70934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574855,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":212992,"data_stored":71094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574855,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":811008,"data_stored":661940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:27:00.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:27:00.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.173+0000 7f6a6fd0f640 1 Processor -- start 2024-03-04T07:27:00.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.173+0000 7f6a6fd0f640 1 -- start start 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6fd0f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6a680b2c50 0x7f6a680d2ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6fd0f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a680ce970 con 0x7f6a68071c50 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6fd0f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a680cef30 con 0x7f6a6815e000 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6fd0f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6a680cece0 con 0x7f6a680b2c50 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6d2aa640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6a68071c50 0x7f6a680d25d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42876/0 (socket says 172.21.15.64:42876) 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6d2aa640 1 -- 172.21.15.64:0/530155669 learned_addr learned my addr 172.21.15.64:0/530155669 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6e2ac640 1 --2- 172.21.15.64:0/530155669 >> v2:172.21.15.66:3300/0 conn(0x7f6a680b2c50 0x7f6a680d2ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6caa9640 1 -- 172.21.15.64:0/530155669 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 351674438 0 0) 0x7f6a680ce970 con 0x7f6a68071c50 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6e2ac640 1 -- 172.21.15.64:0/530155669 >> v1:172.21.15.66:6789/0 conn(0x7f6a6815e000 legacy=0x7f6a68071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6e2ac640 1 -- 172.21.15.64:0/530155669 >> v1:172.21.15.64:6789/0 conn(0x7f6a68071c50 legacy=0x7f6a680d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.174+0000 7f6a6e2ac640 1 -- 172.21.15.64:0/530155669 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a680cf210 con 0x7f6a680b2c50 2024-03-04T07:27:00.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.175+0000 7f6a6e2ac640 1 --2- 172.21.15.64:0/530155669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 0x7f6a680d2ce0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f6a640097a0 tx=0x7f6a6402b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7bdc1fb071da41d0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.175+0000 7f6a6caa9640 1 -- 172.21.15.64:0/530155669 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a64035020 con 0x7f6a680b2c50 2024-03-04T07:27:00.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.175+0000 7f6a6caa9640 1 -- 172.21.15.64:0/530155669 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a64031830 con 0x7f6a680b2c50 2024-03-04T07:27:00.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.175+0000 7f6a6caa9640 1 -- 172.21.15.64:0/530155669 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a64031c00 con 0x7f6a680b2c50 2024-03-04T07:27:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.176+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/530155669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 msgr2=0x7f6a680d2ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.176+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/530155669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 0x7f6a680d2ce0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f6a640097a0 tx=0x7f6a6402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.176+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/530155669 shutdown_connections 2024-03-04T07:27:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.176+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/530155669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 0x7f6a680d2ce0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.176+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/530155669 >> 172.21.15.64:0/530155669 conn(0x7f6a680d3c80 msgr2=0x7f6a680d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.176+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/530155669 shutdown_connections 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.176+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/530155669 wait complete. 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.177+0000 7f6a6fd0f640 1 Processor -- start 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.177+0000 7f6a6fd0f640 1 -- start start 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6fd0f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a68071c50 0x7f6a6814d1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6fd0f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 0x7f6a681e6700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6fd0f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a6815e000 0x7f6a681e6c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6fd0f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a680b3320 con 0x7f6a68071c50 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6fd0f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a680b31a0 con 0x7f6a680b2c50 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6fd0f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a6815e3e0 con 0x7f6a6815e000 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a68071c50 0x7f6a6814d1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a68071c50 0x7f6a6814d1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41074/0 (socket says 172.21.15.64:41074) 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 -- 172.21.15.64:0/2813722808 learned_addr learned my addr 172.21.15.64:0/2813722808 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6d2aa640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 0x7f6a681e6700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6e2ac640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a6815e000 0x7f6a681e6c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 -- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a6815e000 msgr2=0x7f6a681e6c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a6815e000 0x7f6a681e6c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 -- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 msgr2=0x7f6a681e6700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 0x7f6a681e6700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 -- 172.21.15.64:0/2813722808 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a680c81a0 con 0x7f6a68071c50 2024-03-04T07:27:00.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.178+0000 7f6a6daab640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a68071c50 0x7f6a6814d1b0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f6a6000c870 tx=0x7f6a6000cd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.179+0000 7f6a5e7fc640 1 -- 172.21.15.64:0/2813722808 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a60003b70 con 0x7f6a68071c50 2024-03-04T07:27:00.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.179+0000 7f6a5e7fc640 1 -- 172.21.15.64:0/2813722808 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a60003d30 con 0x7f6a68071c50 2024-03-04T07:27:00.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.179+0000 7f6a5e7fc640 1 -- 172.21.15.64:0/2813722808 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a6000aa30 con 0x7f6a68071c50 2024-03-04T07:27:00.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.179+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a680c8480 con 0x7f6a68071c50 2024-03-04T07:27:00.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.179+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a680c8a40 con 0x7f6a68071c50 2024-03-04T07:27:00.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.180+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6a68067ff0 con 0x7f6a68071c50 2024-03-04T07:27:00.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.180+0000 7f6a5e7fc640 1 -- 172.21.15.64:0/2813722808 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6a6000abd0 con 0x7f6a68071c50 2024-03-04T07:27:00.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.180+0000 7f6a5e7fc640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a4c06b4d0 0x7f6a4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.181+0000 7f6a6d2aa640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a4c06b4d0 0x7f6a4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.181+0000 7f6a5e7fc640 1 -- 172.21.15.64:0/2813722808 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6a60089fd0 con 0x7f6a68071c50 2024-03-04T07:27:00.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.181+0000 7f6a6d2aa640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a4c06b4d0 0x7f6a4c06d990 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f6a680d1db0 tx=0x7f6a58007e40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.185+0000 7f6a5e7fc640 1 -- 172.21.15.64:0/2813722808 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6a60058800 con 0x7f6a68071c50 2024-03-04T07:27:00.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.363+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f6a680cfa80 con 0x7f6a4c06b4d0 2024-03-04T07:27:00.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.369+0000 7f6a5e7fc640 1 -- 172.21.15.64:0/2813722808 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251836 (secure 0 0 0) 0x7f6a680cfa80 con 0x7f6a4c06b4d0 2024-03-04T07:27:00.371 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a4c06b4d0 msgr2=0x7f6a4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a4c06b4d0 0x7f6a4c06d990 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f6a680d1db0 tx=0x7f6a58007e40 comp rx=0 tx=0).stop 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a68071c50 msgr2=0x7f6a6814d1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a68071c50 0x7f6a6814d1b0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f6a6000c870 tx=0x7f6a6000cd30 comp rx=0 tx=0).stop 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 shutdown_connections 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a4c06b4d0 0x7f6a4c06d990 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a6815e000 0x7f6a681e6c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a680b2c50 0x7f6a681e6700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 --2- 172.21.15.64:0/2813722808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a68071c50 0x7f6a6814d1b0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 >> 172.21.15.64:0/2813722808 conn(0x7f6a680d3c80 msgr2=0x7f6a6815e770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 shutdown_connections 2024-03-04T07:27:00.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.372+0000 7f6a6fd0f640 1 -- 172.21.15.64:0/2813722808 wait complete. 2024-03-04T07:27:00.382 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":22,"stamp":"2024-03-04T07:26:59.143040+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":190,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219008,"kb_used_data":2824,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754755712,"statfs":{"total":773094113280,"available":772869849088,"internally_reserved":0,"allocated":2891776,"data_stored":1753028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.839614"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389967+0000","last_change":"2024-03-04T07:26:53.331782+0000","last_active":"2024-03-04T07:26:54.389967+0000","last_peered":"2024-03-04T07:26:54.389967+0000","last_clean":"2024-03-04T07:26:54.389967+0000","last_became_active":"2024-03-04T07:26:53.331449+0000","last_became_peered":"2024-03-04T07:26:53.331449+0000","last_unstale":"2024-03-04T07:26:54.389967+0000","last_undegraded":"2024-03-04T07:26:54.389967+0000","last_fullsized":"2024-03-04T07:26:54.389967+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305647+0000","last_change":"2024-03-04T07:26:53.321240+0000","last_active":"2024-03-04T07:26:54.305647+0000","last_peered":"2024-03-04T07:26:54.305647+0000","last_clean":"2024-03-04T07:26:54.305647+0000","last_became_active":"2024-03-04T07:26:53.320945+0000","last_became_peered":"2024-03-04T07:26:53.320945+0000","last_unstale":"2024-03-04T07:26:54.305647+0000","last_undegraded":"2024-03-04T07:26:54.305647+0000","last_fullsized":"2024-03-04T07:26:54.305647+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390253+0000","last_change":"2024-03-04T07:26:53.317216+0000","last_active":"2024-03-04T07:26:54.390253+0000","last_peered":"2024-03-04T07:26:54.390253+0000","last_clean":"2024-03-04T07:26:54.390253+0000","last_became_active":"2024-03-04T07:26:53.316875+0000","last_became_peered":"2024-03-04T07:26:53.316875+0000","last_unstale":"2024-03-04T07:26:54.390253+0000","last_undegraded":"2024-03-04T07:26:54.390253+0000","last_fullsized":"2024-03-04T07:26:54.390253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306389+0000","last_change":"2024-03-04T07:26:53.326427+0000","last_active":"2024-03-04T07:26:54.306389+0000","last_peered":"2024-03-04T07:26:54.306389+0000","last_clean":"2024-03-04T07:26:54.306389+0000","last_became_active":"2024-03-04T07:26:53.326097+0000","last_became_peered":"2024-03-04T07:26:53.326097+0000","last_unstale":"2024-03-04T07:26:54.306389+0000","last_undegraded":"2024-03-04T07:26:54.306389+0000","last_fullsized":"2024-03-04T07:26:54.306389+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389857+0000","last_change":"2024-03-04T07:26:53.334207+0000","last_active":"2024-03-04T07:26:54.389857+0000","last_peered":"2024-03-04T07:26:54.389857+0000","last_clean":"2024-03-04T07:26:54.389857+0000","last_became_active":"2024-03-04T07:26:53.333629+0000","last_became_peered":"2024-03-04T07:26:53.333629+0000","last_unstale":"2024-03-04T07:26:54.389857+0000","last_undegraded":"2024-03-04T07:26:54.389857+0000","last_fullsized":"2024-03-04T07:26:54.389857+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305566+0000","last_change":"2024-03-04T07:26:53.322181+0000","last_active":"2024-03-04T07:26:54.305566+0000","last_peered":"2024-03-04T07:26:54.305566+0000","last_clean":"2024-03-04T07:26:54.305566+0000","last_became_active":"2024-03-04T07:26:53.321619+0000","last_became_peered":"2024-03-04T07:26:53.321619+0000","last_unstale":"2024-03-04T07:26:54.305566+0000","last_undegraded":"2024-03-04T07:26:54.305566+0000","last_fullsized":"2024-03-04T07:26:54.305566+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389617+0000","last_change":"2024-03-04T07:26:53.329485+0000","last_active":"2024-03-04T07:26:54.389617+0000","last_peered":"2024-03-04T07:26:54.389617+0000","last_clean":"2024-03-04T07:26:54.389617+0000","last_became_active":"2024-03-04T07:26:53.329174+0000","last_became_peered":"2024-03-04T07:26:53.329174+0000","last_unstale":"2024-03-04T07:26:54.389617+0000","last_undegraded":"2024-03-04T07:26:54.389617+0000","last_fullsized":"2024-03-04T07:26:54.389617+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389430+0000","last_change":"2024-03-04T07:26:53.327402+0000","last_active":"2024-03-04T07:26:54.389430+0000","last_peered":"2024-03-04T07:26:54.389430+0000","last_clean":"2024-03-04T07:26:54.389430+0000","last_became_active":"2024-03-04T07:26:53.326893+0000","last_became_peered":"2024-03-04T07:26:53.326893+0000","last_unstale":"2024-03-04T07:26:54.389430+0000","last_undegraded":"2024-03-04T07:26:54.389430+0000","last_fullsized":"2024-03-04T07:26:54.389430+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.341109+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.340834+0000","last_became_peered":"2024-03-04T07:26:53.340834+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324909+0000","last_change":"2024-03-04T07:26:53.325503+0000","last_active":"2024-03-04T07:26:54.324909+0000","last_peered":"2024-03-04T07:26:54.324909+0000","last_clean":"2024-03-04T07:26:54.324909+0000","last_became_active":"2024-03-04T07:26:53.325191+0000","last_became_peered":"2024-03-04T07:26:53.325191+0000","last_unstale":"2024-03-04T07:26:54.324909+0000","last_undegraded":"2024-03-04T07:26:54.324909+0000","last_fullsized":"2024-03-04T07:26:54.324909+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324908+0000","last_change":"2024-03-04T07:26:53.327284+0000","last_active":"2024-03-04T07:26:54.324908+0000","last_peered":"2024-03-04T07:26:54.324908+0000","last_clean":"2024-03-04T07:26:54.324908+0000","last_became_active":"2024-03-04T07:26:53.327011+0000","last_became_peered":"2024-03-04T07:26:53.327011+0000","last_unstale":"2024-03-04T07:26:54.324908+0000","last_undegraded":"2024-03-04T07:26:54.324908+0000","last_fullsized":"2024-03-04T07:26:54.324908+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306083+0000","last_change":"2024-03-04T07:26:53.316248+0000","last_active":"2024-03-04T07:26:54.306083+0000","last_peered":"2024-03-04T07:26:54.306083+0000","last_clean":"2024-03-04T07:26:54.306083+0000","last_became_active":"2024-03-04T07:26:53.315903+0000","last_became_peered":"2024-03-04T07:26:53.315903+0000","last_unstale":"2024-03-04T07:26:54.306083+0000","last_undegraded":"2024-03-04T07:26:54.306083+0000","last_fullsized":"2024-03-04T07:26:54.306083+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389868+0000","last_change":"2024-03-04T07:26:53.319814+0000","last_active":"2024-03-04T07:26:54.389868+0000","last_peered":"2024-03-04T07:26:54.389868+0000","last_clean":"2024-03-04T07:26:54.389868+0000","last_became_active":"2024-03-04T07:26:53.319503+0000","last_became_peered":"2024-03-04T07:26:53.319503+0000","last_unstale":"2024-03-04T07:26:54.389868+0000","last_undegraded":"2024-03-04T07:26:54.389868+0000","last_fullsized":"2024-03-04T07:26:54.389868+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389681+0000","last_change":"2024-03-04T07:26:53.318624+0000","last_active":"2024-03-04T07:26:54.389681+0000","last_peered":"2024-03-04T07:26:54.389681+0000","last_clean":"2024-03-04T07:26:54.389681+0000","last_became_active":"2024-03-04T07:26:53.318299+0000","last_became_peered":"2024-03-04T07:26:53.318299+0000","last_unstale":"2024-03-04T07:26:54.389681+0000","last_undegraded":"2024-03-04T07:26:54.389681+0000","last_fullsized":"2024-03-04T07:26:54.389681+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:13:39.358840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305782+0000","last_change":"2024-03-04T07:26:53.319312+0000","last_active":"2024-03-04T07:26:54.305782+0000","last_peered":"2024-03-04T07:26:54.305782+0000","last_clean":"2024-03-04T07:26:54.305782+0000","last_became_active":"2024-03-04T07:26:53.319021+0000","last_became_peered":"2024-03-04T07:26:53.319021+0000","last_unstale":"2024-03-04T07:26:54.305782+0000","last_undegraded":"2024-03-04T07:26:54.305782+0000","last_fullsized":"2024-03-04T07:26:54.305782+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305585+0000","last_change":"2024-03-04T07:26:53.313675+0000","last_active":"2024-03-04T07:26:54.305585+0000","last_peered":"2024-03-04T07:26:54.305585+0000","last_clean":"2024-03-04T07:26:54.305585+0000","last_became_active":"2024-03-04T07:26:53.313325+0000","last_became_peered":"2024-03-04T07:26:53.313325+0000","last_unstale":"2024-03-04T07:26:54.305585+0000","last_undegraded":"2024-03-04T07:26:54.305585+0000","last_fullsized":"2024-03-04T07:26:54.305585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306601+0000","last_change":"2024-03-04T07:26:53.335375+0000","last_active":"2024-03-04T07:26:54.306601+0000","last_peered":"2024-03-04T07:26:54.306601+0000","last_clean":"2024-03-04T07:26:54.306601+0000","last_became_active":"2024-03-04T07:26:53.335101+0000","last_became_peered":"2024-03-04T07:26:53.335101+0000","last_unstale":"2024-03-04T07:26:54.306601+0000","last_undegraded":"2024-03-04T07:26:54.306601+0000","last_fullsized":"2024-03-04T07:26:54.306601+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390303+0000","last_change":"2024-03-04T07:26:53.320188+0000","last_active":"2024-03-04T07:26:54.390303+0000","last_peered":"2024-03-04T07:26:54.390303+0000","last_clean":"2024-03-04T07:26:54.390303+0000","last_became_active":"2024-03-04T07:26:53.319902+0000","last_became_peered":"2024-03-04T07:26:53.319902+0000","last_unstale":"2024-03-04T07:26:54.390303+0000","last_undegraded":"2024-03-04T07:26:54.390303+0000","last_fullsized":"2024-03-04T07:26:54.390303+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:30.037220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390194+0000","last_change":"2024-03-04T07:26:53.334274+0000","last_active":"2024-03-04T07:26:54.390194+0000","last_peered":"2024-03-04T07:26:54.390194+0000","last_clean":"2024-03-04T07:26:54.390194+0000","last_became_active":"2024-03-04T07:26:53.333828+0000","last_became_peered":"2024-03-04T07:26:53.333828+0000","last_unstale":"2024-03-04T07:26:54.390194+0000","last_undegraded":"2024-03-04T07:26:54.390194+0000","last_fullsized":"2024-03-04T07:26:54.390194+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:08:42.738284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325119+0000","last_change":"2024-03-04T07:26:53.322810+0000","last_active":"2024-03-04T07:26:54.325119+0000","last_peered":"2024-03-04T07:26:54.325119+0000","last_clean":"2024-03-04T07:26:54.325119+0000","last_became_active":"2024-03-04T07:26:53.322527+0000","last_became_peered":"2024-03-04T07:26:53.322527+0000","last_unstale":"2024-03-04T07:26:54.325119+0000","last_undegraded":"2024-03-04T07:26:54.325119+0000","last_fullsized":"2024-03-04T07:26:54.325119+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390380+0000","last_change":"2024-03-04T07:26:53.326816+0000","last_active":"2024-03-04T07:26:54.390380+0000","last_peered":"2024-03-04T07:26:54.390380+0000","last_clean":"2024-03-04T07:26:54.390380+0000","last_became_active":"2024-03-04T07:26:53.326028+0000","last_became_peered":"2024-03-04T07:26:53.326028+0000","last_unstale":"2024-03-04T07:26:54.390380+0000","last_undegraded":"2024-03-04T07:26:54.390380+0000","last_fullsized":"2024-03-04T07:26:54.390380+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306215+0000","last_change":"2024-03-04T07:26:53.328614+0000","last_active":"2024-03-04T07:26:54.306215+0000","last_peered":"2024-03-04T07:26:54.306215+0000","last_clean":"2024-03-04T07:26:54.306215+0000","last_became_active":"2024-03-04T07:26:53.328262+0000","last_became_peered":"2024-03-04T07:26:53.328262+0000","last_unstale":"2024-03-04T07:26:54.306215+0000","last_undegraded":"2024-03-04T07:26:54.306215+0000","last_fullsized":"2024-03-04T07:26:54.306215+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390277+0000","last_change":"2024-03-04T07:26:53.322798+0000","last_active":"2024-03-04T07:26:54.390277+0000","last_peered":"2024-03-04T07:26:54.390277+0000","last_clean":"2024-03-04T07:26:54.390277+0000","last_became_active":"2024-03-04T07:26:53.321992+0000","last_became_peered":"2024-03-04T07:26:53.321992+0000","last_unstale":"2024-03-04T07:26:54.390277+0000","last_undegraded":"2024-03-04T07:26:54.390277+0000","last_fullsized":"2024-03-04T07:26:54.390277+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305730+0000","last_change":"2024-03-04T07:26:53.323813+0000","last_active":"2024-03-04T07:26:54.305730+0000","last_peered":"2024-03-04T07:26:54.305730+0000","last_clean":"2024-03-04T07:26:54.305730+0000","last_became_active":"2024-03-04T07:26:53.323530+0000","last_became_peered":"2024-03-04T07:26:53.323530+0000","last_unstale":"2024-03-04T07:26:54.305730+0000","last_undegraded":"2024-03-04T07:26:54.305730+0000","last_fullsized":"2024-03-04T07:26:54.305730+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390125+0000","last_change":"2024-03-04T07:26:53.322537+0000","last_active":"2024-03-04T07:26:54.390125+0000","last_peered":"2024-03-04T07:26:54.390125+0000","last_clean":"2024-03-04T07:26:54.390125+0000","last_became_active":"2024-03-04T07:26:53.322254+0000","last_became_peered":"2024-03-04T07:26:53.322254+0000","last_unstale":"2024-03-04T07:26:54.390125+0000","last_undegraded":"2024-03-04T07:26:54.390125+0000","last_fullsized":"2024-03-04T07:26:54.390125+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325071+0000","last_change":"2024-03-04T07:26:53.332147+0000","last_active":"2024-03-04T07:26:54.325071+0000","last_peered":"2024-03-04T07:26:54.325071+0000","last_clean":"2024-03-04T07:26:54.325071+0000","last_became_active":"2024-03-04T07:26:53.331346+0000","last_became_peered":"2024-03-04T07:26:53.331346+0000","last_unstale":"2024-03-04T07:26:54.325071+0000","last_undegraded":"2024-03-04T07:26:54.325071+0000","last_fullsized":"2024-03-04T07:26:54.325071+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.520316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305687+0000","last_change":"2024-03-04T07:26:53.325767+0000","last_active":"2024-03-04T07:26:54.305687+0000","last_peered":"2024-03-04T07:26:54.305687+0000","last_clean":"2024-03-04T07:26:54.305687+0000","last_became_active":"2024-03-04T07:26:53.325486+0000","last_became_peered":"2024-03-04T07:26:53.325486+0000","last_unstale":"2024-03-04T07:26:54.305687+0000","last_undegraded":"2024-03-04T07:26:54.305687+0000","last_fullsized":"2024-03-04T07:26:54.305687+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389819+0000","last_change":"2024-03-04T07:26:53.323242+0000","last_active":"2024-03-04T07:26:54.389819+0000","last_peered":"2024-03-04T07:26:54.389819+0000","last_clean":"2024-03-04T07:26:54.389819+0000","last_became_active":"2024-03-04T07:26:53.322945+0000","last_became_peered":"2024-03-04T07:26:53.322945+0000","last_unstale":"2024-03-04T07:26:54.389819+0000","last_undegraded":"2024-03-04T07:26:54.389819+0000","last_fullsized":"2024-03-04T07:26:54.389819+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390452+0000","last_change":"2024-03-04T07:26:53.321011+0000","last_active":"2024-03-04T07:26:54.390452+0000","last_peered":"2024-03-04T07:26:54.390452+0000","last_clean":"2024-03-04T07:26:54.390452+0000","last_became_active":"2024-03-04T07:26:53.320663+0000","last_became_peered":"2024-03-04T07:26:53.320663+0000","last_unstale":"2024-03-04T07:26:54.390452+0000","last_undegraded":"2024-03-04T07:26:54.390452+0000","last_fullsized":"2024-03-04T07:26:54.390452+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306658+0000","last_change":"2024-03-04T07:26:53.321889+0000","last_active":"2024-03-04T07:26:54.306658+0000","last_peered":"2024-03-04T07:26:54.306658+0000","last_clean":"2024-03-04T07:26:54.306658+0000","last_became_active":"2024-03-04T07:26:53.321304+0000","last_became_peered":"2024-03-04T07:26:53.321304+0000","last_unstale":"2024-03-04T07:26:54.306658+0000","last_undegraded":"2024-03-04T07:26:54.306658+0000","last_fullsized":"2024-03-04T07:26:54.306658+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390060+0000","last_change":"2024-03-04T07:26:53.323893+0000","last_active":"2024-03-04T07:26:54.390060+0000","last_peered":"2024-03-04T07:26:54.390060+0000","last_clean":"2024-03-04T07:26:54.390060+0000","last_became_active":"2024-03-04T07:26:53.323608+0000","last_became_peered":"2024-03-04T07:26:53.323608+0000","last_unstale":"2024-03-04T07:26:54.390060+0000","last_undegraded":"2024-03-04T07:26:54.390060+0000","last_fullsized":"2024-03-04T07:26:54.390060+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324678+0000","last_change":"2024-03-04T07:26:53.318669+0000","last_active":"2024-03-04T07:26:54.324678+0000","last_peered":"2024-03-04T07:26:54.324678+0000","last_clean":"2024-03-04T07:26:54.324678+0000","last_became_active":"2024-03-04T07:26:53.318412+0000","last_became_peered":"2024-03-04T07:26:53.318412+0000","last_unstale":"2024-03-04T07:26:54.324678+0000","last_undegraded":"2024-03-04T07:26:54.324678+0000","last_fullsized":"2024-03-04T07:26:54.324678+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:53.317343+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:53.317041+0000","last_became_peered":"2024-03-04T07:26:53.317041+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:05.230078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390061+0000","last_change":"2024-03-04T07:26:52.326593+0000","last_active":"2024-03-04T07:26:54.390061+0000","last_peered":"2024-03-04T07:26:54.390061+0000","last_clean":"2024-03-04T07:26:54.390061+0000","last_became_active":"2024-03-04T07:26:52.326321+0000","last_became_peered":"2024-03-04T07:26:52.326321+0000","last_unstale":"2024-03-04T07:26:54.390061+0000","last_undegraded":"2024-03-04T07:26:54.390061+0000","last_fullsized":"2024-03-04T07:26:54.390061+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390516+0000","last_change":"2024-03-04T07:26:53.317929+0000","last_active":"2024-03-04T07:26:54.390516+0000","last_peered":"2024-03-04T07:26:54.390516+0000","last_clean":"2024-03-04T07:26:54.390516+0000","last_became_active":"2024-03-04T07:26:53.317631+0000","last_became_peered":"2024-03-04T07:26:53.317631+0000","last_unstale":"2024-03-04T07:26:54.390516+0000","last_undegraded":"2024-03-04T07:26:54.390516+0000","last_fullsized":"2024-03-04T07:26:54.390516+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324662+0000","last_change":"2024-03-04T07:26:52.321659+0000","last_active":"2024-03-04T07:26:54.324662+0000","last_peered":"2024-03-04T07:26:54.324662+0000","last_clean":"2024-03-04T07:26:54.324662+0000","last_became_active":"2024-03-04T07:26:52.321247+0000","last_became_peered":"2024-03-04T07:26:52.321247+0000","last_unstale":"2024-03-04T07:26:54.324662+0000","last_undegraded":"2024-03-04T07:26:54.324662+0000","last_fullsized":"2024-03-04T07:26:54.324662+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390279+0000","last_change":"2024-03-04T07:26:52.316474+0000","last_active":"2024-03-04T07:26:54.390279+0000","last_peered":"2024-03-04T07:26:54.390279+0000","last_clean":"2024-03-04T07:26:54.390279+0000","last_became_active":"2024-03-04T07:26:52.316148+0000","last_became_peered":"2024-03-04T07:26:52.316148+0000","last_unstale":"2024-03-04T07:26:54.390279+0000","last_undegraded":"2024-03-04T07:26:54.390279+0000","last_fullsized":"2024-03-04T07:26:54.390279+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389444+0000","last_change":"2024-03-04T07:26:53.325862+0000","last_active":"2024-03-04T07:26:54.389444+0000","last_peered":"2024-03-04T07:26:54.389444+0000","last_clean":"2024-03-04T07:26:54.389444+0000","last_became_active":"2024-03-04T07:26:53.325450+0000","last_became_peered":"2024-03-04T07:26:53.325450+0000","last_unstale":"2024-03-04T07:26:54.389444+0000","last_undegraded":"2024-03-04T07:26:54.389444+0000","last_fullsized":"2024-03-04T07:26:54.389444+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305831+0000","last_change":"2024-03-04T07:26:52.312838+0000","last_active":"2024-03-04T07:26:54.305831+0000","last_peered":"2024-03-04T07:26:54.305831+0000","last_clean":"2024-03-04T07:26:54.305831+0000","last_became_active":"2024-03-04T07:26:52.312335+0000","last_became_peered":"2024-03-04T07:26:52.312335+0000","last_unstale":"2024-03-04T07:26:54.305831+0000","last_undegraded":"2024-03-04T07:26:54.305831+0000","last_fullsized":"2024-03-04T07:26:54.305831+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305454+0000","last_change":"2024-03-04T07:26:53.324974+0000","last_active":"2024-03-04T07:26:54.305454+0000","last_peered":"2024-03-04T07:26:54.305454+0000","last_clean":"2024-03-04T07:26:54.305454+0000","last_became_active":"2024-03-04T07:26:53.324662+0000","last_became_peered":"2024-03-04T07:26:53.324662+0000","last_unstale":"2024-03-04T07:26:54.305454+0000","last_undegraded":"2024-03-04T07:26:54.305454+0000","last_fullsized":"2024-03-04T07:26:54.305454+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:19.692690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324840+0000","last_change":"2024-03-04T07:26:52.317328+0000","last_active":"2024-03-04T07:26:54.324840+0000","last_peered":"2024-03-04T07:26:54.324840+0000","last_clean":"2024-03-04T07:26:54.324840+0000","last_became_active":"2024-03-04T07:26:52.317039+0000","last_became_peered":"2024-03-04T07:26:52.317039+0000","last_unstale":"2024-03-04T07:26:54.324840+0000","last_undegraded":"2024-03-04T07:26:54.324840+0000","last_fullsized":"2024-03-04T07:26:54.324840+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306365+0000","last_change":"2024-03-04T07:26:53.329677+0000","last_active":"2024-03-04T07:26:54.306365+0000","last_peered":"2024-03-04T07:26:54.306365+0000","last_clean":"2024-03-04T07:26:54.306365+0000","last_became_active":"2024-03-04T07:26:53.327273+0000","last_became_peered":"2024-03-04T07:26:53.327273+0000","last_unstale":"2024-03-04T07:26:54.306365+0000","last_undegraded":"2024-03-04T07:26:54.306365+0000","last_fullsized":"2024-03-04T07:26:54.306365+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306307+0000","last_change":"2024-03-04T07:26:52.317388+0000","last_active":"2024-03-04T07:26:54.306307+0000","last_peered":"2024-03-04T07:26:54.306307+0000","last_clean":"2024-03-04T07:26:54.306307+0000","last_became_active":"2024-03-04T07:26:52.316946+0000","last_became_peered":"2024-03-04T07:26:52.316946+0000","last_unstale":"2024-03-04T07:26:54.306307+0000","last_undegraded":"2024-03-04T07:26:54.306307+0000","last_fullsized":"2024-03-04T07:26:54.306307+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390169+0000","last_change":"2024-03-04T07:26:53.324201+0000","last_active":"2024-03-04T07:26:54.390169+0000","last_peered":"2024-03-04T07:26:54.390169+0000","last_clean":"2024-03-04T07:26:54.390169+0000","last_became_active":"2024-03-04T07:26:53.323923+0000","last_became_peered":"2024-03-04T07:26:53.323923+0000","last_unstale":"2024-03-04T07:26:54.390169+0000","last_undegraded":"2024-03-04T07:26:54.390169+0000","last_fullsized":"2024-03-04T07:26:54.390169+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:50.898693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"19'3","reported_seq":17,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.317701+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.317411+0000","last_became_peered":"2024-03-04T07:26:52.317411+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306245+0000","last_change":"2024-03-04T07:26:53.334757+0000","last_active":"2024-03-04T07:26:54.306245+0000","last_peered":"2024-03-04T07:26:54.306245+0000","last_clean":"2024-03-04T07:26:54.306245+0000","last_became_active":"2024-03-04T07:26:53.331793+0000","last_became_peered":"2024-03-04T07:26:53.331793+0000","last_unstale":"2024-03-04T07:26:54.306245+0000","last_undegraded":"2024-03-04T07:26:54.306245+0000","last_fullsized":"2024-03-04T07:26:54.306245+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:23.419725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"19'6","reported_seq":20,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.325093+0000","last_change":"2024-03-04T07:26:52.318084+0000","last_active":"2024-03-04T07:26:54.325093+0000","last_peered":"2024-03-04T07:26:54.325093+0000","last_clean":"2024-03-04T07:26:54.325093+0000","last_became_active":"2024-03-04T07:26:52.317701+0000","last_became_peered":"2024-03-04T07:26:52.317701+0000","last_unstale":"2024-03-04T07:26:54.325093+0000","last_undegraded":"2024-03-04T07:26:54.325093+0000","last_fullsized":"2024-03-04T07:26:54.325093+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390023+0000","last_change":"2024-03-04T07:26:53.331720+0000","last_active":"2024-03-04T07:26:54.390023+0000","last_peered":"2024-03-04T07:26:54.390023+0000","last_clean":"2024-03-04T07:26:54.390023+0000","last_became_active":"2024-03-04T07:26:53.331397+0000","last_became_peered":"2024-03-04T07:26:53.331397+0000","last_unstale":"2024-03-04T07:26:54.390023+0000","last_undegraded":"2024-03-04T07:26:54.390023+0000","last_fullsized":"2024-03-04T07:26:54.390023+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:23.820563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390546+0000","last_change":"2024-03-04T07:26:52.319865+0000","last_active":"2024-03-04T07:26:54.390546+0000","last_peered":"2024-03-04T07:26:54.390546+0000","last_clean":"2024-03-04T07:26:54.390546+0000","last_became_active":"2024-03-04T07:26:52.319514+0000","last_became_peered":"2024-03-04T07:26:52.319514+0000","last_unstale":"2024-03-04T07:26:54.390546+0000","last_undegraded":"2024-03-04T07:26:54.390546+0000","last_fullsized":"2024-03-04T07:26:54.390546+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389831+0000","last_change":"2024-03-04T07:26:53.327475+0000","last_active":"2024-03-04T07:26:54.389831+0000","last_peered":"2024-03-04T07:26:54.389831+0000","last_clean":"2024-03-04T07:26:54.389831+0000","last_became_active":"2024-03-04T07:26:53.325978+0000","last_became_peered":"2024-03-04T07:26:53.325978+0000","last_unstale":"2024-03-04T07:26:54.389831+0000","last_undegraded":"2024-03-04T07:26:54.389831+0000","last_fullsized":"2024-03-04T07:26:54.389831+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:14:35.134882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306345+0000","last_change":"2024-03-04T07:26:52.322481+0000","last_active":"2024-03-04T07:26:54.306345+0000","last_peered":"2024-03-04T07:26:54.306345+0000","last_clean":"2024-03-04T07:26:54.306345+0000","last_became_active":"2024-03-04T07:26:52.322193+0000","last_became_peered":"2024-03-04T07:26:52.322193+0000","last_unstale":"2024-03-04T07:26:54.306345+0000","last_undegraded":"2024-03-04T07:26:54.306345+0000","last_fullsized":"2024-03-04T07:26:54.306345+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389953+0000","last_change":"2024-03-04T07:26:53.339564+0000","last_active":"2024-03-04T07:26:54.389953+0000","last_peered":"2024-03-04T07:26:54.389953+0000","last_clean":"2024-03-04T07:26:54.389953+0000","last_became_active":"2024-03-04T07:26:53.338911+0000","last_became_peered":"2024-03-04T07:26:53.338911+0000","last_unstale":"2024-03-04T07:26:54.389953+0000","last_undegraded":"2024-03-04T07:26:54.389953+0000","last_fullsized":"2024-03-04T07:26:54.389953+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324657+0000","last_change":"2024-03-04T07:26:52.321570+0000","last_active":"2024-03-04T07:26:54.324657+0000","last_peered":"2024-03-04T07:26:54.324657+0000","last_clean":"2024-03-04T07:26:54.324657+0000","last_became_active":"2024-03-04T07:26:52.321193+0000","last_became_peered":"2024-03-04T07:26:52.321193+0000","last_unstale":"2024-03-04T07:26:54.324657+0000","last_undegraded":"2024-03-04T07:26:54.324657+0000","last_fullsized":"2024-03-04T07:26:54.324657+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324683+0000","last_change":"2024-03-04T07:26:53.318654+0000","last_active":"2024-03-04T07:26:54.324683+0000","last_peered":"2024-03-04T07:26:54.324683+0000","last_clean":"2024-03-04T07:26:54.324683+0000","last_became_active":"2024-03-04T07:26:53.318357+0000","last_became_peered":"2024-03-04T07:26:53.318357+0000","last_unstale":"2024-03-04T07:26:54.324683+0000","last_undegraded":"2024-03-04T07:26:54.324683+0000","last_fullsized":"2024-03-04T07:26:54.324683+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:14.945807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"19'8","reported_seq":22,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390444+0000","last_change":"2024-03-04T07:26:52.317772+0000","last_active":"2024-03-04T07:26:54.390444+0000","last_peered":"2024-03-04T07:26:54.390444+0000","last_clean":"2024-03-04T07:26:54.390444+0000","last_became_active":"2024-03-04T07:26:52.316794+0000","last_became_peered":"2024-03-04T07:26:52.316794+0000","last_unstale":"2024-03-04T07:26:54.390444+0000","last_undegraded":"2024-03-04T07:26:54.390444+0000","last_fullsized":"2024-03-04T07:26:54.390444+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389575+0000","last_change":"2024-03-04T07:26:53.340074+0000","last_active":"2024-03-04T07:26:54.389575+0000","last_peered":"2024-03-04T07:26:54.389575+0000","last_clean":"2024-03-04T07:26:54.389575+0000","last_became_active":"2024-03-04T07:26:53.339265+0000","last_became_peered":"2024-03-04T07:26:53.339265+0000","last_unstale":"2024-03-04T07:26:54.389575+0000","last_undegraded":"2024-03-04T07:26:54.389575+0000","last_fullsized":"2024-03-04T07:26:54.389575+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306140+0000","last_change":"2024-03-04T07:26:52.313758+0000","last_active":"2024-03-04T07:26:54.306140+0000","last_peered":"2024-03-04T07:26:54.306140+0000","last_clean":"2024-03-04T07:26:54.306140+0000","last_became_active":"2024-03-04T07:26:52.313362+0000","last_became_peered":"2024-03-04T07:26:52.313362+0000","last_unstale":"2024-03-04T07:26:54.306140+0000","last_undegraded":"2024-03-04T07:26:54.306140+0000","last_fullsized":"2024-03-04T07:26:54.306140+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:47:38.677880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389920+0000","last_change":"2024-03-04T07:26:53.337849+0000","last_active":"2024-03-04T07:26:54.389920+0000","last_peered":"2024-03-04T07:26:54.389920+0000","last_clean":"2024-03-04T07:26:54.389920+0000","last_became_active":"2024-03-04T07:26:53.337529+0000","last_became_peered":"2024-03-04T07:26:53.337529+0000","last_unstale":"2024-03-04T07:26:54.389920+0000","last_undegraded":"2024-03-04T07:26:54.389920+0000","last_fullsized":"2024-03-04T07:26:54.389920+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390527+0000","last_change":"2024-03-04T07:26:52.318490+0000","last_active":"2024-03-04T07:26:54.390527+0000","last_peered":"2024-03-04T07:26:54.390527+0000","last_clean":"2024-03-04T07:26:54.390527+0000","last_became_active":"2024-03-04T07:26:52.318217+0000","last_became_peered":"2024-03-04T07:26:52.318217+0000","last_unstale":"2024-03-04T07:26:54.390527+0000","last_undegraded":"2024-03-04T07:26:54.390527+0000","last_fullsized":"2024-03-04T07:26:54.390527+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390128+0000","last_change":"2024-03-04T07:26:53.338814+0000","last_active":"2024-03-04T07:26:54.390128+0000","last_peered":"2024-03-04T07:26:54.390128+0000","last_clean":"2024-03-04T07:26:54.390128+0000","last_became_active":"2024-03-04T07:26:53.338090+0000","last_became_peered":"2024-03-04T07:26:53.338090+0000","last_unstale":"2024-03-04T07:26:54.390128+0000","last_undegraded":"2024-03-04T07:26:54.390128+0000","last_fullsized":"2024-03-04T07:26:54.390128+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:33.083393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305758+0000","last_change":"2024-03-04T07:26:52.320092+0000","last_active":"2024-03-04T07:26:54.305758+0000","last_peered":"2024-03-04T07:26:54.305758+0000","last_clean":"2024-03-04T07:26:54.305758+0000","last_became_active":"2024-03-04T07:26:52.319835+0000","last_became_peered":"2024-03-04T07:26:52.319835+0000","last_unstale":"2024-03-04T07:26:54.305758+0000","last_undegraded":"2024-03-04T07:26:54.305758+0000","last_fullsized":"2024-03-04T07:26:54.305758+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306585+0000","last_change":"2024-03-04T07:26:53.325995+0000","last_active":"2024-03-04T07:26:54.306585+0000","last_peered":"2024-03-04T07:26:54.306585+0000","last_clean":"2024-03-04T07:26:54.306585+0000","last_became_active":"2024-03-04T07:26:53.325550+0000","last_became_peered":"2024-03-04T07:26:53.325550+0000","last_unstale":"2024-03-04T07:26:54.306585+0000","last_undegraded":"2024-03-04T07:26:54.306585+0000","last_fullsized":"2024-03-04T07:26:54.306585+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:20.565392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389741+0000","last_change":"2024-03-04T07:26:52.318023+0000","last_active":"2024-03-04T07:26:54.389741+0000","last_peered":"2024-03-04T07:26:54.389741+0000","last_clean":"2024-03-04T07:26:54.389741+0000","last_became_active":"2024-03-04T07:26:52.317127+0000","last_became_peered":"2024-03-04T07:26:52.317127+0000","last_unstale":"2024-03-04T07:26:54.389741+0000","last_undegraded":"2024-03-04T07:26:54.389741+0000","last_fullsized":"2024-03-04T07:26:54.389741+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:10.212684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390046+0000","last_change":"2024-03-04T07:26:53.337973+0000","last_active":"2024-03-04T07:26:54.390046+0000","last_peered":"2024-03-04T07:26:54.390046+0000","last_clean":"2024-03-04T07:26:54.390046+0000","last_became_active":"2024-03-04T07:26:53.337551+0000","last_became_peered":"2024-03-04T07:26:53.337551+0000","last_unstale":"2024-03-04T07:26:54.390046+0000","last_undegraded":"2024-03-04T07:26:54.390046+0000","last_fullsized":"2024-03-04T07:26:54.390046+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:59:11.216123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":170,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389506+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:26:54.389506+0000","last_peered":"2024-03-04T07:26:54.389506+0000","last_clean":"2024-03-04T07:26:54.389506+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:26:54.389506+0000","last_undegraded":"2024-03-04T07:26:54.389506+0000","last_fullsized":"2024-03-04T07:26:54.389506+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306129+0000","last_change":"2024-03-04T07:26:52.317678+0000","last_active":"2024-03-04T07:26:54.306129+0000","last_peered":"2024-03-04T07:26:54.306129+0000","last_clean":"2024-03-04T07:26:54.306129+0000","last_became_active":"2024-03-04T07:26:52.317008+0000","last_became_peered":"2024-03-04T07:26:52.317008+0000","last_unstale":"2024-03-04T07:26:54.306129+0000","last_undegraded":"2024-03-04T07:26:54.306129+0000","last_fullsized":"2024-03-04T07:26:54.306129+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:52:45.342736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389685+0000","last_change":"2024-03-04T07:26:53.327257+0000","last_active":"2024-03-04T07:26:54.389685+0000","last_peered":"2024-03-04T07:26:54.389685+0000","last_clean":"2024-03-04T07:26:54.389685+0000","last_became_active":"2024-03-04T07:26:53.326960+0000","last_became_peered":"2024-03-04T07:26:53.326960+0000","last_unstale":"2024-03-04T07:26:54.389685+0000","last_undegraded":"2024-03-04T07:26:54.389685+0000","last_fullsized":"2024-03-04T07:26:54.389685+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389451+0000","last_change":"2024-03-04T07:26:52.325578+0000","last_active":"2024-03-04T07:26:54.389451+0000","last_peered":"2024-03-04T07:26:54.389451+0000","last_clean":"2024-03-04T07:26:54.389451+0000","last_became_active":"2024-03-04T07:26:52.325291+0000","last_became_peered":"2024-03-04T07:26:52.325291+0000","last_unstale":"2024-03-04T07:26:54.389451+0000","last_undegraded":"2024-03-04T07:26:54.389451+0000","last_fullsized":"2024-03-04T07:26:54.389451+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:58.480243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389676+0000","last_change":"2024-03-04T07:26:53.309927+0000","last_active":"2024-03-04T07:26:54.389676+0000","last_peered":"2024-03-04T07:26:54.389676+0000","last_clean":"2024-03-04T07:26:54.389676+0000","last_became_active":"2024-03-04T07:26:53.309679+0000","last_became_peered":"2024-03-04T07:26:53.309679+0000","last_unstale":"2024-03-04T07:26:54.389676+0000","last_undegraded":"2024-03-04T07:26:54.389676+0000","last_fullsized":"2024-03-04T07:26:54.389676+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390148+0000","last_change":"2024-03-04T07:26:52.319140+0000","last_active":"2024-03-04T07:26:54.390148+0000","last_peered":"2024-03-04T07:26:54.390148+0000","last_clean":"2024-03-04T07:26:54.390148+0000","last_became_active":"2024-03-04T07:26:52.318861+0000","last_became_peered":"2024-03-04T07:26:52.318861+0000","last_unstale":"2024-03-04T07:26:54.390148+0000","last_undegraded":"2024-03-04T07:26:54.390148+0000","last_fullsized":"2024-03-04T07:26:54.390148+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:03.812984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305804+0000","last_change":"2024-03-04T07:26:53.317717+0000","last_active":"2024-03-04T07:26:54.305804+0000","last_peered":"2024-03-04T07:26:54.305804+0000","last_clean":"2024-03-04T07:26:54.305804+0000","last_became_active":"2024-03-04T07:26:53.317374+0000","last_became_peered":"2024-03-04T07:26:53.317374+0000","last_unstale":"2024-03-04T07:26:54.305804+0000","last_undegraded":"2024-03-04T07:26:54.305804+0000","last_fullsized":"2024-03-04T07:26:54.305804+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:56.820666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306441+0000","last_change":"2024-03-04T07:26:52.318464+0000","last_active":"2024-03-04T07:26:54.306441+0000","last_peered":"2024-03-04T07:26:54.306441+0000","last_clean":"2024-03-04T07:26:54.306441+0000","last_became_active":"2024-03-04T07:26:52.318141+0000","last_became_peered":"2024-03-04T07:26:52.318141+0000","last_unstale":"2024-03-04T07:26:54.306441+0000","last_undegraded":"2024-03-04T07:26:54.306441+0000","last_fullsized":"2024-03-04T07:26:54.306441+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306253+0000","last_change":"2024-03-04T07:26:53.313219+0000","last_active":"2024-03-04T07:26:54.306253+0000","last_peered":"2024-03-04T07:26:54.306253+0000","last_clean":"2024-03-04T07:26:54.306253+0000","last_became_active":"2024-03-04T07:26:53.312894+0000","last_became_peered":"2024-03-04T07:26:53.312894+0000","last_unstale":"2024-03-04T07:26:54.306253+0000","last_undegraded":"2024-03-04T07:26:54.306253+0000","last_fullsized":"2024-03-04T07:26:54.306253+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:25:59.482007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305506+0000","last_change":"2024-03-04T07:26:52.315301+0000","last_active":"2024-03-04T07:26:54.305506+0000","last_peered":"2024-03-04T07:26:54.305506+0000","last_clean":"2024-03-04T07:26:54.305506+0000","last_became_active":"2024-03-04T07:26:52.315023+0000","last_became_peered":"2024-03-04T07:26:52.315023+0000","last_unstale":"2024-03-04T07:26:54.305506+0000","last_undegraded":"2024-03-04T07:26:54.305506+0000","last_fullsized":"2024-03-04T07:26:54.305506+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:22.889965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306519+0000","last_change":"2024-03-04T07:26:53.324968+0000","last_active":"2024-03-04T07:26:54.306519+0000","last_peered":"2024-03-04T07:26:54.306519+0000","last_clean":"2024-03-04T07:26:54.306519+0000","last_became_active":"2024-03-04T07:26:53.324658+0000","last_became_peered":"2024-03-04T07:26:53.324658+0000","last_unstale":"2024-03-04T07:26:54.306519+0000","last_undegraded":"2024-03-04T07:26:54.306519+0000","last_fullsized":"2024-03-04T07:26:54.306519+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:04.018484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390203+0000","last_change":"2024-03-04T07:26:52.325400+0000","last_active":"2024-03-04T07:26:54.390203+0000","last_peered":"2024-03-04T07:26:54.390203+0000","last_clean":"2024-03-04T07:26:54.390203+0000","last_became_active":"2024-03-04T07:26:52.325067+0000","last_became_peered":"2024-03-04T07:26:52.325067+0000","last_unstale":"2024-03-04T07:26:54.390203+0000","last_undegraded":"2024-03-04T07:26:54.390203+0000","last_fullsized":"2024-03-04T07:26:54.390203+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305544+0000","last_change":"2024-03-04T07:26:53.317693+0000","last_active":"2024-03-04T07:26:54.305544+0000","last_peered":"2024-03-04T07:26:54.305544+0000","last_clean":"2024-03-04T07:26:54.305544+0000","last_became_active":"2024-03-04T07:26:53.317394+0000","last_became_peered":"2024-03-04T07:26:53.317394+0000","last_unstale":"2024-03-04T07:26:54.305544+0000","last_undegraded":"2024-03-04T07:26:54.305544+0000","last_fullsized":"2024-03-04T07:26:54.305544+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:13.263383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"20'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324149+0000","last_change":"2024-03-04T07:26:52.316749+0000","last_active":"2024-03-04T07:26:54.324149+0000","last_peered":"2024-03-04T07:26:54.324149+0000","last_clean":"2024-03-04T07:26:54.324149+0000","last_became_active":"2024-03-04T07:26:52.316450+0000","last_became_peered":"2024-03-04T07:26:52.316450+0000","last_unstale":"2024-03-04T07:26:54.324149+0000","last_undegraded":"2024-03-04T07:26:54.324149+0000","last_fullsized":"2024-03-04T07:26:54.324149+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390022+0000","last_change":"2024-03-04T07:26:53.331415+0000","last_active":"2024-03-04T07:26:54.390022+0000","last_peered":"2024-03-04T07:26:54.390022+0000","last_clean":"2024-03-04T07:26:54.390022+0000","last_became_active":"2024-03-04T07:26:53.331128+0000","last_became_peered":"2024-03-04T07:26:53.331128+0000","last_unstale":"2024-03-04T07:26:54.390022+0000","last_undegraded":"2024-03-04T07:26:54.390022+0000","last_fullsized":"2024-03-04T07:26:54.390022+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:55:48.884048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324940+0000","last_change":"2024-03-04T07:26:52.325268+0000","last_active":"2024-03-04T07:26:54.324940+0000","last_peered":"2024-03-04T07:26:54.324940+0000","last_clean":"2024-03-04T07:26:54.324940+0000","last_became_active":"2024-03-04T07:26:52.324890+0000","last_became_peered":"2024-03-04T07:26:52.324890+0000","last_unstale":"2024-03-04T07:26:54.324940+0000","last_undegraded":"2024-03-04T07:26:54.324940+0000","last_fullsized":"2024-03-04T07:26:54.324940+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390532+0000","last_change":"2024-03-04T07:26:53.323671+0000","last_active":"2024-03-04T07:26:54.390532+0000","last_peered":"2024-03-04T07:26:54.390532+0000","last_clean":"2024-03-04T07:26:54.390532+0000","last_became_active":"2024-03-04T07:26:53.323120+0000","last_became_peered":"2024-03-04T07:26:53.323120+0000","last_unstale":"2024-03-04T07:26:54.390532+0000","last_undegraded":"2024-03-04T07:26:54.390532+0000","last_fullsized":"2024-03-04T07:26:54.390532+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:09:21.082022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389452+0000","last_change":"2024-03-04T07:26:52.324959+0000","last_active":"2024-03-04T07:26:54.389452+0000","last_peered":"2024-03-04T07:26:54.389452+0000","last_clean":"2024-03-04T07:26:54.389452+0000","last_became_active":"2024-03-04T07:26:52.324641+0000","last_became_peered":"2024-03-04T07:26:52.324641+0000","last_unstale":"2024-03-04T07:26:54.389452+0000","last_undegraded":"2024-03-04T07:26:54.389452+0000","last_fullsized":"2024-03-04T07:26:54.389452+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:22.417961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305752+0000","last_change":"2024-03-04T07:26:53.318712+0000","last_active":"2024-03-04T07:26:54.305752+0000","last_peered":"2024-03-04T07:26:54.305752+0000","last_clean":"2024-03-04T07:26:54.305752+0000","last_became_active":"2024-03-04T07:26:53.317676+0000","last_became_peered":"2024-03-04T07:26:53.317676+0000","last_unstale":"2024-03-04T07:26:54.305752+0000","last_undegraded":"2024-03-04T07:26:54.305752+0000","last_fullsized":"2024-03-04T07:26:54.305752+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:57.300981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305560+0000","last_change":"2024-03-04T07:26:52.324864+0000","last_active":"2024-03-04T07:26:54.305560+0000","last_peered":"2024-03-04T07:26:54.305560+0000","last_clean":"2024-03-04T07:26:54.305560+0000","last_became_active":"2024-03-04T07:26:52.324553+0000","last_became_peered":"2024-03-04T07:26:52.324553+0000","last_unstale":"2024-03-04T07:26:54.305560+0000","last_undegraded":"2024-03-04T07:26:54.305560+0000","last_fullsized":"2024-03-04T07:26:54.305560+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390011+0000","last_change":"2024-03-04T07:26:53.320516+0000","last_active":"2024-03-04T07:26:54.390011+0000","last_peered":"2024-03-04T07:26:54.390011+0000","last_clean":"2024-03-04T07:26:54.390011+0000","last_became_active":"2024-03-04T07:26:53.320216+0000","last_became_peered":"2024-03-04T07:26:53.320216+0000","last_unstale":"2024-03-04T07:26:54.390011+0000","last_undegraded":"2024-03-04T07:26:54.390011+0000","last_fullsized":"2024-03-04T07:26:54.390011+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:56:28.489133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.305704+0000","last_change":"2024-03-04T07:26:52.318968+0000","last_active":"2024-03-04T07:26:54.305704+0000","last_peered":"2024-03-04T07:26:54.305704+0000","last_clean":"2024-03-04T07:26:54.305704+0000","last_became_active":"2024-03-04T07:26:52.318681+0000","last_became_peered":"2024-03-04T07:26:52.318681+0000","last_unstale":"2024-03-04T07:26:54.305704+0000","last_undegraded":"2024-03-04T07:26:54.305704+0000","last_fullsized":"2024-03-04T07:26:54.305704+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:36:29.035456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390005+0000","last_change":"2024-03-04T07:26:53.322137+0000","last_active":"2024-03-04T07:26:54.390005+0000","last_peered":"2024-03-04T07:26:54.390005+0000","last_clean":"2024-03-04T07:26:54.390005+0000","last_became_active":"2024-03-04T07:26:53.321716+0000","last_became_peered":"2024-03-04T07:26:53.321716+0000","last_unstale":"2024-03-04T07:26:54.390005+0000","last_undegraded":"2024-03-04T07:26:54.390005+0000","last_fullsized":"2024-03-04T07:26:54.390005+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:37.574862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389972+0000","last_change":"2024-03-04T07:26:52.325680+0000","last_active":"2024-03-04T07:26:54.389972+0000","last_peered":"2024-03-04T07:26:54.389972+0000","last_clean":"2024-03-04T07:26:54.389972+0000","last_became_active":"2024-03-04T07:26:52.324823+0000","last_became_peered":"2024-03-04T07:26:52.324823+0000","last_unstale":"2024-03-04T07:26:54.389972+0000","last_undegraded":"2024-03-04T07:26:54.389972+0000","last_fullsized":"2024-03-04T07:26:54.389972+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:04.228314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.324811+0000","last_change":"2024-03-04T07:26:53.322865+0000","last_active":"2024-03-04T07:26:54.324811+0000","last_peered":"2024-03-04T07:26:54.324811+0000","last_clean":"2024-03-04T07:26:54.324811+0000","last_became_active":"2024-03-04T07:26:53.322579+0000","last_became_peered":"2024-03-04T07:26:53.322579+0000","last_unstale":"2024-03-04T07:26:54.324811+0000","last_undegraded":"2024-03-04T07:26:54.324811+0000","last_fullsized":"2024-03-04T07:26:54.324811+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306571+0000","last_change":"2024-03-04T07:26:53.322261+0000","last_active":"2024-03-04T07:26:54.306571+0000","last_peered":"2024-03-04T07:26:54.306571+0000","last_clean":"2024-03-04T07:26:54.306571+0000","last_became_active":"2024-03-04T07:26:53.321959+0000","last_became_peered":"2024-03-04T07:26:53.321959+0000","last_unstale":"2024-03-04T07:26:54.306571+0000","last_undegraded":"2024-03-04T07:26:54.306571+0000","last_fullsized":"2024-03-04T07:26:54.306571+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:38:23.833207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389738+0000","last_change":"2024-03-04T07:26:52.330362+0000","last_active":"2024-03-04T07:26:54.389738+0000","last_peered":"2024-03-04T07:26:54.389738+0000","last_clean":"2024-03-04T07:26:54.389738+0000","last_became_active":"2024-03-04T07:26:52.329955+0000","last_became_peered":"2024-03-04T07:26:52.329955+0000","last_unstale":"2024-03-04T07:26:54.389738+0000","last_undegraded":"2024-03-04T07:26:54.389738+0000","last_fullsized":"2024-03-04T07:26:54.389738+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:54:14.546431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.306535+0000","last_change":"2024-03-04T07:26:53.321372+0000","last_active":"2024-03-04T07:26:54.306535+0000","last_peered":"2024-03-04T07:26:54.306535+0000","last_clean":"2024-03-04T07:26:54.306535+0000","last_became_active":"2024-03-04T07:26:53.321099+0000","last_became_peered":"2024-03-04T07:26:53.321099+0000","last_unstale":"2024-03-04T07:26:54.306535+0000","last_undegraded":"2024-03-04T07:26:54.306535+0000","last_fullsized":"2024-03-04T07:26:54.306535+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:25.834042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1d","version":"19'2","reported_seq":16,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390131+0000","last_change":"2024-03-04T07:26:52.327378+0000","last_active":"2024-03-04T07:26:54.390131+0000","last_peered":"2024-03-04T07:26:54.390131+0000","last_clean":"2024-03-04T07:26:54.390131+0000","last_became_active":"2024-03-04T07:26:52.327042+0000","last_became_peered":"2024-03-04T07:26:52.327042+0000","last_unstale":"2024-03-04T07:26:54.390131+0000","last_undegraded":"2024-03-04T07:26:54.390131+0000","last_fullsized":"2024-03-04T07:26:54.390131+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:56.299216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390026+0000","last_change":"2024-03-04T07:26:53.325350+0000","last_active":"2024-03-04T07:26:54.390026+0000","last_peered":"2024-03-04T07:26:54.390026+0000","last_clean":"2024-03-04T07:26:54.390026+0000","last_became_active":"2024-03-04T07:26:53.325079+0000","last_became_peered":"2024-03-04T07:26:53.325079+0000","last_unstale":"2024-03-04T07:26:54.390026+0000","last_undegraded":"2024-03-04T07:26:54.390026+0000","last_fullsized":"2024-03-04T07:26:54.390026+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:30:21.821563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"19'1","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389813+0000","last_change":"2024-03-04T07:26:52.322256+0000","last_active":"2024-03-04T07:26:54.389813+0000","last_peered":"2024-03-04T07:26:54.389813+0000","last_clean":"2024-03-04T07:26:54.389813+0000","last_became_active":"2024-03-04T07:26:52.321972+0000","last_became_peered":"2024-03-04T07:26:52.321972+0000","last_unstale":"2024-03-04T07:26:54.389813+0000","last_undegraded":"2024-03-04T07:26:54.389813+0000","last_fullsized":"2024-03-04T07:26:54.389813+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:40:05.595953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.389568+0000","last_change":"2024-03-04T07:26:53.324396+0000","last_active":"2024-03-04T07:26:54.389568+0000","last_peered":"2024-03-04T07:26:54.389568+0000","last_clean":"2024-03-04T07:26:54.389568+0000","last_became_active":"2024-03-04T07:26:53.320387+0000","last_became_peered":"2024-03-04T07:26:53.320387+0000","last_unstale":"2024-03-04T07:26:54.389568+0000","last_undegraded":"2024-03-04T07:26:54.389568+0000","last_fullsized":"2024-03-04T07:26:54.389568+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:52.296728+0000","last_clean_scrub_stamp":"2024-03-04T07:26:52.296728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:10.780686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"19'4","reported_seq":18,"reported_epoch":20,"state":"active+clean","last_fresh":"2024-03-04T07:26:54.390337+0000","last_change":"2024-03-04T07:26:52.324964+0000","last_active":"2024-03-04T07:26:54.390337+0000","last_peered":"2024-03-04T07:26:54.390337+0000","last_clean":"2024-03-04T07:26:54.390337+0000","last_became_active":"2024-03-04T07:26:52.324642+0000","last_became_peered":"2024-03-04T07:26:52.324642+0000","last_unstale":"2024-03-04T07:26:54.390337+0000","last_undegraded":"2024-03-04T07:26:54.390337+0000","last_fullsized":"2024-03-04T07:26:54.390337+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:51.294963+0000","last_clean_scrub_stamp":"2024-03-04T07:26:51.294963+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:49.281484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574856,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":784,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":802816,"data_stored":661884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574855,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574855,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27228,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344612,"statfs":{"total":96636764160,"available":96608882688,"internally_reserved":0,"allocated":208896,"data_stored":71024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574856,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":217088,"data_stored":71060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574855,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27236,"kb_used_data":212,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344604,"statfs":{"total":96636764160,"available":96608874496,"internally_reserved":0,"allocated":217088,"data_stored":72546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574855,"num_pgs":13,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":204800,"data_stored":70934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574855,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":212992,"data_stored":71094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574855,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":811008,"data_stored":661940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:27:00.386 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-03-04T07:27:00.386 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-04T07:27:00.386 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-03-04T07:27:00.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-03-04T07:27:00.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.524+0000 7f3a3a60e640 1 Processor -- start 2024-03-04T07:27:00.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.524+0000 7f3a3a60e640 1 -- start start 2024-03-04T07:27:00.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.524+0000 7f3a3a60e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3a34072660 0x7f3a3415ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.524+0000 7f3a3a60e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3a340b55d0 0x7f3a34071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.524+0000 7f3a3a60e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3a340b4160 con 0x7f3a3406e700 2024-03-04T07:27:00.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.524+0000 7f3a3a60e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a340b5b40 con 0x7f3a34072660 2024-03-04T07:27:00.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.524+0000 7f3a3a60e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a340b3ba0 con 0x7f3a340b55d0 2024-03-04T07:27:00.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a337fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3a34072660 0x7f3a3415ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a33fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3a3406e700 0x7f3a3406d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42882/0 (socket says 172.21.15.64:42882) 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a337fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3a34072660 0x7f3a3415ef80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41076/0 (socket says 172.21.15.64:41076) 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a33fff640 1 -- 172.21.15.64:0/3625857099 learned_addr learned my addr 172.21.15.64:0/3625857099 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a38bab640 1 --2- 172.21.15.64:0/3625857099 >> v2:172.21.15.66:3300/0 conn(0x7f3a340b55d0 0x7f3a34071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a337fe640 1 -- 172.21.15.64:0/3625857099 >> v2:172.21.15.66:3300/0 conn(0x7f3a340b55d0 msgr2=0x7f3a34071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a337fe640 1 --2- 172.21.15.64:0/3625857099 >> v2:172.21.15.66:3300/0 conn(0x7f3a340b55d0 0x7f3a34071980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a337fe640 1 -- 172.21.15.64:0/3625857099 >> v1:172.21.15.64:6789/0 conn(0x7f3a3406e700 legacy=0x7f3a3406d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a32ffd640 1 -- 172.21.15.64:0/3625857099 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1271637330 0 0) 0x7f3a340b4160 con 0x7f3a3406e700 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.525+0000 7f3a337fe640 1 -- 172.21.15.64:0/3625857099 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a340b3e80 con 0x7f3a34072660 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a337fe640 1 --2- 172.21.15.64:0/3625857099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a34072660 0x7f3a3415ef80 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f3a34071580 tx=0x7f3a2802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=23cc46b4e13b75b1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a32ffd640 1 -- 172.21.15.64:0/3625857099 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a28035020 con 0x7f3a34072660 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a32ffd640 1 -- 172.21.15.64:0/3625857099 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a28031830 con 0x7f3a34072660 2024-03-04T07:27:00.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a32ffd640 1 -- 172.21.15.64:0/3625857099 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a28031c00 con 0x7f3a34072660 2024-03-04T07:27:00.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 -- 172.21.15.64:0/3625857099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a34072660 msgr2=0x7f3a3415ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/3625857099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a34072660 0x7f3a3415ef80 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f3a34071580 tx=0x7f3a2802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 -- 172.21.15.64:0/3625857099 shutdown_connections 2024-03-04T07:27:00.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/3625857099 >> v2:172.21.15.66:3300/0 conn(0x7f3a340b55d0 0x7f3a34071980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/3625857099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a34072660 0x7f3a3415ef80 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 -- 172.21.15.64:0/3625857099 >> 172.21.15.64:0/3625857099 conn(0x7f3a340b2720 msgr2=0x7f3a340b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 -- 172.21.15.64:0/3625857099 shutdown_connections 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.526+0000 7f3a3a60e640 1 -- 172.21.15.64:0/3625857099 wait complete. 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.527+0000 7f3a3a60e640 1 Processor -- start 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a3a60e640 1 -- start start 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a3a60e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a3406e700 0x7f3a341f4690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a3a60e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a34072660 0x7f3a340cfe00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a3a60e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a340b55d0 0x7f3a3414e0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a3a60e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a34072040 con 0x7f3a3406e700 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a3a60e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a34071ec0 con 0x7f3a340b55d0 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a3a60e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3a340721c0 con 0x7f3a34072660 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a3406e700 0x7f3a341f4690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a3406e700 0x7f3a341f4690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41080/0 (socket says 172.21.15.64:41080) 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a33fff640 1 -- 172.21.15.64:0/453945723 learned_addr learned my addr 172.21.15.64:0/453945723 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a337fe640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a34072660 0x7f3a340cfe00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.528+0000 7f3a38bab640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a340b55d0 0x7f3a3414e0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a33fff640 1 -- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a34072660 msgr2=0x7f3a340cfe00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a33fff640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a34072660 0x7f3a340cfe00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a33fff640 1 -- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a340b55d0 msgr2=0x7f3a3414e0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a33fff640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a340b55d0 0x7f3a3414e0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a33fff640 1 -- 172.21.15.64:0/453945723 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a3415a000 con 0x7f3a3406e700 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a337fe640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a34072660 0x7f3a340cfe00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a33fff640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a3406e700 0x7f3a341f4690 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f3a2400c500 tx=0x7f3a2400c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a30ff9640 1 -- 172.21.15.64:0/453945723 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a24018070 con 0x7f3a3406e700 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a30ff9640 1 -- 172.21.15.64:0/453945723 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a24007580 con 0x7f3a3406e700 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a34159d20 con 0x7f3a3406e700 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a30ff9640 1 -- 172.21.15.64:0/453945723 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a24004a20 con 0x7f3a3406e700 2024-03-04T07:27:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.529+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a340cd810 con 0x7f3a3406e700 2024-03-04T07:27:00.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.530+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a34067ff0 con 0x7f3a3406e700 2024-03-04T07:27:00.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.531+0000 7f3a30ff9640 1 -- 172.21.15.64:0/453945723 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3a2401b070 con 0x7f3a3406e700 2024-03-04T07:27:00.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.531+0000 7f3a30ff9640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a2006b500 0x7f3a2006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.531+0000 7f3a30ff9640 1 -- 172.21.15.64:0/453945723 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3a240893a0 con 0x7f3a3406e700 2024-03-04T07:27:00.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.531+0000 7f3a337fe640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a2006b500 0x7f3a2006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.531+0000 7f3a337fe640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a2006b500 0x7f3a2006d9c0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f3a28036000 tx=0x7f3a28008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.535+0000 7f3a30ff9640 1 -- 172.21.15.64:0/453945723 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3a24057c60 con 0x7f3a3406e700 2024-03-04T07:27:00.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.754+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) v1 -- 0x7f3a34072c30 con 0x7f3a3406e700 2024-03-04T07:27:00.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.754+0000 7f3a30ff9640 1 -- 172.21.15.64:0/453945723 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) v1 ==== 72+0+46 (secure 0 0 0) 0x7f3a2405bd00 con 0x7f3a3406e700 2024-03-04T07:27:00.754 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:00.754 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-04T07:27:00.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a2006b500 msgr2=0x7f3a2006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a2006b500 0x7f3a2006d9c0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f3a28036000 tx=0x7f3a28008030 comp rx=0 tx=0).stop 2024-03-04T07:27:00.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a3406e700 msgr2=0x7f3a341f4690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a3406e700 0x7f3a341f4690 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f3a2400c500 tx=0x7f3a2400c9c0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 shutdown_connections 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a2006b500 0x7f3a2006d9c0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a340b55d0 0x7f3a3414e0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a34072660 0x7f3a340cfe00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.756+0000 7f3a3a60e640 1 --2- 172.21.15.64:0/453945723 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a3406e700 0x7f3a341f4690 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.757+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 >> 172.21.15.64:0/453945723 conn(0x7f3a340b2720 msgr2=0x7f3a340b1e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.757+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 shutdown_connections 2024-03-04T07:27:00.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.757+0000 7f3a3a60e640 1 -- 172.21.15.64:0/453945723 wait complete. 2024-03-04T07:27:00.767 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-03-04T07:27:00.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:00.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.869+0000 7f5b3c9a0640 1 Processor -- start 2024-03-04T07:27:00.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.869+0000 7f5b3c9a0640 1 -- start start 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b3c9a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b3815dd20 0x7f5b380d1bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b3c9a0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5b380d1140 0x7f5b380b6c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b3c9a0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5b3815e100 con 0x7f5b380d2100 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b3c9a0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5b380d3190 con 0x7f5b380d1140 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b3c9a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b380d2580 con 0x7f5b3815dd20 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b37fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5b380d1140 0x7f5b380b6c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b37fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5b380d1140 0x7f5b380b6c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41084/0 (socket says 172.21.15.64:41084) 2024-03-04T07:27:00.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b36ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5b380d2100 0x7f5b380d0a30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42892/0 (socket says 172.21.15.64:42892) 2024-03-04T07:27:00.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b37fff640 1 -- 172.21.15.64:0/666273898 learned_addr learned my addr 172.21.15.64:0/666273898 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:00.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b377fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b3815dd20 0x7f5b380d1bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b377fe640 1 -- 172.21.15.64:0/666273898 >> v2:172.21.15.64:3300/0 conn(0x7f5b380d1140 msgr2=0x7f5b380b6c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:00.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b377fe640 1 --2- 172.21.15.64:0/666273898 >> v2:172.21.15.64:3300/0 conn(0x7f5b380d1140 0x7f5b380b6c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b377fe640 1 -- 172.21.15.64:0/666273898 >> v1:172.21.15.64:6789/0 conn(0x7f5b380d2100 legacy=0x7f5b380d0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.870+0000 7f5b377fe640 1 -- 172.21.15.64:0/666273898 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b380d2860 con 0x7f5b3815dd20 2024-03-04T07:27:00.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.871+0000 7f5b377fe640 1 --2- 172.21.15.64:0/666273898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b3815dd20 0x7f5b380d1bc0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f5b28009d90 tx=0x7f5b2802bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ebaba46f74c65a9b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.871+0000 7f5b367fc640 1 -- 172.21.15.64:0/666273898 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b28039070 con 0x7f5b3815dd20 2024-03-04T07:27:00.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.871+0000 7f5b367fc640 1 -- 172.21.15.64:0/666273898 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b2802cc50 con 0x7f5b3815dd20 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.871+0000 7f5b367fc640 1 -- 172.21.15.64:0/666273898 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b28034440 con 0x7f5b3815dd20 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/666273898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b3815dd20 msgr2=0x7f5b380d1bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/666273898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b3815dd20 0x7f5b380d1bc0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f5b28009d90 tx=0x7f5b2802bea0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/666273898 shutdown_connections 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/666273898 >> v2:172.21.15.64:3300/0 conn(0x7f5b380d1140 0x7f5b380b6c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/666273898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b3815dd20 0x7f5b380d1bc0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/666273898 >> 172.21.15.64:0/666273898 conn(0x7f5b38071960 msgr2=0x7f5b38071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/666273898 shutdown_connections 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.872+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/666273898 wait complete. 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.873+0000 7f5b3c9a0640 1 Processor -- start 2024-03-04T07:27:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.873+0000 7f5b3c9a0640 1 -- start start 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b3c9a0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b380d1140 0x7f5b380b48d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b3c9a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b380d2100 0x7f5b38152dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b3c9a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b3815dd20 0x7f5b38153300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b3c9a0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b380b9580 con 0x7f5b3815dd20 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b3c9a0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b380d2e20 con 0x7f5b380d1140 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b3c9a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b380b9700 con 0x7f5b380d2100 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b3815dd20 0x7f5b38153300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b377fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b380d1140 0x7f5b380b48d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b3815dd20 0x7f5b38153300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41090/0 (socket says 172.21.15.64:41090) 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 -- 172.21.15.64:0/293899917 learned_addr learned my addr 172.21.15.64:0/293899917 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b36ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b380d2100 0x7f5b38152dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 -- 172.21.15.64:0/293899917 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b380d2100 msgr2=0x7f5b38152dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b380d2100 0x7f5b38152dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 -- 172.21.15.64:0/293899917 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b380d1140 msgr2=0x7f5b380b48d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b380d1140 0x7f5b380b48d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 -- 172.21.15.64:0/293899917 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b381556e0 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b37fff640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b3815dd20 0x7f5b38153300 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5b3000e8a0 tx=0x7f5b3000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b1ffff640 1 -- 172.21.15.64:0/293899917 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b30017070 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.874+0000 7f5b1ffff640 1 -- 172.21.15.64:0/293899917 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b3000c940 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.875+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b381559c0 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.875+0000 7f5b1ffff640 1 -- 172.21.15.64:0/293899917 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b3000cd30 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.875+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b38155f80 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.875+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5b3804f330 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.876+0000 7f5b1ffff640 1 -- 172.21.15.64:0/293899917 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5b3001a070 con 0x7f5b3815dd20 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.876+0000 7f5b1ffff640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b2406b500 0x7f5b2406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.876+0000 7f5b377fe640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b2406b500 0x7f5b2406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:00.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.876+0000 7f5b1ffff640 1 -- 172.21.15.64:0/293899917 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5b3008a560 con 0x7f5b3815dd20 2024-03-04T07:27:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.877+0000 7f5b377fe640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b2406b500 0x7f5b2406d9c0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f5b28036000 tx=0x7f5b28030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:00.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:00.881+0000 7f5b1ffff640 1 -- 172.21.15.64:0/293899917 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5b30058d10 con 0x7f5b3815dd20 2024-03-04T07:27:01.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.061+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f5b380b4120 con 0x7f5b3815dd20 2024-03-04T07:27:01.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.063+0000 7f5b1ffff640 1 -- 172.21.15.64:0/293899917 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f5b3005cdb0 con 0x7f5b3815dd20 2024-03-04T07:27:01.062 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:01.063 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:01.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b2406b500 msgr2=0x7f5b2406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b2406b500 0x7f5b2406d9c0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f5b28036000 tx=0x7f5b28030000 comp rx=0 tx=0).stop 2024-03-04T07:27:01.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b3815dd20 msgr2=0x7f5b38153300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b3815dd20 0x7f5b38153300 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5b3000e8a0 tx=0x7f5b3000ed60 comp rx=0 tx=0).stop 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 shutdown_connections 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b2406b500 0x7f5b2406d9c0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b3815dd20 0x7f5b38153300 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b380d2100 0x7f5b38152dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 --2- 172.21.15.64:0/293899917 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b380d1140 0x7f5b380b48d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.064+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 >> 172.21.15.64:0/293899917 conn(0x7f5b38071960 msgr2=0x7f5b380b3320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.065+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 shutdown_connections 2024-03-04T07:27:01.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.065+0000 7f5b3c9a0640 1 -- 172.21.15.64:0/293899917 wait complete. 2024-03-04T07:27:01.076 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:01.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.181+0000 7f630ad65640 1 Processor -- start 2024-03-04T07:27:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.181+0000 7f630ad65640 1 -- start start 2024-03-04T07:27:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f630ad65640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f63040b7420 0x7f63040d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f630ad65640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63040d3740 con 0x7f63040ceea0 2024-03-04T07:27:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f630ad65640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63040d3120 con 0x7f630415dd90 2024-03-04T07:27:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f630ad65640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63040d3400 con 0x7f63040b7420 2024-03-04T07:27:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f6309302640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f63040b7420 0x7f63040d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f6308b01640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f630415dd90 0x7f63040d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43842/0 (socket says 172.21.15.64:43842) 2024-03-04T07:27:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f6309302640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f63040b7420 0x7f63040d29d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41094/0 (socket says 172.21.15.64:41094) 2024-03-04T07:27:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f6308b01640 1 -- 172.21.15.64:0/2939980037 learned_addr learned my addr 172.21.15.64:0/2939980037 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f6303fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f63040ceea0 0x7f63040b6d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42908/0 (socket says 172.21.15.64:42908) 2024-03-04T07:27:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f6309302640 1 -- 172.21.15.64:0/2939980037 >> v1:172.21.15.66:6789/0 conn(0x7f630415dd90 legacy=0x7f63040d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.183+0000 7f6309302640 1 -- 172.21.15.64:0/2939980037 >> v1:172.21.15.64:6789/0 conn(0x7f63040ceea0 legacy=0x7f63040b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.182+0000 7f63037fe640 1 -- 172.21.15.64:0/2939980037 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2879736819 0 0) 0x7f63040d3120 con 0x7f630415dd90 2024-03-04T07:27:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.183+0000 7f6309302640 1 -- 172.21.15.64:0/2939980037 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63040d39e0 con 0x7f63040b7420 2024-03-04T07:27:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.183+0000 7f6309302640 1 --2- 172.21.15.64:0/2939980037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b7420 0x7f63040d29d0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f62fc009d90 tx=0x7f62fc02bd80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3da9e9a945b00a0c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.183+0000 7f63037fe640 1 -- 172.21.15.64:0/2939980037 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62fc039070 con 0x7f63040b7420 2024-03-04T07:27:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.183+0000 7f63037fe640 1 -- 172.21.15.64:0/2939980037 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62fc02cab0 con 0x7f63040b7420 2024-03-04T07:27:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.183+0000 7f63037fe640 1 -- 172.21.15.64:0/2939980037 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62fc034440 con 0x7f63040b7420 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.184+0000 7f630ad65640 1 -- 172.21.15.64:0/2939980037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b7420 msgr2=0x7f63040d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.184+0000 7f630ad65640 1 --2- 172.21.15.64:0/2939980037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b7420 0x7f63040d29d0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f62fc009d90 tx=0x7f62fc02bd80 comp rx=0 tx=0).stop 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.184+0000 7f630ad65640 1 -- 172.21.15.64:0/2939980037 shutdown_connections 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.184+0000 7f630ad65640 1 --2- 172.21.15.64:0/2939980037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b7420 0x7f63040d29d0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.184+0000 7f630ad65640 1 -- 172.21.15.64:0/2939980037 >> 172.21.15.64:0/2939980037 conn(0x7f6304071960 msgr2=0x7f6304071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.184+0000 7f630ad65640 1 -- 172.21.15.64:0/2939980037 shutdown_connections 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.184+0000 7f630ad65640 1 -- 172.21.15.64:0/2939980037 wait complete. 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.185+0000 7f630ad65640 1 Processor -- start 2024-03-04T07:27:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.185+0000 7f630ad65640 1 -- start start 2024-03-04T07:27:01.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f630ad65640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63040ceea0 0x7f63041f7930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f630ad65640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f630415dd90 0x7f6304152390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f630ad65640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b8650 0x7f63040b8b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f630ad65640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63040d48f0 con 0x7f63040b8650 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f630ad65640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f630415e170 con 0x7f63040ceea0 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f630ad65640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63040d4a70 con 0x7f630415dd90 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b8650 0x7f63040b8b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b8650 0x7f63040b8b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41098/0 (socket says 172.21.15.64:41098) 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 -- 172.21.15.64:0/1702293466 learned_addr learned my addr 172.21.15.64:0/1702293466 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6308b01640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63040ceea0 0x7f63041f7930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6303fff640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f630415dd90 0x7f6304152390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 -- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f630415dd90 msgr2=0x7f6304152390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f630415dd90 0x7f6304152390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 -- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63040ceea0 msgr2=0x7f63041f7930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63040ceea0 0x7f63041f7930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 -- 172.21.15.64:0/1702293466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63041f7e70 con 0x7f63040b8650 2024-03-04T07:27:01.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.186+0000 7f6309302640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b8650 0x7f63040b8b00 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f62fc009d90 tx=0x7f62fc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.187+0000 7f63017fa640 1 -- 172.21.15.64:0/1702293466 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62fc044070 con 0x7f63040b8650 2024-03-04T07:27:01.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.187+0000 7f63017fa640 1 -- 172.21.15.64:0/1702293466 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62fc003bb0 con 0x7f63040b8650 2024-03-04T07:27:01.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.187+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63040cb110 con 0x7f63040b8650 2024-03-04T07:27:01.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.187+0000 7f63017fa640 1 -- 172.21.15.64:0/1702293466 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62fc039040 con 0x7f63040b8650 2024-03-04T07:27:01.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.187+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63040cb6d0 con 0x7f63040b8650 2024-03-04T07:27:01.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.188+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6304072a30 con 0x7f63040b8650 2024-03-04T07:27:01.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.188+0000 7f63017fa640 1 -- 172.21.15.64:0/1702293466 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f62fc02c930 con 0x7f63040b8650 2024-03-04T07:27:01.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.188+0000 7f63017fa640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62f006b500 0x7f62f006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.189+0000 7f63017fa640 1 -- 172.21.15.64:0/1702293466 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f62fc0aca70 con 0x7f63040b8650 2024-03-04T07:27:01.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.189+0000 7f6308b01640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62f006b500 0x7f62f006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.189+0000 7f6308b01640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62f006b500 0x7f62f006d9c0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f62f4000f60 tx=0x7f62f4007e40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.193+0000 7f63017fa640 1 -- 172.21.15.64:0/1702293466 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f62fc04e020 con 0x7f63040b8650 2024-03-04T07:27:01.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.376+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f630404f410 con 0x7f63040b8650 2024-03-04T07:27:01.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.376+0000 7f63017fa640 1 -- 172.21.15.64:0/1702293466 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f63041f7e70 con 0x7f63040b8650 2024-03-04T07:27:01.376 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62f006b500 msgr2=0x7f62f006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62f006b500 0x7f62f006d9c0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f62f4000f60 tx=0x7f62f4007e40 comp rx=0 tx=0).stop 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b8650 msgr2=0x7f63040b8b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b8650 0x7f63040b8b00 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f62fc009d90 tx=0x7f62fc004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 shutdown_connections 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62f006b500 0x7f62f006d9c0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63040b8650 0x7f63040b8b00 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f630415dd90 0x7f6304152390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 --2- 172.21.15.64:0/1702293466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63040ceea0 0x7f63041f7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 >> 172.21.15.64:0/1702293466 conn(0x7f6304071960 msgr2=0x7f63040d0c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:01.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 shutdown_connections 2024-03-04T07:27:01.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.378+0000 7f630ad65640 1 -- 172.21.15.64:0/1702293466 wait complete. 2024-03-04T07:27:01.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:01.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.493+0000 7f1d72d9a640 1 Processor -- start 2024-03-04T07:27:01.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.493+0000 7f1d72d9a640 1 -- start start 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d72d9a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1d6c1618e0 0x7f1d6c06b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d72d9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d72d9a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1d6c0cee70 con 0x7f1d6c161d40 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d72d9a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1d6c0cfce0 con 0x7f1d6c1618e0 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d72d9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1d6c15f730 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d70b36640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1d6c1618e0 0x7f1d6c06b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d70b36640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1d6c1618e0 0x7f1d6c06b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41110/0 (socket says 172.21.15.64:41110) 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d70b36640 1 -- 172.21.15.64:0/849203498 learned_addr learned my addr 172.21.15.64:0/849203498 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d6bfff640 1 --2- 172.21.15.64:0/849203498 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d70b36640 1 -- 172.21.15.64:0/849203498 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c0ce4f0 msgr2=0x7f1d6c0ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d70b36640 1 --2- 172.21.15.64:0/849203498 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0ce930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d70b36640 1 -- 172.21.15.64:0/849203498 >> v1:172.21.15.66:6789/0 conn(0x7f1d6c161d40 legacy=0x7f1d6c1621d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.494+0000 7f1d70b36640 1 -- 172.21.15.64:0/849203498 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d6c15ebb0 con 0x7f1d6c1618e0 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d70b36640 1 --2- 172.21.15.64:0/849203498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c1618e0 0x7f1d6c06b130 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f1d60009870 tx=0x7f1d6002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d67ff51b7b72385 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d6b7fe640 1 -- 172.21.15.64:0/849203498 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d6002f840 con 0x7f1d6c1618e0 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d6b7fe640 1 -- 172.21.15.64:0/849203498 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1d6002f9e0 con 0x7f1d6c1618e0 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d6b7fe640 1 -- 172.21.15.64:0/849203498 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d6002fdb0 con 0x7f1d6c1618e0 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 -- 172.21.15.64:0/849203498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c1618e0 msgr2=0x7f1d6c06b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/849203498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c1618e0 0x7f1d6c06b130 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f1d60009870 tx=0x7f1d6002ba20 comp rx=0 tx=0).stop 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 -- 172.21.15.64:0/849203498 shutdown_connections 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/849203498 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/849203498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c1618e0 0x7f1d6c06b130 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 -- 172.21.15.64:0/849203498 >> 172.21.15.64:0/849203498 conn(0x7f1d6c0b3660 msgr2=0x7f1d6c0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:01.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 -- 172.21.15.64:0/849203498 shutdown_connections 2024-03-04T07:27:01.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.495+0000 7f1d72d9a640 1 -- 172.21.15.64:0/849203498 wait complete. 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 Processor -- start 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 -- start start 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0b6890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d6c161d40 0x7f1d6c0b5180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c1541a0 0x7f1d6c0c78d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d6c15f450 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d6c15f170 con 0x7f1d6c161d40 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.497+0000 7f1d72d9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1d6c06b670 con 0x7f1d6c1541a0 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0b6890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0b6890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41124/0 (socket says 172.21.15.64:41124) 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 -- 172.21.15.64:0/3476339333 learned_addr learned my addr 172.21.15.64:0/3476339333 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d6bfff640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d6c161d40 0x7f1d6c0b5180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d71337640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c1541a0 0x7f1d6c0c78d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 -- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c1541a0 msgr2=0x7f1d6c0c78d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c1541a0 0x7f1d6c0c78d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 -- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d6c161d40 msgr2=0x7f1d6c0b5180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d6c161d40 0x7f1d6c0b5180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 -- 172.21.15.64:0/3476339333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d6c154630 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d70b36640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0b6890 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f1d60000c00 tx=0x7f1d60004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d697fa640 1 -- 172.21.15.64:0/3476339333 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d60037bc0 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d697fa640 1 -- 172.21.15.64:0/3476339333 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1d60046070 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d6c0b57f0 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d697fa640 1 -- 172.21.15.64:0/3476339333 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d60041400 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.498+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d6c0b5db0 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.499+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1d6c09fc50 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.500+0000 7f1d697fa640 1 -- 172.21.15.64:0/3476339333 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1d60040070 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.500+0000 7f1d697fa640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1d5006b4d0 0x7f1d5006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.500+0000 7f1d6bfff640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1d5006b4d0 0x7f1d5006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.500+0000 7f1d697fa640 1 -- 172.21.15.64:0/3476339333 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1d600ac630 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.501+0000 7f1d6bfff640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1d5006b4d0 0x7f1d5006d990 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f1d6c0b4500 tx=0x7f1d58009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.504+0000 7f1d697fa640 1 -- 172.21.15.64:0/3476339333 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1d60033030 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.685+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f1d6c160120 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.686+0000 7f1d697fa640 1 -- 172.21.15.64:0/3476339333 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f1d6007aef0 con 0x7f1d6c0ce4f0 2024-03-04T07:27:01.685 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1d5006b4d0 msgr2=0x7f1d5006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1d5006b4d0 0x7f1d5006d990 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f1d6c0b4500 tx=0x7f1d58009190 comp rx=0 tx=0).stop 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c0ce4f0 msgr2=0x7f1d6c0b6890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0b6890 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f1d60000c00 tx=0x7f1d60004730 comp rx=0 tx=0).stop 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 shutdown_connections 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1d5006b4d0 0x7f1d5006d990 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1d6c1541a0 0x7f1d6c0c78d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d6c161d40 0x7f1d6c0b5180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 --2- 172.21.15.64:0/3476339333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1d6c0ce4f0 0x7f1d6c0b6890 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 >> 172.21.15.64:0/3476339333 conn(0x7f1d6c0b3660 msgr2=0x7f1d6c15e160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:01.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 shutdown_connections 2024-03-04T07:27:01.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.688+0000 7f1d72d9a640 1 -- 172.21.15.64:0/3476339333 wait complete. 2024-03-04T07:27:01.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:01.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.803+0000 7f9fe288a640 1 Processor -- start 2024-03-04T07:27:01.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.803+0000 7f9fe288a640 1 -- start start 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fe288a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9fdc1618e0 0x7f9fdc06b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fe288a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce960 0x7f9fdc162450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fe288a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9fdc15f730 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fe288a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9fdc0cfce0 con 0x7f9fdc1618e0 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fe288a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9fdc15ebb0 con 0x7f9fdc0ce960 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fdbfff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9fdc1618e0 0x7f9fdc06b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fe0e27640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce960 0x7f9fdc162450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fdb7fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f9fdc0ce4f0 0x7f9fdc161d40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43864/0 (socket says 172.21.15.64:43864) 2024-03-04T07:27:01.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fdb7fe640 1 -- 172.21.15.64:0/3764856115 learned_addr learned my addr 172.21.15.64:0/3764856115 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:01.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fdbfff640 1 -- 172.21.15.64:0/3764856115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce960 msgr2=0x7f9fdc162450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:01.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fdbfff640 1 --2- 172.21.15.64:0/3764856115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce960 0x7f9fdc162450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fdbfff640 1 -- 172.21.15.64:0/3764856115 >> v1:172.21.15.66:6789/0 conn(0x7f9fdc0ce4f0 legacy=0x7f9fdc161d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.804+0000 7f9fdbfff640 1 -- 172.21.15.64:0/3764856115 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9fdc15ee90 con 0x7f9fdc1618e0 2024-03-04T07:27:01.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.805+0000 7f9fdbfff640 1 --2- 172.21.15.64:0/3764856115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc1618e0 0x7f9fdc06b130 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f9fc40097a0 tx=0x7f9fc402b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8fa08713a9886501 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.805+0000 7f9fdaffd640 1 -- 172.21.15.64:0/3764856115 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fc4034070 con 0x7f9fdc1618e0 2024-03-04T07:27:01.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.805+0000 7f9fdaffd640 1 -- 172.21.15.64:0/3764856115 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9fc4030610 con 0x7f9fdc1618e0 2024-03-04T07:27:01.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.805+0000 7f9fdaffd640 1 -- 172.21.15.64:0/3764856115 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fc40309e0 con 0x7f9fdc1618e0 2024-03-04T07:27:01.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 -- 172.21.15.64:0/3764856115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc1618e0 msgr2=0x7f9fdc06b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 --2- 172.21.15.64:0/3764856115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc1618e0 0x7f9fdc06b130 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f9fc40097a0 tx=0x7f9fc402b960 comp rx=0 tx=0).stop 2024-03-04T07:27:01.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 -- 172.21.15.64:0/3764856115 shutdown_connections 2024-03-04T07:27:01.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 --2- 172.21.15.64:0/3764856115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce960 0x7f9fdc162450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 --2- 172.21.15.64:0/3764856115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc1618e0 0x7f9fdc06b130 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 -- 172.21.15.64:0/3764856115 >> 172.21.15.64:0/3764856115 conn(0x7f9fdc0b3660 msgr2=0x7f9fdc0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:01.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 -- 172.21.15.64:0/3764856115 shutdown_connections 2024-03-04T07:27:01.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.806+0000 7f9fe288a640 1 -- 172.21.15.64:0/3764856115 wait complete. 2024-03-04T07:27:01.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.807+0000 7f9fe288a640 1 Processor -- start 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.807+0000 7f9fe288a640 1 -- start start 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe288a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce4f0 0x7f9fdc0c9fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe288a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fdc0ce960 0x7f9fdc0cb0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe288a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc14d230 0x7f9fdc0cb600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe288a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9fdc06b670 con 0x7f9fdc0ce960 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe288a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9fdc15f450 con 0x7f9fdc14d230 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe288a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9fdc0cedf0 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fdc0ce960 0x7f9fdc0cb0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce4f0 0x7f9fdc0c9fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe0e27640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc14d230 0x7f9fdc0cb600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce4f0 0x7f9fdc0c9fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:48440/0 (socket says 172.21.15.64:48440) 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 -- 172.21.15.64:0/1128932384 learned_addr learned my addr 172.21.15.64:0/1128932384 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fdc0ce960 0x7f9fdc0cb0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41134/0 (socket says 172.21.15.64:41134) 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 -- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc14d230 msgr2=0x7f9fdc0cb600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc14d230 0x7f9fdc0cb600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 -- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fdc0ce960 msgr2=0x7f9fdc0cb0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fdc0ce960 0x7f9fdc0cb0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fdbfff640 1 -- 172.21.15.64:0/1128932384 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9fdc0c8c20 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.808+0000 7f9fe0e27640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc14d230 0x7f9fdc0cb600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.809+0000 7f9fdbfff640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce4f0 0x7f9fdc0c9fa0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f9fc402bf70 tx=0x7f9fc40046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.809+0000 7f9fd8ff9640 1 -- 172.21.15.64:0/1128932384 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fc4034040 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.809+0000 7f9fd8ff9640 1 -- 172.21.15.64:0/1128932384 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9fc403aaa0 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.809+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9fdc0c8f00 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.809+0000 7f9fd8ff9640 1 -- 172.21.15.64:0/1128932384 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fc40443f0 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.809+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9fdc0c94c0 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.810+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9fdc09fc50 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.810+0000 7f9fd8ff9640 1 -- 172.21.15.64:0/1128932384 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9fc404b060 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.810+0000 7f9fd8ff9640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fbc06b4d0 0x7f9fbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.811+0000 7f9fdb7fe640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fbc06b4d0 0x7f9fbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.811+0000 7f9fd8ff9640 1 -- 172.21.15.64:0/1128932384 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9fc40abe10 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.811+0000 7f9fdb7fe640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fbc06b4d0 0x7f9fbc06d990 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f9fdc0c61d0 tx=0x7f9fcc007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:01.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.815+0000 7f9fd8ff9640 1 -- 172.21.15.64:0/1128932384 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9fc403f070 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.998+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f9fdc0b2d30 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:01.998+0000 7f9fd8ff9640 1 -- 172.21.15.64:0/1128932384 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f9fc407a640 con 0x7f9fdc0ce4f0 2024-03-04T07:27:01.998 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:02.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fbc06b4d0 msgr2=0x7f9fbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fbc06b4d0 0x7f9fbc06d990 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f9fdc0c61d0 tx=0x7f9fcc007e80 comp rx=0 tx=0).stop 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce4f0 msgr2=0x7f9fdc0c9fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce4f0 0x7f9fdc0c9fa0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f9fc402bf70 tx=0x7f9fc40046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 shutdown_connections 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fbc06b4d0 0x7f9fbc06d990 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fdc14d230 0x7f9fdc0cb600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fdc0ce960 0x7f9fdc0cb0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 --2- 172.21.15.64:0/1128932384 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fdc0ce4f0 0x7f9fdc0c9fa0 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.000+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 >> 172.21.15.64:0/1128932384 conn(0x7f9fdc0b3660 msgr2=0x7f9fdc0cc690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.001+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 shutdown_connections 2024-03-04T07:27:02.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.001+0000 7f9fe288a640 1 -- 172.21.15.64:0/1128932384 wait complete. 2024-03-04T07:27:02.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:02.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.113+0000 7f1852e46640 1 Processor -- start 2024-03-04T07:27:02.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.113+0000 7f1852e46640 1 -- start start 2024-03-04T07:27:02.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.113+0000 7f1852e46640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f184c15dd70 0x7f184c0b9650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f1852e46640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f184c0b7c70 0x7f184c0b5b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f1852e46640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f184c0b83c0 con 0x7f184c0b5700 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f1852e46640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f184c0b60b0 con 0x7f184c15dd70 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f1852e46640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f184c0b8fd0 con 0x7f184c0b7c70 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f1850be2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f184c15dd70 0x7f184c0b9650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f184bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f184c0b5700 0x7f184c0d1e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42918/0 (socket says 172.21.15.64:42918) 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f184bfff640 1 -- 172.21.15.64:0/430190413 learned_addr learned my addr 172.21.15.64:0/430190413 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f18513e3640 1 --2- 172.21.15.64:0/430190413 >> v2:172.21.15.66:3300/0 conn(0x7f184c0b7c70 0x7f184c0b5b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f18513e3640 1 -- 172.21.15.64:0/430190413 >> v2:172.21.15.64:3300/0 conn(0x7f184c15dd70 msgr2=0x7f184c0b9650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f18513e3640 1 --2- 172.21.15.64:0/430190413 >> v2:172.21.15.64:3300/0 conn(0x7f184c15dd70 0x7f184c0b9650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f18513e3640 1 -- 172.21.15.64:0/430190413 >> v1:172.21.15.64:6789/0 conn(0x7f184c0b5700 legacy=0x7f184c0d1e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f184b7fe640 1 -- 172.21.15.64:0/430190413 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3412741187 0 0) 0x7f184c0b83c0 con 0x7f184c0b5700 2024-03-04T07:27:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f18513e3640 1 -- 172.21.15.64:0/430190413 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f184c0b86a0 con 0x7f184c0b7c70 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.114+0000 7f1850be2640 1 --2- 172.21.15.64:0/430190413 >> v2:172.21.15.64:3300/0 conn(0x7f184c15dd70 0x7f184c0b9650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.115+0000 7f18513e3640 1 --2- 172.21.15.64:0/430190413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c0b7c70 0x7f184c0b5b70 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f184c0d1a40 tx=0x7f184402bf50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6efe1be71dbcffde server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.115+0000 7f184b7fe640 1 -- 172.21.15.64:0/430190413 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1844039070 con 0x7f184c0b7c70 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.115+0000 7f184b7fe640 1 -- 172.21.15.64:0/430190413 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f184402cd00 con 0x7f184c0b7c70 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.115+0000 7f184b7fe640 1 -- 172.21.15.64:0/430190413 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1844034660 con 0x7f184c0b7c70 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 -- 172.21.15.64:0/430190413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c0b7c70 msgr2=0x7f184c0b5b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 --2- 172.21.15.64:0/430190413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c0b7c70 0x7f184c0b5b70 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f184c0d1a40 tx=0x7f184402bf50 comp rx=0 tx=0).stop 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 -- 172.21.15.64:0/430190413 shutdown_connections 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 --2- 172.21.15.64:0/430190413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c0b7c70 0x7f184c0b5b70 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 --2- 172.21.15.64:0/430190413 >> v2:172.21.15.64:3300/0 conn(0x7f184c15dd70 0x7f184c0b9650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 -- 172.21.15.64:0/430190413 >> 172.21.15.64:0/430190413 conn(0x7f184c071960 msgr2=0x7f184c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:02.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 -- 172.21.15.64:0/430190413 shutdown_connections 2024-03-04T07:27:02.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.116+0000 7f1852e46640 1 -- 172.21.15.64:0/430190413 wait complete. 2024-03-04T07:27:02.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.117+0000 7f1852e46640 1 Processor -- start 2024-03-04T07:27:02.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.117+0000 7f1852e46640 1 -- start start 2024-03-04T07:27:02.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.117+0000 7f1852e46640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b5700 0x7f184c0c77b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1852e46640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f184c0b7c70 0x7f184c1f4460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1852e46640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15dd70 0x7f184c15f920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1852e46640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f184c0b9b90 con 0x7f184c0b7c70 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1852e46640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f184c0b8c60 con 0x7f184c15dd70 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1852e46640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f184c0b80e0 con 0x7f184c0b5700 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f184c0b7c70 0x7f184c1f4460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f184c0b7c70 0x7f184c1f4460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41154/0 (socket says 172.21.15.64:41154) 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1850be2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b5700 0x7f184c0c77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f18513e3640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15dd70 0x7f184c15f920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 -- 172.21.15.64:0/137495194 learned_addr learned my addr 172.21.15.64:0/137495194 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 -- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b5700 msgr2=0x7f184c0c77b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b5700 0x7f184c0c77b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 -- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15dd70 msgr2=0x7f184c15f920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15dd70 0x7f184c15f920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 -- 172.21.15.64:0/137495194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f184c1f49a0 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1850be2640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b5700 0x7f184c0c77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f184bfff640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f184c0b7c70 0x7f184c1f4460 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f183800c4d0 tx=0x7f183800c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f18497fa640 1 -- 172.21.15.64:0/137495194 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1838018070 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f18497fa640 1 -- 172.21.15.64:0/137495194 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1838007500 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f184c0c7e00 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f18497fa640 1 -- 172.21.15.64:0/137495194 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1838004a20 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.118+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f184c0c83c0 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.119+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f184c067ff0 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.120+0000 7f18497fa640 1 -- 172.21.15.64:0/137495194 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f183801b070 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.120+0000 7f18497fa640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.120+0000 7f18497fa640 1 -- 172.21.15.64:0/137495194 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1838089400 con 0x7f184c0b7c70 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.120+0000 7f1850be2640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.120+0000 7f1850be2640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f1840009560 tx=0x7f1840009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.124+0000 7f18497fa640 1 -- 172.21.15.64:0/137495194 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1838057c30 con 0x7f184c0b7c70 2024-03-04T07:27:02.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.341+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f184c0b3140 con 0x7f184c0b7c70 2024-03-04T07:27:02.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.342+0000 7f18497fa640 1 -- 172.21.15.64:0/137495194 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f183805bcd0 con 0x7f184c0b7c70 2024-03-04T07:27:02.342 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:02.342 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:27:02.342 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 msgr2=0x7f183006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f1840009560 tx=0x7f1840009190 comp rx=0 tx=0).stop 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f184c0b7c70 msgr2=0x7f184c1f4460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f184c0b7c70 0x7f184c1f4460 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f183800c4d0 tx=0x7f183800c990 comp rx=0 tx=0).stop 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 shutdown_connections 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15dd70 0x7f184c15f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f184c0b7c70 0x7f184c1f4460 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 --2- 172.21.15.64:0/137495194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b5700 0x7f184c0c77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 >> 172.21.15.64:0/137495194 conn(0x7f184c071960 msgr2=0x7f184c0d2110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:02.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 shutdown_connections 2024-03-04T07:27:02.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.344+0000 7f1852e46640 1 -- 172.21.15.64:0/137495194 wait complete. 2024-03-04T07:27:02.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.458+0000 7f7f35246640 1 Processor -- start 2024-03-04T07:27:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.458+0000 7f7f35246640 1 -- start start 2024-03-04T07:27:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f35246640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f300cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f35246640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7f300b4260 0x7f7f300b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f35246640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7f300b0dd0 con 0x7f7f300d06c0 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f35246640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7f300b0b80 con 0x7f7f300b4260 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f35246640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7f300aeec0 con 0x7f7f300d02e0 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7f300b4260 0x7f7f300b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7f300b4260 0x7f7f300b1500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41160/0 (socket says 172.21.15.64:41160) 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2e59c640 1 -- 172.21.15.64:0/2909448208 learned_addr learned my addr 172.21.15.64:0/2909448208 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2ed9d640 1 --2- 172.21.15.64:0/2909448208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f300cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2e59c640 1 -- 172.21.15.64:0/2909448208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 msgr2=0x7f7f300cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2e59c640 1 --2- 172.21.15.64:0/2909448208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f300cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2e59c640 1 -- 172.21.15.64:0/2909448208 >> v1:172.21.15.66:6789/0 conn(0x7f7f300d06c0 legacy=0x7f7f300b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.459+0000 7f7f2e59c640 1 -- 172.21.15.64:0/2909448208 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f300b10b0 con 0x7f7f300b4260 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.460+0000 7f7f2e59c640 1 --2- 172.21.15.64:0/2909448208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300b4260 0x7f7f300b1500 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f7f18009870 tx=0x7f7f1802ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=35996664fdbf092 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.460+0000 7f7f2dd9b640 1 -- 172.21.15.64:0/2909448208 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f1802f840 con 0x7f7f300b4260 2024-03-04T07:27:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.460+0000 7f7f2dd9b640 1 -- 172.21.15.64:0/2909448208 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f1802f9e0 con 0x7f7f300b4260 2024-03-04T07:27:02.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.460+0000 7f7f2dd9b640 1 -- 172.21.15.64:0/2909448208 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f1802fdb0 con 0x7f7f300b4260 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.460+0000 7f7f35246640 1 -- 172.21.15.64:0/2909448208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300b4260 msgr2=0x7f7f300b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.460+0000 7f7f35246640 1 --2- 172.21.15.64:0/2909448208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300b4260 0x7f7f300b1500 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f7f18009870 tx=0x7f7f1802ba20 comp rx=0 tx=0).stop 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.461+0000 7f7f35246640 1 -- 172.21.15.64:0/2909448208 shutdown_connections 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.461+0000 7f7f35246640 1 --2- 172.21.15.64:0/2909448208 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300b4260 0x7f7f300b1500 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.461+0000 7f7f35246640 1 --2- 172.21.15.64:0/2909448208 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f300cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.461+0000 7f7f35246640 1 -- 172.21.15.64:0/2909448208 >> 172.21.15.64:0/2909448208 conn(0x7f7f300b36d0 msgr2=0x7f7f300b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.461+0000 7f7f35246640 1 -- 172.21.15.64:0/2909448208 shutdown_connections 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.461+0000 7f7f35246640 1 -- 172.21.15.64:0/2909448208 wait complete. 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.462+0000 7f7f35246640 1 Processor -- start 2024-03-04T07:27:02.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.462+0000 7f7f35246640 1 -- start start 2024-03-04T07:27:02.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f35246640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f300b4260 0x7f7f300c72f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f35246640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f301e6140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f35246640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300d06c0 0x7f7f301e6680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f35246640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f300b22d0 con 0x7f7f300d06c0 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f35246640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f300b2150 con 0x7f7f300b4260 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f35246640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7f300af350 con 0x7f7f300d02e0 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300d06c0 0x7f7f301e6680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300d06c0 0x7f7f301e6680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41168/0 (socket says 172.21.15.64:41168) 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 -- 172.21.15.64:0/1273845993 learned_addr learned my addr 172.21.15.64:0/1273845993 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2ed9d640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f300b4260 0x7f7f300c72f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2e59c640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f301e6140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 -- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 msgr2=0x7f7f301e6140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f301e6140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 -- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f300b4260 msgr2=0x7f7f300c72f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f300b4260 0x7f7f300c72f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 -- 172.21.15.64:0/1273845993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f300c9750 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f2f59e640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300d06c0 0x7f7f301e6680 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f7f2000c680 tx=0x7f7f2000cb40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f277fe640 1 -- 172.21.15.64:0/1273845993 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f20003b70 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f277fe640 1 -- 172.21.15.64:0/1273845993 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f20003d10 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7f300c9a30 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.463+0000 7f7f277fe640 1 -- 172.21.15.64:0/1273845993 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f20005310 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.464+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7f300c7830 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.464+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7f300743c0 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.465+0000 7f7f277fe640 1 -- 172.21.15.64:0/1273845993 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7f200054b0 con 0x7f7f300d06c0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.465+0000 7f7f277fe640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1006b4d0 0x7f7f1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.465+0000 7f7f2ed9d640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1006b4d0 0x7f7f1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.465+0000 7f7f277fe640 1 -- 172.21.15.64:0/1273845993 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7f20089ef0 con 0x7f7f300d06c0 2024-03-04T07:27:02.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.466+0000 7f7f2ed9d640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1006b4d0 0x7f7f1006d990 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f7f28009b40 tx=0x7f7f28009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.469+0000 7f7f277fe640 1 -- 172.21.15.64:0/1273845993 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7f300743c0 con 0x7f7f300d06c0 2024-03-04T07:27:02.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.649+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7f30068700 con 0x7f7f300d06c0 2024-03-04T07:27:02.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.650+0000 7f7f277fe640 1 -- 172.21.15.64:0/1273845993 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f7f20058720 con 0x7f7f300d06c0 2024-03-04T07:27:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:02.650 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1006b4d0 msgr2=0x7f7f1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1006b4d0 0x7f7f1006d990 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f7f28009b40 tx=0x7f7f28009190 comp rx=0 tx=0).stop 2024-03-04T07:27:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300d06c0 msgr2=0x7f7f301e6680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300d06c0 0x7f7f301e6680 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f7f2000c680 tx=0x7f7f2000cb40 comp rx=0 tx=0).stop 2024-03-04T07:27:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 shutdown_connections 2024-03-04T07:27:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1006b4d0 0x7f7f1006d990 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f300d06c0 0x7f7f301e6680 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f300d02e0 0x7f7f301e6140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 --2- 172.21.15.64:0/1273845993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f300b4260 0x7f7f300c72f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 >> 172.21.15.64:0/1273845993 conn(0x7f7f300b36d0 msgr2=0x7f7f300b1740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:02.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 shutdown_connections 2024-03-04T07:27:02.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.652+0000 7f7f35246640 1 -- 172.21.15.64:0/1273845993 wait complete. 2024-03-04T07:27:02.663 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:02.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:02.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.768+0000 7fa69c7e2640 1 Processor -- start 2024-03-04T07:27:02.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.768+0000 7fa69c7e2640 1 -- start start 2024-03-04T07:27:02.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa69c7e2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6940b4830 0x7fa694160280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa69c7e2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa694070980 con 0x7fa6940b29a0 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa69c7e2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa6941607c0 con 0x7fa694142170 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa69c7e2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6941458e0 con 0x7fa6940b4830 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa699d7d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6940b4830 0x7fa694160280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa699d7d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6940b4830 0x7fa694160280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41176/0 (socket says 172.21.15.64:41176) 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa699d7d640 1 -- 172.21.15.64:0/732163156 learned_addr learned my addr 172.21.15.64:0/732163156 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa69ad7f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa6940b29a0 0x7fa6940b2db0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42924/0 (socket says 172.21.15.64:42924) 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa699d7d640 1 -- 172.21.15.64:0/732163156 >> v1:172.21.15.66:6789/0 conn(0x7fa694142170 legacy=0x7fa694142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa699d7d640 1 -- 172.21.15.64:0/732163156 >> v1:172.21.15.64:6789/0 conn(0x7fa6940b29a0 legacy=0x7fa6940b2db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa69957c640 1 -- 172.21.15.64:0/732163156 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2060645803 0 0) 0x7fa690001110 con 0x7fa694142170 2024-03-04T07:27:02.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.769+0000 7fa699d7d640 1 -- 172.21.15.64:0/732163156 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa694070bf0 con 0x7fa6940b4830 2024-03-04T07:27:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.770+0000 7fa699d7d640 1 --2- 172.21.15.64:0/732163156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa694160280 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7fa684009870 tx=0x7fa68402b7e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=636c216212587a8e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.770+0000 7fa69957c640 1 -- 172.21.15.64:0/732163156 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa684037070 con 0x7fa6940b4830 2024-03-04T07:27:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.770+0000 7fa69957c640 1 -- 172.21.15.64:0/732163156 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa684004230 con 0x7fa6940b4830 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.770+0000 7fa69957c640 1 -- 172.21.15.64:0/732163156 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa684003610 con 0x7fa6940b4830 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.770+0000 7fa69c7e2640 1 -- 172.21.15.64:0/732163156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 msgr2=0x7fa694160280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.770+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/732163156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa694160280 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7fa684009870 tx=0x7fa68402b7e0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.771+0000 7fa69c7e2640 1 -- 172.21.15.64:0/732163156 shutdown_connections 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.771+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/732163156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa694160280 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.771+0000 7fa69c7e2640 1 -- 172.21.15.64:0/732163156 >> 172.21.15.64:0/732163156 conn(0x7fa694066d00 msgr2=0x7fa694067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.771+0000 7fa69c7e2640 1 -- 172.21.15.64:0/732163156 shutdown_connections 2024-03-04T07:27:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.771+0000 7fa69c7e2640 1 -- 172.21.15.64:0/732163156 wait complete. 2024-03-04T07:27:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 Processor -- start 2024-03-04T07:27:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 -- start start 2024-03-04T07:27:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6940b29a0 0x7fa6941f3da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa69406cbb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa694142170 0x7fa69415b520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6940b4df0 con 0x7fa6940b4830 2024-03-04T07:27:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6940711b0 con 0x7fa694142170 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69c7e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa694142550 con 0x7fa6940b29a0 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa699d7d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa69406cbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa699d7d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa69406cbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41184/0 (socket says 172.21.15.64:41184) 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa699d7d640 1 -- 172.21.15.64:0/213621199 learned_addr learned my addr 172.21.15.64:0/213621199 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69ad7f640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa694142170 0x7fa69415b520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.772+0000 7fa69a57e640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6940b29a0 0x7fa6941f3da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa699d7d640 1 -- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6940b29a0 msgr2=0x7fa6941f3da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa699d7d640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6940b29a0 0x7fa6941f3da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa699d7d640 1 -- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa694142170 msgr2=0x7fa69415b520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa699d7d640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa694142170 0x7fa69415b520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa699d7d640 1 -- 172.21.15.64:0/213621199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6941f44f0 con 0x7fa6940b4830 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa699d7d640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa69406cbb0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fa684009870 tx=0x7fa6840046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa68affd640 1 -- 172.21.15.64:0/213621199 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa684042070 con 0x7fa6940b4830 2024-03-04T07:27:02.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa68affd640 1 -- 172.21.15.64:0/213621199 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa684030730 con 0x7fa6940b4830 2024-03-04T07:27:02.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa6941f47d0 con 0x7fa6940b4830 2024-03-04T07:27:02.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa68affd640 1 -- 172.21.15.64:0/213621199 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa684037040 con 0x7fa6940b4830 2024-03-04T07:27:02.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.773+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6941f5090 con 0x7fa6940b4830 2024-03-04T07:27:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.774+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa6940733c0 con 0x7fa6940b4830 2024-03-04T07:27:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.775+0000 7fa68affd640 1 -- 172.21.15.64:0/213621199 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa684045070 con 0x7fa6940b4830 2024-03-04T07:27:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.775+0000 7fa68affd640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa67806b4d0 0x7fa67806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.775+0000 7fa69a57e640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa67806b4d0 0x7fa67806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.775+0000 7fa68affd640 1 -- 172.21.15.64:0/213621199 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa6840abec0 con 0x7fa6940b4830 2024-03-04T07:27:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.776+0000 7fa69a57e640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa67806b4d0 0x7fa67806d990 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fa690000f60 tx=0x7fa690009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:02.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.780+0000 7fa68affd640 1 -- 172.21.15.64:0/213621199 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa68407a780 con 0x7fa6940b4830 2024-03-04T07:27:02.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.958+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fa69404f410 con 0x7fa6940b4830 2024-03-04T07:27:02.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.959+0000 7fa68affd640 1 -- 172.21.15.64:0/213621199 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7fa68407e820 con 0x7fa6940b4830 2024-03-04T07:27:02.958 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:02.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa67806b4d0 msgr2=0x7fa67806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa67806b4d0 0x7fa67806d990 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fa690000f60 tx=0x7fa690009040 comp rx=0 tx=0).stop 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 msgr2=0x7fa69406cbb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa69406cbb0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fa684009870 tx=0x7fa6840046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 shutdown_connections 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa67806b4d0 0x7fa67806d990 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa694142170 0x7fa69415b520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6940b4830 0x7fa69406cbb0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 --2- 172.21.15.64:0/213621199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6940b29a0 0x7fa6941f3da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 >> 172.21.15.64:0/213621199 conn(0x7fa694066d00 msgr2=0x7fa69415f5b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 shutdown_connections 2024-03-04T07:27:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:02.961+0000 7fa69c7e2640 1 -- 172.21.15.64:0/213621199 wait complete. 2024-03-04T07:27:02.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:03.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.075+0000 7f20f88e1640 1 Processor -- start 2024-03-04T07:27:03.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.075+0000 7f20f88e1640 1 -- start start 2024-03-04T07:27:03.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f88e1640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f20f4144640 0x7f20f406e400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f88e1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f415b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f88e1640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f20f415bc40 con 0x7f20f415b670 2024-03-04T07:27:03.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f88e1640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f20f406df70 con 0x7f20f4144640 2024-03-04T07:27:03.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f88e1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20f415bf20 con 0x7f20f406e940 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f2d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f20f415b670 0x7f20f40ce900 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42938/0 (socket says 172.21.15.64:42938) 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f2d9d640 1 -- 172.21.15.64:0/42436371 learned_addr learned my addr 172.21.15.64:0/42436371 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f259c640 1 --2- 172.21.15.64:0/42436371 >> v2:172.21.15.66:3300/0 conn(0x7f20f4144640 0x7f20f406e400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f1d9b640 1 --2- 172.21.15.64:0/42436371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f415b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f159a640 1 -- 172.21.15.64:0/42436371 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 607504355 0 0) 0x7f20f415bc40 con 0x7f20f415b670 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f159a640 1 -- 172.21.15.64:0/42436371 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f20e0003520 con 0x7f20f415b670 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f1d9b640 1 -- 172.21.15.64:0/42436371 >> v2:172.21.15.66:3300/0 conn(0x7f20f4144640 msgr2=0x7f20f406e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f1d9b640 1 --2- 172.21.15.64:0/42436371 >> v2:172.21.15.66:3300/0 conn(0x7f20f4144640 0x7f20f406e400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f1d9b640 1 -- 172.21.15.64:0/42436371 >> v1:172.21.15.64:6789/0 conn(0x7f20f415b670 legacy=0x7f20f40ce900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f1d9b640 1 -- 172.21.15.64:0/42436371 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20f406dc00 con 0x7f20f406e940 2024-03-04T07:27:03.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.076+0000 7f20f259c640 1 --2- 172.21.15.64:0/42436371 >> v2:172.21.15.66:3300/0 conn(0x7f20f4144640 0x7f20f406e400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:03.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.077+0000 7f20f1d9b640 1 --2- 172.21.15.64:0/42436371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f415b130 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f20dc0097a0 tx=0x7f20dc02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eddc9af32512e95d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.077+0000 7f20f159a640 1 -- 172.21.15.64:0/42436371 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20dc035020 con 0x7f20f406e940 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.077+0000 7f20f159a640 1 -- 172.21.15.64:0/42436371 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20dc031830 con 0x7f20f406e940 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.077+0000 7f20f159a640 1 -- 172.21.15.64:0/42436371 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20dc031be0 con 0x7f20f406e940 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 -- 172.21.15.64:0/42436371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 msgr2=0x7f20f415b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 --2- 172.21.15.64:0/42436371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f415b130 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f20dc0097a0 tx=0x7f20dc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 -- 172.21.15.64:0/42436371 shutdown_connections 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 --2- 172.21.15.64:0/42436371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f415b130 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 --2- 172.21.15.64:0/42436371 >> v2:172.21.15.66:3300/0 conn(0x7f20f4144640 0x7f20f406e400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 -- 172.21.15.64:0/42436371 >> 172.21.15.64:0/42436371 conn(0x7f20f4066d00 msgr2=0x7f20f4067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 -- 172.21.15.64:0/42436371 shutdown_connections 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.078+0000 7f20f88e1640 1 -- 172.21.15.64:0/42436371 wait complete. 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.079+0000 7f20f88e1640 1 Processor -- start 2024-03-04T07:27:03.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.079+0000 7f20f88e1640 1 -- start start 2024-03-04T07:27:03.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f88e1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f41504d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f88e1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20f4144640 0x7f20f40c4910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f88e1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20f415b670 0x7f20f40c6bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f88e1640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20f40cf190 con 0x7f20f4144640 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f88e1640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20f40cf010 con 0x7f20f415b670 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f88e1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20f40b36d0 con 0x7f20f406e940 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20f4144640 0x7f20f40c4910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f259c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f41504d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20f4144640 0x7f20f40c4910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41192/0 (socket says 172.21.15.64:41192) 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 -- 172.21.15.64:0/2060239506 learned_addr learned my addr 172.21.15.64:0/2060239506 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f2d9d640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20f415b670 0x7f20f40c6bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 -- 172.21.15.64:0/2060239506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 msgr2=0x7f20f41504d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f41504d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 -- 172.21.15.64:0/2060239506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20f415b670 msgr2=0x7f20f40c6bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20f415b670 0x7f20f40c6bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.080+0000 7f20f1d9b640 1 -- 172.21.15.64:0/2060239506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20f4150a10 con 0x7f20f4144640 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.081+0000 7f20f1d9b640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20f4144640 0x7f20f40c4910 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f20dc0098c0 tx=0x7f20dc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.081+0000 7f20eaffd640 1 -- 172.21.15.64:0/2060239506 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20dc035040 con 0x7f20f4144640 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.081+0000 7f20eaffd640 1 -- 172.21.15.64:0/2060239506 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20dc03b900 con 0x7f20f4144640 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.081+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20f40c5260 con 0x7f20f4144640 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.081+0000 7f20eaffd640 1 -- 172.21.15.64:0/2060239506 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20dc03bcf0 con 0x7f20f4144640 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.081+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20f40c5820 con 0x7f20f4144640 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.082+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f20f40733c0 con 0x7f20f4144640 2024-03-04T07:27:03.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.082+0000 7f20eaffd640 1 -- 172.21.15.64:0/2060239506 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f20dc038020 con 0x7f20f4144640 2024-03-04T07:27:03.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.082+0000 7f20eaffd640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d406b4d0 0x7f20d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.083+0000 7f20eaffd640 1 -- 172.21.15.64:0/2060239506 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f20dc0ac070 con 0x7f20f4144640 2024-03-04T07:27:03.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.083+0000 7f20f259c640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d406b4d0 0x7f20d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.083+0000 7f20f259c640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d406b4d0 0x7f20d406d990 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f20ec00b800 tx=0x7f20ec00b3a0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.087+0000 7f20eaffd640 1 -- 172.21.15.64:0/2060239506 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f20dc07a930 con 0x7f20f4144640 2024-03-04T07:27:03.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.269+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f20f4141df0 con 0x7f20f4144640 2024-03-04T07:27:03.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.269+0000 7f20eaffd640 1 -- 172.21.15.64:0/2060239506 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f20dc07e9d0 con 0x7f20f4144640 2024-03-04T07:27:03.269 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:03.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d406b4d0 msgr2=0x7f20d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d406b4d0 0x7f20d406d990 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f20ec00b800 tx=0x7f20ec00b3a0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20f4144640 msgr2=0x7f20f40c4910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20f4144640 0x7f20f40c4910 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f20dc0098c0 tx=0x7f20dc0046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 shutdown_connections 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d406b4d0 0x7f20d406d990 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20f415b670 0x7f20f40c6bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20f4144640 0x7f20f40c4910 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 --2- 172.21.15.64:0/2060239506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20f406e940 0x7f20f41504d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 >> 172.21.15.64:0/2060239506 conn(0x7f20f4066d00 msgr2=0x7f20f415c160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 shutdown_connections 2024-03-04T07:27:03.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.272+0000 7f20f88e1640 1 -- 172.21.15.64:0/2060239506 wait complete. 2024-03-04T07:27:03.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:03.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.388+0000 7fd7d0179640 1 Processor -- start 2024-03-04T07:27:03.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.388+0000 7fd7d0179640 1 -- start start 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.388+0000 7fd7d0179640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd7c815ae50 0x7fd7c80708e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.388+0000 7fd7d0179640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd7c8070e20 con 0x7fd7c815b230 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.388+0000 7fd7d0179640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd7c80b2e30 con 0x7fd7c8071870 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.388+0000 7fd7d0179640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7c806dfe0 con 0x7fd7c815ae50 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7cd714640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd7c815b230 0x7fd7c8071160 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42944/0 (socket says 172.21.15.64:42944) 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7cd714640 1 -- 172.21.15.64:0/1308299833 learned_addr learned my addr 172.21.15.64:0/1308299833 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7cdf15640 1 --2- 172.21.15.64:0/1308299833 >> v2:172.21.15.66:3300/0 conn(0x7fd7c815ae50 0x7fd7c80708e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7ccf13640 1 -- 172.21.15.64:0/1308299833 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1431901517 0 0) 0x7fd7c8070e20 con 0x7fd7c815b230 2024-03-04T07:27:03.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7ccf13640 1 -- 172.21.15.64:0/1308299833 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd7b4003520 con 0x7fd7c815b230 2024-03-04T07:27:03.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7ccf13640 1 -- 172.21.15.64:0/1308299833 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2386287480 0 0) 0x7fd7c0001490 con 0x7fd7c8071870 2024-03-04T07:27:03.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7ccf13640 1 -- 172.21.15.64:0/1308299833 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd7c8070e20 con 0x7fd7c8071870 2024-03-04T07:27:03.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7cdf15640 1 -- 172.21.15.64:0/1308299833 >> v1:172.21.15.66:6789/0 conn(0x7fd7c8071870 legacy=0x7fd7c8150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7cdf15640 1 -- 172.21.15.64:0/1308299833 >> v1:172.21.15.64:6789/0 conn(0x7fd7c815b230 legacy=0x7fd7c8071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.389+0000 7fd7cdf15640 1 -- 172.21.15.64:0/1308299833 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7c80b30b0 con 0x7fd7c815ae50 2024-03-04T07:27:03.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7cdf15640 1 --2- 172.21.15.64:0/1308299833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815ae50 0x7fd7c80708e0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fd7c400a940 tx=0x7fd7c402bc80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a51c6270c096bc92 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7ccf13640 1 -- 172.21.15.64:0/1308299833 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7c4039070 con 0x7fd7c815ae50 2024-03-04T07:27:03.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7ccf13640 1 -- 172.21.15.64:0/1308299833 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd7c402ca30 con 0x7fd7c815ae50 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7ccf13640 1 -- 172.21.15.64:0/1308299833 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7c400b040 con 0x7fd7c815ae50 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7d0179640 1 -- 172.21.15.64:0/1308299833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815ae50 msgr2=0x7fd7c80708e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7d0179640 1 --2- 172.21.15.64:0/1308299833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815ae50 0x7fd7c80708e0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fd7c400a940 tx=0x7fd7c402bc80 comp rx=0 tx=0).stop 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7d0179640 1 -- 172.21.15.64:0/1308299833 shutdown_connections 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7d0179640 1 --2- 172.21.15.64:0/1308299833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815ae50 0x7fd7c80708e0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.390+0000 7fd7d0179640 1 -- 172.21.15.64:0/1308299833 >> 172.21.15.64:0/1308299833 conn(0x7fd7c8066d00 msgr2=0x7fd7c8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.391+0000 7fd7d0179640 1 -- 172.21.15.64:0/1308299833 shutdown_connections 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.391+0000 7fd7d0179640 1 -- 172.21.15.64:0/1308299833 wait complete. 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 Processor -- start 2024-03-04T07:27:03.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 -- start start 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7c8071870 0x7fd7c8155750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7c815ae50 0x7fd7c814c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815b230 0x7fd7c8152260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7c80b3950 con 0x7fd7c815ae50 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7c80b3670 con 0x7fd7c815b230 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7d0179640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7c80b4150 con 0x7fd7c8071870 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.392+0000 7fd7cd714640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7c815ae50 0x7fd7c814c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cdf15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7c8071870 0x7fd7c8155750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7ce716640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815b230 0x7fd7c8152260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cd714640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7c815ae50 0x7fd7c814c760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41204/0 (socket says 172.21.15.64:41204) 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cd714640 1 -- 172.21.15.64:0/3020521598 learned_addr learned my addr 172.21.15.64:0/3020521598 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cdf15640 1 -- 172.21.15.64:0/3020521598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815b230 msgr2=0x7fd7c8152260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cdf15640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815b230 0x7fd7c8152260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cdf15640 1 -- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7c815ae50 msgr2=0x7fd7c814c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cdf15640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7c815ae50 0x7fd7c814c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cdf15640 1 -- 172.21.15.64:0/3020521598 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7c8153310 con 0x7fd7c8071870 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cd714640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7c815ae50 0x7fd7c814c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7cdf15640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7c8071870 0x7fd7c8155750 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fd7c4033000 tx=0x7fd7c40046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7be7fc640 1 -- 172.21.15.64:0/3020521598 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7c4039040 con 0x7fd7c8071870 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7be7fc640 1 -- 172.21.15.64:0/3020521598 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd7c4044070 con 0x7fd7c8071870 2024-03-04T07:27:03.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7be7fc640 1 -- 172.21.15.64:0/3020521598 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7c4035d30 con 0x7fd7c8071870 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd7c81527a0 con 0x7fd7c8071870 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.393+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd7c8152d50 con 0x7fd7c8071870 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.394+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd7c80733c0 con 0x7fd7c8071870 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.394+0000 7fd7be7fc640 1 -- 172.21.15.64:0/3020521598 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd7c4049460 con 0x7fd7c8071870 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.395+0000 7fd7be7fc640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd7ac06b4d0 0x7fd7ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.395+0000 7fd7be7fc640 1 -- 172.21.15.64:0/3020521598 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd7c40ac0f0 con 0x7fd7c8071870 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.395+0000 7fd7cd714640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd7ac06b4d0 0x7fd7ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.395+0000 7fd7cd714640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd7ac06b4d0 0x7fd7ac06d990 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7fd7c815e620 tx=0x7fd7b8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.399+0000 7fd7be7fc640 1 -- 172.21.15.64:0/3020521598 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd7c407a920 con 0x7fd7c8071870 2024-03-04T07:27:03.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.578+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fd7c804f410 con 0x7fd7c8071870 2024-03-04T07:27:03.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.578+0000 7fd7be7fc640 1 -- 172.21.15.64:0/3020521598 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7fd7c407e9c0 con 0x7fd7c8071870 2024-03-04T07:27:03.578 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd7ac06b4d0 msgr2=0x7fd7ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd7ac06b4d0 0x7fd7ac06d990 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7fd7c815e620 tx=0x7fd7b8009040 comp rx=0 tx=0).stop 2024-03-04T07:27:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7c8071870 msgr2=0x7fd7c8155750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7c8071870 0x7fd7c8155750 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fd7c4033000 tx=0x7fd7c40046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 shutdown_connections 2024-03-04T07:27:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd7ac06b4d0 0x7fd7ac06d990 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7c815b230 0x7fd7c8152260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7c815ae50 0x7fd7c814c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 --2- 172.21.15.64:0/3020521598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7c8071870 0x7fd7c8155750 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 >> 172.21.15.64:0/3020521598 conn(0x7fd7c8066d00 msgr2=0x7fd7c815cd00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 shutdown_connections 2024-03-04T07:27:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.580+0000 7fd7d0179640 1 -- 172.21.15.64:0/3020521598 wait complete. 2024-03-04T07:27:03.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:03.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.693+0000 7f73aacdf640 1 Processor -- start 2024-03-04T07:27:03.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.693+0000 7f73aacdf640 1 -- start start 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73aacdf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f73a415e000 0x7f73a4071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73aacdf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a4071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73aacdf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f73a40ce970 con 0x7f73a40b2c50 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73aacdf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f73a40cece0 con 0x7f73a415e000 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73aacdf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f73a40cef30 con 0x7f73a40d35e0 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a8a7b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f73a415e000 0x7f73a4071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a927c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a4071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a3fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f73a40b2c50 0x7f73a4161cb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:33058/0 (socket says 172.21.15.64:33058) 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a8a7b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f73a415e000 0x7f73a4071540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57254/0 (socket says 172.21.15.64:57254) 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a3fff640 1 -- 172.21.15.64:0/1455938520 learned_addr learned my addr 172.21.15.64:0/1455938520 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a8a7b640 1 -- 172.21.15.64:0/1455938520 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 msgr2=0x7f73a4071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a8a7b640 1 --2- 172.21.15.64:0/1455938520 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a4071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a8a7b640 1 -- 172.21.15.64:0/1455938520 >> v1:172.21.15.64:6789/0 conn(0x7f73a40b2c50 legacy=0x7f73a4161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a8a7b640 1 -- 172.21.15.64:0/1455938520 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73a40cf210 con 0x7f73a415e000 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a37fe640 1 -- 172.21.15.64:0/1455938520 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3382966060 0 0) 0x7f73a40ce970 con 0x7f73a40b2c50 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.694+0000 7f73a927c640 1 --2- 172.21.15.64:0/1455938520 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a4071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73a8a7b640 1 --2- 172.21.15.64:0/1455938520 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a4071540 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f73980097a0 tx=0x7f739802b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2312ecf2d2e38ce server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73a37fe640 1 -- 172.21.15.64:0/1455938520 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73980315a0 con 0x7f73a415e000 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73a37fe640 1 -- 172.21.15.64:0/1455938520 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7398037070 con 0x7f73a415e000 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73a37fe640 1 -- 172.21.15.64:0/1455938520 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7398031960 con 0x7f73a415e000 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73aacdf640 1 -- 172.21.15.64:0/1455938520 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 msgr2=0x7f73a4071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1455938520 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a4071540 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f73980097a0 tx=0x7f739802b840 comp rx=0 tx=0).stop 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73aacdf640 1 -- 172.21.15.64:0/1455938520 shutdown_connections 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1455938520 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a4071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1455938520 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a4071540 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.695+0000 7f73aacdf640 1 -- 172.21.15.64:0/1455938520 >> 172.21.15.64:0/1455938520 conn(0x7f73a40d3c80 msgr2=0x7f73a40d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.696+0000 7f73aacdf640 1 -- 172.21.15.64:0/1455938520 shutdown_connections 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.696+0000 7f73aacdf640 1 -- 172.21.15.64:0/1455938520 wait complete. 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 Processor -- start 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 -- start start 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73a40b2c50 0x7f73a40cd3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a41f3e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a41f5030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73a40b3320 con 0x7f73a415e000 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73a40b31a0 con 0x7f73a40b2c50 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.697+0000 7f73aacdf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f73a415e3e0 con 0x7f73a40d35e0 2024-03-04T07:27:03.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a41f5030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a41f5030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57262/0 (socket says 172.21.15.64:57262) 2024-03-04T07:27:03.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 -- 172.21.15.64:0/1980592648 learned_addr learned my addr 172.21.15.64:0/1980592648 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:03.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a8a7b640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73a40b2c50 0x7f73a40cd3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a3fff640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a41f3e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 -- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 msgr2=0x7f73a41f3e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a41f3e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 -- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73a40b2c50 msgr2=0x7f73a40cd3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73a40b2c50 0x7f73a40cd3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 -- 172.21.15.64:0/1980592648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73a40cd8f0 con 0x7f73a415e000 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a927c640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a41f5030 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f739c00efc0 tx=0x7f739c00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a17fa640 1 -- 172.21.15.64:0/1980592648 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f739c018020 con 0x7f73a415e000 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a17fa640 1 -- 172.21.15.64:0/1980592648 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f739c014480 con 0x7f73a415e000 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f73a41f4850 con 0x7f73a415e000 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73a17fa640 1 -- 172.21.15.64:0/1980592648 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f739c014870 con 0x7f73a415e000 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.698+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f73a40cae80 con 0x7f73a415e000 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.699+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f73a4067ff0 con 0x7f73a415e000 2024-03-04T07:27:03.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.700+0000 7f73a17fa640 1 -- 172.21.15.64:0/1980592648 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f739c005d10 con 0x7f73a415e000 2024-03-04T07:27:03.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.700+0000 7f73a17fa640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738806b4d0 0x7f738806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:03.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.700+0000 7f73a17fa640 1 -- 172.21.15.64:0/1980592648 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f739c08adc0 con 0x7f73a415e000 2024-03-04T07:27:03.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.700+0000 7f73a8a7b640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738806b4d0 0x7f738806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:03.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.700+0000 7f73a8a7b640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738806b4d0 0x7f738806d990 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f7398036000 tx=0x7f739803d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:03.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.704+0000 7f73a17fa640 1 -- 172.21.15.64:0/1980592648 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f739c022030 con 0x7f73a415e000 2024-03-04T07:27:03.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.922+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f73a4162640 con 0x7f73a415e000 2024-03-04T07:27:03.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.923+0000 7f73a17fa640 1 -- 172.21.15.64:0/1980592648 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f739c010040 con 0x7f73a415e000 2024-03-04T07:27:03.923 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:03.923 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:27:03.923 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:27:03.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.924+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738806b4d0 msgr2=0x7f738806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.924+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738806b4d0 0x7f738806d990 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f7398036000 tx=0x7f739803d040 comp rx=0 tx=0).stop 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.924+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 msgr2=0x7f73a41f5030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a41f5030 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f739c00efc0 tx=0x7f739c00c490 comp rx=0 tx=0).stop 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 shutdown_connections 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738806b4d0 0x7f738806d990 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f73a415e000 0x7f73a41f5030 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73a40d35e0 0x7f73a41f3e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 --2- 172.21.15.64:0/1980592648 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73a40b2c50 0x7f73a40cd3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 >> 172.21.15.64:0/1980592648 conn(0x7f73a40d3c80 msgr2=0x7f73a40d1b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 shutdown_connections 2024-03-04T07:27:03.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:03.925+0000 7f73aacdf640 1 -- 172.21.15.64:0/1980592648 wait complete. 2024-03-04T07:27:03.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:04.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.037+0000 7f7b9971e640 1 Processor -- start 2024-03-04T07:27:04.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.037+0000 7f7b9971e640 1 -- start start 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b9971e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b940d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b9971e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7b940b9a30 0x7f7b940d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b9971e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7b940cea30 con 0x7f7b940b95a0 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b9971e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7b940d2af0 con 0x7f7b940b9a30 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b9971e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7b940cf3c0 con 0x7f7b9415df00 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b937fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7b940b9a30 0x7f7b940d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b927fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7b940b95a0 0x7f7b940b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:33064/0 (socket says 172.21.15.64:33064) 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b927fc640 1 -- 172.21.15.64:0/1322520179 learned_addr learned my addr 172.21.15.64:0/1322520179 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b92ffd640 1 --2- 172.21.15.64:0/1322520179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b940d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b937fe640 1 -- 172.21.15.64:0/1322520179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 msgr2=0x7f7b940d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b937fe640 1 --2- 172.21.15.64:0/1322520179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b940d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b937fe640 1 -- 172.21.15.64:0/1322520179 >> v1:172.21.15.64:6789/0 conn(0x7f7b940b95a0 legacy=0x7f7b940b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b91ffb640 1 -- 172.21.15.64:0/1322520179 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1032951431 0 0) 0x7f7b940cea30 con 0x7f7b940b95a0 2024-03-04T07:27:04.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.038+0000 7f7b937fe640 1 -- 172.21.15.64:0/1322520179 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b940cf0e0 con 0x7f7b940b9a30 2024-03-04T07:27:04.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.039+0000 7f7b937fe640 1 --2- 172.21.15.64:0/1322520179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940b9a30 0x7f7b940d3220 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f7b84009d90 tx=0x7f7b8402bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=196693a05532299a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:04.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.039+0000 7f7b91ffb640 1 -- 172.21.15.64:0/1322520179 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b84039070 con 0x7f7b940b9a30 2024-03-04T07:27:04.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.039+0000 7f7b91ffb640 1 -- 172.21.15.64:0/1322520179 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7b8402cc50 con 0x7f7b940b9a30 2024-03-04T07:27:04.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.039+0000 7f7b91ffb640 1 -- 172.21.15.64:0/1322520179 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b84034440 con 0x7f7b940b9a30 2024-03-04T07:27:04.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 -- 172.21.15.64:0/1322520179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940b9a30 msgr2=0x7f7b940d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 --2- 172.21.15.64:0/1322520179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940b9a30 0x7f7b940d3220 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f7b84009d90 tx=0x7f7b8402bea0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 -- 172.21.15.64:0/1322520179 shutdown_connections 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 --2- 172.21.15.64:0/1322520179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940b9a30 0x7f7b940d3220 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 --2- 172.21.15.64:0/1322520179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b940d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 -- 172.21.15.64:0/1322520179 >> 172.21.15.64:0/1322520179 conn(0x7f7b94071960 msgr2=0x7f7b94071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 -- 172.21.15.64:0/1322520179 shutdown_connections 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.040+0000 7f7b9971e640 1 -- 172.21.15.64:0/1322520179 wait complete. 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.041+0000 7f7b9971e640 1 Processor -- start 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.041+0000 7f7b9971e640 1 -- start start 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b9971e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7b940b95a0 0x7f7b941f3c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b9971e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b94151f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b9971e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940c9510 0x7f7b940b5670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b9971e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b940cec70 con 0x7f7b940c9510 2024-03-04T07:27:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b9971e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b940ce750 con 0x7f7b940b95a0 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b9971e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7b940cedf0 con 0x7f7b9415df00 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940c9510 0x7f7b940b5670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940c9510 0x7f7b940b5670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57288/0 (socket says 172.21.15.64:57288) 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 -- 172.21.15.64:0/3545644704 learned_addr learned my addr 172.21.15.64:0/3545644704 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b927fc640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b94151f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b92ffd640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7b940b95a0 0x7f7b941f3c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 -- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 msgr2=0x7f7b94151f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b94151f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 -- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7b940b95a0 msgr2=0x7f7b941f3c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7b940b95a0 0x7f7b941f3c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 -- 172.21.15.64:0/3545644704 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b940b3020 con 0x7f7b940c9510 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b937fe640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940c9510 0x7f7b940b5670 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f7b84009d90 tx=0x7f7b84004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:04.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b7b7fe640 1 -- 172.21.15.64:0/3545644704 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b84039050 con 0x7f7b940c9510 2024-03-04T07:27:04.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.042+0000 7f7b7b7fe640 1 -- 172.21.15.64:0/3545644704 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7b84045070 con 0x7f7b940c9510 2024-03-04T07:27:04.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.043+0000 7f7b7b7fe640 1 -- 172.21.15.64:0/3545644704 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b84009450 con 0x7f7b940c9510 2024-03-04T07:27:04.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.043+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7b940b32c0 con 0x7f7b940c9510 2024-03-04T07:27:04.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.043+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7b940b3880 con 0x7f7b940c9510 2024-03-04T07:27:04.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.044+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7b94067ff0 con 0x7f7b940c9510 2024-03-04T07:27:04.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.044+0000 7f7b7b7fe640 1 -- 172.21.15.64:0/3545644704 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7b8403e610 con 0x7f7b940c9510 2024-03-04T07:27:04.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.044+0000 7f7b7b7fe640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7b8006b4d0 0x7f7b8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.045+0000 7f7b7b7fe640 1 -- 172.21.15.64:0/3545644704 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7b840aca40 con 0x7f7b940c9510 2024-03-04T07:27:04.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.045+0000 7f7b92ffd640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7b8006b4d0 0x7f7b8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.045+0000 7f7b92ffd640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7b8006b4d0 0x7f7b8006d990 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f7b9415c900 tx=0x7f7b8c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:04.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.049+0000 7f7b7b7fe640 1 -- 172.21.15.64:0/3545644704 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7b8407b300 con 0x7f7b940c9510 2024-03-04T07:27:04.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.230+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7b940c92f0 con 0x7f7b940c9510 2024-03-04T07:27:04.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.231+0000 7f7b7b7fe640 1 -- 172.21.15.64:0/3545644704 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f7b840ace20 con 0x7f7b940c9510 2024-03-04T07:27:04.231 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:04.231 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:04.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7b8006b4d0 msgr2=0x7f7b8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7b8006b4d0 0x7f7b8006d990 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f7b9415c900 tx=0x7f7b8c009190 comp rx=0 tx=0).stop 2024-03-04T07:27:04.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940c9510 msgr2=0x7f7b940b5670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940c9510 0x7f7b940b5670 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f7b84009d90 tx=0x7f7b84004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:04.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 shutdown_connections 2024-03-04T07:27:04.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7b8006b4d0 0x7f7b8006d990 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7b940c9510 0x7f7b940b5670 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7b9415df00 0x7f7b94151f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 --2- 172.21.15.64:0/3545644704 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7b940b95a0 0x7f7b941f3c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 >> 172.21.15.64:0/3545644704 conn(0x7f7b94071960 msgr2=0x7f7b940b8b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:04.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 shutdown_connections 2024-03-04T07:27:04.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.233+0000 7f7b9971e640 1 -- 172.21.15.64:0/3545644704 wait complete. 2024-03-04T07:27:04.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:04.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.346+0000 7f813d404640 1 Processor -- start 2024-03-04T07:27:04.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.346+0000 7f813d404640 1 -- start start 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f813d404640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f813815aa90 0x7f813806d810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f813d404640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81380b5850 0x7f813806ef40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f813d404640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81380d03c0 con 0x7f8138071810 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f813d404640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f813806eab0 con 0x7f81380b5850 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f813d404640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f813806dea0 con 0x7f813815aa90 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f81367fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81380b5850 0x7f813806ef40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f81377fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8138071810 0x7f81380cd920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:33076/0 (socket says 172.21.15.64:33076) 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f81377fe640 1 -- 172.21.15.64:0/1955873437 learned_addr learned my addr 172.21.15.64:0/1955873437 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f81367fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81380b5850 0x7f813806ef40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57302/0 (socket says 172.21.15.64:57302) 2024-03-04T07:27:04.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.347+0000 7f8136ffd640 1 --2- 172.21.15.64:0/1955873437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f813815aa90 0x7f813806d810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f81367fc640 1 -- 172.21.15.64:0/1955873437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f813815aa90 msgr2=0x7f813806d810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f81367fc640 1 --2- 172.21.15.64:0/1955873437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f813815aa90 0x7f813806d810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f81367fc640 1 -- 172.21.15.64:0/1955873437 >> v1:172.21.15.64:6789/0 conn(0x7f8138071810 legacy=0x7f81380cd920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f81367fc640 1 -- 172.21.15.64:0/1955873437 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f813806e180 con 0x7f81380b5850 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f81367fc640 1 --2- 172.21.15.64:0/1955873437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f813806ef40 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f812c0097a0 tx=0x7f812c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80dc1a61e121d467 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f8135ffb640 1 -- 172.21.15.64:0/1955873437 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f812c035020 con 0x7f81380b5850 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f8135ffb640 1 -- 172.21.15.64:0/1955873437 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f812c031830 con 0x7f81380b5850 2024-03-04T07:27:04.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.348+0000 7f8135ffb640 1 -- 172.21.15.64:0/1955873437 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f812c031c00 con 0x7f81380b5850 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 -- 172.21.15.64:0/1955873437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 msgr2=0x7f813806ef40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 --2- 172.21.15.64:0/1955873437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f813806ef40 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f812c0097a0 tx=0x7f812c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 -- 172.21.15.64:0/1955873437 shutdown_connections 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 --2- 172.21.15.64:0/1955873437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f813806ef40 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 --2- 172.21.15.64:0/1955873437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f813815aa90 0x7f813806d810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 -- 172.21.15.64:0/1955873437 >> 172.21.15.64:0/1955873437 conn(0x7f81380d07f0 msgr2=0x7f81380d0c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 -- 172.21.15.64:0/1955873437 shutdown_connections 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.349+0000 7f813d404640 1 -- 172.21.15.64:0/1955873437 wait complete. 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.350+0000 7f813d404640 1 Processor -- start 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.350+0000 7f813d404640 1 -- start start 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f813d404640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8138071810 0x7f813814b770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f813d404640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f81380ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f813d404640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f813815aa90 0x7f81380b0df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f813d404640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f813806f480 con 0x7f81380b5850 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f813d404640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f813806e740 con 0x7f813815aa90 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f813d404640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8138071ca0 con 0x7f8138071810 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f81380ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f81380ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57308/0 (socket says 172.21.15.64:57308) 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 -- 172.21.15.64:0/2058615437 learned_addr learned my addr 172.21.15.64:0/2058615437 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f8136ffd640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8138071810 0x7f813814b770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81377fe640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f813815aa90 0x7f81380b0df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 -- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8138071810 msgr2=0x7f813814b770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8138071810 0x7f813814b770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 -- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f813815aa90 msgr2=0x7f81380b0df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f813815aa90 0x7f81380b0df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.351+0000 7f81367fc640 1 -- 172.21.15.64:0/2058615437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81380ae290 con 0x7f81380b5850 2024-03-04T07:27:04.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.352+0000 7f81367fc640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f81380ad8b0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f812c02bee0 tx=0x7f812c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:04.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.352+0000 7f811f7fe640 1 -- 172.21.15.64:0/2058615437 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f812c035040 con 0x7f81380b5850 2024-03-04T07:27:04.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.352+0000 7f811f7fe640 1 -- 172.21.15.64:0/2058615437 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f812c03b920 con 0x7f81380b5850 2024-03-04T07:27:04.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.352+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81380ae570 con 0x7f81380b5850 2024-03-04T07:27:04.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.352+0000 7f811f7fe640 1 -- 172.21.15.64:0/2058615437 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f812c03bd10 con 0x7f81380b5850 2024-03-04T07:27:04.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.352+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81380af4f0 con 0x7f81380b5850 2024-03-04T07:27:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.353+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f813804f330 con 0x7f81380b5850 2024-03-04T07:27:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.353+0000 7f811f7fe640 1 -- 172.21.15.64:0/2058615437 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f812c038020 con 0x7f81380b5850 2024-03-04T07:27:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.353+0000 7f811f7fe640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f812406b4d0 0x7f812406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.354+0000 7f811f7fe640 1 -- 172.21.15.64:0/2058615437 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f812c0ac190 con 0x7f81380b5850 2024-03-04T07:27:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.354+0000 7f8136ffd640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f812406b4d0 0x7f812406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.354+0000 7f8136ffd640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f812406b4d0 0x7f812406d990 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f8128009aa0 tx=0x7f8128009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:04.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.358+0000 7f811f7fe640 1 -- 172.21.15.64:0/2058615437 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f812c031800 con 0x7f81380b5850 2024-03-04T07:27:04.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.576+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f81380cce30 con 0x7f81380b5850 2024-03-04T07:27:04.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.577+0000 7f811f7fe640 1 -- 172.21.15.64:0/2058615437 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f812c07a9d0 con 0x7f81380b5850 2024-03-04T07:27:04.576 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:04.577 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:27:04.577 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f812406b4d0 msgr2=0x7f812406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f812406b4d0 0x7f812406d990 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f8128009aa0 tx=0x7f8128009190 comp rx=0 tx=0).stop 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 msgr2=0x7f81380ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f81380ad8b0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f812c02bee0 tx=0x7f812c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 shutdown_connections 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f812406b4d0 0x7f812406d990 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f813815aa90 0x7f81380b0df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81380b5850 0x7f81380ad8b0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 --2- 172.21.15.64:0/2058615437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8138071810 0x7f813814b770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 >> 172.21.15.64:0/2058615437 conn(0x7f81380d07f0 msgr2=0x7f81380b3e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 shutdown_connections 2024-03-04T07:27:04.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:04.579+0000 7f813d404640 1 -- 172.21.15.64:0/2058615437 wait complete. 2024-03-04T07:27:04.991 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 7, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:26:52.660771+0000', 'modified': '2024-03-04T07:26:55.666036+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 4293}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_4293': {'gid': 4293, 'name': 'b', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6835/1775037311', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 1775037311}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 1775037311}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 4293, 'qdb_cluster': [4293]} 2024-03-04T07:27:04.992 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:27:04.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-03-04T07:27:05.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.099+0000 7f0993ca3640 1 Processor -- start 2024-03-04T07:27:05.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.099+0000 7f0993ca3640 1 -- start start 2024-03-04T07:27:05.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0993ca3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c15f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0993ca3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f098c0c8420 0x7f098c0b0af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0993ca3640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f098c06c730 con 0x7f098c15ba00 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0993ca3640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f098c070800 con 0x7f098c0c8420 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0993ca3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f098c06cf70 con 0x7f098c15bde0 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0992240640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f098c0c8420 0x7f098c0b0af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0991a3f640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f098c15ba00 0x7f098c070c90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:41172/0 (socket says 172.21.15.64:41172) 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f099123e640 1 --2- 172.21.15.64:0/1591631919 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c15f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.100+0000 7f0991a3f640 1 -- 172.21.15.64:0/1591631919 learned_addr learned my addr 172.21.15.64:0/1591631919 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.101+0000 7f0992240640 1 -- 172.21.15.64:0/1591631919 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 msgr2=0x7f098c15f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.101+0000 7f0992240640 1 --2- 172.21.15.64:0/1591631919 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c15f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.101+0000 7f0992240640 1 -- 172.21.15.64:0/1591631919 >> v1:172.21.15.66:6789/0 conn(0x7f098c15ba00 legacy=0x7f098c070c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.101+0000 7f0992240640 1 -- 172.21.15.64:0/1591631919 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f098c06c9b0 con 0x7f098c0c8420 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.101+0000 7f0990a3d640 1 -- 172.21.15.64:0/1591631919 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3178237866 0 0) 0x7f0984001490 con 0x7f098c15ba00 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.101+0000 7f0992240640 1 --2- 172.21.15.64:0/1591631919 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c0c8420 0x7f098c0b0af0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f097c0097a0 tx=0x7f097c02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=95acd3873435e0be server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.101+0000 7f0990a3d640 1 -- 172.21.15.64:0/1591631919 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f097c0315a0 con 0x7f098c0c8420 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0990a3d640 1 -- 172.21.15.64:0/1591631919 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f097c037070 con 0x7f098c0c8420 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0990a3d640 1 -- 172.21.15.64:0/1591631919 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f097c031960 con 0x7f098c0c8420 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 -- 172.21.15.64:0/1591631919 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c0c8420 msgr2=0x7f098c0b0af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 --2- 172.21.15.64:0/1591631919 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c0c8420 0x7f098c0b0af0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f097c0097a0 tx=0x7f097c02b840 comp rx=0 tx=0).stop 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 -- 172.21.15.64:0/1591631919 shutdown_connections 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 --2- 172.21.15.64:0/1591631919 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c0c8420 0x7f098c0b0af0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 --2- 172.21.15.64:0/1591631919 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c15f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 -- 172.21.15.64:0/1591631919 >> 172.21.15.64:0/1591631919 conn(0x7f098c0b3870 msgr2=0x7f098c0b3c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 -- 172.21.15.64:0/1591631919 shutdown_connections 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.102+0000 7f0993ca3640 1 -- 172.21.15.64:0/1591631919 wait complete. 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.103+0000 7f0993ca3640 1 Processor -- start 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.103+0000 7f0993ca3640 1 -- start start 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0993ca3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f098c0c8420 0x7f098c14e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0993ca3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c15ba00 0x7f098c0b6a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0993ca3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c0ae0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0993ca3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f098c0b1030 con 0x7f098c15ba00 2024-03-04T07:27:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0993ca3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f098c070490 con 0x7f098c0c8420 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0993ca3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f098c0b11b0 con 0x7f098c15bde0 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c15ba00 0x7f098c0b6a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c15ba00 0x7f098c0b6a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57314/0 (socket says 172.21.15.64:57314) 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 -- 172.21.15.64:0/3613477905 learned_addr learned my addr 172.21.15.64:0/3613477905 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f098c0c8420 0x7f098c14e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0992240640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c0ae0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 -- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 msgr2=0x7f098c0ae0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c0ae0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 -- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f098c0c8420 msgr2=0x7f098c14e7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f098c0c8420 0x7f098c14e7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 -- 172.21.15.64:0/3613477905 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f098c0ae5e0 con 0x7f098c15ba00 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f098c0c8420 0x7f098c14e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.104+0000 7f099123e640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c15ba00 0x7f098c0b6a50 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f098000e8a0 tx=0x7f098000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.105+0000 7f098a7fc640 1 -- 172.21.15.64:0/3613477905 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0980017070 con 0x7f098c15ba00 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.105+0000 7f098a7fc640 1 -- 172.21.15.64:0/3613477905 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f098000c940 con 0x7f098c15ba00 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.105+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f098c15c780 con 0x7f098c15ba00 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.105+0000 7f098a7fc640 1 -- 172.21.15.64:0/3613477905 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f098000cd30 con 0x7f098c15ba00 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.105+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f098c15cd40 con 0x7f098c15ba00 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.106+0000 7f098a7fc640 1 -- 172.21.15.64:0/3613477905 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f098001a070 con 0x7f098c15ba00 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.107+0000 7f098a7fc640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f097806b500 0x7f097806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.107+0000 7f098a7fc640 1 -- 172.21.15.64:0/3613477905 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0980089fd0 con 0x7f098c15ba00 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.107+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f097806b500 0x7f097806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.107+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f097806b500 0x7f097806d9c0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f09840012e0 tx=0x7f0984007ed0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 Processor -- start 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 -- start start 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ba870 0x5643f96bac50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96be8c0 0x5643f96c0360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f96eaa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643f96246b0 con 0x5643f96ea600 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643f95cda60 con 0x5643f96be8c0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5643f9620750 con 0x5643f96ba870 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0992240640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f96eaa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0992240640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f96eaa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57322/0 (socket says 172.21.15.64:57322) 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f099123e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96be8c0 0x5643f96c0360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0992240640 1 -- 172.21.15.64:0/994959105 learned_addr learned my addr 172.21.15.64:0/994959105 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.108+0000 7f0991a3f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ba870 0x5643f96bac50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f099123e640 1 -- 172.21.15.64:0/994959105 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ba870 msgr2=0x5643f96bac50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f099123e640 1 --2- 172.21.15.64:0/994959105 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ba870 0x5643f96bac50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f099123e640 1 -- 172.21.15.64:0/994959105 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 msgr2=0x5643f96eaa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f099123e640 1 --2- 172.21.15.64:0/994959105 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f96eaa80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f099123e640 1 -- 172.21.15.64:0/994959105 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5643f95ebf30 con 0x5643f96be8c0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f099123e640 1 --2- 172.21.15.64:0/994959105 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96be8c0 0x5643f96c0360 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f09800584f0 tx=0x7f0980058150 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ef985ff9bb2c3285 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f09737fe640 1 -- 172.21.15.64:0/994959105 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0980021040 con 0x5643f96be8c0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f09737fe640 1 -- 172.21.15.64:0/994959105 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0980081e50 con 0x5643f96be8c0 2024-03-04T07:27:05.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.109+0000 7f09737fe640 1 -- 172.21.15.64:0/994959105 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0980060e50 con 0x5643f96be8c0 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 -- 172.21.15.64:0/994959105 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96be8c0 msgr2=0x5643f96c0360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 --2- 172.21.15.64:0/994959105 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96be8c0 0x5643f96c0360 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f09800584f0 tx=0x7f0980058150 comp rx=0 tx=0).stop 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 -- 172.21.15.64:0/994959105 shutdown_connections 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 --2- 172.21.15.64:0/994959105 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f96eaa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 --2- 172.21.15.64:0/994959105 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96be8c0 0x5643f96c0360 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 --2- 172.21.15.64:0/994959105 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ba870 0x5643f96bac50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 -- 172.21.15.64:0/994959105 >> 172.21.15.64:0/994959105 conn(0x5643f96bc8f0 msgr2=0x5643f96be1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 -- 172.21.15.64:0/994959105 shutdown_connections 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 -- 172.21.15.64:0/994959105 wait complete. 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 -- start start 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.110+0000 7f0996885740 1 Processor -- start 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96ea600 0x5643f96c0610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0996885740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96bf210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f9716500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0996885740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643f96bed50 con 0x5643f96dfb80 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643f95e0ea0 con 0x5643f96ea600 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5643f96bf030 con 0x5643f96bf750 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0996885740 10 client.0 _ll_register_callbacks cb 0x5643f95e0870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f099123e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96bf210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f099123e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96bf210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57334/0 (socket says 172.21.15.64:57334) 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f099123e640 1 -- 172.21.15.64:0/3325064195 learned_addr learned my addr 172.21.15.64:0/3325064195 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96ea600 0x5643f96c0610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.111+0000 7f0992240640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f9716500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f099123e640 1 -- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 msgr2=0x5643f9716500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f099123e640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f9716500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f099123e640 1 -- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96ea600 msgr2=0x5643f96c0610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f099123e640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96ea600 0x5643f96c0610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f099123e640 1 -- 172.21.15.64:0/3325064195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5643f95e1270 con 0x5643f96dfb80 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96ea600 0x5643f96c0610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f099123e640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96bf210 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f09800584f0 tx=0x7f0980000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=40514fd03359cab2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0970ff9640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0970ff9640 1 -- 172.21.15.64:0/3325064195 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0980075060 con 0x5643f96dfb80 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0980081e50 con 0x5643f96dfb80 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0970ff9640 1 -- 172.21.15.64:0/3325064195 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f09800701d0 con 0x5643f96dfb80 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0970ff9640 1 -- 172.21.15.64:0/3325064195 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=7) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0980087030 con 0x5643f96dfb80 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 10 client.4632 fetch_fsmap learned FSMap version 7 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=7,monmap=2+}) v3 -- 0x5643f93392d0 con 0x5643f96dfb80 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0970ff9640 1 -- 172.21.15.64:0/3325064195 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0980081e50 con 0x5643f96dfb80 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0970ff9640 1 -- 172.21.15.64:0/3325064195 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 7) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f0980024060 con 0x5643f96dfb80 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 10 client.4632 fetch_fsmap finished waiting for FSMap version 7 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 10 client.4632 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 20 client.4632 populate_metadata read hostname 'smithi064' 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96e1960 0x5643f96e1d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 4 client.4632 mds_command: new command op to 4293 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.112+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 --> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5643f9631bf0 con 0x5643f96e1960 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.113+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96e1960 0x5643f96e1d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.113+0000 7f0970ff9640 10 client.4632 ms_handle_connect on v2:172.21.15.66:6833/1775037311 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.113+0000 7f0991a3f640 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96e1960 0x5643f96e1d40 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2777a35f212f823e server_cookie=622fc85f555a356b in_seq=0 out_seq=0 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0970ff9640 1 -- 172.21.15.64:0/3325064195 <== mds.0 v2:172.21.15.66:6833/1775037311 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x5643f9631bf0 con 0x5643f96e1960 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0970ff9640 10 client.4632 handle_command_reply: tid=0 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 client.4632 shutdown 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 msgr2=0x5643f96bf210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96bf210 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f09800584f0 tx=0x7f0980000bc0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 shutdown_connections 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96e1960 0x5643f96e1d40 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f9716500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96bf210 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0970ff9640 0 client.4632 ms_handle_reset on v2:172.21.15.66:6833/1775037311 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 --2- 172.21.15.64:0/3325064195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96ea600 0x5643f96c0610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 >> 172.21.15.64:0/3325064195 conn(0x5643f96ba870 msgr2=0x5643f96b9510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 shutdown_connections 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 1 -- 172.21.15.64:0/3325064195 wait complete. 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.114+0000 7f0996885740 20 client.4632 trim_cache size 0 max 16384 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 Processor -- start 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 -- start start 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96bf750 0x5643f96ed680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96e2d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ea600 0x5643f96e3260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f098c1ff370 con 0x5643f96dfb80 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f098c1f73a0 con 0x5643f96bf750 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.131+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f098c1f79a0 con 0x5643f96ea600 2024-03-04T07:27:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96e2d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0991a3f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96bf750 0x5643f96ed680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96e2d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57340/0 (socket says 172.21.15.64:57340) 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 -- 172.21.15.64:0/4017798344 learned_addr learned my addr 172.21.15.64:0/4017798344 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0992240640 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ea600 0x5643f96e3260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 -- 172.21.15.64:0/4017798344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ea600 msgr2=0x5643f96e3260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ea600 0x5643f96e3260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 -- 172.21.15.64:0/4017798344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96bf750 msgr2=0x5643f96ed680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96bf750 0x5643f96ed680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 -- 172.21.15.64:0/4017798344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5643f9716870 con 0x5643f96dfb80 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f099123e640 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96e2d20 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f09800773d0 tx=0x7f0980083000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8aff4c7751ef5984 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0972ffd640 1 -- 172.21.15.64:0/4017798344 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0980021040 con 0x5643f96dfb80 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0972ffd640 1 -- 172.21.15.64:0/4017798344 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0980081e50 con 0x5643f96dfb80 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0972ffd640 1 -- 172.21.15.64:0/4017798344 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0980060e50 con 0x5643f96dfb80 2024-03-04T07:27:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0996885740 1 -- 172.21.15.64:0/4017798344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 msgr2=0x5643f96e2d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0996885740 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96e2d20 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f09800773d0 tx=0x7f0980083000 comp rx=0 tx=0).stop 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0996885740 1 -- 172.21.15.64:0/4017798344 shutdown_connections 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0996885740 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96ea600 0x5643f96e3260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0996885740 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96dfb80 0x5643f96e2d20 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0996885740 1 --2- 172.21.15.64:0/4017798344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96bf750 0x5643f96ed680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.132+0000 7f0996885740 1 -- 172.21.15.64:0/4017798344 >> 172.21.15.64:0/4017798344 conn(0x5643f96ba870 msgr2=0x5643f96c0420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.133+0000 7f0996885740 1 -- 172.21.15.64:0/4017798344 shutdown_connections 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.133+0000 7f0996885740 1 -- 172.21.15.64:0/4017798344 wait complete. 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.133+0000 7f0996885740 1 -- start start 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.133+0000 7f0996885740 1 Processor -- start 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f96e2aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0996885740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96dfb80 0x5643f96e2fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0996885740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f97773a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0996885740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643f96bed50 con 0x5643f96ea600 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643f95e0ea0 con 0x5643f96dfb80 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0996885740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5643f96e27d0 con 0x5643f96bf750 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0996885740 10 client.0 _ll_register_callbacks cb 0x7f0980058650 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f97773a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f97773a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57346/0 (socket says 172.21.15.64:57346) 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 -- 172.21.15.64:0/403616055 learned_addr learned my addr 172.21.15.64:0/403616055 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0991a3f640 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f96e2aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f099123e640 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96dfb80 0x5643f96e2fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 -- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 msgr2=0x5643f96e2aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f96e2aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 -- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96dfb80 msgr2=0x5643f96e2fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96dfb80 0x5643f96e2fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.134+0000 7f0992240640 1 -- 172.21.15.64:0/403616055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5643f94036b0 con 0x5643f96ea600 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0992240640 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f97773a0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f097c0098c0 tx=0x7f097c02e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=91c5589b1f654edf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0970ff9640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0970ff9640 1 -- 172.21.15.64:0/403616055 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f097c04c070 con 0x5643f96ea600 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0970ff9640 1 -- 172.21.15.64:0/403616055 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f097c003d50 con 0x5643f96ea600 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0980081e50 con 0x5643f96ea600 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0970ff9640 1 -- 172.21.15.64:0/403616055 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=7) v2 ==== 24+0+0 (secure 0 0 0) 0x7f097c057050 con 0x5643f96ea600 2024-03-04T07:27:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 10 client.4638 fetch_fsmap learned FSMap version 7 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=7,monmap=2+}) v3 -- 0x5643f9709620 con 0x5643f96ea600 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0970ff9640 1 -- 172.21.15.64:0/403616055 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0980081e50 con 0x5643f96ea600 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0970ff9640 1 -- 172.21.15.64:0/403616055 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 7) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f097c05a020 con 0x5643f96ea600 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 10 client.4638 fetch_fsmap finished waiting for FSMap version 7 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 10 client.4638 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 20 client.4638 populate_metadata read hostname 'smithi064' 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96de430 0x5643f96de810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 4 client.4638 mds_command: new command op to 4293 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 --> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5643f96ee380 con 0x5643f96de430 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.135+0000 7f0991a3f640 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96de430 0x5643f96de810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0991a3f640 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96de430 0x5643f96de810 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ba3dbf91eee3e26e server_cookie=ed0597cee0509419 in_seq=0 out_seq=0 2024-03-04T07:27:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0970ff9640 10 client.4638 ms_handle_connect on v2:172.21.15.66:6833/1775037311 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0970ff9640 1 -- 172.21.15.64:0/403616055 <== mds.0 v2:172.21.15.66:6833/1775037311 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+324 (crc 0 0 0) 0x5643f96ee380 con 0x5643f96de430 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0970ff9640 10 client.4638 handle_command_reply: tid=0 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 client.4638 shutdown 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 msgr2=0x5643f97773a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f97773a0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f097c0098c0 tx=0x7f097c02e000 comp rx=0 tx=0).stop 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 shutdown_connections 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5643f96ea600 0x5643f97773a0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5643f96dfb80 0x5643f96e2fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:6833/1775037311,v1:172.21.15.66:6835/1775037311] conn(0x5643f96de430 0x5643f96de810 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 --2- 172.21.15.64:0/403616055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5643f96bf750 0x5643f96e2aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0970ff9640 0 client.4638 ms_handle_reset on v2:172.21.15.66:6833/1775037311 2024-03-04T07:27:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 >> 172.21.15.64:0/403616055 conn(0x5643f96ba870 msgr2=0x5643f96e7300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 shutdown_connections 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 1 -- 172.21.15.64:0/403616055 wait complete. 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.136+0000 7f0996885740 20 client.4638 trim_cache size 0 max 16384 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 4293, 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:27:05.211 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:27:05.212 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:27:05.212 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 11.820907731, 2024-03-04T07:27:05.212 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 7, 2024-03-04T07:27:05.212 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 20, 2024-03-04T07:27:05.212 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 20, 2024-03-04T07:27:05.212 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 18.416201144999999 2024-03-04T07:27:05.212 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f097806b500 msgr2=0x7f097806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f097806b500 0x7f097806d9c0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f09840012e0 tx=0x7f0984007ed0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c15ba00 msgr2=0x7f098c0b6a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c15ba00 0x7f098c0b6a50 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f098000e8a0 tx=0x7f098000ed60 comp rx=0 tx=0).stop 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 shutdown_connections 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f097806b500 0x7f097806d9c0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f098c15bde0 0x7f098c0ae0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f098c15ba00 0x7f098c0b6a50 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 --2- 172.21.15.64:0/3613477905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f098c0c8420 0x7f098c14e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 >> 172.21.15.64:0/3613477905 conn(0x7f098c0b3870 msgr2=0x7f098c0b2de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 shutdown_connections 2024-03-04T07:27:05.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.137+0000 7f0993ca3640 1 -- 172.21.15.64:0/3613477905 wait complete. 2024-03-04T07:27:05.215 INFO:teuthology.run_tasks:Running task ceph-fuse... 2024-03-04T07:27:05.242 INFO:tasks.ceph_fuse:Running ceph_fuse task... 2024-03-04T07:27:05.242 INFO:tasks.ceph_fuse:config is {'client.0': {}, 'client.2': {}, 'client.1': {}, 'client.3': {}} 2024-03-04T07:27:05.242 INFO:tasks.ceph_fuse:client.0 config is {} 2024-03-04T07:27:05.242 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-04T07:27:05.242 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:05.242 INFO:tasks.ceph_fuse:client.2 config is {} 2024-03-04T07:27:05.242 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-04T07:27:05.243 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:05.243 INFO:tasks.ceph_fuse:client.1 config is {} 2024-03-04T07:27:05.243 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-04T07:27:05.243 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:05.243 INFO:tasks.ceph_fuse:client.3 config is {} 2024-03-04T07:27:05.243 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-04T07:27:05.243 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:05.243 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.243 DEBUG:teuthology.orchestra.run.smithi066:> ip netns list 2024-03-04T07:27:05.259 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.259 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link delete ceph-brx 2024-03-04T07:27:05.332 INFO:teuthology.orchestra.run.smithi066.stderr:Cannot find device "ceph-brx" 2024-03-04T07:27:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:27:05.333 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.333 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:27:05.348 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link delete ceph-brx 2024-03-04T07:27:05.417 INFO:teuthology.orchestra.run.smithi064.stderr:Cannot find device "ceph-brx" 2024-03-04T07:27:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:27:05.418 INFO:tasks.ceph_fuse:Mounting ceph-fuse clients... 2024-03-04T07:27:05.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:27:05.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.561+0000 7f6872c22640 1 Processor -- start 2024-03-04T07:27:05.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.561+0000 7f6872c22640 1 -- start start 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.562+0000 7f6872c22640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b9a10 0x7f686c0d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.562+0000 7f6872c22640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f686c0cea30 con 0x7f686c15df00 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.562+0000 7f6872c22640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f686c0cf3c0 con 0x7f686c0b95a0 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.562+0000 7f6872c22640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f686c0cf0e0 con 0x7f686c0b9a10 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.562+0000 7f68709be640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f686c15df00 0x7f686c0d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:33078/0 (socket says 172.21.15.64:33078) 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.562+0000 7f68709be640 1 -- 172.21.15.64:0/3902751449 learned_addr learned my addr 172.21.15.64:0/3902751449 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.562+0000 7f68711bf640 1 --2- 172.21.15.64:0/3902751449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b9a10 0x7f686c0d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f686b7fe640 1 -- 172.21.15.64:0/3902751449 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3404434842 0 0) 0x7f686c0cea30 con 0x7f686c15df00 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f686b7fe640 1 -- 172.21.15.64:0/3902751449 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6854003520 con 0x7f686c15df00 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f686b7fe640 1 -- 172.21.15.64:0/3902751449 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2299160632 0 0) 0x7f686c0cf3c0 con 0x7f686c0b95a0 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f686b7fe640 1 -- 172.21.15.64:0/3902751449 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f686c0cea30 con 0x7f686c0b95a0 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f68711bf640 1 -- 172.21.15.64:0/3902751449 >> v1:172.21.15.66:6789/0 conn(0x7f686c0b95a0 legacy=0x7f686c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f68711bf640 1 -- 172.21.15.64:0/3902751449 >> v1:172.21.15.64:6789/0 conn(0x7f686c15df00 legacy=0x7f686c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f68711bf640 1 -- 172.21.15.64:0/3902751449 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f686c0ce470 con 0x7f686c0b9a10 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.563+0000 7f68711bf640 1 --2- 172.21.15.64:0/3902751449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b9a10 0x7f686c0d3220 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f685c0097d0 tx=0x7f685c02b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b310489f97e16178 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f686b7fe640 1 -- 172.21.15.64:0/3902751449 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c035020 con 0x7f686c0b9a10 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f686b7fe640 1 -- 172.21.15.64:0/3902751449 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f685c031890 con 0x7f686c0b9a10 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f686b7fe640 1 -- 172.21.15.64:0/3902751449 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c031c40 con 0x7f686c0b9a10 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f6872c22640 1 -- 172.21.15.64:0/3902751449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b9a10 msgr2=0x7f686c0d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f6872c22640 1 --2- 172.21.15.64:0/3902751449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b9a10 0x7f686c0d3220 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f685c0097d0 tx=0x7f685c02b930 comp rx=0 tx=0).stop 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f6872c22640 1 -- 172.21.15.64:0/3902751449 shutdown_connections 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f6872c22640 1 --2- 172.21.15.64:0/3902751449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b9a10 0x7f686c0d3220 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f6872c22640 1 -- 172.21.15.64:0/3902751449 >> 172.21.15.64:0/3902751449 conn(0x7f686c071960 msgr2=0x7f686c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f6872c22640 1 -- 172.21.15.64:0/3902751449 shutdown_connections 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.564+0000 7f6872c22640 1 -- 172.21.15.64:0/3902751449 wait complete. 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.565+0000 7f6872c22640 1 Processor -- start 2024-03-04T07:27:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.565+0000 7f6872c22640 1 -- start start 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f6872c22640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b95a0 0x7f686c0b8030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f6872c22640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f686c0b9a10 0x7f686c0c9ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f6872c22640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f686c15df00 0x7f686c0ca420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f6872c22640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f686c0cedf0 con 0x7f686c15df00 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f6872c22640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f686c0cec70 con 0x7f686c0b9a10 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f6872c22640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f686c0b5e10 con 0x7f686c0b95a0 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f68711bf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f686c15df00 0x7f686c0ca420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f68709be640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b95a0 0x7f686c0b8030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f686bfff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f686c0b9a10 0x7f686c0c9ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f68711bf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f686c15df00 0x7f686c0ca420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57352/0 (socket says 172.21.15.64:57352) 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.566+0000 7f68711bf640 1 -- 172.21.15.64:0/1197366329 learned_addr learned my addr 172.21.15.64:0/1197366329 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68711bf640 1 -- 172.21.15.64:0/1197366329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b95a0 msgr2=0x7f686c0b8030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68711bf640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b95a0 0x7f686c0b8030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68711bf640 1 -- 172.21.15.64:0/1197366329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f686c0b9a10 msgr2=0x7f686c0c9ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68711bf640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f686c0b9a10 0x7f686c0c9ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68711bf640 1 -- 172.21.15.64:0/1197366329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f686c1f3400 con 0x7f686c15df00 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f686bfff640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f686c0b9a10 0x7f686c0c9ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:05.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68711bf640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f686c15df00 0x7f686c0ca420 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f685c02b970 tx=0x7f685c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68697fa640 1 -- 172.21.15.64:0/1197366329 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c035040 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68697fa640 1 -- 172.21.15.64:0/1197366329 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f685c030d20 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f686c1f3690 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f68697fa640 1 -- 172.21.15.64:0/1197366329 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c006e10 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.567+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f686c1f3c50 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.568+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f686c067ff0 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.568+0000 7f68697fa640 1 -- 172.21.15.64:0/1197366329 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f685c038020 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.569+0000 7f68697fa640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685806b4d0 0x7f685806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.569+0000 7f68697fa640 1 -- 172.21.15.64:0/1197366329 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f685c0ac6d0 con 0x7f686c15df00 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.569+0000 7f68709be640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685806b4d0 0x7f685806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:05.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.569+0000 7f68709be640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685806b4d0 0x7f685806d990 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f68640012e0 tx=0x7f6864001a90 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.573+0000 7f68697fa640 1 -- 172.21.15.64:0/1197366329 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f685c02d040 con 0x7f686c15df00 2024-03-04T07:27:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.790+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f686c06b780 con 0x7f686c15df00 2024-03-04T07:27:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.790+0000 7f68697fa640 1 -- 172.21.15.64:0/1197366329 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v7) v1 ==== 53+0+73 (secure 0 0 0) 0x7f685c07af90 con 0x7f686c15df00 2024-03-04T07:27:05.790 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:27:05.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685806b4d0 msgr2=0x7f685806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685806b4d0 0x7f685806d990 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f68640012e0 tx=0x7f6864001a90 comp rx=0 tx=0).stop 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f686c15df00 msgr2=0x7f686c0ca420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f686c15df00 0x7f686c0ca420 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f685c02b970 tx=0x7f685c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 shutdown_connections 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685806b4d0 0x7f685806d990 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f686c15df00 0x7f686c0ca420 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f686c0b9a10 0x7f686c0c9ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 --2- 172.21.15.64:0/1197366329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f686c0b95a0 0x7f686c0b8030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 >> 172.21.15.64:0/1197366329 conn(0x7f686c071960 msgr2=0x7f686c15d1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 shutdown_connections 2024-03-04T07:27:05.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:05.792+0000 7f6872c22640 1 -- 172.21.15.64:0/1197366329 wait complete. 2024-03-04T07:27:05.802 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:27:05.802 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:27:05.802 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:27:05.802 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:27:05.803 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:05.803 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:27:05.803 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:27:05.803 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:27:05.803 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:27:05.803 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.803 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:27:05.818 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:05.818 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:05.819 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:27:05.820 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42686sec preferred_lft 42686sec 2024-03-04T07:27:05.820 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:27:05.820 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:05.820 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-03-04T07:27:05.820 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.820 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:05.820 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:05.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link add name ceph-brx type bridge 2024-03-04T07:27:05.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip addr flush dev ceph-brx 2024-03-04T07:27:05.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set ceph-brx up 2024-03-04T07:27:05.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-03-04T07:27:05.821 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:05.971 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.971 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:27:05.997 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:27:05.998 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:05.999 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:27:06.066 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:27:06.066 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:27:06.066 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:27:06.066 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:27:06.066 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:27:06.066 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:06.066 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:06.066 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:06.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:27:06.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:27:06.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:27:06.067 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:06.338 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:06.338 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:27:06.353 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:06.353 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list-id 2024-03-04T07:27:06.409 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:06.409 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:06.409 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:06.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:06.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.0 0 2024-03-04T07:27:06.410 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:06.496 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.0' with 192.168.0.1/16 2024-03-04T07:27:06.497 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.0 type veth peer name brx.0 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set veth0 up 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set lo up 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip route add default via 192.168.255.254 2024-03-04T07:27:06.497 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:06.610 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:06.610 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:06.610 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:06.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:27:06.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set dev brx.0 master ceph-brx 2024-03-04T07:27:06.610 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:06.653 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:27:06.653 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:06.653 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:06.709 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:27:06.709 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:06.709 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:06.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:27:06.831 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:06.886 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:27:06.886 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:27:06.887 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:27:06.888 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:27:06.889 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:06.889 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:06.889 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:06.890 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:27:06.943 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:27:06.943 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:27:06.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:27:07.012 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:27:07.068 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:07.069 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:07.070 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.070 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:27:07.074 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:27:07.074+0000 7f109503a480 -1 init, newargv = 0x55ef16e117e0 newargc=15 2024-03-04T07:27:07.074 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[47048]: starting ceph client 2024-03-04T07:27:07.079 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[47048]: starting fuse 2024-03-04T07:27:07.089 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:27:07.089 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:27:07.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:27:07.089 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:27:07.089 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:27:07.089 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:07.090 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:27:07.091 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:27:07.211 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.47048.asok 2024-03-04T07:27:07.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:27:07.221 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.47048.asok 2024-03-04T07:27:07.221 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.47048.asok status 2024-03-04T07:27:07.359 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:27:07.359 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "47048", 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 4647, 2024-03-04T07:27:07.360 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 4647 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 1567164805 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 1567164805 2024-03-04T07:27:07.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.4647 192.168.0.1:0/1567164805", 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/1567164805", 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 7, 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 20, 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:27:07.362 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:27:07.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:27:07.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.470+0000 7fb7603f6640 1 Processor -- start 2024-03-04T07:27:07.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.470+0000 7fb7603f6640 1 -- start start 2024-03-04T07:27:07.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.471+0000 7fb7603f6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb75806b5c0 0x7fb7580b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.471+0000 7fb7603f6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb7580b3340 con 0x7fb7580b0e90 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.471+0000 7fb7603f6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb7580cee90 con 0x7fb7580ae9b0 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.471+0000 7fb7603f6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb7580cfa00 con 0x7fb75806b5c0 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.471+0000 7fb75d991640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb7580b0e90 0x7fb75806b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:33088/0 (socket says 172.21.15.64:33088) 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.471+0000 7fb75d991640 1 -- 172.21.15.64:0/3960351902 learned_addr learned my addr 172.21.15.64:0/3960351902 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.471+0000 7fb75e192640 1 --2- 172.21.15.64:0/3960351902 >> v2:172.21.15.66:3300/0 conn(0x7fb75806b5c0 0x7fb7580b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.472+0000 7fb75d190640 1 -- 172.21.15.64:0/3960351902 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 826189542 0 0) 0x7fb7580b3340 con 0x7fb7580b0e90 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.472+0000 7fb75d190640 1 -- 172.21.15.64:0/3960351902 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb744003520 con 0x7fb7580b0e90 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.472+0000 7fb75e192640 1 -- 172.21.15.64:0/3960351902 >> v1:172.21.15.66:6789/0 conn(0x7fb7580ae9b0 legacy=0x7fb7580d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.472+0000 7fb75d190640 1 -- 172.21.15.64:0/3960351902 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1068435536 0 0) 0x7fb7580cee90 con 0x7fb7580ae9b0 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.472+0000 7fb75e192640 1 -- 172.21.15.64:0/3960351902 >> v1:172.21.15.64:6789/0 conn(0x7fb7580b0e90 legacy=0x7fb75806b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:07.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.472+0000 7fb75e192640 1 -- 172.21.15.64:0/3960351902 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7580cfc50 con 0x7fb75806b5c0 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb75e192640 1 --2- 172.21.15.64:0/3960351902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75806b5c0 0x7fb7580b2e00 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fb7540097a0 tx=0x7fb75402b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=78676a4039457be5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb75d190640 1 -- 172.21.15.64:0/3960351902 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb754035020 con 0x7fb75806b5c0 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb75d190640 1 -- 172.21.15.64:0/3960351902 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb754031830 con 0x7fb75806b5c0 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb75d190640 1 -- 172.21.15.64:0/3960351902 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb754031c00 con 0x7fb75806b5c0 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb7603f6640 1 -- 172.21.15.64:0/3960351902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75806b5c0 msgr2=0x7fb7580b2e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3960351902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75806b5c0 0x7fb7580b2e00 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fb7540097a0 tx=0x7fb75402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb7603f6640 1 -- 172.21.15.64:0/3960351902 shutdown_connections 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3960351902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75806b5c0 0x7fb7580b2e00 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb7603f6640 1 -- 172.21.15.64:0/3960351902 >> 172.21.15.64:0/3960351902 conn(0x7fb7580b36d0 msgr2=0x7fb7580b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:07.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb7603f6640 1 -- 172.21.15.64:0/3960351902 shutdown_connections 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.473+0000 7fb7603f6640 1 -- 172.21.15.64:0/3960351902 wait complete. 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.474+0000 7fb7603f6640 1 Processor -- start 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.474+0000 7fb7603f6640 1 -- start start 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb7603f6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb75806b5c0 0x7fb758159010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb7603f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580ae9b0 0x7fb7580b46c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb7603f6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7580b0e90 0x7fb75815d560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb7603f6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb75806c910 con 0x7fb75806b5c0 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb7603f6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb75806c790 con 0x7fb7580b0e90 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb7603f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb7580d0ab0 con 0x7fb7580ae9b0 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb75e192640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb75806b5c0 0x7fb758159010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:07.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb75e192640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb75806b5c0 0x7fb758159010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57368/0 (socket says 172.21.15.64:57368) 2024-03-04T07:27:07.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb75e192640 1 -- 172.21.15.64:0/3398055351 learned_addr learned my addr 172.21.15.64:0/3398055351 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:07.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb75e993640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7580b0e90 0x7fb75815d560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:07.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.475+0000 7fb75d991640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580ae9b0 0x7fb7580b46c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:07.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb75e192640 1 -- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580ae9b0 msgr2=0x7fb7580b46c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:07.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb75e192640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580ae9b0 0x7fb7580b46c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb75e192640 1 -- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7580b0e90 msgr2=0x7fb75815d560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:07.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb75e192640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7580b0e90 0x7fb75815d560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb75e192640 1 -- 172.21.15.64:0/3398055351 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7580b5180 con 0x7fb75806b5c0 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb75e993640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7580b0e90 0x7fb75815d560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb75e192640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb75806b5c0 0x7fb758159010 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fb75402b910 tx=0x7fb7540046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb74effd640 1 -- 172.21.15.64:0/3398055351 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb754040070 con 0x7fb75806b5c0 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb74effd640 1 -- 172.21.15.64:0/3398055351 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb75403e070 con 0x7fb75806b5c0 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb74effd640 1 -- 172.21.15.64:0/3398055351 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb754035040 con 0x7fb75806b5c0 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb7580b5460 con 0x7fb75806b5c0 2024-03-04T07:27:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.476+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7580b5a20 con 0x7fb75806b5c0 2024-03-04T07:27:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.477+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb758068700 con 0x7fb75806b5c0 2024-03-04T07:27:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.477+0000 7fb74effd640 1 -- 172.21.15.64:0/3398055351 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb754038020 con 0x7fb75806b5c0 2024-03-04T07:27:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.478+0000 7fb74effd640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b500 0x7fb73c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.478+0000 7fb74effd640 1 -- 172.21.15.64:0/3398055351 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb7540ab9d0 con 0x7fb75806b5c0 2024-03-04T07:27:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.478+0000 7fb75d991640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b500 0x7fb73c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.478+0000 7fb75d991640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b500 0x7fb73c06d9c0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fb7580c8650 tx=0x7fb748000bc0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:07.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.482+0000 7fb74effd640 1 -- 172.21.15.64:0/3398055351 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb75404a060 con 0x7fb75806b5c0 2024-03-04T07:27:07.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.700+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fb75815e780 con 0x7fb75806b5c0 2024-03-04T07:27:07.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.700+0000 7fb74effd640 1 -- 172.21.15.64:0/3398055351 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v7) v1 ==== 53+0+73 (secure 0 0 0) 0x7fb754031800 con 0x7fb75806b5c0 2024-03-04T07:27:07.700 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b500 msgr2=0x7fb73c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b500 0x7fb73c06d9c0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fb7580c8650 tx=0x7fb748000bc0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb75806b5c0 msgr2=0x7fb758159010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb75806b5c0 0x7fb758159010 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fb75402b910 tx=0x7fb7540046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 shutdown_connections 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b500 0x7fb73c06d9c0 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7580b0e90 0x7fb75815d560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580ae9b0 0x7fb7580b46c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 --2- 172.21.15.64:0/3398055351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb75806b5c0 0x7fb758159010 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:07.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 >> 172.21.15.64:0/3398055351 conn(0x7fb7580b36d0 msgr2=0x7fb7580afcb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:07.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 shutdown_connections 2024-03-04T07:27:07.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:07.702+0000 7fb7603f6640 1 -- 172.21.15.64:0/3398055351 wait complete. 2024-03-04T07:27:07.712 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:27:07.712 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:27:07.712 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:27:07.712 INFO:tasks.cephfs.mount:self.client.name = client.2 2024-03-04T07:27:07.713 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:07.713 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:27:07.713 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:27:07.713 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:27:07.713 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.2' 2024-03-04T07:27:07.713 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.713 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:27:07.728 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:07.728 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:07.728 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:07.728 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:07.728 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:07.728 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42685sec preferred_lft 42685sec 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link tentative 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:07.730 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.731 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:27:07.793 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:27:07.795 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.795 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:27:07.850 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:27:07.851 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:27:07.851 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:27:07.851 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:27:07.851 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:27:07.851 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.851 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:07.851 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:07.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:27:07.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:27:07.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:27:07.852 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:07.946 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.946 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:27:07.961 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:27:07.961 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:07.961 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list-id 2024-03-04T07:27:08.016 INFO:teuthology.orchestra.run.smithi064.stdout:nsid 0 (iproute2 netns name: ceph-ns--home-ubuntu-cephtest-mnt.0) 2024-03-04T07:27:08.016 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.016 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:08.017 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:08.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:08.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.2 1 2024-03-04T07:27:08.017 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:08.098 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr 2024-03-04T07:27:08.164 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether ee:00:16:5b:33:7e brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec00:16ff:fe5b:337e/64 scope link 2024-03-04T07:27:08.165 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.166 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether ee:00:16:5b:33:7e brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec00:16ff:fe5b:337e/64 scope link 2024-03-04T07:27:08.233 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:08.235 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.2' with 192.168.0.2/16 2024-03-04T07:27:08.235 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.2 type veth peer name brx.1 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip addr add 192.168.0.2/16 brd 192.168.255.255 dev veth0 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip link set veth0 up 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip link set lo up 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.2 ip route add default via 192.168.255.254 2024-03-04T07:27:08.235 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:08.360 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.360 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:08.360 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:08.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.1 up 2024-03-04T07:27:08.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set dev brx.1 master ceph-brx 2024-03-04T07:27:08.361 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:08.402 INFO:tasks.cephfs.fuse_mount:Client client.2 config is {} 2024-03-04T07:27:08.402 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:08.402 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:08.457 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.2' 2024-03-04T07:27:08.457 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:08.457 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:08.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:27:08.579 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:08.634 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:27:08.635 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:08.636 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:08.637 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:08.638 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:08.638 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.638 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:27:08.689 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:27:08.689 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [41] 2024-03-04T07:27:08.690 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.2 --id 2) 2024-03-04T07:27:08.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:27:08.757 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:08.813 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:27:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:27:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:27:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:27:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:27:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:27:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:27:08.815 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:27:08.815 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:27:08.815 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:27:08.815 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:27:08.815 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:27:08.815 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:08.816 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:27:08.816 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:27:08.816 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:27:08.816 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:27:08.816 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:08.816 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:27:08.817 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:08.817 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:08.817 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:08.817 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:08.817 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:27:08.818 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:27:08.818 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:27:08.818 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:27:08.818 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:27:08.818 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:27:08.818 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:27:08.819 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:08.819 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:08.819 INFO:teuthology.orchestra.run.smithi064.stdout:/home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:08.819 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:08.819 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:08.820 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.820 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:27:08.821 INFO:tasks.cephfs.fuse_mount.ceph-fuse.2.smithi064.stderr:2024-03-04T07:27:08.816+0000 7f2f5243b480 -1 init, newargv = 0x5621f4a46510 newargc=15 2024-03-04T07:27:08.822 INFO:tasks.cephfs.fuse_mount.ceph-fuse.2.smithi064.stderr:ceph-fuse[47543]: starting ceph client 2024-03-04T07:27:08.822 INFO:tasks.cephfs.fuse_mount.ceph-fuse.2.smithi064.stderr:ceph-fuse[47543]: starting fuse 2024-03-04T07:27:08.836 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:27:08.836 INFO:teuthology.orchestra.run.smithi064.stdout:44 2024-03-04T07:27:08.837 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41, 44] 2024-03-04T07:27:08.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:27:08.837 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:27:08.837 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:27:08.837 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:27:08.838 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:27:08.838 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:08.838 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:27:08.838 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.2.*.asok" 2024-03-04T07:27:08.838 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:27:08.838 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.2" 2024-03-04T07:27:08.838 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:08.839 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:27:08.839 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:27:08.839 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:27:08.839 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:27:08.839 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:08.839 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:27:08.840 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:27:08.840 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:27:08.840 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:27:08.840 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:27:08.840 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:27:08.840 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:27:08.841 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:27:08.841 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:08.841 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.2")) 2024-03-04T07:27:08.841 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:27:08.956 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.2.47543.asok 2024-03-04T07:27:08.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:08 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:27:08.967 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.2.47543.asok 2024-03-04T07:27:08.967 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:08.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.2.47543.asok status 2024-03-04T07:27:09.104 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:27:09.104 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "2", 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.2", 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "47543", 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 4662, 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 4662 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:09.105 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.2:0", 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 747032474 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.2:0", 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 747032474 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.4662 192.168.0.2:0/747032474", 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.2:0/747032474", 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 7, 2024-03-04T07:27:09.106 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 20, 2024-03-04T07:27:09.107 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:27:09.107 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:27:09.107 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:27:09.107 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:27:09.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:27:09.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.214+0000 7f31126eb640 1 Processor -- start 2024-03-04T07:27:09.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.214+0000 7f31126eb640 1 -- start start 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f31126eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c06d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f31126eb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f310c06dd60 0x7f310c0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f31126eb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f310c0af080 con 0x7f310c0aec40 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f31126eb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f310c06e1c0 con 0x7f310c06dd60 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f31126eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f310c071b70 con 0x7f310c158520 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f3110c88640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f310c0aec40 0x7f310c15e670 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:33090/0 (socket says 172.21.15.64:33090) 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f3110c88640 1 -- 172.21.15.64:0/561022502 learned_addr learned my addr 172.21.15.64:0/561022502 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310b7fe640 1 --2- 172.21.15.64:0/561022502 >> v2:172.21.15.66:3300/0 conn(0x7f310c06dd60 0x7f310c0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310bfff640 1 --2- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c06d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310affd640 1 -- 172.21.15.64:0/561022502 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1896662096 0 0) 0x7f310c0af080 con 0x7f310c0aec40 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310affd640 1 -- 172.21.15.64:0/561022502 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f30f4003520 con 0x7f310c0aec40 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310b7fe640 1 -- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 msgr2=0x7f310c06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310b7fe640 1 --2- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c06d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310b7fe640 1 -- 172.21.15.64:0/561022502 >> v1:172.21.15.64:6789/0 conn(0x7f310c0aec40 legacy=0x7f310c15e670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.215+0000 7f310b7fe640 1 -- 172.21.15.64:0/561022502 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f310c071240 con 0x7f310c06dd60 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.216+0000 7f310b7fe640 1 --2- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0ae700 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f30fc0097a0 tx=0x7f30fc02b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ae6cd72f9d964256 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.216+0000 7f310affd640 1 -- 172.21.15.64:0/561022502 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30fc035020 con 0x7f310c06dd60 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.216+0000 7f310affd640 1 -- 172.21.15.64:0/561022502 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f30fc031830 con 0x7f310c06dd60 2024-03-04T07:27:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.216+0000 7f310affd640 1 -- 172.21.15.64:0/561022502 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30fc031be0 con 0x7f310c06dd60 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 -- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 msgr2=0x7f310c0ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 --2- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0ae700 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f30fc0097a0 tx=0x7f30fc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 -- 172.21.15.64:0/561022502 shutdown_connections 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 --2- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0ae700 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 --2- 172.21.15.64:0/561022502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c06d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 -- 172.21.15.64:0/561022502 >> 172.21.15.64:0/561022502 conn(0x7f310c066d00 msgr2=0x7f310c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 -- 172.21.15.64:0/561022502 shutdown_connections 2024-03-04T07:27:09.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.217+0000 7f31126eb640 1 -- 172.21.15.64:0/561022502 wait complete. 2024-03-04T07:27:09.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.218+0000 7f31126eb640 1 Processor -- start 2024-03-04T07:27:09.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.218+0000 7f31126eb640 1 -- start start 2024-03-04T07:27:09.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f31126eb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0b23c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:09.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f31126eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f310c0aec40 0x7f310c1472a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:09.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f31126eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c0a97f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f31126eb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f310c0b4650 con 0x7f310c0aec40 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f31126eb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f310c071800 con 0x7f310c06dd60 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f31126eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f310c072000 con 0x7f310c158520 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f310c0aec40 0x7f310c1472a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310bfff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0b23c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f310c0aec40 0x7f310c1472a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57378/0 (socket says 172.21.15.64:57378) 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 -- 172.21.15.64:0/1774509666 learned_addr learned my addr 172.21.15.64:0/1774509666 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f3110c88640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c0a97f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 -- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 msgr2=0x7f310c0a97f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c0a97f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 -- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 msgr2=0x7f310c0b23c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:09.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0b23c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310b7fe640 1 -- 172.21.15.64:0/1774509666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f310c0b1320 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.219+0000 7f310bfff640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0b23c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.220+0000 7f310b7fe640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f310c0aec40 0x7f310c1472a0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f30fc0098c0 tx=0x7f30fc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.220+0000 7f3108ff9640 1 -- 172.21.15.64:0/1774509666 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30fc035040 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.220+0000 7f3108ff9640 1 -- 172.21.15.64:0/1774509666 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f30fc03b900 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.220+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f310c0b1600 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.220+0000 7f3108ff9640 1 -- 172.21.15.64:0/1774509666 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30fc03bcf0 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.220+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f310c0b1bc0 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.221+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f310c067ff0 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.221+0000 7f3108ff9640 1 -- 172.21.15.64:0/1774509666 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f30fc038020 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.221+0000 7f3108ff9640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f30f806b4d0 0x7f30f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.222+0000 7f3108ff9640 1 -- 172.21.15.64:0/1774509666 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f30fc0ac0f0 con 0x7f310c0aec40 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.222+0000 7f310bfff640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f30f806b4d0 0x7f30f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:09.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.222+0000 7f310bfff640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f30f806b4d0 0x7f30f806d990 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f3100009e90 tx=0x7f3100009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:09.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.226+0000 7f3108ff9640 1 -- 172.21.15.64:0/1774509666 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f30fc07a920 con 0x7f310c0aec40 2024-03-04T07:27:09.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.446+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f310c06e600 con 0x7f310c0aec40 2024-03-04T07:27:09.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.447+0000 7f3108ff9640 1 -- 172.21.15.64:0/1774509666 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v7) v1 ==== 53+0+73 (secure 0 0 0) 0x7f30fc07e9c0 con 0x7f310c0aec40 2024-03-04T07:27:09.446 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:27:09.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f30f806b4d0 msgr2=0x7f30f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:09.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f30f806b4d0 0x7f30f806d990 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f3100009e90 tx=0x7f3100009190 comp rx=0 tx=0).stop 2024-03-04T07:27:09.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f310c0aec40 msgr2=0x7f310c1472a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f310c0aec40 0x7f310c1472a0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f30fc0098c0 tx=0x7f30fc0046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 shutdown_connections 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f30f806b4d0 0x7f30f806d990 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f310c158520 0x7f310c0a97f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f310c0aec40 0x7f310c1472a0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 --2- 172.21.15.64:0/1774509666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f310c06dd60 0x7f310c0b23c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.448+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 >> 172.21.15.64:0/1774509666 conn(0x7f310c066d00 msgr2=0x7f310c06ce80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.449+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 shutdown_connections 2024-03-04T07:27:09.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:09.449+0000 7f31126eb640 1 -- 172.21.15.64:0/1774509666 wait complete. 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi066.front.sepia.ceph.com 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:self.client.name = client.1 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:27:09.459 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.1' 2024-03-04T07:27:09.459 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:09.459 DEBUG:teuthology.orchestra.run.smithi066:> ip addr 2024-03-04T07:27:09.475 INFO:teuthology.orchestra.run.smithi066.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:09.475 INFO:teuthology.orchestra.run.smithi066.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:09.475 INFO:teuthology.orchestra.run.smithi066.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:09.475 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:09.475 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:09.475 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:a8:b4:84 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: altname enp7s0f0 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:a8:b4:85 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: altname enp7s0f1 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:b6:f8:bc brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:b6:f8:bd brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: inet 172.21.15.66/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft 42684sec preferred_lft 42684sec 2024-03-04T07:27:09.476 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 fe80::ec4:7aff:feb6:f8bd/64 scope link 2024-03-04T07:27:09.477 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:09.477 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-03-04T07:27:09.477 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:09.477 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:09.477 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:09.477 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link add name ceph-brx type bridge 2024-03-04T07:27:09.477 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip addr flush dev ceph-brx 2024-03-04T07:27:09.477 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link set ceph-brx up 2024-03-04T07:27:09.477 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-03-04T07:27:09.477 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:09.652 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:09.652 DEBUG:teuthology.orchestra.run.smithi066:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:27:09.683 INFO:teuthology.orchestra.run.smithi066.stdout:1 2024-03-04T07:27:09.685 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:09.685 DEBUG:teuthology.orchestra.run.smithi066:> route 2024-03-04T07:27:09.757 INFO:teuthology.orchestra.run.smithi066.stdout:Kernel IP routing table 2024-03-04T07:27:09.757 INFO:teuthology.orchestra.run.smithi066.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:27:09.757 INFO:teuthology.orchestra.run.smithi066.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:27:09.757 INFO:teuthology.orchestra.run.smithi066.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:27:09.757 INFO:teuthology.orchestra.run.smithi066.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:27:09.758 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:09.758 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:09.758 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:09.758 DEBUG:teuthology.orchestra.run.smithi066:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:27:09.758 DEBUG:teuthology.orchestra.run.smithi066:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:27:09.758 DEBUG:teuthology.orchestra.run.smithi066:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:27:09.758 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:10.028 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.028 DEBUG:teuthology.orchestra.run.smithi066:> ip netns list 2024-03-04T07:27:10.043 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.043 DEBUG:teuthology.orchestra.run.smithi066:> ip netns list-id 2024-03-04T07:27:10.098 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.098 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:10.098 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:10.098 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:10.098 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.1 0 2024-03-04T07:27:10.099 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:10.189 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.1' with 192.168.0.1/16 2024-03-04T07:27:10.190 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.1 type veth peer name brx.0 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip link set veth0 up 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip link set lo up 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip route add default via 192.168.255.254 2024-03-04T07:27:10.190 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:10.308 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.308 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:10.308 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:10.308 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link set brx.0 up 2024-03-04T07:27:10.308 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link set dev brx.0 master ceph-brx 2024-03-04T07:27:10.308 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:10.362 INFO:tasks.cephfs.fuse_mount:Client client.1 config is {} 2024-03-04T07:27:10.363 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:10.363 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -v /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:10.391 INFO:teuthology.orchestra.run.smithi066.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.1' 2024-03-04T07:27:10.391 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:10.391 DEBUG:teuthology.orchestra.run.smithi066:> chmod 0000 /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:10.446 DEBUG:teuthology.orchestra.run.smithi066:> sudo modprobe fuse 2024-03-04T07:27:10.518 DEBUG:teuthology.orchestra.run.smithi066:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/proc 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/sys 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/dev 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/security 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/shm 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/pts 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/run 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/cgroup 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/pstore 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/bpf 2024-03-04T07:27:10.574 INFO:teuthology.orchestra.run.smithi066.stdout:/ 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/selinux 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/debug 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/mqueue 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/hugepages 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/tracing 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/config 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/run/user/1000 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph 2024-03-04T07:27:10.575 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-4 2024-03-04T07:27:10.576 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-5 2024-03-04T07:27:10.576 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-6 2024-03-04T07:27:10.576 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-7 2024-03-04T07:27:10.576 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns 2024-03-04T07:27:10.576 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:10.576 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:10.576 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.576 DEBUG:teuthology.orchestra.run.smithi066:> ls /sys/fs/fuse/connections 2024-03-04T07:27:10.629 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:27:10.629 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.1 --id 1) 2024-03-04T07:27:10.673 DEBUG:teuthology.orchestra.run.smithi066:> sudo modprobe fuse 2024-03-04T07:27:10.704 DEBUG:teuthology.orchestra.run.smithi066:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:10.761 INFO:teuthology.orchestra.run.smithi066.stdout:/proc 2024-03-04T07:27:10.761 INFO:teuthology.orchestra.run.smithi066.stdout:/sys 2024-03-04T07:27:10.761 INFO:teuthology.orchestra.run.smithi066.stdout:/dev 2024-03-04T07:27:10.761 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/security 2024-03-04T07:27:10.761 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/shm 2024-03-04T07:27:10.761 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/pts 2024-03-04T07:27:10.761 INFO:teuthology.orchestra.run.smithi066.stdout:/run 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/cgroup 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/pstore 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/bpf 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/ 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/selinux 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/debug 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/mqueue 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/hugepages 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/tracing 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/config 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:10.762 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/run/user/1000 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-4 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-5 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-6 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-7 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:10.763 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.763 DEBUG:teuthology.orchestra.run.smithi066:> ls /sys/fs/fuse/connections 2024-03-04T07:27:10.772 INFO:tasks.cephfs.fuse_mount.ceph-fuse.1.smithi066.stderr:2024-03-04T07:27:10.772+0000 7efd61e39480 -1 init, newargv = 0x559cdef08620 newargc=15 2024-03-04T07:27:10.772 INFO:tasks.cephfs.fuse_mount.ceph-fuse.1.smithi066.stderr:ceph-fuse[44371]: starting ceph client 2024-03-04T07:27:10.777 INFO:tasks.cephfs.fuse_mount.ceph-fuse.1.smithi066.stderr:ceph-fuse[44371]: starting fuse 2024-03-04T07:27:10.787 INFO:teuthology.orchestra.run.smithi066.stdout:41 2024-03-04T07:27:10.788 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:27:10.788 DEBUG:teuthology.orchestra.run.smithi066:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:27:10.788 DEBUG:teuthology.orchestra.run.smithi066:> import glob 2024-03-04T07:27:10.788 DEBUG:teuthology.orchestra.run.smithi066:> import re 2024-03-04T07:27:10.788 DEBUG:teuthology.orchestra.run.smithi066:> import os 2024-03-04T07:27:10.788 DEBUG:teuthology.orchestra.run.smithi066:> import subprocess 2024-03-04T07:27:10.788 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:10.788 DEBUG:teuthology.orchestra.run.smithi066:> def _find_admin_socket(client_name): 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> asok_path = "/var/run/ceph/ceph-client.1.*.asok" 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> files = glob.glob(asok_path) 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> mountpoint = "/home/ubuntu/cephtest/mnt.1" 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> # Given a non-glob path, it better be there 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> if "*" not in asok_path: 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> assert(len(files) == 1) 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> return files[0] 2024-03-04T07:27:10.789 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> for f in files: 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> contents = proc_f.read() 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> if mountpoint in contents: 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> return f 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> print(_find_admin_socket("client.1")) 2024-03-04T07:27:10.790 DEBUG:teuthology.orchestra.run.smithi066:> ' 2024-03-04T07:27:10.918 INFO:teuthology.orchestra.run.smithi066.stdout:/var/run/ceph/ceph-client.1.44371.asok 2024-03-04T07:27:10.921 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:27:10 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:27:10.929 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.1.44371.asok 2024-03-04T07:27:10.929 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:10.929 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.1.44371.asok status 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout:{ 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "metadata": { 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "entity_id": "1", 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "hostname": "smithi066", 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.1", 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "pid": "44371", 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "root": "/" 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "dentry_count": 0, 2024-03-04T07:27:11.072 INFO:teuthology.orchestra.run.smithi066.stdout: "dentry_pinned_count": 0, 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "id": 4603, 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "inst": { 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "name": { 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "type": "client", 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "num": 4603 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": { 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "type": "v1", 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "nonce": 4165226083 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: } 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": { 2024-03-04T07:27:11.073 INFO:teuthology.orchestra.run.smithi066.stdout: "type": "v1", 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "nonce": 4165226083 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "inst_str": "client.4603 192.168.0.1:0/4165226083", 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "addr_str": "192.168.0.1:0/4165226083", 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "inode_count": 1, 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "mds_epoch": 7, 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "osd_epoch": 20, 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "blocklisted": false, 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout: "fs_name": "cephfs" 2024-03-04T07:27:11.074 INFO:teuthology.orchestra.run.smithi066.stdout:} 2024-03-04T07:27:11.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:27:11.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.196+0000 7f06aafbc640 1 Processor -- start 2024-03-04T07:27:11.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.196+0000 7f06aafbc640 1 -- start start 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06aafbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06aafbc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f06a40cea30 con 0x7f06a40b95a0 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06aafbc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f06a40d2af0 con 0x7f06a40b9a30 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06aafbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f06a40cf3c0 con 0x7f06a415df00 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a3fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f06a40b95a0 0x7f06a40b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:33096/0 (socket says 172.21.15.64:33096) 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a3fff640 1 -- 172.21.15.64:0/855132792 learned_addr learned my addr 172.21.15.64:0/855132792 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a8d58640 1 --2- 172.21.15.64:0/855132792 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:11.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a37fe640 1 -- 172.21.15.64:0/855132792 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2311564070 0 0) 0x7f06a40cea30 con 0x7f06a40b95a0 2024-03-04T07:27:11.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a37fe640 1 -- 172.21.15.64:0/855132792 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0690003520 con 0x7f06a40b95a0 2024-03-04T07:27:11.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a37fe640 1 -- 172.21.15.64:0/855132792 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3932824917 0 0) 0x7f06a40d2af0 con 0x7f06a40b9a30 2024-03-04T07:27:11.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a37fe640 1 -- 172.21.15.64:0/855132792 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f06a40cea30 con 0x7f06a40b9a30 2024-03-04T07:27:11.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a8d58640 1 -- 172.21.15.64:0/855132792 >> v1:172.21.15.66:6789/0 conn(0x7f06a40b9a30 legacy=0x7f06a40d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:11.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a8d58640 1 -- 172.21.15.64:0/855132792 >> v1:172.21.15.64:6789/0 conn(0x7f06a40b95a0 legacy=0x7f06a40b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:11.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.197+0000 7f06a8d58640 1 -- 172.21.15.64:0/855132792 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f06a40cf0e0 con 0x7f06a415df00 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.198+0000 7f06a8d58640 1 --2- 172.21.15.64:0/855132792 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40d25b0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f069c009dc0 tx=0x7f069c02bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b7b0d7579752cf67 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.198+0000 7f06a37fe640 1 -- 172.21.15.64:0/855132792 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f069c039070 con 0x7f06a415df00 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.198+0000 7f06a37fe640 1 -- 172.21.15.64:0/855132792 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f069c02ccd0 con 0x7f06a415df00 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.198+0000 7f06a37fe640 1 -- 172.21.15.64:0/855132792 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f069c034660 con 0x7f06a415df00 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.199+0000 7f06aafbc640 1 -- 172.21.15.64:0/855132792 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 msgr2=0x7f06a40d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.199+0000 7f06aafbc640 1 --2- 172.21.15.64:0/855132792 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40d25b0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f069c009dc0 tx=0x7f069c02bf20 comp rx=0 tx=0).stop 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.199+0000 7f06aafbc640 1 -- 172.21.15.64:0/855132792 shutdown_connections 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.199+0000 7f06aafbc640 1 --2- 172.21.15.64:0/855132792 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40d25b0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.199+0000 7f06aafbc640 1 -- 172.21.15.64:0/855132792 >> 172.21.15.64:0/855132792 conn(0x7f06a4071960 msgr2=0x7f06a4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.199+0000 7f06aafbc640 1 -- 172.21.15.64:0/855132792 shutdown_connections 2024-03-04T07:27:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.199+0000 7f06aafbc640 1 -- 172.21.15.64:0/855132792 wait complete. 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.200+0000 7f06aafbc640 1 Processor -- start 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.200+0000 7f06aafbc640 1 -- start start 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06aafbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06a40b95a0 0x7f06a41550a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06aafbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40cc630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06aafbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f06a40ccb70 0x7f06a415bb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06aafbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06a40cec70 con 0x7f06a40ccb70 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06aafbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06a40ce750 con 0x7f06a40b95a0 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06aafbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f06a40cedf0 con 0x7f06a415df00 2024-03-04T07:27:11.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a9559640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f06a40ccb70 0x7f06a415bb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a8d58640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06a40b95a0 0x7f06a41550a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a3fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40cc630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a8d58640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06a40b95a0 0x7f06a41550a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:40080/0 (socket says 172.21.15.64:40080) 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a9559640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f06a40ccb70 0x7f06a415bb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57390/0 (socket says 172.21.15.64:57390) 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a8d58640 1 -- 172.21.15.64:0/2663360377 learned_addr learned my addr 172.21.15.64:0/2663360377 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a9559640 1 -- 172.21.15.64:0/2663360377 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 msgr2=0x7f06a40cc630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a9559640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40cc630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a9559640 1 -- 172.21.15.64:0/2663360377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06a40b95a0 msgr2=0x7f06a41550a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a9559640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06a40b95a0 0x7f06a41550a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a9559640 1 -- 172.21.15.64:0/2663360377 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f06a40c88b0 con 0x7f06a40ccb70 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.201+0000 7f06a8d58640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06a40b95a0 0x7f06a41550a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.202+0000 7f06a9559640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f06a40ccb70 0x7f06a415bb50 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f068c00c310 tx=0x7f068c00c7d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.202+0000 7f06a17fa640 1 -- 172.21.15.64:0/2663360377 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f068c017070 con 0x7f06a40ccb70 2024-03-04T07:27:11.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.202+0000 7f06a17fa640 1 -- 172.21.15.64:0/2663360377 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f068c0052d0 con 0x7f06a40ccb70 2024-03-04T07:27:11.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.202+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f06a40c7850 con 0x7f06a40ccb70 2024-03-04T07:27:11.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.202+0000 7f06a17fa640 1 -- 172.21.15.64:0/2663360377 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f068c003830 con 0x7f06a40ccb70 2024-03-04T07:27:11.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.202+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f06a40c80b0 con 0x7f06a40ccb70 2024-03-04T07:27:11.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.203+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f06a4067ff0 con 0x7f06a40ccb70 2024-03-04T07:27:11.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.203+0000 7f06a17fa640 1 -- 172.21.15.64:0/2663360377 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f068c01a070 con 0x7f06a40ccb70 2024-03-04T07:27:11.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.204+0000 7f06a17fa640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f068406b4d0 0x7f068406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:11.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.204+0000 7f06a17fa640 1 -- 172.21.15.64:0/2663360377 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f068c0894d0 con 0x7f06a40ccb70 2024-03-04T07:27:11.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.204+0000 7f06a8d58640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f068406b4d0 0x7f068406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:11.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.204+0000 7f06a8d58640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f068406b4d0 0x7f068406d990 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f069c02bf60 tx=0x7f069c030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:11.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.208+0000 7f06a17fa640 1 -- 172.21.15.64:0/2663360377 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f06a40c88b0 con 0x7f06a40ccb70 2024-03-04T07:27:11.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.429+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f06a415d490 con 0x7f06a40ccb70 2024-03-04T07:27:11.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.430+0000 7f06a17fa640 1 -- 172.21.15.64:0/2663360377 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v7) v1 ==== 53+0+73 (secure 0 0 0) 0x7f068c087e20 con 0x7f06a40ccb70 2024-03-04T07:27:11.429 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:27:11.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.431+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f068406b4d0 msgr2=0x7f068406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:11.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.431+0000 7f06aafbc640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f068406b4d0 0x7f068406d990 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f069c02bf60 tx=0x7f069c030000 comp rx=0 tx=0).stop 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.431+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f06a40ccb70 msgr2=0x7f06a415bb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f06a40ccb70 0x7f06a415bb50 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f068c00c310 tx=0x7f068c00c7d0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 shutdown_connections 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f068406b4d0 0x7f068406d990 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f06a40ccb70 0x7f06a415bb50 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f06a415df00 0x7f06a40cc630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 --2- 172.21.15.64:0/2663360377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06a40b95a0 0x7f06a41550a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 >> 172.21.15.64:0/2663360377 conn(0x7f06a4071960 msgr2=0x7f06a40b8bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 shutdown_connections 2024-03-04T07:27:11.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:11.432+0000 7f06aafbc640 1 -- 172.21.15.64:0/2663360377 wait complete. 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi066.front.sepia.ceph.com 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:self.client.name = client.3 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:27:11.442 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.3' 2024-03-04T07:27:11.442 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.442 DEBUG:teuthology.orchestra.run.smithi066:> ip addr 2024-03-04T07:27:11.460 INFO:teuthology.orchestra.run.smithi066.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:a8:b4:84 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:11.461 INFO:teuthology.orchestra.run.smithi066.stdout: altname enp7s0f0 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:a8:b4:85 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout: altname enp7s0f1 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:b6:f8:bc brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 0c:c4:7a:b6:f8:bd brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:11.462 INFO:teuthology.orchestra.run.smithi066.stdout: inet 172.21.15.66/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:27:11.463 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft 42682sec preferred_lft 42682sec 2024-03-04T07:27:11.463 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 fe80::ec4:7aff:feb6:f8bd/64 scope link 2024-03-04T07:27:11.463 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.463 INFO:teuthology.orchestra.run.smithi066.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:27:11.463 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether ee:00:62:37:51:b4 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:11.463 INFO:teuthology.orchestra.run.smithi066.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:27:11.464 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.464 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 fe80::4c5a:e5ff:fe0c:461d/64 scope link 2024-03-04T07:27:11.464 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.464 INFO:teuthology.orchestra.run.smithi066.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:27:11.464 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether ee:00:62:37:51:b4 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:11.464 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 fe80::ec00:62ff:fe37:51b4/64 scope link tentative 2024-03-04T07:27:11.464 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.465 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.465 DEBUG:teuthology.orchestra.run.smithi066:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:27:11.530 INFO:teuthology.orchestra.run.smithi066.stdout:1 2024-03-04T07:27:11.531 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.532 DEBUG:teuthology.orchestra.run.smithi066:> route 2024-03-04T07:27:11.590 INFO:teuthology.orchestra.run.smithi066.stdout:Kernel IP routing table 2024-03-04T07:27:11.590 INFO:teuthology.orchestra.run.smithi066.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:27:11.590 INFO:teuthology.orchestra.run.smithi066.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:27:11.590 INFO:teuthology.orchestra.run.smithi066.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:27:11.590 INFO:teuthology.orchestra.run.smithi066.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:27:11.591 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.591 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:11.591 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:11.591 DEBUG:teuthology.orchestra.run.smithi066:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:27:11.591 DEBUG:teuthology.orchestra.run.smithi066:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:27:11.591 DEBUG:teuthology.orchestra.run.smithi066:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:27:11.591 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:11.696 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.697 DEBUG:teuthology.orchestra.run.smithi066:> ip netns list 2024-03-04T07:27:11.712 INFO:teuthology.orchestra.run.smithi066.stdout:ceph-ns--home-ubuntu-cephtest-mnt.1 (id: 0) 2024-03-04T07:27:11.713 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.713 DEBUG:teuthology.orchestra.run.smithi066:> ip netns list-id 2024-03-04T07:27:11.767 INFO:teuthology.orchestra.run.smithi066.stdout:nsid 0 (iproute2 netns name: ceph-ns--home-ubuntu-cephtest-mnt.1) 2024-03-04T07:27:11.767 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.767 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:11.767 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:11.768 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.3 2024-03-04T07:27:11.768 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.3 1 2024-03-04T07:27:11.768 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:11.857 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.857 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip addr 2024-03-04T07:27:11.887 INFO:teuthology.orchestra.run.smithi066.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:11.887 INFO:teuthology.orchestra.run.smithi066.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:11.887 INFO:teuthology.orchestra.run.smithi066.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 72:c4:6a:b4:27:57 brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 fe80::70c4:6aff:feb4:2757/64 scope link 2024-03-04T07:27:11.888 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.889 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.889 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip addr 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout:2: veth0@if7: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:27:11.961 INFO:teuthology.orchestra.run.smithi066.stdout: link/ether 72:c4:6a:b4:27:57 brd ff:ff:ff:ff:ff:ff link-netnsid 0 2024-03-04T07:27:11.962 INFO:teuthology.orchestra.run.smithi066.stdout: inet 192.168.0.1/16 brd 192.168.255.255 scope global veth0 2024-03-04T07:27:11.962 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.962 INFO:teuthology.orchestra.run.smithi066.stdout: inet6 fe80::70c4:6aff:feb4:2757/64 scope link 2024-03-04T07:27:11.962 INFO:teuthology.orchestra.run.smithi066.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:11.963 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.3' with 192.168.0.2/16 2024-03-04T07:27:11.963 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:11.963 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:11.963 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:11.963 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.3 type veth peer name brx.1 2024-03-04T07:27:11.963 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip addr add 192.168.0.2/16 brd 192.168.255.255 dev veth0 2024-03-04T07:27:11.963 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip link set veth0 up 2024-03-04T07:27:11.963 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip link set lo up 2024-03-04T07:27:11.963 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.3 ip route add default via 192.168.255.254 2024-03-04T07:27:11.964 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:12.109 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:12.110 DEBUG:teuthology.orchestra.run.smithi066:> (cd / && exec bash -c ' 2024-03-04T07:27:12.110 DEBUG:teuthology.orchestra.run.smithi066:> set -e 2024-03-04T07:27:12.110 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link set brx.1 up 2024-03-04T07:27:12.110 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link set dev brx.1 master ceph-brx 2024-03-04T07:27:12.110 DEBUG:teuthology.orchestra.run.smithi066:> ') 2024-03-04T07:27:12.161 INFO:tasks.cephfs.fuse_mount:Client client.3 config is {} 2024-03-04T07:27:12.161 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:12.161 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -v /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:12.176 INFO:teuthology.orchestra.run.smithi066.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.3' 2024-03-04T07:27:12.176 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:12.176 DEBUG:teuthology.orchestra.run.smithi066:> chmod 0000 /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:12.231 DEBUG:teuthology.orchestra.run.smithi066:> sudo modprobe fuse 2024-03-04T07:27:12.302 DEBUG:teuthology.orchestra.run.smithi066:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/proc 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/sys 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/dev 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/security 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/shm 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/pts 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/run 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/cgroup 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/pstore 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/bpf 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/ 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/selinux 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/debug 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/mqueue 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/hugepages 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/tracing 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/config 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/run/user/1000 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph 2024-03-04T07:27:12.360 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-4 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-5 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-6 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-7 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 2024-03-04T07:27:12.361 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:12.361 DEBUG:teuthology.orchestra.run.smithi066:> ls /sys/fs/fuse/connections 2024-03-04T07:27:12.414 INFO:teuthology.orchestra.run.smithi066.stdout:41 2024-03-04T07:27:12.414 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [41] 2024-03-04T07:27:12.414 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.3 --id 3) 2024-03-04T07:27:12.456 DEBUG:teuthology.orchestra.run.smithi066:> sudo modprobe fuse 2024-03-04T07:27:12.487 DEBUG:teuthology.orchestra.run.smithi066:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:12.545 INFO:teuthology.orchestra.run.smithi066.stdout:/proc 2024-03-04T07:27:12.545 INFO:teuthology.orchestra.run.smithi066.stdout:/sys 2024-03-04T07:27:12.545 INFO:teuthology.orchestra.run.smithi066.stdout:/dev 2024-03-04T07:27:12.545 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/security 2024-03-04T07:27:12.545 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/shm 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/pts 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/run 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/cgroup 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/pstore 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/bpf 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/ 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/selinux 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/debug 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/mqueue 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/dev/hugepages 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/tracing 2024-03-04T07:27:12.546 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:12.547 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:12.547 INFO:teuthology.orchestra.run.smithi066.stdout:/sys/kernel/config 2024-03-04T07:27:12.547 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:12.547 INFO:teuthology.orchestra.run.smithi066.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:12.547 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/run/user/1000 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-4 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-5 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-6 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/var/lib/ceph/osd/ceph-7 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 2024-03-04T07:27:12.548 INFO:teuthology.orchestra.run.smithi066.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 2024-03-04T07:27:12.549 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:12.549 DEBUG:teuthology.orchestra.run.smithi066:> ls /sys/fs/fuse/connections 2024-03-04T07:27:12.550 INFO:tasks.cephfs.fuse_mount.ceph-fuse.3.smithi066.stderr:2024-03-04T07:27:12.550+0000 7fe7fb531480 -1 init, newargv = 0x5556222e3420 newargc=15 2024-03-04T07:27:12.550 INFO:tasks.cephfs.fuse_mount.ceph-fuse.3.smithi066.stderr:ceph-fuse[44823]: starting ceph client 2024-03-04T07:27:12.555 INFO:tasks.cephfs.fuse_mount.ceph-fuse.3.smithi066.stderr:ceph-fuse[44823]: starting fuse 2024-03-04T07:27:12.567 INFO:teuthology.orchestra.run.smithi066.stdout:41 2024-03-04T07:27:12.567 INFO:teuthology.orchestra.run.smithi066.stdout:44 2024-03-04T07:27:12.567 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41, 44] 2024-03-04T07:27:12.567 DEBUG:teuthology.orchestra.run.smithi066:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:27:12.567 DEBUG:teuthology.orchestra.run.smithi066:> import glob 2024-03-04T07:27:12.567 DEBUG:teuthology.orchestra.run.smithi066:> import re 2024-03-04T07:27:12.567 DEBUG:teuthology.orchestra.run.smithi066:> import os 2024-03-04T07:27:12.567 DEBUG:teuthology.orchestra.run.smithi066:> import subprocess 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> def _find_admin_socket(client_name): 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> asok_path = "/var/run/ceph/ceph-client.3.*.asok" 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> files = glob.glob(asok_path) 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> mountpoint = "/home/ubuntu/cephtest/mnt.3" 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> # Given a non-glob path, it better be there 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> if "*" not in asok_path: 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> assert(len(files) == 1) 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> return files[0] 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> for f in files: 2024-03-04T07:27:12.568 DEBUG:teuthology.orchestra.run.smithi066:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> contents = proc_f.read() 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> if mountpoint in contents: 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> return f 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> print(_find_admin_socket("client.3")) 2024-03-04T07:27:12.569 DEBUG:teuthology.orchestra.run.smithi066:> ' 2024-03-04T07:27:12.707 INFO:teuthology.orchestra.run.smithi066.stdout:/var/run/ceph/ceph-client.3.44823.asok 2024-03-04T07:27:12.710 INFO:teuthology.orchestra.run.smithi066.stderr:2024-03-04T07:27:12 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:27:12.720 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.3.44823.asok 2024-03-04T07:27:12.720 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:12.720 DEBUG:teuthology.orchestra.run.smithi066:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.3.44823.asok status 2024-03-04T07:27:12.867 INFO:teuthology.orchestra.run.smithi066.stdout:{ 2024-03-04T07:27:12.867 INFO:teuthology.orchestra.run.smithi066.stdout: "metadata": { 2024-03-04T07:27:12.868 INFO:teuthology.orchestra.run.smithi066.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:27:12.868 INFO:teuthology.orchestra.run.smithi066.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:27:12.868 INFO:teuthology.orchestra.run.smithi066.stdout: "entity_id": "3", 2024-03-04T07:27:12.868 INFO:teuthology.orchestra.run.smithi066.stdout: "hostname": "smithi066", 2024-03-04T07:27:12.868 INFO:teuthology.orchestra.run.smithi066.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.3", 2024-03-04T07:27:12.869 INFO:teuthology.orchestra.run.smithi066.stdout: "pid": "44823", 2024-03-04T07:27:12.869 INFO:teuthology.orchestra.run.smithi066.stdout: "root": "/" 2024-03-04T07:27:12.869 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:12.869 INFO:teuthology.orchestra.run.smithi066.stdout: "dentry_count": 0, 2024-03-04T07:27:12.869 INFO:teuthology.orchestra.run.smithi066.stdout: "dentry_pinned_count": 0, 2024-03-04T07:27:12.869 INFO:teuthology.orchestra.run.smithi066.stdout: "id": 4499, 2024-03-04T07:27:12.870 INFO:teuthology.orchestra.run.smithi066.stdout: "inst": { 2024-03-04T07:27:12.870 INFO:teuthology.orchestra.run.smithi066.stdout: "name": { 2024-03-04T07:27:12.870 INFO:teuthology.orchestra.run.smithi066.stdout: "type": "client", 2024-03-04T07:27:12.870 INFO:teuthology.orchestra.run.smithi066.stdout: "num": 4499 2024-03-04T07:27:12.870 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:12.870 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": { 2024-03-04T07:27:12.871 INFO:teuthology.orchestra.run.smithi066.stdout: "type": "v1", 2024-03-04T07:27:12.871 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": "192.168.0.2:0", 2024-03-04T07:27:12.871 INFO:teuthology.orchestra.run.smithi066.stdout: "nonce": 896635265 2024-03-04T07:27:12.871 INFO:teuthology.orchestra.run.smithi066.stdout: } 2024-03-04T07:27:12.871 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:12.871 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": { 2024-03-04T07:27:12.872 INFO:teuthology.orchestra.run.smithi066.stdout: "type": "v1", 2024-03-04T07:27:12.872 INFO:teuthology.orchestra.run.smithi066.stdout: "addr": "192.168.0.2:0", 2024-03-04T07:27:12.872 INFO:teuthology.orchestra.run.smithi066.stdout: "nonce": 896635265 2024-03-04T07:27:12.872 INFO:teuthology.orchestra.run.smithi066.stdout: }, 2024-03-04T07:27:12.872 INFO:teuthology.orchestra.run.smithi066.stdout: "inst_str": "client.4499 192.168.0.2:0/896635265", 2024-03-04T07:27:12.872 INFO:teuthology.orchestra.run.smithi066.stdout: "addr_str": "192.168.0.2:0/896635265", 2024-03-04T07:27:12.873 INFO:teuthology.orchestra.run.smithi066.stdout: "inode_count": 1, 2024-03-04T07:27:12.873 INFO:teuthology.orchestra.run.smithi066.stdout: "mds_epoch": 7, 2024-03-04T07:27:12.873 INFO:teuthology.orchestra.run.smithi066.stdout: "osd_epoch": 20, 2024-03-04T07:27:12.873 INFO:teuthology.orchestra.run.smithi066.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:27:12.873 INFO:teuthology.orchestra.run.smithi066.stdout: "blocklisted": false, 2024-03-04T07:27:12.873 INFO:teuthology.orchestra.run.smithi066.stdout: "fs_name": "cephfs" 2024-03-04T07:27:12.874 INFO:teuthology.orchestra.run.smithi066.stdout:} 2024-03-04T07:27:12.875 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:12.875 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:27:12.875 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:12.894 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:27:12.894 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:12.894 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:12.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:12.966 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:12.967 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:27:12.967 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:13.023 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:27:13.023 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:13.024 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:13.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:13.095 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:13.095 DEBUG:teuthology.orchestra.run.smithi066:> stat --file-system '--printf=%T 2024-03-04T07:27:13.095 DEBUG:teuthology.orchestra.run.smithi066:> ' -- /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:13.110 INFO:teuthology.orchestra.run.smithi066.stdout:fuseblk 2024-03-04T07:27:13.111 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:13.111 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:13.111 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:13.182 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:13.182 DEBUG:teuthology.orchestra.run.smithi066:> stat --file-system '--printf=%T 2024-03-04T07:27:13.182 DEBUG:teuthology.orchestra.run.smithi066:> ' -- /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:13.239 INFO:teuthology.orchestra.run.smithi066.stdout:fuseblk 2024-03-04T07:27:13.240 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:13.240 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:13.240 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:13.313 INFO:teuthology.run_tasks:Running task cephfs_test_runner... 2024-03-04T07:27:13.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:13.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.444+0000 7f3085c5c640 1 Processor -- start 2024-03-04T07:27:13.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.444+0000 7f3085c5c640 1 -- start start 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.444+0000 7f3085c5c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f30800b2b50 0x7f30800b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.444+0000 7f3085c5c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f30800ce610 con 0x7f30800d3400 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.444+0000 7f3085c5c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f30800d2920 con 0x7f3080071540 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.444+0000 7f3085c5c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f30800d2f70 con 0x7f30800b2b50 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.445+0000 7f307ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f30800b2b50 0x7f30800b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.445+0000 7f307effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f30800d3400 0x7f30800b6e20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48356/0 (socket says 172.21.15.64:48356) 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.445+0000 7f307effd640 1 -- 172.21.15.64:0/740466830 learned_addr learned my addr 172.21.15.64:0/740466830 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.445+0000 7f307ffff640 1 -- 172.21.15.64:0/740466830 >> v1:172.21.15.66:6789/0 conn(0x7f3080071540 legacy=0x7f3080071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.445+0000 7f307ffff640 1 -- 172.21.15.64:0/740466830 >> v1:172.21.15.64:6789/0 conn(0x7f30800d3400 legacy=0x7f30800b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.445+0000 7f307ffff640 1 -- 172.21.15.64:0/740466830 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30800d2c00 con 0x7f30800b2b50 2024-03-04T07:27:13.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.445+0000 7f307e7fc640 1 -- 172.21.15.64:0/740466830 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 796555441 0 0) 0x7f30800ce610 con 0x7f30800d3400 2024-03-04T07:27:13.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.446+0000 7f307ffff640 1 --2- 172.21.15.64:0/740466830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30800b2b50 0x7f30800b30a0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f3078009d90 tx=0x7f307802bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d53b41ef75e208f1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.446+0000 7f307e7fc640 1 -- 172.21.15.64:0/740466830 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3078039070 con 0x7f30800b2b50 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.446+0000 7f307e7fc640 1 -- 172.21.15.64:0/740466830 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f307802cd00 con 0x7f30800b2b50 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.446+0000 7f307e7fc640 1 -- 172.21.15.64:0/740466830 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3078034660 con 0x7f30800b2b50 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.447+0000 7f3085c5c640 1 -- 172.21.15.64:0/740466830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30800b2b50 msgr2=0x7f30800b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.447+0000 7f3085c5c640 1 --2- 172.21.15.64:0/740466830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30800b2b50 0x7f30800b30a0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f3078009d90 tx=0x7f307802bf50 comp rx=0 tx=0).stop 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.447+0000 7f3085c5c640 1 -- 172.21.15.64:0/740466830 shutdown_connections 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.447+0000 7f3085c5c640 1 --2- 172.21.15.64:0/740466830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f30800b2b50 0x7f30800b30a0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.447+0000 7f3085c5c640 1 -- 172.21.15.64:0/740466830 >> 172.21.15.64:0/740466830 conn(0x7f30800d3ca0 msgr2=0x7f30800d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.447+0000 7f3085c5c640 1 -- 172.21.15.64:0/740466830 shutdown_connections 2024-03-04T07:27:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.447+0000 7f3085c5c640 1 -- 172.21.15.64:0/740466830 wait complete. 2024-03-04T07:27:13.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.448+0000 7f3085c5c640 1 Processor -- start 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.448+0000 7f3085c5c640 1 -- start start 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f3085c5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3080071540 0x7f30800c82a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f3085c5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30800b2b50 0x7f3080151110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f3085c5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30800d3400 0x7f308015be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f3085c5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f308015e370 con 0x7f3080071540 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f3085c5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30800d4be0 con 0x7f30800b2b50 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f3085c5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f30800c87e0 con 0x7f30800d3400 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3080071540 0x7f30800c82a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3080071540 0x7f30800c82a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39764/0 (socket says 172.21.15.64:39764) 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 -- 172.21.15.64:0/2364018019 learned_addr learned my addr 172.21.15.64:0/2364018019 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307effd640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30800b2b50 0x7f3080151110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307ffff640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30800d3400 0x7f308015be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 -- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30800d3400 msgr2=0x7f308015be60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30800d3400 0x7f308015be60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 -- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30800b2b50 msgr2=0x7f3080151110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30800b2b50 0x7f3080151110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 -- 172.21.15.64:0/2364018019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30800b3f70 con 0x7f3080071540 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.449+0000 7f307f7fe640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3080071540 0x7f30800c82a0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f307000c450 tx=0x7f307000c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.450+0000 7f3067fff640 1 -- 172.21.15.64:0/2364018019 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30700052d0 con 0x7f3080071540 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.450+0000 7f3067fff640 1 -- 172.21.15.64:0/2364018019 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3070005470 con 0x7f3080071540 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.450+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f30800c74e0 con 0x7f3080071540 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.450+0000 7f3067fff640 1 -- 172.21.15.64:0/2364018019 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3070012870 con 0x7f3080071540 2024-03-04T07:27:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.450+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f30800c7aa0 con 0x7f3080071540 2024-03-04T07:27:13.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.451+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3080068700 con 0x7f3080071540 2024-03-04T07:27:13.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.451+0000 7f3067fff640 1 -- 172.21.15.64:0/2364018019 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3070012a10 con 0x7f3080071540 2024-03-04T07:27:13.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.451+0000 7f3067fff640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f306c06b4d0 0x7f306c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.452+0000 7f307effd640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f306c06b4d0 0x7f306c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.452+0000 7f3067fff640 1 -- 172.21.15.64:0/2364018019 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3070089cb0 con 0x7f3080071540 2024-03-04T07:27:13.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.452+0000 7f307effd640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f306c06b4d0 0x7f306c06d990 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f308015caa0 tx=0x7f3074007e40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:13.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.456+0000 7f3067fff640 1 -- 172.21.15.64:0/2364018019 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f30700584e0 con 0x7f3080071540 2024-03-04T07:27:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.636+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f308006b8e0 con 0x7f3080071540 2024-03-04T07:27:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.637+0000 7f3067fff640 1 -- 172.21.15.64:0/2364018019 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f307005c580 con 0x7f3080071540 2024-03-04T07:27:13.637 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:13.637 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:13.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f306c06b4d0 msgr2=0x7f306c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f306c06b4d0 0x7f306c06d990 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f308015caa0 tx=0x7f3074007e40 comp rx=0 tx=0).stop 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3080071540 msgr2=0x7f30800c82a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3080071540 0x7f30800c82a0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f307000c450 tx=0x7f307000c910 comp rx=0 tx=0).stop 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 shutdown_connections 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f306c06b4d0 0x7f306c06d990 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f30800d3400 0x7f308015be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f30800b2b50 0x7f3080151110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 --2- 172.21.15.64:0/2364018019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3080071540 0x7f30800c82a0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 >> 172.21.15.64:0/2364018019 conn(0x7f30800d3ca0 msgr2=0x7f308015fa10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 shutdown_connections 2024-03-04T07:27:13.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.640+0000 7f3085c5c640 1 -- 172.21.15.64:0/2364018019 wait complete. 2024-03-04T07:27:13.650 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:13.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:13.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.755+0000 7fa979ec2640 1 Processor -- start 2024-03-04T07:27:13.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.755+0000 7fa979ec2640 1 -- start start 2024-03-04T07:27:13.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa979ec2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa97415dd90 0x7fa9740ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa979ec2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9740b6d10 0x7fa9740b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa979ec2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa9740d3740 con 0x7fa9740d29d0 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa979ec2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa9740d3400 con 0x7fa97415dd90 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa979ec2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9740d3120 con 0x7fa9740b6d10 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa9737fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa97415dd90 0x7fa9740ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa973fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9740b6d10 0x7fa9740b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa9737fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa97415dd90 0x7fa9740ceea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37516/0 (socket says 172.21.15.64:37516) 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.756+0000 7fa9737fe640 1 -- 172.21.15.64:0/1156963918 learned_addr learned my addr 172.21.15.64:0/1156963918 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.757+0000 7fa9737fe640 1 -- 172.21.15.64:0/1156963918 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9740b6d10 msgr2=0x7fa9740b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.757+0000 7fa9737fe640 1 --2- 172.21.15.64:0/1156963918 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9740b6d10 0x7fa9740b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.757+0000 7fa9737fe640 1 -- 172.21.15.64:0/1156963918 >> v1:172.21.15.66:6789/0 conn(0x7fa9740d29d0 legacy=0x7fa9740d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.757+0000 7fa9737fe640 1 -- 172.21.15.64:0/1156963918 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9740d39e0 con 0x7fa97415dd90 2024-03-04T07:27:13.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.757+0000 7fa9737fe640 1 --2- 172.21.15.64:0/1156963918 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa97415dd90 0x7fa9740ceea0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fa964009870 tx=0x7fa96402ba20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=67365b3c2667c575 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa9727fc640 1 -- 172.21.15.64:0/1156963918 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa96402f840 con 0x7fa97415dd90 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa9727fc640 1 -- 172.21.15.64:0/1156963918 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa96402f9e0 con 0x7fa97415dd90 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa9727fc640 1 -- 172.21.15.64:0/1156963918 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa96402fdb0 con 0x7fa97415dd90 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 -- 172.21.15.64:0/1156963918 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa97415dd90 msgr2=0x7fa9740ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 --2- 172.21.15.64:0/1156963918 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa97415dd90 0x7fa9740ceea0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fa964009870 tx=0x7fa96402ba20 comp rx=0 tx=0).stop 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 -- 172.21.15.64:0/1156963918 shutdown_connections 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 --2- 172.21.15.64:0/1156963918 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9740b6d10 0x7fa9740b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 --2- 172.21.15.64:0/1156963918 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa97415dd90 0x7fa9740ceea0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 -- 172.21.15.64:0/1156963918 >> 172.21.15.64:0/1156963918 conn(0x7fa974071960 msgr2=0x7fa974071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 -- 172.21.15.64:0/1156963918 shutdown_connections 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.758+0000 7fa979ec2640 1 -- 172.21.15.64:0/1156963918 wait complete. 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.759+0000 7fa979ec2640 1 Processor -- start 2024-03-04T07:27:13.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.759+0000 7fa979ec2640 1 -- start start 2024-03-04T07:27:13.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa979ec2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9740b6d10 0x7fa974160240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa979ec2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9740d29d0 0x7fa974151aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa979ec2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa97415dd90 0x7fa9740ca040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa979ec2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9740d2df0 con 0x7fa9740b6d10 2024-03-04T07:27:13.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa979ec2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa97415e170 con 0x7fa9740d29d0 2024-03-04T07:27:13.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa979ec2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9740d48f0 con 0x7fa97415dd90 2024-03-04T07:27:13.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa9737fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9740b6d10 0x7fa974160240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa9737fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9740b6d10 0x7fa974160240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39780/0 (socket says 172.21.15.64:39780) 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa9737fe640 1 -- 172.21.15.64:0/3161879292 learned_addr learned my addr 172.21.15.64:0/3161879292 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa972ffd640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9740d29d0 0x7fa974151aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.760+0000 7fa973fff640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa97415dd90 0x7fa9740ca040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa9737fe640 1 -- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa97415dd90 msgr2=0x7fa9740ca040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa9737fe640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa97415dd90 0x7fa9740ca040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa9737fe640 1 -- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9740d29d0 msgr2=0x7fa974151aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa9737fe640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9740d29d0 0x7fa974151aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa9737fe640 1 -- 172.21.15.64:0/3161879292 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa974160820 con 0x7fa9740b6d10 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa9737fe640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9740b6d10 0x7fa974160240 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fa964000c00 tx=0x7fa964004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa95bfff640 1 -- 172.21.15.64:0/3161879292 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa964037bc0 con 0x7fa9740b6d10 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa974160b00 con 0x7fa9740b6d10 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa95bfff640 1 -- 172.21.15.64:0/3161879292 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa964046070 con 0x7fa9740b6d10 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa95bfff640 1 -- 172.21.15.64:0/3161879292 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa964041400 con 0x7fa9740b6d10 2024-03-04T07:27:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.761+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9740c8b80 con 0x7fa9740b6d10 2024-03-04T07:27:13.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.762+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa974072a30 con 0x7fa9740b6d10 2024-03-04T07:27:13.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.762+0000 7fa95bfff640 1 -- 172.21.15.64:0/3161879292 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa964040070 con 0x7fa9740b6d10 2024-03-04T07:27:13.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.763+0000 7fa95bfff640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa96006b500 0x7fa96006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:13.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.763+0000 7fa95bfff640 1 -- 172.21.15.64:0/3161879292 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa9640ac690 con 0x7fa9740b6d10 2024-03-04T07:27:13.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.763+0000 7fa972ffd640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa96006b500 0x7fa96006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:13.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.763+0000 7fa972ffd640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa96006b500 0x7fa96006d9c0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fa9741f71c0 tx=0x7fa968009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:13.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.767+0000 7fa95bfff640 1 -- 172.21.15.64:0/3161879292 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa964033030 con 0x7fa9740b6d10 2024-03-04T07:27:13.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.945+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fa9740c8dc0 con 0x7fa9740b6d10 2024-03-04T07:27:13.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.945+0000 7fa95bfff640 1 -- 172.21.15.64:0/3161879292 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7fa96407aed0 con 0x7fa9740b6d10 2024-03-04T07:27:13.945 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:13.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa96006b500 msgr2=0x7fa96006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa96006b500 0x7fa96006d9c0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fa9741f71c0 tx=0x7fa968009040 comp rx=0 tx=0).stop 2024-03-04T07:27:13.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9740b6d10 msgr2=0x7fa974160240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9740b6d10 0x7fa974160240 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fa964000c00 tx=0x7fa964004730 comp rx=0 tx=0).stop 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 shutdown_connections 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa96006b500 0x7fa96006d9c0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa97415dd90 0x7fa9740ca040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9740d29d0 0x7fa974151aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 --2- 172.21.15.64:0/3161879292 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9740b6d10 0x7fa974160240 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.947+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 >> 172.21.15.64:0/3161879292 conn(0x7fa974071960 msgr2=0x7fa9740d0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.948+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 shutdown_connections 2024-03-04T07:27:13.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:13.948+0000 7fa979ec2640 1 -- 172.21.15.64:0/3161879292 wait complete. 2024-03-04T07:27:13.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:14.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.063+0000 7f7d611c7640 1 Processor -- start 2024-03-04T07:27:14.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.063+0000 7f7d611c7640 1 -- start start 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d611c7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7d5c0b3060 0x7f7d5c071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d611c7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d5c15e4b0 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d611c7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d5c0b97e0 con 0x7f7d5c0717b0 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d611c7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7d5c0b7c70 con 0x7f7d5c0b3060 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d53fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7d5c0b3060 0x7f7d5c071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d5b59e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7d5c0b2b10 0x7f7d5c0b7560 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48370/0 (socket says 172.21.15.64:48370) 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d53fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7d5c0b3060 0x7f7d5c071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39794/0 (socket says 172.21.15.64:39794) 2024-03-04T07:27:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d5b59e640 1 -- 172.21.15.64:0/2800269195 learned_addr learned my addr 172.21.15.64:0/2800269195 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:14.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d53fff640 1 -- 172.21.15.64:0/2800269195 >> v1:172.21.15.66:6789/0 conn(0x7f7d5c0717b0 legacy=0x7f7d5c0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d53fff640 1 -- 172.21.15.64:0/2800269195 >> v1:172.21.15.64:6789/0 conn(0x7f7d5c0b2b10 legacy=0x7f7d5c0b7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d53fff640 1 -- 172.21.15.64:0/2800269195 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d5c0b9a80 con 0x7f7d5c0b3060 2024-03-04T07:27:14.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.064+0000 7f7d5a59c640 1 -- 172.21.15.64:0/2800269195 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2141108270 0 0) 0x7f7d54001490 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d53fff640 1 --2- 172.21.15.64:0/2800269195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b3060 0x7f7d5c071b90 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f7d440097a0 tx=0x7f7d4402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=69ab7d0b4c907ad0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d5a59c640 1 -- 172.21.15.64:0/2800269195 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d44034070 con 0x7f7d5c0b3060 2024-03-04T07:27:14.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d5a59c640 1 -- 172.21.15.64:0/2800269195 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d44030610 con 0x7f7d5c0b3060 2024-03-04T07:27:14.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d5a59c640 1 -- 172.21.15.64:0/2800269195 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d440309e0 con 0x7f7d5c0b3060 2024-03-04T07:27:14.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d611c7640 1 -- 172.21.15.64:0/2800269195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b3060 msgr2=0x7f7d5c071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d611c7640 1 --2- 172.21.15.64:0/2800269195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b3060 0x7f7d5c071b90 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f7d440097a0 tx=0x7f7d4402b960 comp rx=0 tx=0).stop 2024-03-04T07:27:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d611c7640 1 -- 172.21.15.64:0/2800269195 shutdown_connections 2024-03-04T07:27:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d611c7640 1 --2- 172.21.15.64:0/2800269195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b3060 0x7f7d5c071b90 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.065+0000 7f7d611c7640 1 -- 172.21.15.64:0/2800269195 >> 172.21.15.64:0/2800269195 conn(0x7f7d5c0d3d20 msgr2=0x7f7d5c0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.066+0000 7f7d611c7640 1 -- 172.21.15.64:0/2800269195 shutdown_connections 2024-03-04T07:27:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.066+0000 7f7d611c7640 1 -- 172.21.15.64:0/2800269195 wait complete. 2024-03-04T07:27:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.067+0000 7f7d611c7640 1 Processor -- start 2024-03-04T07:27:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.067+0000 7f7d611c7640 1 -- start start 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.067+0000 7f7d611c7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d5c0717b0 0x7f7d5c146b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.067+0000 7f7d611c7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b2b10 0x7f7d5c1474c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d611c7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d5c0b3060 0x7f7d5c147a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d611c7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d5c0d4bf0 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d611c7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d5c0d2e10 con 0x7f7d5c0717b0 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d611c7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d5c0d32a0 con 0x7f7d5c0b3060 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b2b10 0x7f7d5c1474c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b2b10 0x7f7d5c1474c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39806/0 (socket says 172.21.15.64:39806) 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 -- 172.21.15.64:0/3837917061 learned_addr learned my addr 172.21.15.64:0/3837917061 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d5ad9d640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d5c0717b0 0x7f7d5c146b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d5b59e640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d5c0b3060 0x7f7d5c147a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 -- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d5c0b3060 msgr2=0x7f7d5c147a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d5c0b3060 0x7f7d5c147a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 -- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d5c0717b0 msgr2=0x7f7d5c146b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d5c0717b0 0x7f7d5c146b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 -- 172.21.15.64:0/3837917061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d5c159870 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d53fff640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b2b10 0x7f7d5c1474c0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f7d44009890 tx=0x7f7d440046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d537fe640 1 -- 172.21.15.64:0/3837917061 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d44034040 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d537fe640 1 -- 172.21.15.64:0/3837917061 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d4403acc0 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d5c148030 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d537fe640 1 -- 172.21.15.64:0/3837917061 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d44044630 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.068+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d5c1485f0 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.069+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d5c0b6940 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.070+0000 7f7d537fe640 1 -- 172.21.15.64:0/3837917061 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7d4404b060 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.070+0000 7f7d537fe640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d3c06b4d0 0x7f7d3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.070+0000 7f7d5ad9d640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d3c06b4d0 0x7f7d3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.070+0000 7f7d537fe640 1 -- 172.21.15.64:0/3837917061 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7d440abf90 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.071+0000 7f7d5ad9d640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d3c06b4d0 0x7f7d3c06d990 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f7d4c000f60 tx=0x7f7d4c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.075+0000 7f7d537fe640 1 -- 172.21.15.64:0/3837917061 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7d5c0b6940 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.256+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f7d5c04f410 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.256+0000 7f7d537fe640 1 -- 172.21.15.64:0/3837917061 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f7d4403f070 con 0x7f7d5c0b2b10 2024-03-04T07:27:14.256 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d3c06b4d0 msgr2=0x7f7d3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d3c06b4d0 0x7f7d3c06d990 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f7d4c000f60 tx=0x7f7d4c009040 comp rx=0 tx=0).stop 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b2b10 msgr2=0x7f7d5c1474c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b2b10 0x7f7d5c1474c0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f7d44009890 tx=0x7f7d440046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 shutdown_connections 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d3c06b4d0 0x7f7d3c06d990 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d5c0b3060 0x7f7d5c147a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d5c0b2b10 0x7f7d5c1474c0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 --2- 172.21.15.64:0/3837917061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d5c0717b0 0x7f7d5c146b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 >> 172.21.15.64:0/3837917061 conn(0x7f7d5c0d3d20 msgr2=0x7f7d5c068e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 shutdown_connections 2024-03-04T07:27:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.258+0000 7f7d611c7640 1 -- 172.21.15.64:0/3837917061 wait complete. 2024-03-04T07:27:14.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:14.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.374+0000 7f8cb273a640 1 Processor -- start 2024-03-04T07:27:14.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.374+0000 7f8cb273a640 1 -- start start 2024-03-04T07:27:14.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cb273a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8cac15d2c0 0x7f8cac15d9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cb273a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8cac0b4340 0x7f8cac0b2ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cb273a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac15fb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cb273a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8cac0d0a30 con 0x7f8cac0b4340 2024-03-04T07:27:14.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cb273a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8cac0d01d0 con 0x7f8cac15d2c0 2024-03-04T07:27:14.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cb273a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8cac0cc5c0 con 0x7f8cac0b3000 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8cac0b4340 0x7f8cac0b2ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8cac0b4340 0x7f8cac0b2ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39820/0 (socket says 172.21.15.64:39820) 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 -- 172.21.15.64:0/1998096032 learned_addr learned my addr 172.21.15.64:0/1998096032 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cb0cd7640 1 --2- 172.21.15.64:0/1998096032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac15fb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cabfff640 1 --2- 172.21.15.64:0/1998096032 >> v2:172.21.15.66:3300/0 conn(0x7f8cac15d2c0 0x7f8cac15d9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 -- 172.21.15.64:0/1998096032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 msgr2=0x7f8cac15fb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 --2- 172.21.15.64:0/1998096032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac15fb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 -- 172.21.15.64:0/1998096032 >> v2:172.21.15.66:3300/0 conn(0x7f8cac15d2c0 msgr2=0x7f8cac15d9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 --2- 172.21.15.64:0/1998096032 >> v2:172.21.15.66:3300/0 conn(0x7f8cac15d2c0 0x7f8cac15d9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.375+0000 7f8cab7fe640 1 -- 172.21.15.64:0/1998096032 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8cac0d0470 con 0x7f8cac0b4340 2024-03-04T07:27:14.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.376+0000 7f8cab7fe640 1 --2- 172.21.15.64:0/1998096032 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac0b4340 0x7f8cac0b2ac0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f8cac067ce0 tx=0x7f8c9c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6bb092fefc44b16b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.376+0000 7f8caaffd640 1 -- 172.21.15.64:0/1998096032 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c9c035020 con 0x7f8cac0b4340 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.376+0000 7f8caaffd640 1 -- 172.21.15.64:0/1998096032 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c9c031870 con 0x7f8cac0b4340 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.376+0000 7f8caaffd640 1 -- 172.21.15.64:0/1998096032 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c9c031c40 con 0x7f8cac0b4340 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 -- 172.21.15.64:0/1998096032 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac0b4340 msgr2=0x7f8cac0b2ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 --2- 172.21.15.64:0/1998096032 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac0b4340 0x7f8cac0b2ac0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f8cac067ce0 tx=0x7f8c9c02b910 comp rx=0 tx=0).stop 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 -- 172.21.15.64:0/1998096032 shutdown_connections 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 --2- 172.21.15.64:0/1998096032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac15fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 --2- 172.21.15.64:0/1998096032 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac0b4340 0x7f8cac0b2ac0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 --2- 172.21.15.64:0/1998096032 >> v2:172.21.15.66:3300/0 conn(0x7f8cac15d2c0 0x7f8cac15d9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 -- 172.21.15.64:0/1998096032 >> 172.21.15.64:0/1998096032 conn(0x7f8cac0b3660 msgr2=0x7f8cac0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 -- 172.21.15.64:0/1998096032 shutdown_connections 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.377+0000 7f8cb273a640 1 -- 172.21.15.64:0/1998096032 wait complete. 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.378+0000 7f8cb273a640 1 Processor -- start 2024-03-04T07:27:14.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.378+0000 7f8cb273a640 1 -- start start 2024-03-04T07:27:14.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb273a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac14c9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb273a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8cac0b4340 0x7f8cac0c5680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb273a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac15d2c0 0x7f8cac14bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb273a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cac1600a0 con 0x7f8cac15d2c0 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb273a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cac0cc860 con 0x7f8cac0b4340 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb273a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8cac160220 con 0x7f8cac0b3000 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac15d2c0 0x7f8cac14bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac15d2c0 0x7f8cac14bf20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39826/0 (socket says 172.21.15.64:39826) 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cabfff640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac14c9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 -- 172.21.15.64:0/3129963680 learned_addr learned my addr 172.21.15.64:0/3129963680 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cab7fe640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8cac0b4340 0x7f8cac0c5680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 -- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 msgr2=0x7f8cac14c9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac14c9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 -- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8cac0b4340 msgr2=0x7f8cac0c5680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8cac0b4340 0x7f8cac0c5680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cb0cd7640 1 -- 172.21.15.64:0/3129963680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8cac0b5a30 con 0x7f8cac15d2c0 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.379+0000 7f8cab7fe640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8cac0b4340 0x7f8cac0c5680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:14.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.380+0000 7f8cb0cd7640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac15d2c0 0x7f8cac14bf20 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f8ca400efc0 tx=0x7f8ca400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.380+0000 7f8ca8ff9640 1 -- 172.21.15.64:0/3129963680 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ca4018020 con 0x7f8cac15d2c0 2024-03-04T07:27:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.380+0000 7f8ca8ff9640 1 -- 172.21.15.64:0/3129963680 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ca4014480 con 0x7f8cac15d2c0 2024-03-04T07:27:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.380+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8cac0b5d10 con 0x7f8cac15d2c0 2024-03-04T07:27:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.380+0000 7f8ca8ff9640 1 -- 172.21.15.64:0/3129963680 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ca4014870 con 0x7f8cac15d2c0 2024-03-04T07:27:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.380+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8cac14c460 con 0x7f8cac15d2c0 2024-03-04T07:27:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.381+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8cac09fc50 con 0x7f8cac15d2c0 2024-03-04T07:27:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.381+0000 7f8ca8ff9640 1 -- 172.21.15.64:0/3129963680 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8ca4005d10 con 0x7f8cac15d2c0 2024-03-04T07:27:14.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.382+0000 7f8ca8ff9640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c9806b4d0 0x7f8c9806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.382+0000 7f8ca8ff9640 1 -- 172.21.15.64:0/3129963680 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8ca408aeb0 con 0x7f8cac15d2c0 2024-03-04T07:27:14.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.382+0000 7f8cabfff640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c9806b4d0 0x7f8c9806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.382+0000 7f8cabfff640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c9806b4d0 0x7f8c9806d990 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f8ca0009a20 tx=0x7f8ca0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.386+0000 7f8ca8ff9640 1 -- 172.21.15.64:0/3129963680 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8ca4022030 con 0x7f8cac15d2c0 2024-03-04T07:27:14.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.567+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f8cac072cb0 con 0x7f8cac15d2c0 2024-03-04T07:27:14.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.567+0000 7f8ca8ff9640 1 -- 172.21.15.64:0/3129963680 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f8ca4010040 con 0x7f8cac15d2c0 2024-03-04T07:27:14.567 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:14.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c9806b4d0 msgr2=0x7f8c9806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c9806b4d0 0x7f8c9806d990 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f8ca0009a20 tx=0x7f8ca0009190 comp rx=0 tx=0).stop 2024-03-04T07:27:14.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac15d2c0 msgr2=0x7f8cac14bf20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac15d2c0 0x7f8cac14bf20 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f8ca400efc0 tx=0x7f8ca400c490 comp rx=0 tx=0).stop 2024-03-04T07:27:14.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 shutdown_connections 2024-03-04T07:27:14.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c9806b4d0 0x7f8c9806d990 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8cac15d2c0 0x7f8cac14bf20 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8cac0b4340 0x7f8cac0c5680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 --2- 172.21.15.64:0/3129963680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8cac0b3000 0x7f8cac14c9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.569+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 >> 172.21.15.64:0/3129963680 conn(0x7f8cac0b3660 msgr2=0x7f8cac06ef80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:14.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.570+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 shutdown_connections 2024-03-04T07:27:14.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.570+0000 7f8cb273a640 1 -- 172.21.15.64:0/3129963680 wait complete. 2024-03-04T07:27:14.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:14.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.682+0000 7ff469839640 1 Processor -- start 2024-03-04T07:27:14.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.682+0000 7ff469839640 1 -- start start 2024-03-04T07:27:14.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff469839640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff464071c50 0x7ff4640b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff469839640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4640b6f20 con 0x7ff46415df70 2024-03-04T07:27:14.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff469839640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4640d3030 con 0x7ff4640d18c0 2024-03-04T07:27:14.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff469839640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4640d3310 con 0x7ff464071c50 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff462ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff46415df70 0x7ff4640d27b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48372/0 (socket says 172.21.15.64:48372) 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff462ffd640 1 -- 172.21.15.64:0/328856144 learned_addr learned my addr 172.21.15.64:0/328856144 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff4637fe640 1 --2- 172.21.15.64:0/328856144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff464071c50 0x7ff4640b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff461ffb640 1 -- 172.21.15.64:0/328856144 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3671616843 0 0) 0x7ff4640b6f20 con 0x7ff46415df70 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff461ffb640 1 -- 172.21.15.64:0/328856144 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff450003520 con 0x7ff46415df70 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff461ffb640 1 -- 172.21.15.64:0/328856144 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2195709477 0 0) 0x7ff4640d3030 con 0x7ff4640d18c0 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff461ffb640 1 -- 172.21.15.64:0/328856144 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff4640b6f20 con 0x7ff4640d18c0 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff4637fe640 1 -- 172.21.15.64:0/328856144 >> v1:172.21.15.66:6789/0 conn(0x7ff4640d18c0 legacy=0x7ff464071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff4637fe640 1 -- 172.21.15.64:0/328856144 >> v1:172.21.15.64:6789/0 conn(0x7ff46415df70 legacy=0x7ff4640d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.683+0000 7ff4637fe640 1 -- 172.21.15.64:0/328856144 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4640d35f0 con 0x7ff464071c50 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.684+0000 7ff4637fe640 1 --2- 172.21.15.64:0/328856144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff464071c50 0x7ff4640b2700 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7ff454009dc0 tx=0x7ff45402bfa0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7a8005446bb55933 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.684+0000 7ff461ffb640 1 -- 172.21.15.64:0/328856144 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff454039070 con 0x7ff464071c50 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.684+0000 7ff461ffb640 1 -- 172.21.15.64:0/328856144 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff45402cd50 con 0x7ff464071c50 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.684+0000 7ff461ffb640 1 -- 172.21.15.64:0/328856144 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff454034640 con 0x7ff464071c50 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.685+0000 7ff469839640 1 -- 172.21.15.64:0/328856144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff464071c50 msgr2=0x7ff4640b2700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.685+0000 7ff469839640 1 --2- 172.21.15.64:0/328856144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff464071c50 0x7ff4640b2700 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7ff454009dc0 tx=0x7ff45402bfa0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.685+0000 7ff469839640 1 -- 172.21.15.64:0/328856144 shutdown_connections 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.685+0000 7ff469839640 1 --2- 172.21.15.64:0/328856144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff464071c50 0x7ff4640b2700 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.685+0000 7ff469839640 1 -- 172.21.15.64:0/328856144 >> 172.21.15.64:0/328856144 conn(0x7ff4640d3cf0 msgr2=0x7ff4640d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:14.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.685+0000 7ff469839640 1 -- 172.21.15.64:0/328856144 shutdown_connections 2024-03-04T07:27:14.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.685+0000 7ff469839640 1 -- 172.21.15.64:0/328856144 wait complete. 2024-03-04T07:27:14.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.686+0000 7ff469839640 1 Processor -- start 2024-03-04T07:27:14.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.686+0000 7ff469839640 1 -- start start 2024-03-04T07:27:14.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff469839640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff464071c50 0x7ff4640cc2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff469839640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4640d18c0 0x7ff4640b91f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff469839640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff46415df70 0x7ff4641f3c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff469839640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff46415e4d0 con 0x7ff4640d18c0 2024-03-04T07:27:14.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff469839640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff46415e350 con 0x7ff464071c50 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff469839640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4640b8410 con 0x7ff46415df70 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4640d18c0 0x7ff4640b91f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4640d18c0 0x7ff4640b91f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39834/0 (socket says 172.21.15.64:39834) 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 -- 172.21.15.64:0/4053012293 learned_addr learned my addr 172.21.15.64:0/4053012293 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff462ffd640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff464071c50 0x7ff4640cc2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4637fe640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff46415df70 0x7ff4641f3c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 -- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff46415df70 msgr2=0x7ff4641f3c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff46415df70 0x7ff4641f3c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 -- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff464071c50 msgr2=0x7ff4640cc2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff464071c50 0x7ff4640cc2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 -- 172.21.15.64:0/4053012293 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4640cb4e0 con 0x7ff4640d18c0 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff462ffd640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff464071c50 0x7ff4640cc2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.687+0000 7ff4627fc640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4640d18c0 0x7ff4640b91f0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff44c00c500 tx=0x7ff44c00c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.688+0000 7ff45bfff640 1 -- 172.21.15.64:0/4053012293 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff44c0052d0 con 0x7ff4640d18c0 2024-03-04T07:27:14.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.688+0000 7ff45bfff640 1 -- 172.21.15.64:0/4053012293 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff44c005490 con 0x7ff4640d18c0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.688+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4640cb7c0 con 0x7ff4640d18c0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.688+0000 7ff45bfff640 1 -- 172.21.15.64:0/4053012293 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff44c012870 con 0x7ff4640d18c0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.688+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4640caf50 con 0x7ff4640d18c0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.689+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff464067ff0 con 0x7ff4640d18c0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.689+0000 7ff45bfff640 1 -- 172.21.15.64:0/4053012293 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff44c012a10 con 0x7ff4640d18c0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.689+0000 7ff45bfff640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44406b4d0 0x7ff44406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.690+0000 7ff45bfff640 1 -- 172.21.15.64:0/4053012293 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff44c089da0 con 0x7ff4640d18c0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.690+0000 7ff462ffd640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44406b4d0 0x7ff44406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.690+0000 7ff462ffd640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44406b4d0 0x7ff44406d990 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7ff45c0011e0 tx=0x7ff45c007d30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.694+0000 7ff45bfff640 1 -- 172.21.15.64:0/4053012293 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff44c0585e0 con 0x7ff4640d18c0 2024-03-04T07:27:14.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.875+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7ff4640b3080 con 0x7ff4640d18c0 2024-03-04T07:27:14.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.876+0000 7ff45bfff640 1 -- 172.21.15.64:0/4053012293 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7ff44c05c680 con 0x7ff4640d18c0 2024-03-04T07:27:14.875 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:14.876 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:14.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44406b4d0 msgr2=0x7ff44406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44406b4d0 0x7ff44406d990 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7ff45c0011e0 tx=0x7ff45c007d30 comp rx=0 tx=0).stop 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4640d18c0 msgr2=0x7ff4640b91f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4640d18c0 0x7ff4640b91f0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff44c00c500 tx=0x7ff44c00c9c0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 shutdown_connections 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44406b4d0 0x7ff44406d990 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff46415df70 0x7ff4641f3c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4640d18c0 0x7ff4640b91f0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 --2- 172.21.15.64:0/4053012293 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff464071c50 0x7ff4640cc2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 >> 172.21.15.64:0/4053012293 conn(0x7ff4640d3cf0 msgr2=0x7ff4640b5fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 shutdown_connections 2024-03-04T07:27:14.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.878+0000 7ff469839640 1 -- 172.21.15.64:0/4053012293 wait complete. 2024-03-04T07:27:14.888 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:14.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:14.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.994+0000 7f965709a640 1 Processor -- start 2024-03-04T07:27:14.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.994+0000 7f965709a640 1 -- start start 2024-03-04T07:27:14.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.994+0000 7f965709a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f965015f450 0x7f96500cf690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.994+0000 7f965709a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96500d0930 0x7f96500cfc50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.994+0000 7f965709a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f96500cd310 con 0x7f96500d0190 2024-03-04T07:27:14.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.994+0000 7f965709a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96500b1430 con 0x7f965015f450 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.994+0000 7f965709a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f965006ec20 con 0x7f96500d0930 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f9654e36640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f965015f450 0x7f96500cf690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f964ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96500d0930 0x7f96500cfc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f9655637640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f96500d0190 0x7f96500ce370 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:41322/0 (socket says 172.21.15.64:41322) 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f9655637640 1 -- 172.21.15.64:0/2012397041 learned_addr learned my addr 172.21.15.64:0/2012397041 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f9654e36640 1 -- 172.21.15.64:0/2012397041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96500d0930 msgr2=0x7f96500cfc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f9654e36640 1 --2- 172.21.15.64:0/2012397041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96500d0930 0x7f96500cfc50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f9654e36640 1 -- 172.21.15.64:0/2012397041 >> v1:172.21.15.66:6789/0 conn(0x7f96500d0190 legacy=0x7f96500ce370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.995+0000 7f9654e36640 1 -- 172.21.15.64:0/2012397041 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96500cd5f0 con 0x7f965015f450 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f9654e36640 1 --2- 172.21.15.64:0/2012397041 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f965015f450 0x7f96500cf690 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f96440097a0 tx=0x7f964402b7a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e872a980dde58136 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f964f7fe640 1 -- 172.21.15.64:0/2012397041 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9644038070 con 0x7f965015f450 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f964f7fe640 1 -- 172.21.15.64:0/2012397041 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9644031700 con 0x7f965015f450 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f964f7fe640 1 -- 172.21.15.64:0/2012397041 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9644031ad0 con 0x7f965015f450 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 -- 172.21.15.64:0/2012397041 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f965015f450 msgr2=0x7f96500cf690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 --2- 172.21.15.64:0/2012397041 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f965015f450 0x7f96500cf690 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f96440097a0 tx=0x7f964402b7a0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 -- 172.21.15.64:0/2012397041 shutdown_connections 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 --2- 172.21.15.64:0/2012397041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96500d0930 0x7f96500cfc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 --2- 172.21.15.64:0/2012397041 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f965015f450 0x7f96500cf690 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 -- 172.21.15.64:0/2012397041 >> 172.21.15.64:0/2012397041 conn(0x7f96500b3670 msgr2=0x7f96500b3a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 -- 172.21.15.64:0/2012397041 shutdown_connections 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.996+0000 7f965709a640 1 -- 172.21.15.64:0/2012397041 wait complete. 2024-03-04T07:27:14.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.997+0000 7f965709a640 1 Processor -- start 2024-03-04T07:27:14.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.997+0000 7f965709a640 1 -- start start 2024-03-04T07:27:14.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.998+0000 7f965709a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96500d0190 0x7f96500b45b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:14.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.998+0000 7f965709a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96500d0930 0x7f965015df60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.998+0000 7f965709a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965015f450 0x7f965014db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.998+0000 7f965709a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96500b1140 con 0x7f96500d0930 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.998+0000 7f965709a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96500b0fc0 con 0x7f96500d0190 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.998+0000 7f965709a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96500700c0 con 0x7f965015f450 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.998+0000 7f964ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96500d0930 0x7f965015df60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96500d0190 0x7f96500b45b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9655637640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965015f450 0x7f965014db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96500d0190 0x7f96500b45b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37576/0 (socket says 172.21.15.64:37576) 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 -- 172.21.15.64:0/81445141 learned_addr learned my addr 172.21.15.64:0/81445141 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:15.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f964ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96500d0930 0x7f965015df60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39838/0 (socket says 172.21.15.64:39838) 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 -- 172.21.15.64:0/81445141 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965015f450 msgr2=0x7f965014db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965015f450 0x7f965014db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 -- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96500d0930 msgr2=0x7f965015df60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96500d0930 0x7f965015df60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 -- 172.21.15.64:0/81445141 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96500b4f00 con 0x7f96500d0190 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f9654e36640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96500d0190 0x7f96500b45b0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f96440096f0 tx=0x7f96440046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f964d7fa640 1 -- 172.21.15.64:0/81445141 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9644042070 con 0x7f96500d0190 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f964d7fa640 1 -- 172.21.15.64:0/81445141 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f964403d870 con 0x7f96500d0190 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96500b51e0 con 0x7f96500d0190 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f964d7fa640 1 -- 172.21.15.64:0/81445141 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9644038040 con 0x7f96500d0190 2024-03-04T07:27:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:14.999+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96500b5a80 con 0x7f96500d0190 2024-03-04T07:27:15.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.000+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f965009fcd0 con 0x7f96500d0190 2024-03-04T07:27:15.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.001+0000 7f964d7fa640 1 -- 172.21.15.64:0/81445141 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9644045070 con 0x7f96500d0190 2024-03-04T07:27:15.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.001+0000 7f964d7fa640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963406b4d0 0x7f963406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.001+0000 7f964d7fa640 1 -- 172.21.15.64:0/81445141 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f96440abf10 con 0x7f96500d0190 2024-03-04T07:27:15.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.001+0000 7f964ffff640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963406b4d0 0x7f963406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.002+0000 7f964ffff640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963406b4d0 0x7f963406d990 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f965015afe0 tx=0x7f963c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.006+0000 7f964d7fa640 1 -- 172.21.15.64:0/81445141 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9644030070 con 0x7f96500d0190 2024-03-04T07:27:15.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.188+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f965015ec70 con 0x7f96500d0190 2024-03-04T07:27:15.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.189+0000 7f964d7fa640 1 -- 172.21.15.64:0/81445141 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f964407a860 con 0x7f96500d0190 2024-03-04T07:27:15.188 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:15.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.190+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963406b4d0 msgr2=0x7f963406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.190+0000 7f965709a640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963406b4d0 0x7f963406d990 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f965015afe0 tx=0x7f963c009190 comp rx=0 tx=0).stop 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.190+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96500d0190 msgr2=0x7f96500b45b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96500d0190 0x7f96500b45b0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f96440096f0 tx=0x7f96440046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 shutdown_connections 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963406b4d0 0x7f963406d990 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965015f450 0x7f965014db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96500d0930 0x7f965015df60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 --2- 172.21.15.64:0/81445141 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96500d0190 0x7f96500b45b0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 >> 172.21.15.64:0/81445141 conn(0x7f96500b3670 msgr2=0x7f965015cf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 shutdown_connections 2024-03-04T07:27:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.191+0000 7f965709a640 1 -- 172.21.15.64:0/81445141 wait complete. 2024-03-04T07:27:15.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:15.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.306+0000 7f9fd0d5c640 1 Processor -- start 2024-03-04T07:27:15.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.306+0000 7f9fd0d5c640 1 -- start start 2024-03-04T07:27:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fd0d5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc142170 0x7f9fcc142a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fd0d5c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9fcc0b4830 0x7f9fcc160280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fd0d5c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9fcc070980 con 0x7f9fcc0b29a0 2024-03-04T07:27:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fd0d5c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9fcc1458e0 con 0x7f9fcc0b4830 2024-03-04T07:27:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fd0d5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9fcc1607c0 con 0x7f9fcc142170 2024-03-04T07:27:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fc9d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9fcc0b4830 0x7f9fcc160280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fcad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9fcc0b29a0 0x7f9fcc0716d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48376/0 (socket says 172.21.15.64:48376) 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fcad9d640 1 -- 172.21.15.64:0/3456429350 learned_addr learned my addr 172.21.15.64:0/3456429350 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fca59c640 1 --2- 172.21.15.64:0/3456429350 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc142170 0x7f9fcc142a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.307+0000 7f9fc9d9b640 1 -- 172.21.15.64:0/3456429350 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc142170 msgr2=0x7f9fcc142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc9d9b640 1 --2- 172.21.15.64:0/3456429350 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc142170 0x7f9fcc142a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc9d9b640 1 -- 172.21.15.64:0/3456429350 >> v1:172.21.15.64:6789/0 conn(0x7f9fcc0b29a0 legacy=0x7f9fcc0716d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc9d9b640 1 -- 172.21.15.64:0/3456429350 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9fcc142f60 con 0x7f9fcc0b4830 2024-03-04T07:27:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc959a640 1 -- 172.21.15.64:0/3456429350 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3179242555 0 0) 0x7f9fcc070980 con 0x7f9fcc0b29a0 2024-03-04T07:27:15.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc9d9b640 1 --2- 172.21.15.64:0/3456429350 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc160280 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f9fb40097a0 tx=0x7f9fb402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=954193ac33885a02 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc959a640 1 -- 172.21.15.64:0/3456429350 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fb4035020 con 0x7f9fcc0b4830 2024-03-04T07:27:15.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc959a640 1 -- 172.21.15.64:0/3456429350 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9fb4031830 con 0x7f9fcc0b4830 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.308+0000 7f9fc959a640 1 -- 172.21.15.64:0/3456429350 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fb4031c00 con 0x7f9fcc0b4830 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3456429350 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 msgr2=0x7f9fcc160280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3456429350 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc160280 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f9fb40097a0 tx=0x7f9fb402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3456429350 shutdown_connections 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3456429350 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc160280 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3456429350 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc142170 0x7f9fcc142a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3456429350 >> 172.21.15.64:0/3456429350 conn(0x7f9fcc066d00 msgr2=0x7f9fcc067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:15.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3456429350 shutdown_connections 2024-03-04T07:27:15.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.309+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3456429350 wait complete. 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.310+0000 7f9fd0d5c640 1 Processor -- start 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.310+0000 7f9fd0d5c640 1 -- start start 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fd0d5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc0b29a0 0x7f9fcc15b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fd0d5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc1438f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fd0d5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fcc142170 0x7f9fcc143e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fd0d5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9fcc0711b0 con 0x7f9fcc0b4830 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fd0d5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9fcc070ed0 con 0x7f9fcc142170 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fd0d5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9fcc0b4df0 con 0x7f9fcc0b29a0 2024-03-04T07:27:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fc9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc1438f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fc9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc1438f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39866/0 (socket says 172.21.15.64:39866) 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fc9d9b640 1 -- 172.21.15.64:0/3353154245 learned_addr learned my addr 172.21.15.64:0/3353154245 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fca59c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc0b29a0 0x7f9fcc15b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.311+0000 7f9fcad9d640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fcc142170 0x7f9fcc143e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc9d9b640 1 -- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc0b29a0 msgr2=0x7f9fcc15b4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc9d9b640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc0b29a0 0x7f9fcc15b4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc9d9b640 1 -- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fcc142170 msgr2=0x7f9fcc143e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc9d9b640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fcc142170 0x7f9fcc143e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc9d9b640 1 -- 172.21.15.64:0/3353154245 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9fcc1e6710 con 0x7f9fcc0b4830 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc9d9b640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc1438f0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f9fb402bee0 tx=0x7f9fb40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc2ffd640 1 -- 172.21.15.64:0/3353154245 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fb4035040 con 0x7f9fcc0b4830 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc2ffd640 1 -- 172.21.15.64:0/3353154245 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9fb403b920 con 0x7f9fcc0b4830 2024-03-04T07:27:15.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9fcc1e69f0 con 0x7f9fcc0b4830 2024-03-04T07:27:15.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fc2ffd640 1 -- 172.21.15.64:0/3353154245 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9fb403bd10 con 0x7f9fcc0b4830 2024-03-04T07:27:15.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.312+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9fcc1e6fb0 con 0x7f9fcc0b4830 2024-03-04T07:27:15.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.313+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9fcc0733c0 con 0x7f9fcc0b4830 2024-03-04T07:27:15.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.313+0000 7f9fc2ffd640 1 -- 172.21.15.64:0/3353154245 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9fb4038020 con 0x7f9fcc0b4830 2024-03-04T07:27:15.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.314+0000 7f9fc2ffd640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fac06b4d0 0x7f9fac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.314+0000 7f9fca59c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fac06b4d0 0x7f9fac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.314+0000 7f9fc2ffd640 1 -- 172.21.15.64:0/3353154245 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9fb40ac1e0 con 0x7f9fcc0b4830 2024-03-04T07:27:15.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.314+0000 7f9fca59c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fac06b4d0 0x7f9fac06d990 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f9fc4009b40 tx=0x7f9fc4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.318+0000 7f9fc2ffd640 1 -- 172.21.15.64:0/3353154245 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9fb4031800 con 0x7f9fcc0b4830 2024-03-04T07:27:15.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.502+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f9fcc0cef70 con 0x7f9fcc0b4830 2024-03-04T07:27:15.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.503+0000 7f9fc2ffd640 1 -- 172.21.15.64:0/3353154245 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f9fb407aa20 con 0x7f9fcc0b4830 2024-03-04T07:27:15.503 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:15.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fac06b4d0 msgr2=0x7f9fac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fac06b4d0 0x7f9fac06d990 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f9fc4009b40 tx=0x7f9fc4009190 comp rx=0 tx=0).stop 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 msgr2=0x7f9fcc1438f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc1438f0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f9fb402bee0 tx=0x7f9fb40046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 shutdown_connections 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9fac06b4d0 0x7f9fac06d990 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9fcc142170 0x7f9fcc143e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9fcc0b4830 0x7f9fcc1438f0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 --2- 172.21.15.64:0/3353154245 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9fcc0b29a0 0x7f9fcc15b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 >> 172.21.15.64:0/3353154245 conn(0x7f9fcc066d00 msgr2=0x7f9fcc0715d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 shutdown_connections 2024-03-04T07:27:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.505+0000 7f9fd0d5c640 1 -- 172.21.15.64:0/3353154245 wait complete. 2024-03-04T07:27:15.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:15.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.620+0000 7f3231815640 1 Processor -- start 2024-03-04T07:27:15.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.620+0000 7f3231815640 1 -- start start 2024-03-04T07:27:15.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f3231815640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f322c15dd50 0x7f322c0ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f3231815640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f322c072680 0x7f322c0d2760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f3231815640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f322c072cf0 con 0x7f322c0cef20 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f3231815640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f322c072a90 con 0x7f322c072680 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f3231815640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f322c072fd0 con 0x7f322c15dd50 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f322c072680 0x7f322c0d2760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f322c072680 0x7f322c0d2760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39878/0 (socket says 172.21.15.64:39878) 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322b7fe640 1 -- 172.21.15.64:0/1377648514 learned_addr learned my addr 172.21.15.64:0/1377648514 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322affd640 1 --2- 172.21.15.64:0/1377648514 >> v2:172.21.15.66:3300/0 conn(0x7f322c15dd50 0x7f322c0ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322b7fe640 1 -- 172.21.15.64:0/1377648514 >> v2:172.21.15.66:3300/0 conn(0x7f322c15dd50 msgr2=0x7f322c0ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322b7fe640 1 --2- 172.21.15.64:0/1377648514 >> v2:172.21.15.66:3300/0 conn(0x7f322c15dd50 0x7f322c0ce9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322b7fe640 1 -- 172.21.15.64:0/1377648514 >> v1:172.21.15.66:6789/0 conn(0x7f322c0cef20 legacy=0x7f322c071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.621+0000 7f322b7fe640 1 -- 172.21.15.64:0/1377648514 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f322c0d1f60 con 0x7f322c072680 2024-03-04T07:27:15.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.622+0000 7f322b7fe640 1 --2- 172.21.15.64:0/1377648514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c072680 0x7f322c0d2760 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f322c08c610 tx=0x7f322402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4e16bc65b4fbbc89 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.622+0000 7f3229ffb640 1 -- 172.21.15.64:0/1377648514 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3224035020 con 0x7f322c072680 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.622+0000 7f3229ffb640 1 -- 172.21.15.64:0/1377648514 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3224031870 con 0x7f322c072680 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.622+0000 7f3229ffb640 1 -- 172.21.15.64:0/1377648514 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3224031c40 con 0x7f322c072680 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 -- 172.21.15.64:0/1377648514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c072680 msgr2=0x7f322c0d2760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 --2- 172.21.15.64:0/1377648514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c072680 0x7f322c0d2760 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f322c08c610 tx=0x7f322402b910 comp rx=0 tx=0).stop 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 -- 172.21.15.64:0/1377648514 shutdown_connections 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 --2- 172.21.15.64:0/1377648514 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c072680 0x7f322c0d2760 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 --2- 172.21.15.64:0/1377648514 >> v2:172.21.15.66:3300/0 conn(0x7f322c15dd50 0x7f322c0ce9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 -- 172.21.15.64:0/1377648514 >> 172.21.15.64:0/1377648514 conn(0x7f322c0d3ef0 msgr2=0x7f322c0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 -- 172.21.15.64:0/1377648514 shutdown_connections 2024-03-04T07:27:15.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.623+0000 7f3231815640 1 -- 172.21.15.64:0/1377648514 wait complete. 2024-03-04T07:27:15.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.624+0000 7f3231815640 1 Processor -- start 2024-03-04T07:27:15.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.624+0000 7f3231815640 1 -- start start 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f3231815640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f322c072680 0x7f322c150890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f3231815640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c0cef20 0x7f322c150dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f3231815640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f322c15dd50 0x7f322c0ccc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f3231815640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f322c15e2b0 con 0x7f322c0cef20 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f3231815640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f322c15e130 con 0x7f322c15dd50 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f3231815640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f322c0d2ca0 con 0x7f322c072680 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c0cef20 0x7f322c150dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c0cef20 0x7f322c150dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39892/0 (socket says 172.21.15.64:39892) 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 -- 172.21.15.64:0/2202594045 learned_addr learned my addr 172.21.15.64:0/2202594045 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322b7fe640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f322c15dd50 0x7f322c0ccc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322affd640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f322c072680 0x7f322c150890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 -- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f322c072680 msgr2=0x7f322c150890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f322c072680 0x7f322c150890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 -- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f322c15dd50 msgr2=0x7f322c0ccc90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f322c15dd50 0x7f322c0ccc90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.625+0000 7f322a7fc640 1 -- 172.21.15.64:0/2202594045 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f322c0cdb10 con 0x7f322c0cef20 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.626+0000 7f322a7fc640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c0cef20 0x7f322c150dd0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f321400c400 tx=0x7f321400c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.626+0000 7f3223fff640 1 -- 172.21.15.64:0/2202594045 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32140052d0 con 0x7f322c0cef20 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.626+0000 7f3223fff640 1 -- 172.21.15.64:0/2202594045 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3214005490 con 0x7f322c0cef20 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.626+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f322c160830 con 0x7f322c0cef20 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.626+0000 7f3223fff640 1 -- 172.21.15.64:0/2202594045 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3214007260 con 0x7f322c0cef20 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.626+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f322c160df0 con 0x7f322c0cef20 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.627+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f322c06cdd0 con 0x7f322c0cef20 2024-03-04T07:27:15.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.627+0000 7f3223fff640 1 -- 172.21.15.64:0/2202594045 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3214007400 con 0x7f322c0cef20 2024-03-04T07:27:15.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.628+0000 7f3223fff640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f320c06b4d0 0x7f320c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.628+0000 7f322affd640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f320c06b4d0 0x7f320c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.628+0000 7f3223fff640 1 -- 172.21.15.64:0/2202594045 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f32140895f0 con 0x7f322c0cef20 2024-03-04T07:27:15.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.628+0000 7f322affd640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f320c06b4d0 0x7f320c06d990 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f321c011340 tx=0x7f321c00b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.632+0000 7f3223fff640 1 -- 172.21.15.64:0/2202594045 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3214057eb0 con 0x7f322c0cef20 2024-03-04T07:27:15.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.814+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f322c160fc0 con 0x7f322c0cef20 2024-03-04T07:27:15.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.815+0000 7f3223fff640 1 -- 172.21.15.64:0/2202594045 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f321405bf50 con 0x7f322c0cef20 2024-03-04T07:27:15.815 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:15.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f320c06b4d0 msgr2=0x7f320c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f320c06b4d0 0x7f320c06d990 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f321c011340 tx=0x7f321c00b040 comp rx=0 tx=0).stop 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c0cef20 msgr2=0x7f322c150dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c0cef20 0x7f322c150dd0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f321400c400 tx=0x7f321400c8c0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 shutdown_connections 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f320c06b4d0 0x7f320c06d990 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f322c15dd50 0x7f322c0ccc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f322c0cef20 0x7f322c150dd0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 --2- 172.21.15.64:0/2202594045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f322c072680 0x7f322c150890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 >> 172.21.15.64:0/2202594045 conn(0x7f322c0d3ef0 msgr2=0x7f322c0d3000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 shutdown_connections 2024-03-04T07:27:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.817+0000 7f3231815640 1 -- 172.21.15.64:0/2202594045 wait complete. 2024-03-04T07:27:15.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:15.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.929+0000 7f15a7d0f640 1 Processor -- start 2024-03-04T07:27:15.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.929+0000 7f15a7d0f640 1 -- start start 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a7d0f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f15a015df70 0x7f15a00d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a7d0f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a00d18c0 0x7f15a0071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a7d0f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f15a0071a80 0x7f15a00b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a7d0f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f15a00b6f20 con 0x7f15a015df70 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a7d0f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f15a00d3030 con 0x7f15a0071a80 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a7d0f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f15a00d3310 con 0x7f15a00d18c0 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f15a015df70 0x7f15a00d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f15a015df70 0x7f15a00d27b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39900/0 (socket says 172.21.15.64:39900) 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 -- 172.21.15.64:0/3392149057 learned_addr learned my addr 172.21.15.64:0/3392149057 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a62ac640 1 --2- 172.21.15.64:0/3392149057 >> v2:172.21.15.66:3300/0 conn(0x7f15a0071a80 0x7f15a00b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a52aa640 1 --2- 172.21.15.64:0/3392149057 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a00d18c0 0x7f15a0071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 -- 172.21.15.64:0/3392149057 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a00d18c0 msgr2=0x7f15a0071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:15.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 --2- 172.21.15.64:0/3392149057 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a00d18c0 0x7f15a0071540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 -- 172.21.15.64:0/3392149057 >> v2:172.21.15.66:3300/0 conn(0x7f15a0071a80 msgr2=0x7f15a00b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:15.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 --2- 172.21.15.64:0/3392149057 >> v2:172.21.15.66:3300/0 conn(0x7f15a0071a80 0x7f15a00b2700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.930+0000 7f15a5aab640 1 -- 172.21.15.64:0/3392149057 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15a00d35f0 con 0x7f15a015df70 2024-03-04T07:27:15.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.931+0000 7f15a5aab640 1 --2- 172.21.15.64:0/3392149057 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a00d27b0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f15a00b68c0 tx=0x7f159c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=742557a62ea7075d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.931+0000 7f15a4aa9640 1 -- 172.21.15.64:0/3392149057 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f159c034070 con 0x7f15a015df70 2024-03-04T07:27:15.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.931+0000 7f15a4aa9640 1 -- 172.21.15.64:0/3392149057 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f159c030860 con 0x7f15a015df70 2024-03-04T07:27:15.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.931+0000 7f15a4aa9640 1 -- 172.21.15.64:0/3392149057 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f159c030c30 con 0x7f15a015df70 2024-03-04T07:27:15.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 -- 172.21.15.64:0/3392149057 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 msgr2=0x7f15a00d27b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/3392149057 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a00d27b0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f15a00b68c0 tx=0x7f159c02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 -- 172.21.15.64:0/3392149057 shutdown_connections 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/3392149057 >> v2:172.21.15.66:3300/0 conn(0x7f15a0071a80 0x7f15a00b2700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/3392149057 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a00d18c0 0x7f15a0071540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/3392149057 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a00d27b0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 -- 172.21.15.64:0/3392149057 >> 172.21.15.64:0/3392149057 conn(0x7f15a00d3cf0 msgr2=0x7f15a00d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 -- 172.21.15.64:0/3392149057 shutdown_connections 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.932+0000 7f15a7d0f640 1 -- 172.21.15.64:0/3392149057 wait complete. 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.933+0000 7f15a7d0f640 1 Processor -- start 2024-03-04T07:27:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.933+0000 7f15a7d0f640 1 -- start start 2024-03-04T07:27:15.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a0071a80 0x7f15a014b100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15a00d18c0 0x7f15a015a350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a015a890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15a015e350 con 0x7f15a015df70 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15a00d2cf0 con 0x7f15a00d18c0 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f15a015e4d0 con 0x7f15a0071a80 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a015a890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a015a890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39912/0 (socket says 172.21.15.64:39912) 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 -- 172.21.15.64:0/173515115 learned_addr learned my addr 172.21.15.64:0/173515115 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a5aab640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a0071a80 0x7f15a014b100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a52aa640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15a00d18c0 0x7f15a015a350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 -- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a0071a80 msgr2=0x7f15a014b100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a0071a80 0x7f15a014b100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 -- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15a00d18c0 msgr2=0x7f15a015a350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15a00d18c0 0x7f15a015a350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 -- 172.21.15.64:0/173515115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15a014b640 con 0x7f15a015df70 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a62ac640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a015a890 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f159800eed0 tx=0x7f159800b480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15967fc640 1 -- 172.21.15.64:0/173515115 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1598018020 con 0x7f15a015df70 2024-03-04T07:27:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15967fc640 1 -- 172.21.15.64:0/173515115 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f159800bd20 con 0x7f15a015df70 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15967fc640 1 -- 172.21.15.64:0/173515115 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1598014640 con 0x7f15a015df70 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f15a0157b90 con 0x7f15a015df70 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.934+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15a00c8970 con 0x7f15a015df70 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.935+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f15a0067ff0 con 0x7f15a015df70 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.936+0000 7f15967fc640 1 -- 172.21.15.64:0/173515115 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1598005d10 con 0x7f15a015df70 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.936+0000 7f15967fc640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f158406b500 0x7f158406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.936+0000 7f15967fc640 1 -- 172.21.15.64:0/173515115 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f159808a640 con 0x7f15a015df70 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.936+0000 7f15a5aab640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f158406b500 0x7f158406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.937+0000 7f15a5aab640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f158406b500 0x7f158406d9c0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f159c036000 tx=0x7f159c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:15.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:15.940+0000 7f15967fc640 1 -- 172.21.15.64:0/173515115 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1598022070 con 0x7f15a015df70 2024-03-04T07:27:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.122+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f15a00b3eb0 con 0x7f15a015df70 2024-03-04T07:27:16.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.123+0000 7f15967fc640 1 -- 172.21.15.64:0/173515115 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f15a014b640 con 0x7f15a015df70 2024-03-04T07:27:16.123 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:16.123 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f158406b500 msgr2=0x7f158406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f158406b500 0x7f158406d9c0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f159c036000 tx=0x7f159c008030 comp rx=0 tx=0).stop 2024-03-04T07:27:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 msgr2=0x7f15a015a890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a015a890 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f159800eed0 tx=0x7f159800b480 comp rx=0 tx=0).stop 2024-03-04T07:27:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 shutdown_connections 2024-03-04T07:27:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f158406b500 0x7f158406d9c0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f15a015df70 0x7f15a015a890 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15a00d18c0 0x7f15a015a350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 --2- 172.21.15.64:0/173515115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15a0071a80 0x7f15a014b100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 >> 172.21.15.64:0/173515115 conn(0x7f15a00d3cf0 msgr2=0x7f15a00b5b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:16.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 shutdown_connections 2024-03-04T07:27:16.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.125+0000 7f15a7d0f640 1 -- 172.21.15.64:0/173515115 wait complete. 2024-03-04T07:27:16.136 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:16.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:16.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.242+0000 7f55565ac640 1 Processor -- start 2024-03-04T07:27:16.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.242+0000 7f55565ac640 1 -- start start 2024-03-04T07:27:16.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.242+0000 7f55565ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555015d2c0 0x7f555015d9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.242+0000 7f55565ac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f555015fb60 0x7f555015ffd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.242+0000 7f55565ac640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55500d0a30 con 0x7f55500b4340 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.242+0000 7f55565ac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5550160510 con 0x7f555015fb60 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.242+0000 7f55565ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55500d01d0 con 0x7f555015d2c0 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f5554b49640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f555015fb60 0x7f555015ffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f554ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555015d2c0 0x7f555015d9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f554f7fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f55500b4340 0x7f55500b2ac0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:41332/0 (socket says 172.21.15.64:41332) 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f5554b49640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f555015fb60 0x7f555015ffd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37630/0 (socket says 172.21.15.64:37630) 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f554ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555015d2c0 0x7f555015d9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:43006/0 (socket says 172.21.15.64:43006) 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f554f7fe640 1 -- 172.21.15.64:0/1096519594 learned_addr learned my addr 172.21.15.64:0/1096519594 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f5554b49640 1 -- 172.21.15.64:0/1096519594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555015d2c0 msgr2=0x7f555015d9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f5554b49640 1 --2- 172.21.15.64:0/1096519594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555015d2c0 0x7f555015d9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f5554b49640 1 -- 172.21.15.64:0/1096519594 >> v1:172.21.15.66:6789/0 conn(0x7f55500b4340 legacy=0x7f55500b2ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.243+0000 7f5554b49640 1 -- 172.21.15.64:0/1096519594 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55500cc5c0 con 0x7f555015fb60 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f5554b49640 1 --2- 172.21.15.64:0/1096519594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015fb60 0x7f555015ffd0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f5548009d90 tx=0x7f554802bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7700406d870c6a76 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f554effd640 1 -- 172.21.15.64:0/1096519594 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5548039070 con 0x7f555015fb60 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f554effd640 1 -- 172.21.15.64:0/1096519594 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f554802cc50 con 0x7f555015fb60 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f554effd640 1 -- 172.21.15.64:0/1096519594 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5548034440 con 0x7f555015fb60 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 -- 172.21.15.64:0/1096519594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015fb60 msgr2=0x7f555015ffd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 --2- 172.21.15.64:0/1096519594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015fb60 0x7f555015ffd0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f5548009d90 tx=0x7f554802bea0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 -- 172.21.15.64:0/1096519594 shutdown_connections 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 --2- 172.21.15.64:0/1096519594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015fb60 0x7f555015ffd0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 --2- 172.21.15.64:0/1096519594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555015d2c0 0x7f555015d9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 -- 172.21.15.64:0/1096519594 >> 172.21.15.64:0/1096519594 conn(0x7f55500b3660 msgr2=0x7f55500b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 -- 172.21.15.64:0/1096519594 shutdown_connections 2024-03-04T07:27:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.244+0000 7f55565ac640 1 -- 172.21.15.64:0/1096519594 wait complete. 2024-03-04T07:27:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 Processor -- start 2024-03-04T07:27:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 -- start start 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55500b4340 0x7f55500b51e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015d2c0 0x7f555015e960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555015fb60 0x7f55500caae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55500cc860 con 0x7f555015fb60 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55500d0750 con 0x7f555015d2c0 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.246+0000 7f55565ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55500b31d0 con 0x7f55500b4340 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015d2c0 0x7f555015e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f5554b49640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555015fb60 0x7f55500caae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55500b4340 0x7f55500b51e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f5554b49640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555015fb60 0x7f55500caae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39920/0 (socket says 172.21.15.64:39920) 2024-03-04T07:27:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f5554b49640 1 -- 172.21.15.64:0/3353260935 learned_addr learned my addr 172.21.15.64:0/3353260935 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015d2c0 0x7f555015e960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37640/0 (socket says 172.21.15.64:37640) 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 -- 172.21.15.64:0/3353260935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55500b4340 msgr2=0x7f55500b51e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55500b4340 0x7f55500b51e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 -- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555015fb60 msgr2=0x7f55500caae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555015fb60 0x7f55500caae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 -- 172.21.15.64:0/3353260935 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55500b4740 con 0x7f555015d2c0 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f5554b49640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555015fb60 0x7f55500caae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554f7fe640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015d2c0 0x7f555015e960 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f5540000f60 tx=0x7f554000c800 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554cff9640 1 -- 172.21.15.64:0/3353260935 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5540017070 con 0x7f555015d2c0 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554cff9640 1 -- 172.21.15.64:0/3353260935 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5540005ce0 con 0x7f555015d2c0 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f55500cb650 con 0x7f555015d2c0 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f554cff9640 1 -- 172.21.15.64:0/3353260935 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5540007310 con 0x7f555015d2c0 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.247+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55500b49e0 con 0x7f555015d2c0 2024-03-04T07:27:16.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.248+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f555009fc50 con 0x7f555015d2c0 2024-03-04T07:27:16.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.249+0000 7f554cff9640 1 -- 172.21.15.64:0/3353260935 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f554001a070 con 0x7f555015d2c0 2024-03-04T07:27:16.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.249+0000 7f554cff9640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553006b500 0x7f553006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.249+0000 7f554cff9640 1 -- 172.21.15.64:0/3353260935 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5540089430 con 0x7f555015d2c0 2024-03-04T07:27:16.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.249+0000 7f554ffff640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553006b500 0x7f553006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.250+0000 7f554ffff640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553006b500 0x7f553006d9c0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f555015f2e0 tx=0x7f5538009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.253+0000 7f554cff9640 1 -- 172.21.15.64:0/3353260935 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5540057cf0 con 0x7f555015d2c0 2024-03-04T07:27:16.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.435+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f55500c4fd0 con 0x7f555015d2c0 2024-03-04T07:27:16.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.436+0000 7f554cff9640 1 -- 172.21.15.64:0/3353260935 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f554005bd90 con 0x7f555015d2c0 2024-03-04T07:27:16.436 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553006b500 msgr2=0x7f553006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553006b500 0x7f553006d9c0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f555015f2e0 tx=0x7f5538009190 comp rx=0 tx=0).stop 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015d2c0 msgr2=0x7f555015e960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015d2c0 0x7f555015e960 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f5540000f60 tx=0x7f554000c800 comp rx=0 tx=0).stop 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 shutdown_connections 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553006b500 0x7f553006d9c0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555015fb60 0x7f55500caae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555015d2c0 0x7f555015e960 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 --2- 172.21.15.64:0/3353260935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55500b4340 0x7f55500b51e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 >> 172.21.15.64:0/3353260935 conn(0x7f55500b3660 msgr2=0x7f55500cd720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 shutdown_connections 2024-03-04T07:27:16.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.438+0000 7f55565ac640 1 -- 172.21.15.64:0/3353260935 wait complete. 2024-03-04T07:27:16.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:16.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.552+0000 7f6b7b833640 1 Processor -- start 2024-03-04T07:27:16.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.552+0000 7f6b7b833640 1 -- start start 2024-03-04T07:27:16.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b7b833640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b740b79f0 0x7f6b740b7ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b7b833640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b740d4960 con 0x7f6b74071540 2024-03-04T07:27:16.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b7b833640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6b740d36c0 con 0x7f6b740b2d30 2024-03-04T07:27:16.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b7b833640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b740d3230 con 0x7f6b740b79f0 2024-03-04T07:27:16.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b79dd0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b740b79f0 0x7f6b740b7ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b795cf640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6b74071540 0x7f6b74071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48382/0 (socket says 172.21.15.64:48382) 2024-03-04T07:27:16.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b795cf640 1 -- 172.21.15.64:0/2674693818 learned_addr learned my addr 172.21.15.64:0/2674693818 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:16.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b79dd0640 1 -- 172.21.15.64:0/2674693818 >> v1:172.21.15.66:6789/0 conn(0x7f6b740b2d30 legacy=0x7f6b740b72e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b79dd0640 1 -- 172.21.15.64:0/2674693818 >> v1:172.21.15.64:6789/0 conn(0x7f6b74071540 legacy=0x7f6b74071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.553+0000 7f6b79dd0640 1 -- 172.21.15.64:0/2674693818 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b740d2620 con 0x7f6b740b79f0 2024-03-04T07:27:16.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.554+0000 7f6b79dd0640 1 --2- 172.21.15.64:0/2674693818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b79f0 0x7f6b740b7ea0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f6b640097a0 tx=0x7f6b6402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=50233c46b208d8ad server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.554+0000 7f6b73fff640 1 -- 172.21.15.64:0/2674693818 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b64034070 con 0x7f6b740b79f0 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.554+0000 7f6b73fff640 1 -- 172.21.15.64:0/2674693818 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b64030610 con 0x7f6b740b79f0 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.554+0000 7f6b73fff640 1 -- 172.21.15.64:0/2674693818 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b640309e0 con 0x7f6b740b79f0 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.555+0000 7f6b7b833640 1 -- 172.21.15.64:0/2674693818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b79f0 msgr2=0x7f6b740b7ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.555+0000 7f6b7b833640 1 --2- 172.21.15.64:0/2674693818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b79f0 0x7f6b740b7ea0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f6b640097a0 tx=0x7f6b6402b960 comp rx=0 tx=0).stop 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.555+0000 7f6b7b833640 1 -- 172.21.15.64:0/2674693818 shutdown_connections 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.555+0000 7f6b7b833640 1 --2- 172.21.15.64:0/2674693818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b79f0 0x7f6b740b7ea0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.555+0000 7f6b7b833640 1 -- 172.21.15.64:0/2674693818 >> 172.21.15.64:0/2674693818 conn(0x7f6b740d3d60 msgr2=0x7f6b740d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.555+0000 7f6b7b833640 1 -- 172.21.15.64:0/2674693818 shutdown_connections 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.555+0000 7f6b7b833640 1 -- 172.21.15.64:0/2674693818 wait complete. 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.556+0000 7f6b7b833640 1 Processor -- start 2024-03-04T07:27:16.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.556+0000 7f6b7b833640 1 -- start start 2024-03-04T07:27:16.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b7b833640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b74071540 0x7f6b74159470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b7b833640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b2d30 0x7f6b740cc320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b7b833640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b740b79f0 0x7f6b740cc860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b7b833640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b740d2be0 con 0x7f6b740b2d30 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b7b833640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b740b83e0 con 0x7f6b74071540 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b7b833640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b740d2ec0 con 0x7f6b740b79f0 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b2d30 0x7f6b740cc320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b2d30 0x7f6b740cc320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39946/0 (socket says 172.21.15.64:39946) 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 -- 172.21.15.64:0/1217409215 learned_addr learned my addr 172.21.15.64:0/1217409215 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b795cf640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b74071540 0x7f6b74159470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b79dd0640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b740b79f0 0x7f6b740cc860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 -- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b740b79f0 msgr2=0x7f6b740cc860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b740b79f0 0x7f6b740cc860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 -- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b74071540 msgr2=0x7f6b74159470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b74071540 0x7f6b74159470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 -- 172.21.15.64:0/1217409215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b740c9d80 con 0x7f6b740b2d30 2024-03-04T07:27:16.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.557+0000 7f6b78dce640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b2d30 0x7f6b740cc320 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f6b6800c440 tx=0x7f6b68001e70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.558+0000 7f6b71ffb640 1 -- 172.21.15.64:0/1217409215 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b680052d0 con 0x7f6b740b2d30 2024-03-04T07:27:16.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.558+0000 7f6b71ffb640 1 -- 172.21.15.64:0/1217409215 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b68005470 con 0x7f6b740b2d30 2024-03-04T07:27:16.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.558+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b740ca060 con 0x7f6b740b2d30 2024-03-04T07:27:16.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.558+0000 7f6b71ffb640 1 -- 172.21.15.64:0/1217409215 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b68007260 con 0x7f6b740b2d30 2024-03-04T07:27:16.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.558+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b740ca620 con 0x7f6b740b2d30 2024-03-04T07:27:16.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.559+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b740b7f00 con 0x7f6b740b2d30 2024-03-04T07:27:16.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.559+0000 7f6b71ffb640 1 -- 172.21.15.64:0/1217409215 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b68007400 con 0x7f6b740b2d30 2024-03-04T07:27:16.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.559+0000 7f6b71ffb640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b6006b4d0 0x7f6b6006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.560+0000 7f6b71ffb640 1 -- 172.21.15.64:0/1217409215 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6b68089690 con 0x7f6b740b2d30 2024-03-04T07:27:16.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.560+0000 7f6b795cf640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b6006b4d0 0x7f6b6006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.560+0000 7f6b795cf640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b6006b4d0 0x7f6b6006d990 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f6b6c005c00 tx=0x7f6b6c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.564+0000 7f6b71ffb640 1 -- 172.21.15.64:0/1217409215 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6b68057ec0 con 0x7f6b740b2d30 2024-03-04T07:27:16.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.744+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f6b74072070 con 0x7f6b740b2d30 2024-03-04T07:27:16.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.744+0000 7f6b71ffb640 1 -- 172.21.15.64:0/1217409215 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f6b6805bf60 con 0x7f6b740b2d30 2024-03-04T07:27:16.744 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:16.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b6006b4d0 msgr2=0x7f6b6006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b6006b4d0 0x7f6b6006d990 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f6b6c005c00 tx=0x7f6b6c009040 comp rx=0 tx=0).stop 2024-03-04T07:27:16.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b2d30 msgr2=0x7f6b740cc320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b2d30 0x7f6b740cc320 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f6b6800c440 tx=0x7f6b68001e70 comp rx=0 tx=0).stop 2024-03-04T07:27:16.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 shutdown_connections 2024-03-04T07:27:16.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b6006b4d0 0x7f6b6006d990 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b740b79f0 0x7f6b740cc860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b740b2d30 0x7f6b740cc320 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 --2- 172.21.15.64:0/1217409215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b74071540 0x7f6b74159470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 >> 172.21.15.64:0/1217409215 conn(0x7f6b740d3d60 msgr2=0x7f6b7415b1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:16.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 shutdown_connections 2024-03-04T07:27:16.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.746+0000 7f6b7b833640 1 -- 172.21.15.64:0/1217409215 wait complete. 2024-03-04T07:27:16.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:16.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.863+0000 7fa58a393640 1 Processor -- start 2024-03-04T07:27:16.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.863+0000 7fa58a393640 1 -- start start 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa58a393640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa5840ceea0 0x7fa5840d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa58a393640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5840b6d10 0x7fa5840b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa58a393640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa5840d3740 con 0x7fa58415dd90 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa58a393640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa5840d3400 con 0x7fa5840ceea0 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa58a393640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa5840d3120 con 0x7fa5840b6d10 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa588930640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5840b6d10 0x7fa5840b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa583fff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fa58415dd90 0x7fa5840d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:41354/0 (socket says 172.21.15.64:41354) 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa583fff640 1 -- 172.21.15.64:0/765358674 learned_addr learned my addr 172.21.15.64:0/765358674 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa5837fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa5840ceea0 0x7fa5840d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa582ffd640 1 -- 172.21.15.64:0/765358674 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 994510006 0 0) 0x7fa5840d3740 con 0x7fa58415dd90 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa588930640 1 -- 172.21.15.64:0/765358674 >> v2:172.21.15.66:3300/0 conn(0x7fa5840ceea0 msgr2=0x7fa5840d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:16.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa588930640 1 --2- 172.21.15.64:0/765358674 >> v2:172.21.15.66:3300/0 conn(0x7fa5840ceea0 0x7fa5840d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa588930640 1 -- 172.21.15.64:0/765358674 >> v1:172.21.15.66:6789/0 conn(0x7fa58415dd90 legacy=0x7fa5840d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.864+0000 7fa588930640 1 -- 172.21.15.64:0/765358674 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa5840d39e0 con 0x7fa5840b6d10 2024-03-04T07:27:16.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.865+0000 7fa588930640 1 --2- 172.21.15.64:0/765358674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5840b6d10 0x7fa5840b71a0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fa57c009d90 tx=0x7fa57c02bf30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=db10a145d558e16e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.865+0000 7fa582ffd640 1 -- 172.21.15.64:0/765358674 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa57c039070 con 0x7fa5840b6d10 2024-03-04T07:27:16.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.865+0000 7fa582ffd640 1 -- 172.21.15.64:0/765358674 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa57c02cce0 con 0x7fa5840b6d10 2024-03-04T07:27:16.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.865+0000 7fa582ffd640 1 -- 172.21.15.64:0/765358674 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa57c034660 con 0x7fa5840b6d10 2024-03-04T07:27:16.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.865+0000 7fa58a393640 1 -- 172.21.15.64:0/765358674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5840b6d10 msgr2=0x7fa5840b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.865+0000 7fa58a393640 1 --2- 172.21.15.64:0/765358674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5840b6d10 0x7fa5840b71a0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fa57c009d90 tx=0x7fa57c02bf30 comp rx=0 tx=0).stop 2024-03-04T07:27:16.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.866+0000 7fa58a393640 1 -- 172.21.15.64:0/765358674 shutdown_connections 2024-03-04T07:27:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.866+0000 7fa58a393640 1 --2- 172.21.15.64:0/765358674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5840b6d10 0x7fa5840b71a0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.866+0000 7fa58a393640 1 --2- 172.21.15.64:0/765358674 >> v2:172.21.15.66:3300/0 conn(0x7fa5840ceea0 0x7fa5840d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.866+0000 7fa58a393640 1 -- 172.21.15.64:0/765358674 >> 172.21.15.64:0/765358674 conn(0x7fa584071960 msgr2=0x7fa584071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.866+0000 7fa58a393640 1 -- 172.21.15.64:0/765358674 shutdown_connections 2024-03-04T07:27:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.866+0000 7fa58a393640 1 -- 172.21.15.64:0/765358674 wait complete. 2024-03-04T07:27:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.867+0000 7fa58a393640 1 Processor -- start 2024-03-04T07:27:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.867+0000 7fa58a393640 1 -- start start 2024-03-04T07:27:16.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa58a393640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5840b6d10 0x7fa5841f3300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa58a393640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5840ceea0 0x7fa5840b4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa58a393640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa58415dd90 0x7fa5840b5010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa58a393640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa5840d48f0 con 0x7fa5840ceea0 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa58a393640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa58415e170 con 0x7fa5840b6d10 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa58a393640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa5840d4a70 con 0x7fa58415dd90 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5840ceea0 0x7fa5840b4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5840ceea0 0x7fa5840b4ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39950/0 (socket says 172.21.15.64:39950) 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 -- 172.21.15.64:0/275851937 learned_addr learned my addr 172.21.15.64:0/275851937 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa588930640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa58415dd90 0x7fa5840b5010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa583fff640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5840b6d10 0x7fa5841f3300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 -- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa58415dd90 msgr2=0x7fa5840b5010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa58415dd90 0x7fa5840b5010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 -- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5840b6d10 msgr2=0x7fa5841f3300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5840b6d10 0x7fa5841f3300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 -- 172.21.15.64:0/275851937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa58415f8d0 con 0x7fa5840ceea0 2024-03-04T07:27:16.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa583fff640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5840b6d10 0x7fa5841f3300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:16.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa5837fe640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5840ceea0 0x7fa5840b4ad0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fa57800e9b0 tx=0x7fa57800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.868+0000 7fa580ff9640 1 -- 172.21.15.64:0/275851937 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa578017070 con 0x7fa5840ceea0 2024-03-04T07:27:16.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.869+0000 7fa580ff9640 1 -- 172.21.15.64:0/275851937 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa57800c9a0 con 0x7fa5840ceea0 2024-03-04T07:27:16.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.869+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa5840c8360 con 0x7fa5840ceea0 2024-03-04T07:27:16.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.869+0000 7fa580ff9640 1 -- 172.21.15.64:0/275851937 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa57800cd90 con 0x7fa5840ceea0 2024-03-04T07:27:16.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.869+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa5840c8920 con 0x7fa5840ceea0 2024-03-04T07:27:16.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.870+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa584072a30 con 0x7fa5840ceea0 2024-03-04T07:27:16.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.870+0000 7fa580ff9640 1 -- 172.21.15.64:0/275851937 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa57801a070 con 0x7fa5840ceea0 2024-03-04T07:27:16.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.870+0000 7fa580ff9640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa57006b4d0 0x7fa57006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:16.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.871+0000 7fa580ff9640 1 -- 172.21.15.64:0/275851937 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa57808a500 con 0x7fa5840ceea0 2024-03-04T07:27:16.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.871+0000 7fa583fff640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa57006b4d0 0x7fa57006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:16.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.871+0000 7fa583fff640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa57006b4d0 0x7fa57006d990 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fa574000f60 tx=0x7fa574009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:16.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:16.875+0000 7fa580ff9640 1 -- 172.21.15.64:0/275851937 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa58415f8d0 con 0x7fa5840ceea0 2024-03-04T07:27:17.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.055+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fa58404f410 con 0x7fa5840ceea0 2024-03-04T07:27:17.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.056+0000 7fa580ff9640 1 -- 172.21.15.64:0/275851937 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7fa578058d30 con 0x7fa5840ceea0 2024-03-04T07:27:17.056 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa57006b4d0 msgr2=0x7fa57006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa57006b4d0 0x7fa57006d990 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fa574000f60 tx=0x7fa574009040 comp rx=0 tx=0).stop 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5840ceea0 msgr2=0x7fa5840b4ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5840ceea0 0x7fa5840b4ad0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fa57800e9b0 tx=0x7fa57800ee70 comp rx=0 tx=0).stop 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 shutdown_connections 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa57006b4d0 0x7fa57006d990 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa58415dd90 0x7fa5840b5010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5840ceea0 0x7fa5840b4ad0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 --2- 172.21.15.64:0/275851937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5840b6d10 0x7fa5841f3300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 >> 172.21.15.64:0/275851937 conn(0x7fa584071960 msgr2=0x7fa5840b3d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:17.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 shutdown_connections 2024-03-04T07:27:17.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.058+0000 7fa58a393640 1 -- 172.21.15.64:0/275851937 wait complete. 2024-03-04T07:27:17.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:17.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.171+0000 7f997f0fb640 1 Processor -- start 2024-03-04T07:27:17.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.171+0000 7f997f0fb640 1 -- start start 2024-03-04T07:27:17.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.171+0000 7f997f0fb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99780aee10 0x7f9978165a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.171+0000 7f997f0fb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9978071240 con 0x7f997806df30 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.171+0000 7f997f0fb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9978071520 con 0x7f9978158520 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.171+0000 7f997f0fb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9978071b70 con 0x7f99780aee10 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.172+0000 7f997d698640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99780aee10 0x7f9978165a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.172+0000 7f9977fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f997806df30 0x7f99780ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48396/0 (socket says 172.21.15.64:48396) 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.172+0000 7f9977fff640 1 -- 172.21.15.64:0/394890786 learned_addr learned my addr 172.21.15.64:0/394890786 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.172+0000 7f997d698640 1 -- 172.21.15.64:0/394890786 >> v1:172.21.15.66:6789/0 conn(0x7f9978158520 legacy=0x7f997806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.172+0000 7f997d698640 1 -- 172.21.15.64:0/394890786 >> v1:172.21.15.64:6789/0 conn(0x7f997806df30 legacy=0x7f99780ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.172+0000 7f99777fe640 1 -- 172.21.15.64:0/394890786 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1564645254 0 0) 0x7f9978071520 con 0x7f9978158520 2024-03-04T07:27:17.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.172+0000 7f997d698640 1 -- 172.21.15.64:0/394890786 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9978071800 con 0x7f99780aee10 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997d698640 1 --2- 172.21.15.64:0/394890786 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99780aee10 0x7f9978165a00 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f9970009d90 tx=0x7f997002e0c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e087872ec15d9763 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f99777fe640 1 -- 172.21.15.64:0/394890786 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f997003b070 con 0x7f99780aee10 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f99777fe640 1 -- 172.21.15.64:0/394890786 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9970036440 con 0x7f99780aee10 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f99777fe640 1 -- 172.21.15.64:0/394890786 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9970036810 con 0x7f99780aee10 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997f0fb640 1 -- 172.21.15.64:0/394890786 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99780aee10 msgr2=0x7f9978165a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997f0fb640 1 --2- 172.21.15.64:0/394890786 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99780aee10 0x7f9978165a00 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f9970009d90 tx=0x7f997002e0c0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997f0fb640 1 -- 172.21.15.64:0/394890786 shutdown_connections 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997f0fb640 1 --2- 172.21.15.64:0/394890786 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99780aee10 0x7f9978165a00 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997f0fb640 1 -- 172.21.15.64:0/394890786 >> 172.21.15.64:0/394890786 conn(0x7f9978066d00 msgr2=0x7f99780670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997f0fb640 1 -- 172.21.15.64:0/394890786 shutdown_connections 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.173+0000 7f997f0fb640 1 -- 172.21.15.64:0/394890786 wait complete. 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.174+0000 7f997f0fb640 1 Processor -- start 2024-03-04T07:27:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.175+0000 7f997f0fb640 1 -- start start 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.175+0000 7f997f0fb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f997806df30 0x7f99781571f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.175+0000 7f997f0fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99780aee10 0x7f9978154540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.175+0000 7f997f0fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9978158520 0x7f99781492e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.175+0000 7f997f0fb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9978158900 con 0x7f9978158520 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.175+0000 7f997f0fb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9978072000 con 0x7f997806df30 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.175+0000 7f997f0fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9978158a80 con 0x7f99780aee10 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9978158520 0x7f99781492e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f9977fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99780aee10 0x7f9978154540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997ce97640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f997806df30 0x7f99781571f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997ce97640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f997806df30 0x7f99781571f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37670/0 (socket says 172.21.15.64:37670) 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9978158520 0x7f99781492e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39964/0 (socket says 172.21.15.64:39964) 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f9977fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99780aee10 0x7f9978154540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:43060/0 (socket says 172.21.15.64:43060) 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997ce97640 1 -- 172.21.15.64:0/2459366024 learned_addr learned my addr 172.21.15.64:0/2459366024 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 -- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99780aee10 msgr2=0x7f9978154540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99780aee10 0x7f9978154540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 -- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f997806df30 msgr2=0x7f99781571f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f997806df30 0x7f99781571f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 -- 172.21.15.64:0/2459366024 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9978154a80 con 0x7f9978158520 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f9977fff640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99780aee10 0x7f9978154540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997ce97640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f997806df30 0x7f99781571f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997d698640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9978158520 0x7f99781492e0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f9970035000 tx=0x7f9970004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f99757fa640 1 -- 172.21.15.64:0/2459366024 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f997003b040 con 0x7f9978158520 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f99757fa640 1 -- 172.21.15.64:0/2459366024 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9970044070 con 0x7f9978158520 2024-03-04T07:27:17.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99781578d0 con 0x7f9978158520 2024-03-04T07:27:17.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f99757fa640 1 -- 172.21.15.64:0/2459366024 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f997003f820 con 0x7f9978158520 2024-03-04T07:27:17.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.176+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9978155480 con 0x7f9978158520 2024-03-04T07:27:17.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.177+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9978067ff0 con 0x7f9978158520 2024-03-04T07:27:17.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.178+0000 7f99757fa640 1 -- 172.21.15.64:0/2459366024 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f997003f9c0 con 0x7f9978158520 2024-03-04T07:27:17.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.178+0000 7f99757fa640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f995c06b4d0 0x7f995c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.178+0000 7f997ce97640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f995c06b4d0 0x7f995c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.178+0000 7f99757fa640 1 -- 172.21.15.64:0/2459366024 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f99700ac530 con 0x7f9978158520 2024-03-04T07:27:17.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.179+0000 7f997ce97640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f995c06b4d0 0x7f995c06d990 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f996c000f60 tx=0x7f996c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.183+0000 7f99757fa640 1 -- 172.21.15.64:0/2459366024 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9978154a80 con 0x7f9978158520 2024-03-04T07:27:17.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:27:17.275 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:27:17.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.367+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f99780b1330 con 0x7f9978158520 2024-03-04T07:27:17.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.372+0000 7f99757fa640 1 -- 172.21.15.64:0/2459366024 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f997007ad60 con 0x7f9978158520 2024-03-04T07:27:17.372 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:17.372 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f995c06b4d0 msgr2=0x7f995c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f995c06b4d0 0x7f995c06d990 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f996c000f60 tx=0x7f996c009040 comp rx=0 tx=0).stop 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9978158520 msgr2=0x7f99781492e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9978158520 0x7f99781492e0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f9970035000 tx=0x7f9970004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 shutdown_connections 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f995c06b4d0 0x7f995c06d990 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9978158520 0x7f99781492e0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99780aee10 0x7f9978154540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 --2- 172.21.15.64:0/2459366024 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f997806df30 0x7f99781571f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 >> 172.21.15.64:0/2459366024 conn(0x7f9978066d00 msgr2=0x7f9978158fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:17.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 shutdown_connections 2024-03-04T07:27:17.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.374+0000 7f997f0fb640 1 -- 172.21.15.64:0/2459366024 wait complete. 2024-03-04T07:27:17.386 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:17.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:17.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.530+0000 7f038c9dd640 1 Processor -- start 2024-03-04T07:27:17.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.530+0000 7f038c9dd640 1 -- start start 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f038c9dd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f038815dd90 0x7f03880ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f038c9dd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f03880d29d0 0x7f03880d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f038c9dd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f03880d4720 0x7f03880b6d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f038c9dd640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f03880d3400 con 0x7f03880d4720 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f038c9dd640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f03880d3740 con 0x7f03880d29d0 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f038c9dd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03880d3120 con 0x7f038815dd90 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f03880d4720 0x7f03880b6d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f03880d4720 0x7f03880b6d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39966/0 (socket says 172.21.15.64:39966) 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 -- 172.21.15.64:0/2209188815 learned_addr learned my addr 172.21.15.64:0/2209188815 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0385d9b640 1 --2- 172.21.15.64:0/2209188815 >> v2:172.21.15.66:3300/0 conn(0x7f03880d29d0 0x7f03880d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f038659c640 1 --2- 172.21.15.64:0/2209188815 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f038815dd90 0x7f03880ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 -- 172.21.15.64:0/2209188815 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f038815dd90 msgr2=0x7f03880ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 --2- 172.21.15.64:0/2209188815 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f038815dd90 0x7f03880ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 -- 172.21.15.64:0/2209188815 >> v2:172.21.15.66:3300/0 conn(0x7f03880d29d0 msgr2=0x7f03880d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 --2- 172.21.15.64:0/2209188815 >> v2:172.21.15.66:3300/0 conn(0x7f03880d29d0 0x7f03880d41e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.531+0000 7f0386d9d640 1 -- 172.21.15.64:0/2209188815 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03880d39e0 con 0x7f03880d4720 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.532+0000 7f0386d9d640 1 --2- 172.21.15.64:0/2209188815 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03880d4720 0x7f03880b6d10 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f03880b6890 tx=0x7f037802bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec76eb4d34cc2ff2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.532+0000 7f038559a640 1 -- 172.21.15.64:0/2209188815 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f037802cb00 con 0x7f03880d4720 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.532+0000 7f038559a640 1 -- 172.21.15.64:0/2209188815 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f037802cca0 con 0x7f03880d4720 2024-03-04T07:27:17.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.532+0000 7f038559a640 1 -- 172.21.15.64:0/2209188815 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0378034610 con 0x7f03880d4720 2024-03-04T07:27:17.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 -- 172.21.15.64:0/2209188815 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03880d4720 msgr2=0x7f03880b6d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2209188815 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03880d4720 0x7f03880b6d10 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f03880b6890 tx=0x7f037802bf60 comp rx=0 tx=0).stop 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 -- 172.21.15.64:0/2209188815 shutdown_connections 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2209188815 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03880d4720 0x7f03880b6d10 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2209188815 >> v2:172.21.15.66:3300/0 conn(0x7f03880d29d0 0x7f03880d41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2209188815 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f038815dd90 0x7f03880ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 -- 172.21.15.64:0/2209188815 >> 172.21.15.64:0/2209188815 conn(0x7f0388071960 msgr2=0x7f0388071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 -- 172.21.15.64:0/2209188815 shutdown_connections 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.533+0000 7f038c9dd640 1 -- 172.21.15.64:0/2209188815 wait complete. 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.534+0000 7f038c9dd640 1 Processor -- start 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.534+0000 7f038c9dd640 1 -- start start 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f038c9dd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03880d29d0 0x7f038815e770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f038c9dd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03880d4720 0x7f038814c2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f038c9dd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f038815dd90 0x7f03881493e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f038c9dd640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03880d2df0 con 0x7f038815dd90 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f038c9dd640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f038815e170 con 0x7f03880d29d0 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f038c9dd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03880b7250 con 0x7f03880d4720 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f038815dd90 0x7f03881493e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f038659c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03880d29d0 0x7f038815e770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f038815dd90 0x7f03881493e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39982/0 (socket says 172.21.15.64:39982) 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 -- 172.21.15.64:0/2224965104 learned_addr learned my addr 172.21.15.64:0/2224965104 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0385d9b640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03880d4720 0x7f038814c2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 -- 172.21.15.64:0/2224965104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03880d4720 msgr2=0x7f038814c2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03880d4720 0x7f038814c2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 -- 172.21.15.64:0/2224965104 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03880d29d0 msgr2=0x7f038815e770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03880d29d0 0x7f038815e770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.535+0000 7f0386d9d640 1 -- 172.21.15.64:0/2224965104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f038815ee30 con 0x7f038815dd90 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.536+0000 7f0386d9d640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f038815dd90 0x7f03881493e0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f0378036040 tx=0x7f0378004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.536+0000 7f037effd640 1 -- 172.21.15.64:0/2224965104 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0378043020 con 0x7f038815dd90 2024-03-04T07:27:17.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.536+0000 7f037effd640 1 -- 172.21.15.64:0/2224965104 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f037802ce50 con 0x7f038815dd90 2024-03-04T07:27:17.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.536+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f038815f110 con 0x7f038815dd90 2024-03-04T07:27:17.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.536+0000 7f037effd640 1 -- 172.21.15.64:0/2224965104 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0378009510 con 0x7f038815dd90 2024-03-04T07:27:17.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.536+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f038815f6d0 con 0x7f038815dd90 2024-03-04T07:27:17.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.537+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0388072a30 con 0x7f038815dd90 2024-03-04T07:27:17.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.537+0000 7f037effd640 1 -- 172.21.15.64:0/2224965104 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0378046020 con 0x7f038815dd90 2024-03-04T07:27:17.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.538+0000 7f037effd640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f036806b4d0 0x7f036806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.538+0000 7f037effd640 1 -- 172.21.15.64:0/2224965104 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f03780ac620 con 0x7f038815dd90 2024-03-04T07:27:17.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.538+0000 7f038659c640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f036806b4d0 0x7f036806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.538+0000 7f038659c640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f036806b4d0 0x7f036806d990 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f0380009aa0 tx=0x7f0380009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.542+0000 7f037effd640 1 -- 172.21.15.64:0/2224965104 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f037804d030 con 0x7f038815dd90 2024-03-04T07:27:17.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.723+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f038804f410 con 0x7f038815dd90 2024-03-04T07:27:17.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.723+0000 7f037effd640 1 -- 172.21.15.64:0/2224965104 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f037807add0 con 0x7f038815dd90 2024-03-04T07:27:17.723 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:17.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f036806b4d0 msgr2=0x7f036806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f036806b4d0 0x7f036806d990 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f0380009aa0 tx=0x7f0380009190 comp rx=0 tx=0).stop 2024-03-04T07:27:17.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f038815dd90 msgr2=0x7f03881493e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f038815dd90 0x7f03881493e0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f0378036040 tx=0x7f0378004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:17.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 shutdown_connections 2024-03-04T07:27:17.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f036806b4d0 0x7f036806d990 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f038815dd90 0x7f03881493e0 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03880d4720 0x7f038814c2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 --2- 172.21.15.64:0/2224965104 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03880d29d0 0x7f038815e770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.725+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 >> 172.21.15.64:0/2224965104 conn(0x7f0388071960 msgr2=0x7f03880b3700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:17.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.726+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 shutdown_connections 2024-03-04T07:27:17.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.726+0000 7f038c9dd640 1 -- 172.21.15.64:0/2224965104 wait complete. 2024-03-04T07:27:17.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.841+0000 7f32f3f93640 1 Processor -- start 2024-03-04T07:27:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.841+0000 7f32f3f93640 1 -- start start 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f3f93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f32ec15dd90 0x7f32ec0ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f3f93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec0d29d0 0x7f32ec0d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f3f93640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f32ec0d4720 0x7f32ec0b6d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f3f93640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f32ec0d3400 con 0x7f32ec15dd90 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f3f93640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f32ec0d3740 con 0x7f32ec0d4720 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f3f93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32ec0d3120 con 0x7f32ec0d29d0 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f32ec15dd90 0x7f32ec0ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f32ec15dd90 0x7f32ec0ceea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39990/0 (socket says 172.21.15.64:39990) 2024-03-04T07:27:17.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 -- 172.21.15.64:0/1866419931 learned_addr learned my addr 172.21.15.64:0/1866419931 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f2530640 1 --2- 172.21.15.64:0/1866419931 >> v2:172.21.15.66:3300/0 conn(0x7f32ec0d4720 0x7f32ec0b6d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f152e640 1 --2- 172.21.15.64:0/1866419931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec0d29d0 0x7f32ec0d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 -- 172.21.15.64:0/1866419931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec0d29d0 msgr2=0x7f32ec0d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 --2- 172.21.15.64:0/1866419931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec0d29d0 0x7f32ec0d41e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 -- 172.21.15.64:0/1866419931 >> v2:172.21.15.66:3300/0 conn(0x7f32ec0d4720 msgr2=0x7f32ec0b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 --2- 172.21.15.64:0/1866419931 >> v2:172.21.15.66:3300/0 conn(0x7f32ec0d4720 0x7f32ec0b6d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f1d2f640 1 -- 172.21.15.64:0/1866419931 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32ec0d39e0 con 0x7f32ec15dd90 2024-03-04T07:27:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.842+0000 7f32f2530640 1 --2- 172.21.15.64:0/1866419931 >> v2:172.21.15.66:3300/0 conn(0x7f32ec0d4720 0x7f32ec0b6d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:17.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.843+0000 7f32f1d2f640 1 --2- 172.21.15.64:0/1866419931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec15dd90 0x7f32ec0ceea0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f32ec0b6c60 tx=0x7f32e802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=13a812deb03da5a1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.843+0000 7f32f0d2d640 1 -- 172.21.15.64:0/1866419931 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32e8034070 con 0x7f32ec15dd90 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.843+0000 7f32f0d2d640 1 -- 172.21.15.64:0/1866419931 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f32e8030860 con 0x7f32ec15dd90 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.843+0000 7f32f0d2d640 1 -- 172.21.15.64:0/1866419931 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32e8030c30 con 0x7f32ec15dd90 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 -- 172.21.15.64:0/1866419931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec15dd90 msgr2=0x7f32ec0ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 --2- 172.21.15.64:0/1866419931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec15dd90 0x7f32ec0ceea0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f32ec0b6c60 tx=0x7f32e802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 -- 172.21.15.64:0/1866419931 shutdown_connections 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 --2- 172.21.15.64:0/1866419931 >> v2:172.21.15.66:3300/0 conn(0x7f32ec0d4720 0x7f32ec0b6d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 --2- 172.21.15.64:0/1866419931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec0d29d0 0x7f32ec0d41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 --2- 172.21.15.64:0/1866419931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec15dd90 0x7f32ec0ceea0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 -- 172.21.15.64:0/1866419931 >> 172.21.15.64:0/1866419931 conn(0x7f32ec071960 msgr2=0x7f32ec071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 -- 172.21.15.64:0/1866419931 shutdown_connections 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.844+0000 7f32f3f93640 1 -- 172.21.15.64:0/1866419931 wait complete. 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.845+0000 7f32f3f93640 1 Processor -- start 2024-03-04T07:27:17.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.845+0000 7f32f3f93640 1 -- start start 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32ec0d29d0 0x7f32ec1f4a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec0d4720 0x7f32ec1f66a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec15dd90 0x7f32ec1551e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32ec0d2df0 con 0x7f32ec0d4720 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32ec15e170 con 0x7f32ec0d29d0 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32ec0b7250 con 0x7f32ec15dd90 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec0d4720 0x7f32ec1f66a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec0d4720 0x7f32ec1f66a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39998/0 (socket says 172.21.15.64:39998) 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 -- 172.21.15.64:0/3728057765 learned_addr learned my addr 172.21.15.64:0/3728057765 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f1d2f640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32ec0d29d0 0x7f32ec1f4a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f2530640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec15dd90 0x7f32ec1551e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 -- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec15dd90 msgr2=0x7f32ec1551e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec15dd90 0x7f32ec1551e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 -- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32ec0d29d0 msgr2=0x7f32ec1f4a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32ec0d29d0 0x7f32ec1f4a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 -- 172.21.15.64:0/3728057765 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32ec1f4810 con 0x7f32ec0d4720 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f1d2f640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32ec0d29d0 0x7f32ec1f4a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f152e640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec0d4720 0x7f32ec1f66a0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f32dc00e8c0 tx=0x7f32dc00ecc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32e27fc640 1 -- 172.21.15.64:0/3728057765 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32dc017070 con 0x7f32ec0d4720 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32e27fc640 1 -- 172.21.15.64:0/3728057765 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f32dc00c940 con 0x7f32ec0d4720 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32ec152fe0 con 0x7f32ec0d4720 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32e27fc640 1 -- 172.21.15.64:0/3728057765 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32dc00cd10 con 0x7f32ec0d4720 2024-03-04T07:27:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.846+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f32ec1535a0 con 0x7f32ec0d4720 2024-03-04T07:27:17.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.847+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f32ec072a30 con 0x7f32ec0d4720 2024-03-04T07:27:17.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.848+0000 7f32e27fc640 1 -- 172.21.15.64:0/3728057765 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f32dc01a070 con 0x7f32ec0d4720 2024-03-04T07:27:17.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.848+0000 7f32e27fc640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32d006b4d0 0x7f32d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:17.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.848+0000 7f32e27fc640 1 -- 172.21.15.64:0/3728057765 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f32dc08a400 con 0x7f32ec0d4720 2024-03-04T07:27:17.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.848+0000 7f32f1d2f640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32d006b4d0 0x7f32d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:17.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.849+0000 7f32f1d2f640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32d006b4d0 0x7f32d006d990 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f32e8036000 tx=0x7f32e803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:17.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:17.852+0000 7f32e27fc640 1 -- 172.21.15.64:0/3728057765 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f32dc058c30 con 0x7f32ec0d4720 2024-03-04T07:27:18.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.030+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f32ec1f4fb0 con 0x7f32ec0d4720 2024-03-04T07:27:18.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.031+0000 7f32e27fc640 1 -- 172.21.15.64:0/3728057765 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f32dc05ccd0 con 0x7f32ec0d4720 2024-03-04T07:27:18.030 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:18.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32d006b4d0 msgr2=0x7f32d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32d006b4d0 0x7f32d006d990 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f32e8036000 tx=0x7f32e803b040 comp rx=0 tx=0).stop 2024-03-04T07:27:18.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec0d4720 msgr2=0x7f32ec1f66a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec0d4720 0x7f32ec1f66a0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f32dc00e8c0 tx=0x7f32dc00ecc0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 shutdown_connections 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32d006b4d0 0x7f32d006d990 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32ec15dd90 0x7f32ec1551e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32ec0d4720 0x7f32ec1f66a0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 --2- 172.21.15.64:0/3728057765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32ec0d29d0 0x7f32ec1f4a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.032+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 >> 172.21.15.64:0/3728057765 conn(0x7f32ec071960 msgr2=0x7f32ec1616d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.033+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 shutdown_connections 2024-03-04T07:27:18.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.033+0000 7f32f3f93640 1 -- 172.21.15.64:0/3728057765 wait complete. 2024-03-04T07:27:18.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.148+0000 7fa93a1d7640 1 Processor -- start 2024-03-04T07:27:18.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.148+0000 7fa93a1d7640 1 -- start start 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa93a1d7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa93415ae50 0x7fa9340708e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa93a1d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa934071870 0x7fa934150a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa93a1d7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa934070e20 con 0x7fa93415b230 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa93a1d7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa9340b2e30 con 0x7fa93415ae50 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa93a1d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa93406dfe0 con 0x7fa934071870 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa932ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa93415b230 0x7fa934071160 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48402/0 (socket says 172.21.15.64:48402) 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa932ffd640 1 -- 172.21.15.64:0/4195173441 learned_addr learned my addr 172.21.15.64:0/4195173441 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa9337fe640 1 --2- 172.21.15.64:0/4195173441 >> v2:172.21.15.66:3300/0 conn(0x7fa93415ae50 0x7fa9340708e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa933fff640 1 --2- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa934071870 0x7fa934150a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa9327fc640 1 -- 172.21.15.64:0/4195173441 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2373273146 0 0) 0x7fa934070e20 con 0x7fa93415b230 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa9327fc640 1 -- 172.21.15.64:0/4195173441 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa91c003520 con 0x7fa93415b230 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa9337fe640 1 -- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa934071870 msgr2=0x7fa934150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa9337fe640 1 --2- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa934071870 0x7fa934150a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa9337fe640 1 -- 172.21.15.64:0/4195173441 >> v1:172.21.15.64:6789/0 conn(0x7fa93415b230 legacy=0x7fa934071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa9337fe640 1 -- 172.21.15.64:0/4195173441 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9340b30b0 con 0x7fa93415ae50 2024-03-04T07:27:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.149+0000 7fa933fff640 1 --2- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa934071870 0x7fa934150a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.150+0000 7fa9337fe640 1 --2- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415ae50 0x7fa9340708e0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fa9240097a0 tx=0x7fa92402b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c9da033768adb415 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.150+0000 7fa9327fc640 1 -- 172.21.15.64:0/4195173441 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9240315a0 con 0x7fa93415ae50 2024-03-04T07:27:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.150+0000 7fa9327fc640 1 -- 172.21.15.64:0/4195173441 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa924037070 con 0x7fa93415ae50 2024-03-04T07:27:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.150+0000 7fa9327fc640 1 -- 172.21.15.64:0/4195173441 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa924031960 con 0x7fa93415ae50 2024-03-04T07:27:18.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 -- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415ae50 msgr2=0x7fa9340708e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415ae50 0x7fa9340708e0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fa9240097a0 tx=0x7fa92402b840 comp rx=0 tx=0).stop 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 -- 172.21.15.64:0/4195173441 shutdown_connections 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa934071870 0x7fa934150a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/4195173441 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415ae50 0x7fa9340708e0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 -- 172.21.15.64:0/4195173441 >> 172.21.15.64:0/4195173441 conn(0x7fa934066d00 msgr2=0x7fa934067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 -- 172.21.15.64:0/4195173441 shutdown_connections 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.151+0000 7fa93a1d7640 1 -- 172.21.15.64:0/4195173441 wait complete. 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.152+0000 7fa93a1d7640 1 Processor -- start 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.152+0000 7fa93a1d7640 1 -- start start 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa93a1d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa934071870 0x7fa93413f260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa93a1d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa93415ae50 0x7fa93414b000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa93a1d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415b230 0x7fa9341501a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa93a1d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9340b3950 con 0x7fa934071870 2024-03-04T07:27:18.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa93a1d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9340b3670 con 0x7fa93415b230 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa93a1d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9340b4150 con 0x7fa93415ae50 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa934071870 0x7fa93413f260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa934071870 0x7fa93413f260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40002/0 (socket says 172.21.15.64:40002) 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 -- 172.21.15.64:0/60499284 learned_addr learned my addr 172.21.15.64:0/60499284 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa933fff640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415b230 0x7fa9341501a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa932ffd640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa93415ae50 0x7fa93414b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 -- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa93415ae50 msgr2=0x7fa93414b000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa93415ae50 0x7fa93414b000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 -- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415b230 msgr2=0x7fa9341501a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415b230 0x7fa9341501a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 -- 172.21.15.64:0/60499284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa93414b540 con 0x7fa934071870 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.153+0000 7fa9337fe640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa934071870 0x7fa93413f260 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fa9240098c0 tx=0x7fa9240046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.154+0000 7fa91bfff640 1 -- 172.21.15.64:0/60499284 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa924032430 con 0x7fa934071870 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.154+0000 7fa91bfff640 1 -- 172.21.15.64:0/60499284 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa9240325d0 con 0x7fa934071870 2024-03-04T07:27:18.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.154+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa93414f100 con 0x7fa934071870 2024-03-04T07:27:18.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.154+0000 7fa91bfff640 1 -- 172.21.15.64:0/60499284 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9240329c0 con 0x7fa934071870 2024-03-04T07:27:18.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.154+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa93414f6c0 con 0x7fa934071870 2024-03-04T07:27:18.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.155+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa9340733c0 con 0x7fa934071870 2024-03-04T07:27:18.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.155+0000 7fa91bfff640 1 -- 172.21.15.64:0/60499284 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa924032b60 con 0x7fa934071870 2024-03-04T07:27:18.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.155+0000 7fa91bfff640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa92006b4d0 0x7fa92006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.156+0000 7fa91bfff640 1 -- 172.21.15.64:0/60499284 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa92402f030 con 0x7fa934071870 2024-03-04T07:27:18.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.156+0000 7fa932ffd640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa92006b4d0 0x7fa92006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.156+0000 7fa932ffd640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa92006b4d0 0x7fa92006d990 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fa9340b0dd0 tx=0x7fa928009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.160+0000 7fa91bfff640 1 -- 172.21.15.64:0/60499284 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa924040070 con 0x7fa934071870 2024-03-04T07:27:18.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.340+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fa9340ccf80 con 0x7fa934071870 2024-03-04T07:27:18.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.340+0000 7fa91bfff640 1 -- 172.21.15.64:0/60499284 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7fa92407ad00 con 0x7fa934071870 2024-03-04T07:27:18.340 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:18.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa92006b4d0 msgr2=0x7fa92006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa92006b4d0 0x7fa92006d990 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fa9340b0dd0 tx=0x7fa928009040 comp rx=0 tx=0).stop 2024-03-04T07:27:18.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa934071870 msgr2=0x7fa93413f260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa934071870 0x7fa93413f260 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fa9240098c0 tx=0x7fa9240046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 shutdown_connections 2024-03-04T07:27:18.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa92006b4d0 0x7fa92006d990 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa93415b230 0x7fa9341501a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa93415ae50 0x7fa93414b000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 --2- 172.21.15.64:0/60499284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa934071870 0x7fa93413f260 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 >> 172.21.15.64:0/60499284 conn(0x7fa934066d00 msgr2=0x7fa9340af840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:18.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 shutdown_connections 2024-03-04T07:27:18.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.342+0000 7fa93a1d7640 1 -- 172.21.15.64:0/60499284 wait complete. 2024-03-04T07:27:18.362 INFO:tasks.cephfs_test_runner:Starting test: test_dir_deletion (tasks.cephfs.test_strays.TestStrays) That when deleting a bunch of dentries and the containing 2024-03-04T07:27:18.362 INFO:tasks.cephfs_test_runner:test_dir_deletion (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:27:18.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:18.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.464+0000 7f8e357ae640 1 Processor -- start 2024-03-04T07:27:18.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.464+0000 7f8e357ae640 1 -- start start 2024-03-04T07:27:18.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e357ae640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e300d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e357ae640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e300cea30 con 0x7f8e300b95a0 2024-03-04T07:27:18.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e357ae640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e300d2af0 con 0x7f8e300b9a30 2024-03-04T07:27:18.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e357ae640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e300cf3c0 con 0x7f8e3015df00 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2e7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8e300b95a0 0x7f8e300b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48406/0 (socket says 172.21.15.64:48406) 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2e7fc640 1 -- 172.21.15.64:0/801281190 learned_addr learned my addr 172.21.15.64:0/801281190 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2effd640 1 --2- 172.21.15.64:0/801281190 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e300d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2dffb640 1 -- 172.21.15.64:0/801281190 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3426909505 0 0) 0x7f8e300cea30 con 0x7f8e300b95a0 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2dffb640 1 -- 172.21.15.64:0/801281190 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8e18003520 con 0x7f8e300b95a0 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2dffb640 1 -- 172.21.15.64:0/801281190 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2860095546 0 0) 0x7f8e300d2af0 con 0x7f8e300b9a30 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2dffb640 1 -- 172.21.15.64:0/801281190 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8e300cea30 con 0x7f8e300b9a30 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2effd640 1 -- 172.21.15.64:0/801281190 >> v1:172.21.15.66:6789/0 conn(0x7f8e300b9a30 legacy=0x7f8e300d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2effd640 1 -- 172.21.15.64:0/801281190 >> v1:172.21.15.64:6789/0 conn(0x7f8e300b95a0 legacy=0x7f8e300b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.465+0000 7f8e2effd640 1 -- 172.21.15.64:0/801281190 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e300cf0e0 con 0x7f8e3015df00 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.466+0000 7f8e2effd640 1 --2- 172.21.15.64:0/801281190 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e300d25b0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f8e28009dc0 tx=0x7f8e2802bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f294ed3c547653f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.466+0000 7f8e2dffb640 1 -- 172.21.15.64:0/801281190 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e28039070 con 0x7f8e3015df00 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.466+0000 7f8e2dffb640 1 -- 172.21.15.64:0/801281190 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e2802ccd0 con 0x7f8e3015df00 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.466+0000 7f8e2dffb640 1 -- 172.21.15.64:0/801281190 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e28034640 con 0x7f8e3015df00 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.467+0000 7f8e357ae640 1 -- 172.21.15.64:0/801281190 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 msgr2=0x7f8e300d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.467+0000 7f8e357ae640 1 --2- 172.21.15.64:0/801281190 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e300d25b0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f8e28009dc0 tx=0x7f8e2802bf20 comp rx=0 tx=0).stop 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.467+0000 7f8e357ae640 1 -- 172.21.15.64:0/801281190 shutdown_connections 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.467+0000 7f8e357ae640 1 --2- 172.21.15.64:0/801281190 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e300d25b0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.467+0000 7f8e357ae640 1 -- 172.21.15.64:0/801281190 >> 172.21.15.64:0/801281190 conn(0x7f8e30071960 msgr2=0x7f8e30071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.467+0000 7f8e357ae640 1 -- 172.21.15.64:0/801281190 shutdown_connections 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.467+0000 7f8e357ae640 1 -- 172.21.15.64:0/801281190 wait complete. 2024-03-04T07:27:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.468+0000 7f8e357ae640 1 Processor -- start 2024-03-04T07:27:18.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.468+0000 7f8e357ae640 1 -- start start 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e357ae640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b95a0 0x7f8e30160cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e357ae640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e30150f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e357ae640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e300cc1e0 0x7f8e300cc690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e357ae640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e300cec70 con 0x7f8e300cc1e0 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e357ae640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e300ce750 con 0x7f8e300b95a0 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e357ae640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e300cedf0 con 0x7f8e3015df00 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e300cc1e0 0x7f8e300cc690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e300cc1e0 0x7f8e300cc690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40012/0 (socket says 172.21.15.64:40012) 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 -- 172.21.15.64:0/344231171 learned_addr learned my addr 172.21.15.64:0/344231171 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2e7fc640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e30150f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2effd640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b95a0 0x7f8e30160cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 -- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 msgr2=0x7f8e30150f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e30150f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 -- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b95a0 msgr2=0x7f8e30160cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b95a0 0x7f8e30160cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.469+0000 7f8e2f7fe640 1 -- 172.21.15.64:0/344231171 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e30161230 con 0x7f8e300cc1e0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.470+0000 7f8e2f7fe640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e300cc1e0 0x7f8e300cc690 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f8e2000c430 tx=0x7f8e2000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.470+0000 7f8e177fe640 1 -- 172.21.15.64:0/344231171 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e20017070 con 0x7f8e300cc1e0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.470+0000 7f8e177fe640 1 -- 172.21.15.64:0/344231171 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e200052d0 con 0x7f8e300cc1e0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.470+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e3015ff70 con 0x7f8e300cc1e0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.470+0000 7f8e177fe640 1 -- 172.21.15.64:0/344231171 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e20003830 con 0x7f8e300cc1e0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.470+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e301604f0 con 0x7f8e300cc1e0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.471+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e30067ff0 con 0x7f8e300cc1e0 2024-03-04T07:27:18.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.471+0000 7f8e177fe640 1 -- 172.21.15.64:0/344231171 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8e2001a070 con 0x7f8e300cc1e0 2024-03-04T07:27:18.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.472+0000 7f8e177fe640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b4d0 0x7f8e1c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.472+0000 7f8e177fe640 1 -- 172.21.15.64:0/344231171 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8e20089540 con 0x7f8e300cc1e0 2024-03-04T07:27:18.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.472+0000 7f8e2effd640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b4d0 0x7f8e1c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.472+0000 7f8e2effd640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b4d0 0x7f8e1c06d990 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f8e28036000 tx=0x7f8e28030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.476+0000 7f8e177fe640 1 -- 172.21.15.64:0/344231171 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8e30161230 con 0x7f8e300cc1e0 2024-03-04T07:27:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.659+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f8e300ca950 con 0x7f8e300cc1e0 2024-03-04T07:27:18.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.660+0000 7f8e177fe640 1 -- 172.21.15.64:0/344231171 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f8e20057d70 con 0x7f8e300cc1e0 2024-03-04T07:27:18.660 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:18.660 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b4d0 msgr2=0x7f8e1c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b4d0 0x7f8e1c06d990 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f8e28036000 tx=0x7f8e28030000 comp rx=0 tx=0).stop 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e300cc1e0 msgr2=0x7f8e300cc690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e300cc1e0 0x7f8e300cc690 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f8e2000c430 tx=0x7f8e2000c8f0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 shutdown_connections 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b4d0 0x7f8e1c06d990 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e300cc1e0 0x7f8e300cc690 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e3015df00 0x7f8e30150f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 --2- 172.21.15.64:0/344231171 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b95a0 0x7f8e30160cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 >> 172.21.15.64:0/344231171 conn(0x7f8e30071960 msgr2=0x7f8e3015d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:18.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 shutdown_connections 2024-03-04T07:27:18.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.662+0000 7f8e357ae640 1 -- 172.21.15.64:0/344231171 wait complete. 2024-03-04T07:27:18.672 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:18.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:18.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.778+0000 7f045dedf640 1 Processor -- start 2024-03-04T07:27:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.778+0000 7f045dedf640 1 -- start start 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f045dedf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f04580ce960 0x7f04580d0860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f045dedf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f045815f730 con 0x7f04580ce4f0 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f045dedf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f045815ebb0 con 0x7f04581618e0 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f045dedf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04580cfce0 con 0x7f04580ce960 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f0456ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f04580ce4f0 0x7f04580d0150 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48414/0 (socket says 172.21.15.64:48414) 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f0456ffd640 1 -- 172.21.15.64:0/3678885700 learned_addr learned my addr 172.21.15.64:0/3678885700 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f0457fff640 1 --2- 172.21.15.64:0/3678885700 >> v2:172.21.15.66:3300/0 conn(0x7f04580ce960 0x7f04580d0860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f04567fc640 1 -- 172.21.15.64:0/3678885700 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3474868242 0 0) 0x7f045815f730 con 0x7f04580ce4f0 2024-03-04T07:27:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f04567fc640 1 -- 172.21.15.64:0/3678885700 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0440003520 con 0x7f04580ce4f0 2024-03-04T07:27:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f04567fc640 1 -- 172.21.15.64:0/3678885700 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3938392991 0 0) 0x7f045815ebb0 con 0x7f04581618e0 2024-03-04T07:27:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f04567fc640 1 -- 172.21.15.64:0/3678885700 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f045815f730 con 0x7f04581618e0 2024-03-04T07:27:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f0457fff640 1 -- 172.21.15.64:0/3678885700 >> v1:172.21.15.66:6789/0 conn(0x7f04581618e0 legacy=0x7f045806b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f0457fff640 1 -- 172.21.15.64:0/3678885700 >> v1:172.21.15.64:6789/0 conn(0x7f04580ce4f0 legacy=0x7f04580d0150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.779+0000 7f0457fff640 1 -- 172.21.15.64:0/3678885700 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f045815ee90 con 0x7f04580ce960 2024-03-04T07:27:18.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.780+0000 7f0457fff640 1 --2- 172.21.15.64:0/3678885700 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce960 0x7f04580d0860 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f0450009d90 tx=0x7f045002bfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6ec51b1be1dae3f6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.780+0000 7f04567fc640 1 -- 172.21.15.64:0/3678885700 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450039070 con 0x7f04580ce960 2024-03-04T07:27:18.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.780+0000 7f04567fc640 1 -- 172.21.15.64:0/3678885700 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f045002cd80 con 0x7f04580ce960 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.780+0000 7f04567fc640 1 -- 172.21.15.64:0/3678885700 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450034640 con 0x7f04580ce960 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.781+0000 7f045dedf640 1 -- 172.21.15.64:0/3678885700 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce960 msgr2=0x7f04580d0860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.781+0000 7f045dedf640 1 --2- 172.21.15.64:0/3678885700 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce960 0x7f04580d0860 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f0450009d90 tx=0x7f045002bfd0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.781+0000 7f045dedf640 1 -- 172.21.15.64:0/3678885700 shutdown_connections 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.781+0000 7f045dedf640 1 --2- 172.21.15.64:0/3678885700 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce960 0x7f04580d0860 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.781+0000 7f045dedf640 1 -- 172.21.15.64:0/3678885700 >> 172.21.15.64:0/3678885700 conn(0x7f04580b3660 msgr2=0x7f04580b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.781+0000 7f045dedf640 1 -- 172.21.15.64:0/3678885700 shutdown_connections 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.781+0000 7f045dedf640 1 -- 172.21.15.64:0/3678885700 wait complete. 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.782+0000 7f045dedf640 1 Processor -- start 2024-03-04T07:27:18.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.782+0000 7f045dedf640 1 -- start start 2024-03-04T07:27:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f045dedf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce4f0 0x7f0458151ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f045dedf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04580ce960 0x7f04580b5020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f045dedf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0458152630 0x7f04580c5660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f045dedf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04580cedb0 con 0x7f04580ce960 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f045dedf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f045815f450 con 0x7f04580ce4f0 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f045dedf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f045815e7d0 con 0x7f0458152630 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04580ce960 0x7f04580b5020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04580ce960 0x7f04580b5020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40020/0 (socket says 172.21.15.64:40020) 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 -- 172.21.15.64:0/286835993 learned_addr learned my addr 172.21.15.64:0/286835993 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0457fff640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0458152630 0x7f04580c5660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f04577fe640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce4f0 0x7f0458151ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 -- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0458152630 msgr2=0x7f04580c5660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0458152630 0x7f04580c5660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 -- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce4f0 msgr2=0x7f0458151ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce4f0 0x7f0458151ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.783+0000 7f0456ffd640 1 -- 172.21.15.64:0/286835993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04580ca960 con 0x7f04580ce960 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.784+0000 7f0456ffd640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04580ce960 0x7f04580b5020 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f044c00c500 tx=0x7f044c00c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.784+0000 7f043ffff640 1 -- 172.21.15.64:0/286835993 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f044c017070 con 0x7f04580ce960 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.784+0000 7f043ffff640 1 -- 172.21.15.64:0/286835993 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f044c005ce0 con 0x7f04580ce960 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.784+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04580cac40 con 0x7f04580ce960 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.784+0000 7f043ffff640 1 -- 172.21.15.64:0/286835993 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f044c0072f0 con 0x7f04580ce960 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.784+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04580cb200 con 0x7f04580ce960 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.785+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f045809fc50 con 0x7f04580ce960 2024-03-04T07:27:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.785+0000 7f043ffff640 1 -- 172.21.15.64:0/286835993 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f044c01a070 con 0x7f04580ce960 2024-03-04T07:27:18.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.785+0000 7f043ffff640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044406b4d0 0x7f044406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:18.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.786+0000 7f043ffff640 1 -- 172.21.15.64:0/286835993 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f044c0894e0 con 0x7f04580ce960 2024-03-04T07:27:18.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.786+0000 7f04577fe640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044406b4d0 0x7f044406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:18.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.786+0000 7f04577fe640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044406b4d0 0x7f044406d990 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f0448007fd0 tx=0x7f0448009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:18.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.790+0000 7f043ffff640 1 -- 172.21.15.64:0/286835993 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f044c057da0 con 0x7f04580ce960 2024-03-04T07:27:18.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.970+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f04580cb3d0 con 0x7f04580ce960 2024-03-04T07:27:18.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.970+0000 7f043ffff640 1 -- 172.21.15.64:0/286835993 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f044c05be40 con 0x7f04580ce960 2024-03-04T07:27:18.970 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044406b4d0 msgr2=0x7f044406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044406b4d0 0x7f044406d990 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f0448007fd0 tx=0x7f0448009040 comp rx=0 tx=0).stop 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04580ce960 msgr2=0x7f04580b5020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04580ce960 0x7f04580b5020 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f044c00c500 tx=0x7f044c00c9c0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 shutdown_connections 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044406b4d0 0x7f044406d990 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0458152630 0x7f04580c5660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04580ce960 0x7f04580b5020 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 --2- 172.21.15.64:0/286835993 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04580ce4f0 0x7f0458151ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 >> 172.21.15.64:0/286835993 conn(0x7f04580b3660 msgr2=0x7f04580cc840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 shutdown_connections 2024-03-04T07:27:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:18.972+0000 7f045dedf640 1 -- 172.21.15.64:0/286835993 wait complete. 2024-03-04T07:27:18.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:19.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.088+0000 7fca6029d640 1 Processor -- start 2024-03-04T07:27:19.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.088+0000 7fca6029d640 1 -- start start 2024-03-04T07:27:19.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca6029d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b3060 0x7fca58071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca6029d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fca580b2b10 0x7fca580d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca6029d640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fca5815e4b0 con 0x7fca580717b0 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca6029d640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fca580d2e10 con 0x7fca580b2b10 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca6029d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca580b97e0 con 0x7fca580b3060 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e83a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fca580b2b10 0x7fca580d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e83a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fca580b2b10 0x7fca580d32a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40022/0 (socket says 172.21.15.64:40022) 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e83a640 1 -- 172.21.15.64:0/2619107206 learned_addr learned my addr 172.21.15.64:0/2619107206 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e039640 1 --1- 172.21.15.64:0/2619107206 >> v1:172.21.15.66:6789/0 conn(0x7fca580717b0 0x7fca580b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:41380/0 (socket says 172.21.15.64:41380) 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5d838640 1 --2- 172.21.15.64:0/2619107206 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b3060 0x7fca58071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e83a640 1 -- 172.21.15.64:0/2619107206 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b3060 msgr2=0x7fca58071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e83a640 1 --2- 172.21.15.64:0/2619107206 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b3060 0x7fca58071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e83a640 1 -- 172.21.15.64:0/2619107206 >> v1:172.21.15.66:6789/0 conn(0x7fca580717b0 legacy=0x7fca580b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5e83a640 1 -- 172.21.15.64:0/2619107206 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca580b9a80 con 0x7fca580b2b10 2024-03-04T07:27:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.089+0000 7fca5d037640 1 -- 172.21.15.64:0/2619107206 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2087965976 0 0) 0x7fca54001110 con 0x7fca580717b0 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.090+0000 7fca5e83a640 1 --2- 172.21.15.64:0/2619107206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580b2b10 0x7fca580d32a0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fca50009e50 tx=0x7fca5002bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f48ba162a3f5daf3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.090+0000 7fca5d037640 1 -- 172.21.15.64:0/2619107206 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca5002cb00 con 0x7fca580b2b10 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.090+0000 7fca5d037640 1 -- 172.21.15.64:0/2619107206 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca5002cca0 con 0x7fca580b2b10 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.090+0000 7fca5d037640 1 -- 172.21.15.64:0/2619107206 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca50034610 con 0x7fca580b2b10 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 -- 172.21.15.64:0/2619107206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580b2b10 msgr2=0x7fca580d32a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 --2- 172.21.15.64:0/2619107206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580b2b10 0x7fca580d32a0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fca50009e50 tx=0x7fca5002bf60 comp rx=0 tx=0).stop 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 -- 172.21.15.64:0/2619107206 shutdown_connections 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 --2- 172.21.15.64:0/2619107206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580b2b10 0x7fca580d32a0 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 --2- 172.21.15.64:0/2619107206 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b3060 0x7fca58071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 -- 172.21.15.64:0/2619107206 >> 172.21.15.64:0/2619107206 conn(0x7fca580d3d20 msgr2=0x7fca580d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 -- 172.21.15.64:0/2619107206 shutdown_connections 2024-03-04T07:27:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.091+0000 7fca6029d640 1 -- 172.21.15.64:0/2619107206 wait complete. 2024-03-04T07:27:19.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.092+0000 7fca6029d640 1 Processor -- start 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.092+0000 7fca6029d640 1 -- start start 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.092+0000 7fca6029d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580717b0 0x7fca581f3300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.092+0000 7fca6029d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b2b10 0x7fca580cd330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca6029d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca580b3060 0x7fca5814a480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca6029d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca580b7560 con 0x7fca580717b0 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca6029d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca580d4bf0 con 0x7fca580b3060 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca6029d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca580b76e0 con 0x7fca580b2b10 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580717b0 0x7fca581f3300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580717b0 0x7fca581f3300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40036/0 (socket says 172.21.15.64:40036) 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 -- 172.21.15.64:0/4168920145 learned_addr learned my addr 172.21.15.64:0/4168920145 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5d838640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b2b10 0x7fca580cd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e83a640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca580b3060 0x7fca5814a480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 -- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b2b10 msgr2=0x7fca580cd330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b2b10 0x7fca580cd330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 -- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca580b3060 msgr2=0x7fca5814a480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca580b3060 0x7fca5814a480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 -- 172.21.15.64:0/4168920145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca581f3840 con 0x7fca580717b0 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca5e039640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580717b0 0x7fca581f3300 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fca5400c780 tx=0x7fca5400cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca4effd640 1 -- 172.21.15.64:0/4168920145 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca54018070 con 0x7fca580717b0 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca4effd640 1 -- 172.21.15.64:0/4168920145 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca540072d0 con 0x7fca580717b0 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca580cd870 con 0x7fca580717b0 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca4effd640 1 -- 172.21.15.64:0/4168920145 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca540047e0 con 0x7fca580717b0 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.093+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca580cc570 con 0x7fca580717b0 2024-03-04T07:27:19.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.094+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca5806ce40 con 0x7fca580717b0 2024-03-04T07:27:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.095+0000 7fca4effd640 1 -- 172.21.15.64:0/4168920145 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fca5401b070 con 0x7fca580717b0 2024-03-04T07:27:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.095+0000 7fca4effd640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca3c06b4d0 0x7fca3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.095+0000 7fca4effd640 1 -- 172.21.15.64:0/4168920145 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fca54089460 con 0x7fca580717b0 2024-03-04T07:27:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.095+0000 7fca5d838640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca3c06b4d0 0x7fca3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.096+0000 7fca5d838640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca3c06b4d0 0x7fca3c06d990 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fca580ce110 tx=0x7fca48009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.099+0000 7fca4effd640 1 -- 172.21.15.64:0/4168920145 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fca5815e4b0 con 0x7fca580717b0 2024-03-04T07:27:19.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.278+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fca5804f410 con 0x7fca580717b0 2024-03-04T07:27:19.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.279+0000 7fca4effd640 1 -- 172.21.15.64:0/4168920145 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7fca54057c90 con 0x7fca580717b0 2024-03-04T07:27:19.279 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca3c06b4d0 msgr2=0x7fca3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca3c06b4d0 0x7fca3c06d990 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fca580ce110 tx=0x7fca48009190 comp rx=0 tx=0).stop 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580717b0 msgr2=0x7fca581f3300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580717b0 0x7fca581f3300 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fca5400c780 tx=0x7fca5400cc40 comp rx=0 tx=0).stop 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 shutdown_connections 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca3c06b4d0 0x7fca3c06d990 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca580b3060 0x7fca5814a480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca580b2b10 0x7fca580cd330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 --2- 172.21.15.64:0/4168920145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca580717b0 0x7fca581f3300 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 >> 172.21.15.64:0/4168920145 conn(0x7fca580d3d20 msgr2=0x7fca5815cfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:19.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 shutdown_connections 2024-03-04T07:27:19.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.281+0000 7fca6029d640 1 -- 172.21.15.64:0/4168920145 wait complete. 2024-03-04T07:27:19.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:19.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.399+0000 7f5fe6781640 1 Processor -- start 2024-03-04T07:27:19.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.399+0000 7f5fe6781640 1 -- start start 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.399+0000 7f5fe6781640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5fe0144640 0x7f5fe006e400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.399+0000 7f5fe6781640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe00ced70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.399+0000 7f5fe6781640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5fe015bc40 con 0x7f5fe006e940 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.399+0000 7f5fe6781640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5fe00cf2b0 con 0x7f5fe0144640 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.399+0000 7f5fe6781640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5fe006df70 con 0x7f5fe00ce900 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fdf7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5fe006e940 0x7f5fe015b130 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48422/0 (socket says 172.21.15.64:48422) 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fdf7fe640 1 -- 172.21.15.64:0/548511802 learned_addr learned my addr 172.21.15.64:0/548511802 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fdffff640 1 --2- 172.21.15.64:0/548511802 >> v2:172.21.15.66:3300/0 conn(0x7f5fe0144640 0x7f5fe006e400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fe4d1e640 1 --2- 172.21.15.64:0/548511802 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe00ced70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fdeffd640 1 -- 172.21.15.64:0/548511802 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2071388142 0 0) 0x7f5fe015bc40 con 0x7f5fe006e940 2024-03-04T07:27:19.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fdeffd640 1 -- 172.21.15.64:0/548511802 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5fc8003520 con 0x7f5fe006e940 2024-03-04T07:27:19.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fe4d1e640 1 -- 172.21.15.64:0/548511802 >> v2:172.21.15.66:3300/0 conn(0x7f5fe0144640 msgr2=0x7f5fe006e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:19.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fe4d1e640 1 --2- 172.21.15.64:0/548511802 >> v2:172.21.15.66:3300/0 conn(0x7f5fe0144640 0x7f5fe006e400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fe4d1e640 1 -- 172.21.15.64:0/548511802 >> v1:172.21.15.64:6789/0 conn(0x7f5fe006e940 legacy=0x7f5fe015b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.400+0000 7f5fe4d1e640 1 -- 172.21.15.64:0/548511802 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5fe015bf20 con 0x7f5fe00ce900 2024-03-04T07:27:19.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fe4d1e640 1 --2- 172.21.15.64:0/548511802 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe00ced70 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f5fd8009d90 tx=0x7f5fd802bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c66378863f70c4b8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fdeffd640 1 -- 172.21.15.64:0/548511802 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fd8039070 con 0x7f5fe00ce900 2024-03-04T07:27:19.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fdeffd640 1 -- 172.21.15.64:0/548511802 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5fd802cc50 con 0x7f5fe00ce900 2024-03-04T07:27:19.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fdeffd640 1 -- 172.21.15.64:0/548511802 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fd8034440 con 0x7f5fe00ce900 2024-03-04T07:27:19.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fe6781640 1 -- 172.21.15.64:0/548511802 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 msgr2=0x7f5fe00ced70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fe6781640 1 --2- 172.21.15.64:0/548511802 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe00ced70 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f5fd8009d90 tx=0x7f5fd802bea0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fe6781640 1 -- 172.21.15.64:0/548511802 shutdown_connections 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fe6781640 1 --2- 172.21.15.64:0/548511802 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe00ced70 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fe6781640 1 --2- 172.21.15.64:0/548511802 >> v2:172.21.15.66:3300/0 conn(0x7f5fe0144640 0x7f5fe006e400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.401+0000 7f5fe6781640 1 -- 172.21.15.64:0/548511802 >> 172.21.15.64:0/548511802 conn(0x7f5fe0066d00 msgr2=0x7f5fe0067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.402+0000 7f5fe6781640 1 -- 172.21.15.64:0/548511802 shutdown_connections 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.402+0000 7f5fe6781640 1 -- 172.21.15.64:0/548511802 wait complete. 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.403+0000 7f5fe6781640 1 Processor -- start 2024-03-04T07:27:19.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.403+0000 7f5fe6781640 1 -- start start 2024-03-04T07:27:19.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.403+0000 7f5fe6781640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5fe006e940 0x7f5fe0138ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe6781640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe01f7dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe6781640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5fe0144640 0x7f5fe01e7a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe6781640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5fe0070c20 con 0x7f5fe006e940 2024-03-04T07:27:19.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe6781640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5fe0070aa0 con 0x7f5fe0144640 2024-03-04T07:27:19.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe6781640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5fe015b840 con 0x7f5fe00ce900 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5fe006e940 0x7f5fe0138ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5fe006e940 0x7f5fe0138ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40040/0 (socket says 172.21.15.64:40040) 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 -- 172.21.15.64:0/214666040 learned_addr learned my addr 172.21.15.64:0/214666040 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe4d1e640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5fe0144640 0x7f5fe01e7a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdf7fe640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe01f7dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 -- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 msgr2=0x7f5fe01f7dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe01f7dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 -- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5fe0144640 msgr2=0x7f5fe01e7a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5fe0144640 0x7f5fe01e7a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 -- 172.21.15.64:0/214666040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5fe0139df0 con 0x7f5fe006e940 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdffff640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5fe006e940 0x7f5fe0138ee0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f5fd000e8a0 tx=0x7f5fd000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdcff9640 1 -- 172.21.15.64:0/214666040 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fd0017070 con 0x7f5fe006e940 2024-03-04T07:27:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdcff9640 1 -- 172.21.15.64:0/214666040 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5fd000c940 con 0x7f5fe006e940 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5fe0139550 con 0x7f5fe006e940 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fdcff9640 1 -- 172.21.15.64:0/214666040 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5fd000cd30 con 0x7f5fe006e940 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.404+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5fe0139b10 con 0x7f5fe006e940 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.405+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5fe00733c0 con 0x7f5fe006e940 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.406+0000 7f5fdcff9640 1 -- 172.21.15.64:0/214666040 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5fd001a070 con 0x7f5fe006e940 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.406+0000 7f5fdcff9640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fcc06b4d0 0x7f5fcc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.406+0000 7f5fdcff9640 1 -- 172.21.15.64:0/214666040 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5fd0089f50 con 0x7f5fe006e940 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.406+0000 7f5fdf7fe640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fcc06b4d0 0x7f5fcc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.407+0000 7f5fdf7fe640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fcc06b4d0 0x7f5fcc06d990 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f5fe00cd340 tx=0x7f5fd4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.411+0000 7f5fdcff9640 1 -- 172.21.15.64:0/214666040 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5fd0058810 con 0x7f5fe006e940 2024-03-04T07:27:19.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.590+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f5fe004f410 con 0x7f5fe006e940 2024-03-04T07:27:19.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.590+0000 7f5fdcff9640 1 -- 172.21.15.64:0/214666040 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f5fd005c8b0 con 0x7f5fe006e940 2024-03-04T07:27:19.590 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fcc06b4d0 msgr2=0x7f5fcc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fcc06b4d0 0x7f5fcc06d990 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f5fe00cd340 tx=0x7f5fd4009040 comp rx=0 tx=0).stop 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5fe006e940 msgr2=0x7f5fe0138ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5fe006e940 0x7f5fe0138ee0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f5fd000e8a0 tx=0x7f5fd000ed60 comp rx=0 tx=0).stop 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 shutdown_connections 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fcc06b4d0 0x7f5fcc06d990 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5fe0144640 0x7f5fe01e7a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5fe00ce900 0x7f5fe01f7dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 --2- 172.21.15.64:0/214666040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5fe006e940 0x7f5fe0138ee0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 >> 172.21.15.64:0/214666040 conn(0x7f5fe0066d00 msgr2=0x7f5fe00670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:19.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 shutdown_connections 2024-03-04T07:27:19.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.592+0000 7f5fe6781640 1 -- 172.21.15.64:0/214666040 wait complete. 2024-03-04T07:27:19.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dir_deletion' 2024-03-04T07:27:19.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.705+0000 7efdddc5c640 1 Processor -- start 2024-03-04T07:27:19.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.705+0000 7efdddc5c640 1 -- start start 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.705+0000 7efdddc5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80ce9e0 0x7efdd806eac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.705+0000 7efdddc5c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efdd80d0990 con 0x7efdd806f890 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.705+0000 7efdddc5c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efdd80d0c70 con 0x7efdd80b0f40 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.705+0000 7efdddc5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efdd806af20 con 0x7efdd80ce9e0 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7efdd806f890 0x7efdd815d6f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48432/0 (socket says 172.21.15.64:48432) 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd7fff640 1 -- 172.21.15.64:0/1622968216 learned_addr learned my addr 172.21.15.64:0/1622968216 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd77fe640 1 --2- 172.21.15.64:0/1622968216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80ce9e0 0x7efdd806eac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd67fc640 1 -- 172.21.15.64:0/1622968216 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2235825882 0 0) 0x7efdd80d0990 con 0x7efdd806f890 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd67fc640 1 -- 172.21.15.64:0/1622968216 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7efdc0003520 con 0x7efdd806f890 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd67fc640 1 -- 172.21.15.64:0/1622968216 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 377252291 0 0) 0x7efdd80d0c70 con 0x7efdd80b0f40 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd67fc640 1 -- 172.21.15.64:0/1622968216 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7efdd80d0990 con 0x7efdd80b0f40 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd77fe640 1 -- 172.21.15.64:0/1622968216 >> v1:172.21.15.66:6789/0 conn(0x7efdd80b0f40 legacy=0x7efdd806f180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd77fe640 1 -- 172.21.15.64:0/1622968216 >> v1:172.21.15.64:6789/0 conn(0x7efdd806f890 legacy=0x7efdd815d6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.706+0000 7efdd77fe640 1 -- 172.21.15.64:0/1622968216 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efdd80d0f50 con 0x7efdd80ce9e0 2024-03-04T07:27:19.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.707+0000 7efdd77fe640 1 --2- 172.21.15.64:0/1622968216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80ce9e0 0x7efdd806eac0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7efdc80097d0 tx=0x7efdc802b8a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b6dd992d4173586b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.707+0000 7efdd67fc640 1 -- 172.21.15.64:0/1622968216 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdc8035020 con 0x7efdd80ce9e0 2024-03-04T07:27:19.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.707+0000 7efdd67fc640 1 -- 172.21.15.64:0/1622968216 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efdc8031800 con 0x7efdd80ce9e0 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.707+0000 7efdd67fc640 1 -- 172.21.15.64:0/1622968216 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdc8031bb0 con 0x7efdd80ce9e0 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.707+0000 7efdddc5c640 1 -- 172.21.15.64:0/1622968216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80ce9e0 msgr2=0x7efdd806eac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.707+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1622968216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80ce9e0 0x7efdd806eac0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7efdc80097d0 tx=0x7efdc802b8a0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.708+0000 7efdddc5c640 1 -- 172.21.15.64:0/1622968216 shutdown_connections 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.708+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1622968216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80ce9e0 0x7efdd806eac0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.708+0000 7efdddc5c640 1 -- 172.21.15.64:0/1622968216 >> 172.21.15.64:0/1622968216 conn(0x7efdd80d1a80 msgr2=0x7efdd80d1e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.708+0000 7efdddc5c640 1 -- 172.21.15.64:0/1622968216 shutdown_connections 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.708+0000 7efdddc5c640 1 -- 172.21.15.64:0/1622968216 wait complete. 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.709+0000 7efdddc5c640 1 Processor -- start 2024-03-04T07:27:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.709+0000 7efdddc5c640 1 -- start start 2024-03-04T07:27:19.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdddc5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdd806f890 0x7efdd81c98a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdddc5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80b0f40 0x7efdd81e03c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdddc5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdd80ce9e0 0x7efdd81e7f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdddc5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdd806c6a0 con 0x7efdd80ce9e0 2024-03-04T07:27:19.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdddc5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdd806b200 con 0x7efdd806f890 2024-03-04T07:27:19.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdddc5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efdd806c820 con 0x7efdd80b0f40 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdd80ce9e0 0x7efdd81e7f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd6ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80b0f40 0x7efdd81e03c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd77fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdd806f890 0x7efdd81c98a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdd80ce9e0 0x7efdd81e7f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40052/0 (socket says 172.21.15.64:40052) 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd6ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80b0f40 0x7efdd81e03c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:43186/0 (socket says 172.21.15.64:43186) 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd7fff640 1 -- 172.21.15.64:0/1975905410 learned_addr learned my addr 172.21.15.64:0/1975905410 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd77fe640 1 -- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80b0f40 msgr2=0x7efdd81e03c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd77fe640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80b0f40 0x7efdd81e03c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd77fe640 1 -- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdd80ce9e0 msgr2=0x7efdd81e7f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd77fe640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdd80ce9e0 0x7efdd81e7f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd77fe640 1 -- 172.21.15.64:0/1975905410 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efdd81e0900 con 0x7efdd806f890 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd6ffd640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80b0f40 0x7efdd81e03c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd7fff640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdd80ce9e0 0x7efdd81e7f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdd77fe640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdd806f890 0x7efdd81c98a0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7efdc8033000 tx=0x7efdc80046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdbffff640 1 -- 172.21.15.64:0/1975905410 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdc8040070 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.710+0000 7efdbffff640 1 -- 172.21.15.64:0/1975905410 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efdc8039cf0 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.711+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efdd81e0b80 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.711+0000 7efdbffff640 1 -- 172.21.15.64:0/1975905410 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdc8035040 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.711+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efdd81e1140 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.712+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efdd8067ff0 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.712+0000 7efdbffff640 1 -- 172.21.15.64:0/1975905410 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efdc8038020 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.712+0000 7efdbffff640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efdc406b4d0 0x7efdc406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.712+0000 7efdbffff640 1 -- 172.21.15.64:0/1975905410 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7efdc80abb40 con 0x7efdd806f890 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.712+0000 7efdd6ffd640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efdc406b4d0 0x7efdc406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.713+0000 7efdd6ffd640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efdc406b4d0 0x7efdc406d990 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7efdcc007fd0 tx=0x7efdcc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:19.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.717+0000 7efdbffff640 1 -- 172.21.15.64:0/1975905410 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efdd81e0900 con 0x7efdd806f890 2024-03-04T07:27:19.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:19.966+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]} v 0) v1 -- 0x7efdd80b7900 con 0x7efdd806f890 2024-03-04T07:27:20.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.379+0000 7efdbffff640 1 -- 172.21.15.64:0/1975905410 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]}]=0 v45) v1 ==== 135+0+0 (secure 0 0 0) 0x7efdc807a350 con 0x7efdd806f890 2024-03-04T07:27:20.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.381+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efdc406b4d0 msgr2=0x7efdc406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efdc406b4d0 0x7efdc406d990 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7efdcc007fd0 tx=0x7efdcc009040 comp rx=0 tx=0).stop 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdd806f890 msgr2=0x7efdd81c98a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdd806f890 0x7efdd81c98a0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7efdc8033000 tx=0x7efdc80046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 shutdown_connections 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efdc406b4d0 0x7efdc406d990 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdd80ce9e0 0x7efdd81e7f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdd80b0f40 0x7efdd81e03c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 --2- 172.21.15.64:0/1975905410 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdd806f890 0x7efdd81c98a0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 >> 172.21.15.64:0/1975905410 conn(0x7efdd80d1a80 msgr2=0x7efdd81e4ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 shutdown_connections 2024-03-04T07:27:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.382+0000 7efdddc5c640 1 -- 172.21.15.64:0/1975905410 wait complete. 2024-03-04T07:27:20.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:27:20.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.498+0000 7f3569b13640 1 Processor -- start 2024-03-04T07:27:20.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.498+0000 7f3569b13640 1 -- start start 2024-03-04T07:27:20.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3569b13640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f356415dd90 0x7f35640ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:20.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3569b13640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f35640d29d0 0x7f35640d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:20.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3569b13640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f35640d3740 con 0x7f35640d4720 2024-03-04T07:27:20.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3569b13640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35640d3400 con 0x7f356415dd90 2024-03-04T07:27:20.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3569b13640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35640d3120 con 0x7f35640d29d0 2024-03-04T07:27:20.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f35637fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f356415dd90 0x7f35640ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:20.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3563fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f35640d4720 0x7f35640b6d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48444/0 (socket says 172.21.15.64:48444) 2024-03-04T07:27:20.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3563fff640 1 -- 172.21.15.64:0/2776758828 learned_addr learned my addr 172.21.15.64:0/2776758828 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:20.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f3562ffd640 1 --2- 172.21.15.64:0/2776758828 >> v2:172.21.15.66:3300/0 conn(0x7f35640d29d0 0x7f35640d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:20.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f35637fe640 1 -- 172.21.15.64:0/2776758828 >> v2:172.21.15.66:3300/0 conn(0x7f35640d29d0 msgr2=0x7f35640d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:20.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f35637fe640 1 --2- 172.21.15.64:0/2776758828 >> v2:172.21.15.66:3300/0 conn(0x7f35640d29d0 0x7f35640d41e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.499+0000 7f35637fe640 1 -- 172.21.15.64:0/2776758828 >> v1:172.21.15.64:6789/0 conn(0x7f35640d4720 legacy=0x7f35640b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.500+0000 7f35627fc640 1 -- 172.21.15.64:0/2776758828 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3695086317 0 0) 0x7f35640d3740 con 0x7f35640d4720 2024-03-04T07:27:20.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.500+0000 7f35637fe640 1 -- 172.21.15.64:0/2776758828 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35640d39e0 con 0x7f356415dd90 2024-03-04T07:27:20.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.500+0000 7f35637fe640 1 --2- 172.21.15.64:0/2776758828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f35640ceea0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f35640724d0 tx=0x7f355402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3734261824924882 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.500+0000 7f35627fc640 1 -- 172.21.15.64:0/2776758828 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3554034070 con 0x7f356415dd90 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.500+0000 7f35627fc640 1 -- 172.21.15.64:0/2776758828 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3554030610 con 0x7f356415dd90 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.500+0000 7f35627fc640 1 -- 172.21.15.64:0/2776758828 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35540309e0 con 0x7f356415dd90 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 -- 172.21.15.64:0/2776758828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 msgr2=0x7f35640ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 --2- 172.21.15.64:0/2776758828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f35640ceea0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f35640724d0 tx=0x7f355402b960 comp rx=0 tx=0).stop 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 -- 172.21.15.64:0/2776758828 shutdown_connections 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 --2- 172.21.15.64:0/2776758828 >> v2:172.21.15.66:3300/0 conn(0x7f35640d29d0 0x7f35640d41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 --2- 172.21.15.64:0/2776758828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f35640ceea0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 -- 172.21.15.64:0/2776758828 >> 172.21.15.64:0/2776758828 conn(0x7f3564071960 msgr2=0x7f3564071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 -- 172.21.15.64:0/2776758828 shutdown_connections 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.501+0000 7f3569b13640 1 -- 172.21.15.64:0/2776758828 wait complete. 2024-03-04T07:27:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.502+0000 7f3569b13640 1 Processor -- start 2024-03-04T07:27:20.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.502+0000 7f3569b13640 1 -- start start 2024-03-04T07:27:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3569b13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35640d29d0 0x7f3564149ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3569b13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35640d4720 0x7f35640c9d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3569b13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f356414a1e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3569b13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35640d2df0 con 0x7f356415dd90 2024-03-04T07:27:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3569b13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f356415e170 con 0x7f35640d4720 2024-03-04T07:27:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3569b13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35640b7420 con 0x7f35640d29d0 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f356414a1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f356414a1e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40070/0 (socket says 172.21.15.64:40070) 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 -- 172.21.15.64:0/666510951 learned_addr learned my addr 172.21.15.64:0/666510951 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f35637fe640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35640d29d0 0x7f3564149ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3562ffd640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35640d4720 0x7f35640c9d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 -- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35640d29d0 msgr2=0x7f3564149ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35640d29d0 0x7f3564149ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 -- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35640d4720 msgr2=0x7f35640c9d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35640d4720 0x7f35640c9d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.503+0000 7f3563fff640 1 -- 172.21.15.64:0/666510951 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35640c8cb0 con 0x7f356415dd90 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.504+0000 7f3563fff640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f356414a1e0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f355c00c7d0 tx=0x7f355c00cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.504+0000 7f354bfff640 1 -- 172.21.15.64:0/666510951 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f355c018070 con 0x7f356415dd90 2024-03-04T07:27:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.504+0000 7f354bfff640 1 -- 172.21.15.64:0/666510951 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f355c00a5e0 con 0x7f356415dd90 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.504+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35640c8f90 con 0x7f356415dd90 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.504+0000 7f354bfff640 1 -- 172.21.15.64:0/666510951 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f355c00aa10 con 0x7f356415dd90 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.504+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35640c9550 con 0x7f356415dd90 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.505+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3564072a30 con 0x7f356415dd90 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.505+0000 7f354bfff640 1 -- 172.21.15.64:0/666510951 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f355c01b070 con 0x7f356415dd90 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.506+0000 7f354bfff640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f355006b4d0 0x7f355006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.506+0000 7f354bfff640 1 -- 172.21.15.64:0/666510951 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f355c089730 con 0x7f356415dd90 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.506+0000 7f35637fe640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f355006b4d0 0x7f355006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.506+0000 7f35637fe640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f355006b4d0 0x7f355006d990 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f3554036000 tx=0x7f3554008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.510+0000 7f354bfff640 1 -- 172.21.15.64:0/666510951 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f355c057f60 con 0x7f356415dd90 2024-03-04T07:27:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.690+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f356415d740 con 0x7f356415dd90 2024-03-04T07:27:20.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.693+0000 7f354bfff640 1 -- 172.21.15.64:0/666510951 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v10)=0 v10) v1 ==== 122+0+0 (secure 0 0 0) 0x7f355c05c000 con 0x7f356415dd90 2024-03-04T07:27:20.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f355006b4d0 msgr2=0x7f355006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f355006b4d0 0x7f355006d990 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f3554036000 tx=0x7f3554008030 comp rx=0 tx=0).stop 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 msgr2=0x7f356414a1e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f356414a1e0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f355c00c7d0 tx=0x7f355c00cc90 comp rx=0 tx=0).stop 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 shutdown_connections 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f355006b4d0 0x7f355006d990 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f356415dd90 0x7f356414a1e0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35640d4720 0x7f35640c9d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 --2- 172.21.15.64:0/666510951 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35640d29d0 0x7f3564149ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.695+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 >> 172.21.15.64:0/666510951 conn(0x7f3564071960 msgr2=0x7f35640b8130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.696+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 shutdown_connections 2024-03-04T07:27:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:20.696+0000 7f3569b13640 1 -- 172.21.15.64:0/666510951 wait complete. 2024-03-04T07:27:20.706 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:27:20.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:27:20.745 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:27:20.794 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:27:20.795 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:27:20.795 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:20.810 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:27:20.810 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:20.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:21.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.008+0000 7f96aa2fa640 1 Processor -- start 2024-03-04T07:27:21.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.008+0000 7f96aa2fa640 1 -- start start 2024-03-04T07:27:21.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.008+0000 7f96aa2fa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96a40b4260 0x7f96a40cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.008+0000 7f96aa2fa640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f96a40d06c0 0x7f96a40b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.008+0000 7f96aa2fa640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f96a40b0dd0 con 0x7f96a40d02e0 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.008+0000 7f96aa2fa640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96a40b0b80 con 0x7f96a40b4260 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.008+0000 7f96aa2fa640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96a40aeec0 con 0x7f96a40d06c0 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a37fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96a40b4260 0x7f96a40cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a37fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96a40b4260 0x7f96a40cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40080/0 (socket says 172.21.15.64:40080) 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a37fe640 1 -- 172.21.15.64:0/389053009 learned_addr learned my addr 172.21.15.64:0/389053009 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a8897640 1 --2- 172.21.15.64:0/389053009 >> v2:172.21.15.66:3300/0 conn(0x7f96a40d06c0 0x7f96a40b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a37fe640 1 -- 172.21.15.64:0/389053009 >> v2:172.21.15.66:3300/0 conn(0x7f96a40d06c0 msgr2=0x7f96a40b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a37fe640 1 --2- 172.21.15.64:0/389053009 >> v2:172.21.15.66:3300/0 conn(0x7f96a40d06c0 0x7f96a40b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a37fe640 1 -- 172.21.15.64:0/389053009 >> v1:172.21.15.64:6789/0 conn(0x7f96a40d02e0 legacy=0x7f96a40b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.009+0000 7f96a37fe640 1 -- 172.21.15.64:0/389053009 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96a40b10b0 con 0x7f96a40b4260 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96a37fe640 1 --2- 172.21.15.64:0/389053009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40b4260 0x7f96a40cfa00 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f96a40cc0e0 tx=0x7f969802ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=99876512a9de5985 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96a2ffd640 1 -- 172.21.15.64:0/389053009 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969802f850 con 0x7f96a40b4260 2024-03-04T07:27:21.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96a2ffd640 1 -- 172.21.15.64:0/389053009 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f969802f9f0 con 0x7f96a40b4260 2024-03-04T07:27:21.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96a2ffd640 1 -- 172.21.15.64:0/389053009 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969802fdc0 con 0x7f96a40b4260 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 -- 172.21.15.64:0/389053009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40b4260 msgr2=0x7f96a40cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/389053009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40b4260 0x7f96a40cfa00 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f96a40cc0e0 tx=0x7f969802ba30 comp rx=0 tx=0).stop 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 -- 172.21.15.64:0/389053009 shutdown_connections 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/389053009 >> v2:172.21.15.66:3300/0 conn(0x7f96a40d06c0 0x7f96a40b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/389053009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40b4260 0x7f96a40cfa00 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 -- 172.21.15.64:0/389053009 >> 172.21.15.64:0/389053009 conn(0x7f96a40b36d0 msgr2=0x7f96a40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 -- 172.21.15.64:0/389053009 shutdown_connections 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.010+0000 7f96aa2fa640 1 -- 172.21.15.64:0/389053009 wait complete. 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.011+0000 7f96aa2fa640 1 Processor -- start 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96aa2fa640 1 -- start start 2024-03-04T07:27:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96aa2fa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a40b4260 0x7f96a4152f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96aa2fa640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96a40d02e0 0x7f96a413ef20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96aa2fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40d06c0 0x7f96a415a460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96aa2fa640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96a40b22d0 con 0x7f96a40d06c0 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96aa2fa640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96a40b2150 con 0x7f96a40d02e0 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96aa2fa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96a40af350 con 0x7f96a40b4260 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96a8897640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40d06c0 0x7f96a415a460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96a8897640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40d06c0 0x7f96a415a460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40082/0 (socket says 172.21.15.64:40082) 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.012+0000 7f96a8897640 1 -- 172.21.15.64:0/1514365788 learned_addr learned my addr 172.21.15.64:0/1514365788 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a3fff640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a40b4260 0x7f96a4152f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a37fe640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96a40d02e0 0x7f96a413ef20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a8897640 1 -- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a40b4260 msgr2=0x7f96a4152f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a8897640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a40b4260 0x7f96a4152f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a8897640 1 -- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96a40d02e0 msgr2=0x7f96a413ef20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a8897640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96a40d02e0 0x7f96a413ef20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a8897640 1 -- 172.21.15.64:0/1514365788 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96a41596a0 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a8897640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40d06c0 0x7f96a415a460 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f969c00efc0 tx=0x7f969c00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a0ff9640 1 -- 172.21.15.64:0/1514365788 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969c018020 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a0ff9640 1 -- 172.21.15.64:0/1514365788 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f969c014480 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96a4159980 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96a0ff9640 1 -- 172.21.15.64:0/1514365788 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969c014870 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.013+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96a4153480 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.014+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f96a41e6d00 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.014+0000 7f96a0ff9640 1 -- 172.21.15.64:0/1514365788 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f969c005d10 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.015+0000 7f96a0ff9640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b500 0x7f969006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.015+0000 7f96a0ff9640 1 -- 172.21.15.64:0/1514365788 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f969c08ad60 con 0x7f96a40d06c0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.015+0000 7f96a3fff640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b500 0x7f969006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.015+0000 7f96a3fff640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b500 0x7f969006d9c0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f9694001780 tx=0x7f9694009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.019+0000 7f96a0ff9640 1 -- 172.21.15.64:0/1514365788 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f96a41e6d00 con 0x7f96a40d06c0 2024-03-04T07:27:21.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.201+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f96a40cf2e0 con 0x7f96a40d06c0 2024-03-04T07:27:21.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.202+0000 7f96a0ff9640 1 -- 172.21.15.64:0/1514365788 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f96a40cf2e0 con 0x7f96a40d06c0 2024-03-04T07:27:21.202 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:21.202 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:21.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b500 msgr2=0x7f969006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b500 0x7f969006d9c0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f9694001780 tx=0x7f9694009040 comp rx=0 tx=0).stop 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40d06c0 msgr2=0x7f96a415a460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40d06c0 0x7f96a415a460 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f969c00efc0 tx=0x7f969c00c490 comp rx=0 tx=0).stop 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 shutdown_connections 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b500 0x7f969006d9c0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a40d06c0 0x7f96a415a460 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96a40d02e0 0x7f96a413ef20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 --2- 172.21.15.64:0/1514365788 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a40b4260 0x7f96a4152f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.204+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 >> 172.21.15.64:0/1514365788 conn(0x7f96a40b36d0 msgr2=0x7f96a40cc6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.205+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 shutdown_connections 2024-03-04T07:27:21.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.205+0000 7f96aa2fa640 1 -- 172.21.15.64:0/1514365788 wait complete. 2024-03-04T07:27:21.215 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:21.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:21.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.321+0000 7f99103ac640 1 Processor -- start 2024-03-04T07:27:21.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.321+0000 7f99103ac640 1 -- start start 2024-03-04T07:27:21.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f99103ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9908072a30 0x7f99081624c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f99103ac640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99080d07c0 0x7f990815d660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f99103ac640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f990806d4d0 con 0x7f99080cf9f0 2024-03-04T07:27:21.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f99103ac640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f990806dba0 con 0x7f99080d07c0 2024-03-04T07:27:21.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f99103ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99080b4950 con 0x7f9908072a30 2024-03-04T07:27:21.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e949640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99080d07c0 0x7f990815d660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e949640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99080d07c0 0x7f990815d660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40090/0 (socket says 172.21.15.64:40090) 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e949640 1 -- 172.21.15.64:0/312323790 learned_addr learned my addr 172.21.15.64:0/312323790 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e148640 1 --2- 172.21.15.64:0/312323790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9908072a30 0x7f99081624c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e949640 1 -- 172.21.15.64:0/312323790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9908072a30 msgr2=0x7f99081624c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e949640 1 --2- 172.21.15.64:0/312323790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9908072a30 0x7f99081624c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e949640 1 -- 172.21.15.64:0/312323790 >> v1:172.21.15.64:6789/0 conn(0x7f99080cf9f0 legacy=0x7f99080d00b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e949640 1 -- 172.21.15.64:0/312323790 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f990806d7b0 con 0x7f99080d07c0 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.322+0000 7f990e148640 1 --2- 172.21.15.64:0/312323790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9908072a30 0x7f99081624c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:21.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.323+0000 7f990e949640 1 --2- 172.21.15.64:0/312323790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99080d07c0 0x7f990815d660 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f9900009e50 tx=0x7f990002be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bf84b75cc6346986 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.323+0000 7f990d146640 1 -- 172.21.15.64:0/312323790 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f990002c9e0 con 0x7f99080d07c0 2024-03-04T07:27:21.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.323+0000 7f990d146640 1 -- 172.21.15.64:0/312323790 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f990002cb80 con 0x7f99080d07c0 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.323+0000 7f990d146640 1 -- 172.21.15.64:0/312323790 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99000343f0 con 0x7f99080d07c0 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 -- 172.21.15.64:0/312323790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99080d07c0 msgr2=0x7f990815d660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 --2- 172.21.15.64:0/312323790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99080d07c0 0x7f990815d660 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f9900009e50 tx=0x7f990002be40 comp rx=0 tx=0).stop 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 -- 172.21.15.64:0/312323790 shutdown_connections 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 --2- 172.21.15.64:0/312323790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99080d07c0 0x7f990815d660 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 --2- 172.21.15.64:0/312323790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9908072a30 0x7f99081624c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 -- 172.21.15.64:0/312323790 >> 172.21.15.64:0/312323790 conn(0x7f990806af20 msgr2=0x7f990806b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 -- 172.21.15.64:0/312323790 shutdown_connections 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.324+0000 7f99103ac640 1 -- 172.21.15.64:0/312323790 wait complete. 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.325+0000 7f99103ac640 1 Processor -- start 2024-03-04T07:27:21.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.325+0000 7f99103ac640 1 -- start start 2024-03-04T07:27:21.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9908072a30 0x7f990814f7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99080cf9f0 0x7f99080ccbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99080d07c0 0x7f99080cd120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9908072e10 con 0x7f9908072a30 2024-03-04T07:27:21.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9908071130 con 0x7f99080d07c0 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9908072f90 con 0x7f99080cf9f0 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9908072a30 0x7f990814f7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9908072a30 0x7f990814f7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40102/0 (socket says 172.21.15.64:40102) 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 -- 172.21.15.64:0/416820748 learned_addr learned my addr 172.21.15.64:0/416820748 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e949640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99080d07c0 0x7f99080cd120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990d947640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99080cf9f0 0x7f99080ccbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 -- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99080cf9f0 msgr2=0x7f99080ccbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99080cf9f0 0x7f99080ccbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 -- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99080d07c0 msgr2=0x7f99080cd120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99080d07c0 0x7f99080cd120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 -- 172.21.15.64:0/416820748 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9908152150 con 0x7f9908072a30 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f990e148640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9908072a30 0x7f990814f7c0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f990400e960 tx=0x7f990400ee20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f98feffd640 1 -- 172.21.15.64:0/416820748 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9904017070 con 0x7f9908072a30 2024-03-04T07:27:21.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f98feffd640 1 -- 172.21.15.64:0/416820748 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f990400ca00 con 0x7f9908072a30 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9908152430 con 0x7f9908072a30 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f98feffd640 1 -- 172.21.15.64:0/416820748 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f990400cdd0 con 0x7f9908072a30 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.326+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99081529f0 con 0x7f9908072a30 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.327+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f990809e320 con 0x7f9908072a30 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.328+0000 7f98feffd640 1 -- 172.21.15.64:0/416820748 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f990401a070 con 0x7f9908072a30 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.328+0000 7f98feffd640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98ec06b500 0x7f98ec06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.328+0000 7f98feffd640 1 -- 172.21.15.64:0/416820748 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f990408a380 con 0x7f9908072a30 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.328+0000 7f990d947640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98ec06b500 0x7f98ec06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.329+0000 7f990d947640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98ec06b500 0x7f98ec06d9c0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f99080cd6c0 tx=0x7f98f8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.332+0000 7f98feffd640 1 -- 172.21.15.64:0/416820748 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9904058bb0 con 0x7f9908072a30 2024-03-04T07:27:21.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.516+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f990804f410 con 0x7f9908072a30 2024-03-04T07:27:21.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.516+0000 7f98feffd640 1 -- 172.21.15.64:0/416820748 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f990405cc50 con 0x7f9908072a30 2024-03-04T07:27:21.516 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:21.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.518+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98ec06b500 msgr2=0x7f98ec06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.518+0000 7f99103ac640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98ec06b500 0x7f98ec06d9c0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f99080cd6c0 tx=0x7f98f8009040 comp rx=0 tx=0).stop 2024-03-04T07:27:21.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.518+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9908072a30 msgr2=0x7f990814f7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.518+0000 7f99103ac640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9908072a30 0x7f990814f7c0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f990400e960 tx=0x7f990400ee20 comp rx=0 tx=0).stop 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 shutdown_connections 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98ec06b500 0x7f98ec06d9c0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99080d07c0 0x7f99080cd120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99080cf9f0 0x7f99080ccbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 --2- 172.21.15.64:0/416820748 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9908072a30 0x7f990814f7c0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 >> 172.21.15.64:0/416820748 conn(0x7f990806af20 msgr2=0x7f99081609d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 shutdown_connections 2024-03-04T07:27:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.519+0000 7f99103ac640 1 -- 172.21.15.64:0/416820748 wait complete. 2024-03-04T07:27:21.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:21.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.634+0000 7fce225d7640 1 Processor -- start 2024-03-04T07:27:21.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.634+0000 7fce225d7640 1 -- start start 2024-03-04T07:27:21.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.634+0000 7fce225d7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fce1c15dd50 0x7fce1c0ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.634+0000 7fce225d7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fce1c0cef20 0x7fce1c071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.634+0000 7fce225d7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fce1c072cf0 con 0x7fce1c0724b0 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce225d7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fce1c072fd0 con 0x7fce1c0cef20 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce225d7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fce1c0d1f60 con 0x7fce1c15dd50 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fce1c0cef20 0x7fce1c071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fce1c0cef20 0x7fce1c071f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40112/0 (socket says 172.21.15.64:40112) 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1b7fe640 1 -- 172.21.15.64:0/306695140 learned_addr learned my addr 172.21.15.64:0/306695140 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1bfff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fce1c15dd50 0x7fce1c0ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1b7fe640 1 -- 172.21.15.64:0/306695140 >> v2:172.21.15.66:3300/0 conn(0x7fce1c15dd50 msgr2=0x7fce1c0ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1b7fe640 1 --2- 172.21.15.64:0/306695140 >> v2:172.21.15.66:3300/0 conn(0x7fce1c15dd50 0x7fce1c0ce9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1b7fe640 1 -- 172.21.15.64:0/306695140 >> v1:172.21.15.66:6789/0 conn(0x7fce1c0724b0 legacy=0x7fce1c161a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.635+0000 7fce1b7fe640 1 -- 172.21.15.64:0/306695140 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce1c0d2240 con 0x7fce1c0cef20 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce1b7fe640 1 --2- 172.21.15.64:0/306695140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c071f70 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fce1c08c610 tx=0x7fce1002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6e0db280d282e0d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce1affd640 1 -- 172.21.15.64:0/306695140 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce10035020 con 0x7fce1c0cef20 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce1affd640 1 -- 172.21.15.64:0/306695140 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fce10031870 con 0x7fce1c0cef20 2024-03-04T07:27:21.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce1affd640 1 -- 172.21.15.64:0/306695140 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce10031c40 con 0x7fce1c0cef20 2024-03-04T07:27:21.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 -- 172.21.15.64:0/306695140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 msgr2=0x7fce1c071f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 --2- 172.21.15.64:0/306695140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c071f70 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fce1c08c610 tx=0x7fce1002b910 comp rx=0 tx=0).stop 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 -- 172.21.15.64:0/306695140 shutdown_connections 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 --2- 172.21.15.64:0/306695140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c071f70 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 --2- 172.21.15.64:0/306695140 >> v2:172.21.15.66:3300/0 conn(0x7fce1c15dd50 0x7fce1c0ce9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 -- 172.21.15.64:0/306695140 >> 172.21.15.64:0/306695140 conn(0x7fce1c0d3ef0 msgr2=0x7fce1c0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 -- 172.21.15.64:0/306695140 shutdown_connections 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.636+0000 7fce225d7640 1 -- 172.21.15.64:0/306695140 wait complete. 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.638+0000 7fce225d7640 1 Processor -- start 2024-03-04T07:27:21.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.638+0000 7fce225d7640 1 -- start start 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce225d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce1c0724b0 0x7fce1c0c9a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce225d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c0c9f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce225d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce1c15dd50 0x7fce1c0c7a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce225d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce1c15e130 con 0x7fce1c0cef20 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce225d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce1c072ad0 con 0x7fce1c0724b0 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce225d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fce1c15e2b0 con 0x7fce1c15dd50 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c0c9f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c0c9f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40128/0 (socket says 172.21.15.64:40128) 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 -- 172.21.15.64:0/1115183120 learned_addr learned my addr 172.21.15.64:0/1115183120 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1bfff640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce1c0724b0 0x7fce1c0c9a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce20b74640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce1c15dd50 0x7fce1c0c7a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 -- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce1c15dd50 msgr2=0x7fce1c0c7a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce1c15dd50 0x7fce1c0c7a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 -- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce1c0724b0 msgr2=0x7fce1c0c9a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce1c0724b0 0x7fce1c0c9a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 -- 172.21.15.64:0/1115183120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce1c0c8770 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce1b7fe640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c0c9f80 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fce10000c00 tx=0x7fce100046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce18ff9640 1 -- 172.21.15.64:0/1115183120 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce10035040 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.639+0000 7fce18ff9640 1 -- 172.21.15.64:0/1115183120 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fce10030d00 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.640+0000 7fce18ff9640 1 -- 172.21.15.64:0/1115183120 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce100456a0 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.640+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fce1c0c8a50 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.640+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fce1c0c9010 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.640+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fce1c06cdd0 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.641+0000 7fce18ff9640 1 -- 172.21.15.64:0/1115183120 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fce10038020 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.641+0000 7fce18ff9640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce0806b4d0 0x7fce0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.641+0000 7fce18ff9640 1 -- 172.21.15.64:0/1115183120 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fce100ac460 con 0x7fce1c0cef20 2024-03-04T07:27:21.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.642+0000 7fce1bfff640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce0806b4d0 0x7fce0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.642+0000 7fce1bfff640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce0806b4d0 0x7fce0806d990 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fce0c009aa0 tx=0x7fce0c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.646+0000 7fce18ff9640 1 -- 172.21.15.64:0/1115183120 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fce1007ac00 con 0x7fce1c0cef20 2024-03-04T07:27:21.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.825+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fce1c0d4770 con 0x7fce1c0cef20 2024-03-04T07:27:21.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.826+0000 7fce18ff9640 1 -- 172.21.15.64:0/1115183120 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7fce1007eca0 con 0x7fce1c0cef20 2024-03-04T07:27:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:21.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce0806b4d0 msgr2=0x7fce0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce0806b4d0 0x7fce0806d990 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fce0c009aa0 tx=0x7fce0c009190 comp rx=0 tx=0).stop 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 msgr2=0x7fce1c0c9f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c0c9f80 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fce10000c00 tx=0x7fce100046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 shutdown_connections 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce0806b4d0 0x7fce0806d990 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce1c15dd50 0x7fce1c0c7a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce1c0cef20 0x7fce1c0c9f80 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 --2- 172.21.15.64:0/1115183120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce1c0724b0 0x7fce1c0c9a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.827+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 >> 172.21.15.64:0/1115183120 conn(0x7fce1c0d3ef0 msgr2=0x7fce1c0d2660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.828+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 shutdown_connections 2024-03-04T07:27:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.828+0000 7fce225d7640 1 -- 172.21.15.64:0/1115183120 wait complete. 2024-03-04T07:27:21.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:21.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.943+0000 7f79cda55640 1 Processor -- start 2024-03-04T07:27:21.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.943+0000 7f79cda55640 1 -- start start 2024-03-04T07:27:21.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79cda55640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80717b0 0x7f79c8071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79cda55640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f79c80b3060 0x7f79c80d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79cda55640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f79c80b2b10 0x7f79c80b6e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79cda55640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f79c80d2e10 con 0x7f79c80b2b10 2024-03-04T07:27:21.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79cda55640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f79c815e4b0 con 0x7f79c80b3060 2024-03-04T07:27:21.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79cda55640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79c80b97e0 con 0x7f79c80717b0 2024-03-04T07:27:21.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c77fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f79c80b2b10 0x7f79c80b6e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c6ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80717b0 0x7f79c8071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c77fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f79c80b2b10 0x7f79c80b6e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40144/0 (socket says 172.21.15.64:40144) 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c77fe640 1 -- 172.21.15.64:0/1641199575 learned_addr learned my addr 172.21.15.64:0/1641199575 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79bffff640 1 --2- 172.21.15.64:0/1641199575 >> v2:172.21.15.66:3300/0 conn(0x7f79c80b3060 0x7f79c80d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c6ffd640 1 -- 172.21.15.64:0/1641199575 >> v2:172.21.15.66:3300/0 conn(0x7f79c80b3060 msgr2=0x7f79c80d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c6ffd640 1 --2- 172.21.15.64:0/1641199575 >> v2:172.21.15.66:3300/0 conn(0x7f79c80b3060 0x7f79c80d32a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c6ffd640 1 -- 172.21.15.64:0/1641199575 >> v2:172.21.15.64:3300/0 conn(0x7f79c80b2b10 msgr2=0x7f79c80b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c6ffd640 1 --2- 172.21.15.64:0/1641199575 >> v2:172.21.15.64:3300/0 conn(0x7f79c80b2b10 0x7f79c80b6e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c6ffd640 1 -- 172.21.15.64:0/1641199575 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79c80b9a80 con 0x7f79c80717b0 2024-03-04T07:27:21.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.944+0000 7f79c77fe640 1 --2- 172.21.15.64:0/1641199575 >> v2:172.21.15.64:3300/0 conn(0x7f79c80b2b10 0x7f79c80b6e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:21.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.945+0000 7f79c6ffd640 1 --2- 172.21.15.64:0/1641199575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80717b0 0x7f79c8071b90 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f79c808c7a0 tx=0x7f79b802b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ddd1684146e51b74 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.945+0000 7f79c67fc640 1 -- 172.21.15.64:0/1641199575 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79b80315a0 con 0x7f79c80717b0 2024-03-04T07:27:21.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.945+0000 7f79c67fc640 1 -- 172.21.15.64:0/1641199575 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f79b8037070 con 0x7f79c80717b0 2024-03-04T07:27:21.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.945+0000 7f79c67fc640 1 -- 172.21.15.64:0/1641199575 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79b8031960 con 0x7f79c80717b0 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 -- 172.21.15.64:0/1641199575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80717b0 msgr2=0x7f79c8071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 --2- 172.21.15.64:0/1641199575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80717b0 0x7f79c8071b90 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f79c808c7a0 tx=0x7f79b802b840 comp rx=0 tx=0).stop 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 -- 172.21.15.64:0/1641199575 shutdown_connections 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 --2- 172.21.15.64:0/1641199575 >> v2:172.21.15.64:3300/0 conn(0x7f79c80b2b10 0x7f79c80b6e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 --2- 172.21.15.64:0/1641199575 >> v2:172.21.15.66:3300/0 conn(0x7f79c80b3060 0x7f79c80d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 --2- 172.21.15.64:0/1641199575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80717b0 0x7f79c8071b90 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 -- 172.21.15.64:0/1641199575 >> 172.21.15.64:0/1641199575 conn(0x7f79c80d3d20 msgr2=0x7f79c80d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 -- 172.21.15.64:0/1641199575 shutdown_connections 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.946+0000 7f79cda55640 1 -- 172.21.15.64:0/1641199575 wait complete. 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.947+0000 7f79cda55640 1 Processor -- start 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.947+0000 7f79cda55640 1 -- start start 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.947+0000 7f79cda55640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79c80717b0 0x7f79c8149c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79cda55640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79c80b2b10 0x7f79c81f6970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79cda55640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80b3060 0x7f79c81f6eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79cda55640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79c80b7390 con 0x7f79c80717b0 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79cda55640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79c80d4bf0 con 0x7f79c80b2b10 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79cda55640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79c80b7510 con 0x7f79c80b3060 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79c80717b0 0x7f79c8149c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79c80717b0 0x7f79c8149c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40152/0 (socket says 172.21.15.64:40152) 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 -- 172.21.15.64:0/2310160029 learned_addr learned my addr 172.21.15.64:0/2310160029 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79bffff640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79c80b2b10 0x7f79c81f6970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c77fe640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80b3060 0x7f79c81f6eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 -- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80b3060 msgr2=0x7f79c81f6eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80b3060 0x7f79c81f6eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 -- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79c80b2b10 msgr2=0x7f79c81f6970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79c80b2b10 0x7f79c81f6970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 -- 172.21.15.64:0/2310160029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79c80ccdf0 con 0x7f79c80717b0 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c6ffd640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79c80717b0 0x7f79c8149c10 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f79b8009890 tx=0x7f79b80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c4ff9640 1 -- 172.21.15.64:0/2310160029 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79b8032630 con 0x7f79c80717b0 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c4ff9640 1 -- 172.21.15.64:0/2310160029 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f79b80327d0 con 0x7f79c80717b0 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f79c80cc270 con 0x7f79c80717b0 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79c4ff9640 1 -- 172.21.15.64:0/2310160029 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79b8032bc0 con 0x7f79c80717b0 2024-03-04T07:27:21.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.948+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f79c80cc830 con 0x7f79c80717b0 2024-03-04T07:27:21.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.949+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f79c806ce40 con 0x7f79c80717b0 2024-03-04T07:27:21.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.950+0000 7f79c4ff9640 1 -- 172.21.15.64:0/2310160029 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f79b803b3f0 con 0x7f79c80717b0 2024-03-04T07:27:21.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.950+0000 7f79c4ff9640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79a806b4d0 0x7f79a806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:21.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.950+0000 7f79c4ff9640 1 -- 172.21.15.64:0/2310160029 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f79b802f030 con 0x7f79c80717b0 2024-03-04T07:27:21.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.950+0000 7f79bffff640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79a806b4d0 0x7f79a806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:21.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.950+0000 7f79bffff640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79a806b4d0 0x7f79a806d990 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f79c80cd0e0 tx=0x7f79b0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:21.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:21.954+0000 7f79c4ff9640 1 -- 172.21.15.64:0/2310160029 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f79b8040070 con 0x7f79c80717b0 2024-03-04T07:27:22.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.135+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f79c804f410 con 0x7f79c80717b0 2024-03-04T07:27:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.135+0000 7f79c4ff9640 1 -- 172.21.15.64:0/2310160029 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f79b807a630 con 0x7f79c80717b0 2024-03-04T07:27:22.135 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:22.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79a806b4d0 msgr2=0x7f79a806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79a806b4d0 0x7f79a806d990 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f79c80cd0e0 tx=0x7f79b0009190 comp rx=0 tx=0).stop 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79c80717b0 msgr2=0x7f79c8149c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79c80717b0 0x7f79c8149c10 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f79b8009890 tx=0x7f79b80046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 shutdown_connections 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79a806b4d0 0x7f79a806d990 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79c80b3060 0x7f79c81f6eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79c80b2b10 0x7f79c81f6970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 --2- 172.21.15.64:0/2310160029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79c80717b0 0x7f79c8149c10 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 >> 172.21.15.64:0/2310160029 conn(0x7f79c80d3d20 msgr2=0x7f79c8071b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 shutdown_connections 2024-03-04T07:27:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.137+0000 7f79cda55640 1 -- 172.21.15.64:0/2310160029 wait complete. 2024-03-04T07:27:22.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:22.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.250+0000 7f0bffb48640 1 Processor -- start 2024-03-04T07:27:22.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.250+0000 7f0bffb48640 1 -- start start 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bffb48640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf815e000 0x7f0bf8071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bffb48640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0bf80d35e0 0x7f0bf8071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bffb48640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0bf80ce970 con 0x7f0bf80b2c50 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bffb48640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0bf80cece0 con 0x7f0bf80d35e0 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bffb48640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0bf80cef30 con 0x7f0bf815e000 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfe0e5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0bf80d35e0 0x7f0bf8071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfd8e4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf815e000 0x7f0bf8071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfe0e5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0bf80d35e0 0x7f0bf8071a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40158/0 (socket says 172.21.15.64:40158) 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfe0e5640 1 -- 172.21.15.64:0/468736194 learned_addr learned my addr 172.21.15.64:0/468736194 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfd8e4640 1 -- 172.21.15.64:0/468736194 >> v2:172.21.15.64:3300/0 conn(0x7f0bf80d35e0 msgr2=0x7f0bf8071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfd8e4640 1 --2- 172.21.15.64:0/468736194 >> v2:172.21.15.64:3300/0 conn(0x7f0bf80d35e0 0x7f0bf8071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfd8e4640 1 -- 172.21.15.64:0/468736194 >> v1:172.21.15.66:6789/0 conn(0x7f0bf80b2c50 legacy=0x7f0bf8161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.251+0000 7f0bfd8e4640 1 -- 172.21.15.64:0/468736194 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bf80cf210 con 0x7f0bf815e000 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.252+0000 7f0bfd8e4640 1 --2- 172.21.15.64:0/468736194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf815e000 0x7f0bf8071540 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0bf40097a0 tx=0x7f0bf402b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ec7d787810edff67 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.252+0000 7f0bfc8e2640 1 -- 172.21.15.64:0/468736194 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf40315a0 con 0x7f0bf815e000 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.252+0000 7f0bfc8e2640 1 -- 172.21.15.64:0/468736194 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0bf4037070 con 0x7f0bf815e000 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.252+0000 7f0bfc8e2640 1 -- 172.21.15.64:0/468736194 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf4031960 con 0x7f0bf815e000 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 -- 172.21.15.64:0/468736194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf815e000 msgr2=0x7f0bf8071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 --2- 172.21.15.64:0/468736194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf815e000 0x7f0bf8071540 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0bf40097a0 tx=0x7f0bf402b840 comp rx=0 tx=0).stop 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 -- 172.21.15.64:0/468736194 shutdown_connections 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 --2- 172.21.15.64:0/468736194 >> v2:172.21.15.64:3300/0 conn(0x7f0bf80d35e0 0x7f0bf8071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 --2- 172.21.15.64:0/468736194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf815e000 0x7f0bf8071540 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 -- 172.21.15.64:0/468736194 >> 172.21.15.64:0/468736194 conn(0x7f0bf80d3c80 msgr2=0x7f0bf80d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 -- 172.21.15.64:0/468736194 shutdown_connections 2024-03-04T07:27:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.253+0000 7f0bffb48640 1 -- 172.21.15.64:0/468736194 wait complete. 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.254+0000 7f0bffb48640 1 Processor -- start 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.254+0000 7f0bffb48640 1 -- start start 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bffb48640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bf80b2c50 0x7f0bf80b35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bffb48640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf80d35e0 0x7f0bf8147ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bffb48640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bf815e000 0x7f0bf80ccb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bffb48640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bf80b3320 con 0x7f0bf815e000 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bffb48640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bf80b31a0 con 0x7f0bf80b2c50 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bffb48640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0bf815e3e0 con 0x7f0bf80d35e0 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bf815e000 0x7f0bf80ccb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bf815e000 0x7f0bf80ccb30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40160/0 (socket says 172.21.15.64:40160) 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 -- 172.21.15.64:0/817403058 learned_addr learned my addr 172.21.15.64:0/817403058 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:22.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfd8e4640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bf80b2c50 0x7f0bf80b35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfd0e3640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf80d35e0 0x7f0bf8147ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 -- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf80d35e0 msgr2=0x7f0bf8147ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf80d35e0 0x7f0bf8147ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 -- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bf80b2c50 msgr2=0x7f0bf80b35e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bf80b2c50 0x7f0bf80b35e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 -- 172.21.15.64:0/817403058 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bf80b3b20 con 0x7f0bf815e000 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bfe0e5640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bf815e000 0x7f0bf80ccb30 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f0bf000efc0 tx=0x7f0bf000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bee7fc640 1 -- 172.21.15.64:0/817403058 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0009e60 con 0x7f0bf815e000 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.255+0000 7f0bee7fc640 1 -- 172.21.15.64:0/817403058 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0bf000cd20 con 0x7f0bf815e000 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.256+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0bf80b3dc0 con 0x7f0bf815e000 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.256+0000 7f0bee7fc640 1 -- 172.21.15.64:0/817403058 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0003bd0 con 0x7f0bf815e000 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.256+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0bf80b4380 con 0x7f0bf815e000 2024-03-04T07:27:22.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.256+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0bf8067ff0 con 0x7f0bf815e000 2024-03-04T07:27:22.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.257+0000 7f0bee7fc640 1 -- 172.21.15.64:0/817403058 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0bf0012050 con 0x7f0bf815e000 2024-03-04T07:27:22.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.257+0000 7f0bee7fc640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bdc06b4d0 0x7f0bdc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.257+0000 7f0bfd8e4640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bdc06b4d0 0x7f0bdc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.257+0000 7f0bee7fc640 1 -- 172.21.15.64:0/817403058 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0bf008a520 con 0x7f0bf815e000 2024-03-04T07:27:22.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.258+0000 7f0bfd8e4640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bdc06b4d0 0x7f0bdc06d990 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0bf4036000 tx=0x7f0bf403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.262+0000 7f0bee7fc640 1 -- 172.21.15.64:0/817403058 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0bf0010040 con 0x7f0bf815e000 2024-03-04T07:27:22.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.459+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f0bf815d4c0 con 0x7f0bf815e000 2024-03-04T07:27:22.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.459+0000 7f0bee7fc640 1 -- 172.21.15.64:0/817403058 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v20) v1 ==== 80+0+0 (secure 0 0 0) 0x7f0bf0058d50 con 0x7f0bf815e000 2024-03-04T07:27:22.459 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:22.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bdc06b4d0 msgr2=0x7f0bdc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bdc06b4d0 0x7f0bdc06d990 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0bf4036000 tx=0x7f0bf403d040 comp rx=0 tx=0).stop 2024-03-04T07:27:22.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bf815e000 msgr2=0x7f0bf80ccb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bf815e000 0x7f0bf80ccb30 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f0bf000efc0 tx=0x7f0bf000c490 comp rx=0 tx=0).stop 2024-03-04T07:27:22.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 shutdown_connections 2024-03-04T07:27:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bdc06b4d0 0x7f0bdc06d990 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bf815e000 0x7f0bf80ccb30 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bf80d35e0 0x7f0bf8147ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 --2- 172.21.15.64:0/817403058 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bf80b2c50 0x7f0bf80b35e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 >> 172.21.15.64:0/817403058 conn(0x7f0bf80d3c80 msgr2=0x7f0bf815e770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 shutdown_connections 2024-03-04T07:27:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.461+0000 7f0bffb48640 1 -- 172.21.15.64:0/817403058 wait complete. 2024-03-04T07:27:22.471 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:27:22.472 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:22.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:22.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.628+0000 7feae6839640 1 Processor -- start 2024-03-04T07:27:22.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.628+0000 7feae6839640 1 -- start start 2024-03-04T07:27:22.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.628+0000 7feae6839640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feae015df70 0x7feae00d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.628+0000 7feae6839640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae00b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae6839640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feae00b6f20 con 0x7feae00d18c0 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae6839640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feae00d3030 con 0x7feae015df70 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae6839640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feae00d3310 con 0x7feae0071c50 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae4dd6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae00b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feadffff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feae015df70 0x7feae00d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feadf7fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7feae00d18c0 0x7feae0071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:41428/0 (socket says 172.21.15.64:41428) 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae4dd6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae00b2700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:43272/0 (socket says 172.21.15.64:43272) 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feadffff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feae015df70 0x7feae00d27b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37882/0 (socket says 172.21.15.64:37882) 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feadf7fe640 1 -- 172.21.15.64:0/1822072557 learned_addr learned my addr 172.21.15.64:0/1822072557 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feadeffd640 1 -- 172.21.15.64:0/1822072557 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2413142466 0 0) 0x7feae00b6f20 con 0x7feae00d18c0 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae4dd6640 1 -- 172.21.15.64:0/1822072557 >> v2:172.21.15.66:3300/0 conn(0x7feae015df70 msgr2=0x7feae00d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae4dd6640 1 --2- 172.21.15.64:0/1822072557 >> v2:172.21.15.66:3300/0 conn(0x7feae015df70 0x7feae00d27b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae4dd6640 1 -- 172.21.15.64:0/1822072557 >> v1:172.21.15.66:6789/0 conn(0x7feae00d18c0 legacy=0x7feae0071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feae4dd6640 1 -- 172.21.15.64:0/1822072557 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feae00d35f0 con 0x7feae0071c50 2024-03-04T07:27:22.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.629+0000 7feadffff640 1 --2- 172.21.15.64:0/1822072557 >> v2:172.21.15.66:3300/0 conn(0x7feae015df70 0x7feae00d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:22.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.630+0000 7feae4dd6640 1 --2- 172.21.15.64:0/1822072557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae00b2700 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fead8009d90 tx=0x7fead802bf30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a52def02c09814d8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.630+0000 7feadeffd640 1 -- 172.21.15.64:0/1822072557 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fead8039070 con 0x7feae0071c50 2024-03-04T07:27:22.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.630+0000 7feadeffd640 1 -- 172.21.15.64:0/1822072557 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fead802cce0 con 0x7feae0071c50 2024-03-04T07:27:22.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.630+0000 7feadeffd640 1 -- 172.21.15.64:0/1822072557 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fead8034660 con 0x7feae0071c50 2024-03-04T07:27:22.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.630+0000 7feae6839640 1 -- 172.21.15.64:0/1822072557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 msgr2=0x7feae00b2700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.630+0000 7feae6839640 1 --2- 172.21.15.64:0/1822072557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae00b2700 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fead8009d90 tx=0x7fead802bf30 comp rx=0 tx=0).stop 2024-03-04T07:27:22.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.631+0000 7feae6839640 1 -- 172.21.15.64:0/1822072557 shutdown_connections 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.631+0000 7feae6839640 1 --2- 172.21.15.64:0/1822072557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae00b2700 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.631+0000 7feae6839640 1 --2- 172.21.15.64:0/1822072557 >> v2:172.21.15.66:3300/0 conn(0x7feae015df70 0x7feae00d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.631+0000 7feae6839640 1 -- 172.21.15.64:0/1822072557 >> 172.21.15.64:0/1822072557 conn(0x7feae00d3cf0 msgr2=0x7feae00d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.631+0000 7feae6839640 1 -- 172.21.15.64:0/1822072557 shutdown_connections 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.631+0000 7feae6839640 1 -- 172.21.15.64:0/1822072557 wait complete. 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.632+0000 7feae6839640 1 Processor -- start 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.632+0000 7feae6839640 1 -- start start 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae6839640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae014bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae6839640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feae00d18c0 0x7feae00c96c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae6839640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae015df70 0x7feae00c9c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae6839640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feae015e350 con 0x7feae00d18c0 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae6839640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feae00d2cf0 con 0x7feae015df70 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae6839640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feae015e4d0 con 0x7feae0071c50 2024-03-04T07:27:22.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feae00d18c0 0x7feae00c96c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feae00d18c0 0x7feae00c96c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40168/0 (socket says 172.21.15.64:40168) 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 -- 172.21.15.64:0/4211389626 learned_addr learned my addr 172.21.15.64:0/4211389626 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae014bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae4dd6640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae015df70 0x7feae00c9c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 -- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 msgr2=0x7feae014bf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae014bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 -- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae015df70 msgr2=0x7feae00c9c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae015df70 0x7feae00c9c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 -- 172.21.15.64:0/4211389626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feae01f4a80 con 0x7feae00d18c0 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadffff640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae014bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadf7fe640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feae00d18c0 0x7feae00c96c0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fead400c4d0 tx=0x7fead400c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadcff9640 1 -- 172.21.15.64:0/4211389626 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fead4018070 con 0x7feae00d18c0 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadcff9640 1 -- 172.21.15.64:0/4211389626 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fead4007500 con 0x7feae00d18c0 2024-03-04T07:27:22.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feadcff9640 1 -- 172.21.15.64:0/4211389626 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fead4004a20 con 0x7feae00d18c0 2024-03-04T07:27:22.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.633+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feae00c8900 con 0x7feae00d18c0 2024-03-04T07:27:22.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.634+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feae00c8ec0 con 0x7feae00d18c0 2024-03-04T07:27:22.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.635+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feae0067ff0 con 0x7feae00d18c0 2024-03-04T07:27:22.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.635+0000 7feadcff9640 1 -- 172.21.15.64:0/4211389626 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fead401b070 con 0x7feae00d18c0 2024-03-04T07:27:22.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.635+0000 7feadcff9640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feacc06b500 0x7feacc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.636+0000 7feadcff9640 1 -- 172.21.15.64:0/4211389626 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fead40893d0 con 0x7feae00d18c0 2024-03-04T07:27:22.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.636+0000 7feadffff640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feacc06b500 0x7feacc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.636+0000 7feadffff640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feacc06b500 0x7feacc06d9c0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fead0009560 tx=0x7fead0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.640+0000 7feadcff9640 1 -- 172.21.15.64:0/4211389626 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fead4057c90 con 0x7feae00d18c0 2024-03-04T07:27:22.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.822+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7feae015fde0 con 0x7feae00d18c0 2024-03-04T07:27:22.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.823+0000 7feadcff9640 1 -- 172.21.15.64:0/4211389626 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7fead405bd30 con 0x7feae00d18c0 2024-03-04T07:27:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:22.823 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:22.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feacc06b500 msgr2=0x7feacc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feacc06b500 0x7feacc06d9c0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fead0009560 tx=0x7fead0009190 comp rx=0 tx=0).stop 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feae00d18c0 msgr2=0x7feae00c96c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feae00d18c0 0x7feae00c96c0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fead400c4d0 tx=0x7fead400c990 comp rx=0 tx=0).stop 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 shutdown_connections 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feacc06b500 0x7feacc06d9c0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae015df70 0x7feae00c9c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feae00d18c0 0x7feae00c96c0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 --2- 172.21.15.64:0/4211389626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae0071c50 0x7feae014bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 >> 172.21.15.64:0/4211389626 conn(0x7feae00d3cf0 msgr2=0x7feae0160300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 shutdown_connections 2024-03-04T07:27:22.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.826+0000 7feae6839640 1 -- 172.21.15.64:0/4211389626 wait complete. 2024-03-04T07:27:22.837 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:22.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:22.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.942+0000 7fe30e84e640 1 Processor -- start 2024-03-04T07:27:22.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.942+0000 7fe30e84e640 1 -- start start 2024-03-04T07:27:22.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30e84e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3081618e0 0x7fe30806b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30e84e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe3080ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30e84e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe308161d40 0x7fe3081621d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30e84e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe3080cfce0 con 0x7fe3081618e0 2024-03-04T07:27:22.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30e84e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe3080cee70 con 0x7fe308161d40 2024-03-04T07:27:22.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30e84e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe30815f730 con 0x7fe3080ce4f0 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe307fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3081618e0 0x7fe30806b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe307fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3081618e0 0x7fe30806b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40174/0 (socket says 172.21.15.64:40174) 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30cdeb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe308161d40 0x7fe3081621d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe307fff640 1 -- 172.21.15.64:0/4129811560 learned_addr learned my addr 172.21.15.64:0/4129811560 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe3077fe640 1 --2- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe3080ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30cdeb640 1 -- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 msgr2=0x7fe3080ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30cdeb640 1 --2- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe3080ce930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30cdeb640 1 -- 172.21.15.64:0/4129811560 >> v2:172.21.15.64:3300/0 conn(0x7fe3081618e0 msgr2=0x7fe30806b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30cdeb640 1 --2- 172.21.15.64:0/4129811560 >> v2:172.21.15.64:3300/0 conn(0x7fe3081618e0 0x7fe30806b130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe30cdeb640 1 -- 172.21.15.64:0/4129811560 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe30815ebb0 con 0x7fe308161d40 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe307fff640 1 --2- 172.21.15.64:0/4129811560 >> v2:172.21.15.64:3300/0 conn(0x7fe3081618e0 0x7fe30806b130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:22.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.943+0000 7fe3077fe640 1 --2- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe3080ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.944+0000 7fe30cdeb640 1 --2- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 0x7fe3081621d0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fe308067c90 tx=0x7fe30002beb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=340083284b445bd9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.944+0000 7fe306ffd640 1 -- 172.21.15.64:0/4129811560 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe300039070 con 0x7fe308161d40 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.944+0000 7fe306ffd640 1 -- 172.21.15.64:0/4129811560 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe30002cc60 con 0x7fe308161d40 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.944+0000 7fe306ffd640 1 -- 172.21.15.64:0/4129811560 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe300034660 con 0x7fe308161d40 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 -- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 msgr2=0x7fe3081621d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 --2- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 0x7fe3081621d0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fe308067c90 tx=0x7fe30002beb0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 -- 172.21.15.64:0/4129811560 shutdown_connections 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 --2- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 0x7fe3081621d0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 --2- 172.21.15.64:0/4129811560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe3080ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 --2- 172.21.15.64:0/4129811560 >> v2:172.21.15.64:3300/0 conn(0x7fe3081618e0 0x7fe30806b130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 -- 172.21.15.64:0/4129811560 >> 172.21.15.64:0/4129811560 conn(0x7fe3080b3660 msgr2=0x7fe3080b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:22.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 -- 172.21.15.64:0/4129811560 shutdown_connections 2024-03-04T07:27:22.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.945+0000 7fe30e84e640 1 -- 172.21.15.64:0/4129811560 wait complete. 2024-03-04T07:27:22.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.946+0000 7fe30e84e640 1 Processor -- start 2024-03-04T07:27:22.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.946+0000 7fe30e84e640 1 -- start start 2024-03-04T07:27:22.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30e84e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe30814efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30e84e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 0x7fe3080c7f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30e84e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3080c57a0 0x7fe3080b3a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30e84e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe30815f450 con 0x7fe3080c57a0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30e84e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe30815f170 con 0x7fe308161d40 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30e84e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe30806b670 con 0x7fe3080ce4f0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3080c57a0 0x7fe3080b3a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3080c57a0 0x7fe3080b3a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40178/0 (socket says 172.21.15.64:40178) 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 -- 172.21.15.64:0/807779221 learned_addr learned my addr 172.21.15.64:0/807779221 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe3077fe640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 0x7fe3080c7f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe307fff640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe30814efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 -- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 msgr2=0x7fe30814efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe30814efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 -- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 msgr2=0x7fe3080c7f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 0x7fe3080c7f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.947+0000 7fe30cdeb640 1 -- 172.21.15.64:0/807779221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3080c5cf0 con 0x7fe3080c57a0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.948+0000 7fe30cdeb640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3080c57a0 0x7fe3080b3a40 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fe300009e80 tx=0x7fe300004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.948+0000 7fe304ff9640 1 -- 172.21.15.64:0/807779221 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe300044020 con 0x7fe3080c57a0 2024-03-04T07:27:22.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.948+0000 7fe304ff9640 1 -- 172.21.15.64:0/807779221 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe30003f470 con 0x7fe3080c57a0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.948+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe3080b6c90 con 0x7fe3080c57a0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.948+0000 7fe304ff9640 1 -- 172.21.15.64:0/807779221 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe300039040 con 0x7fe3080c57a0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.948+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe3080b7250 con 0x7fe3080c57a0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.949+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe30809fc50 con 0x7fe3080c57a0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.949+0000 7fe304ff9640 1 -- 172.21.15.64:0/807779221 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe300047020 con 0x7fe3080c57a0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.950+0000 7fe304ff9640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe2f406b4d0 0x7fe2f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.950+0000 7fe304ff9640 1 -- 172.21.15.64:0/807779221 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe3000acb00 con 0x7fe3080c57a0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.950+0000 7fe307fff640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe2f406b4d0 0x7fe2f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.950+0000 7fe307fff640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe2f406b4d0 0x7fe2f406d990 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fe2f8006ca0 tx=0x7fe2f8009240 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:22.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:22.954+0000 7fe304ff9640 1 -- 172.21.15.64:0/807779221 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe30004e030 con 0x7fe3080c57a0 2024-03-04T07:27:23.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.134+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fe3080cbf00 con 0x7fe3080c57a0 2024-03-04T07:27:23.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.134+0000 7fe304ff9640 1 -- 172.21.15.64:0/807779221 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7fe30007b330 con 0x7fe3080c57a0 2024-03-04T07:27:23.134 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe2f406b4d0 msgr2=0x7fe2f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe2f406b4d0 0x7fe2f406d990 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fe2f8006ca0 tx=0x7fe2f8009240 comp rx=0 tx=0).stop 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3080c57a0 msgr2=0x7fe3080b3a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3080c57a0 0x7fe3080b3a40 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fe300009e80 tx=0x7fe300004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 shutdown_connections 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe2f406b4d0 0x7fe2f406d990 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3080c57a0 0x7fe3080b3a40 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe308161d40 0x7fe3080c7f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 --2- 172.21.15.64:0/807779221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3080ce4f0 0x7fe30814efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 >> 172.21.15.64:0/807779221 conn(0x7fe3080b3660 msgr2=0x7fe3080ccc90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:23.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 shutdown_connections 2024-03-04T07:27:23.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.136+0000 7fe30e84e640 1 -- 172.21.15.64:0/807779221 wait complete. 2024-03-04T07:27:23.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:23.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.251+0000 7f335e6b5640 1 Processor -- start 2024-03-04T07:27:23.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.251+0000 7f335e6b5640 1 -- start start 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f335e6b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f335815dbd0 0x7f335815ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f335e6b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f335815f3b0 con 0x7f33580b5840 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f335e6b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f335806f070 con 0x7f33580b40c0 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f335e6b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3358071940 con 0x7f335815dbd0 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f3357fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f335815dbd0 0x7f335815ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f3357fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f335815dbd0 0x7f335815ee70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40190/0 (socket says 172.21.15.64:40190) 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f3357fff640 1 -- 172.21.15.64:0/2524614194 learned_addr learned my addr 172.21.15.64:0/2524614194 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f33577fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f33580b5840 0x7f33580b39b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48474/0 (socket says 172.21.15.64:48474) 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f3357fff640 1 -- 172.21.15.64:0/2524614194 >> v1:172.21.15.66:6789/0 conn(0x7f33580b40c0 legacy=0x7f3358066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f3357fff640 1 -- 172.21.15.64:0/2524614194 >> v1:172.21.15.64:6789/0 conn(0x7f33580b5840 legacy=0x7f33580b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f3356ffd640 1 -- 172.21.15.64:0/2524614194 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1681603705 0 0) 0x7f335806f070 con 0x7f33580b40c0 2024-03-04T07:27:23.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.252+0000 7f3357fff640 1 -- 172.21.15.64:0/2524614194 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33580b5db0 con 0x7f335815dbd0 2024-03-04T07:27:23.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.253+0000 7f3357fff640 1 --2- 172.21.15.64:0/2524614194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f335815dbd0 0x7f335815ee70 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f335806f240 tx=0x7f334c02ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7e0c953d600365d1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.253+0000 7f3356ffd640 1 -- 172.21.15.64:0/2524614194 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f334c02f840 con 0x7f335815dbd0 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.253+0000 7f3356ffd640 1 -- 172.21.15.64:0/2524614194 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f334c02f9e0 con 0x7f335815dbd0 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.253+0000 7f3356ffd640 1 -- 172.21.15.64:0/2524614194 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f334c02fdb0 con 0x7f335815dbd0 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.254+0000 7f335e6b5640 1 -- 172.21.15.64:0/2524614194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f335815dbd0 msgr2=0x7f335815ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.254+0000 7f335e6b5640 1 --2- 172.21.15.64:0/2524614194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f335815dbd0 0x7f335815ee70 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f335806f240 tx=0x7f334c02ba20 comp rx=0 tx=0).stop 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.254+0000 7f335e6b5640 1 -- 172.21.15.64:0/2524614194 shutdown_connections 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.254+0000 7f335e6b5640 1 --2- 172.21.15.64:0/2524614194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f335815dbd0 0x7f335815ee70 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.254+0000 7f335e6b5640 1 -- 172.21.15.64:0/2524614194 >> 172.21.15.64:0/2524614194 conn(0x7f33580b26b0 msgr2=0x7f33580b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.254+0000 7f335e6b5640 1 -- 172.21.15.64:0/2524614194 shutdown_connections 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.254+0000 7f335e6b5640 1 -- 172.21.15.64:0/2524614194 wait complete. 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.255+0000 7f335e6b5640 1 Processor -- start 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.255+0000 7f335e6b5640 1 -- start start 2024-03-04T07:27:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f335e6b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f33580b40c0 0x7f3358156380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f335e6b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33580b5840 0x7f33581568c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f335e6b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f335815dbd0 0x7f33580ae1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f335e6b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33580721c0 con 0x7f33580b40c0 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f335e6b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3358071ee0 con 0x7f33580b5840 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f335e6b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f33580726e0 con 0x7f335815dbd0 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f33580b40c0 0x7f3358156380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f33580b40c0 0x7f3358156380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40202/0 (socket says 172.21.15.64:40202) 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 -- 172.21.15.64:0/1974548727 learned_addr learned my addr 172.21.15.64:0/1974548727 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f33577fe640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33580b5840 0x7f33581568c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f335cc52640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f335815dbd0 0x7f33580ae1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 -- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f335815dbd0 msgr2=0x7f33580ae1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f335815dbd0 0x7f33580ae1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 -- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33580b5840 msgr2=0x7f33581568c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33580b5840 0x7f33581568c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 -- 172.21.15.64:0/1974548727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3358156ef0 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3357fff640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f33580b40c0 0x7f3358156380 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f334c0090b0 tx=0x7f334c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3354ff9640 1 -- 172.21.15.64:0/1974548727 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f334c047070 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.256+0000 7f3354ff9640 1 -- 172.21.15.64:0/1974548727 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f334c037cc0 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.257+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f33581571d0 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.257+0000 7f3354ff9640 1 -- 172.21.15.64:0/1974548727 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f334c004270 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.257+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3358157790 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.258+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3358073380 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.258+0000 7f3354ff9640 1 -- 172.21.15.64:0/1974548727 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f334c04a070 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.259+0000 7f3354ff9640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f334406b500 0x7f334406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.259+0000 7f3354ff9640 1 -- 172.21.15.64:0/1974548727 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f334c0ac370 con 0x7f33580b40c0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.259+0000 7f33577fe640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f334406b500 0x7f334406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.259+0000 7f33577fe640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f334406b500 0x7f334406d9c0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f33581562f0 tx=0x7f3348007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.263+0000 7f3354ff9640 1 -- 172.21.15.64:0/1974548727 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f334c033030 con 0x7f33580b40c0 2024-03-04T07:27:23.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.442+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f335804f410 con 0x7f33580b40c0 2024-03-04T07:27:23.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.442+0000 7f3354ff9640 1 -- 172.21.15.64:0/1974548727 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f334c07ac30 con 0x7f33580b40c0 2024-03-04T07:27:23.442 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f334406b500 msgr2=0x7f334406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f334406b500 0x7f334406d9c0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f33581562f0 tx=0x7f3348007e80 comp rx=0 tx=0).stop 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f33580b40c0 msgr2=0x7f3358156380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f33580b40c0 0x7f3358156380 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f334c0090b0 tx=0x7f334c004730 comp rx=0 tx=0).stop 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 shutdown_connections 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f334406b500 0x7f334406d9c0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f335815dbd0 0x7f33580ae1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33580b5840 0x7f33581568c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 --2- 172.21.15.64:0/1974548727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f33580b40c0 0x7f3358156380 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 >> 172.21.15.64:0/1974548727 conn(0x7f33580b26b0 msgr2=0x7f335815e9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:23.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 shutdown_connections 2024-03-04T07:27:23.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.444+0000 7f335e6b5640 1 -- 172.21.15.64:0/1974548727 wait complete. 2024-03-04T07:27:23.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:23.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.560+0000 7f624921c640 1 Processor -- start 2024-03-04T07:27:23.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.560+0000 7f624921c640 1 -- start start 2024-03-04T07:27:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624921c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f624415f5c0 0x7f62440cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624921c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62440cff40 0x7f62440d03d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624921c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440b18a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624921c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62440b0af0 con 0x7f62440cff40 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624921c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62440b33b0 con 0x7f624415f5c0 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624921c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62440b0d40 con 0x7f62440b1470 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62440cff40 0x7f62440d03d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62440cff40 0x7f62440d03d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51344/0 (socket says 172.21.15.64:51344) 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 -- 172.21.15.64:0/1201217574 learned_addr learned my addr 172.21.15.64:0/1201217574 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f6242d9d640 1 --2- 172.21.15.64:0/1201217574 >> v2:172.21.15.66:3300/0 conn(0x7f624415f5c0 0x7f62440cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624359e640 1 --2- 172.21.15.64:0/1201217574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440b18a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 -- 172.21.15.64:0/1201217574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 msgr2=0x7f62440b18a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 --2- 172.21.15.64:0/1201217574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440b18a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 -- 172.21.15.64:0/1201217574 >> v2:172.21.15.66:3300/0 conn(0x7f624415f5c0 msgr2=0x7f62440cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 --2- 172.21.15.64:0/1201217574 >> v2:172.21.15.66:3300/0 conn(0x7f624415f5c0 0x7f62440cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.561+0000 7f624259c640 1 -- 172.21.15.64:0/1201217574 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62440b1020 con 0x7f62440cff40 2024-03-04T07:27:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.562+0000 7f624259c640 1 --2- 172.21.15.64:0/1201217574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62440cff40 0x7f62440d03d0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f6244071e70 tx=0x7f623802b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a7d5ac6e5b02db24 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.562+0000 7f6241d9b640 1 -- 172.21.15.64:0/1201217574 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f623802e070 con 0x7f62440cff40 2024-03-04T07:27:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.562+0000 7f6241d9b640 1 -- 172.21.15.64:0/1201217574 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6238037070 con 0x7f62440cff40 2024-03-04T07:27:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.562+0000 7f6241d9b640 1 -- 172.21.15.64:0/1201217574 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6238031a00 con 0x7f62440cff40 2024-03-04T07:27:23.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.562+0000 7f624921c640 1 -- 172.21.15.64:0/1201217574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62440cff40 msgr2=0x7f62440d03d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.562+0000 7f624921c640 1 --2- 172.21.15.64:0/1201217574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62440cff40 0x7f62440d03d0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f6244071e70 tx=0x7f623802b880 comp rx=0 tx=0).stop 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.563+0000 7f624921c640 1 -- 172.21.15.64:0/1201217574 shutdown_connections 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.563+0000 7f624921c640 1 --2- 172.21.15.64:0/1201217574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440b18a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.563+0000 7f624921c640 1 --2- 172.21.15.64:0/1201217574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62440cff40 0x7f62440d03d0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.563+0000 7f624921c640 1 --2- 172.21.15.64:0/1201217574 >> v2:172.21.15.66:3300/0 conn(0x7f624415f5c0 0x7f62440cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.563+0000 7f624921c640 1 -- 172.21.15.64:0/1201217574 >> 172.21.15.64:0/1201217574 conn(0x7f62440b36d0 msgr2=0x7f62440b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.563+0000 7f624921c640 1 -- 172.21.15.64:0/1201217574 shutdown_connections 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.563+0000 7f624921c640 1 -- 172.21.15.64:0/1201217574 wait complete. 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.564+0000 7f624921c640 1 Processor -- start 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.564+0000 7f624921c640 1 -- start start 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624921c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440c81c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624921c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62440cff40 0x7f624409c480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624921c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f624415f5c0 0x7f624409c9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624921c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f624415f9a0 con 0x7f624415f5c0 2024-03-04T07:27:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624921c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62440ced80 con 0x7f62440cff40 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624921c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62440cf2a0 con 0x7f62440b1470 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624359e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f624415f5c0 0x7f624409c9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624359e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f624415f5c0 0x7f624409c9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51356/0 (socket says 172.21.15.64:51356) 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624359e640 1 -- 172.21.15.64:0/2650691747 learned_addr learned my addr 172.21.15.64:0/2650691747 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f6242d9d640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440c81c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624259c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62440cff40 0x7f624409c480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f6242d9d640 1 -- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62440cff40 msgr2=0x7f624409c480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f6242d9d640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62440cff40 0x7f624409c480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f6242d9d640 1 -- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f624415f5c0 msgr2=0x7f624409c9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f6242d9d640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f624415f5c0 0x7f624409c9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f6242d9d640 1 -- 172.21.15.64:0/2650691747 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f624409f3f0 con 0x7f62440b1470 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f624359e640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f624415f5c0 0x7f624409c9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.565+0000 7f6242d9d640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440c81c0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f623c00e870 tx=0x7f623c00ed30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.566+0000 7f622b7fe640 1 -- 172.21.15.64:0/2650691747 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f623c017070 con 0x7f62440b1470 2024-03-04T07:27:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.566+0000 7f622b7fe640 1 -- 172.21.15.64:0/2650691747 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f623c00ee60 con 0x7f62440b1470 2024-03-04T07:27:23.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.566+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f624409f6d0 con 0x7f62440b1470 2024-03-04T07:27:23.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.566+0000 7f622b7fe640 1 -- 172.21.15.64:0/2650691747 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f623c00cb60 con 0x7f62440b1470 2024-03-04T07:27:23.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.566+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62441f3070 con 0x7f62440b1470 2024-03-04T07:27:23.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.567+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f624409fd30 con 0x7f62440b1470 2024-03-04T07:27:23.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.567+0000 7f622b7fe640 1 -- 172.21.15.64:0/2650691747 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f623c01a070 con 0x7f62440b1470 2024-03-04T07:27:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.567+0000 7f622b7fe640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f623006b500 0x7f623006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.568+0000 7f622b7fe640 1 -- 172.21.15.64:0/2650691747 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f623c08a360 con 0x7f62440b1470 2024-03-04T07:27:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.568+0000 7f624259c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f623006b500 0x7f623006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.568+0000 7f624259c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f623006b500 0x7f623006d9c0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f6238036000 tx=0x7f623803d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.572+0000 7f622b7fe640 1 -- 172.21.15.64:0/2650691747 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f623c058b00 con 0x7f62440b1470 2024-03-04T07:27:23.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.752+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f624404f410 con 0x7f62440b1470 2024-03-04T07:27:23.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.753+0000 7f622b7fe640 1 -- 172.21.15.64:0/2650691747 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f623c05cba0 con 0x7f62440b1470 2024-03-04T07:27:23.753 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f623006b500 msgr2=0x7f623006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f623006b500 0x7f623006d9c0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f6238036000 tx=0x7f623803d040 comp rx=0 tx=0).stop 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 msgr2=0x7f62440c81c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440c81c0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f623c00e870 tx=0x7f623c00ed30 comp rx=0 tx=0).stop 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 shutdown_connections 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f623006b500 0x7f623006d9c0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f624415f5c0 0x7f624409c9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62440cff40 0x7f624409c480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 --2- 172.21.15.64:0/2650691747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62440b1470 0x7f62440c81c0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 >> 172.21.15.64:0/2650691747 conn(0x7f62440b36d0 msgr2=0x7f624415c310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 shutdown_connections 2024-03-04T07:27:23.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.755+0000 7f624921c640 1 -- 172.21.15.64:0/2650691747 wait complete. 2024-03-04T07:27:23.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:23.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f549138a640 1 Processor -- start 2024-03-04T07:27:23.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f549138a640 1 -- start start 2024-03-04T07:27:23.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f549138a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c071c50 0x7f548c0d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f549138a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f548c0b2c50 0x7f548c0d2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f549138a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f548c0ce970 con 0x7f548c15e000 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f549138a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f548c0cece0 con 0x7f548c0b2c50 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f549138a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f548c0cef30 con 0x7f548c071c50 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.869+0000 7f548affd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f548c15e000 0x7f548c071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48554/0 (socket says 172.21.15.64:48554) 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548affd640 1 -- 172.21.15.64:0/2946010184 learned_addr learned my addr 172.21.15.64:0/2946010184 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548b7fe640 1 --2- 172.21.15.64:0/2946010184 >> v2:172.21.15.66:3300/0 conn(0x7f548c0b2c50 0x7f548c0d2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548a7fc640 1 --2- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c071c50 0x7f548c0d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f5489ffb640 1 -- 172.21.15.64:0/2946010184 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2332699035 0 0) 0x7f548c0ce970 con 0x7f548c15e000 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f5489ffb640 1 -- 172.21.15.64:0/2946010184 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5478003520 con 0x7f548c15e000 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548b7fe640 1 -- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c071c50 msgr2=0x7f548c0d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548b7fe640 1 --2- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c071c50 0x7f548c0d25d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548b7fe640 1 -- 172.21.15.64:0/2946010184 >> v1:172.21.15.64:6789/0 conn(0x7f548c15e000 legacy=0x7f548c071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548b7fe640 1 -- 172.21.15.64:0/2946010184 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f548c0cf210 con 0x7f548c0b2c50 2024-03-04T07:27:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.870+0000 7f548a7fc640 1 --2- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c071c50 0x7f548c0d25d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f548b7fe640 1 --2- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 0x7f548c0d2b10 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f547c009d90 tx=0x7f547c02be10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=10e1ce506dd927bd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f5489ffb640 1 -- 172.21.15.64:0/2946010184 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f547c039070 con 0x7f548c0b2c50 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f5489ffb640 1 -- 172.21.15.64:0/2946010184 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f547c02cbc0 con 0x7f548c0b2c50 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f5489ffb640 1 -- 172.21.15.64:0/2946010184 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f547c034440 con 0x7f548c0b2c50 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 -- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 msgr2=0x7f548c0d2b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 --2- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 0x7f548c0d2b10 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f547c009d90 tx=0x7f547c02be10 comp rx=0 tx=0).stop 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 -- 172.21.15.64:0/2946010184 shutdown_connections 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 --2- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 0x7f548c0d2b10 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 --2- 172.21.15.64:0/2946010184 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c071c50 0x7f548c0d25d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 -- 172.21.15.64:0/2946010184 >> 172.21.15.64:0/2946010184 conn(0x7f548c0d3c80 msgr2=0x7f548c0d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 -- 172.21.15.64:0/2946010184 shutdown_connections 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.871+0000 7f549138a640 1 -- 172.21.15.64:0/2946010184 wait complete. 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 Processor -- start 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 -- start start 2024-03-04T07:27:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f548c071c50 0x7f548c14f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 0x7f548c0c9910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c15e000 0x7f548c0c9e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f548c0b3320 con 0x7f548c071c50 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f548c0b31a0 con 0x7f548c0b2c50 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f549138a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f548c15e3e0 con 0x7f548c15e000 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f548affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f548c071c50 0x7f548c14f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f548affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f548c071c50 0x7f548c14f0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51366/0 (socket says 172.21.15.64:51366) 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.873+0000 7f548affd640 1 -- 172.21.15.64:0/1115606197 learned_addr learned my addr 172.21.15.64:0/1115606197 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548b7fe640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c15e000 0x7f548c0c9e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548a7fc640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 0x7f548c0c9910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548affd640 1 -- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c15e000 msgr2=0x7f548c0c9e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548affd640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c15e000 0x7f548c0c9e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548affd640 1 -- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 msgr2=0x7f548c0c9910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548affd640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 0x7f548c0c9910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548affd640 1 -- 172.21.15.64:0/1115606197 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f548c0c8f90 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f548affd640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f548c071c50 0x7f548c14f0b0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f548400c390 tx=0x7f548400c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f54837fe640 1 -- 172.21.15.64:0/1115606197 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5484017070 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f54837fe640 1 -- 172.21.15.64:0/1115606197 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5484005ce0 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f548c0c8410 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f54837fe640 1 -- 172.21.15.64:0/1115606197 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54840072f0 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.874+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f548c0c89d0 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.875+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f548c067ff0 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.876+0000 7f54837fe640 1 -- 172.21.15.64:0/1115606197 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f548401a070 con 0x7f548c071c50 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.876+0000 7f54837fe640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f546c06b4d0 0x7f546c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:23.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.876+0000 7f548a7fc640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f546c06b4d0 0x7f546c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:23.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.876+0000 7f54837fe640 1 -- 172.21.15.64:0/1115606197 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f54840894c0 con 0x7f548c071c50 2024-03-04T07:27:23.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.877+0000 7f548a7fc640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f546c06b4d0 0x7f546c06d990 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f5474009560 tx=0x7f5474009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:23.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:23.881+0000 7f54837fe640 1 -- 172.21.15.64:0/1115606197 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5484057d80 con 0x7f548c071c50 2024-03-04T07:27:24.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.074+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f548c0d1c30 con 0x7f548c071c50 2024-03-04T07:27:24.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.074+0000 7f54837fe640 1 -- 172.21.15.64:0/1115606197 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v20) v1 ==== 80+0+0 (secure 0 0 0) 0x7f548405e0d0 con 0x7f548c071c50 2024-03-04T07:27:24.074 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f546c06b4d0 msgr2=0x7f546c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f546c06b4d0 0x7f546c06d990 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f5474009560 tx=0x7f5474009190 comp rx=0 tx=0).stop 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f548c071c50 msgr2=0x7f548c14f0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f548c071c50 0x7f548c14f0b0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f548400c390 tx=0x7f548400c850 comp rx=0 tx=0).stop 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 shutdown_connections 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f546c06b4d0 0x7f546c06d990 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f548c15e000 0x7f548c0c9e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f548c0b2c50 0x7f548c0c9910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 --2- 172.21.15.64:0/1115606197 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f548c071c50 0x7f548c14f0b0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 >> 172.21.15.64:0/1115606197 conn(0x7f548c0d3c80 msgr2=0x7f548c15f9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:24.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 shutdown_connections 2024-03-04T07:27:24.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.077+0000 7f549138a640 1 -- 172.21.15.64:0/1115606197 wait complete. 2024-03-04T07:27:24.087 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:27:24.087 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:24.087 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:24.114 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:24.114 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:27:24.114 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:24.114 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:27:24.192 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:27:24.192 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:24.209 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:27:24.209 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:24.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:24.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.367+0000 7f2e22488640 1 Processor -- start 2024-03-04T07:27:24.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.367+0000 7f2e22488640 1 -- start start 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e22488640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c15e000 0x7f2e1c071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e22488640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e1c0d35e0 0x7f2e1c071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e22488640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2e1c0ce970 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e22488640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2e1c0cece0 con 0x7f2e1c0d35e0 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e22488640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e1c0cef30 con 0x7f2e1c15e000 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e20a25640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e1c0d35e0 0x7f2e1c071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e20a25640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2e1c0d35e0 0x7f2e1c071a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51376/0 (socket says 172.21.15.64:51376) 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e20a25640 1 -- 172.21.15.64:0/4236227834 learned_addr learned my addr 172.21.15.64:0/4236227834 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e1bfff640 1 --2- 172.21.15.64:0/4236227834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c15e000 0x7f2e1c071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e20a25640 1 -- 172.21.15.64:0/4236227834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c15e000 msgr2=0x7f2e1c071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:24.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e20a25640 1 --2- 172.21.15.64:0/4236227834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c15e000 0x7f2e1c071540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e20a25640 1 -- 172.21.15.64:0/4236227834 >> v1:172.21.15.66:6789/0 conn(0x7f2e1c0b2c50 legacy=0x7f2e1c161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.368+0000 7f2e20a25640 1 -- 172.21.15.64:0/4236227834 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e1c0cf210 con 0x7f2e1c0d35e0 2024-03-04T07:27:24.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.369+0000 7f2e20a25640 1 --2- 172.21.15.64:0/4236227834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0d35e0 0x7f2e1c071a80 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f2e14009e50 tx=0x7f2e1402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b9e4f1d463f070cc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.369+0000 7f2e1affd640 1 -- 172.21.15.64:0/4236227834 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1402ca70 con 0x7f2e1c0d35e0 2024-03-04T07:27:24.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.369+0000 7f2e1affd640 1 -- 172.21.15.64:0/4236227834 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e1402cc10 con 0x7f2e1c0d35e0 2024-03-04T07:27:24.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.369+0000 7f2e1affd640 1 -- 172.21.15.64:0/4236227834 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e140343f0 con 0x7f2e1c0d35e0 2024-03-04T07:27:24.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 -- 172.21.15.64:0/4236227834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0d35e0 msgr2=0x7f2e1c071a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 --2- 172.21.15.64:0/4236227834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0d35e0 0x7f2e1c071a80 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f2e14009e50 tx=0x7f2e1402bed0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 -- 172.21.15.64:0/4236227834 shutdown_connections 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 --2- 172.21.15.64:0/4236227834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0d35e0 0x7f2e1c071a80 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 --2- 172.21.15.64:0/4236227834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c15e000 0x7f2e1c071540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 -- 172.21.15.64:0/4236227834 >> 172.21.15.64:0/4236227834 conn(0x7f2e1c0d3c80 msgr2=0x7f2e1c0d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 -- 172.21.15.64:0/4236227834 shutdown_connections 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.370+0000 7f2e22488640 1 -- 172.21.15.64:0/4236227834 wait complete. 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.371+0000 7f2e22488640 1 Processor -- start 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.371+0000 7f2e22488640 1 -- start start 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e22488640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0b2c50 0x7f2e1c1f3fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e22488640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c0d35e0 0x7f2e1c154b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e22488640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1c15e000 0x7f2e1c159590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e22488640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e1c0b3320 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e22488640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e1c0b31a0 con 0x7f2e1c15e000 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e22488640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2e1c15e3e0 con 0x7f2e1c0d35e0 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0b2c50 0x7f2e1c1f3fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0b2c50 0x7f2e1c1f3fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51378/0 (socket says 172.21.15.64:51378) 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 -- 172.21.15.64:0/144005747 learned_addr learned my addr 172.21.15.64:0/144005747 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e20a25640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1c15e000 0x7f2e1c159590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1b7fe640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c0d35e0 0x7f2e1c154b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 -- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c0d35e0 msgr2=0x7f2e1c154b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c0d35e0 0x7f2e1c154b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 -- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1c15e000 msgr2=0x7f2e1c159590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1c15e000 0x7f2e1c159590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 -- 172.21.15.64:0/144005747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e1c1f4500 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.372+0000 7f2e1bfff640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0b2c50 0x7f2e1c1f3fc0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f2e1000e9e0 tx=0x7f2e1000eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.373+0000 7f2e18ff9640 1 -- 172.21.15.64:0/144005747 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e10017070 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.373+0000 7f2e18ff9640 1 -- 172.21.15.64:0/144005747 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2e1000c9d0 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.373+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2e1c155060 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.373+0000 7f2e18ff9640 1 -- 172.21.15.64:0/144005747 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2e1000cdc0 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.373+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2e1c1f4a80 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.374+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2e1c067ff0 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.374+0000 7f2e18ff9640 1 -- 172.21.15.64:0/144005747 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2e1001a070 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.374+0000 7f2e18ff9640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e0806b4d0 0x7f2e0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.375+0000 7f2e18ff9640 1 -- 172.21.15.64:0/144005747 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2e1008a1e0 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.375+0000 7f2e1b7fe640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e0806b4d0 0x7f2e0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.375+0000 7f2e1b7fe640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e0806b4d0 0x7f2e0806d990 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f2e1c160610 tx=0x7f2e0c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.379+0000 7f2e18ff9640 1 -- 172.21.15.64:0/144005747 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2e10058a10 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.560+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2e1c0cdd80 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.561+0000 7f2e18ff9640 1 -- 172.21.15.64:0/144005747 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f2e1005cab0 con 0x7f2e1c0b2c50 2024-03-04T07:27:24.561 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:24.561 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:24.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e0806b4d0 msgr2=0x7f2e0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e0806b4d0 0x7f2e0806d990 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f2e1c160610 tx=0x7f2e0c009040 comp rx=0 tx=0).stop 2024-03-04T07:27:24.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0b2c50 msgr2=0x7f2e1c1f3fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0b2c50 0x7f2e1c1f3fc0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f2e1000e9e0 tx=0x7f2e1000eea0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 shutdown_connections 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2e0806b4d0 0x7f2e0806d990 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2e1c15e000 0x7f2e1c159590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2e1c0d35e0 0x7f2e1c154b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 --2- 172.21.15.64:0/144005747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2e1c0b2c50 0x7f2e1c1f3fc0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.563+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 >> 172.21.15.64:0/144005747 conn(0x7f2e1c0d3c80 msgr2=0x7f2e1c1607a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.564+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 shutdown_connections 2024-03-04T07:27:24.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.564+0000 7f2e22488640 1 -- 172.21.15.64:0/144005747 wait complete. 2024-03-04T07:27:24.574 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:24.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:24.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.679+0000 7f3cfaecf640 1 Processor -- start 2024-03-04T07:27:24.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.679+0000 7f3cfaecf640 1 -- start start 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cfaecf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3cf40ce4f0 0x7f3cf40ce960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cfaecf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3cf40d0150 0x7f3cf40d05a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cfaecf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3cf415f730 con 0x7f3cf41618e0 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cfaecf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3cf40cfce0 con 0x7f3cf40d0150 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cfaecf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3cf40ceea0 con 0x7f3cf40ce4f0 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf946c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3cf40d0150 0x7f3cf40d05a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf946c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3cf40d0150 0x7f3cf40d05a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51392/0 (socket says 172.21.15.64:51392) 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf946c640 1 -- 172.21.15.64:0/4103279869 learned_addr learned my addr 172.21.15.64:0/4103279869 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf3fff640 1 --2- 172.21.15.64:0/4103279869 >> v2:172.21.15.66:3300/0 conn(0x7f3cf40ce4f0 0x7f3cf40ce960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf946c640 1 -- 172.21.15.64:0/4103279869 >> v2:172.21.15.66:3300/0 conn(0x7f3cf40ce4f0 msgr2=0x7f3cf40ce960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:24.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf946c640 1 --2- 172.21.15.64:0/4103279869 >> v2:172.21.15.66:3300/0 conn(0x7f3cf40ce4f0 0x7f3cf40ce960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf946c640 1 -- 172.21.15.64:0/4103279869 >> v1:172.21.15.64:6789/0 conn(0x7f3cf41618e0 legacy=0x7f3cf406b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.680+0000 7f3cf946c640 1 -- 172.21.15.64:0/4103279869 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3cf415ebb0 con 0x7f3cf40d0150 2024-03-04T07:27:24.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.681+0000 7f3cf946c640 1 --2- 172.21.15.64:0/4103279869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40d05a0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f3cf4071ab0 tx=0x7f3cec02c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1f769bfaca1ae97b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.681+0000 7f3cf37fe640 1 -- 172.21.15.64:0/4103279869 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cec02cbb0 con 0x7f3cf40d0150 2024-03-04T07:27:24.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.681+0000 7f3cf37fe640 1 -- 172.21.15.64:0/4103279869 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3cec02cd50 con 0x7f3cf40d0150 2024-03-04T07:27:24.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.681+0000 7f3cf37fe640 1 -- 172.21.15.64:0/4103279869 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3cec034610 con 0x7f3cf40d0150 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 -- 172.21.15.64:0/4103279869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 msgr2=0x7f3cf40d05a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/4103279869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40d05a0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f3cf4071ab0 tx=0x7f3cec02c010 comp rx=0 tx=0).stop 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 -- 172.21.15.64:0/4103279869 shutdown_connections 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/4103279869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40d05a0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/4103279869 >> v2:172.21.15.66:3300/0 conn(0x7f3cf40ce4f0 0x7f3cf40ce960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 -- 172.21.15.64:0/4103279869 >> 172.21.15.64:0/4103279869 conn(0x7f3cf40b3660 msgr2=0x7f3cf40b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 -- 172.21.15.64:0/4103279869 shutdown_connections 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.682+0000 7f3cfaecf640 1 -- 172.21.15.64:0/4103279869 wait complete. 2024-03-04T07:27:24.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.683+0000 7f3cfaecf640 1 Processor -- start 2024-03-04T07:27:24.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.683+0000 7f3cfaecf640 1 -- start start 2024-03-04T07:27:24.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cfaecf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3cf40ce4f0 0x7f3cf41f41e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cfaecf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40cc8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cfaecf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3cf40ccdf0 0x7f3cf40cadf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cfaecf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3cf415f450 con 0x7f3cf40d0150 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cfaecf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3cf415f170 con 0x7f3cf40ccdf0 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cfaecf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3cf415e7d0 con 0x7f3cf40ce4f0 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40cc8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40cc8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51402/0 (socket says 172.21.15.64:51402) 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 -- 172.21.15.64:0/344628583 learned_addr learned my addr 172.21.15.64:0/344628583 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf946c640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3cf40ccdf0 0x7f3cf40cadf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf8c6b640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3cf40ce4f0 0x7f3cf41f41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 -- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3cf40ce4f0 msgr2=0x7f3cf41f41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3cf40ce4f0 0x7f3cf41f41e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 -- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3cf40ccdf0 msgr2=0x7f3cf40cadf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3cf40ccdf0 0x7f3cf40cadf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.684+0000 7f3cf3fff640 1 -- 172.21.15.64:0/344628583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3cf41f4720 con 0x7f3cf40d0150 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.685+0000 7f3cf3fff640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40cc8b0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f3ce000e870 tx=0x7f3ce000ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.685+0000 7f3cf17fa640 1 -- 172.21.15.64:0/344628583 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ce0017070 con 0x7f3cf40d0150 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.685+0000 7f3cf17fa640 1 -- 172.21.15.64:0/344628583 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ce000c8e0 con 0x7f3cf40d0150 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.685+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3cf4152b00 con 0x7f3cf40d0150 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.685+0000 7f3cf17fa640 1 -- 172.21.15.64:0/344628583 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ce000ccd0 con 0x7f3cf40d0150 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.685+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3cf40c7ce0 con 0x7f3cf40d0150 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.686+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3cf4074980 con 0x7f3cf40d0150 2024-03-04T07:27:24.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.686+0000 7f3cf17fa640 1 -- 172.21.15.64:0/344628583 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3ce001a070 con 0x7f3cf40d0150 2024-03-04T07:27:24.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.687+0000 7f3cf17fa640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cd806b4d0 0x7f3cd806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.687+0000 7f3cf17fa640 1 -- 172.21.15.64:0/344628583 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3ce0089f80 con 0x7f3cf40d0150 2024-03-04T07:27:24.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.687+0000 7f3cf8c6b640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cd806b4d0 0x7f3cd806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.687+0000 7f3cf8c6b640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cd806b4d0 0x7f3cd806d990 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f3ce8001780 tx=0x7f3ce8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.691+0000 7f3cf17fa640 1 -- 172.21.15.64:0/344628583 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3cf4074980 con 0x7f3cf40d0150 2024-03-04T07:27:24.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.870+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f3cf409fc50 con 0x7f3cf40d0150 2024-03-04T07:27:24.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.871+0000 7f3cf17fa640 1 -- 172.21.15.64:0/344628583 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f3ce00587b0 con 0x7f3cf40d0150 2024-03-04T07:27:24.870 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.872+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cd806b4d0 msgr2=0x7f3cd806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.872+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cd806b4d0 0x7f3cd806d990 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f3ce8001780 tx=0x7f3ce8009040 comp rx=0 tx=0).stop 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.872+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 msgr2=0x7f3cf40cc8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.872+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40cc8b0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f3ce000e870 tx=0x7f3ce000ed30 comp rx=0 tx=0).stop 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 shutdown_connections 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cd806b4d0 0x7f3cd806d990 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3cf40ccdf0 0x7f3cf40cadf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3cf40d0150 0x7f3cf40cc8b0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 --2- 172.21.15.64:0/344628583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3cf40ce4f0 0x7f3cf41f41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 >> 172.21.15.64:0/344628583 conn(0x7f3cf40b3660 msgr2=0x7f3cf40b3a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 shutdown_connections 2024-03-04T07:27:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.873+0000 7f3cfaecf640 1 -- 172.21.15.64:0/344628583 wait complete. 2024-03-04T07:27:24.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:24.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.987+0000 7f661e4f4640 1 Processor -- start 2024-03-04T07:27:24.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.987+0000 7f661e4f4640 1 -- start start 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f661e4f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180c6e40 0x7f6618166b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f661e4f4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66180cf120 con 0x7f66180cf740 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f661e4f4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66180cd310 con 0x7f661815f500 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f661e4f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f661815f8e0 con 0x7f66180c6e40 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f66177fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f66180cf740 0x7f66180cea10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48560/0 (socket says 172.21.15.64:48560) 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f66177fe640 1 -- 172.21.15.64:0/762484485 learned_addr learned my addr 172.21.15.64:0/762484485 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f661ca91640 1 --2- 172.21.15.64:0/762484485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180c6e40 0x7f6618166b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.988+0000 7f6616ffd640 1 -- 172.21.15.64:0/762484485 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 254428779 0 0) 0x7f66180cf120 con 0x7f66180cf740 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.989+0000 7f6616ffd640 1 -- 172.21.15.64:0/762484485 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6600003520 con 0x7f66180cf740 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.989+0000 7f6616ffd640 1 -- 172.21.15.64:0/762484485 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2951319055 0 0) 0x7f66180cd310 con 0x7f661815f500 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.989+0000 7f6616ffd640 1 -- 172.21.15.64:0/762484485 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f66180cf120 con 0x7f661815f500 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.989+0000 7f661ca91640 1 -- 172.21.15.64:0/762484485 >> v1:172.21.15.66:6789/0 conn(0x7f661815f500 legacy=0x7f66180ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.989+0000 7f661ca91640 1 -- 172.21.15.64:0/762484485 >> v1:172.21.15.64:6789/0 conn(0x7f66180cf740 legacy=0x7f66180cea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.989+0000 7f661ca91640 1 -- 172.21.15.64:0/762484485 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66180cd5b0 con 0x7f66180c6e40 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.989+0000 7f661ca91640 1 --2- 172.21.15.64:0/762484485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180c6e40 0x7f6618166b30 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f6610009dc0 tx=0x7f661002be90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c4241f02ae5297e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f6616ffd640 1 -- 172.21.15.64:0/762484485 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6610039070 con 0x7f66180c6e40 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f6616ffd640 1 -- 172.21.15.64:0/762484485 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f661002cc40 con 0x7f66180c6e40 2024-03-04T07:27:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f6616ffd640 1 -- 172.21.15.64:0/762484485 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6610034440 con 0x7f66180c6e40 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f661e4f4640 1 -- 172.21.15.64:0/762484485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180c6e40 msgr2=0x7f6618166b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f661e4f4640 1 --2- 172.21.15.64:0/762484485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180c6e40 0x7f6618166b30 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f6610009dc0 tx=0x7f661002be90 comp rx=0 tx=0).stop 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f661e4f4640 1 -- 172.21.15.64:0/762484485 shutdown_connections 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f661e4f4640 1 --2- 172.21.15.64:0/762484485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180c6e40 0x7f6618166b30 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f661e4f4640 1 -- 172.21.15.64:0/762484485 >> 172.21.15.64:0/762484485 conn(0x7f66180b3700 msgr2=0x7f66180b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f661e4f4640 1 -- 172.21.15.64:0/762484485 shutdown_connections 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.990+0000 7f661e4f4640 1 -- 172.21.15.64:0/762484485 wait complete. 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.991+0000 7f661e4f4640 1 Processor -- start 2024-03-04T07:27:24.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.991+0000 7f661e4f4640 1 -- start start 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f661e4f4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66180c6e40 0x7f6618151f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f661e4f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180cf740 0x7f66180c9b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f661e4f4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f661815f500 0x7f66180ca060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f661e4f4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66180c59a0 con 0x7f66180c6e40 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f661e4f4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66180c56c0 con 0x7f661815f500 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f661e4f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66180b43d0 con 0x7f66180cf740 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f6617fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66180c6e40 0x7f6618151f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f6617fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66180c6e40 0x7f6618151f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51416/0 (socket says 172.21.15.64:51416) 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f6617fff640 1 -- 172.21.15.64:0/1974139828 learned_addr learned my addr 172.21.15.64:0/1974139828 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f661ca91640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f661815f500 0x7f66180ca060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.992+0000 7f66177fe640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180cf740 0x7f66180c9b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6617fff640 1 -- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180cf740 msgr2=0x7f66180c9b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6617fff640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180cf740 0x7f66180c9b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6617fff640 1 -- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f661815f500 msgr2=0x7f66180ca060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6617fff640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f661815f500 0x7f66180ca060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6617fff640 1 -- 172.21.15.64:0/1974139828 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66180c8f20 con 0x7f66180c6e40 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f661ca91640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f661815f500 0x7f66180ca060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6617fff640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66180c6e40 0x7f6618151f80 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f660800c3c0 tx=0x7f660800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6614ff9640 1 -- 172.21.15.64:0/1974139828 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6608017070 con 0x7f66180c6e40 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6614ff9640 1 -- 172.21.15.64:0/1974139828 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66080052d0 con 0x7f66180c6e40 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66180ca6d0 con 0x7f66180c6e40 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f6614ff9640 1 -- 172.21.15.64:0/1974139828 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6608003810 con 0x7f66180c6e40 2024-03-04T07:27:24.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.993+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66180b4ad0 con 0x7f66180c6e40 2024-03-04T07:27:24.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.994+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f661809fcf0 con 0x7f66180c6e40 2024-03-04T07:27:24.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.994+0000 7f6614ff9640 1 -- 172.21.15.64:0/1974139828 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f660801a070 con 0x7f66180c6e40 2024-03-04T07:27:24.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.995+0000 7f6614ff9640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f660406b500 0x7f660406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:24.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.995+0000 7f6614ff9640 1 -- 172.21.15.64:0/1974139828 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f66080893b0 con 0x7f66180c6e40 2024-03-04T07:27:24.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.995+0000 7f66177fe640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f660406b500 0x7f660406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:24.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.995+0000 7f66177fe640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f660406b500 0x7f660406d9c0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f6618070bf0 tx=0x7f660c007d30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:24.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:24.999+0000 7f6614ff9640 1 -- 172.21.15.64:0/1974139828 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6608057c70 con 0x7f66180c6e40 2024-03-04T07:27:25.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.182+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f661815ead0 con 0x7f66180c6e40 2024-03-04T07:27:25.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.182+0000 7f6614ff9640 1 -- 172.21.15.64:0/1974139828 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f660805bd10 con 0x7f66180c6e40 2024-03-04T07:27:25.182 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:25.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f660406b500 msgr2=0x7f660406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f660406b500 0x7f660406d9c0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f6618070bf0 tx=0x7f660c007d30 comp rx=0 tx=0).stop 2024-03-04T07:27:25.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66180c6e40 msgr2=0x7f6618151f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66180c6e40 0x7f6618151f80 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f660800c3c0 tx=0x7f660800c880 comp rx=0 tx=0).stop 2024-03-04T07:27:25.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 shutdown_connections 2024-03-04T07:27:25.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f660406b500 0x7f660406d9c0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f661815f500 0x7f66180ca060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66180cf740 0x7f66180c9b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 --2- 172.21.15.64:0/1974139828 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66180c6e40 0x7f6618151f80 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 >> 172.21.15.64:0/1974139828 conn(0x7f66180b3700 msgr2=0x7f66180ce940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:25.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 shutdown_connections 2024-03-04T07:27:25.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.184+0000 7f661e4f4640 1 -- 172.21.15.64:0/1974139828 wait complete. 2024-03-04T07:27:25.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:25.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.299+0000 7ff459edf640 1 Processor -- start 2024-03-04T07:27:25.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.299+0000 7ff459edf640 1 -- start start 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff459edf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff45415dd50 0x7ff4540ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff459edf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4540d2760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff459edf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff454072cf0 con 0x7ff4540cef20 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff459edf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff454072ab0 con 0x7ff45415dd50 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff459edf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff454072fd0 con 0x7ff454072680 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff4537fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff45415dd50 0x7ff4540ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff4537fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff45415dd50 0x7ff4540ce9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51420/0 (socket says 172.21.15.64:51420) 2024-03-04T07:27:25.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff4537fe640 1 -- 172.21.15.64:0/1890302472 learned_addr learned my addr 172.21.15.64:0/1890302472 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff453fff640 1 --2- 172.21.15.64:0/1890302472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4540d2760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff4537fe640 1 -- 172.21.15.64:0/1890302472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 msgr2=0x7ff4540d2760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff4537fe640 1 --2- 172.21.15.64:0/1890302472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4540d2760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff4537fe640 1 -- 172.21.15.64:0/1890302472 >> v1:172.21.15.66:6789/0 conn(0x7ff4540cef20 legacy=0x7ff454071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.300+0000 7ff4537fe640 1 -- 172.21.15.64:0/1890302472 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4540d1f60 con 0x7ff45415dd50 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.301+0000 7ff4537fe640 1 --2- 172.21.15.64:0/1890302472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff4540ce9e0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7ff444009870 tx=0x7ff44402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a97b49df148c541 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.301+0000 7ff4527fc640 1 -- 172.21.15.64:0/1890302472 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff444035020 con 0x7ff45415dd50 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.301+0000 7ff4527fc640 1 -- 172.21.15.64:0/1890302472 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff444031640 con 0x7ff45415dd50 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.301+0000 7ff4527fc640 1 -- 172.21.15.64:0/1890302472 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff44402d040 con 0x7ff45415dd50 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 -- 172.21.15.64:0/1890302472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 msgr2=0x7ff4540ce9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 --2- 172.21.15.64:0/1890302472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff4540ce9e0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7ff444009870 tx=0x7ff44402b8f0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 -- 172.21.15.64:0/1890302472 shutdown_connections 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 --2- 172.21.15.64:0/1890302472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4540d2760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 --2- 172.21.15.64:0/1890302472 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff4540ce9e0 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 -- 172.21.15.64:0/1890302472 >> 172.21.15.64:0/1890302472 conn(0x7ff4540d3ef0 msgr2=0x7ff4540d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 -- 172.21.15.64:0/1890302472 shutdown_connections 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.302+0000 7ff459edf640 1 -- 172.21.15.64:0/1890302472 wait complete. 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.303+0000 7ff459edf640 1 Processor -- start 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.303+0000 7ff459edf640 1 -- start start 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff459edf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4541e6a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff459edf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4540cef20 0x7ff454160870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff459edf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff454148b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff459edf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff45415e2b0 con 0x7ff45415dd50 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff459edf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff45415e130 con 0x7ff4540cef20 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff459edf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4540d2ca0 con 0x7ff454072680 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff454148b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff454148b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51432/0 (socket says 172.21.15.64:51432) 2024-03-04T07:27:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 -- 172.21.15.64:0/2541333356 learned_addr learned my addr 172.21.15.64:0/2541333356 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff4537fe640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4541e6a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff452ffd640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4540cef20 0x7ff454160870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 -- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 msgr2=0x7ff4541e6a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4541e6a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 -- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4540cef20 msgr2=0x7ff454160870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4540cef20 0x7ff454160870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.304+0000 7ff453fff640 1 -- 172.21.15.64:0/2541333356 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4541e6f70 con 0x7ff45415dd50 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.305+0000 7ff453fff640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff454148b60 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7ff44c00efc0 tx=0x7ff44c00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.305+0000 7ff43bfff640 1 -- 172.21.15.64:0/2541333356 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff44c018020 con 0x7ff45415dd50 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.305+0000 7ff43bfff640 1 -- 172.21.15.64:0/2541333356 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff44c014480 con 0x7ff45415dd50 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.305+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4541f7d60 con 0x7ff45415dd50 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.305+0000 7ff43bfff640 1 -- 172.21.15.64:0/2541333356 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff44c014870 con 0x7ff45415dd50 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.305+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4541f8600 con 0x7ff45415dd50 2024-03-04T07:27:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.306+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff45406cdd0 con 0x7ff45415dd50 2024-03-04T07:27:25.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.306+0000 7ff43bfff640 1 -- 172.21.15.64:0/2541333356 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff44c005d10 con 0x7ff45415dd50 2024-03-04T07:27:25.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.306+0000 7ff43bfff640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44006b4d0 0x7ff44006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.307+0000 7ff43bfff640 1 -- 172.21.15.64:0/2541333356 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff44c08ab80 con 0x7ff45415dd50 2024-03-04T07:27:25.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.307+0000 7ff4537fe640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44006b4d0 0x7ff44006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.307+0000 7ff4537fe640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44006b4d0 0x7ff44006d990 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7ff444036000 tx=0x7ff44403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.311+0000 7ff43bfff640 1 -- 172.21.15.64:0/2541333356 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff44c022070 con 0x7ff45415dd50 2024-03-04T07:27:25.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.496+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7ff45404f410 con 0x7ff45415dd50 2024-03-04T07:27:25.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.496+0000 7ff43bfff640 1 -- 172.21.15.64:0/2541333356 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7ff4541e6f70 con 0x7ff45415dd50 2024-03-04T07:27:25.496 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44006b4d0 msgr2=0x7ff44006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44006b4d0 0x7ff44006d990 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7ff444036000 tx=0x7ff44403c040 comp rx=0 tx=0).stop 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 msgr2=0x7ff454148b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff454148b60 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7ff44c00efc0 tx=0x7ff44c00c490 comp rx=0 tx=0).stop 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 shutdown_connections 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff44006b4d0 0x7ff44006d990 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff45415dd50 0x7ff454148b60 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4540cef20 0x7ff454160870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 --2- 172.21.15.64:0/2541333356 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff454072680 0x7ff4541e6a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 >> 172.21.15.64:0/2541333356 conn(0x7ff4540d3ef0 msgr2=0x7ff4540c92e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 shutdown_connections 2024-03-04T07:27:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.498+0000 7ff459edf640 1 -- 172.21.15.64:0/2541333356 wait complete. 2024-03-04T07:27:25.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:25.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 Processor -- start 2024-03-04T07:27:25.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 -- start start 2024-03-04T07:27:25.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8188158520 0x7f818806d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f818806dd60 0x7f81880ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f81880aec40 0x7f818815e670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f818806e1c0 con 0x7f818806dd60 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8188071b70 con 0x7f81880aec40 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.615+0000 7f818ed56640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81880af080 con 0x7f8188158520 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f818806dd60 0x7f81880ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f818806dd60 0x7f81880ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51442/0 (socket says 172.21.15.64:51442) 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 -- 172.21.15.64:0/441779691 learned_addr learned my addr 172.21.15.64:0/441779691 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f818d2f3640 1 --2- 172.21.15.64:0/441779691 >> v2:172.21.15.66:3300/0 conn(0x7f81880aec40 0x7f818815e670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f818caf2640 1 --2- 172.21.15.64:0/441779691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8188158520 0x7f818806d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 -- 172.21.15.64:0/441779691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8188158520 msgr2=0x7f818806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 --2- 172.21.15.64:0/441779691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8188158520 0x7f818806d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 -- 172.21.15.64:0/441779691 >> v2:172.21.15.66:3300/0 conn(0x7f81880aec40 msgr2=0x7f818815e670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 --2- 172.21.15.64:0/441779691 >> v2:172.21.15.66:3300/0 conn(0x7f81880aec40 0x7f818815e670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.616+0000 7f8187fff640 1 -- 172.21.15.64:0/441779691 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8188071240 con 0x7f818806dd60 2024-03-04T07:27:25.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f8187fff640 1 --2- 172.21.15.64:0/441779691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818806dd60 0x7f81880ae700 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f8188072c70 tx=0x7f817402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ca30d76ef05f72f9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f81877fe640 1 -- 172.21.15.64:0/441779691 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8174035020 con 0x7f818806dd60 2024-03-04T07:27:25.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f81877fe640 1 -- 172.21.15.64:0/441779691 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8174031870 con 0x7f818806dd60 2024-03-04T07:27:25.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f81877fe640 1 -- 172.21.15.64:0/441779691 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8174031c40 con 0x7f818806dd60 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 -- 172.21.15.64:0/441779691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818806dd60 msgr2=0x7f81880ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 --2- 172.21.15.64:0/441779691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818806dd60 0x7f81880ae700 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f8188072c70 tx=0x7f817402b910 comp rx=0 tx=0).stop 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 -- 172.21.15.64:0/441779691 shutdown_connections 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 --2- 172.21.15.64:0/441779691 >> v2:172.21.15.66:3300/0 conn(0x7f81880aec40 0x7f818815e670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 --2- 172.21.15.64:0/441779691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818806dd60 0x7f81880ae700 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 --2- 172.21.15.64:0/441779691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8188158520 0x7f818806d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 -- 172.21.15.64:0/441779691 >> 172.21.15.64:0/441779691 conn(0x7f8188066d00 msgr2=0x7f81880670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 -- 172.21.15.64:0/441779691 shutdown_connections 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.617+0000 7f818ed56640 1 -- 172.21.15.64:0/441779691 wait complete. 2024-03-04T07:27:25.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.618+0000 7f818ed56640 1 Processor -- start 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.619+0000 7f818ed56640 1 -- start start 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.619+0000 7f818ed56640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f818806dd60 0x7f8188149470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.619+0000 7f818ed56640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81880aec40 0x7f81880cf660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.619+0000 7f818ed56640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8188158520 0x7f818815d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.619+0000 7f818ed56640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8188072000 con 0x7f8188158520 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.619+0000 7f818ed56640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8188071800 con 0x7f818806dd60 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.619+0000 7f818ed56640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8188158900 con 0x7f81880aec40 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8188158520 0x7f818815d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8188158520 0x7f818815d850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51450/0 (socket says 172.21.15.64:51450) 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818caf2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f818806dd60 0x7f8188149470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 -- 172.21.15.64:0/1836174913 learned_addr learned my addr 172.21.15.64:0/1836174913 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f8187fff640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81880aec40 0x7f81880cf660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 -- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81880aec40 msgr2=0x7f81880cf660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81880aec40 0x7f81880cf660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 -- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f818806dd60 msgr2=0x7f8188149470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f818806dd60 0x7f8188149470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 -- 172.21.15.64:0/1836174913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81880cfba0 con 0x7f8188158520 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f8187fff640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81880aec40 0x7f81880cf660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818caf2640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f818806dd60 0x7f8188149470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818d2f3640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8188158520 0x7f818815d850 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f818000efc0 tx=0x7f818000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f81857fa640 1 -- 172.21.15.64:0/1836174913 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8180018020 con 0x7f8188158520 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f81857fa640 1 -- 172.21.15.64:0/1836174913 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8180014480 con 0x7f8188158520 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f818815c4d0 con 0x7f8188158520 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f81857fa640 1 -- 172.21.15.64:0/1836174913 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8180014870 con 0x7f8188158520 2024-03-04T07:27:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.620+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f818815ca90 con 0x7f8188158520 2024-03-04T07:27:25.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.621+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8188067ff0 con 0x7f8188158520 2024-03-04T07:27:25.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.622+0000 7f81857fa640 1 -- 172.21.15.64:0/1836174913 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8180005d10 con 0x7f8188158520 2024-03-04T07:27:25.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.622+0000 7f81857fa640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816c06b4d0 0x7f816c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.622+0000 7f818caf2640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816c06b4d0 0x7f816c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.622+0000 7f81857fa640 1 -- 172.21.15.64:0/1836174913 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f818008aaf0 con 0x7f8188158520 2024-03-04T07:27:25.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.623+0000 7f818caf2640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816c06b4d0 0x7f816c06d990 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f817c009e10 tx=0x7f817c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.626+0000 7f81857fa640 1 -- 172.21.15.64:0/1836174913 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8180022070 con 0x7f8188158520 2024-03-04T07:27:25.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.821+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f81880b4b20 con 0x7f8188158520 2024-03-04T07:27:25.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.822+0000 7f81857fa640 1 -- 172.21.15.64:0/1836174913 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v20) v1 ==== 80+0+0 (secure 0 0 0) 0x7f8180010040 con 0x7f8188158520 2024-03-04T07:27:25.821 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816c06b4d0 msgr2=0x7f816c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816c06b4d0 0x7f816c06d990 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f817c009e10 tx=0x7f817c009190 comp rx=0 tx=0).stop 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8188158520 msgr2=0x7f818815d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8188158520 0x7f818815d850 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f818000efc0 tx=0x7f818000c490 comp rx=0 tx=0).stop 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 shutdown_connections 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816c06b4d0 0x7f816c06d990 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8188158520 0x7f818815d850 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81880aec40 0x7f81880cf660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 --2- 172.21.15.64:0/1836174913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f818806dd60 0x7f8188149470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 >> 172.21.15.64:0/1836174913 conn(0x7f8188066d00 msgr2=0x7f8188159fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 shutdown_connections 2024-03-04T07:27:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.824+0000 7f818ed56640 1 -- 172.21.15.64:0/1836174913 wait complete. 2024-03-04T07:27:25.834 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:27:25.834 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:25.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:25.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.953+0000 7f4f65678640 1 Processor -- start 2024-03-04T07:27:25.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.953+0000 7f4f65678640 1 -- start start 2024-03-04T07:27:25.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.953+0000 7f4f65678640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f6006fe20 0x7f4f600cea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.953+0000 7f4f65678640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4f600d0880 0x7f4f600ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.953+0000 7f4f65678640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4f600d0dd0 con 0x7f4f6015e060 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.953+0000 7f4f65678640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f600d10b0 con 0x7f4f600d0880 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.953+0000 7f4f65678640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f600cc0e0 con 0x7f4f6006fe20 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5f7fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4f600d0880 0x7f4f600ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5e7fc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f6006fe20 0x7f4f600cea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5effd640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f4f6015e060 0x7f4f600ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:40174/0 (socket says 172.21.15.64:40174) 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5effd640 1 -- 172.21.15.64:0/2363095372 learned_addr learned my addr 172.21.15.64:0/2363095372 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5e7fc640 1 -- 172.21.15.64:0/2363095372 >> v2:172.21.15.66:3300/0 conn(0x7f4f600d0880 msgr2=0x7f4f600ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5dffb640 1 -- 172.21.15.64:0/2363095372 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4112034383 0 0) 0x7f4f600d0dd0 con 0x7f4f6015e060 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5e7fc640 1 --2- 172.21.15.64:0/2363095372 >> v2:172.21.15.66:3300/0 conn(0x7f4f600d0880 0x7f4f600ccbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5e7fc640 1 -- 172.21.15.64:0/2363095372 >> v1:172.21.15.66:6789/0 conn(0x7f4f6015e060 legacy=0x7f4f600ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5e7fc640 1 -- 172.21.15.64:0/2363095372 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f600cc3c0 con 0x7f4f6006fe20 2024-03-04T07:27:25.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.954+0000 7f4f5f7fe640 1 --2- 172.21.15.64:0/2363095372 >> v2:172.21.15.66:3300/0 conn(0x7f4f600d0880 0x7f4f600ccbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f5e7fc640 1 --2- 172.21.15.64:0/2363095372 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f6006fe20 0x7f4f600cea20 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4f540097a0 tx=0x7f4f5402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1252425ce3bd1a54 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f5dffb640 1 -- 172.21.15.64:0/2363095372 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f54035020 con 0x7f4f6006fe20 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f5dffb640 1 -- 172.21.15.64:0/2363095372 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f54031830 con 0x7f4f6006fe20 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f5dffb640 1 -- 172.21.15.64:0/2363095372 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f54031c00 con 0x7f4f6006fe20 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 -- 172.21.15.64:0/2363095372 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f6006fe20 msgr2=0x7f4f600cea20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 --2- 172.21.15.64:0/2363095372 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f6006fe20 0x7f4f600cea20 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4f540097a0 tx=0x7f4f5402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 -- 172.21.15.64:0/2363095372 shutdown_connections 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 --2- 172.21.15.64:0/2363095372 >> v2:172.21.15.66:3300/0 conn(0x7f4f600d0880 0x7f4f600ccbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 --2- 172.21.15.64:0/2363095372 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f6006fe20 0x7f4f600cea20 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 -- 172.21.15.64:0/2363095372 >> 172.21.15.64:0/2363095372 conn(0x7f4f600d1a90 msgr2=0x7f4f600d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 -- 172.21.15.64:0/2363095372 shutdown_connections 2024-03-04T07:27:25.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.955+0000 7f4f65678640 1 -- 172.21.15.64:0/2363095372 wait complete. 2024-03-04T07:27:25.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 Processor -- start 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 -- start start 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f6006fe20 0x7f4f600b2a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f600d0880 0x7f4f60158310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f6015e060 0x7f4f6006c0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f600cef60 con 0x7f4f6015e060 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f6015e440 con 0x7f4f6006fe20 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.957+0000 7f4f65678640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f600cf0e0 con 0x7f4f600d0880 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f6006fe20 0x7f4f600b2a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5e7fc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f600d0880 0x7f4f60158310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f6015e060 0x7f4f6006c0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f6006fe20 0x7f4f600b2a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:56126/0 (socket says 172.21.15.64:56126) 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 -- 172.21.15.64:0/2182676333 learned_addr learned my addr 172.21.15.64:0/2182676333 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 -- 172.21.15.64:0/2182676333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f600d0880 msgr2=0x7f4f60158310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f600d0880 0x7f4f60158310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 -- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f6015e060 msgr2=0x7f4f6006c0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f6015e060 0x7f4f6006c0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 -- 172.21.15.64:0/2182676333 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f600b22b0 con 0x7f4f6006fe20 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f5effd640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f6006fe20 0x7f4f600b2a90 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f4f5000c4d0 tx=0x7f4f5000c8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f477fe640 1 -- 172.21.15.64:0/2182676333 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f500074e0 con 0x7f4f6006fe20 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f477fe640 1 -- 172.21.15.64:0/2182676333 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f500047e0 con 0x7f4f6006fe20 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f600cace0 con 0x7f4f6006fe20 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f477fe640 1 -- 172.21.15.64:0/2182676333 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f50004bd0 con 0x7f4f6006fe20 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.958+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f600b0540 con 0x7f4f6006fe20 2024-03-04T07:27:25.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.959+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f60067ff0 con 0x7f4f6006fe20 2024-03-04T07:27:25.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.960+0000 7f4f477fe640 1 -- 172.21.15.64:0/2182676333 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4f50011070 con 0x7f4f6006fe20 2024-03-04T07:27:25.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.960+0000 7f4f477fe640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f4c06b4d0 0x7f4f4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:25.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.960+0000 7f4f477fe640 1 -- 172.21.15.64:0/2182676333 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4f50089d40 con 0x7f4f6006fe20 2024-03-04T07:27:25.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.960+0000 7f4f5e7fc640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f4c06b4d0 0x7f4f4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:25.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.961+0000 7f4f5e7fc640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f4c06b4d0 0x7f4f4c06d990 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f4f54036000 tx=0x7f4f5403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:25.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:25.965+0000 7f4f477fe640 1 -- 172.21.15.64:0/2182676333 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4f50058bd0 con 0x7f4f6006fe20 2024-03-04T07:27:26.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.147+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f4f600cd260 con 0x7f4f6006fe20 2024-03-04T07:27:26.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.148+0000 7f4f477fe640 1 -- 172.21.15.64:0/2182676333 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f4f500584d0 con 0x7f4f6006fe20 2024-03-04T07:27:26.148 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:26.148 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:26.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f4c06b4d0 msgr2=0x7f4f4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f4c06b4d0 0x7f4f4c06d990 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f4f54036000 tx=0x7f4f5403c040 comp rx=0 tx=0).stop 2024-03-04T07:27:26.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f6006fe20 msgr2=0x7f4f600b2a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f6006fe20 0x7f4f600b2a90 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f4f5000c4d0 tx=0x7f4f5000c8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 shutdown_connections 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f4c06b4d0 0x7f4f4c06d990 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f6015e060 0x7f4f6006c0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f600d0880 0x7f4f60158310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 --2- 172.21.15.64:0/2182676333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f6006fe20 0x7f4f600b2a90 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 >> 172.21.15.64:0/2182676333 conn(0x7f4f600d1a90 msgr2=0x7f4f600d12f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 shutdown_connections 2024-03-04T07:27:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.150+0000 7f4f65678640 1 -- 172.21.15.64:0/2182676333 wait complete. 2024-03-04T07:27:26.161 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:26.162 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:26.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.266+0000 7fa8518d4640 1 Processor -- start 2024-03-04T07:27:26.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.266+0000 7fa8518d4640 1 -- start start 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa8518d4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa84c0cf740 0x7fa84c0cea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa8518d4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa84c0cd310 con 0x7fa84c0c5e30 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa8518d4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa84c0cd5b0 con 0x7fa84c15f500 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa8518d4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa84c15f8e0 con 0x7fa84c0cf740 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa84a7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa84c0cf740 0x7fa84c0cea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa84a7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa84c0cf740 0x7fa84c0cea10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51456/0 (socket says 172.21.15.64:51456) 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2643916751 learned_addr learned my addr 172.21.15.64:0/2643916751 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:26.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.267+0000 7fa84affd640 1 --1- 172.21.15.64:0/2643916751 >> v1:172.21.15.66:6789/0 conn(0x7fa84c15f500 0x7fa84c0ce300 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:40182/0 (socket says 172.21.15.64:40182) 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2643916751 >> v1:172.21.15.66:6789/0 conn(0x7fa84c15f500 legacy=0x7fa84c0ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2643916751 >> v1:172.21.15.64:6789/0 conn(0x7fa84c0c5e30 legacy=0x7fa84c0c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2643916751 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa84c0cd890 con 0x7fa84c0cf740 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa849ffb640 1 -- 172.21.15.64:0/2643916751 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2254451304 0 0) 0x7fa84c0cd5b0 con 0x7fa84c15f500 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa84a7fc640 1 --2- 172.21.15.64:0/2643916751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c0cea10 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fa834009870 tx=0x7fa83402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7c323f019cc7cfd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa849ffb640 1 -- 172.21.15.64:0/2643916751 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa834034070 con 0x7fa84c0cf740 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa849ffb640 1 -- 172.21.15.64:0/2643916751 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa834030860 con 0x7fa84c0cf740 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.268+0000 7fa849ffb640 1 -- 172.21.15.64:0/2643916751 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa834030c30 con 0x7fa84c0cf740 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.269+0000 7fa8518d4640 1 -- 172.21.15.64:0/2643916751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 msgr2=0x7fa84c0cea10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.269+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2643916751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c0cea10 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fa834009870 tx=0x7fa83402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.269+0000 7fa8518d4640 1 -- 172.21.15.64:0/2643916751 shutdown_connections 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.269+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2643916751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c0cea10 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.269+0000 7fa8518d4640 1 -- 172.21.15.64:0/2643916751 >> 172.21.15.64:0/2643916751 conn(0x7fa84c0b3700 msgr2=0x7fa84c0b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:26.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.269+0000 7fa8518d4640 1 -- 172.21.15.64:0/2643916751 shutdown_connections 2024-03-04T07:27:26.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.269+0000 7fa8518d4640 1 -- 172.21.15.64:0/2643916751 wait complete. 2024-03-04T07:27:26.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.270+0000 7fa8518d4640 1 Processor -- start 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.270+0000 7fa8518d4640 1 -- start start 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa8518d4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa84c0c5e30 0x7fa84c1f85a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa8518d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c06c0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa8518d4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa84c15f500 0x7fa84c1f7a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa8518d4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa84c0c59a0 con 0x7fa84c0cf740 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa8518d4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa84c0c56c0 con 0x7fa84c0c5e30 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa8518d4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa84c0cef50 con 0x7fa84c15f500 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c06c0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c06c0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51462/0 (socket says 172.21.15.64:51462) 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2640083063 learned_addr learned my addr 172.21.15.64:0/2640083063 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84affd640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa84c0c5e30 0x7fa84c1f85a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84b7fe640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa84c15f500 0x7fa84c1f7a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa84c15f500 msgr2=0x7fa84c1f7a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa84c15f500 0x7fa84c1f7a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa84c0c5e30 msgr2=0x7fa84c1f85a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa84c0c5e30 0x7fa84c1f85a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.271+0000 7fa84a7fc640 1 -- 172.21.15.64:0/2640083063 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa84c0c8ce0 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.272+0000 7fa84a7fc640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c06c0a0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fa834009870 tx=0x7fa834004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.272+0000 7fa843fff640 1 -- 172.21.15.64:0/2640083063 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa834048070 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.272+0000 7fa843fff640 1 -- 172.21.15.64:0/2640083063 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa834039d90 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.272+0000 7fa843fff640 1 -- 172.21.15.64:0/2640083063 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa834034040 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.272+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa84c0c8fc0 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.272+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa84c06c5e0 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.273+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa84c09fcf0 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.273+0000 7fa843fff640 1 -- 172.21.15.64:0/2640083063 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa83404b070 con 0x7fa84c0cf740 2024-03-04T07:27:26.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.274+0000 7fa843fff640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa82c06b4d0 0x7fa82c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.274+0000 7fa843fff640 1 -- 172.21.15.64:0/2640083063 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa8340ac270 con 0x7fa84c0cf740 2024-03-04T07:27:26.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.274+0000 7fa84affd640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa82c06b4d0 0x7fa82c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.274+0000 7fa84affd640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa82c06b4d0 0x7fa82c06d990 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fa844000f60 tx=0x7fa844007e40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.278+0000 7fa843fff640 1 -- 172.21.15.64:0/2640083063 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa83407aaa0 con 0x7fa84c0cf740 2024-03-04T07:27:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.460+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fa84c04f410 con 0x7fa84c0cf740 2024-03-04T07:27:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.461+0000 7fa843fff640 1 -- 172.21.15.64:0/2640083063 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7fa83407eb40 con 0x7fa84c0cf740 2024-03-04T07:27:26.460 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa82c06b4d0 msgr2=0x7fa82c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa82c06b4d0 0x7fa82c06d990 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fa844000f60 tx=0x7fa844007e40 comp rx=0 tx=0).stop 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 msgr2=0x7fa84c06c0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c06c0a0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fa834009870 tx=0x7fa834004730 comp rx=0 tx=0).stop 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 shutdown_connections 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa82c06b4d0 0x7fa82c06d990 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa84c15f500 0x7fa84c1f7a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa84c0cf740 0x7fa84c06c0a0 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 --2- 172.21.15.64:0/2640083063 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa84c0c5e30 0x7fa84c1f85a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 >> 172.21.15.64:0/2640083063 conn(0x7fa84c0b3700 msgr2=0x7fa84c0b2e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 shutdown_connections 2024-03-04T07:27:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.463+0000 7fa8518d4640 1 -- 172.21.15.64:0/2640083063 wait complete. 2024-03-04T07:27:26.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:26.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.578+0000 7f4f8fb1e640 1 Processor -- start 2024-03-04T07:27:26.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.578+0000 7f4f8fb1e640 1 -- start start 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8fb1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8815e2f0 0x7f4f880b41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8fb1e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4f8806b4e0 0x7f4f8806b940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8fb1e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f8806be80 0x7f4f880cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8fb1e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f8815e6d0 con 0x7f4f8806be80 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8fb1e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f880ccd00 con 0x7f4f8806b4e0 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8fb1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f880ccfe0 con 0x7f4f8815e2f0 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8e0bb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f8806be80 0x7f4f880cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8e0bb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f8806be80 0x7f4f880cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51468/0 (socket says 172.21.15.64:51468) 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8e0bb640 1 -- 172.21.15.64:0/2977762472 learned_addr learned my addr 172.21.15.64:0/2977762472 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8d0b9640 1 --2- 172.21.15.64:0/2977762472 >> v2:172.21.15.66:3300/0 conn(0x7f4f8806b4e0 0x7f4f8806b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8d8ba640 1 --2- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8815e2f0 0x7f4f880b41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8d0b9640 1 -- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8815e2f0 msgr2=0x7f4f880b41f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:26.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8d0b9640 1 --2- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8815e2f0 0x7f4f880b41f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8d0b9640 1 -- 172.21.15.64:0/2977762472 >> v2:172.21.15.64:3300/0 conn(0x7f4f8806be80 msgr2=0x7f4f880cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:26.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8d0b9640 1 --2- 172.21.15.64:0/2977762472 >> v2:172.21.15.64:3300/0 conn(0x7f4f8806be80 0x7f4f880cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8d0b9640 1 -- 172.21.15.64:0/2977762472 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f880cd2c0 con 0x7f4f8806b4e0 2024-03-04T07:27:26.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.579+0000 7f4f8e0bb640 1 --2- 172.21.15.64:0/2977762472 >> v2:172.21.15.64:3300/0 conn(0x7f4f8806be80 0x7f4f880cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:26.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.580+0000 7f4f8d0b9640 1 --2- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806b4e0 0x7f4f8806b940 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f4f8806eaa0 tx=0x7f4f7802b710 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5c3a17ca62f8e53c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.580+0000 7f4f8c8b8640 1 -- 172.21.15.64:0/2977762472 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f78037070 con 0x7f4f8806b4e0 2024-03-04T07:27:26.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.580+0000 7f4f8c8b8640 1 -- 172.21.15.64:0/2977762472 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f78030710 con 0x7f4f8806b4e0 2024-03-04T07:27:26.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.580+0000 7f4f8c8b8640 1 -- 172.21.15.64:0/2977762472 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f78030ae0 con 0x7f4f8806b4e0 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806b4e0 msgr2=0x7f4f8806b940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806b4e0 0x7f4f8806b940 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f4f8806eaa0 tx=0x7f4f7802b710 comp rx=0 tx=0).stop 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/2977762472 shutdown_connections 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/2977762472 >> v2:172.21.15.64:3300/0 conn(0x7f4f8806be80 0x7f4f880cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806b4e0 0x7f4f8806b940 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/2977762472 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8815e2f0 0x7f4f880b41f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/2977762472 >> 172.21.15.64:0/2977762472 conn(0x7f4f880b36d0 msgr2=0x7f4f880b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/2977762472 shutdown_connections 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.581+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/2977762472 wait complete. 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.582+0000 7f4f8fb1e640 1 Processor -- start 2024-03-04T07:27:26.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.582+0000 7f4f8fb1e640 1 -- start start 2024-03-04T07:27:26.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8806b4e0 0x7f4f881f86a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806be80 0x7f4f88161130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f8815e2f0 0x7f4f880c8e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f880cff40 con 0x7f4f8815e2f0 2024-03-04T07:27:26.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f8806c310 con 0x7f4f8806be80 2024-03-04T07:27:26.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f880d00c0 con 0x7f4f8806b4e0 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f8815e2f0 0x7f4f880c8e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f8815e2f0 0x7f4f880c8e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51484/0 (socket says 172.21.15.64:51484) 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8d8ba640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8806b4e0 0x7f4f881f86a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 -- 172.21.15.64:0/3788547846 learned_addr learned my addr 172.21.15.64:0/3788547846 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8d0b9640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806be80 0x7f4f88161130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 -- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8806b4e0 msgr2=0x7f4f881f86a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8806b4e0 0x7f4f881f86a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 -- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806be80 msgr2=0x7f4f88161130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806be80 0x7f4f88161130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 -- 172.21.15.64:0/3788547846 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f881f54f0 con 0x7f4f8815e2f0 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8d8ba640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8806b4e0 0x7f4f881f86a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8d0b9640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806be80 0x7f4f88161130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8e0bb640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f8815e2f0 0x7f4f880c8e40 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f4f8000eed0 tx=0x7f4f8000b480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f7e7fc640 1 -- 172.21.15.64:0/3788547846 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f8000bb10 con 0x7f4f8815e2f0 2024-03-04T07:27:26.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f7e7fc640 1 -- 172.21.15.64:0/3788547846 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f8000bcb0 con 0x7f4f8815e2f0 2024-03-04T07:27:26.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f881618b0 con 0x7f4f8815e2f0 2024-03-04T07:27:26.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f7e7fc640 1 -- 172.21.15.64:0/3788547846 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f8000be60 con 0x7f4f8815e2f0 2024-03-04T07:27:26.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.583+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f881f4a10 con 0x7f4f8815e2f0 2024-03-04T07:27:26.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.584+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f8809fd30 con 0x7f4f8815e2f0 2024-03-04T07:27:26.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.585+0000 7f4f7e7fc640 1 -- 172.21.15.64:0/3788547846 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4f8001b030 con 0x7f4f8815e2f0 2024-03-04T07:27:26.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.585+0000 7f4f7e7fc640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f6c06b4d0 0x7f4f6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.585+0000 7f4f8d8ba640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f6c06b4d0 0x7f4f6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.585+0000 7f4f7e7fc640 1 -- 172.21.15.64:0/3788547846 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4f8008a020 con 0x7f4f8815e2f0 2024-03-04T07:27:26.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.586+0000 7f4f8d8ba640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f6c06b4d0 0x7f4f6c06d990 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f4f84009e90 tx=0x7f4f84009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.590+0000 7f4f7e7fc640 1 -- 172.21.15.64:0/3788547846 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4f80022020 con 0x7f4f8815e2f0 2024-03-04T07:27:26.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.770+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f4f8804f410 con 0x7f4f8815e2f0 2024-03-04T07:27:26.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.771+0000 7f4f7e7fc640 1 -- 172.21.15.64:0/3788547846 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f4f800588e0 con 0x7f4f8815e2f0 2024-03-04T07:27:26.770 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f6c06b4d0 msgr2=0x7f4f6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f6c06b4d0 0x7f4f6c06d990 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f4f84009e90 tx=0x7f4f84009190 comp rx=0 tx=0).stop 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f8815e2f0 msgr2=0x7f4f880c8e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f8815e2f0 0x7f4f880c8e40 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f4f8000eed0 tx=0x7f4f8000b480 comp rx=0 tx=0).stop 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 shutdown_connections 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f6c06b4d0 0x7f4f6c06d990 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f8815e2f0 0x7f4f880c8e40 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f8806be80 0x7f4f88161130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 --2- 172.21.15.64:0/3788547846 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f8806b4e0 0x7f4f881f86a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 >> 172.21.15.64:0/3788547846 conn(0x7f4f880b36d0 msgr2=0x7f4f880c7eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 shutdown_connections 2024-03-04T07:27:26.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.773+0000 7f4f8fb1e640 1 -- 172.21.15.64:0/3788547846 wait complete. 2024-03-04T07:27:26.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:26.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.888+0000 7f4854ff7640 1 Processor -- start 2024-03-04T07:27:26.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.888+0000 7f4854ff7640 1 -- start start 2024-03-04T07:27:26.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f4854ff7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f485015b130 0x7f485015b590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f4854ff7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f48500ce900 0x7f48500ced90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f4854ff7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f485015bc40 con 0x7f4850144640 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f4854ff7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f485006df70 con 0x7f48500ce900 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f4854ff7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f485015bf20 con 0x7f485015b130 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f484ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f48500ce900 0x7f48500ced90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f484ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f48500ce900 0x7f48500ced90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51496/0 (socket says 172.21.15.64:51496) 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f484ed9d640 1 -- 172.21.15.64:0/2638525996 learned_addr learned my addr 172.21.15.64:0/2638525996 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.889+0000 7f4847fff640 1 --2- 172.21.15.64:0/2638525996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f485015b130 0x7f485015b590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.890+0000 7f484ed9d640 1 -- 172.21.15.64:0/2638525996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f485015b130 msgr2=0x7f485015b590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.890+0000 7f484ed9d640 1 --2- 172.21.15.64:0/2638525996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f485015b130 0x7f485015b590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.890+0000 7f484ed9d640 1 -- 172.21.15.64:0/2638525996 >> v1:172.21.15.64:6789/0 conn(0x7f4850144640 legacy=0x7f485006e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.890+0000 7f484ed9d640 1 -- 172.21.15.64:0/2638525996 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f485006dc00 con 0x7f48500ce900 2024-03-04T07:27:26.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f484ed9d640 1 --2- 172.21.15.64:0/2638525996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48500ce900 0x7f48500ced90 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f484800d150 tx=0x7f484802e160 comp rx=0 tx=0).ready entity=mon.0 client_cookie=85612a61c6c8790a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f484dd9b640 1 -- 172.21.15.64:0/2638525996 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f484800b0d0 con 0x7f48500ce900 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f484dd9b640 1 -- 172.21.15.64:0/2638525996 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f484800b270 con 0x7f48500ce900 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f484dd9b640 1 -- 172.21.15.64:0/2638525996 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4848009600 con 0x7f48500ce900 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 -- 172.21.15.64:0/2638525996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48500ce900 msgr2=0x7f48500ced90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 --2- 172.21.15.64:0/2638525996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48500ce900 0x7f48500ced90 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f484800d150 tx=0x7f484802e160 comp rx=0 tx=0).stop 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 -- 172.21.15.64:0/2638525996 shutdown_connections 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 --2- 172.21.15.64:0/2638525996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48500ce900 0x7f48500ced90 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 --2- 172.21.15.64:0/2638525996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f485015b130 0x7f485015b590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 -- 172.21.15.64:0/2638525996 >> 172.21.15.64:0/2638525996 conn(0x7f4850066d00 msgr2=0x7f4850067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 -- 172.21.15.64:0/2638525996 shutdown_connections 2024-03-04T07:27:26.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.891+0000 7f4854ff7640 1 -- 172.21.15.64:0/2638525996 wait complete. 2024-03-04T07:27:26.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 Processor -- start 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 -- start start 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48500ce900 0x7f485015a3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4850144640 0x7f4850137b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f485015b130 0x7f4850139380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4850070c20 con 0x7f485015b130 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4850070aa0 con 0x7f48500ce900 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.893+0000 7f4854ff7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f48500b36d0 con 0x7f4850144640 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f485015b130 0x7f4850139380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f485015b130 0x7f4850139380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51504/0 (socket says 172.21.15.64:51504) 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 -- 172.21.15.64:0/82278047 learned_addr learned my addr 172.21.15.64:0/82278047 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484e59c640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48500ce900 0x7f485015a3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f4847fff640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4850144640 0x7f4850137b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 -- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4850144640 msgr2=0x7f4850137b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4850144640 0x7f4850137b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 -- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48500ce900 msgr2=0x7f485015a3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48500ce900 0x7f485015a3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 -- 172.21.15.64:0/82278047 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48501385b0 con 0x7f485015b130 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f484ed9d640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f485015b130 0x7f4850139380 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f4848038040 tx=0x7f4848004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f4846ffd640 1 -- 172.21.15.64:0/82278047 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4848045020 con 0x7f485015b130 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f4846ffd640 1 -- 172.21.15.64:0/82278047 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4848009e30 con 0x7f485015b130 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4850138890 con 0x7f485015b130 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f4846ffd640 1 -- 172.21.15.64:0/82278047 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f484800b0d0 con 0x7f485015b130 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.894+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f485013a940 con 0x7f485015b130 2024-03-04T07:27:26.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.895+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f48500733c0 con 0x7f485015b130 2024-03-04T07:27:26.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.896+0000 7f4846ffd640 1 -- 172.21.15.64:0/82278047 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4848048020 con 0x7f485015b130 2024-03-04T07:27:26.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.896+0000 7f4846ffd640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f483006b4d0 0x7f483006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:26.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.896+0000 7f484e59c640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f483006b4d0 0x7f483006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:26.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.896+0000 7f4846ffd640 1 -- 172.21.15.64:0/82278047 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f48480aea80 con 0x7f485015b130 2024-03-04T07:27:26.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.897+0000 7f484e59c640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f483006b4d0 0x7f483006d990 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f4840001780 tx=0x7f4840009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:26.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:26.901+0000 7f4846ffd640 1 -- 172.21.15.64:0/82278047 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f484804f030 con 0x7f485015b130 2024-03-04T07:27:27.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.082+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f485004f410 con 0x7f485015b130 2024-03-04T07:27:27.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.082+0000 7f4846ffd640 1 -- 172.21.15.64:0/82278047 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f484807d340 con 0x7f485015b130 2024-03-04T07:27:27.082 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f483006b4d0 msgr2=0x7f483006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f483006b4d0 0x7f483006d990 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f4840001780 tx=0x7f4840009040 comp rx=0 tx=0).stop 2024-03-04T07:27:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f485015b130 msgr2=0x7f4850139380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f485015b130 0x7f4850139380 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f4848038040 tx=0x7f4848004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 shutdown_connections 2024-03-04T07:27:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f483006b4d0 0x7f483006d990 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f485015b130 0x7f4850139380 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4850144640 0x7f4850137b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 --2- 172.21.15.64:0/82278047 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48500ce900 0x7f485015a3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 >> 172.21.15.64:0/82278047 conn(0x7f4850066d00 msgr2=0x7f485015d240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 shutdown_connections 2024-03-04T07:27:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.084+0000 7f4854ff7640 1 -- 172.21.15.64:0/82278047 wait complete. 2024-03-04T07:27:27.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:27.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.198+0000 7fa521afb640 1 Processor -- start 2024-03-04T07:27:27.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.198+0000 7fa521afb640 1 -- start start 2024-03-04T07:27:27.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa521afb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c1614b0 0x7fa51c0d05d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa521afb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa51c0b9ad0 0x7fa51c0ceaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa521afb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa51c0d1590 con 0x7fa51c0d0b10 2024-03-04T07:27:27.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa521afb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa51c161890 con 0x7fa51c0b9ad0 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa521afb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa51c0d0fd0 con 0x7fa51c1614b0 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa51c0d0b10 0x7fa51c161a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48588/0 (socket says 172.21.15.64:48588) 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51bfff640 1 -- 172.21.15.64:0/1349905989 learned_addr learned my addr 172.21.15.64:0/1349905989 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51affd640 1 --2- 172.21.15.64:0/1349905989 >> v2:172.21.15.66:3300/0 conn(0x7fa51c0b9ad0 0x7fa51c0ceaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51b7fe640 1 --2- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c1614b0 0x7fa51c0d05d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51a7fc640 1 -- 172.21.15.64:0/1349905989 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4163456946 0 0) 0x7fa51c0d1590 con 0x7fa51c0d0b10 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51a7fc640 1 -- 172.21.15.64:0/1349905989 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa504003520 con 0x7fa51c0d0b10 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51affd640 1 -- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c1614b0 msgr2=0x7fa51c0d05d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51affd640 1 --2- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c1614b0 0x7fa51c0d05d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51affd640 1 -- 172.21.15.64:0/1349905989 >> v1:172.21.15.64:6789/0 conn(0x7fa51c0d0b10 legacy=0x7fa51c161a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.199+0000 7fa51affd640 1 -- 172.21.15.64:0/1349905989 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa51c0d12b0 con 0x7fa51c0b9ad0 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.200+0000 7fa51affd640 1 --2- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0b9ad0 0x7fa51c0ceaf0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fa5100097a0 tx=0x7fa51002b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c38e769c6de3762c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.200+0000 7fa51a7fc640 1 -- 172.21.15.64:0/1349905989 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5100315a0 con 0x7fa51c0b9ad0 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.200+0000 7fa51a7fc640 1 -- 172.21.15.64:0/1349905989 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa510037070 con 0x7fa51c0b9ad0 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.200+0000 7fa51a7fc640 1 -- 172.21.15.64:0/1349905989 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa510031940 con 0x7fa51c0b9ad0 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 -- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0b9ad0 msgr2=0x7fa51c0ceaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 --2- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0b9ad0 0x7fa51c0ceaf0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fa5100097a0 tx=0x7fa51002b840 comp rx=0 tx=0).stop 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 -- 172.21.15.64:0/1349905989 shutdown_connections 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 --2- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0b9ad0 0x7fa51c0ceaf0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 --2- 172.21.15.64:0/1349905989 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c1614b0 0x7fa51c0d05d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 -- 172.21.15.64:0/1349905989 >> 172.21.15.64:0/1349905989 conn(0x7fa51c071930 msgr2=0x7fa51c071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:27.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 -- 172.21.15.64:0/1349905989 shutdown_connections 2024-03-04T07:27:27.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.201+0000 7fa521afb640 1 -- 172.21.15.64:0/1349905989 wait complete. 2024-03-04T07:27:27.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.202+0000 7fa521afb640 1 Processor -- start 2024-03-04T07:27:27.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.202+0000 7fa521afb640 1 -- start start 2024-03-04T07:27:27.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa521afb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0d0b10 0x7fa51c1f3050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa521afb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa51c1614b0 0x7fa51c0d2540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa521afb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c0ad1c0 0x7fa51c0d2a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa521afb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa51c0d37d0 con 0x7fa51c1614b0 2024-03-04T07:27:27.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa521afb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa51c0d3650 con 0x7fa51c0d0b10 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa521afb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa51c0d3950 con 0x7fa51c0ad1c0 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa51c1614b0 0x7fa51c0d2540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa51c1614b0 0x7fa51c0d2540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51516/0 (socket says 172.21.15.64:51516) 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 -- 172.21.15.64:0/2615029593 learned_addr learned my addr 172.21.15.64:0/2615029593 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51b7fe640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0d0b10 0x7fa51c1f3050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51bfff640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c0ad1c0 0x7fa51c0d2a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 -- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c0ad1c0 msgr2=0x7fa51c0d2a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c0ad1c0 0x7fa51c0d2a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 -- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0d0b10 msgr2=0x7fa51c1f3050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0d0b10 0x7fa51c1f3050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.203+0000 7fa51affd640 1 -- 172.21.15.64:0/2615029593 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa51c0abfe0 con 0x7fa51c1614b0 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.204+0000 7fa51b7fe640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0d0b10 0x7fa51c1f3050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.204+0000 7fa51affd640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa51c1614b0 0x7fa51c0d2540 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7fa5100098c0 tx=0x7fa5100046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:27.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.204+0000 7fa503fff640 1 -- 172.21.15.64:0/2615029593 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa510032430 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.204+0000 7fa503fff640 1 -- 172.21.15.64:0/2615029593 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa5100325d0 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.204+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa51c0ac2c0 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.204+0000 7fa503fff640 1 -- 172.21.15.64:0/2615029593 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa510032980 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.204+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa51c0ac880 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.205+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa51c067ff0 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.205+0000 7fa503fff640 1 -- 172.21.15.64:0/2615029593 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa510032b20 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.206+0000 7fa503fff640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa50806b4d0 0x7fa50806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.206+0000 7fa503fff640 1 -- 172.21.15.64:0/2615029593 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa51002f030 con 0x7fa51c1614b0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.206+0000 7fa51b7fe640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa50806b4d0 0x7fa50806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.206+0000 7fa51b7fe640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa50806b4d0 0x7fa50806d990 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fa50c009f30 tx=0x7fa50c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:27.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.210+0000 7fa503fff640 1 -- 172.21.15.64:0/2615029593 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa5100b0070 con 0x7fa51c1614b0 2024-03-04T07:27:27.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.404+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fa51c0d3330 con 0x7fa51c1614b0 2024-03-04T07:27:27.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.404+0000 7fa503fff640 1 -- 172.21.15.64:0/2615029593 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v20) v1 ==== 80+0+0 (secure 0 0 0) 0x7fa510040070 con 0x7fa51c1614b0 2024-03-04T07:27:27.404 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa50806b4d0 msgr2=0x7fa50806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa50806b4d0 0x7fa50806d990 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fa50c009f30 tx=0x7fa50c009190 comp rx=0 tx=0).stop 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa51c1614b0 msgr2=0x7fa51c0d2540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa51c1614b0 0x7fa51c0d2540 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7fa5100098c0 tx=0x7fa5100046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 shutdown_connections 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa50806b4d0 0x7fa50806d990 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa51c0ad1c0 0x7fa51c0d2a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa51c1614b0 0x7fa51c0d2540 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 --2- 172.21.15.64:0/2615029593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa51c0d0b10 0x7fa51c1f3050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 >> 172.21.15.64:0/2615029593 conn(0x7fa51c071930 msgr2=0x7fa51c067590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 shutdown_connections 2024-03-04T07:27:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.406+0000 7fa521afb640 1 -- 172.21.15.64:0/2615029593 wait complete. 2024-03-04T07:27:27.416 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi066.front.sepia.ceph.com... 2024-03-04T07:27:27.416 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:27.416 DEBUG:teuthology.orchestra.run.smithi066:> sudo fusermount -u /home/ubuntu/cephtest/mnt.1 2024-03-04T07:27:27.448 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:27.448 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:27:27.448 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:27.448 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:27:27.514 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:27:27.514 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:27.529 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:27:27.529 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:27.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:27.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.726+0000 7f738bc84640 1 Processor -- start 2024-03-04T07:27:27.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.726+0000 7f738bc84640 1 -- start start 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738bc84640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f738415dca0 0x7f738406fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738bc84640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f738415e080 0x7f7384066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738bc84640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7384161c20 con 0x7f73840b00c0 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738bc84640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f73841619d0 con 0x7f738415e080 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738bc84640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7384161f00 con 0x7f738415dca0 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738921f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f738415e080 0x7f7384066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738921f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f738415e080 0x7f7384066d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51522/0 (socket says 172.21.15.64:51522) 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738a221640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f73840b00c0 0x7f73840719b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48600/0 (socket says 172.21.15.64:48600) 2024-03-04T07:27:27.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f7389a20640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f738415dca0 0x7f738406fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.727+0000 7f738a221640 1 -- 172.21.15.64:0/1710564118 learned_addr learned my addr 172.21.15.64:0/1710564118 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:27.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f738921f640 1 -- 172.21.15.64:0/1710564118 >> v2:172.21.15.66:3300/0 conn(0x7f738415dca0 msgr2=0x7f738406fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:27.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f738921f640 1 --2- 172.21.15.64:0/1710564118 >> v2:172.21.15.66:3300/0 conn(0x7f738415dca0 0x7f738406fd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f738921f640 1 -- 172.21.15.64:0/1710564118 >> v1:172.21.15.64:6789/0 conn(0x7f73840b00c0 legacy=0x7f73840719b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f738921f640 1 -- 172.21.15.64:0/1710564118 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73841621e0 con 0x7f738415e080 2024-03-04T07:27:27.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f7389a20640 1 --2- 172.21.15.64:0/1710564118 >> v2:172.21.15.66:3300/0 conn(0x7f738415dca0 0x7f738406fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:27.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f738921f640 1 --2- 172.21.15.64:0/1710564118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415e080 0x7f7384066d00 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f738406c050 tx=0x7f737402b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c57458171cb896ad server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:27.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f7388a1e640 1 -- 172.21.15.64:0/1710564118 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73740315a0 con 0x7f738415e080 2024-03-04T07:27:27.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.728+0000 7f7388a1e640 1 -- 172.21.15.64:0/1710564118 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7374037070 con 0x7f738415e080 2024-03-04T07:27:27.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f7388a1e640 1 -- 172.21.15.64:0/1710564118 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7374031960 con 0x7f738415e080 2024-03-04T07:27:27.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 -- 172.21.15.64:0/1710564118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415e080 msgr2=0x7f7384066d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 --2- 172.21.15.64:0/1710564118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415e080 0x7f7384066d00 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f738406c050 tx=0x7f737402b840 comp rx=0 tx=0).stop 2024-03-04T07:27:27.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 -- 172.21.15.64:0/1710564118 shutdown_connections 2024-03-04T07:27:27.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 --2- 172.21.15.64:0/1710564118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415e080 0x7f7384066d00 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 --2- 172.21.15.64:0/1710564118 >> v2:172.21.15.66:3300/0 conn(0x7f738415dca0 0x7f738406fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 -- 172.21.15.64:0/1710564118 >> 172.21.15.64:0/1710564118 conn(0x7f73840d0990 msgr2=0x7f73840d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:27.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 -- 172.21.15.64:0/1710564118 shutdown_connections 2024-03-04T07:27:27.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.729+0000 7f738bc84640 1 -- 172.21.15.64:0/1710564118 wait complete. 2024-03-04T07:27:27.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.730+0000 7f738bc84640 1 Processor -- start 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.730+0000 7f738bc84640 1 -- start start 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738bc84640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73840b00c0 0x7f7384145500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738bc84640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415dca0 0x7f73840b1730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738bc84640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73840b1c70 0x7f738415fdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738bc84640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73840703d0 con 0x7f738415dca0 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738bc84640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7384070250 con 0x7f73840b1c70 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738bc84640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f73840b4cb0 con 0x7f73840b00c0 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415dca0 0x7f73840b1730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415dca0 0x7f73840b1730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51526/0 (socket says 172.21.15.64:51526) 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 -- 172.21.15.64:0/3656744262 learned_addr learned my addr 172.21.15.64:0/3656744262 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738a221640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73840b1c70 0x7f738415fdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f7389a20640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73840b00c0 0x7f7384145500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 -- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73840b00c0 msgr2=0x7f7384145500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73840b00c0 0x7f7384145500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 -- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73840b1c70 msgr2=0x7f738415fdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73840b1c70 0x7f738415fdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f738921f640 1 -- 172.21.15.64:0/3656744262 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7384145a40 con 0x7f738415dca0 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.731+0000 7f7389a20640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73840b00c0 0x7f7384145500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.732+0000 7f738921f640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415dca0 0x7f73840b1730 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f737402be50 tx=0x7f73740046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.732+0000 7f737a7fc640 1 -- 172.21.15.64:0/3656744262 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7374032630 con 0x7f738415dca0 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.732+0000 7f737a7fc640 1 -- 172.21.15.64:0/3656744262 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f73740327f0 con 0x7f738415dca0 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.732+0000 7f737a7fc640 1 -- 172.21.15.64:0/3656744262 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7374032be0 con 0x7f738415dca0 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.732+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f738415e610 con 0x7f738415dca0 2024-03-04T07:27:27.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.732+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f738415ebd0 con 0x7f738415dca0 2024-03-04T07:27:27.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.733+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7384068700 con 0x7f738415dca0 2024-03-04T07:27:27.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.733+0000 7f737a7fc640 1 -- 172.21.15.64:0/3656744262 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f737404c020 con 0x7f738415dca0 2024-03-04T07:27:27.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.734+0000 7f737a7fc640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f736806b500 0x7f736806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:27.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.734+0000 7f7389a20640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f736806b500 0x7f736806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:27.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.734+0000 7f737a7fc640 1 -- 172.21.15.64:0/3656744262 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f737402f020 con 0x7f738415dca0 2024-03-04T07:27:27.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.734+0000 7f7389a20640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f736806b500 0x7f736806d9c0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f737c009ba0 tx=0x7f737c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:27.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.738+0000 7f737a7fc640 1 -- 172.21.15.64:0/3656744262 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7384145a40 con 0x7f738415dca0 2024-03-04T07:27:27.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.921+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f73840aee50 con 0x7f738415dca0 2024-03-04T07:27:27.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.922+0000 7f737a7fc640 1 -- 172.21.15.64:0/3656744262 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f7374040070 con 0x7f738415dca0 2024-03-04T07:27:27.921 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f736806b500 msgr2=0x7f736806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f736806b500 0x7f736806d9c0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f737c009ba0 tx=0x7f737c008040 comp rx=0 tx=0).stop 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415dca0 msgr2=0x7f73840b1730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415dca0 0x7f73840b1730 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f737402be50 tx=0x7f73740046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 shutdown_connections 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f736806b500 0x7f736806d9c0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f73840b1c70 0x7f738415fdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f738415dca0 0x7f73840b1730 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 --2- 172.21.15.64:0/3656744262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f73840b00c0 0x7f7384145500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:27.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 >> 172.21.15.64:0/3656744262 conn(0x7f73840d0990 msgr2=0x7f73840ae740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:27.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 shutdown_connections 2024-03-04T07:27:27.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:27.924+0000 7f738bc84640 1 -- 172.21.15.64:0/3656744262 wait complete. 2024-03-04T07:27:27.934 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:27.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:28.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.040+0000 7fc232c40640 1 Processor -- start 2024-03-04T07:27:28.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.040+0000 7fc232c40640 1 -- start start 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.040+0000 7fc232c40640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc22c0717b0 0x7fc22c071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.040+0000 7fc232c40640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c0d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.040+0000 7fc232c40640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc22c15e4b0 con 0x7fc22c0b3060 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.040+0000 7fc232c40640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc22c0d2e10 con 0x7fc22c0717b0 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.040+0000 7fc232c40640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc22c0b97e0 con 0x7fc22c0b2b10 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2309dc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc22c0717b0 0x7fc22c071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2309dc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc22c0717b0 0x7fc22c071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51536/0 (socket says 172.21.15.64:51536) 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc22bfff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fc22c0b3060 0x7fc22c0b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:40186/0 (socket says 172.21.15.64:40186) 2024-03-04T07:27:28.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2309dc640 1 -- 172.21.15.64:0/188244762 learned_addr learned my addr 172.21.15.64:0/188244762 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:28.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2311dd640 1 --2- 172.21.15.64:0/188244762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c0d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2309dc640 1 -- 172.21.15.64:0/188244762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 msgr2=0x7fc22c0d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:28.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2309dc640 1 --2- 172.21.15.64:0/188244762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c0d32a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2309dc640 1 -- 172.21.15.64:0/188244762 >> v1:172.21.15.66:6789/0 conn(0x7fc22c0b3060 legacy=0x7fc22c0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.041+0000 7fc2309dc640 1 -- 172.21.15.64:0/188244762 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc22c0b9a80 con 0x7fc22c0717b0 2024-03-04T07:27:28.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.042+0000 7fc2309dc640 1 --2- 172.21.15.64:0/188244762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c071b90 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fc220009870 tx=0x7fc22002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=455c216631ba273a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.042+0000 7fc22b7fe640 1 -- 172.21.15.64:0/188244762 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc220034070 con 0x7fc22c0717b0 2024-03-04T07:27:28.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.042+0000 7fc22b7fe640 1 -- 172.21.15.64:0/188244762 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc220030840 con 0x7fc22c0717b0 2024-03-04T07:27:28.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.042+0000 7fc22b7fe640 1 -- 172.21.15.64:0/188244762 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc220030c10 con 0x7fc22c0717b0 2024-03-04T07:27:28.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.042+0000 7fc232c40640 1 -- 172.21.15.64:0/188244762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 msgr2=0x7fc22c071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.042+0000 7fc232c40640 1 --2- 172.21.15.64:0/188244762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c071b90 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fc220009870 tx=0x7fc22002b980 comp rx=0 tx=0).stop 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.043+0000 7fc232c40640 1 -- 172.21.15.64:0/188244762 shutdown_connections 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.043+0000 7fc232c40640 1 --2- 172.21.15.64:0/188244762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c0d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.043+0000 7fc232c40640 1 --2- 172.21.15.64:0/188244762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c071b90 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.043+0000 7fc232c40640 1 -- 172.21.15.64:0/188244762 >> 172.21.15.64:0/188244762 conn(0x7fc22c0d3d20 msgr2=0x7fc22c0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.043+0000 7fc232c40640 1 -- 172.21.15.64:0/188244762 shutdown_connections 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.043+0000 7fc232c40640 1 -- 172.21.15.64:0/188244762 wait complete. 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.044+0000 7fc232c40640 1 Processor -- start 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.044+0000 7fc232c40640 1 -- start start 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c14fea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c159610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc22c0b3060 0x7fc22c0c93f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc22c0b7560 con 0x7fc22c0717b0 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc22c0d4bf0 con 0x7fc22c0b3060 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc22c0b76e0 con 0x7fc22c0b2b10 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c14fea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c14fea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51540/0 (socket says 172.21.15.64:51540) 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 -- 172.21.15.64:0/750137447 learned_addr learned my addr 172.21.15.64:0/750137447 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2311dd640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc22c0b3060 0x7fc22c0c93f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc22bfff640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c159610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 -- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 msgr2=0x7fc22c159610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c159610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 -- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc22c0b3060 msgr2=0x7fc22c0c93f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc22c0b3060 0x7fc22c0c93f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 -- 172.21.15.64:0/750137447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc22c0ca0e0 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2311dd640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc22c0b3060 0x7fc22c0c93f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2309dc640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c14fea0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fc220000c00 tx=0x7fc220004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2297fa640 1 -- 172.21.15.64:0/750137447 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc220048070 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2297fa640 1 -- 172.21.15.64:0/750137447 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc220030640 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc22c0ca3c0 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc2297fa640 1 -- 172.21.15.64:0/750137447 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc220034040 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.045+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc22c156990 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.047+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc22c156dd0 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.047+0000 7fc2297fa640 1 -- 172.21.15.64:0/750137447 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc22004b070 con 0x7fc22c0717b0 2024-03-04T07:27:28.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.047+0000 7fc2297fa640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc21006b4d0 0x7fc21006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.047+0000 7fc2297fa640 1 -- 172.21.15.64:0/750137447 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc2200abf90 con 0x7fc22c0717b0 2024-03-04T07:27:28.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.047+0000 7fc22bfff640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc21006b4d0 0x7fc21006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.048+0000 7fc22bfff640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc21006b4d0 0x7fc21006d990 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fc22c0cfa70 tx=0x7fc218007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.052+0000 7fc2297fa640 1 -- 172.21.15.64:0/750137447 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc22007a970 con 0x7fc22c0717b0 2024-03-04T07:27:28.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.231+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fc22c04f410 con 0x7fc22c0717b0 2024-03-04T07:27:28.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.231+0000 7fc2297fa640 1 -- 172.21.15.64:0/750137447 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7fc22007ea10 con 0x7fc22c0717b0 2024-03-04T07:27:28.231 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:28.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc21006b4d0 msgr2=0x7fc21006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc21006b4d0 0x7fc21006d990 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fc22c0cfa70 tx=0x7fc218007e80 comp rx=0 tx=0).stop 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 msgr2=0x7fc22c14fea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c14fea0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fc220000c00 tx=0x7fc220004730 comp rx=0 tx=0).stop 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 shutdown_connections 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc21006b4d0 0x7fc21006d990 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc22c0b3060 0x7fc22c0c93f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc22c0b2b10 0x7fc22c159610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 --2- 172.21.15.64:0/750137447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc22c0717b0 0x7fc22c14fea0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 >> 172.21.15.64:0/750137447 conn(0x7fc22c0d3d20 msgr2=0x7fc22c0ce7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 shutdown_connections 2024-03-04T07:27:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.233+0000 7fc232c40640 1 -- 172.21.15.64:0/750137447 wait complete. 2024-03-04T07:27:28.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:28.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.348+0000 7f688241c640 1 Processor -- start 2024-03-04T07:27:28.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.348+0000 7f688241c640 1 -- start start 2024-03-04T07:27:28.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.349+0000 7f688241c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0cdcd0 0x7f687c0b1550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.349+0000 7f688241c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f687c0b2cd0 0x7f687c0b1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.349+0000 7f688241c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f687c161a40 con 0x7f687c0b1fd0 2024-03-04T07:27:28.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.349+0000 7f688241c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f687c0d11e0 con 0x7f687c0b2cd0 2024-03-04T07:27:28.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.349+0000 7f688241c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f687c06c2d0 con 0x7f687c0cdcd0 2024-03-04T07:27:28.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.349+0000 7f687b7fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f687c0b2cd0 0x7f687c0b1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.349+0000 7f68809b9640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f687c0b1fd0 0x7f687c0ce3f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:40188/0 (socket says 172.21.15.64:40188) 2024-03-04T07:27:28.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.350+0000 7f687bfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0cdcd0 0x7f687c0b1550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.350+0000 7f68809b9640 1 -- 172.21.15.64:0/3206633730 learned_addr learned my addr 172.21.15.64:0/3206633730 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:28.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.350+0000 7f687b7fe640 1 -- 172.21.15.64:0/3206633730 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0cdcd0 msgr2=0x7f687c0b1550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:28.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.350+0000 7f687b7fe640 1 --2- 172.21.15.64:0/3206633730 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0cdcd0 0x7f687c0b1550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.350+0000 7f687b7fe640 1 -- 172.21.15.64:0/3206633730 >> v1:172.21.15.66:6789/0 conn(0x7f687c0b1fd0 legacy=0x7f687c0ce3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.350+0000 7f687b7fe640 1 -- 172.21.15.64:0/3206633730 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f687c0b2a20 con 0x7f687c0b2cd0 2024-03-04T07:27:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f687b7fe640 1 --2- 172.21.15.64:0/3206633730 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0b2cd0 0x7f687c0b1a90 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f686c0097a0 tx=0x7f686c02b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bdb50607786c6473 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f687affd640 1 -- 172.21.15.64:0/3206633730 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f686c0315a0 con 0x7f687c0b2cd0 2024-03-04T07:27:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f687affd640 1 -- 172.21.15.64:0/3206633730 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f686c037070 con 0x7f687c0b2cd0 2024-03-04T07:27:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f687affd640 1 -- 172.21.15.64:0/3206633730 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f686c031960 con 0x7f687c0b2cd0 2024-03-04T07:27:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 -- 172.21.15.64:0/3206633730 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0b2cd0 msgr2=0x7f687c0b1a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 --2- 172.21.15.64:0/3206633730 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0b2cd0 0x7f687c0b1a90 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f686c0097a0 tx=0x7f686c02b840 comp rx=0 tx=0).stop 2024-03-04T07:27:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 -- 172.21.15.64:0/3206633730 shutdown_connections 2024-03-04T07:27:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 --2- 172.21.15.64:0/3206633730 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0b2cd0 0x7f687c0b1a90 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 --2- 172.21.15.64:0/3206633730 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0cdcd0 0x7f687c0b1550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 -- 172.21.15.64:0/3206633730 >> 172.21.15.64:0/3206633730 conn(0x7f687c0b3700 msgr2=0x7f687c0b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 -- 172.21.15.64:0/3206633730 shutdown_connections 2024-03-04T07:27:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.351+0000 7f688241c640 1 -- 172.21.15.64:0/3206633730 wait complete. 2024-03-04T07:27:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 Processor -- start 2024-03-04T07:27:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 -- start start 2024-03-04T07:27:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0b1fd0 0x7f687c156e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f687c0b2cd0 0x7f687c1f4030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0cdcd0 0x7f687c159b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f687c0d0f00 con 0x7f687c0b2cd0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f687c0d0c20 con 0x7f687c0cdcd0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.353+0000 7f688241c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f687c0cc1c0 con 0x7f687c0b1fd0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0b1fd0 0x7f687c156e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f687c0b2cd0 0x7f687c1f4030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f68809b9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0cdcd0 0x7f687c159b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0b1fd0 0x7f687c156e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:34462/0 (socket says 172.21.15.64:34462) 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 -- 172.21.15.64:0/3020334052 learned_addr learned my addr 172.21.15.64:0/3020334052 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 -- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0cdcd0 msgr2=0x7f687c159b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0cdcd0 0x7f687c159b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 -- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f687c0b2cd0 msgr2=0x7f687c1f4030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f687c0b2cd0 0x7f687c1f4030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 -- 172.21.15.64:0/3020334052 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f687c1f49a0 con 0x7f687c0b1fd0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f68809b9640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0cdcd0 0x7f687c159b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f687bfff640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0b1fd0 0x7f687c156e30 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f687000e9b0 tx=0x7f687000ee70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f6878ff9640 1 -- 172.21.15.64:0/3020334052 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6870017070 con 0x7f687c0b1fd0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f6878ff9640 1 -- 172.21.15.64:0/3020334052 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f687000c9a0 con 0x7f687c0b1fd0 2024-03-04T07:27:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f6878ff9640 1 -- 172.21.15.64:0/3020334052 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f687000cd90 con 0x7f687c0b1fd0 2024-03-04T07:27:28.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f687c1f4c80 con 0x7f687c0b1fd0 2024-03-04T07:27:28.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.354+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f687c1f5640 con 0x7f687c0b1fd0 2024-03-04T07:27:28.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.355+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f687c09fcf0 con 0x7f687c0b1fd0 2024-03-04T07:27:28.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.356+0000 7f6878ff9640 1 -- 172.21.15.64:0/3020334052 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f687001a070 con 0x7f687c0b1fd0 2024-03-04T07:27:28.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.356+0000 7f6878ff9640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f686806b4d0 0x7f686806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.356+0000 7f6878ff9640 1 -- 172.21.15.64:0/3020334052 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f687008a5a0 con 0x7f687c0b1fd0 2024-03-04T07:27:28.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.356+0000 7f687b7fe640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f686806b4d0 0x7f686806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.357+0000 7f687b7fe640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f686806b4d0 0x7f686806d990 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f686c036000 tx=0x7f686c03d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.360+0000 7f6878ff9640 1 -- 172.21.15.64:0/3020334052 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6870058d50 con 0x7f687c0b1fd0 2024-03-04T07:27:28.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.540+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f687c0d1690 con 0x7f687c0b1fd0 2024-03-04T07:27:28.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.541+0000 7f6878ff9640 1 -- 172.21.15.64:0/3020334052 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7f687005cdf0 con 0x7f687c0b1fd0 2024-03-04T07:27:28.541 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f686806b4d0 msgr2=0x7f686806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f686806b4d0 0x7f686806d990 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f686c036000 tx=0x7f686c03d040 comp rx=0 tx=0).stop 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0b1fd0 msgr2=0x7f687c156e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0b1fd0 0x7f687c156e30 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f687000e9b0 tx=0x7f687000ee70 comp rx=0 tx=0).stop 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 shutdown_connections 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f686806b4d0 0x7f686806d990 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f687c0cdcd0 0x7f687c159b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f687c0b2cd0 0x7f687c1f4030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 --2- 172.21.15.64:0/3020334052 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f687c0b1fd0 0x7f687c156e30 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 >> 172.21.15.64:0/3020334052 conn(0x7f687c0b3700 msgr2=0x7f687c06ecb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 shutdown_connections 2024-03-04T07:27:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.543+0000 7f688241c640 1 -- 172.21.15.64:0/3020334052 wait complete. 2024-03-04T07:27:28.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:28.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.657+0000 7fa8a9678640 1 Processor -- start 2024-03-04T07:27:28.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.657+0000 7fa8a9678640 1 -- start start 2024-03-04T07:27:28.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.658+0000 7fa8a9678640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415ae50 0x7fa8a40708e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.658+0000 7fa8a9678640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa8a415b230 0x7fa8a4071160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.658+0000 7fa8a9678640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa8a4070e20 con 0x7fa8a415b690 2024-03-04T07:27:28.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.658+0000 7fa8a9678640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa8a40b2e30 con 0x7fa8a415b230 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.658+0000 7fa8a9678640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa8a406dfe0 con 0x7fa8a415ae50 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.658+0000 7fa8a37fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa8a415b690 0x7fa8a4150a00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48602/0 (socket says 172.21.15.64:48602) 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.658+0000 7fa8a37fe640 1 -- 172.21.15.64:0/3825876277 learned_addr learned my addr 172.21.15.64:0/3825876277 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a27fc640 1 --2- 172.21.15.64:0/3825876277 >> v2:172.21.15.66:3300/0 conn(0x7fa8a415b230 0x7fa8a4071160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a2ffd640 1 --2- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415ae50 0x7fa8a40708e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a1ffb640 1 -- 172.21.15.64:0/3825876277 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 463486000 0 0) 0x7fa8a4070e20 con 0x7fa8a415b690 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a1ffb640 1 -- 172.21.15.64:0/3825876277 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa88c004630 con 0x7fa8a415b690 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a27fc640 1 -- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415ae50 msgr2=0x7fa8a40708e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a27fc640 1 --2- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415ae50 0x7fa8a40708e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a27fc640 1 -- 172.21.15.64:0/3825876277 >> v1:172.21.15.64:6789/0 conn(0x7fa8a415b690 legacy=0x7fa8a4150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a27fc640 1 -- 172.21.15.64:0/3825876277 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8a40b30b0 con 0x7fa8a415b230 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.659+0000 7fa8a2ffd640 1 --2- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415ae50 0x7fa8a40708e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a27fc640 1 --2- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b230 0x7fa8a4071160 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa8980097a0 tx=0x7fa89802b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9b913bbfc990f09e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a1ffb640 1 -- 172.21.15.64:0/3825876277 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8980315a0 con 0x7fa8a415b230 2024-03-04T07:27:28.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a1ffb640 1 -- 172.21.15.64:0/3825876277 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa898037070 con 0x7fa8a415b230 2024-03-04T07:27:28.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a1ffb640 1 -- 172.21.15.64:0/3825876277 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa898031960 con 0x7fa8a415b230 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 -- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b230 msgr2=0x7fa8a4071160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b230 0x7fa8a4071160 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa8980097a0 tx=0x7fa89802b840 comp rx=0 tx=0).stop 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 -- 172.21.15.64:0/3825876277 shutdown_connections 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b230 0x7fa8a4071160 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3825876277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415ae50 0x7fa8a40708e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 -- 172.21.15.64:0/3825876277 >> 172.21.15.64:0/3825876277 conn(0x7fa8a4066d00 msgr2=0x7fa8a4067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 -- 172.21.15.64:0/3825876277 shutdown_connections 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.660+0000 7fa8a9678640 1 -- 172.21.15.64:0/3825876277 wait complete. 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 Processor -- start 2024-03-04T07:27:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 -- start start 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8a415ae50 0x7fa8a4140570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415b230 0x7fa8a40cd7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b690 0x7fa8a40cdd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8a40b3950 con 0x7fa8a415ae50 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8a40b3670 con 0x7fa8a415b690 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.662+0000 7fa8a9678640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa8a40b4150 con 0x7fa8a415b230 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8a415ae50 0x7fa8a4140570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8a415ae50 0x7fa8a4140570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51566/0 (socket says 172.21.15.64:51566) 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a37fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b690 0x7fa8a40cdd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 -- 172.21.15.64:0/3541760284 learned_addr learned my addr 172.21.15.64:0/3541760284 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a27fc640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415b230 0x7fa8a40cd7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 -- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415b230 msgr2=0x7fa8a40cd7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415b230 0x7fa8a40cd7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 -- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b690 msgr2=0x7fa8a40cdd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b690 0x7fa8a40cdd30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 -- 172.21.15.64:0/3541760284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8a41f51a0 con 0x7fa8a415ae50 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a37fe640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b690 0x7fa8a40cdd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a2ffd640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8a415ae50 0x7fa8a4140570 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fa89400ec10 tx=0x7fa89400c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa88b7fe640 1 -- 172.21.15.64:0/3541760284 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa894017070 con 0x7fa8a415ae50 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa88b7fe640 1 -- 172.21.15.64:0/3541760284 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa89400cc30 con 0x7fa8a415ae50 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa88b7fe640 1 -- 172.21.15.64:0/3541760284 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa894013470 con 0x7fa8a415ae50 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa8a41f43e0 con 0x7fa8a415ae50 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.663+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8a41f49a0 con 0x7fa8a415ae50 2024-03-04T07:27:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.664+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa8a40733c0 con 0x7fa8a415ae50 2024-03-04T07:27:28.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.665+0000 7fa88b7fe640 1 -- 172.21.15.64:0/3541760284 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa89401a070 con 0x7fa8a415ae50 2024-03-04T07:27:28.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.665+0000 7fa88b7fe640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa89006b4d0 0x7fa89006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.665+0000 7fa8a27fc640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa89006b4d0 0x7fa89006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.665+0000 7fa88b7fe640 1 -- 172.21.15.64:0/3541760284 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa89408a3f0 con 0x7fa8a415ae50 2024-03-04T07:27:28.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.666+0000 7fa8a27fc640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa89006b4d0 0x7fa89006d990 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fa898036000 tx=0x7fa89803d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.670+0000 7fa88b7fe640 1 -- 172.21.15.64:0/3541760284 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa894058cb0 con 0x7fa8a415ae50 2024-03-04T07:27:28.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.850+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fa8a404f410 con 0x7fa8a415ae50 2024-03-04T07:27:28.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.850+0000 7fa88b7fe640 1 -- 172.21.15.64:0/3541760284 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7fa89405cd50 con 0x7fa8a415ae50 2024-03-04T07:27:28.850 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa89006b4d0 msgr2=0x7fa89006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa89006b4d0 0x7fa89006d990 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fa898036000 tx=0x7fa89803d040 comp rx=0 tx=0).stop 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8a415ae50 msgr2=0x7fa8a4140570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8a415ae50 0x7fa8a4140570 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fa89400ec10 tx=0x7fa89400c520 comp rx=0 tx=0).stop 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 shutdown_connections 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa89006b4d0 0x7fa89006d990 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8a415b690 0x7fa8a40cdd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8a415b230 0x7fa8a40cd7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 --2- 172.21.15.64:0/3541760284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8a415ae50 0x7fa8a4140570 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 >> 172.21.15.64:0/3541760284 conn(0x7fa8a4066d00 msgr2=0x7fa8a4154210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:28.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 shutdown_connections 2024-03-04T07:27:28.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.853+0000 7fa8a9678640 1 -- 172.21.15.64:0/3541760284 wait complete. 2024-03-04T07:27:28.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:28.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.966+0000 7f9e6c90c640 1 Processor -- start 2024-03-04T07:27:28.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.966+0000 7f9e6c90c640 1 -- start start 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6c90c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9e6815e000 0x7f9e68071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6c90c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e680b2c50 0x7f9e68071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6c90c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9e680ce970 con 0x7f9e68161cb0 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6c90c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9e680cece0 con 0x7f9e6815e000 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6c90c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e680cef30 con 0x7f9e680b2c50 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6659c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9e6815e000 0x7f9e68071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e65d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e680b2c50 0x7f9e68071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e66d9d640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f9e68161cb0 0x7f9e68162140 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:40204/0 (socket says 172.21.15.64:40204) 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e66d9d640 1 -- 172.21.15.64:0/2246043056 learned_addr learned my addr 172.21.15.64:0/2246043056 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6659c640 1 -- 172.21.15.64:0/2246043056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e680b2c50 msgr2=0x7f9e68071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:28.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6659c640 1 --2- 172.21.15.64:0/2246043056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e680b2c50 0x7f9e68071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6659c640 1 -- 172.21.15.64:0/2246043056 >> v1:172.21.15.66:6789/0 conn(0x7f9e68161cb0 legacy=0x7f9e68162140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.967+0000 7f9e6659c640 1 -- 172.21.15.64:0/2246043056 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e680cf210 con 0x7f9e6815e000 2024-03-04T07:27:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.968+0000 7f9e6659c640 1 --2- 172.21.15.64:0/2246043056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e6815e000 0x7f9e68071540 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f9e600097a0 tx=0x7f9e6002b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cd008fd20bc56b79 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.968+0000 7f9e6559a640 1 -- 172.21.15.64:0/2246043056 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e600315a0 con 0x7f9e6815e000 2024-03-04T07:27:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.968+0000 7f9e6559a640 1 -- 172.21.15.64:0/2246043056 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e60037070 con 0x7f9e6815e000 2024-03-04T07:27:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.968+0000 7f9e6559a640 1 -- 172.21.15.64:0/2246043056 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e60031960 con 0x7f9e6815e000 2024-03-04T07:27:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 -- 172.21.15.64:0/2246043056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e6815e000 msgr2=0x7f9e68071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/2246043056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e6815e000 0x7f9e68071540 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f9e600097a0 tx=0x7f9e6002b840 comp rx=0 tx=0).stop 2024-03-04T07:27:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 -- 172.21.15.64:0/2246043056 shutdown_connections 2024-03-04T07:27:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/2246043056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e680b2c50 0x7f9e68071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/2246043056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e6815e000 0x7f9e68071540 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 -- 172.21.15.64:0/2246043056 >> 172.21.15.64:0/2246043056 conn(0x7f9e680d3c80 msgr2=0x7f9e680d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 -- 172.21.15.64:0/2246043056 shutdown_connections 2024-03-04T07:27:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.969+0000 7f9e6c90c640 1 -- 172.21.15.64:0/2246043056 wait complete. 2024-03-04T07:27:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.970+0000 7f9e6c90c640 1 Processor -- start 2024-03-04T07:27:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.970+0000 7f9e6c90c640 1 -- start start 2024-03-04T07:27:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e6c90c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e680b2c50 0x7f9e680b35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e6c90c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e6815e000 0x7f9e681e8d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e6c90c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e68161cb0 0x7f9e68143ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e6c90c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e680b3320 con 0x7f9e68161cb0 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e6c90c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e680b31a0 con 0x7f9e680b2c50 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e6c90c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e6815e3e0 con 0x7f9e6815e000 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e66d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e68161cb0 0x7f9e68143ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e66d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e68161cb0 0x7f9e68143ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51570/0 (socket says 172.21.15.64:51570) 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e6659c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e680b2c50 0x7f9e680b35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e66d9d640 1 -- 172.21.15.64:0/1414584479 learned_addr learned my addr 172.21.15.64:0/1414584479 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.971+0000 7f9e65d9b640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e6815e000 0x7f9e681e8d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6659c640 1 -- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e6815e000 msgr2=0x7f9e681e8d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6659c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e6815e000 0x7f9e681e8d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6659c640 1 -- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e68161cb0 msgr2=0x7f9e68143ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6659c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e68161cb0 0x7f9e68143ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6659c640 1 -- 172.21.15.64:0/1414584479 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e680b3b20 con 0x7f9e680b2c50 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e66d9d640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e68161cb0 0x7f9e68143ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:28.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6659c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e680b2c50 0x7f9e680b35e0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f9e6806d100 tx=0x7f9e600046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e56ffd640 1 -- 172.21.15.64:0/1414584479 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e60045640 con 0x7f9e680b2c50 2024-03-04T07:27:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e56ffd640 1 -- 172.21.15.64:0/1414584479 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e60045800 con 0x7f9e680b2c50 2024-03-04T07:27:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e680cda30 con 0x7f9e680b2c50 2024-03-04T07:27:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e56ffd640 1 -- 172.21.15.64:0/1414584479 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e60045bf0 con 0x7f9e680b2c50 2024-03-04T07:27:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.972+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e681e92b0 con 0x7f9e680b2c50 2024-03-04T07:27:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.973+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9e68067ff0 con 0x7f9e680b2c50 2024-03-04T07:27:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.973+0000 7f9e56ffd640 1 -- 172.21.15.64:0/1414584479 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9e6003b3f0 con 0x7f9e680b2c50 2024-03-04T07:27:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.974+0000 7f9e56ffd640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e4806b4d0 0x7f9e4806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.974+0000 7f9e56ffd640 1 -- 172.21.15.64:0/1414584479 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9e6002f070 con 0x7f9e680b2c50 2024-03-04T07:27:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.974+0000 7f9e65d9b640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e4806b4d0 0x7f9e4806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.974+0000 7f9e65d9b640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e4806b4d0 0x7f9e4806d990 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f9e680b9240 tx=0x7f9e5c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:28.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:28.978+0000 7f9e56ffd640 1 -- 172.21.15.64:0/1414584479 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9e60040070 con 0x7f9e680b2c50 2024-03-04T07:27:29.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.171+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f9e680b8670 con 0x7f9e680b2c50 2024-03-04T07:27:29.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.172+0000 7f9e56ffd640 1 -- 172.21.15.64:0/1414584479 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v20) v1 ==== 80+0+0 (secure 0 0 0) 0x7f9e60079f80 con 0x7f9e680b2c50 2024-03-04T07:27:29.171 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:29.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e4806b4d0 msgr2=0x7f9e4806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e4806b4d0 0x7f9e4806d990 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f9e680b9240 tx=0x7f9e5c009190 comp rx=0 tx=0).stop 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e680b2c50 msgr2=0x7f9e680b35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e680b2c50 0x7f9e680b35e0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f9e6806d100 tx=0x7f9e600046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 shutdown_connections 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e4806b4d0 0x7f9e4806d990 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e68161cb0 0x7f9e68143ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e6815e000 0x7f9e681e8d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 --2- 172.21.15.64:0/1414584479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e680b2c50 0x7f9e680b35e0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 >> 172.21.15.64:0/1414584479 conn(0x7f9e680d3c80 msgr2=0x7f9e680d1aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 shutdown_connections 2024-03-04T07:27:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.174+0000 7f9e6c90c640 1 -- 172.21.15.64:0/1414584479 wait complete. 2024-03-04T07:27:29.184 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:27:29.184 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:29.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.341+0000 7f65d6fc7640 1 Processor -- start 2024-03-04T07:27:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.341+0000 7f65d6fc7640 1 -- start start 2024-03-04T07:27:29.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.341+0000 7f65d6fc7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f65d0071c50 0x7f65d00d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.341+0000 7f65d6fc7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f65d00b2c50 0x7f65d00d2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.341+0000 7f65d6fc7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f65d00ce970 con 0x7f65d015e000 2024-03-04T07:27:29.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.341+0000 7f65d6fc7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f65d00cece0 con 0x7f65d00b2c50 2024-03-04T07:27:29.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.341+0000 7f65d6fc7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f65d00cef30 con 0x7f65d0071c50 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65d5564640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f65d00b2c50 0x7f65d00d2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65d5564640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f65d00b2c50 0x7f65d00d2b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51586/0 (socket says 172.21.15.64:51586) 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65d5564640 1 -- 172.21.15.64:0/941852442 learned_addr learned my addr 172.21.15.64:0/941852442 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65cffff640 1 --2- 172.21.15.64:0/941852442 >> v2:172.21.15.66:3300/0 conn(0x7f65d0071c50 0x7f65d00d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65d5564640 1 -- 172.21.15.64:0/941852442 >> v2:172.21.15.66:3300/0 conn(0x7f65d0071c50 msgr2=0x7f65d00d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65d5564640 1 --2- 172.21.15.64:0/941852442 >> v2:172.21.15.66:3300/0 conn(0x7f65d0071c50 0x7f65d00d25d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65d5564640 1 -- 172.21.15.64:0/941852442 >> v1:172.21.15.66:6789/0 conn(0x7f65d015e000 legacy=0x7f65d0071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.342+0000 7f65d5564640 1 -- 172.21.15.64:0/941852442 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65d00cf210 con 0x7f65d00b2c50 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d5564640 1 --2- 172.21.15.64:0/941852442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00d2b10 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f65d008c990 tx=0x7f65c802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d5916ec627bcc2ca server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65cf7fe640 1 -- 172.21.15.64:0/941852442 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65c802ca80 con 0x7f65d00b2c50 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65cf7fe640 1 -- 172.21.15.64:0/941852442 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f65c802cc20 con 0x7f65d00b2c50 2024-03-04T07:27:29.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65cf7fe640 1 -- 172.21.15.64:0/941852442 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65c80343f0 con 0x7f65d00b2c50 2024-03-04T07:27:29.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 -- 172.21.15.64:0/941852442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 msgr2=0x7f65d00d2b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/941852442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00d2b10 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f65d008c990 tx=0x7f65c802bee0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 -- 172.21.15.64:0/941852442 shutdown_connections 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/941852442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00d2b10 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/941852442 >> v2:172.21.15.66:3300/0 conn(0x7f65d0071c50 0x7f65d00d25d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 -- 172.21.15.64:0/941852442 >> 172.21.15.64:0/941852442 conn(0x7f65d00d3c80 msgr2=0x7f65d00d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 -- 172.21.15.64:0/941852442 shutdown_connections 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.343+0000 7f65d6fc7640 1 -- 172.21.15.64:0/941852442 wait complete. 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.345+0000 7f65d6fc7640 1 Processor -- start 2024-03-04T07:27:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.345+0000 7f65d6fc7640 1 -- start start 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d6fc7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65d0071c50 0x7f65d015c910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d6fc7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00acf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d6fc7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65d015e000 0x7f65d00ad490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d6fc7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65d00b3320 con 0x7f65d00b2c50 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d6fc7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65d00b31a0 con 0x7f65d015e000 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d6fc7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f65d015e3e0 con 0x7f65d0071c50 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00acf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00acf50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51598/0 (socket says 172.21.15.64:51598) 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 -- 172.21.15.64:0/685292822 learned_addr learned my addr 172.21.15.64:0/685292822 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d4d63640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65d0071c50 0x7f65d015c910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d5564640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65d015e000 0x7f65d00ad490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 -- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65d0071c50 msgr2=0x7f65d015c910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65d0071c50 0x7f65d015c910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 -- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65d015e000 msgr2=0x7f65d00ad490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65d015e000 0x7f65d00ad490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 -- 172.21.15.64:0/685292822 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65d00cd6a0 con 0x7f65d00b2c50 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cffff640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00acf50 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f65bc00e870 tx=0x7f65bc00ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cd7fa640 1 -- 172.21.15.64:0/685292822 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65bc017070 con 0x7f65d00b2c50 2024-03-04T07:27:29.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65cd7fa640 1 -- 172.21.15.64:0/685292822 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f65bc00c940 con 0x7f65d00b2c50 2024-03-04T07:27:29.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.346+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65d00cd980 con 0x7f65d00b2c50 2024-03-04T07:27:29.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.347+0000 7f65cd7fa640 1 -- 172.21.15.64:0/685292822 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65bc00cd30 con 0x7f65d00b2c50 2024-03-04T07:27:29.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.347+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65d015ce50 con 0x7f65d00b2c50 2024-03-04T07:27:29.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.348+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f65d0067ff0 con 0x7f65d00b2c50 2024-03-04T07:27:29.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.348+0000 7f65cd7fa640 1 -- 172.21.15.64:0/685292822 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f65bc01a070 con 0x7f65d00b2c50 2024-03-04T07:27:29.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.348+0000 7f65cd7fa640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f65b406b4d0 0x7f65b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.348+0000 7f65cd7fa640 1 -- 172.21.15.64:0/685292822 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f65bc08a2e0 con 0x7f65d00b2c50 2024-03-04T07:27:29.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.349+0000 7f65d4d63640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f65b406b4d0 0x7f65b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.349+0000 7f65d4d63640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f65b406b4d0 0x7f65b406d990 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f65c4001780 tx=0x7f65c4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.353+0000 7f65cd7fa640 1 -- 172.21.15.64:0/685292822 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f65bc058ba0 con 0x7f65d00b2c50 2024-03-04T07:27:29.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.533+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f65d0071540 con 0x7f65d00b2c50 2024-03-04T07:27:29.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.534+0000 7f65cd7fa640 1 -- 172.21.15.64:0/685292822 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f65bc05cc40 con 0x7f65d00b2c50 2024-03-04T07:27:29.535 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:29.535 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f65b406b4d0 msgr2=0x7f65b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f65b406b4d0 0x7f65b406d990 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f65c4001780 tx=0x7f65c4009040 comp rx=0 tx=0).stop 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 msgr2=0x7f65d00acf50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00acf50 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f65bc00e870 tx=0x7f65bc00ed30 comp rx=0 tx=0).stop 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 shutdown_connections 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f65b406b4d0 0x7f65b406d990 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65d015e000 0x7f65d00ad490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65d00b2c50 0x7f65d00acf50 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 --2- 172.21.15.64:0/685292822 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65d0071c50 0x7f65d015c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 >> 172.21.15.64:0/685292822 conn(0x7f65d00d3c80 msgr2=0x7f65d00cf730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 shutdown_connections 2024-03-04T07:27:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.537+0000 7f65d6fc7640 1 -- 172.21.15.64:0/685292822 wait complete. 2024-03-04T07:27:29.548 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:29.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:29.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.652+0000 7f6ee82c0640 1 Processor -- start 2024-03-04T07:27:29.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.652+0000 7f6ee82c0640 1 -- start start 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.653+0000 7f6ee82c0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6ee015dd90 0x7f6ee00ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.653+0000 7f6ee82c0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00b6d10 0x7f6ee00b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.653+0000 7f6ee82c0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6ee00d3740 con 0x7f6ee00d29d0 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.653+0000 7f6ee82c0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ee00d3400 con 0x7f6ee015dd90 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.653+0000 7f6ee82c0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ee00d3120 con 0x7f6ee00b6d10 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee605c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6ee015dd90 0x7f6ee00ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee585b640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f6ee00d29d0 0x7f6ee00d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:40228/0 (socket says 172.21.15.64:40228) 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee685d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00b6d10 0x7f6ee00b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee585b640 1 -- 172.21.15.64:0/822687589 learned_addr learned my addr 172.21.15.64:0/822687589 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee605c640 1 -- 172.21.15.64:0/822687589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00b6d10 msgr2=0x7f6ee00b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee605c640 1 --2- 172.21.15.64:0/822687589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00b6d10 0x7f6ee00b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee605c640 1 -- 172.21.15.64:0/822687589 >> v1:172.21.15.66:6789/0 conn(0x7f6ee00d29d0 legacy=0x7f6ee00d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.654+0000 7f6ee605c640 1 -- 172.21.15.64:0/822687589 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ee00d39e0 con 0x7f6ee015dd90 2024-03-04T07:27:29.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee605c640 1 --2- 172.21.15.64:0/822687589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee015dd90 0x7f6ee00ceea0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f6ed80097a0 tx=0x7f6ed802b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5ddb6f92eb87e05d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee505a640 1 -- 172.21.15.64:0/822687589 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ed8034070 con 0x7f6ee015dd90 2024-03-04T07:27:29.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee505a640 1 -- 172.21.15.64:0/822687589 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6ed8030610 con 0x7f6ee015dd90 2024-03-04T07:27:29.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee505a640 1 -- 172.21.15.64:0/822687589 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ed80309e0 con 0x7f6ee015dd90 2024-03-04T07:27:29.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 -- 172.21.15.64:0/822687589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee015dd90 msgr2=0x7f6ee00ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/822687589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee015dd90 0x7f6ee00ceea0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f6ed80097a0 tx=0x7f6ed802b960 comp rx=0 tx=0).stop 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 -- 172.21.15.64:0/822687589 shutdown_connections 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/822687589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00b6d10 0x7f6ee00b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/822687589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee015dd90 0x7f6ee00ceea0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 -- 172.21.15.64:0/822687589 >> 172.21.15.64:0/822687589 conn(0x7f6ee0071960 msgr2=0x7f6ee0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 -- 172.21.15.64:0/822687589 shutdown_connections 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.655+0000 7f6ee82c0640 1 -- 172.21.15.64:0/822687589 wait complete. 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.656+0000 7f6ee82c0640 1 Processor -- start 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee82c0640 1 -- start start 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee82c0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee00b6d10 0x7f6ee0155910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee82c0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00d29d0 0x7f6ee015e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee82c0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ee015dd90 0x7f6ee015e830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee82c0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ee00d2df0 con 0x7f6ee015dd90 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee82c0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ee015e170 con 0x7f6ee00b6d10 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee82c0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ee00d48f0 con 0x7f6ee00d29d0 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee685d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ee015dd90 0x7f6ee015e830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee585b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00d29d0 0x7f6ee015e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee685d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ee015dd90 0x7f6ee015e830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51600/0 (socket says 172.21.15.64:51600) 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee685d640 1 -- 172.21.15.64:0/1099599370 learned_addr learned my addr 172.21.15.64:0/1099599370 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.657+0000 7f6ee605c640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee00b6d10 0x7f6ee0155910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee685d640 1 -- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00d29d0 msgr2=0x7f6ee015e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee685d640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00d29d0 0x7f6ee015e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee685d640 1 -- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee00b6d10 msgr2=0x7f6ee0155910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee685d640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee00b6d10 0x7f6ee0155910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee685d640 1 -- 172.21.15.64:0/1099599370 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ee0155e50 con 0x7f6ee015dd90 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee585b640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00d29d0 0x7f6ee015e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee605c640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee00b6d10 0x7f6ee0155910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee685d640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ee015dd90 0x7f6ee015e830 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f6edc00efc0 tx=0x7f6edc00c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ed6ffd640 1 -- 172.21.15.64:0/1099599370 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6edc018020 con 0x7f6ee015dd90 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ed6ffd640 1 -- 172.21.15.64:0/1099599370 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6edc014480 con 0x7f6ee015dd90 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ed6ffd640 1 -- 172.21.15.64:0/1099599370 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6edc014870 con 0x7f6ee015dd90 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ee01527d0 con 0x7f6ee015dd90 2024-03-04T07:27:29.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.658+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ee0160a60 con 0x7f6ee015dd90 2024-03-04T07:27:29.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.659+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ee0072a30 con 0x7f6ee015dd90 2024-03-04T07:27:29.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.659+0000 7f6ed6ffd640 1 -- 172.21.15.64:0/1099599370 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6edc005d10 con 0x7f6ee015dd90 2024-03-04T07:27:29.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.660+0000 7f6ed6ffd640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b4d0 0x7f6ec406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.660+0000 7f6ed6ffd640 1 -- 172.21.15.64:0/1099599370 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6edc08ae70 con 0x7f6ee015dd90 2024-03-04T07:27:29.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.660+0000 7f6ee605c640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b4d0 0x7f6ec406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.661+0000 7f6ee605c640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b4d0 0x7f6ec406d990 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f6ed8036000 tx=0x7f6ed803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.664+0000 7f6ed6ffd640 1 -- 172.21.15.64:0/1099599370 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6edc022030 con 0x7f6ee015dd90 2024-03-04T07:27:29.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.844+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f6ee004f410 con 0x7f6ee015dd90 2024-03-04T07:27:29.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.844+0000 7f6ed6ffd640 1 -- 172.21.15.64:0/1099599370 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f6edc010040 con 0x7f6ee015dd90 2024-03-04T07:27:29.845 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:29.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b4d0 msgr2=0x7f6ec406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b4d0 0x7f6ec406d990 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f6ed8036000 tx=0x7f6ed803b040 comp rx=0 tx=0).stop 2024-03-04T07:27:29.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ee015dd90 msgr2=0x7f6ee015e830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ee015dd90 0x7f6ee015e830 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f6edc00efc0 tx=0x7f6edc00c490 comp rx=0 tx=0).stop 2024-03-04T07:27:29.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 shutdown_connections 2024-03-04T07:27:29.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b4d0 0x7f6ec406d990 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ee015dd90 0x7f6ee015e830 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ee00d29d0 0x7f6ee015e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 --2- 172.21.15.64:0/1099599370 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ee00b6d10 0x7f6ee0155910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 >> 172.21.15.64:0/1099599370 conn(0x7f6ee0071960 msgr2=0x7f6ee00d0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:29.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 shutdown_connections 2024-03-04T07:27:29.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.846+0000 7f6ee82c0640 1 -- 172.21.15.64:0/1099599370 wait complete. 2024-03-04T07:27:29.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:29.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.962+0000 7fa38f73a640 1 Processor -- start 2024-03-04T07:27:29.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.962+0000 7fa38f73a640 1 -- start start 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38f73a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38f73a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa3880b2b10 0x7fa3880d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38f73a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa38815e4b0 con 0x7fa3880b3060 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38f73a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa3880d2e10 con 0x7fa3880b2b10 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38f73a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa3880b97e0 con 0x7fa3880717b0 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38ccd5640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa3880b3060 0x7fa3880b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48612/0 (socket says 172.21.15.64:48612) 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38ccd5640 1 -- 172.21.15.64:0/1971627238 learned_addr learned my addr 172.21.15.64:0/1971627238 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38dcd7640 1 --2- 172.21.15.64:0/1971627238 >> v2:172.21.15.66:3300/0 conn(0x7fa3880b2b10 0x7fa3880d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38d4d6640 1 --2- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa37ffff640 1 -- 172.21.15.64:0/1971627238 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3300898164 0 0) 0x7fa378001110 con 0x7fa3880b3060 2024-03-04T07:27:29.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38dcd7640 1 -- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 msgr2=0x7fa388071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:29.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38dcd7640 1 --2- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38dcd7640 1 -- 172.21.15.64:0/1971627238 >> v1:172.21.15.64:6789/0 conn(0x7fa3880b3060 legacy=0x7fa3880b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.963+0000 7fa38dcd7640 1 -- 172.21.15.64:0/1971627238 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3880b9a80 con 0x7fa3880b2b10 2024-03-04T07:27:29.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.964+0000 7fa38dcd7640 1 --2- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 0x7fa3880d32a0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fa380009d90 tx=0x7fa38002bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7be3cd68e1f7e2ad server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.964+0000 7fa37ffff640 1 -- 172.21.15.64:0/1971627238 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa380039070 con 0x7fa3880b2b10 2024-03-04T07:27:29.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.964+0000 7fa37ffff640 1 -- 172.21.15.64:0/1971627238 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa38002cc50 con 0x7fa3880b2b10 2024-03-04T07:27:29.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.964+0000 7fa37ffff640 1 -- 172.21.15.64:0/1971627238 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa380034440 con 0x7fa3880b2b10 2024-03-04T07:27:29.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 -- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 msgr2=0x7fa3880d32a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 0x7fa3880d32a0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fa380009d90 tx=0x7fa38002bea0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 -- 172.21.15.64:0/1971627238 shutdown_connections 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 0x7fa3880d32a0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1971627238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 -- 172.21.15.64:0/1971627238 >> 172.21.15.64:0/1971627238 conn(0x7fa3880d3d20 msgr2=0x7fa3880d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 -- 172.21.15.64:0/1971627238 shutdown_connections 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.965+0000 7fa38f73a640 1 -- 172.21.15.64:0/1971627238 wait complete. 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.966+0000 7fa38f73a640 1 Processor -- start 2024-03-04T07:27:29.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.966+0000 7fa38f73a640 1 -- start start 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38f73a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388149ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38f73a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 0x7fa3881e34f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38f73a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3880b3060 0x7fa3881593f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38f73a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3880b7560 con 0x7fa3880b3060 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38f73a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3880d4bf0 con 0x7fa3880b2b10 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38f73a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa3880b76e0 con 0x7fa3880717b0 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3880b3060 0x7fa3881593f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3880b3060 0x7fa3881593f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51616/0 (socket says 172.21.15.64:51616) 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38d4d6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388149ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 -- 172.21.15.64:0/1946227530 learned_addr learned my addr 172.21.15.64:0/1946227530 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38ccd5640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 0x7fa3881e34f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 -- 172.21.15.64:0/1946227530 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 msgr2=0x7fa388149ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388149ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 -- 172.21.15.64:0/1946227530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 msgr2=0x7fa3881e34f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 0x7fa3881e34f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 -- 172.21.15.64:0/1946227530 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3881e3a30 con 0x7fa3880b3060 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38dcd7640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3880b3060 0x7fa3881593f0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7fa380009d90 tx=0x7fa380004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa37dffb640 1 -- 172.21.15.64:0/1946227530 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa380044020 con 0x7fa3880b3060 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa37dffb640 1 -- 172.21.15.64:0/1946227530 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa38003a040 con 0x7fa3880b3060 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa37dffb640 1 -- 172.21.15.64:0/1946227530 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa380039050 con 0x7fa3880b3060 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.967+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa38814a0b0 con 0x7fa3880b3060 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.968+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa38814a670 con 0x7fa3880b3060 2024-03-04T07:27:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.969+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa38806ce40 con 0x7fa3880b3060 2024-03-04T07:27:29.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.969+0000 7fa37dffb640 1 -- 172.21.15.64:0/1946227530 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa380003990 con 0x7fa3880b3060 2024-03-04T07:27:29.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.969+0000 7fa37dffb640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa36c06b4d0 0x7fa36c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:29.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.970+0000 7fa38d4d6640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa36c06b4d0 0x7fa36c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:29.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.970+0000 7fa37dffb640 1 -- 172.21.15.64:0/1946227530 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa3800ac4a0 con 0x7fa3880b3060 2024-03-04T07:27:29.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.970+0000 7fa38d4d6640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa36c06b4d0 0x7fa36c06d990 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fa384009b40 tx=0x7fa384009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:29.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:29.974+0000 7fa37dffb640 1 -- 172.21.15.64:0/1946227530 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa38004e060 con 0x7fa3880b3060 2024-03-04T07:27:30.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.153+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fa3880d4580 con 0x7fa3880b3060 2024-03-04T07:27:30.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.153+0000 7fa37dffb640 1 -- 172.21.15.64:0/1946227530 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7fa38007ad60 con 0x7fa3880b3060 2024-03-04T07:27:30.154 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa36c06b4d0 msgr2=0x7fa36c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa36c06b4d0 0x7fa36c06d990 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fa384009b40 tx=0x7fa384009190 comp rx=0 tx=0).stop 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3880b3060 msgr2=0x7fa3881593f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3880b3060 0x7fa3881593f0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7fa380009d90 tx=0x7fa380004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 shutdown_connections 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa36c06b4d0 0x7fa36c06d990 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3880b3060 0x7fa3881593f0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3880b2b10 0x7fa3881e34f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 --2- 172.21.15.64:0/1946227530 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3880717b0 0x7fa388149ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 >> 172.21.15.64:0/1946227530 conn(0x7fa3880d3d20 msgr2=0x7fa38815d020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 shutdown_connections 2024-03-04T07:27:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.155+0000 7fa38f73a640 1 -- 172.21.15.64:0/1946227530 wait complete. 2024-03-04T07:27:30.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:30.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.270+0000 7f01fa51e640 1 Processor -- start 2024-03-04T07:27:30.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.270+0000 7f01fa51e640 1 -- start start 2024-03-04T07:27:30.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.270+0000 7f01fa51e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f01f415dd50 0x7f01f40ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01fa51e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f01f4072cf0 con 0x7f01f40cef20 2024-03-04T07:27:30.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01fa51e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f01f4072fd0 con 0x7f01f4072680 2024-03-04T07:27:30.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01fa51e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f01f4072a90 con 0x7f01f415dd50 2024-03-04T07:27:30.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f37fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f01f40cef20 0x7f01f4071f70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48618/0 (socket says 172.21.15.64:48618) 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f37fe640 1 -- 172.21.15.64:0/2535036606 learned_addr learned my addr 172.21.15.64:0/2535036606 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f3fff640 1 --2- 172.21.15.64:0/2535036606 >> v2:172.21.15.66:3300/0 conn(0x7f01f415dd50 0x7f01f40ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f2ffd640 1 -- 172.21.15.64:0/2535036606 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3181405480 0 0) 0x7f01f4072cf0 con 0x7f01f40cef20 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f2ffd640 1 -- 172.21.15.64:0/2535036606 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f01dc003520 con 0x7f01f40cef20 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f2ffd640 1 -- 172.21.15.64:0/2535036606 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1858797720 0 0) 0x7f01f4072fd0 con 0x7f01f4072680 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f2ffd640 1 -- 172.21.15.64:0/2535036606 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f01f4072cf0 con 0x7f01f4072680 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f3fff640 1 -- 172.21.15.64:0/2535036606 >> v1:172.21.15.66:6789/0 conn(0x7f01f4072680 legacy=0x7f01f40d2760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f3fff640 1 -- 172.21.15.64:0/2535036606 >> v1:172.21.15.64:6789/0 conn(0x7f01f40cef20 legacy=0x7f01f4071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.271+0000 7f01f3fff640 1 -- 172.21.15.64:0/2535036606 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01f40d1f60 con 0x7f01f415dd50 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.272+0000 7f01f3fff640 1 --2- 172.21.15.64:0/2535036606 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f415dd50 0x7f01f40ce9e0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f01e40097a0 tx=0x7f01e402b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8695e9bb1df296a6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.272+0000 7f01f2ffd640 1 -- 172.21.15.64:0/2535036606 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01e4035020 con 0x7f01f415dd50 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.272+0000 7f01f2ffd640 1 -- 172.21.15.64:0/2535036606 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f01e4031830 con 0x7f01f415dd50 2024-03-04T07:27:30.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.272+0000 7f01f2ffd640 1 -- 172.21.15.64:0/2535036606 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01e4031be0 con 0x7f01f415dd50 2024-03-04T07:27:30.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.273+0000 7f01fa51e640 1 -- 172.21.15.64:0/2535036606 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f415dd50 msgr2=0x7f01f40ce9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.273+0000 7f01fa51e640 1 --2- 172.21.15.64:0/2535036606 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f415dd50 0x7f01f40ce9e0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f01e40097a0 tx=0x7f01e402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.273+0000 7f01fa51e640 1 -- 172.21.15.64:0/2535036606 shutdown_connections 2024-03-04T07:27:30.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.273+0000 7f01fa51e640 1 --2- 172.21.15.64:0/2535036606 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f415dd50 0x7f01f40ce9e0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.273+0000 7f01fa51e640 1 -- 172.21.15.64:0/2535036606 >> 172.21.15.64:0/2535036606 conn(0x7f01f40d3ef0 msgr2=0x7f01f40d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:30.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.273+0000 7f01fa51e640 1 -- 172.21.15.64:0/2535036606 shutdown_connections 2024-03-04T07:27:30.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.273+0000 7f01fa51e640 1 -- 172.21.15.64:0/2535036606 wait complete. 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.274+0000 7f01fa51e640 1 Processor -- start 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.274+0000 7f01fa51e640 1 -- start start 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f4072680 0x7f01f40c9800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01f40cef20 0x7f01f4155d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01f415dd50 0x7f01f4160f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01f415e2b0 con 0x7f01f415dd50 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01f415e130 con 0x7f01f4072680 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f01f40d2e70 con 0x7f01f40cef20 2024-03-04T07:27:30.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f4072680 0x7f01f40c9800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f8abb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01f415dd50 0x7f01f4160f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f4072680 0x7f01f40c9800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:56340/0 (socket says 172.21.15.64:56340) 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 -- 172.21.15.64:0/351320271 learned_addr learned my addr 172.21.15.64:0/351320271 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f37fe640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01f40cef20 0x7f01f4155d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 -- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01f40cef20 msgr2=0x7f01f4155d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01f40cef20 0x7f01f4155d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 -- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01f415dd50 msgr2=0x7f01f4160f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01f415dd50 0x7f01f4160f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 -- 172.21.15.64:0/351320271 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01f4154c90 con 0x7f01f4072680 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f3fff640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f4072680 0x7f01f40c9800 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f01e402b910 tx=0x7f01e40046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f0ff9640 1 -- 172.21.15.64:0/351320271 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01e4040070 con 0x7f01f4072680 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f0ff9640 1 -- 172.21.15.64:0/351320271 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f01e403a8f0 con 0x7f01f4072680 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01f4154f70 con 0x7f01f4072680 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01f0ff9640 1 -- 172.21.15.64:0/351320271 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01e4035040 con 0x7f01f4072680 2024-03-04T07:27:30.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.275+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f01f4155530 con 0x7f01f4072680 2024-03-04T07:27:30.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.276+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f01f406cdd0 con 0x7f01f4072680 2024-03-04T07:27:30.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.277+0000 7f01f0ff9640 1 -- 172.21.15.64:0/351320271 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f01e4038020 con 0x7f01f4072680 2024-03-04T07:27:30.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.277+0000 7f01f0ff9640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01e006b4d0 0x7f01e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.277+0000 7f01f0ff9640 1 -- 172.21.15.64:0/351320271 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f01e40ab940 con 0x7f01f4072680 2024-03-04T07:27:30.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.277+0000 7f01f37fe640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01e006b4d0 0x7f01e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.278+0000 7f01f37fe640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01e006b4d0 0x7f01e006d990 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f01f4160260 tx=0x7f01e8007d30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:30.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.287+0000 7f01f0ff9640 1 -- 172.21.15.64:0/351320271 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f01e407a0f0 con 0x7f01f4072680 2024-03-04T07:27:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.467+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f01f40d1970 con 0x7f01f4072680 2024-03-04T07:27:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.467+0000 7f01f0ff9640 1 -- 172.21.15.64:0/351320271 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7f01e407e190 con 0x7f01f4072680 2024-03-04T07:27:30.469 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01e006b4d0 msgr2=0x7f01e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01e006b4d0 0x7f01e006d990 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f01f4160260 tx=0x7f01e8007d30 comp rx=0 tx=0).stop 2024-03-04T07:27:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f4072680 msgr2=0x7f01f40c9800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f4072680 0x7f01f40c9800 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f01e402b910 tx=0x7f01e40046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 shutdown_connections 2024-03-04T07:27:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01e006b4d0 0x7f01e006d990 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01f415dd50 0x7f01f4160f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01f40cef20 0x7f01f4155d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 --2- 172.21.15.64:0/351320271 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01f4072680 0x7f01f40c9800 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 >> 172.21.15.64:0/351320271 conn(0x7f01f40d3ef0 msgr2=0x7f01f415ccf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 shutdown_connections 2024-03-04T07:27:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.469+0000 7f01fa51e640 1 -- 172.21.15.64:0/351320271 wait complete. 2024-03-04T07:27:30.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:30.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.582+0000 7f556089a640 1 Processor -- start 2024-03-04T07:27:30.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.582+0000 7f556089a640 1 -- start start 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f556089a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f555c06af20 0x7f555c15d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f556089a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f555c0cc0e0 con 0x7f555c0b1020 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f556089a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f555c0cc3c0 con 0x7f555c06ed10 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f556089a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f555c0d28b0 con 0x7f555c06af20 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f555ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f555c06af20 0x7f555c15d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f555a59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f555c0b1020 0x7f555c0cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48624/0 (socket says 172.21.15.64:48624) 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f555a59c640 1 -- 172.21.15.64:0/792844191 learned_addr learned my addr 172.21.15.64:0/792844191 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:30.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f555ad9d640 1 -- 172.21.15.64:0/792844191 >> v1:172.21.15.66:6789/0 conn(0x7f555c06ed10 legacy=0x7f555c0cd890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f555ad9d640 1 -- 172.21.15.64:0/792844191 >> v1:172.21.15.64:6789/0 conn(0x7f555c0b1020 legacy=0x7f555c0cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.583+0000 7f555ad9d640 1 -- 172.21.15.64:0/792844191 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f555c0cc6a0 con 0x7f555c06af20 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.584+0000 7f555ad9d640 1 --2- 172.21.15.64:0/792844191 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c06af20 0x7f555c15d700 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f554c009d90 tx=0x7f554c02bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c482c193b6f418a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.584+0000 7f555959a640 1 -- 172.21.15.64:0/792844191 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f554c039070 con 0x7f555c06af20 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.584+0000 7f555959a640 1 -- 172.21.15.64:0/792844191 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f554c02cd00 con 0x7f555c06af20 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.584+0000 7f555959a640 1 -- 172.21.15.64:0/792844191 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f554c034660 con 0x7f555c06af20 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.585+0000 7f556089a640 1 -- 172.21.15.64:0/792844191 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c06af20 msgr2=0x7f555c15d700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.585+0000 7f556089a640 1 --2- 172.21.15.64:0/792844191 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c06af20 0x7f555c15d700 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f554c009d90 tx=0x7f554c02bf50 comp rx=0 tx=0).stop 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.585+0000 7f556089a640 1 -- 172.21.15.64:0/792844191 shutdown_connections 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.585+0000 7f556089a640 1 --2- 172.21.15.64:0/792844191 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c06af20 0x7f555c15d700 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.585+0000 7f556089a640 1 -- 172.21.15.64:0/792844191 >> 172.21.15.64:0/792844191 conn(0x7f555c0d1a90 msgr2=0x7f555c0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:30.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.585+0000 7f556089a640 1 -- 172.21.15.64:0/792844191 shutdown_connections 2024-03-04T07:27:30.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.585+0000 7f556089a640 1 -- 172.21.15.64:0/792844191 wait complete. 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.586+0000 7f556089a640 1 Processor -- start 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.586+0000 7f556089a640 1 -- start start 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f556089a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555c06af20 0x7f555c0b5560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f556089a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555c06ed10 0x7f555c15fc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f556089a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c0b1020 0x7f555c1601d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f556089a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f555c0ccc60 con 0x7f555c0b1020 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f556089a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f555c0cf320 con 0x7f555c06af20 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f556089a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f555c15dc40 con 0x7f555c06ed10 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c0b1020 0x7f555c1601d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c0b1020 0x7f555c1601d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51646/0 (socket says 172.21.15.64:51646) 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 -- 172.21.15.64:0/1446373303 learned_addr learned my addr 172.21.15.64:0/1446373303 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:30.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555a59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555c06af20 0x7f555c0b5560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f5559d9b640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555c06ed10 0x7f555c15fc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 -- 172.21.15.64:0/1446373303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555c06ed10 msgr2=0x7f555c15fc90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555c06ed10 0x7f555c15fc90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 -- 172.21.15.64:0/1446373303 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555c06af20 msgr2=0x7f555c0b5560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555c06af20 0x7f555c0b5560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.587+0000 7f555ad9d640 1 -- 172.21.15.64:0/1446373303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f555c0b7610 con 0x7f555c0b1020 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.588+0000 7f555ad9d640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c0b1020 0x7f555c1601d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f554c009d90 tx=0x7f554c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.588+0000 7f554affd640 1 -- 172.21.15.64:0/1446373303 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f554c039050 con 0x7f555c0b1020 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.588+0000 7f554affd640 1 -- 172.21.15.64:0/1446373303 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f554c045070 con 0x7f555c0b1020 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.588+0000 7f554affd640 1 -- 172.21.15.64:0/1446373303 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f554c03e830 con 0x7f555c0b1020 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.588+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f555c160800 con 0x7f555c0b1020 2024-03-04T07:27:30.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.588+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f555c0b2690 con 0x7f555c0b1020 2024-03-04T07:27:30.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.589+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f555c067ff0 con 0x7f555c0b1020 2024-03-04T07:27:30.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.589+0000 7f554affd640 1 -- 172.21.15.64:0/1446373303 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f554c03e9d0 con 0x7f555c0b1020 2024-03-04T07:27:30.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.590+0000 7f554affd640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553c06b4d0 0x7f553c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:30.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.590+0000 7f555a59c640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553c06b4d0 0x7f553c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:30.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.590+0000 7f554affd640 1 -- 172.21.15.64:0/1446373303 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f554c0ad020 con 0x7f555c0b1020 2024-03-04T07:27:30.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.590+0000 7f555a59c640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553c06b4d0 0x7f553c06d990 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f5554005880 tx=0x7f5554009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:30.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.594+0000 7f554affd640 1 -- 172.21.15.64:0/1446373303 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f554c07ba00 con 0x7f555c0b1020 2024-03-04T07:27:30.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.785+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f555c06f2f0 con 0x7f555c0b1020 2024-03-04T07:27:30.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.786+0000 7f554affd640 1 -- 172.21.15.64:0/1446373303 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v20) v1 ==== 80+0+0 (secure 0 0 0) 0x7f554c07faa0 con 0x7f555c0b1020 2024-03-04T07:27:30.786 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:30.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553c06b4d0 msgr2=0x7f553c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553c06b4d0 0x7f553c06d990 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f5554005880 tx=0x7f5554009040 comp rx=0 tx=0).stop 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c0b1020 msgr2=0x7f555c1601d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c0b1020 0x7f555c1601d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f554c009d90 tx=0x7f554c004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 shutdown_connections 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553c06b4d0 0x7f553c06d990 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f555c0b1020 0x7f555c1601d0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f555c06ed10 0x7f555c15fc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 --2- 172.21.15.64:0/1446373303 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f555c06af20 0x7f555c0b5560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 >> 172.21.15.64:0/1446373303 conn(0x7f555c0d1a90 msgr2=0x7f555c0cd2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 shutdown_connections 2024-03-04T07:27:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:30.788+0000 7f556089a640 1 -- 172.21.15.64:0/1446373303 wait complete. 2024-03-04T07:27:30.799 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:27:30.799 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:30.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.2 2024-03-04T07:27:30.826 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:30.826 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:27:30.826 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:30.827 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:27:30.881 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:27:30.881 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:30.897 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:27:30.897 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:30.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:31.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.053+0000 7f47884fb640 1 Processor -- start 2024-03-04T07:27:31.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.053+0000 7f47884fb640 1 -- start start 2024-03-04T07:27:31.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f47884fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4780158520 0x7f478006d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f47884fb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4780071240 con 0x7f47800aee10 2024-03-04T07:27:31.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f47884fb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4780071520 con 0x7f478006dd60 2024-03-04T07:27:31.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f47884fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f478006e1c0 con 0x7f4780158520 2024-03-04T07:27:31.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4786a98640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f47800aee10 0x7f47800b3640 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48632/0 (socket says 172.21.15.64:48632) 2024-03-04T07:27:31.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4786a98640 1 -- 172.21.15.64:0/3869446560 learned_addr learned my addr 172.21.15.64:0/3869446560 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:31.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4786297640 1 --2- 172.21.15.64:0/3869446560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4780158520 0x7f478006d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4785295640 1 -- 172.21.15.64:0/3869446560 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 384576932 0 0) 0x7f4780071240 con 0x7f47800aee10 2024-03-04T07:27:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4785295640 1 -- 172.21.15.64:0/3869446560 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4768003520 con 0x7f47800aee10 2024-03-04T07:27:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4785295640 1 -- 172.21.15.64:0/3869446560 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3044595965 0 0) 0x7f4780071520 con 0x7f478006dd60 2024-03-04T07:27:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4785295640 1 -- 172.21.15.64:0/3869446560 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4780071240 con 0x7f478006dd60 2024-03-04T07:27:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4786297640 1 -- 172.21.15.64:0/3869446560 >> v1:172.21.15.66:6789/0 conn(0x7f478006dd60 legacy=0x7f47800ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4786297640 1 -- 172.21.15.64:0/3869446560 >> v1:172.21.15.64:6789/0 conn(0x7f47800aee10 legacy=0x7f47800b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.054+0000 7f4786297640 1 -- 172.21.15.64:0/3869446560 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4780071800 con 0x7f4780158520 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.055+0000 7f4786297640 1 --2- 172.21.15.64:0/3869446560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4780158520 0x7f478006d820 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f47780097d0 tx=0x7f477802b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e4730c1ee878d8c8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.055+0000 7f4785295640 1 -- 172.21.15.64:0/3869446560 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4778035020 con 0x7f4780158520 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.055+0000 7f4785295640 1 -- 172.21.15.64:0/3869446560 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4778031890 con 0x7f4780158520 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.055+0000 7f4785295640 1 -- 172.21.15.64:0/3869446560 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4778031c40 con 0x7f4780158520 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.056+0000 7f47884fb640 1 -- 172.21.15.64:0/3869446560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4780158520 msgr2=0x7f478006d820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.056+0000 7f47884fb640 1 --2- 172.21.15.64:0/3869446560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4780158520 0x7f478006d820 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f47780097d0 tx=0x7f477802b930 comp rx=0 tx=0).stop 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.056+0000 7f47884fb640 1 -- 172.21.15.64:0/3869446560 shutdown_connections 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.056+0000 7f47884fb640 1 --2- 172.21.15.64:0/3869446560 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4780158520 0x7f478006d820 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.056+0000 7f47884fb640 1 -- 172.21.15.64:0/3869446560 >> 172.21.15.64:0/3869446560 conn(0x7f4780066d00 msgr2=0x7f47800670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.056+0000 7f47884fb640 1 -- 172.21.15.64:0/3869446560 shutdown_connections 2024-03-04T07:27:31.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.056+0000 7f47884fb640 1 -- 172.21.15.64:0/3869446560 wait complete. 2024-03-04T07:27:31.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.057+0000 7f47884fb640 1 Processor -- start 2024-03-04T07:27:31.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.057+0000 7f47884fb640 1 -- start start 2024-03-04T07:27:31.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f47884fb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f478006dd60 0x7f47801e5830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f47884fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47800aee10 0x7f47800cd5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f47884fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4780158520 0x7f478013edf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f47884fb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4780158900 con 0x7f4780158520 2024-03-04T07:27:31.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f47884fb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4780071b70 con 0x7f478006dd60 2024-03-04T07:27:31.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f47884fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4780158a80 con 0x7f47800aee10 2024-03-04T07:27:31.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786a98640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4780158520 0x7f478013edf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f478006dd60 0x7f47801e5830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f478006dd60 0x7f47801e5830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:56362/0 (socket says 172.21.15.64:56362) 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786a98640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4780158520 0x7f478013edf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51648/0 (socket says 172.21.15.64:51648) 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 -- 172.21.15.64:0/2997942405 learned_addr learned my addr 172.21.15.64:0/2997942405 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4785a96640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47800aee10 0x7f47800cd5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 -- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47800aee10 msgr2=0x7f47800cd5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47800aee10 0x7f47800cd5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 -- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4780158520 msgr2=0x7f478013edf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4780158520 0x7f478013edf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 -- 172.21.15.64:0/2997942405 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47800ce560 con 0x7f478006dd60 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786a98640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4780158520 0x7f478013edf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:31.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.058+0000 7f4786297640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f478006dd60 0x7f47801e5830 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f4778000c00 tx=0x7f47780046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.059+0000 7f477effd640 1 -- 172.21.15.64:0/2997942405 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4778035040 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.059+0000 7f477effd640 1 -- 172.21.15.64:0/2997942405 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4778030d20 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.059+0000 7f477effd640 1 -- 172.21.15.64:0/2997942405 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4778006e10 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.059+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47800ce000 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.059+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47801e3690 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.060+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4780067ff0 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.060+0000 7f477effd640 1 -- 172.21.15.64:0/2997942405 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4778038020 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.060+0000 7f477effd640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f476c06b4d0 0x7f476c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.061+0000 7f477effd640 1 -- 172.21.15.64:0/2997942405 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f47780ac790 con 0x7f478006dd60 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.061+0000 7f4785a96640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f476c06b4d0 0x7f476c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.061+0000 7f4785a96640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f476c06b4d0 0x7f476c06d990 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f4774007fd0 tx=0x7f4774009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.065+0000 7f477effd640 1 -- 172.21.15.64:0/2997942405 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f47800ce000 con 0x7f478006dd60 2024-03-04T07:27:31.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.250+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f47800b31f0 con 0x7f478006dd60 2024-03-04T07:27:31.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.252+0000 7f477effd640 1 -- 172.21.15.64:0/2997942405 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f477802d040 con 0x7f478006dd60 2024-03-04T07:27:31.252 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:31.252 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f476c06b4d0 msgr2=0x7f476c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f476c06b4d0 0x7f476c06d990 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f4774007fd0 tx=0x7f4774009040 comp rx=0 tx=0).stop 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f478006dd60 msgr2=0x7f47801e5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f478006dd60 0x7f47801e5830 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f4778000c00 tx=0x7f47780046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 shutdown_connections 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f476c06b4d0 0x7f476c06d990 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4780158520 0x7f478013edf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47800aee10 0x7f47800cd5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 --2- 172.21.15.64:0/2997942405 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f478006dd60 0x7f47801e5830 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 >> 172.21.15.64:0/2997942405 conn(0x7f4780066d00 msgr2=0x7f478015a140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 shutdown_connections 2024-03-04T07:27:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.254+0000 7f47884fb640 1 -- 172.21.15.64:0/2997942405 wait complete. 2024-03-04T07:27:31.265 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:31.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.369+0000 7fa1b9f1e640 1 Processor -- start 2024-03-04T07:27:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.369+0000 7fa1b9f1e640 1 -- start start 2024-03-04T07:27:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b9f1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b9f1e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa1b40d29d0 0x7fa1b40d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b9f1e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa1b40d3740 con 0x7fa1b40d4720 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b9f1e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa1b40d3400 con 0x7fa1b40d29d0 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b9f1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa1b40d3120 con 0x7fa1b415dd90 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa1b40d29d0 0x7fa1b40d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa1b40d29d0 0x7fa1b40d41e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51656/0 (socket says 172.21.15.64:51656) 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b2ffd640 1 -- 172.21.15.64:0/1096088821 learned_addr learned my addr 172.21.15.64:0/1096088821 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.370+0000 7fa1b37fe640 1 --2- 172.21.15.64:0/1096088821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.371+0000 7fa1b2ffd640 1 -- 172.21.15.64:0/1096088821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 msgr2=0x7fa1b40ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.371+0000 7fa1b2ffd640 1 --2- 172.21.15.64:0/1096088821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.371+0000 7fa1b27fc640 1 -- 172.21.15.64:0/1096088821 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 354337476 0 0) 0x7fa1b40d3740 con 0x7fa1b40d4720 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.371+0000 7fa1b2ffd640 1 -- 172.21.15.64:0/1096088821 >> v1:172.21.15.66:6789/0 conn(0x7fa1b40d4720 legacy=0x7fa1b40b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.371+0000 7fa1b2ffd640 1 -- 172.21.15.64:0/1096088821 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1b40d39e0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.371+0000 7fa1b2ffd640 1 --2- 172.21.15.64:0/1096088821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b40d41e0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fa1a8009870 tx=0x7fa1a802ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7af4201876ff96f2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b27fc640 1 -- 172.21.15.64:0/1096088821 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1a802f840 con 0x7fa1b40d29d0 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b27fc640 1 -- 172.21.15.64:0/1096088821 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa1a802f9e0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b27fc640 1 -- 172.21.15.64:0/1096088821 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1a802fdb0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/1096088821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 msgr2=0x7fa1b40d41e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/1096088821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b40d41e0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fa1a8009870 tx=0x7fa1a802ba20 comp rx=0 tx=0).stop 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/1096088821 shutdown_connections 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/1096088821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b40d41e0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/1096088821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/1096088821 >> 172.21.15.64:0/1096088821 conn(0x7fa1b4071960 msgr2=0x7fa1b4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:31.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/1096088821 shutdown_connections 2024-03-04T07:27:31.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.372+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/1096088821 wait complete. 2024-03-04T07:27:31.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.373+0000 7fa1b9f1e640 1 Processor -- start 2024-03-04T07:27:31.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b9f1e640 1 -- start start 2024-03-04T07:27:31.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b9f1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b414a580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b9f1e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1b40d4720 0x7fa1b415e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b9f1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ccf40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b9f1e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1b40d2df0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b9f1e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1b415e170 con 0x7fa1b40d4720 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b9f1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa1b40b7420 con 0x7fa1b415dd90 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b414a580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b414a580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51670/0 (socket says 172.21.15.64:51670) 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b37fe640 1 -- 172.21.15.64:0/408077475 learned_addr learned my addr 172.21.15.64:0/408077475 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b2ffd640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1b40d4720 0x7fa1b415e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.374+0000 7fa1b3fff640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ccf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b37fe640 1 -- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 msgr2=0x7fa1b40ccf40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b37fe640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ccf40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b37fe640 1 -- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1b40d4720 msgr2=0x7fa1b415e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b37fe640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1b40d4720 0x7fa1b415e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b37fe640 1 -- 172.21.15.64:0/408077475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1b414aac0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b37fe640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b414a580 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fa1a400e9e0 tx=0x7fa1a400eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa19bfff640 1 -- 172.21.15.64:0/408077475 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1a4017070 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa19bfff640 1 -- 172.21.15.64:0/408077475 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa1a400c9d0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa1b415ec60 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa19bfff640 1 -- 172.21.15.64:0/408077475 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1a400cdc0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.375+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa1b415f220 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.376+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa1b4072a30 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.377+0000 7fa19bfff640 1 -- 172.21.15.64:0/408077475 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa1a401a070 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.377+0000 7fa19bfff640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa1a006b500 0x7fa1a006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.377+0000 7fa19bfff640 1 -- 172.21.15.64:0/408077475 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa1a408a590 con 0x7fa1b40d29d0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.377+0000 7fa1b2ffd640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa1a006b500 0x7fa1a006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.378+0000 7fa1b2ffd640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa1a006b500 0x7fa1a006d9c0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fa1a8036000 tx=0x7fa1a8008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.381+0000 7fa19bfff640 1 -- 172.21.15.64:0/408077475 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa1b414aac0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.562+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fa1b40b46b0 con 0x7fa1b40d29d0 2024-03-04T07:27:31.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.563+0000 7fa19bfff640 1 -- 172.21.15.64:0/408077475 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7fa1a4058d30 con 0x7fa1b40d29d0 2024-03-04T07:27:31.564 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa1a006b500 msgr2=0x7fa1a006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa1a006b500 0x7fa1a006d9c0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fa1a8036000 tx=0x7fa1a8008030 comp rx=0 tx=0).stop 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 msgr2=0x7fa1b414a580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b414a580 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fa1a400e9e0 tx=0x7fa1a400eea0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 shutdown_connections 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa1a006b500 0x7fa1a006d9c0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1b415dd90 0x7fa1b40ccf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1b40d4720 0x7fa1b415e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 --2- 172.21.15.64:0/408077475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1b40d29d0 0x7fa1b414a580 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 >> 172.21.15.64:0/408077475 conn(0x7fa1b4071960 msgr2=0x7fa1b40d0c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:31.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 shutdown_connections 2024-03-04T07:27:31.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.565+0000 7fa1b9f1e640 1 -- 172.21.15.64:0/408077475 wait complete. 2024-03-04T07:27:31.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:31.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.680+0000 7fb707cc3640 1 Processor -- start 2024-03-04T07:27:31.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.680+0000 7fb707cc3640 1 -- start start 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb707cc3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7000b29a0 0x7fb7000b2db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb707cc3640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb700070980 con 0x7fb700142170 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb707cc3640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb7000b32f0 con 0x7fb7000b4830 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb707cc3640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb7001458e0 con 0x7fb7000b29a0 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb706260640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7000b29a0 0x7fb7000b2db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb705a5f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb700142170 0x7fb700142a20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48646/0 (socket says 172.21.15.64:48646) 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb705a5f640 1 -- 172.21.15.64:0/3828581535 learned_addr learned my addr 172.21.15.64:0/3828581535 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb706260640 1 -- 172.21.15.64:0/3828581535 >> v1:172.21.15.66:6789/0 conn(0x7fb7000b4830 legacy=0x7fb700160280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb706260640 1 -- 172.21.15.64:0/3828581535 >> v1:172.21.15.64:6789/0 conn(0x7fb700142170 legacy=0x7fb700142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb704a5d640 1 -- 172.21.15.64:0/3828581535 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2628084954 0 0) 0x7fb6f0001110 con 0x7fb7000b4830 2024-03-04T07:27:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.681+0000 7fb706260640 1 -- 172.21.15.64:0/3828581535 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb700070bf0 con 0x7fb7000b29a0 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.682+0000 7fb706260640 1 --2- 172.21.15.64:0/3828581535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b29a0 0x7fb7000b2db0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fb6fc009d90 tx=0x7fb6fc02bcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=717222ab32bd430a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.682+0000 7fb704a5d640 1 -- 172.21.15.64:0/3828581535 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6fc039070 con 0x7fb7000b29a0 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.682+0000 7fb704a5d640 1 -- 172.21.15.64:0/3828581535 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6fc02ca20 con 0x7fb7000b29a0 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.682+0000 7fb704a5d640 1 -- 172.21.15.64:0/3828581535 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6fc02cdf0 con 0x7fb7000b29a0 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.683+0000 7fb707cc3640 1 -- 172.21.15.64:0/3828581535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b29a0 msgr2=0x7fb7000b2db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.683+0000 7fb707cc3640 1 --2- 172.21.15.64:0/3828581535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b29a0 0x7fb7000b2db0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fb6fc009d90 tx=0x7fb6fc02bcf0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.683+0000 7fb707cc3640 1 -- 172.21.15.64:0/3828581535 shutdown_connections 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.683+0000 7fb707cc3640 1 --2- 172.21.15.64:0/3828581535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b29a0 0x7fb7000b2db0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.683+0000 7fb707cc3640 1 -- 172.21.15.64:0/3828581535 >> 172.21.15.64:0/3828581535 conn(0x7fb700066d00 msgr2=0x7fb700067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.683+0000 7fb707cc3640 1 -- 172.21.15.64:0/3828581535 shutdown_connections 2024-03-04T07:27:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.683+0000 7fb707cc3640 1 -- 172.21.15.64:0/3828581535 wait complete. 2024-03-04T07:27:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.684+0000 7fb707cc3640 1 Processor -- start 2024-03-04T07:27:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.684+0000 7fb707cc3640 1 -- start start 2024-03-04T07:27:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb707cc3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7000b29a0 0x7fb70013b3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb707cc3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b4830 0x7fb7001383a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb707cc3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb700142170 0x7fb70015e610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb707cc3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7000b4df0 con 0x7fb7000b4830 2024-03-04T07:27:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb707cc3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7000711b0 con 0x7fb7000b29a0 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb707cc3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb700142550 con 0x7fb700142170 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b4830 0x7fb7001383a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b4830 0x7fb7001383a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51694/0 (socket says 172.21.15.64:51694) 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 -- 172.21.15.64:0/507592156 learned_addr learned my addr 172.21.15.64:0/507592156 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb705a5f640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7000b29a0 0x7fb70013b3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb706260640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb700142170 0x7fb70015e610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 -- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb700142170 msgr2=0x7fb70015e610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb700142170 0x7fb70015e610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 -- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7000b29a0 msgr2=0x7fb70013b3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7000b29a0 0x7fb70013b3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 -- 172.21.15.64:0/507592156 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb700138d20 con 0x7fb7000b4830 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb705a5f640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7000b29a0 0x7fb70013b3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb70525e640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b4830 0x7fb7001383a0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fb6f000c750 tx=0x7fb6f000cc10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.685+0000 7fb6f67fc640 1 -- 172.21.15.64:0/507592156 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f0017070 con 0x7fb7000b4830 2024-03-04T07:27:31.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.686+0000 7fb6f67fc640 1 -- 172.21.15.64:0/507592156 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6f00052b0 con 0x7fb7000b4830 2024-03-04T07:27:31.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.686+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb7001388e0 con 0x7fb7000b4830 2024-03-04T07:27:31.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.686+0000 7fb6f67fc640 1 -- 172.21.15.64:0/507592156 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6f000a880 con 0x7fb7000b4830 2024-03-04T07:27:31.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.686+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7001392a0 con 0x7fb7000b4830 2024-03-04T07:27:31.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.686+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb7000733c0 con 0x7fb7000b4830 2024-03-04T07:27:31.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.687+0000 7fb6f67fc640 1 -- 172.21.15.64:0/507592156 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb6f001a070 con 0x7fb7000b4830 2024-03-04T07:27:31.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.687+0000 7fb6f67fc640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6e406b4d0 0x7fb6e406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.688+0000 7fb705a5f640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6e406b4d0 0x7fb6e406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.688+0000 7fb6f67fc640 1 -- 172.21.15.64:0/507592156 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb6f00893e0 con 0x7fb7000b4830 2024-03-04T07:27:31.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.688+0000 7fb705a5f640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6e406b4d0 0x7fb6e406d990 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fb6f8000f60 tx=0x7fb6f8007e20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.692+0000 7fb6f67fc640 1 -- 172.21.15.64:0/507592156 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb7000b32f0 con 0x7fb7000b4830 2024-03-04T07:27:31.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.875+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fb70004f410 con 0x7fb7000b4830 2024-03-04T07:27:31.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.876+0000 7fb6f67fc640 1 -- 172.21.15.64:0/507592156 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v20) v1 ==== 104+0+11 (secure 0 0 0) 0x7fb6f0007d30 con 0x7fb7000b4830 2024-03-04T07:27:31.876 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6e406b4d0 msgr2=0x7fb6e406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6e406b4d0 0x7fb6e406d990 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fb6f8000f60 tx=0x7fb6f8007e20 comp rx=0 tx=0).stop 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b4830 msgr2=0x7fb7001383a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b4830 0x7fb7001383a0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fb6f000c750 tx=0x7fb6f000cc10 comp rx=0 tx=0).stop 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 shutdown_connections 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6e406b4d0 0x7fb6e406d990 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb700142170 0x7fb70015e610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7000b4830 0x7fb7001383a0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 --2- 172.21.15.64:0/507592156 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7000b29a0 0x7fb70013b3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 >> 172.21.15.64:0/507592156 conn(0x7fb700066d00 msgr2=0x7fb70015c650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 shutdown_connections 2024-03-04T07:27:31.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.878+0000 7fb707cc3640 1 -- 172.21.15.64:0/507592156 wait complete. 2024-03-04T07:27:31.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:31.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.993+0000 7fef18047640 1 Processor -- start 2024-03-04T07:27:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.993+0000 7fef18047640 1 -- start start 2024-03-04T07:27:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef18047640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fef100b4340 0x7fef1015d9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef18047640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef1015fb60 0x7fef1015ff70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef18047640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fef101604b0 con 0x7fef1015d2c0 2024-03-04T07:27:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef18047640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fef100d0a30 con 0x7fef1015fb60 2024-03-04T07:27:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef18047640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fef100d01d0 con 0x7fef100b4340 2024-03-04T07:27:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef165e4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef1015fb60 0x7fef1015ff70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef165e4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef1015fb60 0x7fef1015ff70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51698/0 (socket says 172.21.15.64:51698) 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef15de3640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fef1015d2c0 0x7fef100b2ac0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48656/0 (socket says 172.21.15.64:48656) 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef165e4640 1 -- 172.21.15.64:0/1561399017 learned_addr learned my addr 172.21.15.64:0/1561399017 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef155e2640 1 --2- 172.21.15.64:0/1561399017 >> v2:172.21.15.66:3300/0 conn(0x7fef100b4340 0x7fef1015d9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef165e4640 1 -- 172.21.15.64:0/1561399017 >> v2:172.21.15.66:3300/0 conn(0x7fef100b4340 msgr2=0x7fef1015d9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef165e4640 1 --2- 172.21.15.64:0/1561399017 >> v2:172.21.15.66:3300/0 conn(0x7fef100b4340 0x7fef1015d9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef165e4640 1 -- 172.21.15.64:0/1561399017 >> v1:172.21.15.64:6789/0 conn(0x7fef1015d2c0 legacy=0x7fef100b2ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef165e4640 1 -- 172.21.15.64:0/1561399017 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef100cc5c0 con 0x7fef1015fb60 2024-03-04T07:27:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.994+0000 7fef14de1640 1 -- 172.21.15.64:0/1561399017 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2516184870 0 0) 0x7fef08001110 con 0x7fef1015d2c0 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.995+0000 7fef165e4640 1 --2- 172.21.15.64:0/1561399017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef1015ff70 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fef10072280 tx=0x7fef0002bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=abd3a028f624044b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.995+0000 7fef14de1640 1 -- 172.21.15.64:0/1561399017 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef0002cb20 con 0x7fef1015fb60 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.995+0000 7fef14de1640 1 -- 172.21.15.64:0/1561399017 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fef0002ccc0 con 0x7fef1015fb60 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.995+0000 7fef14de1640 1 -- 172.21.15.64:0/1561399017 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef00034610 con 0x7fef1015fb60 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 -- 172.21.15.64:0/1561399017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 msgr2=0x7fef1015ff70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 --2- 172.21.15.64:0/1561399017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef1015ff70 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fef10072280 tx=0x7fef0002bf80 comp rx=0 tx=0).stop 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 -- 172.21.15.64:0/1561399017 shutdown_connections 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 --2- 172.21.15.64:0/1561399017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef1015ff70 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 --2- 172.21.15.64:0/1561399017 >> v2:172.21.15.66:3300/0 conn(0x7fef100b4340 0x7fef1015d9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 -- 172.21.15.64:0/1561399017 >> 172.21.15.64:0/1561399017 conn(0x7fef100b3660 msgr2=0x7fef100b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 -- 172.21.15.64:0/1561399017 shutdown_connections 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.996+0000 7fef18047640 1 -- 172.21.15.64:0/1561399017 wait complete. 2024-03-04T07:27:31.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.997+0000 7fef18047640 1 Processor -- start 2024-03-04T07:27:31.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.997+0000 7fef18047640 1 -- start start 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef18047640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef100b4340 0x7fef100b47f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef18047640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef1015d2c0 0x7fef101f5250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef18047640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef101f5790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef18047640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef100cc860 con 0x7fef1015fb60 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef18047640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef100d0750 con 0x7fef100b4340 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef18047640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fef100b31d0 con 0x7fef1015d2c0 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef101f5790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef101f5790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51710/0 (socket says 172.21.15.64:51710) 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 -- 172.21.15.64:0/1057472447 learned_addr learned my addr 172.21.15.64:0/1057472447 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef15de3640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef100b4340 0x7fef100b47f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef155e2640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef1015d2c0 0x7fef101f5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 -- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef1015d2c0 msgr2=0x7fef101f5250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef1015d2c0 0x7fef101f5250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 -- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef100b4340 msgr2=0x7fef100b47f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef100b4340 0x7fef100b47f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 -- 172.21.15.64:0/1057472447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef100b5160 con 0x7fef1015fb60 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.998+0000 7fef165e4640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef101f5790 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fef00036040 tx=0x7fef00004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.999+0000 7fef0e7fc640 1 -- 172.21.15.64:0/1057472447 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef00043030 con 0x7fef1015fb60 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.999+0000 7fef0e7fc640 1 -- 172.21.15.64:0/1057472447 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fef0003e450 con 0x7fef1015fb60 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.999+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fef100b5440 con 0x7fef1015fb60 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.999+0000 7fef0e7fc640 1 -- 172.21.15.64:0/1057472447 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef00009510 con 0x7fef1015fb60 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:31.999+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef100b5a00 con 0x7fef1015fb60 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.000+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fef1009fc50 con 0x7fef1015fb60 2024-03-04T07:27:32.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.000+0000 7fef0e7fc640 1 -- 172.21.15.64:0/1057472447 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fef00046070 con 0x7fef1015fb60 2024-03-04T07:27:32.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.000+0000 7fef0e7fc640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feefc06b500 0x7feefc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.001+0000 7fef0e7fc640 1 -- 172.21.15.64:0/1057472447 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fef000ac420 con 0x7fef1015fb60 2024-03-04T07:27:32.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.001+0000 7fef15de3640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feefc06b500 0x7feefc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.001+0000 7fef15de3640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feefc06b500 0x7feefc06d9c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fef08000f60 tx=0x7fef08007b50 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.005+0000 7fef0e7fc640 1 -- 172.21.15.64:0/1057472447 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fef0004d070 con 0x7fef1015fb60 2024-03-04T07:27:32.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.186+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fef10072cb0 con 0x7fef1015fb60 2024-03-04T07:27:32.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.186+0000 7fef0e7fc640 1 -- 172.21.15.64:0/1057472447 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v20) v1 ==== 100+0+11 (secure 0 0 0) 0x7fef0007ad70 con 0x7fef1015fb60 2024-03-04T07:27:32.187 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:32.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feefc06b500 msgr2=0x7feefc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feefc06b500 0x7feefc06d9c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fef08000f60 tx=0x7fef08007b50 comp rx=0 tx=0).stop 2024-03-04T07:27:32.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 msgr2=0x7fef101f5790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef101f5790 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fef00036040 tx=0x7fef00004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:32.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 shutdown_connections 2024-03-04T07:27:32.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feefc06b500 0x7feefc06d9c0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef1015fb60 0x7fef101f5790 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef1015d2c0 0x7fef101f5250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 --2- 172.21.15.64:0/1057472447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef100b4340 0x7fef100b47f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 >> 172.21.15.64:0/1057472447 conn(0x7fef100b3660 msgr2=0x7fef1006eb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:32.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 shutdown_connections 2024-03-04T07:27:32.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.188+0000 7fef18047640 1 -- 172.21.15.64:0/1057472447 wait complete. 2024-03-04T07:27:32.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:32.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.301+0000 7fa4cd32b640 1 Processor -- start 2024-03-04T07:27:32.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.301+0000 7fa4cd32b640 1 -- start start 2024-03-04T07:27:32.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4cd32b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa4c806e700 0x7fa4c815ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4cd32b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa4c80b55d0 0x7fa4c8071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4cd32b640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4c80b4160 con 0x7fa4c8072660 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4cd32b640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4c80b5b40 con 0x7fa4c80b55d0 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4cd32b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4c80b3ba0 con 0x7fa4c806e700 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c77fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa4c80b55d0 0x7fa4c8071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c67fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa4c8072660 0x7fa4c806d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48670/0 (socket says 172.21.15.64:48670) 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c6ffd640 1 --2- 172.21.15.64:0/1854305081 >> v2:172.21.15.66:3300/0 conn(0x7fa4c806e700 0x7fa4c815ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c67fc640 1 -- 172.21.15.64:0/1854305081 learned_addr learned my addr 172.21.15.64:0/1854305081 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c77fe640 1 -- 172.21.15.64:0/1854305081 >> v2:172.21.15.66:3300/0 conn(0x7fa4c806e700 msgr2=0x7fa4c815ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c77fe640 1 --2- 172.21.15.64:0/1854305081 >> v2:172.21.15.66:3300/0 conn(0x7fa4c806e700 0x7fa4c815ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c77fe640 1 -- 172.21.15.64:0/1854305081 >> v1:172.21.15.64:6789/0 conn(0x7fa4c8072660 legacy=0x7fa4c806d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c5ffb640 1 -- 172.21.15.64:0/1854305081 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1573275501 0 0) 0x7fa4c80b4160 con 0x7fa4c8072660 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.302+0000 7fa4c77fe640 1 -- 172.21.15.64:0/1854305081 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4c80b3e80 con 0x7fa4c80b55d0 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.303+0000 7fa4c77fe640 1 --2- 172.21.15.64:0/1854305081 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c80b55d0 0x7fa4c8071980 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fa4c806bf60 tx=0x7fa4b802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5b2b15a5e6899819 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.303+0000 7fa4c5ffb640 1 -- 172.21.15.64:0/1854305081 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4b8039070 con 0x7fa4c80b55d0 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.303+0000 7fa4c5ffb640 1 -- 172.21.15.64:0/1854305081 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa4b802cc80 con 0x7fa4c80b55d0 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.303+0000 7fa4c5ffb640 1 -- 172.21.15.64:0/1854305081 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4b8034660 con 0x7fa4c80b55d0 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1854305081 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c80b55d0 msgr2=0x7fa4c8071980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1854305081 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c80b55d0 0x7fa4c8071980 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fa4c806bf60 tx=0x7fa4b802bed0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1854305081 shutdown_connections 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1854305081 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c80b55d0 0x7fa4c8071980 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1854305081 >> v2:172.21.15.66:3300/0 conn(0x7fa4c806e700 0x7fa4c815ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1854305081 >> 172.21.15.64:0/1854305081 conn(0x7fa4c80b2720 msgr2=0x7fa4c80b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1854305081 shutdown_connections 2024-03-04T07:27:32.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.304+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1854305081 wait complete. 2024-03-04T07:27:32.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.305+0000 7fa4cd32b640 1 Processor -- start 2024-03-04T07:27:32.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.305+0000 7fa4cd32b640 1 -- start start 2024-03-04T07:27:32.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4cd32b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4c806e700 0x7fa4c80b1270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4cd32b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c8072660 0x7fa4c80671a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4cd32b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4c80b55d0 0x7fa4c80ae1a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4cd32b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4c8072040 con 0x7fa4c8072660 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4cd32b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4c8071ec0 con 0x7fa4c80b55d0 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4cd32b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4c80721c0 con 0x7fa4c806e700 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c8072660 0x7fa4c80671a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c8072660 0x7fa4c80671a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51728/0 (socket says 172.21.15.64:51728) 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 -- 172.21.15.64:0/1429540238 learned_addr learned my addr 172.21.15.64:0/1429540238 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c6ffd640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4c806e700 0x7fa4c80b1270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c77fe640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4c80b55d0 0x7fa4c80ae1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 -- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4c806e700 msgr2=0x7fa4c80b1270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4c806e700 0x7fa4c80b1270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 -- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4c80b55d0 msgr2=0x7fa4c80ae1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4c80b55d0 0x7fa4c80ae1a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c67fc640 1 -- 172.21.15.64:0/1429540238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4c80b17b0 con 0x7fa4c8072660 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.306+0000 7fa4c6ffd640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4c806e700 0x7fa4c80b1270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.307+0000 7fa4c67fc640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c8072660 0x7fa4c80671a0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fa4bc00c4b0 tx=0x7fa4bc00c970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.307+0000 7fa4af7fe640 1 -- 172.21.15.64:0/1429540238 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4bc018070 con 0x7fa4c8072660 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.307+0000 7fa4af7fe640 1 -- 172.21.15.64:0/1429540238 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa4bc007500 con 0x7fa4c8072660 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.307+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4c80b0230 con 0x7fa4c8072660 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.307+0000 7fa4af7fe640 1 -- 172.21.15.64:0/1429540238 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4bc004a20 con 0x7fa4c8072660 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.307+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4c80b0a70 con 0x7fa4c8072660 2024-03-04T07:27:32.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.308+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa4c8067ff0 con 0x7fa4c8072660 2024-03-04T07:27:32.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.308+0000 7fa4af7fe640 1 -- 172.21.15.64:0/1429540238 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa4bc01b070 con 0x7fa4c8072660 2024-03-04T07:27:32.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.309+0000 7fa4af7fe640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa4b406b4d0 0x7fa4b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.309+0000 7fa4af7fe640 1 -- 172.21.15.64:0/1429540238 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa4bc0893f0 con 0x7fa4c8072660 2024-03-04T07:27:32.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.309+0000 7fa4c6ffd640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa4b406b4d0 0x7fa4b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.309+0000 7fa4c6ffd640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa4b406b4d0 0x7fa4b406d990 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fa4c0009f30 tx=0x7fa4c0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.313+0000 7fa4af7fe640 1 -- 172.21.15.64:0/1429540238 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa4c80b0230 con 0x7fa4c8072660 2024-03-04T07:27:32.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.511+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fa4c8072c30 con 0x7fa4c8072660 2024-03-04T07:27:32.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.511+0000 7fa4af7fe640 1 -- 172.21.15.64:0/1429540238 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v20) v1 ==== 80+0+0 (secure 0 0 0) 0x7fa4bc057d70 con 0x7fa4c8072660 2024-03-04T07:27:32.512 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:32.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa4b406b4d0 msgr2=0x7fa4b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa4b406b4d0 0x7fa4b406d990 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fa4c0009f30 tx=0x7fa4c0009190 comp rx=0 tx=0).stop 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c8072660 msgr2=0x7fa4c80671a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c8072660 0x7fa4c80671a0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fa4bc00c4b0 tx=0x7fa4bc00c970 comp rx=0 tx=0).stop 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 shutdown_connections 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa4b406b4d0 0x7fa4b406d990 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4c80b55d0 0x7fa4c80ae1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4c8072660 0x7fa4c80671a0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 --2- 172.21.15.64:0/1429540238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4c806e700 0x7fa4c80b1270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 >> 172.21.15.64:0/1429540238 conn(0x7fa4c80b2720 msgr2=0x7fa4c80b2b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 shutdown_connections 2024-03-04T07:27:32.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.514+0000 7fa4cd32b640 1 -- 172.21.15.64:0/1429540238 wait complete. 2024-03-04T07:27:32.525 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:27:32.525 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:27:32.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:32.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.643+0000 7f56f4223640 1 Processor -- start 2024-03-04T07:27:32.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.643+0000 7f56f4223640 1 -- start start 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f4223640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f56ec15e000 0x7f56ec071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f4223640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f56ec162ad0 con 0x7f56ec0d35e0 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f4223640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f56ec0ce970 con 0x7f56ec0b2c50 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f4223640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f56ec0cece0 con 0x7f56ec15e000 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f1fbf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f56ec15e000 0x7f56ec071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f27c0640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f56ec0d35e0 0x7f56ec1623c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48682/0 (socket says 172.21.15.64:48682) 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f27c0640 1 -- 172.21.15.64:0/4049299300 learned_addr learned my addr 172.21.15.64:0/4049299300 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f1fbf640 1 -- 172.21.15.64:0/4049299300 >> v1:172.21.15.66:6789/0 conn(0x7f56ec0b2c50 legacy=0x7f56ec161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f1fbf640 1 -- 172.21.15.64:0/4049299300 >> v1:172.21.15.64:6789/0 conn(0x7f56ec0d35e0 legacy=0x7f56ec1623c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f0fbd640 1 -- 172.21.15.64:0/4049299300 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4096570412 0 0) 0x7f56ec162ad0 con 0x7f56ec0d35e0 2024-03-04T07:27:32.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.644+0000 7f56f1fbf640 1 -- 172.21.15.64:0/4049299300 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56ec0cef30 con 0x7f56ec15e000 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.645+0000 7f56f1fbf640 1 --2- 172.21.15.64:0/4049299300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec15e000 0x7f56ec071540 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f56e80097a0 tx=0x7f56e802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=819ff1dcd1020fbe server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.645+0000 7f56f0fbd640 1 -- 172.21.15.64:0/4049299300 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56e8035020 con 0x7f56ec15e000 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.645+0000 7f56f0fbd640 1 -- 172.21.15.64:0/4049299300 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f56e8031830 con 0x7f56ec15e000 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.645+0000 7f56f0fbd640 1 -- 172.21.15.64:0/4049299300 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56e8031c00 con 0x7f56ec15e000 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.646+0000 7f56f4223640 1 -- 172.21.15.64:0/4049299300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec15e000 msgr2=0x7f56ec071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.646+0000 7f56f4223640 1 --2- 172.21.15.64:0/4049299300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec15e000 0x7f56ec071540 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f56e80097a0 tx=0x7f56e802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.646+0000 7f56f4223640 1 -- 172.21.15.64:0/4049299300 shutdown_connections 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.646+0000 7f56f4223640 1 --2- 172.21.15.64:0/4049299300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec15e000 0x7f56ec071540 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.646+0000 7f56f4223640 1 -- 172.21.15.64:0/4049299300 >> 172.21.15.64:0/4049299300 conn(0x7f56ec0d3c80 msgr2=0x7f56ec0d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.646+0000 7f56f4223640 1 -- 172.21.15.64:0/4049299300 shutdown_connections 2024-03-04T07:27:32.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.646+0000 7f56f4223640 1 -- 172.21.15.64:0/4049299300 wait complete. 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.647+0000 7f56f4223640 1 Processor -- start 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.647+0000 7f56f4223640 1 -- start start 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.647+0000 7f56f4223640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec0b2c50 0x7f56ec14bfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.647+0000 7f56f4223640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56ec0d35e0 0x7f56ec0b60d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f4223640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56ec15e000 0x7f56ec1f7430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f4223640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56ec0b31a0 con 0x7f56ec0b2c50 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f4223640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56ec0ce600 con 0x7f56ec0d35e0 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f4223640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f56ec0b3320 con 0x7f56ec15e000 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec0b2c50 0x7f56ec14bfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec0b2c50 0x7f56ec14bfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51734/0 (socket says 172.21.15.64:51734) 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 -- 172.21.15.64:0/4169451025 learned_addr learned my addr 172.21.15.64:0/4169451025 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f17be640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56ec0d35e0 0x7f56ec0b60d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f27c0640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56ec15e000 0x7f56ec1f7430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 -- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56ec15e000 msgr2=0x7f56ec1f7430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56ec15e000 0x7f56ec1f7430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 -- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56ec0d35e0 msgr2=0x7f56ec0b60d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56ec0d35e0 0x7f56ec0b60d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 -- 172.21.15.64:0/4169451025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56ec0b6610 con 0x7f56ec0b2c50 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f1fbf640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec0b2c50 0x7f56ec14bfb0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f56e80098c0 tx=0x7f56e80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56e27fc640 1 -- 172.21.15.64:0/4169451025 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56e803ba80 con 0x7f56ec0b2c50 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56e27fc640 1 -- 172.21.15.64:0/4169451025 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f56e803bc20 con 0x7f56ec0b2c50 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56ec0cc850 con 0x7f56ec0b2c50 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56e27fc640 1 -- 172.21.15.64:0/4169451025 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56e80453f0 con 0x7f56ec0b2c50 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.648+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56ec0cce10 con 0x7f56ec0b2c50 2024-03-04T07:27:32.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.649+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56ec067ff0 con 0x7f56ec0b2c50 2024-03-04T07:27:32.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.650+0000 7f56e27fc640 1 -- 172.21.15.64:0/4169451025 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f56e804c060 con 0x7f56ec0b2c50 2024-03-04T07:27:32.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.650+0000 7f56e27fc640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56d006b4d0 0x7f56d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.650+0000 7f56e27fc640 1 -- 172.21.15.64:0/4169451025 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f56e80abe80 con 0x7f56ec0b2c50 2024-03-04T07:27:32.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.650+0000 7f56f17be640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56d006b4d0 0x7f56d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.651+0000 7f56f17be640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56d006b4d0 0x7f56d006d990 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f56ec15a040 tx=0x7f56dc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.655+0000 7f56e27fc640 1 -- 172.21.15.64:0/4169451025 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f56e8040070 con 0x7f56ec0b2c50 2024-03-04T07:27:32.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.837+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f56ec160400 con 0x7f56ec0b2c50 2024-03-04T07:27:32.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.838+0000 7f56e27fc640 1 -- 172.21.15.64:0/4169451025 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) v1 ==== 74+0+16557 (secure 0 0 0) 0x7f56e807a6b0 con 0x7f56ec0b2c50 2024-03-04T07:27:32.840 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:32.840 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:26:54.302185+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.839+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56d006b4d0 msgr2=0x7f56d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.839+0000 7f56f4223640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56d006b4d0 0x7f56d006d990 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f56ec15a040 tx=0x7f56dc009040 comp rx=0 tx=0).stop 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.839+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec0b2c50 msgr2=0x7f56ec14bfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.839+0000 7f56f4223640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec0b2c50 0x7f56ec14bfb0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f56e80098c0 tx=0x7f56e80046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 shutdown_connections 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56d006b4d0 0x7f56d006d990 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56ec15e000 0x7f56ec1f7430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56ec0d35e0 0x7f56ec0b60d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 --2- 172.21.15.64:0/4169451025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56ec0b2c50 0x7f56ec14bfb0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 >> 172.21.15.64:0/4169451025 conn(0x7f56ec0d3c80 msgr2=0x7f56ec15f450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 shutdown_connections 2024-03-04T07:27:32.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.840+0000 7f56f4223640 1 -- 172.21.15.64:0/4169451025 wait complete. 2024-03-04T07:27:32.851 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:32.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:32.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.955+0000 7f0d3e89d640 1 Processor -- start 2024-03-04T07:27:32.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.955+0000 7f0d3e89d640 1 -- start start 2024-03-04T07:27:32.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3e89d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d3806bbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3e89d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0d3806c130 0x7f0d380cdac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3e89d640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0d3815e6d0 con 0x7f0d3815e2f0 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3e89d640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d380d0a10 con 0x7f0d3806c130 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3e89d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d380ccd00 con 0x7f0d380b41f0 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3ce3a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0d3806c130 0x7f0d380cdac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d37fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0d3815e2f0 0x7f0d3806b4e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48686/0 (socket says 172.21.15.64:48686) 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d37fff640 1 -- 172.21.15.64:0/1399113489 learned_addr learned my addr 172.21.15.64:0/1399113489 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d377fe640 1 --2- 172.21.15.64:0/1399113489 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d3806bbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3ce3a640 1 -- 172.21.15.64:0/1399113489 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 msgr2=0x7f0d3806bbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3ce3a640 1 --2- 172.21.15.64:0/1399113489 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d3806bbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3ce3a640 1 -- 172.21.15.64:0/1399113489 >> v1:172.21.15.64:6789/0 conn(0x7f0d3815e2f0 legacy=0x7f0d3806b4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d3ce3a640 1 -- 172.21.15.64:0/1399113489 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d380ccfe0 con 0x7f0d3806c130 2024-03-04T07:27:32.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.956+0000 7f0d36ffd640 1 -- 172.21.15.64:0/1399113489 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 147490497 0 0) 0x7f0d3815e6d0 con 0x7f0d3815e2f0 2024-03-04T07:27:32.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.957+0000 7f0d3ce3a640 1 --2- 172.21.15.64:0/1399113489 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3806c130 0x7f0d380cdac0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f0d30009d90 tx=0x7f0d3002be10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eea31fb97466df0a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.957+0000 7f0d36ffd640 1 -- 172.21.15.64:0/1399113489 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d30039070 con 0x7f0d3806c130 2024-03-04T07:27:32.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.957+0000 7f0d36ffd640 1 -- 172.21.15.64:0/1399113489 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d3002cbc0 con 0x7f0d3806c130 2024-03-04T07:27:32.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.957+0000 7f0d36ffd640 1 -- 172.21.15.64:0/1399113489 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d30034440 con 0x7f0d3806c130 2024-03-04T07:27:32.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1399113489 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3806c130 msgr2=0x7f0d380cdac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1399113489 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3806c130 0x7f0d380cdac0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f0d30009d90 tx=0x7f0d3002be10 comp rx=0 tx=0).stop 2024-03-04T07:27:32.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1399113489 shutdown_connections 2024-03-04T07:27:32.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1399113489 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3806c130 0x7f0d380cdac0 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1399113489 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d3806bbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1399113489 >> 172.21.15.64:0/1399113489 conn(0x7f0d380b36d0 msgr2=0x7f0d380b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:32.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1399113489 shutdown_connections 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.958+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1399113489 wait complete. 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.959+0000 7f0d3e89d640 1 Processor -- start 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.959+0000 7f0d3e89d640 1 -- start start 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3e89d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d3806c130 0x7f0d38148890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3e89d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d380b4730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3e89d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3815e2f0 0x7f0d3806eb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3e89d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d380ce000 con 0x7f0d3815e2f0 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3e89d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d380cd5a0 con 0x7f0d3806c130 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3e89d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d380ce180 con 0x7f0d380b41f0 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3815e2f0 0x7f0d3806eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3815e2f0 0x7f0d3806eb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51752/0 (socket says 172.21.15.64:51752) 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 -- 172.21.15.64:0/1906228371 learned_addr learned my addr 172.21.15.64:0/1906228371 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d377fe640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d380b4730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d37fff640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d3806c130 0x7f0d38148890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 -- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 msgr2=0x7f0d380b4730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d380b4730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 -- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d3806c130 msgr2=0x7f0d38148890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d3806c130 0x7f0d38148890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 -- 172.21.15.64:0/1906228371 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d380b6910 con 0x7f0d3815e2f0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d3ce3a640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3815e2f0 0x7f0d3806eb90 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f0d30009d90 tx=0x7f0d30004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.960+0000 7f0d34ff9640 1 -- 172.21.15.64:0/1906228371 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d30039050 con 0x7f0d3815e2f0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.961+0000 7f0d34ff9640 1 -- 172.21.15.64:0/1906228371 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d30045070 con 0x7f0d3815e2f0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.961+0000 7f0d34ff9640 1 -- 172.21.15.64:0/1906228371 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d30009450 con 0x7f0d3815e2f0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.961+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d380b6bf0 con 0x7f0d3815e2f0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.961+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d380b71b0 con 0x7f0d3815e2f0 2024-03-04T07:27:32.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.962+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d3809fd30 con 0x7f0d3815e2f0 2024-03-04T07:27:32.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.962+0000 7f0d34ff9640 1 -- 172.21.15.64:0/1906228371 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0d3003e850 con 0x7f0d3815e2f0 2024-03-04T07:27:32.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.962+0000 7f0d34ff9640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d2406b4d0 0x7f0d2406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:32.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.963+0000 7f0d34ff9640 1 -- 172.21.15.64:0/1906228371 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0d300aceb0 con 0x7f0d3815e2f0 2024-03-04T07:27:32.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.963+0000 7f0d37fff640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d2406b4d0 0x7f0d2406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:32.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.963+0000 7f0d37fff640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d2406b4d0 0x7f0d2406d990 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f0d28007fd0 tx=0x7f0d28007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:32.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:32.967+0000 7f0d34ff9640 1 -- 172.21.15.64:0/1906228371 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0d3007b890 con 0x7f0d3815e2f0 2024-03-04T07:27:33.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.148+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f0d3804f410 con 0x7f0d3815e2f0 2024-03-04T07:27:33.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.148+0000 7f0d34ff9640 1 -- 172.21.15.64:0/1906228371 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v20) v1 ==== 93+0+10 (secure 0 0 0) 0x7f0d3007f930 con 0x7f0d3815e2f0 2024-03-04T07:27:33.149 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d2406b4d0 msgr2=0x7f0d2406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d2406b4d0 0x7f0d2406d990 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f0d28007fd0 tx=0x7f0d28007de0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3815e2f0 msgr2=0x7f0d3806eb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3815e2f0 0x7f0d3806eb90 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f0d30009d90 tx=0x7f0d30004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 shutdown_connections 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d2406b4d0 0x7f0d2406d990 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d3815e2f0 0x7f0d3806eb90 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d380b41f0 0x7f0d380b4730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 --2- 172.21.15.64:0/1906228371 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d3806c130 0x7f0d38148890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 >> 172.21.15.64:0/1906228371 conn(0x7f0d380b36d0 msgr2=0x7f0d380ceb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 shutdown_connections 2024-03-04T07:27:33.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.150+0000 7f0d3e89d640 1 -- 172.21.15.64:0/1906228371 wait complete. 2024-03-04T07:27:33.162 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:33.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.265+0000 7fa38a783640 1 Processor -- start 2024-03-04T07:27:33.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.265+0000 7fa38a783640 1 -- start start 2024-03-04T07:27:33.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.266+0000 7fa38a783640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3841618e0 0x7fa38406b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.266+0000 7fa38a783640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa38415f730 con 0x7fa3840ce4f0 2024-03-04T07:27:33.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.266+0000 7fa38a783640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa38415ebb0 con 0x7fa3840d0150 2024-03-04T07:27:33.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.266+0000 7fa38a783640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa3840cfce0 con 0x7fa3841618e0 2024-03-04T07:27:33.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.266+0000 7fa3837fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa3840ce4f0 0x7fa3840ce930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48688/0 (socket says 172.21.15.64:48688) 2024-03-04T07:27:33.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.266+0000 7fa3837fe640 1 -- 172.21.15.64:0/1575304140 learned_addr learned my addr 172.21.15.64:0/1575304140 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:33.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.266+0000 7fa383fff640 1 --2- 172.21.15.64:0/1575304140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3841618e0 0x7fa38406b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.267+0000 7fa382ffd640 1 -- 172.21.15.64:0/1575304140 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2831686715 0 0) 0x7fa38415ebb0 con 0x7fa3840d0150 2024-03-04T07:27:33.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.267+0000 7fa383fff640 1 -- 172.21.15.64:0/1575304140 >> v1:172.21.15.66:6789/0 conn(0x7fa3840d0150 legacy=0x7fa3840d05e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.267+0000 7fa383fff640 1 -- 172.21.15.64:0/1575304140 >> v1:172.21.15.64:6789/0 conn(0x7fa3840ce4f0 legacy=0x7fa3840ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.267+0000 7fa383fff640 1 -- 172.21.15.64:0/1575304140 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa38415ee90 con 0x7fa3841618e0 2024-03-04T07:27:33.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.267+0000 7fa383fff640 1 --2- 172.21.15.64:0/1575304140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3841618e0 0x7fa38406b130 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fa3780097d0 tx=0x7fa37802b800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e15d21ebcd4b09e0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa382ffd640 1 -- 172.21.15.64:0/1575304140 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa378037070 con 0x7fa3841618e0 2024-03-04T07:27:33.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa382ffd640 1 -- 172.21.15.64:0/1575304140 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa378030710 con 0x7fa3841618e0 2024-03-04T07:27:33.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa382ffd640 1 -- 172.21.15.64:0/1575304140 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa378030ae0 con 0x7fa3841618e0 2024-03-04T07:27:33.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa38a783640 1 -- 172.21.15.64:0/1575304140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3841618e0 msgr2=0x7fa38406b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa38a783640 1 --2- 172.21.15.64:0/1575304140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3841618e0 0x7fa38406b130 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fa3780097d0 tx=0x7fa37802b800 comp rx=0 tx=0).stop 2024-03-04T07:27:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa38a783640 1 -- 172.21.15.64:0/1575304140 shutdown_connections 2024-03-04T07:27:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa38a783640 1 --2- 172.21.15.64:0/1575304140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3841618e0 0x7fa38406b130 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa38a783640 1 -- 172.21.15.64:0/1575304140 >> 172.21.15.64:0/1575304140 conn(0x7fa3840b3660 msgr2=0x7fa3840b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa38a783640 1 -- 172.21.15.64:0/1575304140 shutdown_connections 2024-03-04T07:27:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.268+0000 7fa38a783640 1 -- 172.21.15.64:0/1575304140 wait complete. 2024-03-04T07:27:33.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.269+0000 7fa38a783640 1 Processor -- start 2024-03-04T07:27:33.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.269+0000 7fa38a783640 1 -- start start 2024-03-04T07:27:33.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa38a783640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3840ce4f0 0x7fa3840b57e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa38a783640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3840d0150 0x7fa3840c9ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa38a783640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3840cb2f0 0x7fa3840c7f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa38a783640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa38406b670 con 0x7fa3840cb2f0 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa38a783640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa38415f450 con 0x7fa3840ce4f0 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa38a783640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa38415e7d0 con 0x7fa3840d0150 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa388d20640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3840cb2f0 0x7fa3840c7f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa388d20640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3840cb2f0 0x7fa3840c7f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51762/0 (socket says 172.21.15.64:51762) 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa388d20640 1 -- 172.21.15.64:0/4231901025 learned_addr learned my addr 172.21.15.64:0/4231901025 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa3837fe640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3840d0150 0x7fa3840c9ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.270+0000 7fa383fff640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3840ce4f0 0x7fa3840b57e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa388d20640 1 -- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3840d0150 msgr2=0x7fa3840c9ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa388d20640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3840d0150 0x7fa3840c9ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa388d20640 1 -- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3840ce4f0 msgr2=0x7fa3840b57e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa388d20640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3840ce4f0 0x7fa3840b57e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa388d20640 1 -- 172.21.15.64:0/4231901025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3840ca250 con 0x7fa3840cb2f0 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa388d20640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3840cb2f0 0x7fa3840c7f50 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fa37c00c760 tx=0x7fa37c00cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa380ff9640 1 -- 172.21.15.64:0/4231901025 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa37c003da0 con 0x7fa3840cb2f0 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa380ff9640 1 -- 172.21.15.64:0/4231901025 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa37c005070 con 0x7fa3840cb2f0 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa3840ca530 con 0x7fa3840cb2f0 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa380ff9640 1 -- 172.21.15.64:0/4231901025 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa37c005460 con 0x7fa3840cb2f0 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.271+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa3840caaf0 con 0x7fa3840cb2f0 2024-03-04T07:27:33.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.272+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa38409fc50 con 0x7fa3840cb2f0 2024-03-04T07:27:33.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.272+0000 7fa380ff9640 1 -- 172.21.15.64:0/4231901025 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa37c005600 con 0x7fa3840cb2f0 2024-03-04T07:27:33.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.273+0000 7fa380ff9640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa37006b4d0 0x7fa37006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.273+0000 7fa380ff9640 1 -- 172.21.15.64:0/4231901025 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa37c08a020 con 0x7fa3840cb2f0 2024-03-04T07:27:33.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.273+0000 7fa383fff640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa37006b4d0 0x7fa37006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.273+0000 7fa383fff640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa37006b4d0 0x7fa37006d990 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fa378034000 tx=0x7fa378008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.277+0000 7fa380ff9640 1 -- 172.21.15.64:0/4231901025 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa37c058850 con 0x7fa3840cb2f0 2024-03-04T07:27:33.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.465+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fa38404f410 con 0x7fa3840cb2f0 2024-03-04T07:27:33.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.465+0000 7fa380ff9640 1 -- 172.21.15.64:0/4231901025 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v21) v1 ==== 104+0+11 (secure 0 0 0) 0x7fa37c05c8f0 con 0x7fa3840cb2f0 2024-03-04T07:27:33.466 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:33.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa37006b4d0 msgr2=0x7fa37006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa37006b4d0 0x7fa37006d990 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fa378034000 tx=0x7fa378008030 comp rx=0 tx=0).stop 2024-03-04T07:27:33.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3840cb2f0 msgr2=0x7fa3840c7f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3840cb2f0 0x7fa3840c7f50 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fa37c00c760 tx=0x7fa37c00cc20 comp rx=0 tx=0).stop 2024-03-04T07:27:33.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 shutdown_connections 2024-03-04T07:27:33.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa37006b4d0 0x7fa37006d990 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa3840cb2f0 0x7fa3840c7f50 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa3840d0150 0x7fa3840c9ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 --2- 172.21.15.64:0/4231901025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa3840ce4f0 0x7fa3840b57e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 >> 172.21.15.64:0/4231901025 conn(0x7fa3840b3660 msgr2=0x7fa3840b3a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:33.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 shutdown_connections 2024-03-04T07:27:33.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.467+0000 7fa38a783640 1 -- 172.21.15.64:0/4231901025 wait complete. 2024-03-04T07:27:33.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:33.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.582+0000 7fe4b74c2640 1 Processor -- start 2024-03-04T07:27:33.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.583+0000 7fe4b74c2640 1 -- start start 2024-03-04T07:27:33.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.583+0000 7fe4b74c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b00ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.583+0000 7fe4b74c2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe4b00b6d10 0x7fe4b00b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.583+0000 7fe4b74c2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4b00d3740 con 0x7fe4b00d29d0 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.583+0000 7fe4b74c2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe4b00d3400 con 0x7fe4b00b6d10 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.583+0000 7fe4b74c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4b00d3120 con 0x7fe4b015dd90 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b4a5d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe4b00d29d0 0x7fe4b00d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60596/0 (socket says 172.21.15.64:60596) 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b4a5d640 1 -- 172.21.15.64:0/3212106081 learned_addr learned my addr 172.21.15.64:0/3212106081 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b5a5f640 1 --2- 172.21.15.64:0/3212106081 >> v2:172.21.15.66:3300/0 conn(0x7fe4b00b6d10 0x7fe4b00b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b525e640 1 --2- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b00ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4a3fff640 1 -- 172.21.15.64:0/3212106081 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1669079101 0 0) 0x7fe4b00d3740 con 0x7fe4b00d29d0 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4a3fff640 1 -- 172.21.15.64:0/3212106081 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe49c003520 con 0x7fe4b00d29d0 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b5a5f640 1 -- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 msgr2=0x7fe4b00ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b5a5f640 1 --2- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b00ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b5a5f640 1 -- 172.21.15.64:0/3212106081 >> v1:172.21.15.64:6789/0 conn(0x7fe4b00d29d0 legacy=0x7fe4b00d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.584+0000 7fe4b5a5f640 1 -- 172.21.15.64:0/3212106081 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4b00d39e0 con 0x7fe4b00b6d10 2024-03-04T07:27:33.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b5a5f640 1 --2- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b00b71a0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fe4a8009d90 tx=0x7fe4a802bf30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7f2c2e9b5ea0d624 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4a3fff640 1 -- 172.21.15.64:0/3212106081 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4a8039070 con 0x7fe4b00b6d10 2024-03-04T07:27:33.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4a3fff640 1 -- 172.21.15.64:0/3212106081 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4a802cce0 con 0x7fe4b00b6d10 2024-03-04T07:27:33.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4a3fff640 1 -- 172.21.15.64:0/3212106081 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4a8034660 con 0x7fe4b00b6d10 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 -- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 msgr2=0x7fe4b00b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b00b71a0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fe4a8009d90 tx=0x7fe4a802bf30 comp rx=0 tx=0).stop 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 -- 172.21.15.64:0/3212106081 shutdown_connections 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b00b71a0 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/3212106081 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b00ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 -- 172.21.15.64:0/3212106081 >> 172.21.15.64:0/3212106081 conn(0x7fe4b0071960 msgr2=0x7fe4b0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 -- 172.21.15.64:0/3212106081 shutdown_connections 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.585+0000 7fe4b74c2640 1 -- 172.21.15.64:0/3212106081 wait complete. 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 Processor -- start 2024-03-04T07:27:33.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 -- start start 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b01e7760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4b00d29d0 0x7fe4b0152540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b01e7fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4b00d2df0 con 0x7fe4b00d29d0 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4b015e170 con 0x7fe4b00b6d10 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b74c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4b00d48f0 con 0x7fe4b015dd90 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b4a5d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4b00d29d0 0x7fe4b0152540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b4a5d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4b00d29d0 0x7fe4b0152540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45352/0 (socket says 172.21.15.64:45352) 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b4a5d640 1 -- 172.21.15.64:0/362807369 learned_addr learned my addr 172.21.15.64:0/362807369 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b525e640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b01e7760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.587+0000 7fe4b5a5f640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b01e7fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b4a5d640 1 -- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 msgr2=0x7fe4b01e7fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b4a5d640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b01e7fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b4a5d640 1 -- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 msgr2=0x7fe4b01e7760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b4a5d640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b01e7760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b4a5d640 1 -- 172.21.15.64:0/362807369 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4b01e9190 con 0x7fe4b00d29d0 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b525e640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b01e7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b4a5d640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4b00d29d0 0x7fe4b0152540 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fe4ac00c430 tx=0x7fe4ac00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4a1ffb640 1 -- 172.21.15.64:0/362807369 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4ac017070 con 0x7fe4b00d29d0 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4a1ffb640 1 -- 172.21.15.64:0/362807369 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4ac005ce0 con 0x7fe4b00d29d0 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4b015e5f0 con 0x7fe4b00d29d0 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4a1ffb640 1 -- 172.21.15.64:0/362807369 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4ac0072f0 con 0x7fe4b00d29d0 2024-03-04T07:27:33.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.588+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4b0160df0 con 0x7fe4b00d29d0 2024-03-04T07:27:33.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.589+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe4b0072a30 con 0x7fe4b00d29d0 2024-03-04T07:27:33.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.590+0000 7fe4a1ffb640 1 -- 172.21.15.64:0/362807369 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe4ac01a070 con 0x7fe4b00d29d0 2024-03-04T07:27:33.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.590+0000 7fe4a1ffb640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe49406b4d0 0x7fe49406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.590+0000 7fe4a1ffb640 1 -- 172.21.15.64:0/362807369 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(21..21 src has 1..21) v4 ==== 5825+0+0 (secure 0 0 0) 0x7fe4ac0895f0 con 0x7fe4b00d29d0 2024-03-04T07:27:33.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.590+0000 7fe4b525e640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe49406b4d0 0x7fe49406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.591+0000 7fe4b525e640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe49406b4d0 0x7fe49406d990 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fe4a4009e90 tx=0x7fe4a4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.595+0000 7fe4a1ffb640 1 -- 172.21.15.64:0/362807369 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe4b01e9190 con 0x7fe4b00d29d0 2024-03-04T07:27:33.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.778+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fe4b004f410 con 0x7fe4b00d29d0 2024-03-04T07:27:33.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.778+0000 7fe4a1ffb640 1 -- 172.21.15.64:0/362807369 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v21) v1 ==== 100+0+11 (secure 0 0 0) 0x7fe4ac057e30 con 0x7fe4b00d29d0 2024-03-04T07:27:33.779 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe49406b4d0 msgr2=0x7fe49406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe49406b4d0 0x7fe49406d990 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fe4a4009e90 tx=0x7fe4a4009190 comp rx=0 tx=0).stop 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4b00d29d0 msgr2=0x7fe4b0152540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4b00d29d0 0x7fe4b0152540 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fe4ac00c430 tx=0x7fe4ac00c8f0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 shutdown_connections 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe49406b4d0 0x7fe49406d990 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4b015dd90 0x7fe4b01e7fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4b00d29d0 0x7fe4b0152540 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 --2- 172.21.15.64:0/362807369 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4b00b6d10 0x7fe4b01e7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 >> 172.21.15.64:0/362807369 conn(0x7fe4b0071960 msgr2=0x7fe4b00c78b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:33.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 shutdown_connections 2024-03-04T07:27:33.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.780+0000 7fe4b74c2640 1 -- 172.21.15.64:0/362807369 wait complete. 2024-03-04T07:27:33.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:27:33.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.894+0000 7fd874e4e640 1 Processor -- start 2024-03-04T07:27:33.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.894+0000 7fd874e4e640 1 -- start start 2024-03-04T07:27:33.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd874e4e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd87006e700 0x7fd87015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd874e4e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd870072660 0x7fd87006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd874e4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd874e4e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd8700b4160 con 0x7fd87006e700 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd874e4e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd8700b5b40 con 0x7fd870072660 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd874e4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd8700b3ba0 con 0x7fd8700b55d0 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd86e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd87006e700 0x7fd87015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd86e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd87006e700 0x7fd87015ef80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45364/0 (socket says 172.21.15.64:45364) 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd86e59c640 1 -- 172.21.15.64:0/2186725723 learned_addr learned my addr 172.21.15.64:0/2186725723 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd86dd9b640 1 --2- 172.21.15.64:0/2186725723 >> v2:172.21.15.66:3300/0 conn(0x7fd870072660 0x7fd87006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.895+0000 7fd86ed9d640 1 --2- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86dd9b640 1 -- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 msgr2=0x7fd870071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86dd9b640 1 --2- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870071980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86dd9b640 1 -- 172.21.15.64:0/2186725723 >> v2:172.21.15.64:3300/0 conn(0x7fd87006e700 msgr2=0x7fd87015ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86dd9b640 1 --2- 172.21.15.64:0/2186725723 >> v2:172.21.15.64:3300/0 conn(0x7fd87006e700 0x7fd87015ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86dd9b640 1 -- 172.21.15.64:0/2186725723 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8700b3e80 con 0x7fd870072660 2024-03-04T07:27:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86e59c640 1 --2- 172.21.15.64:0/2186725723 >> v2:172.21.15.64:3300/0 conn(0x7fd87006e700 0x7fd87015ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86dd9b640 1 --2- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd870072660 0x7fd87006d9e0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fd870147ce0 tx=0x7fd86402b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4c8c007f44ba76d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.896+0000 7fd86d59a640 1 -- 172.21.15.64:0/2186725723 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd8640315a0 con 0x7fd870072660 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd86d59a640 1 -- 172.21.15.64:0/2186725723 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd864037070 con 0x7fd870072660 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd86d59a640 1 -- 172.21.15.64:0/2186725723 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd864031960 con 0x7fd870072660 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 -- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd870072660 msgr2=0x7fd87006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 --2- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd870072660 0x7fd87006d9e0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fd870147ce0 tx=0x7fd86402b840 comp rx=0 tx=0).stop 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 -- 172.21.15.64:0/2186725723 shutdown_connections 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 --2- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870071980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 --2- 172.21.15.64:0/2186725723 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd870072660 0x7fd87006d9e0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 --2- 172.21.15.64:0/2186725723 >> v2:172.21.15.64:3300/0 conn(0x7fd87006e700 0x7fd87015ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 -- 172.21.15.64:0/2186725723 >> 172.21.15.64:0/2186725723 conn(0x7fd8700b2720 msgr2=0x7fd8700b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:33.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 -- 172.21.15.64:0/2186725723 shutdown_connections 2024-03-04T07:27:33.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.897+0000 7fd874e4e640 1 -- 172.21.15.64:0/2186725723 wait complete. 2024-03-04T07:27:33.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.898+0000 7fd874e4e640 1 Processor -- start 2024-03-04T07:27:33.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.898+0000 7fd874e4e640 1 -- start start 2024-03-04T07:27:33.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd874e4e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd87006e700 0x7fd8701518f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd874e4e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd870072660 0x7fd87015b470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd874e4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870159c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd874e4e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd87006ec60 con 0x7fd870072660 2024-03-04T07:27:33.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd874e4e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd87006eae0 con 0x7fd87006e700 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd874e4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd870071ec0 con 0x7fd8700b55d0 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd86dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd870072660 0x7fd87015b470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd86dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd870072660 0x7fd87015b470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45366/0 (socket says 172.21.15.64:45366) 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd86e59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd87006e700 0x7fd8701518f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd86dd9b640 1 -- 172.21.15.64:0/1983262912 learned_addr learned my addr 172.21.15.64:0/1983262912 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.899+0000 7fd86ed9d640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870159c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd86dd9b640 1 -- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 msgr2=0x7fd870159c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd86dd9b640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870159c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd86dd9b640 1 -- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd87006e700 msgr2=0x7fd8701518f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd86dd9b640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd87006e700 0x7fd8701518f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd86dd9b640 1 -- 172.21.15.64:0/1983262912 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8701f3b10 con 0x7fd870072660 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd86e59c640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd87006e700 0x7fd8701518f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd86dd9b640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd870072660 0x7fd87015b470 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7fd86402be50 tx=0x7fd8640046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd85effd640 1 -- 172.21.15.64:0/1983262912 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd864032630 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd85effd640 1 -- 172.21.15.64:0/1983262912 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd8640327d0 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd85effd640 1 -- 172.21.15.64:0/1983262912 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd864032bc0 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd8701f3df0 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.900+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd8701f43b0 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.901+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd8701f45f0 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.901+0000 7fd85effd640 1 -- 172.21.15.64:0/1983262912 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd86404c020 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.902+0000 7fd85effd640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd85006b4d0 0x7fd85006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.902+0000 7fd85effd640 1 -- 172.21.15.64:0/1983262912 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(21..21 src has 1..21) v4 ==== 5825+0+0 (secure 0 0 0) 0x7fd86402f020 con 0x7fd870072660 2024-03-04T07:27:33.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.902+0000 7fd86e59c640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd85006b4d0 0x7fd85006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:33.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.902+0000 7fd86e59c640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd85006b4d0 0x7fd85006d990 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fd870152060 tx=0x7fd86000d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:33.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:33.906+0000 7fd85effd640 1 -- 172.21.15.64:0/1983262912 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd864040070 con 0x7fd870072660 2024-03-04T07:27:34.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.099+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fd870067ff0 con 0x7fd870072660 2024-03-04T07:27:34.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.099+0000 7fd85effd640 1 -- 172.21.15.64:0/1983262912 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v21) v1 ==== 80+0+0 (secure 0 0 0) 0x7fd86407a3b0 con 0x7fd870072660 2024-03-04T07:27:34.100 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:27:34.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.101+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd85006b4d0 msgr2=0x7fd85006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.101+0000 7fd874e4e640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd85006b4d0 0x7fd85006d990 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fd870152060 tx=0x7fd86000d040 comp rx=0 tx=0).stop 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.101+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd870072660 msgr2=0x7fd87015b470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.101+0000 7fd874e4e640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd870072660 0x7fd87015b470 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7fd86402be50 tx=0x7fd8640046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.101+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 shutdown_connections 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.101+0000 7fd874e4e640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd85006b4d0 0x7fd85006d990 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.101+0000 7fd874e4e640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8700b55d0 0x7fd870159c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.102+0000 7fd874e4e640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd870072660 0x7fd87015b470 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.102+0000 7fd874e4e640 1 --2- 172.21.15.64:0/1983262912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd87006e700 0x7fd8701518f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.102+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 >> 172.21.15.64:0/1983262912 conn(0x7fd8700b2720 msgr2=0x7fd87015e500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.102+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 shutdown_connections 2024-03-04T07:27:34.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.102+0000 7fd874e4e640 1 -- 172.21.15.64:0/1983262912 wait complete. 2024-03-04T07:27:34.113 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi066.front.sepia.ceph.com... 2024-03-04T07:27:34.113 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:34.113 DEBUG:teuthology.orchestra.run.smithi066:> sudo fusermount -u /home/ubuntu/cephtest/mnt.3 2024-03-04T07:27:34.145 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:34.145 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:27:34.145 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:34.146 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:27:34.200 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:27:34.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:34.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.302+0000 7fa1992a4640 1 Processor -- start 2024-03-04T07:27:34.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.302+0000 7fa1992a4640 1 -- start start 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1992a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa194157f60 0x7fa19415b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1992a4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa1940cbab0 0x7fa194070b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1992a4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa19406c7b0 con 0x7fa194157b80 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1992a4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa19406ca90 con 0x7fa1940cbab0 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1992a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa19406cd70 con 0x7fa194157f60 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1937fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa1940cbab0 0x7fa194070b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1937fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa1940cbab0 0x7fa194070b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45368/0 (socket says 172.21.15.64:45368) 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1937fe640 1 -- 172.21.15.64:0/1346522226 learned_addr learned my addr 172.21.15.64:0/1346522226 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa18bfff640 1 --2- 172.21.15.64:0/1346522226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa194157f60 0x7fa19415b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1937fe640 1 -- 172.21.15.64:0/1346522226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa194157f60 msgr2=0x7fa19415b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1937fe640 1 --2- 172.21.15.64:0/1346522226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa194157f60 0x7fa19415b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1937fe640 1 -- 172.21.15.64:0/1346522226 >> v1:172.21.15.66:6789/0 conn(0x7fa194157b80 legacy=0x7fa19415b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.303+0000 7fa1937fe640 1 -- 172.21.15.64:0/1346522226 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa19406d050 con 0x7fa1940cbab0 2024-03-04T07:27:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.304+0000 7fa1937fe640 1 --2- 172.21.15.64:0/1346522226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1940cbab0 0x7fa194070b40 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7fa18c009e50 tx=0x7fa18c02bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f431d7481795df6e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.304+0000 7fa1927fc640 1 -- 172.21.15.64:0/1346522226 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18c02cb00 con 0x7fa1940cbab0 2024-03-04T07:27:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.304+0000 7fa1927fc640 1 -- 172.21.15.64:0/1346522226 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa18c02cca0 con 0x7fa1940cbab0 2024-03-04T07:27:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.304+0000 7fa1927fc640 1 -- 172.21.15.64:0/1346522226 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18c034610 con 0x7fa1940cbab0 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 -- 172.21.15.64:0/1346522226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1940cbab0 msgr2=0x7fa194070b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 --2- 172.21.15.64:0/1346522226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1940cbab0 0x7fa194070b40 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7fa18c009e50 tx=0x7fa18c02bf60 comp rx=0 tx=0).stop 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 -- 172.21.15.64:0/1346522226 shutdown_connections 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 --2- 172.21.15.64:0/1346522226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1940cbab0 0x7fa194070b40 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 --2- 172.21.15.64:0/1346522226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa194157f60 0x7fa19415b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 -- 172.21.15.64:0/1346522226 >> 172.21.15.64:0/1346522226 conn(0x7fa194066d00 msgr2=0x7fa1940670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 -- 172.21.15.64:0/1346522226 shutdown_connections 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.305+0000 7fa1992a4640 1 -- 172.21.15.64:0/1346522226 wait complete. 2024-03-04T07:27:34.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.306+0000 7fa1992a4640 1 Processor -- start 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.306+0000 7fa1992a4640 1 -- start start 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1940cbab0 0x7fa19413f5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa194157b80 0x7fa194151320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa194157f60 0x7fa1940ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa194071080 con 0x7fa194157f60 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1940cc260 con 0x7fa194157b80 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa194071200 con 0x7fa1940cbab0 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa194157f60 0x7fa1940ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa194157f60 0x7fa1940ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45370/0 (socket says 172.21.15.64:45370) 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 -- 172.21.15.64:0/3953579336 learned_addr learned my addr 172.21.15.64:0/3953579336 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa192ffd640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1940cbab0 0x7fa19413f5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa18bfff640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa194157b80 0x7fa194151320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 -- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1940cbab0 msgr2=0x7fa19413f5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1940cbab0 0x7fa19413f5d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 -- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa194157b80 msgr2=0x7fa194151320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa194157b80 0x7fa194151320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 -- 172.21.15.64:0/3953579336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1940af410 con 0x7fa194157f60 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1937fe640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa194157f60 0x7fa1940ad8b0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7fa18c009f70 tx=0x7fa18c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa18b7fe640 1 -- 172.21.15.64:0/3953579336 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18c043020 con 0x7fa194157f60 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa18b7fe640 1 -- 172.21.15.64:0/3953579336 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa18c02ce50 con 0x7fa194157f60 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa1940af6f0 con 0x7fa194157f60 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa18b7fe640 1 -- 172.21.15.64:0/3953579336 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18c009510 con 0x7fa194157f60 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.307+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa194147250 con 0x7fa194157f60 2024-03-04T07:27:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.309+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa194147450 con 0x7fa194157f60 2024-03-04T07:27:34.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.309+0000 7fa18b7fe640 1 -- 172.21.15.64:0/3953579336 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa18c046020 con 0x7fa194157f60 2024-03-04T07:27:34.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.309+0000 7fa18b7fe640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa17406b500 0x7fa17406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.309+0000 7fa18b7fe640 1 -- 172.21.15.64:0/3953579336 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(21..21 src has 1..21) v4 ==== 5825+0+0 (secure 0 0 0) 0x7fa18c0ac5e0 con 0x7fa194157f60 2024-03-04T07:27:34.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.309+0000 7fa192ffd640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa17406b500 0x7fa17406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.310+0000 7fa192ffd640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa17406b500 0x7fa17406d9c0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7fa1941f75e0 tx=0x7fa184009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.314+0000 7fa18b7fe640 1 -- 172.21.15.64:0/3953579336 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa194147450 con 0x7fa194157f60 2024-03-04T07:27:34.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.536+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa194067ff0 con 0x7fa194157f60 2024-03-04T07:27:34.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.537+0000 7fa18b7fe640 1 -- 172.21.15.64:0/3953579336 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7fa18c04d030 con 0x7fa194157f60 2024-03-04T07:27:34.538 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:34.538 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:27:34.538 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:27:34.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa17406b500 msgr2=0x7fa17406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa17406b500 0x7fa17406d9c0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7fa1941f75e0 tx=0x7fa184009040 comp rx=0 tx=0).stop 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa194157f60 msgr2=0x7fa1940ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa194157f60 0x7fa1940ad8b0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7fa18c009f70 tx=0x7fa18c004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 shutdown_connections 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa17406b500 0x7fa17406d9c0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa194157f60 0x7fa1940ad8b0 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa194157b80 0x7fa194151320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 --2- 172.21.15.64:0/3953579336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1940cbab0 0x7fa19413f5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 >> 172.21.15.64:0/3953579336 conn(0x7fa194066d00 msgr2=0x7fa194153f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 shutdown_connections 2024-03-04T07:27:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.539+0000 7fa1992a4640 1 -- 172.21.15.64:0/3953579336 wait complete. 2024-03-04T07:27:34.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:34.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.653+0000 7f741ce88640 1 Processor -- start 2024-03-04T07:27:34.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.653+0000 7f741ce88640 1 -- start start 2024-03-04T07:27:34.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f741ce88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f741815dca0 0x7f741806fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f741ce88640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f74180b00c0 0x7f74180b3ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f741ce88640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7418161c20 con 0x7f741815e080 2024-03-04T07:27:34.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f741ce88640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f74181619d0 con 0x7f74180b00c0 2024-03-04T07:27:34.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f741ce88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7418161f00 con 0x7f741815dca0 2024-03-04T07:27:34.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f740ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f741815e080 0x7f7418066d00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60600/0 (socket says 172.21.15.64:60600) 2024-03-04T07:27:34.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f740ffff640 1 -- 172.21.15.64:0/4213715390 learned_addr learned my addr 172.21.15.64:0/4213715390 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f7416d9d640 1 --2- 172.21.15.64:0/4213715390 >> v2:172.21.15.66:3300/0 conn(0x7f74180b00c0 0x7f74180b3ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f741659c640 1 --2- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f741815dca0 0x7f741806fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f7415d9b640 1 -- 172.21.15.64:0/4213715390 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1097590543 0 0) 0x7f7418161c20 con 0x7f741815e080 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f7415d9b640 1 -- 172.21.15.64:0/4213715390 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7404003520 con 0x7f741815e080 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f7416d9d640 1 -- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f741815dca0 msgr2=0x7f741806fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f7416d9d640 1 --2- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f741815dca0 0x7f741806fd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f7416d9d640 1 -- 172.21.15.64:0/4213715390 >> v1:172.21.15.64:6789/0 conn(0x7f741815e080 legacy=0x7f7418066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f7416d9d640 1 -- 172.21.15.64:0/4213715390 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74181621e0 con 0x7f74180b00c0 2024-03-04T07:27:34.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.654+0000 7f741659c640 1 --2- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f741815dca0 0x7f741806fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.655+0000 7f7416d9d640 1 --2- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74180b00c0 0x7f74180b3ca0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f7410009d90 tx=0x7f741002bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=694a29abe2c4c3f7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.655+0000 7f7415d9b640 1 -- 172.21.15.64:0/4213715390 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7410039070 con 0x7f74180b00c0 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.655+0000 7f7415d9b640 1 -- 172.21.15.64:0/4213715390 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f741002cc50 con 0x7f74180b00c0 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.655+0000 7f7415d9b640 1 -- 172.21.15.64:0/4213715390 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7410034440 con 0x7f74180b00c0 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 -- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74180b00c0 msgr2=0x7f74180b3ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 --2- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74180b00c0 0x7f74180b3ca0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f7410009d90 tx=0x7f741002bea0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 -- 172.21.15.64:0/4213715390 shutdown_connections 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 --2- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74180b00c0 0x7f74180b3ca0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 --2- 172.21.15.64:0/4213715390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f741815dca0 0x7f741806fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 -- 172.21.15.64:0/4213715390 >> 172.21.15.64:0/4213715390 conn(0x7f74180d0990 msgr2=0x7f74180d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 -- 172.21.15.64:0/4213715390 shutdown_connections 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.656+0000 7f741ce88640 1 -- 172.21.15.64:0/4213715390 wait complete. 2024-03-04T07:27:34.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.657+0000 7f741ce88640 1 Processor -- start 2024-03-04T07:27:34.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.657+0000 7f741ce88640 1 -- start start 2024-03-04T07:27:34.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f741ce88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74180b00c0 0x7f7418151ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f741ce88640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f741815dca0 0x7f741815a9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f741ce88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f741815aee0 0x7f7418160750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f741ce88640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74180703d0 con 0x7f741815aee0 2024-03-04T07:27:34.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f741ce88640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7418070250 con 0x7f741815dca0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f741ce88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7418067410 con 0x7f74180b00c0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f741815aee0 0x7f7418160750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f741815aee0 0x7f7418160750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45386/0 (socket says 172.21.15.64:45386) 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 -- 172.21.15.64:0/1882696344 learned_addr learned my addr 172.21.15.64:0/1882696344 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f740ffff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f741815dca0 0x7f741815a9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f741659c640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74180b00c0 0x7f7418151ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 -- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74180b00c0 msgr2=0x7f7418151ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74180b00c0 0x7f7418151ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 -- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f741815dca0 msgr2=0x7f741815a9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f741815dca0 0x7f741815a9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 -- 172.21.15.64:0/1882696344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f741815e470 con 0x7f741815aee0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f740ffff640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f741815dca0 0x7f741815a9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.658+0000 7f7416d9d640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f741815aee0 0x7f7418160750 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f7410009d90 tx=0x7f7410004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.659+0000 7f740effd640 1 -- 172.21.15.64:0/1882696344 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7410042030 con 0x7f741815aee0 2024-03-04T07:27:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.659+0000 7f740effd640 1 -- 172.21.15.64:0/1882696344 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f741003a040 con 0x7f741815aee0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.659+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f741815e730 con 0x7f741815aee0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.659+0000 7f740effd640 1 -- 172.21.15.64:0/1882696344 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7410039070 con 0x7f741815aee0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.659+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f741815ecf0 con 0x7f741815aee0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.660+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7418068700 con 0x7f741815aee0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.660+0000 7f740effd640 1 -- 172.21.15.64:0/1882696344 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7410045070 con 0x7f741815aee0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.660+0000 7f740effd640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f73f806b500 0x7f73f806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.661+0000 7f740effd640 1 -- 172.21.15.64:0/1882696344 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f74100abd40 con 0x7f741815aee0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.661+0000 7f741659c640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f73f806b500 0x7f73f806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.661+0000 7f741659c640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f73f806b500 0x7f73f806d9c0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f7408009560 tx=0x7f7408009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.665+0000 7f740effd640 1 -- 172.21.15.64:0/1882696344 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f741004c070 con 0x7f741815aee0 2024-03-04T07:27:34.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.849+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f74180afe70 con 0x7f741815aee0 2024-03-04T07:27:34.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.850+0000 7f740effd640 1 -- 172.21.15.64:0/1882696344 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16807 (secure 0 0 0) 0x7f741007a640 con 0x7f741815aee0 2024-03-04T07:27:34.851 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:34.852 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:34.409323+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.12","mappings":[{"from":0,"to":1}]},{"pgid":"3.19","mappings":[{"from":3,"to":2}]},{"pgid":"3.28","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:34.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f73f806b500 msgr2=0x7f73f806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f73f806b500 0x7f73f806d9c0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f7408009560 tx=0x7f7408009190 comp rx=0 tx=0).stop 2024-03-04T07:27:34.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f741815aee0 msgr2=0x7f7418160750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f741815aee0 0x7f7418160750 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f7410009d90 tx=0x7f7410004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 shutdown_connections 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f73f806b500 0x7f73f806d9c0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f741815aee0 0x7f7418160750 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f741815dca0 0x7f741815a9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 --2- 172.21.15.64:0/1882696344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74180b00c0 0x7f7418151ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 >> 172.21.15.64:0/1882696344 conn(0x7f74180d0990 msgr2=0x7f74180719b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.852+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 shutdown_connections 2024-03-04T07:27:34.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.853+0000 7f741ce88640 1 -- 172.21.15.64:0/1882696344 wait complete. 2024-03-04T07:27:34.864 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:26:51.244804+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '19', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:26:51.631023+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '20', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:34.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:34.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.968+0000 7f44445ae640 1 Processor -- start 2024-03-04T07:27:34.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.968+0000 7f44445ae640 1 -- start start 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f44445ae640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c0ce4f0 0x7f443c0ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f44445ae640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f443c0d0150 0x7f443c0d05e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f44445ae640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f443c0cee70 con 0x7f443c1618e0 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f44445ae640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f443c0cfce0 con 0x7f443c0d0150 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f44445ae640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f443c15f730 con 0x7f443c0ce4f0 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4442b4b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f443c0d0150 0x7f443c0d05e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4442b4b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f443c0d0150 0x7f443c0d05e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45400/0 (socket says 172.21.15.64:45400) 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4442b4b640 1 -- 172.21.15.64:0/2251811357 learned_addr learned my addr 172.21.15.64:0/2251811357 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4441b49640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c0ce4f0 0x7f443c0ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f444234a640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f443c1618e0 0x7f443c06b130 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60610/0 (socket says 172.21.15.64:60610) 2024-03-04T07:27:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4442b4b640 1 -- 172.21.15.64:0/2251811357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c0ce4f0 msgr2=0x7f443c0ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:34.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4442b4b640 1 --2- 172.21.15.64:0/2251811357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c0ce4f0 0x7f443c0ce930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4442b4b640 1 -- 172.21.15.64:0/2251811357 >> v1:172.21.15.64:6789/0 conn(0x7f443c1618e0 legacy=0x7f443c06b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4442b4b640 1 -- 172.21.15.64:0/2251811357 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f443c15ebb0 con 0x7f443c0d0150 2024-03-04T07:27:34.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4441348640 1 -- 172.21.15.64:0/2251811357 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 550892366 0 0) 0x7f4438001110 con 0x7f443c1618e0 2024-03-04T07:27:34.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.969+0000 7f4441b49640 1 --2- 172.21.15.64:0/2251811357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c0ce4f0 0x7f443c0ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.970+0000 7f4442b4b640 1 --2- 172.21.15.64:0/2251811357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c0d05e0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f4434009e50 tx=0x7f443402bff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ceefa532f2d23e7f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.970+0000 7f4441348640 1 -- 172.21.15.64:0/2251811357 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f443402cb90 con 0x7f443c0d0150 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.970+0000 7f4441348640 1 -- 172.21.15.64:0/2251811357 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f443402cd30 con 0x7f443c0d0150 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.970+0000 7f4441348640 1 -- 172.21.15.64:0/2251811357 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4434034610 con 0x7f443c0d0150 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 -- 172.21.15.64:0/2251811357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 msgr2=0x7f443c0d05e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 --2- 172.21.15.64:0/2251811357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c0d05e0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f4434009e50 tx=0x7f443402bff0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 -- 172.21.15.64:0/2251811357 shutdown_connections 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 --2- 172.21.15.64:0/2251811357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c0d05e0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 --2- 172.21.15.64:0/2251811357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c0ce4f0 0x7f443c0ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 -- 172.21.15.64:0/2251811357 >> 172.21.15.64:0/2251811357 conn(0x7f443c0b3660 msgr2=0x7f443c0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 -- 172.21.15.64:0/2251811357 shutdown_connections 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.971+0000 7f44445ae640 1 -- 172.21.15.64:0/2251811357 wait complete. 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.972+0000 7f44445ae640 1 Processor -- start 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.972+0000 7f44445ae640 1 -- start start 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f44445ae640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f443c0ce4f0 0x7f443c15fba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f44445ae640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c154f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f44445ae640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c155480 0x7f443c0ca320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f44445ae640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f443c15f450 con 0x7f443c0d0150 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f44445ae640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f443c15f170 con 0x7f443c0ce4f0 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f44445ae640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f443c15e7d0 con 0x7f443c155480 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c154f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c154f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45414/0 (socket says 172.21.15.64:45414) 2024-03-04T07:27:34.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 -- 172.21.15.64:0/1420783832 learned_addr learned my addr 172.21.15.64:0/1420783832 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f444234a640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f443c0ce4f0 0x7f443c15fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4442b4b640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c155480 0x7f443c0ca320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 -- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c155480 msgr2=0x7f443c0ca320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c155480 0x7f443c0ca320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 -- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f443c0ce4f0 msgr2=0x7f443c15fba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f443c0ce4f0 0x7f443c15fba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 -- 172.21.15.64:0/1420783832 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f443c155930 con 0x7f443c0d0150 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f444234a640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f443c0ce4f0 0x7f443c15fba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4441b49640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c154f40 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f442c00e960 tx=0x7f442c00ee20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4432ffd640 1 -- 172.21.15.64:0/1420783832 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f442c017070 con 0x7f443c0d0150 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4432ffd640 1 -- 172.21.15.64:0/1420783832 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f442c00c9a0 con 0x7f443c0d0150 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f443c0b55d0 con 0x7f443c0d0150 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.973+0000 7f4432ffd640 1 -- 172.21.15.64:0/1420783832 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f442c00cd70 con 0x7f443c0d0150 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.974+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f443c0cb380 con 0x7f443c0d0150 2024-03-04T07:27:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.974+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f443c09fc50 con 0x7f443c0d0150 2024-03-04T07:27:34.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.975+0000 7f4432ffd640 1 -- 172.21.15.64:0/1420783832 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f442c01a070 con 0x7f443c0d0150 2024-03-04T07:27:34.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.975+0000 7f4432ffd640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f442006b500 0x7f442006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:34.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.975+0000 7f4432ffd640 1 -- 172.21.15.64:0/1420783832 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f442c08a010 con 0x7f443c0d0150 2024-03-04T07:27:34.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.975+0000 7f444234a640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f442006b500 0x7f442006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:34.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.976+0000 7f444234a640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f442006b500 0x7f442006d9c0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f4438000f60 tx=0x7f4438007b50 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:34.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:34.980+0000 7f4432ffd640 1 -- 172.21.15.64:0/1420783832 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f442c058850 con 0x7f443c0d0150 2024-03-04T07:27:35.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.158+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f443c072cb0 con 0x7f443c0d0150 2024-03-04T07:27:35.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.159+0000 7f4432ffd640 1 -- 172.21.15.64:0/1420783832 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v22) v1 ==== 93+0+10 (secure 0 0 0) 0x7f442c05c8f0 con 0x7f443c0d0150 2024-03-04T07:27:35.159 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:35.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f442006b500 msgr2=0x7f442006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f442006b500 0x7f442006d9c0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f4438000f60 tx=0x7f4438007b50 comp rx=0 tx=0).stop 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 msgr2=0x7f443c154f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c154f40 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f442c00e960 tx=0x7f442c00ee20 comp rx=0 tx=0).stop 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 shutdown_connections 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f442006b500 0x7f442006d9c0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f443c155480 0x7f443c0ca320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f443c0d0150 0x7f443c154f40 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 --2- 172.21.15.64:0/1420783832 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f443c0ce4f0 0x7f443c15fba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 >> 172.21.15.64:0/1420783832 conn(0x7f443c0b3660 msgr2=0x7f443c0cc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 shutdown_connections 2024-03-04T07:27:35.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.161+0000 7f44445ae640 1 -- 172.21.15.64:0/1420783832 wait complete. 2024-03-04T07:27:35.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:27:35.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.275+0000 7f9beaeaf640 1 Processor -- start 2024-03-04T07:27:35.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.275+0000 7f9beaeaf640 1 -- start start 2024-03-04T07:27:35.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.276+0000 7f9beaeaf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9be40b7420 0x7f9be40d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.276+0000 7f9beaeaf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9be40d3740 con 0x7f9be415dd90 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.276+0000 7f9beaeaf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9be40d3120 con 0x7f9be40ceea0 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.276+0000 7f9beaeaf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9be40d3400 con 0x7f9be40b7420 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.276+0000 7f9be8c4b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9be415dd90 0x7f9be40d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60622/0 (socket says 172.21.15.64:60622) 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.276+0000 7f9be8c4b640 1 -- 172.21.15.64:0/1943596747 learned_addr learned my addr 172.21.15.64:0/1943596747 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.276+0000 7f9be944c640 1 --2- 172.21.15.64:0/1943596747 >> v2:172.21.15.66:3300/0 conn(0x7f9be40b7420 0x7f9be40d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.277+0000 7f9be37fe640 1 -- 172.21.15.64:0/1943596747 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 778846952 0 0) 0x7f9be40d3120 con 0x7f9be40ceea0 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.277+0000 7f9be37fe640 1 -- 172.21.15.64:0/1943596747 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9bd4003520 con 0x7f9be40ceea0 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.277+0000 7f9be37fe640 1 -- 172.21.15.64:0/1943596747 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3575904441 0 0) 0x7f9be40d3740 con 0x7f9be415dd90 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.277+0000 7f9be944c640 1 -- 172.21.15.64:0/1943596747 >> v1:172.21.15.66:6789/0 conn(0x7f9be40ceea0 legacy=0x7f9be40b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.277+0000 7f9be944c640 1 -- 172.21.15.64:0/1943596747 >> v1:172.21.15.64:6789/0 conn(0x7f9be415dd90 legacy=0x7f9be40d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.277+0000 7f9be944c640 1 -- 172.21.15.64:0/1943596747 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9be40d39e0 con 0x7f9be40b7420 2024-03-04T07:27:35.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9be944c640 1 --2- 172.21.15.64:0/1943596747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be40b7420 0x7f9be40d29d0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f9bdc009d90 tx=0x7f9bdc02bf50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=42eb44ab3c26a5ab server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9be37fe640 1 -- 172.21.15.64:0/1943596747 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bdc039070 con 0x7f9be40b7420 2024-03-04T07:27:35.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9be37fe640 1 -- 172.21.15.64:0/1943596747 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9bdc02cd00 con 0x7f9be40b7420 2024-03-04T07:27:35.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9be37fe640 1 -- 172.21.15.64:0/1943596747 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bdc034640 con 0x7f9be40b7420 2024-03-04T07:27:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9beaeaf640 1 -- 172.21.15.64:0/1943596747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be40b7420 msgr2=0x7f9be40d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/1943596747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be40b7420 0x7f9be40d29d0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f9bdc009d90 tx=0x7f9bdc02bf50 comp rx=0 tx=0).stop 2024-03-04T07:27:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9beaeaf640 1 -- 172.21.15.64:0/1943596747 shutdown_connections 2024-03-04T07:27:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/1943596747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be40b7420 0x7f9be40d29d0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9beaeaf640 1 -- 172.21.15.64:0/1943596747 >> 172.21.15.64:0/1943596747 conn(0x7f9be4071960 msgr2=0x7f9be4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9beaeaf640 1 -- 172.21.15.64:0/1943596747 shutdown_connections 2024-03-04T07:27:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.278+0000 7f9beaeaf640 1 -- 172.21.15.64:0/1943596747 wait complete. 2024-03-04T07:27:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.279+0000 7f9beaeaf640 1 Processor -- start 2024-03-04T07:27:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.279+0000 7f9beaeaf640 1 -- start start 2024-03-04T07:27:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9beaeaf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9be40ceea0 0x7f9be41589e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9beaeaf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9be415dd90 0x7f9be4147b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9beaeaf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be414b730 0x7f9be4148090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9beaeaf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9be40d48f0 con 0x7f9be40ceea0 2024-03-04T07:27:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9beaeaf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9be415e170 con 0x7f9be414b730 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9beaeaf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9be40d4a70 con 0x7f9be415dd90 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9be8c4b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9be40ceea0 0x7f9be41589e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9be8c4b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9be40ceea0 0x7f9be41589e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45416/0 (socket says 172.21.15.64:45416) 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9be8c4b640 1 -- 172.21.15.64:0/2077341502 learned_addr learned my addr 172.21.15.64:0/2077341502 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9be3fff640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9be415dd90 0x7f9be4147b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.280+0000 7f9be944c640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be414b730 0x7f9be4148090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be8c4b640 1 -- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9be415dd90 msgr2=0x7f9be4147b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be8c4b640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9be415dd90 0x7f9be4147b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be8c4b640 1 -- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be414b730 msgr2=0x7f9be4148090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be8c4b640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be414b730 0x7f9be4148090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be8c4b640 1 -- 172.21.15.64:0/2077341502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9be4156cd0 con 0x7f9be40ceea0 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be8c4b640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9be40ceea0 0x7f9be41589e0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f9bd800c3c0 tx=0x7f9bd800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be17fa640 1 -- 172.21.15.64:0/2077341502 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd8017070 con 0x7f9be40ceea0 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be17fa640 1 -- 172.21.15.64:0/2077341502 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9bd8005ce0 con 0x7f9be40ceea0 2024-03-04T07:27:35.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9be4159120 con 0x7f9be40ceea0 2024-03-04T07:27:35.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9be17fa640 1 -- 172.21.15.64:0/2077341502 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd8007310 con 0x7f9be40ceea0 2024-03-04T07:27:35.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.281+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9be41596e0 con 0x7f9be40ceea0 2024-03-04T07:27:35.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.282+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9be4158f20 con 0x7f9be40ceea0 2024-03-04T07:27:35.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.283+0000 7f9be17fa640 1 -- 172.21.15.64:0/2077341502 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9bd801a070 con 0x7f9be40ceea0 2024-03-04T07:27:35.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.283+0000 7f9be17fa640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9bc806b500 0x7f9bc806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.283+0000 7f9be3fff640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9bc806b500 0x7f9bc806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.283+0000 7f9be17fa640 1 -- 172.21.15.64:0/2077341502 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f9bd80894c0 con 0x7f9be40ceea0 2024-03-04T07:27:35.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.284+0000 7f9be3fff640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9bc806b500 0x7f9bc806d9c0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f9be415c420 tx=0x7f9bd0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.287+0000 7f9be17fa640 1 -- 172.21.15.64:0/2077341502 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9bd8057d00 con 0x7f9be40ceea0 2024-03-04T07:27:35.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.468+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f9be4072a30 con 0x7f9be40ceea0 2024-03-04T07:27:35.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.469+0000 7f9be17fa640 1 -- 172.21.15.64:0/2077341502 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v22) v1 ==== 104+0+11 (secure 0 0 0) 0x7f9bd805bda0 con 0x7f9be40ceea0 2024-03-04T07:27:35.469 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:27:35.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9bc806b500 msgr2=0x7f9bc806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9bc806b500 0x7f9bc806d9c0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f9be415c420 tx=0x7f9bd0009040 comp rx=0 tx=0).stop 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9be40ceea0 msgr2=0x7f9be41589e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9be40ceea0 0x7f9be41589e0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f9bd800c3c0 tx=0x7f9bd800c880 comp rx=0 tx=0).stop 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 shutdown_connections 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9bc806b500 0x7f9bc806d9c0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9be414b730 0x7f9be4148090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9be415dd90 0x7f9be4147b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 --2- 172.21.15.64:0/2077341502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9be40ceea0 0x7f9be41589e0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 >> 172.21.15.64:0/2077341502 conn(0x7f9be4071960 msgr2=0x7f9be415d210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 shutdown_connections 2024-03-04T07:27:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.471+0000 7f9beaeaf640 1 -- 172.21.15.64:0/2077341502 wait complete. 2024-03-04T07:27:35.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:27:35.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.586+0000 7f9e327b5640 1 Processor -- start 2024-03-04T07:27:35.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.586+0000 7f9e327b5640 1 -- start start 2024-03-04T07:27:35.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e327b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0b2b10 0x7f9e2c071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e327b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9e2c15e4b0 con 0x7f9e2c0717b0 2024-03-04T07:27:35.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e327b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9e2c0b97e0 con 0x7f9e2c0b3060 2024-03-04T07:27:35.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e327b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e2c0b7c70 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e2bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9e2c0717b0 0x7f9e2c0b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60628/0 (socket says 172.21.15.64:60628) 2024-03-04T07:27:35.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e2bfff640 1 -- 172.21.15.64:0/3470124411 learned_addr learned my addr 172.21.15.64:0/3470124411 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:35.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e30d52640 1 --2- 172.21.15.64:0/3470124411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0b2b10 0x7f9e2c071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e2affd640 1 -- 172.21.15.64:0/3470124411 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 511673907 0 0) 0x7f9e14001110 con 0x7f9e2c0717b0 2024-03-04T07:27:35.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e2affd640 1 -- 172.21.15.64:0/3470124411 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9e18003520 con 0x7f9e2c0717b0 2024-03-04T07:27:35.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e2affd640 1 -- 172.21.15.64:0/3470124411 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1420948534 0 0) 0x7f9e2c0b97e0 con 0x7f9e2c0b3060 2024-03-04T07:27:35.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e2affd640 1 -- 172.21.15.64:0/3470124411 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9e14001110 con 0x7f9e2c0b3060 2024-03-04T07:27:35.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e30d52640 1 -- 172.21.15.64:0/3470124411 >> v1:172.21.15.66:6789/0 conn(0x7f9e2c0b3060 legacy=0x7f9e2c0b7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e30d52640 1 -- 172.21.15.64:0/3470124411 >> v1:172.21.15.64:6789/0 conn(0x7f9e2c0717b0 legacy=0x7f9e2c0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.587+0000 7f9e30d52640 1 -- 172.21.15.64:0/3470124411 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e2c0b9a80 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.588+0000 7f9e30d52640 1 --2- 172.21.15.64:0/3470124411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0b2b10 0x7f9e2c071b90 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f9e24009dc0 tx=0x7f9e2402bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=527e23d3e4cd327f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.588+0000 7f9e2affd640 1 -- 172.21.15.64:0/3470124411 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e24039070 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.588+0000 7f9e2affd640 1 -- 172.21.15.64:0/3470124411 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e2402ccd0 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.588+0000 7f9e2affd640 1 -- 172.21.15.64:0/3470124411 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e24034640 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.589+0000 7f9e327b5640 1 -- 172.21.15.64:0/3470124411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0b2b10 msgr2=0x7f9e2c071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.589+0000 7f9e327b5640 1 --2- 172.21.15.64:0/3470124411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0b2b10 0x7f9e2c071b90 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f9e24009dc0 tx=0x7f9e2402bf20 comp rx=0 tx=0).stop 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.589+0000 7f9e327b5640 1 -- 172.21.15.64:0/3470124411 shutdown_connections 2024-03-04T07:27:35.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.589+0000 7f9e327b5640 1 --2- 172.21.15.64:0/3470124411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0b2b10 0x7f9e2c071b90 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.589+0000 7f9e327b5640 1 -- 172.21.15.64:0/3470124411 >> 172.21.15.64:0/3470124411 conn(0x7f9e2c0d3d20 msgr2=0x7f9e2c0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:35.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.589+0000 7f9e327b5640 1 -- 172.21.15.64:0/3470124411 shutdown_connections 2024-03-04T07:27:35.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.589+0000 7f9e327b5640 1 -- 172.21.15.64:0/3470124411 wait complete. 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.590+0000 7f9e327b5640 1 Processor -- start 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.590+0000 7f9e327b5640 1 -- start start 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e327b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0717b0 0x7f9e2c156500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e327b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e2c0b2b10 0x7f9e2c0c92b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e327b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e2c0b3060 0x7f9e2c1e7610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e327b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e2c0d4bf0 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e327b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e2c0d2e10 con 0x7f9e2c0b3060 2024-03-04T07:27:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e327b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e2c0d32a0 con 0x7f9e2c0717b0 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e2c0b2b10 0x7f9e2c0c92b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e2c0b2b10 0x7f9e2c0c92b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45430/0 (socket says 172.21.15.64:45430) 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 -- 172.21.15.64:0/111734421 learned_addr learned my addr 172.21.15.64:0/111734421 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2bfff640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0717b0 0x7f9e2c156500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e30d52640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e2c0b3060 0x7f9e2c1e7610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 -- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0717b0 msgr2=0x7f9e2c156500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0717b0 0x7f9e2c156500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 -- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e2c0b3060 msgr2=0x7f9e2c1e7610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e2c0b3060 0x7f9e2c1e7610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 -- 172.21.15.64:0/111734421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e2c156c20 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.591+0000 7f9e2b7fe640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e2c0b2b10 0x7f9e2c0c92b0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f9e1c00c430 tx=0x7f9e1c00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.592+0000 7f9e28ff9640 1 -- 172.21.15.64:0/111734421 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e1c017070 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.592+0000 7f9e28ff9640 1 -- 172.21.15.64:0/111734421 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e1c0052f0 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.592+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e2c0c9b00 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.592+0000 7f9e28ff9640 1 -- 172.21.15.64:0/111734421 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e1c003830 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.592+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e2c0ca0c0 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.593+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9e2c06ce40 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.593+0000 7f9e28ff9640 1 -- 172.21.15.64:0/111734421 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9e1c01a070 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.593+0000 7f9e28ff9640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e0c06b500 0x7f9e0c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.593+0000 7f9e28ff9640 1 -- 172.21.15.64:0/111734421 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f9e1c0895d0 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.594+0000 7f9e2bfff640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e0c06b500 0x7f9e0c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.594+0000 7f9e2bfff640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e0c06b500 0x7f9e0c06d9c0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f9e14000f60 tx=0x7f9e14007d30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.598+0000 7f9e28ff9640 1 -- 172.21.15.64:0/111734421 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9e1c057cf0 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.778+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f9e2c04f410 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.779+0000 7f9e28ff9640 1 -- 172.21.15.64:0/111734421 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v22) v1 ==== 100+0+11 (secure 0 0 0) 0x7f9e1c05bd90 con 0x7f9e2c0b2b10 2024-03-04T07:27:35.805 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e0c06b500 msgr2=0x7f9e0c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e0c06b500 0x7f9e0c06d9c0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f9e14000f60 tx=0x7f9e14007d30 comp rx=0 tx=0).stop 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e2c0b2b10 msgr2=0x7f9e2c0c92b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e2c0b2b10 0x7f9e2c0c92b0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f9e1c00c430 tx=0x7f9e1c00c8f0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 shutdown_connections 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e0c06b500 0x7f9e0c06d9c0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e2c0b3060 0x7f9e2c1e7610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e2c0b2b10 0x7f9e2c0c92b0 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 --2- 172.21.15.64:0/111734421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e2c0717b0 0x7f9e2c156500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 >> 172.21.15.64:0/111734421 conn(0x7f9e2c0d3d20 msgr2=0x7f9e2c15d0b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:35.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 shutdown_connections 2024-03-04T07:27:35.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.781+0000 7f9e327b5640 1 -- 172.21.15.64:0/111734421 wait complete. 2024-03-04T07:27:35.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:35.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.908+0000 7f2834688640 1 Processor -- start 2024-03-04T07:27:35.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.908+0000 7f2834688640 1 -- start start 2024-03-04T07:27:35.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2834688640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f282c0d02e0 0x7f282c0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2834688640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c0b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2834688640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f282c0b0dd0 con 0x7f282c0d06c0 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2834688640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f282c0b0b80 con 0x7f282c0d02e0 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2834688640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f282c0aeec0 con 0x7f282c0b4260 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2832c25640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f282c0d06c0 0x7f282c0b1a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60630/0 (socket says 172.21.15.64:60630) 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2832c25640 1 -- 172.21.15.64:0/2836998195 learned_addr learned my addr 172.21.15.64:0/2836998195 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.909+0000 7f2831c23640 1 --2- 172.21.15.64:0/2836998195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c0b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2832424640 1 --2- 172.21.15.64:0/2836998195 >> v2:172.21.15.66:3300/0 conn(0x7f282c0d02e0 0x7f282c0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2831422640 1 -- 172.21.15.64:0/2836998195 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4083724119 0 0) 0x7f282c0b0dd0 con 0x7f282c0d06c0 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2831422640 1 -- 172.21.15.64:0/2836998195 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2814003520 con 0x7f282c0d06c0 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2831c23640 1 -- 172.21.15.64:0/2836998195 >> v2:172.21.15.66:3300/0 conn(0x7f282c0d02e0 msgr2=0x7f282c0cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2831c23640 1 --2- 172.21.15.64:0/2836998195 >> v2:172.21.15.66:3300/0 conn(0x7f282c0d02e0 0x7f282c0cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2831c23640 1 -- 172.21.15.64:0/2836998195 >> v1:172.21.15.64:6789/0 conn(0x7f282c0d06c0 legacy=0x7f282c0b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2831c23640 1 -- 172.21.15.64:0/2836998195 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f282c0b10b0 con 0x7f282c0b4260 2024-03-04T07:27:35.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.910+0000 7f2832424640 1 --2- 172.21.15.64:0/2836998195 >> v2:172.21.15.66:3300/0 conn(0x7f282c0d02e0 0x7f282c0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:35.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2831c23640 1 --2- 172.21.15.64:0/2836998195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c0b1500 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f28200097a0 tx=0x7f282002b960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f44b860fc372bd59 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2831422640 1 -- 172.21.15.64:0/2836998195 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2820034070 con 0x7f282c0b4260 2024-03-04T07:27:35.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2831422640 1 -- 172.21.15.64:0/2836998195 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2820030610 con 0x7f282c0b4260 2024-03-04T07:27:35.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2831422640 1 -- 172.21.15.64:0/2836998195 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28200309c0 con 0x7f282c0b4260 2024-03-04T07:27:35.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 -- 172.21.15.64:0/2836998195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 msgr2=0x7f282c0b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 --2- 172.21.15.64:0/2836998195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c0b1500 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f28200097a0 tx=0x7f282002b960 comp rx=0 tx=0).stop 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 -- 172.21.15.64:0/2836998195 shutdown_connections 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 --2- 172.21.15.64:0/2836998195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c0b1500 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 --2- 172.21.15.64:0/2836998195 >> v2:172.21.15.66:3300/0 conn(0x7f282c0d02e0 0x7f282c0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 -- 172.21.15.64:0/2836998195 >> 172.21.15.64:0/2836998195 conn(0x7f282c0b36d0 msgr2=0x7f282c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 -- 172.21.15.64:0/2836998195 shutdown_connections 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.911+0000 7f2834688640 1 -- 172.21.15.64:0/2836998195 wait complete. 2024-03-04T07:27:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.912+0000 7f2834688640 1 Processor -- start 2024-03-04T07:27:35.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.912+0000 7f2834688640 1 -- start start 2024-03-04T07:27:35.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2834688640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c14ff10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2834688640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f282c0d02e0 0x7f282c15e9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2834688640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f282c0d06c0 0x7f282c0cafe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2834688640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f282c0b22d0 con 0x7f282c0d02e0 2024-03-04T07:27:35.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2834688640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f282c0b2150 con 0x7f282c0d06c0 2024-03-04T07:27:35.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2834688640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f282c0af350 con 0x7f282c0b4260 2024-03-04T07:27:35.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2831c23640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f282c0d02e0 0x7f282c15e9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2831c23640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f282c0d02e0 0x7f282c15e9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45438/0 (socket says 172.21.15.64:45438) 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2831c23640 1 -- 172.21.15.64:0/3492539097 learned_addr learned my addr 172.21.15.64:0/3492539097 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2832c25640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f282c0d06c0 0x7f282c0cafe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.913+0000 7f2832424640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c14ff10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2831c23640 1 -- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 msgr2=0x7f282c14ff10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2831c23640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c14ff10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2831c23640 1 -- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f282c0d06c0 msgr2=0x7f282c0cafe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2831c23640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f282c0d06c0 0x7f282c0cafe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2831c23640 1 -- 172.21.15.64:0/3492539097 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f282c0c9ba0 con 0x7f282c0d02e0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2831c23640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f282c0d02e0 0x7f282c15e9e0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f282002bf70 tx=0x7f28200046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f282affd640 1 -- 172.21.15.64:0/3492539097 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2820034040 con 0x7f282c0d02e0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f282affd640 1 -- 172.21.15.64:0/3492539097 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2820030410 con 0x7f282c0d02e0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f282c0c9e80 con 0x7f282c0d02e0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f282affd640 1 -- 172.21.15.64:0/3492539097 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2820006e10 con 0x7f282c0d02e0 2024-03-04T07:27:35.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.914+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f282c0ca440 con 0x7f282c0d02e0 2024-03-04T07:27:35.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.915+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f282c068700 con 0x7f282c0d02e0 2024-03-04T07:27:35.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.915+0000 7f282affd640 1 -- 172.21.15.64:0/3492539097 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2820037070 con 0x7f282c0d02e0 2024-03-04T07:27:35.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.916+0000 7f282affd640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281806b4d0 0x7f281806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:35.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.916+0000 7f282affd640 1 -- 172.21.15.64:0/3492539097 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f28200ac090 con 0x7f282c0d02e0 2024-03-04T07:27:35.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.916+0000 7f2832424640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281806b4d0 0x7f281806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:35.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.916+0000 7f2832424640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281806b4d0 0x7f281806d990 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f2824009560 tx=0x7f2824009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:35.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:35.920+0000 7f282affd640 1 -- 172.21.15.64:0/3492539097 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f282007a7a0 con 0x7f282c0d02e0 2024-03-04T07:27:36.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.137+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f282c06ff60 con 0x7f282c0d02e0 2024-03-04T07:27:36.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.138+0000 7f282affd640 1 -- 172.21.15.64:0/3492539097 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f282007e840 con 0x7f282c0d02e0 2024-03-04T07:27:36.139 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:36.139 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:27:36.139 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:27:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281806b4d0 msgr2=0x7f281806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281806b4d0 0x7f281806d990 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f2824009560 tx=0x7f2824009190 comp rx=0 tx=0).stop 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f282c0d02e0 msgr2=0x7f282c15e9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f282c0d02e0 0x7f282c15e9e0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f282002bf70 tx=0x7f28200046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 shutdown_connections 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281806b4d0 0x7f281806d990 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f282c0d06c0 0x7f282c0cafe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f282c0d02e0 0x7f282c15e9e0 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 --2- 172.21.15.64:0/3492539097 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f282c0b4260 0x7f282c14ff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 >> 172.21.15.64:0/3492539097 conn(0x7f282c0b36d0 msgr2=0x7f282c0ccc00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 shutdown_connections 2024-03-04T07:27:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.140+0000 7f2834688640 1 -- 172.21.15.64:0/3492539097 wait complete. 2024-03-04T07:27:36.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:36.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.252+0000 7f43b175e640 1 Processor -- start 2024-03-04T07:27:36.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.252+0000 7f43b175e640 1 -- start start 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43b175e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac15cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43b175e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f43ac06d400 0x7f43ac070870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43b175e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f43ac0729b0 con 0x7f43ac15bac0 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43b175e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43ac06cf70 con 0x7f43ac06d400 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43b175e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43ac06c920 con 0x7f43ac0aee10 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43aaffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f43ac15bac0 0x7f43ac0ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60640/0 (socket says 172.21.15.64:60640) 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43aaffd640 1 -- 172.21.15.64:0/2668293361 learned_addr learned my addr 172.21.15.64:0/2668293361 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43ab7fe640 1 --2- 172.21.15.64:0/2668293361 >> v2:172.21.15.66:3300/0 conn(0x7f43ac06d400 0x7f43ac070870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43aa7fc640 1 --2- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac15cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43a9ffb640 1 -- 172.21.15.64:0/2668293361 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2108256658 0 0) 0x7f43ac0729b0 con 0x7f43ac15bac0 2024-03-04T07:27:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43a9ffb640 1 -- 172.21.15.64:0/2668293361 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4398003520 con 0x7f43ac15bac0 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43ab7fe640 1 -- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 msgr2=0x7f43ac15cbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43ab7fe640 1 --2- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac15cbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43ab7fe640 1 -- 172.21.15.64:0/2668293361 >> v1:172.21.15.64:6789/0 conn(0x7f43ac15bac0 legacy=0x7f43ac0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43ab7fe640 1 -- 172.21.15.64:0/2668293361 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43ac06cc00 con 0x7f43ac06d400 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.253+0000 7f43aa7fc640 1 --2- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac15cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.254+0000 7f43ab7fe640 1 --2- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 0x7f43ac070870 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f439c009d90 tx=0x7f439c02bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=efab24a75b3e16e5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.254+0000 7f43a9ffb640 1 -- 172.21.15.64:0/2668293361 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f439c039070 con 0x7f43ac06d400 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.254+0000 7f43a9ffb640 1 -- 172.21.15.64:0/2668293361 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f439c02cc50 con 0x7f43ac06d400 2024-03-04T07:27:36.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.254+0000 7f43a9ffb640 1 -- 172.21.15.64:0/2668293361 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f439c034440 con 0x7f43ac06d400 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 -- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 msgr2=0x7f43ac070870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 --2- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 0x7f43ac070870 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f439c009d90 tx=0x7f439c02bea0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 -- 172.21.15.64:0/2668293361 shutdown_connections 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 --2- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 0x7f43ac070870 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 --2- 172.21.15.64:0/2668293361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac15cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 -- 172.21.15.64:0/2668293361 >> 172.21.15.64:0/2668293361 conn(0x7f43ac066d00 msgr2=0x7f43ac0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 -- 172.21.15.64:0/2668293361 shutdown_connections 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.255+0000 7f43b175e640 1 -- 172.21.15.64:0/2668293361 wait complete. 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.256+0000 7f43b175e640 1 Processor -- start 2024-03-04T07:27:36.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.256+0000 7f43b175e640 1 -- start start 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43b175e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 0x7f43ac1559f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43b175e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac0adb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43b175e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43ac15bac0 0x7f43ac0b29c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43b175e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43ac15c020 con 0x7f43ac15bac0 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43b175e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43ac15bea0 con 0x7f43ac06d400 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43b175e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43ac072580 con 0x7f43ac0aee10 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43ac15bac0 0x7f43ac0b29c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43ac15bac0 0x7f43ac0b29c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45440/0 (socket says 172.21.15.64:45440) 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 -- 172.21.15.64:0/2562420199 learned_addr learned my addr 172.21.15.64:0/2562420199 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43aa7fc640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac0adb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43aaffd640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 0x7f43ac1559f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 -- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 msgr2=0x7f43ac0adb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac0adb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 -- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 msgr2=0x7f43ac1559f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 0x7f43ac1559f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.257+0000 7f43ab7fe640 1 -- 172.21.15.64:0/2562420199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43ac154210 con 0x7f43ac15bac0 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.258+0000 7f43ab7fe640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43ac15bac0 0x7f43ac0b29c0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f439c009eb0 tx=0x7f439c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.258+0000 7f43a37fe640 1 -- 172.21.15.64:0/2562420199 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f439c039050 con 0x7f43ac15bac0 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.258+0000 7f43a37fe640 1 -- 172.21.15.64:0/2562420199 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f439c045070 con 0x7f43ac15bac0 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.258+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f43ac1544f0 con 0x7f43ac15bac0 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.258+0000 7f43a37fe640 1 -- 172.21.15.64:0/2562420199 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f439c009450 con 0x7f43ac15bac0 2024-03-04T07:27:36.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.258+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f43ac154ab0 con 0x7f43ac15bac0 2024-03-04T07:27:36.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.259+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f43ac067ff0 con 0x7f43ac15bac0 2024-03-04T07:27:36.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.259+0000 7f43a37fe640 1 -- 172.21.15.64:0/2562420199 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f439c03e610 con 0x7f43ac15bac0 2024-03-04T07:27:36.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.260+0000 7f43a37fe640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f438c06b4d0 0x7f438c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.260+0000 7f43a37fe640 1 -- 172.21.15.64:0/2562420199 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f439c0accb0 con 0x7f43ac15bac0 2024-03-04T07:27:36.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.260+0000 7f43aaffd640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f438c06b4d0 0x7f438c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.260+0000 7f43aaffd640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f438c06b4d0 0x7f438c06d990 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f43a4005880 tx=0x7f43a4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.264+0000 7f43a37fe640 1 -- 172.21.15.64:0/2562420199 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f439c07b640 con 0x7f43ac15bac0 2024-03-04T07:27:36.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.446+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f43ac06de50 con 0x7f43ac15bac0 2024-03-04T07:27:36.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.447+0000 7f43a37fe640 1 -- 172.21.15.64:0/2562420199 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16807 (secure 0 0 0) 0x7f439c07f6e0 con 0x7f43ac15bac0 2024-03-04T07:27:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:36.448 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:34.409323+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.12","mappings":[{"from":0,"to":1}]},{"pgid":"3.19","mappings":[{"from":3,"to":2}]},{"pgid":"3.28","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f438c06b4d0 msgr2=0x7f438c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f438c06b4d0 0x7f438c06d990 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f43a4005880 tx=0x7f43a4009040 comp rx=0 tx=0).stop 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43ac15bac0 msgr2=0x7f43ac0b29c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43ac15bac0 0x7f43ac0b29c0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f439c009eb0 tx=0x7f439c004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 shutdown_connections 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f438c06b4d0 0x7f438c06d990 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43ac15bac0 0x7f43ac0b29c0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43ac0aee10 0x7f43ac0adb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 --2- 172.21.15.64:0/2562420199 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43ac06d400 0x7f43ac1559f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 >> 172.21.15.64:0/2562420199 conn(0x7f43ac066d00 msgr2=0x7f43ac0b1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:36.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 shutdown_connections 2024-03-04T07:27:36.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.449+0000 7f43b175e640 1 -- 172.21.15.64:0/2562420199 wait complete. 2024-03-04T07:27:36.461 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:27:36.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:36.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.562+0000 7fb74fbbc640 1 Processor -- start 2024-03-04T07:27:36.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.562+0000 7fb74fbbc640 1 -- start start 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74fbbc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7480aee10 0x7fb74815cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74fbbc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb74806d400 0x7fb748070870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74fbbc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb7480729b0 con 0x7fb74815bac0 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74fbbc640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb74806cf70 con 0x7fb7480aee10 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74fbbc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb74806c920 con 0x7fb74806d400 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74d157640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7480aee10 0x7fb74815cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74d157640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7480aee10 0x7fb74815cbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45452/0 (socket says 172.21.15.64:45452) 2024-03-04T07:27:36.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74d157640 1 -- 172.21.15.64:0/3809234143 learned_addr learned my addr 172.21.15.64:0/3809234143 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74e159640 1 --2- 172.21.15.64:0/3809234143 >> v2:172.21.15.66:3300/0 conn(0x7fb74806d400 0x7fb748070870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74d157640 1 -- 172.21.15.64:0/3809234143 >> v2:172.21.15.66:3300/0 conn(0x7fb74806d400 msgr2=0x7fb748070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74d157640 1 --2- 172.21.15.64:0/3809234143 >> v2:172.21.15.66:3300/0 conn(0x7fb74806d400 0x7fb748070870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74d157640 1 -- 172.21.15.64:0/3809234143 >> v1:172.21.15.66:6789/0 conn(0x7fb74815bac0 legacy=0x7fb7480ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.563+0000 7fb74d157640 1 -- 172.21.15.64:0/3809234143 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb74806cc00 con 0x7fb7480aee10 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.564+0000 7fb74d157640 1 --2- 172.21.15.64:0/3809234143 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7480aee10 0x7fb74815cbe0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fb74806ea40 tx=0x7fb73802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=efccb33390e2ef7f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.564+0000 7fb74c956640 1 -- 172.21.15.64:0/3809234143 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb738034070 con 0x7fb7480aee10 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.564+0000 7fb74c956640 1 -- 172.21.15.64:0/3809234143 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb738030860 con 0x7fb7480aee10 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.564+0000 7fb74c956640 1 -- 172.21.15.64:0/3809234143 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb738030c30 con 0x7fb7480aee10 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3809234143 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7480aee10 msgr2=0x7fb74815cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3809234143 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7480aee10 0x7fb74815cbe0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fb74806ea40 tx=0x7fb73802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3809234143 shutdown_connections 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3809234143 >> v2:172.21.15.66:3300/0 conn(0x7fb74806d400 0x7fb748070870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3809234143 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7480aee10 0x7fb74815cbe0 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3809234143 >> 172.21.15.64:0/3809234143 conn(0x7fb748066d00 msgr2=0x7fb7480670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3809234143 shutdown_connections 2024-03-04T07:27:36.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.565+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3809234143 wait complete. 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.566+0000 7fb74fbbc640 1 Processor -- start 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.566+0000 7fb74fbbc640 1 -- start start 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb74806d400 0x7fb748146290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7480aee10 0x7fb7481e4440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb74815bac0 0x7fb7481f1950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb74815c020 con 0x7fb74806d400 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb74815bea0 con 0x7fb7480aee10 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb748072580 con 0x7fb74815bac0 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb74806d400 0x7fb748146290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb74806d400 0x7fb748146290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45460/0 (socket says 172.21.15.64:45460) 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 -- 172.21.15.64:0/3540339440 learned_addr learned my addr 172.21.15.64:0/3540339440 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:36.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d157640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7480aee10 0x7fb7481e4440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74e159640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb74815bac0 0x7fb7481f1950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 -- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb74815bac0 msgr2=0x7fb7481f1950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb74815bac0 0x7fb7481f1950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 -- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7480aee10 msgr2=0x7fb7481e4440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7480aee10 0x7fb7481e4440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 -- 172.21.15.64:0/3540339440 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb748154550 con 0x7fb74806d400 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74d958640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb74806d400 0x7fb748146290 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fb74400c3f0 tx=0x7fb74400c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb73e7fc640 1 -- 172.21.15.64:0/3540339440 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7440052d0 con 0x7fb74806d400 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb73e7fc640 1 -- 172.21.15.64:0/3540339440 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb744005490 con 0x7fb74806d400 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb748154830 con 0x7fb74806d400 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb73e7fc640 1 -- 172.21.15.64:0/3540339440 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb744007260 con 0x7fb74806d400 2024-03-04T07:27:36.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.567+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb748154df0 con 0x7fb74806d400 2024-03-04T07:27:36.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.568+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb748067ff0 con 0x7fb74806d400 2024-03-04T07:27:36.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.569+0000 7fb73e7fc640 1 -- 172.21.15.64:0/3540339440 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb744007400 con 0x7fb74806d400 2024-03-04T07:27:36.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.569+0000 7fb73e7fc640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb72c06b4d0 0x7fb72c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.569+0000 7fb73e7fc640 1 -- 172.21.15.64:0/3540339440 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7fb744089670 con 0x7fb74806d400 2024-03-04T07:27:36.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.569+0000 7fb74d157640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb72c06b4d0 0x7fb72c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.570+0000 7fb74d157640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb72c06b4d0 0x7fb72c06d990 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fb738036000 tx=0x7fb738008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.574+0000 7fb73e7fc640 1 -- 172.21.15.64:0/3540339440 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb744057e20 con 0x7fb74806d400 2024-03-04T07:27:36.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.794+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb74815e6a0 con 0x7fb74806d400 2024-03-04T07:27:36.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.795+0000 7fb73e7fc640 1 -- 172.21.15.64:0/3540339440 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7fb74405bec0 con 0x7fb74806d400 2024-03-04T07:27:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:36.796 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:27:36.796 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:27:36.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb72c06b4d0 msgr2=0x7fb72c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb72c06b4d0 0x7fb72c06d990 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fb738036000 tx=0x7fb738008030 comp rx=0 tx=0).stop 2024-03-04T07:27:36.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb74806d400 msgr2=0x7fb748146290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb74806d400 0x7fb748146290 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fb74400c3f0 tx=0x7fb74400c8b0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 shutdown_connections 2024-03-04T07:27:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb72c06b4d0 0x7fb72c06d990 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb74815bac0 0x7fb7481f1950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb7480aee10 0x7fb7481e4440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 --2- 172.21.15.64:0/3540339440 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb74806d400 0x7fb748146290 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 >> 172.21.15.64:0/3540339440 conn(0x7fb748066d00 msgr2=0x7fb7480cf410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 shutdown_connections 2024-03-04T07:27:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.797+0000 7fb74fbbc640 1 -- 172.21.15.64:0/3540339440 wait complete. 2024-03-04T07:27:36.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:36.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.908+0000 7f616ef6b640 1 Processor -- start 2024-03-04T07:27:36.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.909+0000 7f616ef6b640 1 -- start start 2024-03-04T07:27:36.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.909+0000 7f616ef6b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f616806b5c0 0x7f61680b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.909+0000 7f616ef6b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f61680b0e90 0x7f616806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.909+0000 7f616ef6b640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f61680b3340 con 0x7f61680ae9b0 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.909+0000 7f616ef6b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f61680cfa00 con 0x7f61680b0e90 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.909+0000 7f616ef6b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f61680cee90 con 0x7f616806b5c0 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.909+0000 7f616cd07640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f616806b5c0 0x7f61680b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f6167fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f61680b0e90 0x7f616806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f616cd07640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f616806b5c0 0x7f61680b2e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:60892/0 (socket says 172.21.15.64:60892) 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f616d508640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f61680ae9b0 0x7f61680d03a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38612/0 (socket says 172.21.15.64:38612) 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f616cd07640 1 -- 172.21.15.64:0/1581051814 learned_addr learned my addr 172.21.15.64:0/1581051814 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f6167fff640 1 -- 172.21.15.64:0/1581051814 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f616806b5c0 msgr2=0x7f61680b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f6167fff640 1 --2- 172.21.15.64:0/1581051814 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f616806b5c0 0x7f61680b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f6167fff640 1 -- 172.21.15.64:0/1581051814 >> v1:172.21.15.66:6789/0 conn(0x7f61680ae9b0 legacy=0x7f61680d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.910+0000 7f6167fff640 1 -- 172.21.15.64:0/1581051814 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61680cfc50 con 0x7f61680b0e90 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f6167fff640 1 --2- 172.21.15.64:0/1581051814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61680b0e90 0x7f616806b9a0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f6158009d90 tx=0x7f615802be10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6b9eb2bed158dd33 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f61677fe640 1 -- 172.21.15.64:0/1581051814 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6158039070 con 0x7f61680b0e90 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f61677fe640 1 -- 172.21.15.64:0/1581051814 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f615802cbc0 con 0x7f61680b0e90 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f61677fe640 1 -- 172.21.15.64:0/1581051814 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6158034440 con 0x7f61680b0e90 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 -- 172.21.15.64:0/1581051814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61680b0e90 msgr2=0x7f616806b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 --2- 172.21.15.64:0/1581051814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61680b0e90 0x7f616806b9a0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f6158009d90 tx=0x7f615802be10 comp rx=0 tx=0).stop 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 -- 172.21.15.64:0/1581051814 shutdown_connections 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 --2- 172.21.15.64:0/1581051814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61680b0e90 0x7f616806b9a0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 --2- 172.21.15.64:0/1581051814 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f616806b5c0 0x7f61680b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 -- 172.21.15.64:0/1581051814 >> 172.21.15.64:0/1581051814 conn(0x7f61680b36d0 msgr2=0x7f61680b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 -- 172.21.15.64:0/1581051814 shutdown_connections 2024-03-04T07:27:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.911+0000 7f616ef6b640 1 -- 172.21.15.64:0/1581051814 wait complete. 2024-03-04T07:27:36.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.912+0000 7f616ef6b640 1 Processor -- start 2024-03-04T07:27:36.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.912+0000 7f616ef6b640 1 -- start start 2024-03-04T07:27:36.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616ef6b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f616806b5c0 0x7f61680b7240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616ef6b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61680ae9b0 0x7f61680b4f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616ef6b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680b0e90 0x7f61680b5440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616ef6b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f616806c790 con 0x7f61680ae9b0 2024-03-04T07:27:36.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616ef6b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f616806bee0 con 0x7f616806b5c0 2024-03-04T07:27:36.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616ef6b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f616806c910 con 0x7f61680b0e90 2024-03-04T07:27:36.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616cd07640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f616806b5c0 0x7f61680b7240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f6167fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61680ae9b0 0x7f61680b4f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616cd07640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f616806b5c0 0x7f61680b7240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37776/0 (socket says 172.21.15.64:37776) 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616cd07640 1 -- 172.21.15.64:0/75751396 learned_addr learned my addr 172.21.15.64:0/75751396 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.913+0000 7f616d508640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680b0e90 0x7f61680b5440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616cd07640 1 -- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680b0e90 msgr2=0x7f61680b5440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616cd07640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680b0e90 0x7f61680b5440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616cd07640 1 -- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61680ae9b0 msgr2=0x7f61680b4f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616cd07640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61680ae9b0 0x7f61680b4f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616cd07640 1 -- 172.21.15.64:0/75751396 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61681f3c00 con 0x7f616806b5c0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616cd07640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f616806b5c0 0x7f61680b7240 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f616000e840 tx=0x7f616000ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f61657fa640 1 -- 172.21.15.64:0/75751396 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6160017070 con 0x7f616806b5c0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f61657fa640 1 -- 172.21.15.64:0/75751396 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f616000c8e0 con 0x7f616806b5c0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61680c9c10 con 0x7f616806b5c0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f61657fa640 1 -- 172.21.15.64:0/75751396 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f616000ccd0 con 0x7f616806b5c0 2024-03-04T07:27:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.914+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61680ca1d0 con 0x7f616806b5c0 2024-03-04T07:27:36.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.915+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6168068700 con 0x7f616806b5c0 2024-03-04T07:27:36.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.916+0000 7f61657fa640 1 -- 172.21.15.64:0/75751396 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f616001a070 con 0x7f616806b5c0 2024-03-04T07:27:36.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.916+0000 7f61657fa640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:36.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.916+0000 7f61657fa640 1 -- 172.21.15.64:0/75751396 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f616008a0f0 con 0x7f616806b5c0 2024-03-04T07:27:36.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.916+0000 7f6167fff640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:36.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.916+0000 7f6167fff640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f6158036000 tx=0x7f6158030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:36.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:36.920+0000 7f61657fa640 1 -- 172.21.15.64:0/75751396 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f61600587f0 con 0x7f616806b5c0 2024-03-04T07:27:37.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.136+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f61680af3d0 con 0x7f616806b5c0 2024-03-04T07:27:37.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.136+0000 7f61657fa640 1 -- 172.21.15.64:0/75751396 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+4968 (secure 0 0 0) 0x7f616005c890 con 0x7f616806b5c0 2024-03-04T07:27:37.137 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:37.137 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4238,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/3217637964","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":3217637964},{"type":"v1","addr":"172.21.15.66:6834","nonce":3217637964}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3},{"gid":4281,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/661746088","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":661746088},{"type":"v1","addr":"172.21.15.64:6836","nonce":661746088}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":2},{"gid":4284,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/1334184920","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1334184920},{"type":"v1","addr":"172.21.15.64:6837","nonce":1334184920}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:26:52.660771+0000","modified":"2024-03-04T07:26:55.666036+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":4293},"failed":[],"damaged":[],"stopped":[],"info":{"gid_4293":{"gid":4293,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/1775037311","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":1775037311},{"type":"v1","addr":"172.21.15.66:6835","nonce":1775037311}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":4293,"qdb_cluster":[4293]},"id":1}]} 2024-03-04T07:27:37.137 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 7 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 msgr2=0x7f614806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f6158036000 tx=0x7f6158030000 comp rx=0 tx=0).stop 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f616806b5c0 msgr2=0x7f61680b7240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f616806b5c0 0x7f61680b7240 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f616000e840 tx=0x7f616000ed00 comp rx=0 tx=0).stop 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 shutdown_connections 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680b0e90 0x7f61680b5440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61680ae9b0 0x7f61680b4f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 --2- 172.21.15.64:0/75751396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f616806b5c0 0x7f61680b7240 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 >> 172.21.15.64:0/75751396 conn(0x7f61680b36d0 msgr2=0x7f61680cdf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:37.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 shutdown_connections 2024-03-04T07:27:37.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.138+0000 7f616ef6b640 1 -- 172.21.15.64:0/75751396 wait complete. 2024-03-04T07:27:37.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:37.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.250+0000 7fab881bc640 1 Processor -- start 2024-03-04T07:27:37.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.250+0000 7fab881bc640 1 -- start start 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab881bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab8006df30 0x7fab800ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab881bc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fab80071240 con 0x7fab800aec40 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab881bc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fab80071520 con 0x7fab80158520 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab881bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab80071b70 con 0x7fab8006df30 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab86759640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fab800aec40 0x7fab800b3640 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60642/0 (socket says 172.21.15.64:60642) 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab86759640 1 -- 172.21.15.64:0/1321471212 learned_addr learned my addr 172.21.15.64:0/1321471212 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab85757640 1 --2- 172.21.15.64:0/1321471212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab8006df30 0x7fab800ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab84f56640 1 -- 172.21.15.64:0/1321471212 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 44543621 0 0) 0x7fab80071240 con 0x7fab800aec40 2024-03-04T07:27:37.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab84f56640 1 -- 172.21.15.64:0/1321471212 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fab6c005730 con 0x7fab800aec40 2024-03-04T07:27:37.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab84f56640 1 -- 172.21.15.64:0/1321471212 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 576443604 0 0) 0x7fab80071520 con 0x7fab80158520 2024-03-04T07:27:37.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab84f56640 1 -- 172.21.15.64:0/1321471212 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fab80071240 con 0x7fab80158520 2024-03-04T07:27:37.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab85757640 1 -- 172.21.15.64:0/1321471212 >> v1:172.21.15.66:6789/0 conn(0x7fab80158520 legacy=0x7fab8006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab85757640 1 -- 172.21.15.64:0/1321471212 >> v1:172.21.15.64:6789/0 conn(0x7fab800aec40 legacy=0x7fab800b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.251+0000 7fab85757640 1 -- 172.21.15.64:0/1321471212 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab80071800 con 0x7fab8006df30 2024-03-04T07:27:37.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.252+0000 7fab85757640 1 --2- 172.21.15.64:0/1321471212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab8006df30 0x7fab800ae700 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fab700097d0 tx=0x7fab7002b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6f66b514425b4e4b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.252+0000 7fab84f56640 1 -- 172.21.15.64:0/1321471212 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab70035020 con 0x7fab8006df30 2024-03-04T07:27:37.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.252+0000 7fab84f56640 1 -- 172.21.15.64:0/1321471212 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab70031890 con 0x7fab8006df30 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.252+0000 7fab84f56640 1 -- 172.21.15.64:0/1321471212 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab70031c60 con 0x7fab8006df30 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.253+0000 7fab881bc640 1 -- 172.21.15.64:0/1321471212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab8006df30 msgr2=0x7fab800ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.253+0000 7fab881bc640 1 --2- 172.21.15.64:0/1321471212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab8006df30 0x7fab800ae700 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fab700097d0 tx=0x7fab7002b930 comp rx=0 tx=0).stop 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.253+0000 7fab881bc640 1 -- 172.21.15.64:0/1321471212 shutdown_connections 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.253+0000 7fab881bc640 1 --2- 172.21.15.64:0/1321471212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab8006df30 0x7fab800ae700 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.253+0000 7fab881bc640 1 -- 172.21.15.64:0/1321471212 >> 172.21.15.64:0/1321471212 conn(0x7fab80066d00 msgr2=0x7fab800670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.253+0000 7fab881bc640 1 -- 172.21.15.64:0/1321471212 shutdown_connections 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.253+0000 7fab881bc640 1 -- 172.21.15.64:0/1321471212 wait complete. 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.254+0000 7fab881bc640 1 Processor -- start 2024-03-04T07:27:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.254+0000 7fab881bc640 1 -- start start 2024-03-04T07:27:37.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab881bc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab8006df30 0x7fab800cf500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab881bc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab800aec40 0x7fab800adb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab881bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab80158520 0x7fab80157120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab881bc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab80158900 con 0x7fab800aec40 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab881bc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab800af0b0 con 0x7fab8006df30 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab881bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab80158a80 con 0x7fab80158520 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85757640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab800aec40 0x7fab800adb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85f58640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab8006df30 0x7fab800cf500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85757640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab800aec40 0x7fab800adb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45492/0 (socket says 172.21.15.64:45492) 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85757640 1 -- 172.21.15.64:0/1391365602 learned_addr learned my addr 172.21.15.64:0/1391365602 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab86759640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab80158520 0x7fab80157120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85f58640 1 -- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab80158520 msgr2=0x7fab80157120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85f58640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab80158520 0x7fab80157120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85f58640 1 -- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab800aec40 msgr2=0x7fab800adb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85f58640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab800aec40 0x7fab800adb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85f58640 1 -- 172.21.15.64:0/1391365602 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab800ce460 con 0x7fab8006df30 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85757640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab800aec40 0x7fab800adb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:37.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.255+0000 7fab85f58640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab8006df30 0x7fab800cf500 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fab7c000f60 tx=0x7fab7c00c820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.256+0000 7fab767fc640 1 -- 172.21.15.64:0/1391365602 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab7c017070 con 0x7fab8006df30 2024-03-04T07:27:37.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.256+0000 7fab767fc640 1 -- 172.21.15.64:0/1391365602 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab7c0052f0 con 0x7fab8006df30 2024-03-04T07:27:37.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.256+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab800ce740 con 0x7fab8006df30 2024-03-04T07:27:37.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.256+0000 7fab767fc640 1 -- 172.21.15.64:0/1391365602 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab7c003830 con 0x7fab8006df30 2024-03-04T07:27:37.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.256+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab800ced00 con 0x7fab8006df30 2024-03-04T07:27:37.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.256+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab80067ff0 con 0x7fab8006df30 2024-03-04T07:27:37.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.257+0000 7fab767fc640 1 -- 172.21.15.64:0/1391365602 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fab7c01a070 con 0x7fab8006df30 2024-03-04T07:27:37.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.257+0000 7fab767fc640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab6406b4d0 0x7fab6406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.257+0000 7fab767fc640 1 -- 172.21.15.64:0/1391365602 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7fab7c0895a0 con 0x7fab8006df30 2024-03-04T07:27:37.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.257+0000 7fab85757640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab6406b4d0 0x7fab6406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.258+0000 7fab85757640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab6406b4d0 0x7fab6406d990 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fab7002b970 tx=0x7fab70008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.262+0000 7fab767fc640 1 -- 172.21.15.64:0/1391365602 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fab7c057d50 con 0x7fab8006df30 2024-03-04T07:27:37.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.446+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fab800b0960 con 0x7fab8006df30 2024-03-04T07:27:37.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.448+0000 7fab767fc640 1 -- 172.21.15.64:0/1391365602 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v22) v1 ==== 74+0+16807 (secure 0 0 0) 0x7fab7c05bdf0 con 0x7fab8006df30 2024-03-04T07:27:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:37.448 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:34.409323+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:26:51.244804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-04T07:26:51.631023+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"20","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.12","mappings":[{"from":0,"to":1}]},{"pgid":"3.19","mappings":[{"from":3,"to":2}]},{"pgid":"3.28","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":5,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab6406b4d0 msgr2=0x7fab6406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab6406b4d0 0x7fab6406d990 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fab7002b970 tx=0x7fab70008030 comp rx=0 tx=0).stop 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab8006df30 msgr2=0x7fab800cf500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab8006df30 0x7fab800cf500 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fab7c000f60 tx=0x7fab7c00c820 comp rx=0 tx=0).stop 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 shutdown_connections 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab6406b4d0 0x7fab6406d990 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab80158520 0x7fab80157120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab800aec40 0x7fab800adb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 --2- 172.21.15.64:0/1391365602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab8006df30 0x7fab800cf500 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 >> 172.21.15.64:0/1391365602 conn(0x7fab80066d00 msgr2=0x7fab800670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:37.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 shutdown_connections 2024-03-04T07:27:37.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.449+0000 7fab881bc640 1 -- 172.21.15.64:0/1391365602 wait complete. 2024-03-04T07:27:37.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:27:37.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.563+0000 7f22c6237640 1 Processor -- start 2024-03-04T07:27:37.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.563+0000 7f22c6237640 1 -- start start 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22c6237640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f22c00ceed0 0x7f22c00d0470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22c6237640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0066d00 0x7f22c00671b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22c6237640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f22c00cf3d0 con 0x7f22c0145780 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22c6237640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f22c0072c30 con 0x7f22c00ceed0 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22c6237640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f22c00cf6b0 con 0x7f22c0066d00 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22beffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f22c00ceed0 0x7f22c00d0470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22bffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0066d00 0x7f22c00671b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22beffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f22c00ceed0 0x7f22c00d0470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45504/0 (socket says 172.21.15.64:45504) 2024-03-04T07:27:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22beffd640 1 -- 172.21.15.64:0/2633401931 learned_addr learned my addr 172.21.15.64:0/2633401931 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:37.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22bf7fe640 1 --1- 172.21.15.64:0/2633401931 >> v1:172.21.15.66:6789/0 conn(0x7f22c0145780 0x7f22c0072520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38624/0 (socket says 172.21.15.64:38624) 2024-03-04T07:27:37.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22beffd640 1 -- 172.21.15.64:0/2633401931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0066d00 msgr2=0x7f22c00671b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:37.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22beffd640 1 --2- 172.21.15.64:0/2633401931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0066d00 0x7f22c00671b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22beffd640 1 -- 172.21.15.64:0/2633401931 >> v1:172.21.15.66:6789/0 conn(0x7f22c0145780 legacy=0x7f22c0072520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22be7fc640 1 -- 172.21.15.64:0/2633401931 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3910989508 0 0) 0x7f22c00cf3d0 con 0x7f22c0145780 2024-03-04T07:27:37.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.564+0000 7f22beffd640 1 -- 172.21.15.64:0/2633401931 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22c00cf990 con 0x7f22c00ceed0 2024-03-04T07:27:37.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.565+0000 7f22beffd640 1 --2- 172.21.15.64:0/2633401931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c00ceed0 0x7f22c00d0470 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f22b4009870 tx=0x7f22b402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da7baa5c6ff004d6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.565+0000 7f22be7fc640 1 -- 172.21.15.64:0/2633401931 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22b4035020 con 0x7f22c00ceed0 2024-03-04T07:27:37.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.565+0000 7f22be7fc640 1 -- 172.21.15.64:0/2633401931 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f22b4031640 con 0x7f22c00ceed0 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.565+0000 7f22be7fc640 1 -- 172.21.15.64:0/2633401931 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22b402d040 con 0x7f22c00ceed0 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 -- 172.21.15.64:0/2633401931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c00ceed0 msgr2=0x7f22c00d0470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 --2- 172.21.15.64:0/2633401931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c00ceed0 0x7f22c00d0470 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f22b4009870 tx=0x7f22b402b8f0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 -- 172.21.15.64:0/2633401931 shutdown_connections 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 --2- 172.21.15.64:0/2633401931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0066d00 0x7f22c00671b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 --2- 172.21.15.64:0/2633401931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c00ceed0 0x7f22c00d0470 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 -- 172.21.15.64:0/2633401931 >> 172.21.15.64:0/2633401931 conn(0x7f22c00b2720 msgr2=0x7f22c00b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 -- 172.21.15.64:0/2633401931 shutdown_connections 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.566+0000 7f22c6237640 1 -- 172.21.15.64:0/2633401931 wait complete. 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.567+0000 7f22c6237640 1 Processor -- start 2024-03-04T07:27:37.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.567+0000 7f22c6237640 1 -- start start 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22c6237640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c0066d00 0x7f22c00c7190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22c6237640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22c00ceed0 0x7f22c01f1e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22c6237640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0145780 0x7f22c01f5e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22c6237640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22c006db60 con 0x7f22c0066d00 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22c6237640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22c006d9e0 con 0x7f22c00ceed0 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22c6237640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f22c006dce0 con 0x7f22c0145780 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22bf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c0066d00 0x7f22c00c7190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22bf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c0066d00 0x7f22c00c7190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45510/0 (socket says 172.21.15.64:45510) 2024-03-04T07:27:37.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22bf7fe640 1 -- 172.21.15.64:0/543400138 learned_addr learned my addr 172.21.15.64:0/543400138 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22beffd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22c00ceed0 0x7f22c01f1e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22bffff640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0145780 0x7f22c01f5e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22beffd640 1 -- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0145780 msgr2=0x7f22c01f5e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22beffd640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0145780 0x7f22c01f5e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22beffd640 1 -- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c0066d00 msgr2=0x7f22c00c7190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22beffd640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c0066d00 0x7f22c00c7190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22beffd640 1 -- 172.21.15.64:0/543400138 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22c00c77c0 con 0x7f22c00ceed0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22bf7fe640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c0066d00 0x7f22c00c7190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.568+0000 7f22beffd640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22c00ceed0 0x7f22c01f1e50 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f22b4033000 tx=0x7f22b40046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.569+0000 7f22a7fff640 1 -- 172.21.15.64:0/543400138 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22b4040070 con 0x7f22c00ceed0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.569+0000 7f22a7fff640 1 -- 172.21.15.64:0/543400138 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f22b40399f0 con 0x7f22c00ceed0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.569+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f22c01f24a0 con 0x7f22c00ceed0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.569+0000 7f22a7fff640 1 -- 172.21.15.64:0/543400138 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22b4035040 con 0x7f22c00ceed0 2024-03-04T07:27:37.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.569+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f22c01f2a60 con 0x7f22c00ceed0 2024-03-04T07:27:37.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.570+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f22c0067ff0 con 0x7f22c00ceed0 2024-03-04T07:27:37.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.570+0000 7f22a7fff640 1 -- 172.21.15.64:0/543400138 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f22b404b060 con 0x7f22c00ceed0 2024-03-04T07:27:37.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.570+0000 7f22a7fff640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22ac06b500 0x7f22ac06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.570+0000 7f22a7fff640 1 -- 172.21.15.64:0/543400138 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(22..22 src has 1..22) v4 ==== 5825+0+0 (secure 0 0 0) 0x7f22b40ab950 con 0x7f22c00ceed0 2024-03-04T07:27:37.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.570+0000 7f22bf7fe640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22ac06b500 0x7f22ac06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.571+0000 7f22bf7fe640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22ac06b500 0x7f22ac06d9c0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f22b0000f60 tx=0x7f22b0007ea0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.575+0000 7f22a7fff640 1 -- 172.21.15.64:0/543400138 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f22b407a080 con 0x7f22c00ceed0 2024-03-04T07:27:37.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.794+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f22c0067460 con 0x7f22c00ceed0 2024-03-04T07:27:37.795 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:27:37.795+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:27:37.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.799+0000 7f22a7fff640 1 -- 172.21.15.64:0/543400138 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v8) v1 ==== 161+0+0 (secure 0 0 0) 0x7f22b407e120 con 0x7f22c00ceed0 2024-03-04T07:27:37.799 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:27:37.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.800+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22ac06b500 msgr2=0x7f22ac06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.800+0000 7f22c6237640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22ac06b500 0x7f22ac06d9c0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f22b0000f60 tx=0x7f22b0007ea0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22c00ceed0 msgr2=0x7f22c01f1e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22c00ceed0 0x7f22c01f1e50 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f22b4033000 tx=0x7f22b40046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 shutdown_connections 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22ac06b500 0x7f22ac06d9c0 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22c0145780 0x7f22c01f5e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22c00ceed0 0x7f22c01f1e50 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 --2- 172.21.15.64:0/543400138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22c0066d00 0x7f22c00c7190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 >> 172.21.15.64:0/543400138 conn(0x7f22c00b2720 msgr2=0x7f22c0143ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 shutdown_connections 2024-03-04T07:27:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.801+0000 7f22c6237640 1 -- 172.21.15.64:0/543400138 wait complete. 2024-03-04T07:27:37.804 INFO:tasks.ceph.mds.b.smithi066.stderr: -2883> 2024-03-04T07:26:57.720+0000 7f1297c9d640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:27:37.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:27:37.835 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:27:37.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.914+0000 7f1b71b93640 1 Processor -- start 2024-03-04T07:27:37.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.914+0000 7f1b71b93640 1 -- start start 2024-03-04T07:27:37.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.915+0000 7f1b71b93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1b6c1445b0 0x7f1b6c0b43c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.915+0000 7f1b71b93640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b6c06f1e0 con 0x7f1b6c072130 2024-03-04T07:27:37.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.915+0000 7f1b71b93640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b6c06f4c0 con 0x7f1b6c06ec40 2024-03-04T07:27:37.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.915+0000 7f1b71b93640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1b6c144990 con 0x7f1b6c1445b0 2024-03-04T07:27:37.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.915+0000 7f1b6b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1b6c1445b0 0x7f1b6c0b43c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.915+0000 7f1b6bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1b6c072130 0x7f1b6c0ce570 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60654/0 (socket says 172.21.15.64:60654) 2024-03-04T07:27:37.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.915+0000 7f1b6bfff640 1 -- 172.21.15.64:0/2868543052 learned_addr learned my addr 172.21.15.64:0/2868543052 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.916+0000 7f1b6b7fe640 1 -- 172.21.15.64:0/2868543052 >> v1:172.21.15.66:6789/0 conn(0x7f1b6c06ec40 legacy=0x7f1b6c071a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.916+0000 7f1b6b7fe640 1 -- 172.21.15.64:0/2868543052 >> v1:172.21.15.64:6789/0 conn(0x7f1b6c072130 legacy=0x7f1b6c0ce570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.916+0000 7f1b6b7fe640 1 -- 172.21.15.64:0/2868543052 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b6c06f7a0 con 0x7f1b6c1445b0 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.916+0000 7f1b6a7fc640 1 -- 172.21.15.64:0/2868543052 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 375177291 0 0) 0x7f1b6c06f1e0 con 0x7f1b6c072130 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.916+0000 7f1b6b7fe640 1 --2- 172.21.15.64:0/2868543052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c0b43c0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f1b5c009d90 tx=0x7f1b5c02beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=202a7de90ab744a5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.916+0000 7f1b6a7fc640 1 -- 172.21.15.64:0/2868543052 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b5c039070 con 0x7f1b6c1445b0 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b6a7fc640 1 -- 172.21.15.64:0/2868543052 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b5c02cc60 con 0x7f1b6c1445b0 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b6a7fc640 1 -- 172.21.15.64:0/2868543052 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b5c034660 con 0x7f1b6c1445b0 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b71b93640 1 -- 172.21.15.64:0/2868543052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c1445b0 msgr2=0x7f1b6c0b43c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b71b93640 1 --2- 172.21.15.64:0/2868543052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c0b43c0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f1b5c009d90 tx=0x7f1b5c02beb0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b71b93640 1 -- 172.21.15.64:0/2868543052 shutdown_connections 2024-03-04T07:27:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b71b93640 1 --2- 172.21.15.64:0/2868543052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c0b43c0 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b71b93640 1 -- 172.21.15.64:0/2868543052 >> 172.21.15.64:0/2868543052 conn(0x7f1b6c0d09c0 msgr2=0x7f1b6c0d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b71b93640 1 -- 172.21.15.64:0/2868543052 shutdown_connections 2024-03-04T07:27:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.917+0000 7f1b71b93640 1 -- 172.21.15.64:0/2868543052 wait complete. 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.918+0000 7f1b71b93640 1 Processor -- start 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.918+0000 7f1b71b93640 1 -- start start 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b71b93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c06ec40 0x7f1b6c1f5f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b71b93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b6c072130 0x7f1b6c142930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b71b93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c1f1450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b71b93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b6c0cdc70 con 0x7f1b6c06ec40 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b71b93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b6c0b4900 con 0x7f1b6c1445b0 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b71b93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1b6c0cddf0 con 0x7f1b6c072130 2024-03-04T07:27:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b6b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c06ec40 0x7f1b6c1f5f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b6b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c06ec40 0x7f1b6c1f5f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45542/0 (socket says 172.21.15.64:45542) 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b6b7fe640 1 -- 172.21.15.64:0/173610169 learned_addr learned my addr 172.21.15.64:0/173610169 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b6bfff640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c1f1450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.919+0000 7f1b6affd640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b6c072130 0x7f1b6c142930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b6b7fe640 1 -- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b6c072130 msgr2=0x7f1b6c142930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b6b7fe640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b6c072130 0x7f1b6c142930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b6b7fe640 1 -- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b6c1445b0 msgr2=0x7f1b6c1f1450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b6b7fe640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c1f1450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b6b7fe640 1 -- 172.21.15.64:0/173610169 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b6c1f6460 con 0x7f1b6c06ec40 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b6bfff640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c1f1450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b6b7fe640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c06ec40 0x7f1b6c1f5f20 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f1b5c009d90 tx=0x7f1b5c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b53fff640 1 -- 172.21.15.64:0/173610169 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b5c044020 con 0x7f1b6c06ec40 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b53fff640 1 -- 172.21.15.64:0/173610169 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b5c03f670 con 0x7f1b6c06ec40 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b6c1e33f0 con 0x7f1b6c06ec40 2024-03-04T07:27:37.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b53fff640 1 -- 172.21.15.64:0/173610169 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b5c039040 con 0x7f1b6c06ec40 2024-03-04T07:27:37.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.920+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b6c1e3970 con 0x7f1b6c06ec40 2024-03-04T07:27:37.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.921+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1b6c0733d0 con 0x7f1b6c06ec40 2024-03-04T07:27:37.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.922+0000 7f1b53fff640 1 -- 172.21.15.64:0/173610169 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1b5c047020 con 0x7f1b6c06ec40 2024-03-04T07:27:37.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.922+0000 7f1b53fff640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b5806b4d0 0x7f1b5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:37.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.923+0000 7f1b53fff640 1 -- 172.21.15.64:0/173610169 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(23..23 src has 1..23) v4 ==== 5911+0+0 (secure 0 0 0) 0x7f1b5c0acec0 con 0x7f1b6c06ec40 2024-03-04T07:27:37.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.924+0000 7f1b6affd640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b5806b4d0 0x7f1b5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:37.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.924+0000 7f1b6affd640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b5806b4d0 0x7f1b5806d990 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f1b6c133cb0 tx=0x7f1b60009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:37.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:37.928+0000 7f1b53fff640 1 -- 172.21.15.64:0/173610169 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1b5c04e030 con 0x7f1b6c06ec40 2024-03-04T07:27:38.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.146+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f1b6c04f410 con 0x7f1b6c06ec40 2024-03-04T07:27:38.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.801+0000 7f1b53fff640 1 -- 172.21.15.64:0/173610169 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v9) v1 ==== 104+0+0 (secure 0 0 0) 0x7f1b6c1f6460 con 0x7f1b6c06ec40 2024-03-04T07:27:38.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b5806b4d0 msgr2=0x7f1b5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:38.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b5806b4d0 0x7f1b5806d990 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f1b6c133cb0 tx=0x7f1b60009040 comp rx=0 tx=0).stop 2024-03-04T07:27:38.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c06ec40 msgr2=0x7f1b6c1f5f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:38.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c06ec40 0x7f1b6c1f5f20 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f1b5c009d90 tx=0x7f1b5c004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:38.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 shutdown_connections 2024-03-04T07:27:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1b5806b4d0 0x7f1b5806d990 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b6c1445b0 0x7f1b6c1f1450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b6c072130 0x7f1b6c142930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 --2- 172.21.15.64:0/173610169 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b6c06ec40 0x7f1b6c1f5f20 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 >> 172.21.15.64:0/173610169 conn(0x7f1b6c0d09c0 msgr2=0x7f1b6c0ce570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 shutdown_connections 2024-03-04T07:27:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.803+0000 7f1b71b93640 1 -- 172.21.15.64:0/173610169 wait complete. 2024-03-04T07:27:38.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:27:38.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.926+0000 7fcfab3b5640 1 Processor -- start 2024-03-04T07:27:38.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.926+0000 7fcfab3b5640 1 -- start start 2024-03-04T07:27:38.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfab3b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa415dbd0 0x7fcfa415ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:38.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfab3b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcfa40b40c0 0x7fcfa4066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:38.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfab3b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcfa415f3b0 con 0x7fcfa40b5840 2024-03-04T07:27:38.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfab3b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcfa406f070 con 0x7fcfa40b40c0 2024-03-04T07:27:38.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfab3b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcfa4071940 con 0x7fcfa415dbd0 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfa9952640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcfa40b40c0 0x7fcfa4066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfa9952640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcfa40b40c0 0x7fcfa4066d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45548/0 (socket says 172.21.15.64:45548) 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfa9952640 1 -- 172.21.15.64:0/857742869 learned_addr learned my addr 172.21.15.64:0/857742869 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.927+0000 7fcfa9151640 1 --2- 172.21.15.64:0/857742869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa415dbd0 0x7fcfa415ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa9952640 1 -- 172.21.15.64:0/857742869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa415dbd0 msgr2=0x7fcfa415ee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa9952640 1 --2- 172.21.15.64:0/857742869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa415dbd0 0x7fcfa415ee70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa9952640 1 -- 172.21.15.64:0/857742869 >> v1:172.21.15.66:6789/0 conn(0x7fcfa40b5840 legacy=0x7fcfa40b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:38.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa9952640 1 -- 172.21.15.64:0/857742869 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcfa40b5db0 con 0x7fcfa40b40c0 2024-03-04T07:27:38.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa9952640 1 --2- 172.21.15.64:0/857742869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa40b40c0 0x7fcfa4066d00 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7fcf94009e50 tx=0x7fcf9402bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c1b0f13588bb165 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:38.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa3fff640 1 -- 172.21.15.64:0/857742869 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf9402cb00 con 0x7fcfa40b40c0 2024-03-04T07:27:38.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa3fff640 1 -- 172.21.15.64:0/857742869 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcf9402cca0 con 0x7fcfa40b40c0 2024-03-04T07:27:38.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.928+0000 7fcfa3fff640 1 -- 172.21.15.64:0/857742869 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf94034610 con 0x7fcfa40b40c0 2024-03-04T07:27:38.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 -- 172.21.15.64:0/857742869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa40b40c0 msgr2=0x7fcfa4066d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:38.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/857742869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa40b40c0 0x7fcfa4066d00 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7fcf94009e50 tx=0x7fcf9402bf60 comp rx=0 tx=0).stop 2024-03-04T07:27:38.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 -- 172.21.15.64:0/857742869 shutdown_connections 2024-03-04T07:27:38.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/857742869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa40b40c0 0x7fcfa4066d00 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/857742869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa415dbd0 0x7fcfa415ee70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 -- 172.21.15.64:0/857742869 >> 172.21.15.64:0/857742869 conn(0x7fcfa40b26b0 msgr2=0x7fcfa40b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:38.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 -- 172.21.15.64:0/857742869 shutdown_connections 2024-03-04T07:27:38.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.929+0000 7fcfab3b5640 1 -- 172.21.15.64:0/857742869 wait complete. 2024-03-04T07:27:38.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.930+0000 7fcfab3b5640 1 Processor -- start 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.930+0000 7fcfab3b5640 1 -- start start 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfab3b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcfa40b40c0 0x7fcfa41555e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfab3b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa40b5840 0x7fcfa41f43d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfab3b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa415dbd0 0x7fcfa415bd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfab3b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfa40721c0 con 0x7fcfa415dbd0 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfab3b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfa4071ee0 con 0x7fcfa40b40c0 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfab3b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcfa40726e0 con 0x7fcfa40b5840 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa415dbd0 0x7fcfa415bd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa415dbd0 0x7fcfa415bd80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45552/0 (socket says 172.21.15.64:45552) 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 -- 172.21.15.64:0/291267202 learned_addr learned my addr 172.21.15.64:0/291267202 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9151640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcfa40b40c0 0x7fcfa41555e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa8950640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa40b5840 0x7fcfa41f43d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 -- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa40b5840 msgr2=0x7fcfa41f43d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:38.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa40b5840 0x7fcfa41f43d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 -- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcfa40b40c0 msgr2=0x7fcfa41555e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcfa40b40c0 0x7fcfa41555e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9952640 1 -- 172.21.15.64:0/291267202 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcfa4156170 con 0x7fcfa415dbd0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa9151640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcfa40b40c0 0x7fcfa41555e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.931+0000 7fcfa8950640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa40b5840 0x7fcfa41f43d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.932+0000 7fcfa9952640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa415dbd0 0x7fcfa415bd80 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fcf94036040 tx=0x7fcf94004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.932+0000 7fcfa1ffb640 1 -- 172.21.15.64:0/291267202 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf94043020 con 0x7fcfa415dbd0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.932+0000 7fcfa1ffb640 1 -- 172.21.15.64:0/291267202 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcf9402ce50 con 0x7fcfa415dbd0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.932+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcfa4156450 con 0x7fcfa415dbd0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.932+0000 7fcfa1ffb640 1 -- 172.21.15.64:0/291267202 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf94009510 con 0x7fcfa415dbd0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.932+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcfa4156a10 con 0x7fcfa415dbd0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.933+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcfa4073380 con 0x7fcfa415dbd0 2024-03-04T07:27:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.933+0000 7fcfa1ffb640 1 -- 172.21.15.64:0/291267202 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcf94046020 con 0x7fcfa415dbd0 2024-03-04T07:27:38.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.934+0000 7fcfa1ffb640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf9006b4d0 0x7fcf9006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:38.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.934+0000 7fcfa1ffb640 1 -- 172.21.15.64:0/291267202 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(23..23 src has 1..23) v4 ==== 5911+0+0 (secure 0 0 0) 0x7fcf940ac850 con 0x7fcfa415dbd0 2024-03-04T07:27:38.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.934+0000 7fcfa9151640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf9006b4d0 0x7fcf9006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:38.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.934+0000 7fcfa9151640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf9006b4d0 0x7fcf9006d990 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fcf9c006fd0 tx=0x7fcf9c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:38.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:38.938+0000 7fcfa1ffb640 1 -- 172.21.15.64:0/291267202 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcf9404d030 con 0x7fcfa415dbd0 2024-03-04T07:27:39.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.118+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fcfa404f410 con 0x7fcfa415dbd0 2024-03-04T07:27:39.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.810+0000 7fcfa1ffb640 1 -- 172.21.15.64:0/291267202 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v24) v1 ==== 181+0+0 (secure 0 0 0) 0x7fcf9407b690 con 0x7fcfa415dbd0 2024-03-04T07:27:39.811 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:27:39.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.813+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf9006b4d0 msgr2=0x7fcf9006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:39.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.813+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf9006b4d0 0x7fcf9006d990 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fcf9c006fd0 tx=0x7fcf9c008040 comp rx=0 tx=0).stop 2024-03-04T07:27:39.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.813+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa415dbd0 msgr2=0x7fcfa415bd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:39.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.813+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa415dbd0 0x7fcfa415bd80 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fcf94036040 tx=0x7fcf94004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:39.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.813+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 shutdown_connections 2024-03-04T07:27:39.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.813+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf9006b4d0 0x7fcf9006d990 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.814+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcfa415dbd0 0x7fcfa415bd80 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.814+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcfa40b5840 0x7fcfa41f43d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.814+0000 7fcfab3b5640 1 --2- 172.21.15.64:0/291267202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcfa40b40c0 0x7fcfa41555e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.814+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 >> 172.21.15.64:0/291267202 conn(0x7fcfa40b26b0 msgr2=0x7fcfa415dfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:39.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.814+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 shutdown_connections 2024-03-04T07:27:39.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.814+0000 7fcfab3b5640 1 -- 172.21.15.64:0/291267202 wait complete. 2024-03-04T07:27:39.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:27:39.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.938+0000 7ff48cb5e640 1 Processor -- start 2024-03-04T07:27:39.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.938+0000 7ff48cb5e640 1 -- start start 2024-03-04T07:27:39.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff48cb5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880b3060 0x7ff488071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:39.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff48cb5e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff4880b2b10 0x7ff4880d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:39.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff48cb5e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff48815e4b0 con 0x7ff4880717b0 2024-03-04T07:27:39.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff48cb5e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff4880d2e10 con 0x7ff4880b2b10 2024-03-04T07:27:39.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff48cb5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4880b97e0 con 0x7ff4880b3060 2024-03-04T07:27:39.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff485d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880b3060 0x7ff488071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:39.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff486d9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff4880b2b10 0x7ff4880d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:39.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff48659c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7ff4880717b0 0x7ff4880b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38634/0 (socket says 172.21.15.64:38634) 2024-03-04T07:27:39.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff48659c640 1 -- 172.21.15.64:0/2778059068 learned_addr learned my addr 172.21.15.64:0/2778059068 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff485d9b640 1 -- 172.21.15.64:0/2778059068 >> v2:172.21.15.66:3300/0 conn(0x7ff4880b2b10 msgr2=0x7ff4880d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff485d9b640 1 --2- 172.21.15.64:0/2778059068 >> v2:172.21.15.66:3300/0 conn(0x7ff4880b2b10 0x7ff4880d32a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff485d9b640 1 -- 172.21.15.64:0/2778059068 >> v1:172.21.15.66:6789/0 conn(0x7ff4880717b0 legacy=0x7ff4880b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff485d9b640 1 -- 172.21.15.64:0/2778059068 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4880b9a80 con 0x7ff4880b3060 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.939+0000 7ff486d9d640 1 --2- 172.21.15.64:0/2778059068 >> v2:172.21.15.66:3300/0 conn(0x7ff4880b2b10 0x7ff4880d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.940+0000 7ff485d9b640 1 --2- 172.21.15.64:0/2778059068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880b3060 0x7ff488071b90 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7ff47c0097a0 tx=0x7ff47c02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ddd5cf1088c701f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.940+0000 7ff48559a640 1 -- 172.21.15.64:0/2778059068 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff47c035020 con 0x7ff4880b3060 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.940+0000 7ff48559a640 1 -- 172.21.15.64:0/2778059068 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff47c031830 con 0x7ff4880b3060 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.940+0000 7ff48559a640 1 -- 172.21.15.64:0/2778059068 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff47c031c00 con 0x7ff4880b3060 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 -- 172.21.15.64:0/2778059068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880b3060 msgr2=0x7ff488071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/2778059068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880b3060 0x7ff488071b90 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7ff47c0097a0 tx=0x7ff47c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 -- 172.21.15.64:0/2778059068 shutdown_connections 2024-03-04T07:27:39.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/2778059068 >> v2:172.21.15.66:3300/0 conn(0x7ff4880b2b10 0x7ff4880d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/2778059068 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880b3060 0x7ff488071b90 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 -- 172.21.15.64:0/2778059068 >> 172.21.15.64:0/2778059068 conn(0x7ff4880d3d20 msgr2=0x7ff4880d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:39.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 -- 172.21.15.64:0/2778059068 shutdown_connections 2024-03-04T07:27:39.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.941+0000 7ff48cb5e640 1 -- 172.21.15.64:0/2778059068 wait complete. 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.942+0000 7ff48cb5e640 1 Processor -- start 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.942+0000 7ff48cb5e640 1 -- start start 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff48cb5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880717b0 0x7ff488158be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff48cb5e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4880b2b10 0x7ff4881597e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff48cb5e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4880b3060 0x7ff488157130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff48cb5e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4880b7560 con 0x7ff4880b2b10 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff48cb5e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4880d4bf0 con 0x7ff4880b3060 2024-03-04T07:27:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff48cb5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4880b76e0 con 0x7ff4880717b0 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff485d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4880b2b10 0x7ff4881597e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff485d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4880b2b10 0x7ff4881597e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45568/0 (socket says 172.21.15.64:45568) 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff485d9b640 1 -- 172.21.15.64:0/184060526 learned_addr learned my addr 172.21.15.64:0/184060526 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff486d9d640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4880b3060 0x7ff488157130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.943+0000 7ff48659c640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880717b0 0x7ff488158be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff485d9b640 1 -- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880717b0 msgr2=0x7ff488158be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff485d9b640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880717b0 0x7ff488158be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff485d9b640 1 -- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4880b3060 msgr2=0x7ff488157130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff485d9b640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4880b3060 0x7ff488157130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff485d9b640 1 -- 172.21.15.64:0/184060526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4880cd390 con 0x7ff4880b2b10 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff485d9b640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4880b2b10 0x7ff4881597e0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7ff47c000c00 tx=0x7ff47c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff476ffd640 1 -- 172.21.15.64:0/184060526 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff47c040070 con 0x7ff4880b2b10 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff476ffd640 1 -- 172.21.15.64:0/184060526 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff47c03a850 con 0x7ff4880b2b10 2024-03-04T07:27:39.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4880cd670 con 0x7ff4880b2b10 2024-03-04T07:27:39.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff476ffd640 1 -- 172.21.15.64:0/184060526 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff47c035040 con 0x7ff4880b2b10 2024-03-04T07:27:39.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.944+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4880cdc30 con 0x7ff4880b2b10 2024-03-04T07:27:39.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.945+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff48806ce40 con 0x7ff4880b2b10 2024-03-04T07:27:39.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.946+0000 7ff476ffd640 1 -- 172.21.15.64:0/184060526 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff47c04b060 con 0x7ff4880b2b10 2024-03-04T07:27:39.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.946+0000 7ff476ffd640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff46806b4d0 0x7ff46806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:39.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.946+0000 7ff476ffd640 1 -- 172.21.15.64:0/184060526 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(24..24 src has 1..24) v4 ==== 5473+0+0 (secure 0 0 0) 0x7ff47c0aba30 con 0x7ff4880b2b10 2024-03-04T07:27:39.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.946+0000 7ff48659c640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff46806b4d0 0x7ff46806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:39.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.947+0000 7ff48659c640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff46806b4d0 0x7ff46806d990 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7ff478007fd0 tx=0x7ff478007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:39.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:39.950+0000 7ff476ffd640 1 -- 172.21.15.64:0/184060526 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff47c031800 con 0x7ff4880b2b10 2024-03-04T07:27:40.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.133+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7ff4880d41a0 con 0x7ff4880b2b10 2024-03-04T07:27:40.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.812+0000 7ff476ffd640 1 -- 172.21.15.64:0/184060526 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v25) v1 ==== 169+0+0 (secure 0 0 0) 0x7ff47c07a900 con 0x7ff4880b2b10 2024-03-04T07:27:40.813 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff46806b4d0 msgr2=0x7ff46806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff46806b4d0 0x7ff46806d990 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7ff478007fd0 tx=0x7ff478007e80 comp rx=0 tx=0).stop 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4880b2b10 msgr2=0x7ff4881597e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4880b2b10 0x7ff4881597e0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7ff47c000c00 tx=0x7ff47c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 shutdown_connections 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff46806b4d0 0x7ff46806d990 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4880b3060 0x7ff488157130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4880b2b10 0x7ff4881597e0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:40.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 --2- 172.21.15.64:0/184060526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4880717b0 0x7ff488158be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:40.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.814+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 >> 172.21.15.64:0/184060526 conn(0x7ff4880d3d20 msgr2=0x7ff48815a1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:40.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.815+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 shutdown_connections 2024-03-04T07:27:40.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:40.815+0000 7ff48cb5e640 1 -- 172.21.15.64:0/184060526 wait complete. 2024-03-04T07:27:40.833 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:27:40.833 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:27:40.834 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:27:40.834 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:40.834 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:27:40.834 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:27:40.834 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:27:40.834 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:40.834 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:27:40.835 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:27:40.835 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:27:40.835 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:40.835 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:27:40.835 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:27:40.835 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:27:40.835 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:27:40.845 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:27:40.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:27:40.846 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:27:40.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:27:40.847 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:27:40.847 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:27:40.848 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:27:40.848 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:27:40.888 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:27:40.888 DEBUG:teuthology.parallel:result is None 2024-03-04T07:27:40.889 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:27:40.889 DEBUG:teuthology.parallel:result is None 2024-03-04T07:27:40.891 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:27:40.891 DEBUG:teuthology.parallel:result is None 2024-03-04T07:27:40.892 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:27:40.892 DEBUG:teuthology.parallel:result is None 2024-03-04T07:27:40.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:27:40.970 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:27:40.971 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:27:40.971 INFO:tasks.ceph.mds.c.smithi064.stderr:2024-03-04T07:27:40.971+0000 7ff8c17eb640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6834/0: (98) Address already in use 2024-03-04T07:27:40.981 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:27:40.981 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:27:41.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.001+0000 7fb38b393640 1 Processor -- start 2024-03-04T07:27:41.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.001+0000 7fb38b393640 1 -- start start 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb38b393640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384067410 0x7fb3840d1540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb38b393640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb384070100 0x7fb384165930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb38b393640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb3840cd8c0 con 0x7fb3841456b0 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb38b393640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb38415d4d0 con 0x7fb384070100 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb38b393640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb38415cd90 con 0x7fb384067410 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb389930640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb384070100 0x7fb384165930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb389930640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb384070100 0x7fb384165930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45590/0 (socket says 172.21.15.64:45590) 2024-03-04T07:27:41.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.002+0000 7fb389930640 1 -- 172.21.15.64:0/16674479 learned_addr learned my addr 172.21.15.64:0/16674479 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.003+0000 7fb38892e640 1 --2- 172.21.15.64:0/16674479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384067410 0x7fb3840d1540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.003+0000 7fb389930640 1 -- 172.21.15.64:0/16674479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384067410 msgr2=0x7fb3840d1540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.003+0000 7fb389930640 1 --2- 172.21.15.64:0/16674479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384067410 0x7fb3840d1540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.003+0000 7fb389930640 1 -- 172.21.15.64:0/16674479 >> v1:172.21.15.66:6789/0 conn(0x7fb3841456b0 legacy=0x7fb384066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.003+0000 7fb389930640 1 -- 172.21.15.64:0/16674479 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb38415d070 con 0x7fb384070100 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.003+0000 7fb38892e640 1 --2- 172.21.15.64:0/16674479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384067410 0x7fb3840d1540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.003+0000 7fb389930640 1 --2- 172.21.15.64:0/16674479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384070100 0x7fb384165930 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fb384072e70 tx=0x7fb37c02be90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d76f8d99dbb829fd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb37bfff640 1 -- 172.21.15.64:0/16674479 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb37c02ca30 con 0x7fb384070100 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb37bfff640 1 -- 172.21.15.64:0/16674479 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb37c02cbd0 con 0x7fb384070100 2024-03-04T07:27:41.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb37bfff640 1 -- 172.21.15.64:0/16674479 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb37c0343f0 con 0x7fb384070100 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 -- 172.21.15.64:0/16674479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384070100 msgr2=0x7fb384165930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 --2- 172.21.15.64:0/16674479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384070100 0x7fb384165930 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fb384072e70 tx=0x7fb37c02be90 comp rx=0 tx=0).stop 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 -- 172.21.15.64:0/16674479 shutdown_connections 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 --2- 172.21.15.64:0/16674479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384070100 0x7fb384165930 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 --2- 172.21.15.64:0/16674479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384067410 0x7fb3840d1540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 -- 172.21.15.64:0/16674479 >> 172.21.15.64:0/16674479 conn(0x7fb3840d0a20 msgr2=0x7fb3840d0e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 -- 172.21.15.64:0/16674479 shutdown_connections 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.004+0000 7fb38b393640 1 -- 172.21.15.64:0/16674479 wait complete. 2024-03-04T07:27:41.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.005+0000 7fb38b393640 1 Processor -- start 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.005+0000 7fb38b393640 1 -- start start 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38b393640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384067410 0x7fb38413c700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38b393640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384070100 0x7fb38413d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38b393640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3841456b0 0x7fb38413d920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38b393640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3840d0220 con 0x7fb384067410 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38b393640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3840cd5e0 con 0x7fb3841456b0 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38b393640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb3840d03a0 con 0x7fb384070100 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38912f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384067410 0x7fb38413c700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38912f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384067410 0x7fb38413c700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45594/0 (socket says 172.21.15.64:45594) 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38912f640 1 -- 172.21.15.64:0/3266700800 learned_addr learned my addr 172.21.15.64:0/3266700800 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb38892e640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384070100 0x7fb38413d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.006+0000 7fb389930640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3841456b0 0x7fb38413d920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38912f640 1 -- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384070100 msgr2=0x7fb38413d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38912f640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384070100 0x7fb38413d3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38912f640 1 -- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3841456b0 msgr2=0x7fb38413d920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38912f640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3841456b0 0x7fb38413d920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38912f640 1 -- 172.21.15.64:0/3266700800 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb38413cc40 con 0x7fb384067410 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb389930640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3841456b0 0x7fb38413d920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38912f640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384067410 0x7fb38413c700 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fb37400b420 tx=0x7fb37400b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb379ffb640 1 -- 172.21.15.64:0/3266700800 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb374011830 con 0x7fb384067410 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb379ffb640 1 -- 172.21.15.64:0/3266700800 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb374016070 con 0x7fb384067410 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb38413df50 con 0x7fb384067410 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb379ffb640 1 -- 172.21.15.64:0/3266700800 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb374011c10 con 0x7fb384067410 2024-03-04T07:27:41.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.007+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb38413e7f0 con 0x7fb384067410 2024-03-04T07:27:41.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.008+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb384068700 con 0x7fb384067410 2024-03-04T07:27:41.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.009+0000 7fb379ffb640 1 -- 172.21.15.64:0/3266700800 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb37401b460 con 0x7fb384067410 2024-03-04T07:27:41.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.009+0000 7fb379ffb640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb36806b4d0 0x7fb36806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.009+0000 7fb379ffb640 1 -- 172.21.15.64:0/3266700800 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(25..25 src has 1..25) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fb3740898d0 con 0x7fb384067410 2024-03-04T07:27:41.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.009+0000 7fb38892e640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb36806b4d0 0x7fb36806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.009+0000 7fb38892e640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb36806b4d0 0x7fb36806d990 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fb38413f030 tx=0x7fb380008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:41.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.013+0000 7fb379ffb640 1 -- 172.21.15.64:0/3266700800 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb374020020 con 0x7fb384067410 2024-03-04T07:27:41.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.218+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7fb38413d0f0 con 0x7fb384067410 2024-03-04T07:27:41.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.822+0000 7fb379ffb640 1 -- 172.21.15.64:0/3266700800 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v27) v1 ==== 97+0+0 (secure 0 0 0) 0x7fb38413cc40 con 0x7fb384067410 2024-03-04T07:27:41.822 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:27:41.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb36806b4d0 msgr2=0x7fb36806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb36806b4d0 0x7fb36806d990 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fb38413f030 tx=0x7fb380008040 comp rx=0 tx=0).stop 2024-03-04T07:27:41.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384067410 msgr2=0x7fb38413c700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384067410 0x7fb38413c700 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fb37400b420 tx=0x7fb37400b8e0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 shutdown_connections 2024-03-04T07:27:41.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb36806b4d0 0x7fb36806d990 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3841456b0 0x7fb38413d920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb384070100 0x7fb38413d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.824+0000 7fb38b393640 1 --2- 172.21.15.64:0/3266700800 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb384067410 0x7fb38413c700 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.825+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 >> 172.21.15.64:0/3266700800 conn(0x7fb3840d0a20 msgr2=0x7fb384066d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:41.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.825+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 shutdown_connections 2024-03-04T07:27:41.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.825+0000 7fb38b393640 1 -- 172.21.15.64:0/3266700800 wait complete. 2024-03-04T07:27:41.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:27:41.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.943+0000 7f0eca335640 1 Processor -- start 2024-03-04T07:27:41.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.943+0000 7f0eca335640 1 -- start start 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0eca335640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0ec415bea0 0x7f0ec40cd330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0eca335640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40cf660 0x7f0ec40cfb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0eca335640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0ec40cf270 con 0x7f0ec40d2580 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0eca335640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0ec415c280 con 0x7f0ec415bea0 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0eca335640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ec40cc850 con 0x7f0ec40cf660 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0ec415bea0 0x7f0ec40cd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec88d2640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0ec40d2580 0x7f0ec406eb00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60674/0 (socket says 172.21.15.64:60674) 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec88d2640 1 -- 172.21.15.64:0/21411707 learned_addr learned my addr 172.21.15.64:0/21411707 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec37fe640 1 --2- 172.21.15.64:0/21411707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40cf660 0x7f0ec40cfb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec3fff640 1 -- 172.21.15.64:0/21411707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40cf660 msgr2=0x7f0ec40cfb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:41.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec3fff640 1 --2- 172.21.15.64:0/21411707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40cf660 0x7f0ec40cfb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec3fff640 1 -- 172.21.15.64:0/21411707 >> v1:172.21.15.64:6789/0 conn(0x7f0ec40d2580 legacy=0x7f0ec406eb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec3fff640 1 -- 172.21.15.64:0/21411707 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ec40ccb30 con 0x7f0ec415bea0 2024-03-04T07:27:41.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.944+0000 7f0ec2ffd640 1 -- 172.21.15.64:0/21411707 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 462891468 0 0) 0x7f0ec40cf270 con 0x7f0ec40d2580 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.945+0000 7f0ec3fff640 1 --2- 172.21.15.64:0/21411707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec40cd330 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f0eac0097d0 tx=0x7f0eac02b810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=53cfe0e89a2f5946 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.945+0000 7f0ec2ffd640 1 -- 172.21.15.64:0/21411707 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0eac031770 con 0x7f0ec415bea0 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.945+0000 7f0ec2ffd640 1 -- 172.21.15.64:0/21411707 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0eac037070 con 0x7f0ec415bea0 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.945+0000 7f0ec2ffd640 1 -- 172.21.15.64:0/21411707 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0eac031b30 con 0x7f0ec415bea0 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 -- 172.21.15.64:0/21411707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 msgr2=0x7f0ec40cd330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 --2- 172.21.15.64:0/21411707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec40cd330 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f0eac0097d0 tx=0x7f0eac02b810 comp rx=0 tx=0).stop 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 -- 172.21.15.64:0/21411707 shutdown_connections 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 --2- 172.21.15.64:0/21411707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40cf660 0x7f0ec40cfb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 --2- 172.21.15.64:0/21411707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec40cd330 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 -- 172.21.15.64:0/21411707 >> 172.21.15.64:0/21411707 conn(0x7f0ec40d1a90 msgr2=0x7f0ec40d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:41.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 -- 172.21.15.64:0/21411707 shutdown_connections 2024-03-04T07:27:41.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.946+0000 7f0eca335640 1 -- 172.21.15.64:0/21411707 wait complete. 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.947+0000 7f0eca335640 1 Processor -- start 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.947+0000 7f0eca335640 1 -- start start 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0eca335640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ec40cf660 0x7f0ec40c5710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0eca335640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40d2580 0x7f0ec40b6a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0eca335640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec415e420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0eca335640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ec40cc500 con 0x7f0ec415bea0 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0eca335640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ec40cd870 con 0x7f0ec40cf660 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0eca335640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ec40ceeb0 con 0x7f0ec40d2580 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec415e420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec415e420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45616/0 (socket says 172.21.15.64:45616) 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 -- 172.21.15.64:0/2551223759 learned_addr learned my addr 172.21.15.64:0/2551223759 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec37fe640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40d2580 0x7f0ec40b6a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec3fff640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ec40cf660 0x7f0ec40c5710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 -- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40d2580 msgr2=0x7f0ec40b6a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40d2580 0x7f0ec40b6a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 -- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ec40cf660 msgr2=0x7f0ec40c5710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ec40cf660 0x7f0ec40c5710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec88d2640 1 -- 172.21.15.64:0/2551223759 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ec40c8fc0 con 0x7f0ec415bea0 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.948+0000 7f0ec3fff640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ec40cf660 0x7f0ec40c5710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.949+0000 7f0ec88d2640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec415e420 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f0ebc00c6d0 tx=0x7f0ebc00cb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.949+0000 7f0ec0ff9640 1 -- 172.21.15.64:0/2551223759 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ebc018070 con 0x7f0ec415bea0 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.949+0000 7f0ec0ff9640 1 -- 172.21.15.64:0/2551223759 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ebc004d60 con 0x7f0ec415bea0 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.949+0000 7f0ec0ff9640 1 -- 172.21.15.64:0/2551223759 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ebc00a880 con 0x7f0ec415bea0 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.949+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0ec40c9300 con 0x7f0ec415bea0 2024-03-04T07:27:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.949+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0ec40c9b40 con 0x7f0ec415bea0 2024-03-04T07:27:41.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.950+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0ec4067ff0 con 0x7f0ec415bea0 2024-03-04T07:27:41.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.950+0000 7f0ec0ff9640 1 -- 172.21.15.64:0/2551223759 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0ebc01b070 con 0x7f0ec415bea0 2024-03-04T07:27:41.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.950+0000 7f0ec0ff9640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ea406b4d0 0x7f0ea406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:41.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.951+0000 7f0ec0ff9640 1 -- 172.21.15.64:0/2551223759 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f0ebc089640 con 0x7f0ec415bea0 2024-03-04T07:27:41.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.951+0000 7f0ec3fff640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ea406b4d0 0x7f0ea406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:41.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.951+0000 7f0ec3fff640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ea406b4d0 0x7f0ea406d990 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f0eac036000 tx=0x7f0eac008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:41.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:41.955+0000 7f0ec0ff9640 1 -- 172.21.15.64:0/2551223759 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0ec40c9300 con 0x7f0ec415bea0 2024-03-04T07:27:42.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.201+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f0ec4070b20 con 0x7f0ec415bea0 2024-03-04T07:27:42.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.201+0000 7f0ec0ff9640 1 -- 172.21.15.64:0/2551223759 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v8) v1 ==== 80+0+6645 (secure 0 0 0) 0x7f0ebc058810 con 0x7f0ec415bea0 2024-03-04T07:27:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:42.202 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:27:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:27:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:27:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:27:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:27:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:27:42.208 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.bootstrap-mds", 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCkd+VlB64lLRAA59E+Ey/ub2EPTslR8r0bRQ==", 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile bootstrap-mds" 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.bootstrap-mgr", 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCkd+VlYLglLRAA0wkgdjVBOU2f3Ik83dtxEg==", 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile bootstrap-mgr" 2024-03-04T07:27:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.bootstrap-osd", 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCkd+VlBcMlLRAAfF3puSBoPFKaUqdSD1AhxQ==", 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile bootstrap-osd" 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.bootstrap-rbd", 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCkd+Vl/sklLRAApEZRuhDMMsLLhZA6KdMYHg==", 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile bootstrap-rbd" 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.bootstrap-rbd-mirror", 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCkd+Vls9AlLRAAT3LGDrdYWdKZrLUE9uJirQ==", 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile bootstrap-rbd-mirror" 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.bootstrap-rgw", 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCkd+VltdclLRAAqb7xdBhulCwz6/cp8ya9Pw==", 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile bootstrap-rgw" 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:27:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:27:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:27:42.218 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:27:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ea406b4d0 msgr2=0x7f0ea406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ea406b4d0 0x7f0ea406d990 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f0eac036000 tx=0x7f0eac008030 comp rx=0 tx=0).stop 2024-03-04T07:27:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 msgr2=0x7f0ec415e420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec415e420 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f0ebc00c6d0 tx=0x7f0ebc00cb90 comp rx=0 tx=0).stop 2024-03-04T07:27:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 shutdown_connections 2024-03-04T07:27:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ea406b4d0 0x7f0ea406d990 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ec415bea0 0x7f0ec415e420 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ec40d2580 0x7f0ec40b6a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 --2- 172.21.15.64:0/2551223759 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ec40cf660 0x7f0ec40c5710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 >> 172.21.15.64:0/2551223759 conn(0x7f0ec40d1a90 msgr2=0x7f0ec40aea40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 shutdown_connections 2024-03-04T07:27:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.203+0000 7f0eca335640 1 -- 172.21.15.64:0/2551223759 wait complete. 2024-03-04T07:27:42.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-mds 2024-03-04T07:27:42.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.322+0000 7fa1635d7640 1 Processor -- start 2024-03-04T07:27:42.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.323+0000 7fa1635d7640 1 -- start start 2024-03-04T07:27:42.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.323+0000 7fa1635d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 0x7fa15c15ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.323+0000 7fa1635d7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa15c072660 0x7fa15c06d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.323+0000 7fa1635d7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa15c0b55d0 0x7fa15c071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.323+0000 7fa1635d7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa15c0b4160 con 0x7fa15c0b55d0 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.323+0000 7fa1635d7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa15c0b5b40 con 0x7fa15c072660 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.323+0000 7fa1635d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa15c0b3ba0 con 0x7fa15c06e700 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa15c0b55d0 0x7fa15c071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa15c0b55d0 0x7fa15c071980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45618/0 (socket says 172.21.15.64:45618) 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 -- 172.21.15.64:0/3647832210 learned_addr learned my addr 172.21.15.64:0/3647832210 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa160b72640 1 --2- 172.21.15.64:0/3647832210 >> v2:172.21.15.66:3300/0 conn(0x7fa15c072660 0x7fa15c06d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161373640 1 --2- 172.21.15.64:0/3647832210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 0x7fa15c15ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 -- 172.21.15.64:0/3647832210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 msgr2=0x7fa15c15ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 --2- 172.21.15.64:0/3647832210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 0x7fa15c15ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 -- 172.21.15.64:0/3647832210 >> v2:172.21.15.66:3300/0 conn(0x7fa15c072660 msgr2=0x7fa15c06d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 --2- 172.21.15.64:0/3647832210 >> v2:172.21.15.66:3300/0 conn(0x7fa15c072660 0x7fa15c06d9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.324+0000 7fa161b74640 1 -- 172.21.15.64:0/3647832210 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa15c0b3e80 con 0x7fa15c0b55d0 2024-03-04T07:27:42.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa161b74640 1 --2- 172.21.15.64:0/3647832210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c0b55d0 0x7fa15c071980 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7fa15c06ccb0 tx=0x7fa15402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4120c772998a8800 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:42.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa15bfff640 1 -- 172.21.15.64:0/3647832210 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa15402ca70 con 0x7fa15c0b55d0 2024-03-04T07:27:42.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa15bfff640 1 -- 172.21.15.64:0/3647832210 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa15402cc10 con 0x7fa15c0b55d0 2024-03-04T07:27:42.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa15bfff640 1 -- 172.21.15.64:0/3647832210 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1540343f0 con 0x7fa15c0b55d0 2024-03-04T07:27:42.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 -- 172.21.15.64:0/3647832210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c0b55d0 msgr2=0x7fa15c071980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 --2- 172.21.15.64:0/3647832210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c0b55d0 0x7fa15c071980 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7fa15c06ccb0 tx=0x7fa15402bed0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 -- 172.21.15.64:0/3647832210 shutdown_connections 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 --2- 172.21.15.64:0/3647832210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c0b55d0 0x7fa15c071980 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 --2- 172.21.15.64:0/3647832210 >> v2:172.21.15.66:3300/0 conn(0x7fa15c072660 0x7fa15c06d9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 --2- 172.21.15.64:0/3647832210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 0x7fa15c15ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 -- 172.21.15.64:0/3647832210 >> 172.21.15.64:0/3647832210 conn(0x7fa15c0b2720 msgr2=0x7fa15c0b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 -- 172.21.15.64:0/3647832210 shutdown_connections 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.325+0000 7fa1635d7640 1 -- 172.21.15.64:0/3647832210 wait complete. 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 Processor -- start 2024-03-04T07:27:42.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 -- start start 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 0x7fa15c0b0a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c072660 0x7fa15c157a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa15c0b55d0 0x7fa15c15a7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa15c06ec60 con 0x7fa15c072660 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa15c06eae0 con 0x7fa15c0b55d0 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa1635d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa15c071ec0 con 0x7fa15c06e700 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa160b72640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c072660 0x7fa15c157a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa160b72640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c072660 0x7fa15c157a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45634/0 (socket says 172.21.15.64:45634) 2024-03-04T07:27:42.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.327+0000 7fa160b72640 1 -- 172.21.15.64:0/2230830848 learned_addr learned my addr 172.21.15.64:0/2230830848 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa160b72640 1 -- 172.21.15.64:0/2230830848 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 msgr2=0x7fa15c0b0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa160b72640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 0x7fa15c0b0a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa160b72640 1 -- 172.21.15.64:0/2230830848 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa15c0b55d0 msgr2=0x7fa15c15a7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa160b72640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa15c0b55d0 0x7fa15c15a7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa160b72640 1 -- 172.21.15.64:0/2230830848 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa15c0b1030 con 0x7fa15c072660 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa160b72640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c072660 0x7fa15c157a80 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fa15000e9b0 tx=0x7fa15000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa159ffb640 1 -- 172.21.15.64:0/2230830848 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa150017070 con 0x7fa15c072660 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa159ffb640 1 -- 172.21.15.64:0/2230830848 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa15000c9a0 con 0x7fa15c072660 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa15c0b1310 con 0x7fa15c072660 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa159ffb640 1 -- 172.21.15.64:0/2230830848 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa15000cd90 con 0x7fa15c072660 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.328+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa15c15ad30 con 0x7fa15c072660 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.329+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa15c0ad8b0 con 0x7fa15c072660 2024-03-04T07:27:42.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.329+0000 7fa159ffb640 1 -- 172.21.15.64:0/2230830848 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa15001a070 con 0x7fa15c072660 2024-03-04T07:27:42.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.330+0000 7fa159ffb640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14806b4d0 0x7fa14806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.330+0000 7fa159ffb640 1 -- 172.21.15.64:0/2230830848 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fa15008a2d0 con 0x7fa15c072660 2024-03-04T07:27:42.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.330+0000 7fa161373640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14806b4d0 0x7fa14806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.330+0000 7fa161373640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14806b4d0 0x7fa14806d990 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fa14c009a40 tx=0x7fa14c009500 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:42.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.334+0000 7fa159ffb640 1 -- 172.21.15.64:0/2230830848 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa150059410 con 0x7fa15c072660 2024-03-04T07:27:42.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.584+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-mds"} v 0) v1 -- 0x7fa15c067ff0 con 0x7fa15c072660 2024-03-04T07:27:42.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.587+0000 7fa159ffb640 1 -- 172.21.15.64:0/2230830848 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-mds"}]=0 v9) v1 ==== 90+0+0 (secure 0 0 0) 0x7fa150058d10 con 0x7fa15c072660 2024-03-04T07:27:42.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.588+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14806b4d0 msgr2=0x7fa14806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14806b4d0 0x7fa14806d990 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fa14c009a40 tx=0x7fa14c009500 comp rx=0 tx=0).stop 2024-03-04T07:27:42.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c072660 msgr2=0x7fa15c157a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c072660 0x7fa15c157a80 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fa15000e9b0 tx=0x7fa15000ee70 comp rx=0 tx=0).stop 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 shutdown_connections 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14806b4d0 0x7fa14806d990 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa15c0b55d0 0x7fa15c15a7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa15c072660 0x7fa15c157a80 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 --2- 172.21.15.64:0/2230830848 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa15c06e700 0x7fa15c0b0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 >> 172.21.15.64:0/2230830848 conn(0x7fa15c0b2720 msgr2=0x7fa15c160cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 shutdown_connections 2024-03-04T07:27:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.589+0000 7fa1635d7640 1 -- 172.21.15.64:0/2230830848 wait complete. 2024-03-04T07:27:42.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-mgr 2024-03-04T07:27:42.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.702+0000 7f1fa3f8a640 1 Processor -- start 2024-03-04T07:27:42.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.702+0000 7f1fa3f8a640 1 -- start start 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa3f8a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1f9c15e060 0x7f1f9c0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa3f8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c06fe20 0x7f1f9c0ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa3f8a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1f9c0d0880 0x7f1f9c0ced90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa3f8a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1f9c0d0dd0 con 0x7f1f9c0d0880 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa3f8a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1f9c0d10b0 con 0x7f1f9c15e060 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa3f8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f9c0cc0e0 con 0x7f1f9c06fe20 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1f9c0d0880 0x7f1f9c0ced90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1f9c0d0880 0x7f1f9c0ced90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45642/0 (socket says 172.21.15.64:45642) 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 -- 172.21.15.64:0/2947936400 learned_addr learned my addr 172.21.15.64:0/2947936400 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa1525640 1 --2- 172.21.15.64:0/2947936400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c06fe20 0x7f1f9c0ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa1d26640 1 --2- 172.21.15.64:0/2947936400 >> v2:172.21.15.66:3300/0 conn(0x7f1f9c15e060 0x7f1f9c0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 -- 172.21.15.64:0/2947936400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c06fe20 msgr2=0x7f1f9c0ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 --2- 172.21.15.64:0/2947936400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c06fe20 0x7f1f9c0ce850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 -- 172.21.15.64:0/2947936400 >> v2:172.21.15.66:3300/0 conn(0x7f1f9c15e060 msgr2=0x7f1f9c0ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 --2- 172.21.15.64:0/2947936400 >> v2:172.21.15.66:3300/0 conn(0x7f1f9c15e060 0x7f1f9c0ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa2527640 1 -- 172.21.15.64:0/2947936400 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f9c0cc3c0 con 0x7f1f9c0d0880 2024-03-04T07:27:42.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.703+0000 7f1fa1525640 1 --2- 172.21.15.64:0/2947936400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c06fe20 0x7f1f9c0ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:42.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.704+0000 7f1fa2527640 1 --2- 172.21.15.64:0/2947936400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c0d0880 0x7f1f9c0ced90 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f1f9c0722f0 tx=0x7f1f8c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1bed14b7e422837c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:42.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.704+0000 7f1fa0d24640 1 -- 172.21.15.64:0/2947936400 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8c02ca70 con 0x7f1f9c0d0880 2024-03-04T07:27:42.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.704+0000 7f1fa0d24640 1 -- 172.21.15.64:0/2947936400 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f8c02cc10 con 0x7f1f9c0d0880 2024-03-04T07:27:42.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.704+0000 7f1fa0d24640 1 -- 172.21.15.64:0/2947936400 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8c0343f0 con 0x7f1f9c0d0880 2024-03-04T07:27:42.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/2947936400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c0d0880 msgr2=0x7f1f9c0ced90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/2947936400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c0d0880 0x7f1f9c0ced90 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f1f9c0722f0 tx=0x7f1f8c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/2947936400 shutdown_connections 2024-03-04T07:27:42.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/2947936400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c0d0880 0x7f1f9c0ced90 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/2947936400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c06fe20 0x7f1f9c0ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/2947936400 >> v2:172.21.15.66:3300/0 conn(0x7f1f9c15e060 0x7f1f9c0ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/2947936400 >> 172.21.15.64:0/2947936400 conn(0x7f1f9c0d1a90 msgr2=0x7f1f9c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/2947936400 shutdown_connections 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.705+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/2947936400 wait complete. 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.706+0000 7f1fa3f8a640 1 Processor -- start 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.706+0000 7f1fa3f8a640 1 -- start start 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa3f8a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9c06fe20 0x7f1f9c15c210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa3f8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c0d0880 0x7f1f9c1579d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa3f8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c15e060 0x7f1f9c157f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa3f8a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f9c06ed30 con 0x7f1f9c15e060 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa3f8a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f9c15e440 con 0x7f1f9c06fe20 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa3f8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f9c06eeb0 con 0x7f1f9c0d0880 2024-03-04T07:27:42.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c15e060 0x7f1f9c157f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c15e060 0x7f1f9c157f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45648/0 (socket says 172.21.15.64:45648) 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 -- 172.21.15.64:0/3911116860 learned_addr learned my addr 172.21.15.64:0/3911116860 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 -- 172.21.15.64:0/3911116860 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c0d0880 msgr2=0x7f1f9c1579d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c0d0880 0x7f1f9c1579d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 -- 172.21.15.64:0/3911116860 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9c06fe20 msgr2=0x7f1f9c15c210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9c06fe20 0x7f1f9c15c210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 -- 172.21.15.64:0/3911116860 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f9c0cd800 con 0x7f1f9c15e060 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1fa2527640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c15e060 0x7f1f9c157f10 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f1f8c036040 tx=0x7f1f8c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.707+0000 7f1f9a7fc640 1 -- 172.21.15.64:0/3911116860 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8c043020 con 0x7f1f9c15e060 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.708+0000 7f1f9a7fc640 1 -- 172.21.15.64:0/3911116860 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f8c03e450 con 0x7f1f9c15e060 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.708+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1f9c0cdae0 con 0x7f1f9c15e060 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.708+0000 7f1f9a7fc640 1 -- 172.21.15.64:0/3911116860 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8c009510 con 0x7f1f9c15e060 2024-03-04T07:27:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.708+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1f9c0ce0a0 con 0x7f1f9c15e060 2024-03-04T07:27:42.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.709+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1f9c067ff0 con 0x7f1f9c15e060 2024-03-04T07:27:42.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.709+0000 7f1f9a7fc640 1 -- 172.21.15.64:0/3911116860 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1f8c046020 con 0x7f1f9c15e060 2024-03-04T07:27:42.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.709+0000 7f1f9a7fc640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f8806b4d0 0x7f1f8806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:42.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.709+0000 7f1f9a7fc640 1 -- 172.21.15.64:0/3911116860 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f1f8c0ac6c0 con 0x7f1f9c15e060 2024-03-04T07:27:42.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.709+0000 7f1fa1d26640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f8806b4d0 0x7f1f8806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:42.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.710+0000 7f1fa1d26640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f8806b4d0 0x7f1f8806d990 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f1f9400b7f0 tx=0x7f1f94008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:42.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.714+0000 7f1f9a7fc640 1 -- 172.21.15.64:0/3911116860 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1f8c04d030 con 0x7f1f9c15e060 2024-03-04T07:27:42.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.965+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-mgr"} v 0) v1 -- 0x7f1f9c15d3c0 con 0x7f1f9c15e060 2024-03-04T07:27:42.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.967+0000 7f1f9a7fc640 1 -- 172.21.15.64:0/3911116860 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-mgr"}]=0 v10) v1 ==== 90+0+0 (secure 0 0 0) 0x7f1f8c07b920 con 0x7f1f9c15e060 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f8806b4d0 msgr2=0x7f1f8806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f8806b4d0 0x7f1f8806d990 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f1f9400b7f0 tx=0x7f1f94008040 comp rx=0 tx=0).stop 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c15e060 msgr2=0x7f1f9c157f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c15e060 0x7f1f9c157f10 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f1f8c036040 tx=0x7f1f8c004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 shutdown_connections 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f8806b4d0 0x7f1f8806d990 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f9c15e060 0x7f1f9c157f10 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f9c0d0880 0x7f1f9c1579d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 --2- 172.21.15.64:0/3911116860 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f9c06fe20 0x7f1f9c15c210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 >> 172.21.15.64:0/3911116860 conn(0x7f1f9c0d1a90 msgr2=0x7f1f9c15c7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 shutdown_connections 2024-03-04T07:27:42.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:42.969+0000 7f1fa3f8a640 1 -- 172.21.15.64:0/3911116860 wait complete. 2024-03-04T07:27:42.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-osd 2024-03-04T07:27:43.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.081+0000 7fcc534c9640 1 Processor -- start 2024-03-04T07:27:43.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.082+0000 7fcc534c9640 1 -- start start 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.082+0000 7fcc534c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcc4c15e060 0x7fcc4c0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.082+0000 7fcc534c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c06fe20 0x7fcc4c0ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.082+0000 7fcc534c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc4c0d0dd0 con 0x7fcc4c0d0880 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.082+0000 7fcc534c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc4c0d10b0 con 0x7fcc4c15e060 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.082+0000 7fcc534c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc4c0cc0e0 con 0x7fcc4c06fe20 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc51a66640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcc4c0d0880 0x7fcc4c06ed30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60684/0 (socket says 172.21.15.64:60684) 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc51a66640 1 -- 172.21.15.64:0/2152996998 learned_addr learned my addr 172.21.15.64:0/2152996998 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc50a64640 1 --2- 172.21.15.64:0/2152996998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c06fe20 0x7fcc4c0ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc51265640 1 --2- 172.21.15.64:0/2152996998 >> v2:172.21.15.66:3300/0 conn(0x7fcc4c15e060 0x7fcc4c0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc3ffff640 1 -- 172.21.15.64:0/2152996998 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 503460697 0 0) 0x7fcc4c0d0dd0 con 0x7fcc4c0d0880 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc3ffff640 1 -- 172.21.15.64:0/2152996998 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcc38003520 con 0x7fcc4c0d0880 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc50a64640 1 -- 172.21.15.64:0/2152996998 >> v2:172.21.15.66:3300/0 conn(0x7fcc4c15e060 msgr2=0x7fcc4c0ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:43.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc50a64640 1 --2- 172.21.15.64:0/2152996998 >> v2:172.21.15.66:3300/0 conn(0x7fcc4c15e060 0x7fcc4c0ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc50a64640 1 -- 172.21.15.64:0/2152996998 >> v1:172.21.15.64:6789/0 conn(0x7fcc4c0d0880 legacy=0x7fcc4c06ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc3ffff640 1 -- 172.21.15.64:0/2152996998 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2106829163 0 0) 0x7fcc38003520 con 0x7fcc4c0d0880 2024-03-04T07:27:43.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.083+0000 7fcc50a64640 1 -- 172.21.15.64:0/2152996998 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc4c0cc3c0 con 0x7fcc4c06fe20 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.084+0000 7fcc50a64640 1 --2- 172.21.15.64:0/2152996998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c06fe20 0x7fcc4c0ce850 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fcc480097a0 tx=0x7fcc4802b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=75142cddaacc10dd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.084+0000 7fcc3ffff640 1 -- 172.21.15.64:0/2152996998 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc48037070 con 0x7fcc4c06fe20 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.084+0000 7fcc3ffff640 1 -- 172.21.15.64:0/2152996998 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc48030510 con 0x7fcc4c06fe20 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.084+0000 7fcc3ffff640 1 -- 172.21.15.64:0/2152996998 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc440019f0 con 0x7fcc4c06fe20 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 -- 172.21.15.64:0/2152996998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c06fe20 msgr2=0x7fcc4c0ce850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 --2- 172.21.15.64:0/2152996998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c06fe20 0x7fcc4c0ce850 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7fcc480097a0 tx=0x7fcc4802b790 comp rx=0 tx=0).stop 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 -- 172.21.15.64:0/2152996998 shutdown_connections 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 --2- 172.21.15.64:0/2152996998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c06fe20 0x7fcc4c0ce850 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 --2- 172.21.15.64:0/2152996998 >> v2:172.21.15.66:3300/0 conn(0x7fcc4c15e060 0x7fcc4c0ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 -- 172.21.15.64:0/2152996998 >> 172.21.15.64:0/2152996998 conn(0x7fcc4c0d1a90 msgr2=0x7fcc4c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:43.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 -- 172.21.15.64:0/2152996998 shutdown_connections 2024-03-04T07:27:43.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.085+0000 7fcc534c9640 1 -- 172.21.15.64:0/2152996998 wait complete. 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.086+0000 7fcc534c9640 1 Processor -- start 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.086+0000 7fcc534c9640 1 -- start start 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc534c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c06fe20 0x7fcc4c15af90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc534c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0d0880 0x7fcc4c0cae40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc534c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c15e060 0x7fcc4c152c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc534c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc4c0cdbd0 con 0x7fcc4c06fe20 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc534c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc4c15e440 con 0x7fcc4c15e060 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc534c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc4c0cdd50 con 0x7fcc4c0d0880 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c06fe20 0x7fcc4c15af90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c06fe20 0x7fcc4c15af90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45662/0 (socket says 172.21.15.64:45662) 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 -- 172.21.15.64:0/3967516126 learned_addr learned my addr 172.21.15.64:0/3967516126 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc50a64640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0d0880 0x7fcc4c0cae40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51a66640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c15e060 0x7fcc4c152c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 -- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0d0880 msgr2=0x7fcc4c0cae40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0d0880 0x7fcc4c0cae40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 -- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c15e060 msgr2=0x7fcc4c152c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c15e060 0x7fcc4c152c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 -- 172.21.15.64:0/3967516126 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc4c15b650 con 0x7fcc4c06fe20 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51a66640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c15e060 0x7fcc4c152c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc51265640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c06fe20 0x7fcc4c15af90 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fcc4000e9b0 tx=0x7fcc4000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc3dffb640 1 -- 172.21.15.64:0/3967516126 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc40017070 con 0x7fcc4c06fe20 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc3dffb640 1 -- 172.21.15.64:0/3967516126 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc4000c9a0 con 0x7fcc4c06fe20 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.087+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc4c15b930 con 0x7fcc4c06fe20 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.088+0000 7fcc3dffb640 1 -- 172.21.15.64:0/3967516126 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc4000cd90 con 0x7fcc4c06fe20 2024-03-04T07:27:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.088+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc4c0c8c90 con 0x7fcc4c06fe20 2024-03-04T07:27:43.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.089+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc4c067ff0 con 0x7fcc4c06fe20 2024-03-04T07:27:43.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.089+0000 7fcc3dffb640 1 -- 172.21.15.64:0/3967516126 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcc4001a070 con 0x7fcc4c06fe20 2024-03-04T07:27:43.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.089+0000 7fcc3dffb640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc3006b4d0 0x7fcc3006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.090+0000 7fcc3dffb640 1 -- 172.21.15.64:0/3967516126 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fcc4008a100 con 0x7fcc4c06fe20 2024-03-04T07:27:43.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.090+0000 7fcc50a64640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc3006b4d0 0x7fcc3006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.090+0000 7fcc50a64640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc3006b4d0 0x7fcc3006d990 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fcc48034000 tx=0x7fcc4802f000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.094+0000 7fcc3dffb640 1 -- 172.21.15.64:0/3967516126 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcc40058d10 con 0x7fcc4c06fe20 2024-03-04T07:27:43.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.349+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-osd"} v 0) v1 -- 0x7fcc4c0cbd70 con 0x7fcc4c06fe20 2024-03-04T07:27:43.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.353+0000 7fcc3dffb640 1 -- 172.21.15.64:0/3967516126 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-osd"}]=0 v11) v1 ==== 90+0+0 (secure 0 0 0) 0x7fcc4005cdb0 con 0x7fcc4c06fe20 2024-03-04T07:27:43.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.355+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc3006b4d0 msgr2=0x7fcc3006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.355+0000 7fcc534c9640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc3006b4d0 0x7fcc3006d990 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fcc48034000 tx=0x7fcc4802f000 comp rx=0 tx=0).stop 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c06fe20 msgr2=0x7fcc4c15af90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c06fe20 0x7fcc4c15af90 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fcc4000e9b0 tx=0x7fcc4000ee70 comp rx=0 tx=0).stop 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 shutdown_connections 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc3006b4d0 0x7fcc3006d990 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c15e060 0x7fcc4c152c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0d0880 0x7fcc4c0cae40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 --2- 172.21.15.64:0/3967516126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c06fe20 0x7fcc4c15af90 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 >> 172.21.15.64:0/3967516126 conn(0x7fcc4c0d1a90 msgr2=0x7fcc4c0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 shutdown_connections 2024-03-04T07:27:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.356+0000 7fcc534c9640 1 -- 172.21.15.64:0/3967516126 wait complete. 2024-03-04T07:27:43.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-rbd 2024-03-04T07:27:43.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.468+0000 7ff6fb18c640 1 Processor -- start 2024-03-04T07:27:43.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6fb18c640 1 -- start start 2024-03-04T07:27:43.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6fb18c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff6f415e030 0x7ff6f406ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6fb18c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f406ef70 0x7ff6f40b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6fb18c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff6f40b2290 0x7ff6f40d0040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6fb18c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff6f4162700 con 0x7ff6f40b2290 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6fb18c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff6f415e410 con 0x7ff6f415e030 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6fb18c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff6f4161af0 con 0x7ff6f406ef70 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6f9729640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff6f40b2290 0x7ff6f40d0040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6f9729640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff6f40b2290 0x7ff6f40d0040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38738/0 (socket says 172.21.15.64:38738) 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.469+0000 7ff6f9729640 1 -- 172.21.15.64:0/2140805713 learned_addr learned my addr 172.21.15.64:0/2140805713 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f8f28640 1 --2- 172.21.15.64:0/2140805713 >> v2:172.21.15.66:3300/0 conn(0x7ff6f415e030 0x7ff6f406ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f3fff640 1 --2- 172.21.15.64:0/2140805713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f406ef70 0x7ff6f40b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f9729640 1 -- 172.21.15.64:0/2140805713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f406ef70 msgr2=0x7ff6f40b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f9729640 1 --2- 172.21.15.64:0/2140805713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f406ef70 0x7ff6f40b1d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f9729640 1 -- 172.21.15.64:0/2140805713 >> v2:172.21.15.66:3300/0 conn(0x7ff6f415e030 msgr2=0x7ff6f406ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f9729640 1 --2- 172.21.15.64:0/2140805713 >> v2:172.21.15.66:3300/0 conn(0x7ff6f415e030 0x7ff6f406ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f9729640 1 -- 172.21.15.64:0/2140805713 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6f4161dd0 con 0x7ff6f40b2290 2024-03-04T07:27:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.470+0000 7ff6f8f28640 1 --2- 172.21.15.64:0/2140805713 >> v2:172.21.15.66:3300/0 conn(0x7ff6f415e030 0x7ff6f406ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6f9729640 1 --2- 172.21.15.64:0/2140805713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f40b2290 0x7ff6f40d0040 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7ff6f4161a50 tx=0x7ff6ec02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=69341a5e928faef9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6f37fe640 1 -- 172.21.15.64:0/2140805713 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6ec02ca70 con 0x7ff6f40b2290 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6f37fe640 1 -- 172.21.15.64:0/2140805713 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff6ec02cc10 con 0x7ff6f40b2290 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6f37fe640 1 -- 172.21.15.64:0/2140805713 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6ec0343f0 con 0x7ff6f40b2290 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2140805713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f40b2290 msgr2=0x7ff6f40d0040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2140805713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f40b2290 0x7ff6f40d0040 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7ff6f4161a50 tx=0x7ff6ec02bed0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2140805713 shutdown_connections 2024-03-04T07:27:43.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2140805713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f40b2290 0x7ff6f40d0040 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2140805713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f406ef70 0x7ff6f40b1d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2140805713 >> v2:172.21.15.66:3300/0 conn(0x7ff6f415e030 0x7ff6f406ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2140805713 >> 172.21.15.64:0/2140805713 conn(0x7ff6f40b55e0 msgr2=0x7ff6f40b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:43.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2140805713 shutdown_connections 2024-03-04T07:27:43.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.471+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2140805713 wait complete. 2024-03-04T07:27:43.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.472+0000 7ff6fb18c640 1 Processor -- start 2024-03-04T07:27:43.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.472+0000 7ff6fb18c640 1 -- start start 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6fb18c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f406ef70 0x7ff6f40c9440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6fb18c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff6f40b2290 0x7ff6f415f8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6fb18c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f415e030 0x7ff6f4146ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6fb18c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6f40b31e0 con 0x7ff6f406ef70 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6fb18c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6f4162390 con 0x7ff6f40b2290 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6fb18c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff6f40b3360 con 0x7ff6f415e030 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6f8f28640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f406ef70 0x7ff6f40c9440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6f8f28640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f406ef70 0x7ff6f40c9440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38754/0 (socket says 172.21.15.64:38754) 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.473+0000 7ff6f8f28640 1 -- 172.21.15.64:0/2660937506 learned_addr learned my addr 172.21.15.64:0/2660937506 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f3fff640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff6f40b2290 0x7ff6f415f8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f9729640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f415e030 0x7ff6f4146ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f8f28640 1 -- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f415e030 msgr2=0x7ff6f4146ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f8f28640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f415e030 0x7ff6f4146ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f8f28640 1 -- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff6f40b2290 msgr2=0x7ff6f415f8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f8f28640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff6f40b2290 0x7ff6f415f8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f8f28640 1 -- 172.21.15.64:0/2660937506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6f40932c0 con 0x7ff6f406ef70 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f9729640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f415e030 0x7ff6f4146ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f8f28640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f406ef70 0x7ff6f40c9440 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7ff6e800d7d0 tx=0x7ff6e800dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f17fa640 1 -- 172.21.15.64:0/2660937506 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6e8004030 con 0x7ff6f406ef70 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f17fa640 1 -- 172.21.15.64:0/2660937506 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff6e80041d0 con 0x7ff6f406ef70 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff6f40935a0 con 0x7ff6f406ef70 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6f17fa640 1 -- 172.21.15.64:0/2660937506 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6e800b6c0 con 0x7ff6f406ef70 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.474+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff6f4093e40 con 0x7ff6f406ef70 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.475+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff6f4093770 con 0x7ff6f406ef70 2024-03-04T07:27:43.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.475+0000 7ff6f17fa640 1 -- 172.21.15.64:0/2660937506 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff6e800b860 con 0x7ff6f406ef70 2024-03-04T07:27:43.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.476+0000 7ff6f17fa640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff6d806b4d0 0x7ff6d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.476+0000 7ff6f17fa640 1 -- 172.21.15.64:0/2660937506 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7ff6e808a200 con 0x7ff6f406ef70 2024-03-04T07:27:43.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.476+0000 7ff6f3fff640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff6d806b4d0 0x7ff6d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.476+0000 7ff6f3fff640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff6d806b4d0 0x7ff6d806d990 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7ff6f40b4d50 tx=0x7ff6e0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.480+0000 7ff6f17fa640 1 -- 172.21.15.64:0/2660937506 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff6e8059340 con 0x7ff6f406ef70 2024-03-04T07:27:43.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.729+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-rbd"} v 0) v1 -- 0x7ff6f4067ff0 con 0x7ff6f406ef70 2024-03-04T07:27:43.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.731+0000 7ff6f17fa640 1 -- 172.21.15.64:0/2660937506 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-rbd"}]=0 v12) v1 ==== 90+0+0 (secure 0 0 0) 0x7ff6e8058c40 con 0x7ff6f406ef70 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff6d806b4d0 msgr2=0x7ff6d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff6d806b4d0 0x7ff6d806d990 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7ff6f40b4d50 tx=0x7ff6e0008040 comp rx=0 tx=0).stop 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f406ef70 msgr2=0x7ff6f40c9440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f406ef70 0x7ff6f40c9440 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7ff6e800d7d0 tx=0x7ff6e800dc90 comp rx=0 tx=0).stop 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 shutdown_connections 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff6d806b4d0 0x7ff6d806d990 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff6f415e030 0x7ff6f4146ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff6f40b2290 0x7ff6f415f8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 --2- 172.21.15.64:0/2660937506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff6f406ef70 0x7ff6f40c9440 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 >> 172.21.15.64:0/2660937506 conn(0x7ff6f40b55e0 msgr2=0x7ff6f40af840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 shutdown_connections 2024-03-04T07:27:43.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.733+0000 7ff6fb18c640 1 -- 172.21.15.64:0/2660937506 wait complete. 2024-03-04T07:27:43.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-rbd-mirror 2024-03-04T07:27:43.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.847+0000 7fe868c88640 1 Processor -- start 2024-03-04T07:27:43.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe868c88640 1 -- start start 2024-03-04T07:27:43.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe868c88640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe8640b0e90 0x7fe86406b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe868c88640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe8640cee90 con 0x7fe86406b5c0 2024-03-04T07:27:43.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe868c88640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe8640cfc50 con 0x7fe8640ae9b0 2024-03-04T07:27:43.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe868c88640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe8640cfa00 con 0x7fe8640b0e90 2024-03-04T07:27:43.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe85bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe8640b0e90 0x7fe86406b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe86259c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe86406b5c0 0x7fe8640b2e00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54598/0 (socket says 172.21.15.64:54598) 2024-03-04T07:27:43.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe86259c640 1 -- 172.21.15.64:0/4044463231 learned_addr learned my addr 172.21.15.64:0/4044463231 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:43.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.848+0000 7fe85bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe8640b0e90 0x7fe86406b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38760/0 (socket says 172.21.15.64:38760) 2024-03-04T07:27:43.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.849+0000 7fe85bfff640 1 -- 172.21.15.64:0/4044463231 >> v1:172.21.15.66:6789/0 conn(0x7fe8640ae9b0 legacy=0x7fe8640d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:43.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.849+0000 7fe861d9b640 1 -- 172.21.15.64:0/4044463231 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1514009841 0 0) 0x7fe8640cee90 con 0x7fe86406b5c0 2024-03-04T07:27:43.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.849+0000 7fe85bfff640 1 -- 172.21.15.64:0/4044463231 >> v1:172.21.15.64:6789/0 conn(0x7fe86406b5c0 legacy=0x7fe8640b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.849+0000 7fe85bfff640 1 -- 172.21.15.64:0/4044463231 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8640cff30 con 0x7fe8640b0e90 2024-03-04T07:27:43.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe85bfff640 1 --2- 172.21.15.64:0/4044463231 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640b0e90 0x7fe86406b9a0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fe84c0097a0 tx=0x7fe84c02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1312bf7a1b671c7b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe861d9b640 1 -- 172.21.15.64:0/4044463231 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe84c0315a0 con 0x7fe8640b0e90 2024-03-04T07:27:43.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe861d9b640 1 -- 172.21.15.64:0/4044463231 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe84c037070 con 0x7fe8640b0e90 2024-03-04T07:27:43.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe861d9b640 1 -- 172.21.15.64:0/4044463231 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe84c031960 con 0x7fe8640b0e90 2024-03-04T07:27:43.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe868c88640 1 -- 172.21.15.64:0/4044463231 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640b0e90 msgr2=0x7fe86406b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe868c88640 1 --2- 172.21.15.64:0/4044463231 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640b0e90 0x7fe86406b9a0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fe84c0097a0 tx=0x7fe84c02b840 comp rx=0 tx=0).stop 2024-03-04T07:27:43.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe868c88640 1 -- 172.21.15.64:0/4044463231 shutdown_connections 2024-03-04T07:27:43.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe868c88640 1 --2- 172.21.15.64:0/4044463231 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640b0e90 0x7fe86406b9a0 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe868c88640 1 -- 172.21.15.64:0/4044463231 >> 172.21.15.64:0/4044463231 conn(0x7fe8640b36d0 msgr2=0x7fe8640b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:43.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe868c88640 1 -- 172.21.15.64:0/4044463231 shutdown_connections 2024-03-04T07:27:43.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.850+0000 7fe868c88640 1 -- 172.21.15.64:0/4044463231 wait complete. 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 Processor -- start 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 -- start start 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe86406b5c0 0x7fe8641f3190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640ae9b0 0x7fe8640b4850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8640b0e90 0x7fe8640b4d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe86406c910 con 0x7fe8640ae9b0 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe86406c790 con 0x7fe86406b5c0 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe868c88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe8640d0ab0 con 0x7fe8640b0e90 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe85bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640ae9b0 0x7fe8640b4850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe85bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640ae9b0 0x7fe8640b4850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38762/0 (socket says 172.21.15.64:38762) 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.852+0000 7fe85bfff640 1 -- 172.21.15.64:0/1429355128 learned_addr learned my addr 172.21.15.64:0/1429355128 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe86259c640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe86406b5c0 0x7fe8641f3190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe862d9d640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8640b0e90 0x7fe8640b4d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85bfff640 1 -- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8640b0e90 msgr2=0x7fe8640b4d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85bfff640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8640b0e90 0x7fe8640b4d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85bfff640 1 -- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe86406b5c0 msgr2=0x7fe8641f3190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85bfff640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe86406b5c0 0x7fe8641f3190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85bfff640 1 -- 172.21.15.64:0/1429355128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8640c7300 con 0x7fe8640ae9b0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe86259c640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe86406b5c0 0x7fe8641f3190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe862d9d640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8640b0e90 0x7fe8640b4d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85bfff640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640ae9b0 0x7fe8640b4850 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7fe84c0097a0 tx=0x7fe84c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85affd640 1 -- 172.21.15.64:0/1429355128 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe84c004230 con 0x7fe8640ae9b0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85affd640 1 -- 172.21.15.64:0/1429355128 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe84c0043f0 con 0x7fe8640ae9b0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe8640c75e0 con 0x7fe8640ae9b0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe85affd640 1 -- 172.21.15.64:0/1429355128 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe84c032880 con 0x7fe8640ae9b0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.853+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8640c7ba0 con 0x7fe8640ae9b0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.854+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe864068700 con 0x7fe8640ae9b0 2024-03-04T07:27:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.854+0000 7fe85affd640 1 -- 172.21.15.64:0/1429355128 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe84c032a20 con 0x7fe8640ae9b0 2024-03-04T07:27:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.855+0000 7fe85affd640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe84406b4d0 0x7fe84406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.855+0000 7fe86259c640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe84406b4d0 0x7fe84406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.855+0000 7fe85affd640 1 -- 172.21.15.64:0/1429355128 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fe84c02f030 con 0x7fe8640ae9b0 2024-03-04T07:27:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.855+0000 7fe86259c640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe84406b4d0 0x7fe84406d990 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fe854000f60 tx=0x7fe854009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:43.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:43.860+0000 7fe85affd640 1 -- 172.21.15.64:0/1429355128 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe84c041070 con 0x7fe8640ae9b0 2024-03-04T07:27:44.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.109+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-rbd-mirror"} v 0) v1 -- 0x7fe86406daf0 con 0x7fe8640ae9b0 2024-03-04T07:27:44.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.114+0000 7fe85affd640 1 -- 172.21.15.64:0/1429355128 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-rbd-mirror"}]=0 v13) v1 ==== 97+0+0 (secure 0 0 0) 0x7fe84c07b0e0 con 0x7fe8640ae9b0 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe84406b4d0 msgr2=0x7fe84406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe84406b4d0 0x7fe84406d990 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fe854000f60 tx=0x7fe854009040 comp rx=0 tx=0).stop 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640ae9b0 msgr2=0x7fe8640b4850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640ae9b0 0x7fe8640b4850 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7fe84c0097a0 tx=0x7fe84c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 shutdown_connections 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe84406b4d0 0x7fe84406d990 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8640b0e90 0x7fe8640b4d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe8640ae9b0 0x7fe8640b4850 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 --2- 172.21.15.64:0/1429355128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe86406b5c0 0x7fe8641f3190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 >> 172.21.15.64:0/1429355128 conn(0x7fe8640b36d0 msgr2=0x7fe8640b3ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 shutdown_connections 2024-03-04T07:27:44.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.116+0000 7fe868c88640 1 -- 172.21.15.64:0/1429355128 wait complete. 2024-03-04T07:27:44.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth del client.bootstrap-rgw 2024-03-04T07:27:44.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.229+0000 7f5494e46640 1 Processor -- start 2024-03-04T07:27:44.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.229+0000 7f5494e46640 1 -- start start 2024-03-04T07:27:44.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f5494e46640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f549015ac30 0x7f54900b3940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f5494e46640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f54900b3e80 con 0x7f54900720c0 2024-03-04T07:27:44.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f5494e46640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f54900b4120 con 0x7f54900afb30 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f5494e46640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f54900b4400 con 0x7f549015ac30 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f548e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f549015ac30 0x7f54900b3940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f548ed9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f54900720c0 0x7f54900cdf90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54604/0 (socket says 172.21.15.64:54604) 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f548e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f549015ac30 0x7f54900b3940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38766/0 (socket says 172.21.15.64:38766) 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.230+0000 7f548ed9d640 1 -- 172.21.15.64:0/2837635259 learned_addr learned my addr 172.21.15.64:0/2837635259 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.231+0000 7f548e59c640 1 -- 172.21.15.64:0/2837635259 >> v1:172.21.15.66:6789/0 conn(0x7f54900afb30 legacy=0x7f54900719b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.231+0000 7f548e59c640 1 -- 172.21.15.64:0/2837635259 >> v1:172.21.15.64:6789/0 conn(0x7f54900720c0 legacy=0x7f54900cdf90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.231+0000 7f548e59c640 1 -- 172.21.15.64:0/2837635259 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54900b46e0 con 0x7f549015ac30 2024-03-04T07:27:44.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.231+0000 7f548d59a640 1 -- 172.21.15.64:0/2837635259 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1016597517 0 0) 0x7f54900b3e80 con 0x7f54900720c0 2024-03-04T07:27:44.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f548e59c640 1 --2- 172.21.15.64:0/2837635259 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f549015ac30 0x7f54900b3940 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f54880097a0 tx=0x7f548802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80769487d04a94fb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f548d59a640 1 -- 172.21.15.64:0/2837635259 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5488034070 con 0x7f549015ac30 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f548d59a640 1 -- 172.21.15.64:0/2837635259 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5488030610 con 0x7f549015ac30 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f548d59a640 1 -- 172.21.15.64:0/2837635259 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54880309e0 con 0x7f549015ac30 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f5494e46640 1 -- 172.21.15.64:0/2837635259 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f549015ac30 msgr2=0x7f54900b3940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f5494e46640 1 --2- 172.21.15.64:0/2837635259 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f549015ac30 0x7f54900b3940 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f54880097a0 tx=0x7f548802b960 comp rx=0 tx=0).stop 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f5494e46640 1 -- 172.21.15.64:0/2837635259 shutdown_connections 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f5494e46640 1 --2- 172.21.15.64:0/2837635259 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f549015ac30 0x7f54900b3940 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f5494e46640 1 -- 172.21.15.64:0/2837635259 >> 172.21.15.64:0/2837635259 conn(0x7f54900d0990 msgr2=0x7f54900d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f5494e46640 1 -- 172.21.15.64:0/2837635259 shutdown_connections 2024-03-04T07:27:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.232+0000 7f5494e46640 1 -- 172.21.15.64:0/2837635259 wait complete. 2024-03-04T07:27:44.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 Processor -- start 2024-03-04T07:27:44.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 -- start start 2024-03-04T07:27:44.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54900720c0 0x7f54901f2780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54900afb30 0x7f5490153550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f549015ac30 0x7f54900aac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54900ce6a0 con 0x7f54900afb30 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54900b4ca0 con 0x7f549015ac30 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f5494e46640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f54900ce820 con 0x7f54900720c0 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f548dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54900afb30 0x7f5490153550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f548dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54900afb30 0x7f5490153550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38774/0 (socket says 172.21.15.64:38774) 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.234+0000 7f548dd9b640 1 -- 172.21.15.64:0/3896827547 learned_addr learned my addr 172.21.15.64:0/3896827547 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548ed9d640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f549015ac30 0x7f54900aac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548e59c640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54900720c0 0x7f54901f2780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548dd9b640 1 -- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54900720c0 msgr2=0x7f54901f2780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548dd9b640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54900720c0 0x7f54901f2780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548dd9b640 1 -- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f549015ac30 msgr2=0x7f54900aac10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548dd9b640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f549015ac30 0x7f54900aac10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548dd9b640 1 -- 172.21.15.64:0/3896827547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5490153a90 con 0x7f54900afb30 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f548dd9b640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54900afb30 0x7f5490153550 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f547800c450 tx=0x7f547800c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f5486ffd640 1 -- 172.21.15.64:0/3896827547 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54780052d0 con 0x7f54900afb30 2024-03-04T07:27:44.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f5486ffd640 1 -- 172.21.15.64:0/3896827547 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5478005470 con 0x7f54900afb30 2024-03-04T07:27:44.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5490156fb0 con 0x7f54900afb30 2024-03-04T07:27:44.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f5486ffd640 1 -- 172.21.15.64:0/3896827547 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5478012870 con 0x7f54900afb30 2024-03-04T07:27:44.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.235+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5490154010 con 0x7f54900afb30 2024-03-04T07:27:44.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.236+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5490068700 con 0x7f54900afb30 2024-03-04T07:27:44.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.237+0000 7f5486ffd640 1 -- 172.21.15.64:0/3896827547 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5478012a10 con 0x7f54900afb30 2024-03-04T07:27:44.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.237+0000 7f5486ffd640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f547006b4d0 0x7f547006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.237+0000 7f548e59c640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f547006b4d0 0x7f547006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.237+0000 7f5486ffd640 1 -- 172.21.15.64:0/3896827547 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f5478089af0 con 0x7f54900afb30 2024-03-04T07:27:44.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.238+0000 7f548e59c640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f547006b4d0 0x7f547006d990 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f5488036000 tx=0x7f5488008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.241+0000 7f5486ffd640 1 -- 172.21.15.64:0/3896827547 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5490153a90 con 0x7f54900afb30 2024-03-04T07:27:44.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.496+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth del", "entity": "client.bootstrap-rgw"} v 0) v1 -- 0x7f549015a170 con 0x7f54900afb30 2024-03-04T07:27:44.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.501+0000 7f5486ffd640 1 -- 172.21.15.64:0/3896827547 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth del", "entity": "client.bootstrap-rgw"}]=0 v14) v1 ==== 90+0+0 (secure 0 0 0) 0x7f5478058cc0 con 0x7f54900afb30 2024-03-04T07:27:44.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f547006b4d0 msgr2=0x7f547006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f547006b4d0 0x7f547006d990 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f5488036000 tx=0x7f5488008030 comp rx=0 tx=0).stop 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54900afb30 msgr2=0x7f5490153550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54900afb30 0x7f5490153550 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f547800c450 tx=0x7f547800c910 comp rx=0 tx=0).stop 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 shutdown_connections 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f547006b4d0 0x7f547006d990 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f549015ac30 0x7f54900aac10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54900afb30 0x7f5490153550 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 --2- 172.21.15.64:0/3896827547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54900720c0 0x7f54901f2780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 >> 172.21.15.64:0/3896827547 conn(0x7f54900d0990 msgr2=0x7f54900cf470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 shutdown_connections 2024-03-04T07:27:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.503+0000 7f5494e46640 1 -- 172.21.15.64:0/3896827547 wait complete. 2024-03-04T07:27:44.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:44.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.615+0000 7f511990f640 1 Processor -- start 2024-03-04T07:27:44.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.615+0000 7f511990f640 1 -- start start 2024-03-04T07:27:44.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f511990f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f511415df70 0x7f51140d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f511990f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5114071c50 0x7f51140b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f511990f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f51140b6f20 con 0x7f51140d18c0 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f511990f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f51140d3030 con 0x7f5114071c50 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f511990f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f51140d3310 con 0x7f511415df70 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f51137fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5114071c50 0x7f51140b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f51137fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5114071c50 0x7f51140b2700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38786/0 (socket says 172.21.15.64:38786) 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f51137fe640 1 -- 172.21.15.64:0/3708087952 learned_addr learned my addr 172.21.15.64:0/3708087952 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f51137fe640 1 -- 172.21.15.64:0/3708087952 >> v2:172.21.15.66:3300/0 conn(0x7f511415df70 msgr2=0x7f51140d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f5112ffd640 1 --2- 172.21.15.64:0/3708087952 >> v2:172.21.15.66:3300/0 conn(0x7f511415df70 0x7f51140d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f51137fe640 1 --2- 172.21.15.64:0/3708087952 >> v2:172.21.15.66:3300/0 conn(0x7f511415df70 0x7f51140d27b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f51137fe640 1 -- 172.21.15.64:0/3708087952 >> v1:172.21.15.66:6789/0 conn(0x7f51140d18c0 legacy=0x7f5114071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:44.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.616+0000 7f51137fe640 1 -- 172.21.15.64:0/3708087952 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51140d35f0 con 0x7f5114071c50 2024-03-04T07:27:44.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.617+0000 7f51137fe640 1 --2- 172.21.15.64:0/3708087952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51140b2700 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f511404f3a0 tx=0x7f510402bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd3d4cd7520df038 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.617+0000 7f5111ffb640 1 -- 172.21.15.64:0/3708087952 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f510402cb20 con 0x7f5114071c50 2024-03-04T07:27:44.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.617+0000 7f5111ffb640 1 -- 172.21.15.64:0/3708087952 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f510402ccc0 con 0x7f5114071c50 2024-03-04T07:27:44.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.617+0000 7f5111ffb640 1 -- 172.21.15.64:0/3708087952 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5104034610 con 0x7f5114071c50 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 -- 172.21.15.64:0/3708087952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 msgr2=0x7f51140b2700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 --2- 172.21.15.64:0/3708087952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51140b2700 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f511404f3a0 tx=0x7f510402bf80 comp rx=0 tx=0).stop 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 -- 172.21.15.64:0/3708087952 shutdown_connections 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 --2- 172.21.15.64:0/3708087952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51140b2700 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 --2- 172.21.15.64:0/3708087952 >> v2:172.21.15.66:3300/0 conn(0x7f511415df70 0x7f51140d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 -- 172.21.15.64:0/3708087952 >> 172.21.15.64:0/3708087952 conn(0x7f51140d3cf0 msgr2=0x7f51140d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 -- 172.21.15.64:0/3708087952 shutdown_connections 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.618+0000 7f511990f640 1 -- 172.21.15.64:0/3708087952 wait complete. 2024-03-04T07:27:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.619+0000 7f511990f640 1 Processor -- start 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.619+0000 7f511990f640 1 -- start start 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f511990f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51141f3700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f511990f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51140d18c0 0x7f51141f6690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f511990f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f511415df70 0x7f51140cbcb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f511990f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f511415e350 con 0x7f5114071c50 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f511990f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51140d2cf0 con 0x7f511415df70 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f511990f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f511415e4d0 con 0x7f51140d18c0 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51141f3700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51141f3700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38798/0 (socket says 172.21.15.64:38798) 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 -- 172.21.15.64:0/878844053 learned_addr learned my addr 172.21.15.64:0/878844053 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f51137fe640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f511415df70 0x7f51140cbcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f51127fc640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51140d18c0 0x7f51141f6690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 -- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51140d18c0 msgr2=0x7f51141f6690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51140d18c0 0x7f51141f6690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 -- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f511415df70 msgr2=0x7f51140cbcb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f511415df70 0x7f51140cbcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.620+0000 7f5112ffd640 1 -- 172.21.15.64:0/878844053 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51140cc530 con 0x7f5114071c50 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.621+0000 7f5112ffd640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51141f3700 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f510c00c820 tx=0x7f510c00cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.621+0000 7f510bfff640 1 -- 172.21.15.64:0/878844053 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f510c015070 con 0x7f5114071c50 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.621+0000 7f510bfff640 1 -- 172.21.15.64:0/878844053 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f510c004020 con 0x7f5114071c50 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.621+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f51140c8a60 con 0x7f5114071c50 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.621+0000 7f510bfff640 1 -- 172.21.15.64:0/878844053 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f510c0043f0 con 0x7f5114071c50 2024-03-04T07:27:44.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.621+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f51140cc1f0 con 0x7f5114071c50 2024-03-04T07:27:44.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.622+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5114067ff0 con 0x7f5114071c50 2024-03-04T07:27:44.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.622+0000 7f510bfff640 1 -- 172.21.15.64:0/878844053 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f510c007830 con 0x7f5114071c50 2024-03-04T07:27:44.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.622+0000 7f510bfff640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f50f406b500 0x7f50f406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.623+0000 7f510bfff640 1 -- 172.21.15.64:0/878844053 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f510c089c60 con 0x7f5114071c50 2024-03-04T07:27:44.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.623+0000 7f51127fc640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f50f406b500 0x7f50f406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.623+0000 7f51127fc640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f50f406b500 0x7f50f406d9c0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f51140cd040 tx=0x7f50fc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.628+0000 7f510bfff640 1 -- 172.21.15.64:0/878844053 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f510c058e40 con 0x7f5114071c50 2024-03-04T07:27:44.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.810+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f51140c8810 con 0x7f5114071c50 2024-03-04T07:27:44.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.811+0000 7f510bfff640 1 -- 172.21.15.64:0/878844053 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v29) v1 ==== 74+0+12843 (secure 0 0 0) 0x7f510c058740 con 0x7f5114071c50 2024-03-04T07:27:44.810 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:44.811 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:41.834755+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":21,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f50f406b500 msgr2=0x7f50f406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f50f406b500 0x7f50f406d9c0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f51140cd040 tx=0x7f50fc009040 comp rx=0 tx=0).stop 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 msgr2=0x7f51141f3700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51141f3700 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f510c00c820 tx=0x7f510c00cce0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 shutdown_connections 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f50f406b500 0x7f50f406d9c0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f511415df70 0x7f51140cbcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51140d18c0 0x7f51141f6690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 --2- 172.21.15.64:0/878844053 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5114071c50 0x7f51141f3700 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 >> 172.21.15.64:0/878844053 conn(0x7f51140d3cf0 msgr2=0x7f51140b3750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 shutdown_connections 2024-03-04T07:27:44.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.813+0000 7f511990f640 1 -- 172.21.15.64:0/878844053 wait complete. 2024-03-04T07:27:44.823 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:27:44.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:27:44.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.928+0000 7fb4a4f0c640 1 Processor -- start 2024-03-04T07:27:44.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.928+0000 7fb4a4f0c640 1 -- start start 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb4a4f0c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4a015ae50 0x7fb4a00708e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb4a4f0c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a0150a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb4a4f0c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb4a0070e20 con 0x7fb4a015b230 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb4a4f0c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4a00b2e30 con 0x7fb4a015ae50 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb4a4f0c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4a006dfe0 con 0x7fb4a0071870 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb49e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4a015ae50 0x7fb4a00708e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb49e59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4a015ae50 0x7fb4a00708e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38812/0 (socket says 172.21.15.64:38812) 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb49e59c640 1 -- 172.21.15.64:0/3382831824 learned_addr learned my addr 172.21.15.64:0/3382831824 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:44.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb49e59c640 1 -- 172.21.15.64:0/3382831824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 msgr2=0x7fb4a0150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:44.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb49e59c640 1 --2- 172.21.15.64:0/3382831824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a0150a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb49e59c640 1 -- 172.21.15.64:0/3382831824 >> v1:172.21.15.66:6789/0 conn(0x7fb4a015b230 legacy=0x7fb4a0071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:44.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.929+0000 7fb49e59c640 1 -- 172.21.15.64:0/3382831824 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4a00b30b0 con 0x7fb4a015ae50 2024-03-04T07:27:44.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.930+0000 7fb49e59c640 1 --2- 172.21.15.64:0/3382831824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015ae50 0x7fb4a00708e0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7fb49000aa10 tx=0x7fb49002de40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=67d9270a23c2e54 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.930+0000 7fb49d59a640 1 -- 172.21.15.64:0/3382831824 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb49003b070 con 0x7fb4a015ae50 2024-03-04T07:27:44.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.930+0000 7fb49d59a640 1 -- 172.21.15.64:0/3382831824 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb49002ebf0 con 0x7fb4a015ae50 2024-03-04T07:27:44.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.930+0000 7fb49d59a640 1 -- 172.21.15.64:0/3382831824 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb49000b040 con 0x7fb4a015ae50 2024-03-04T07:27:44.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/3382831824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015ae50 msgr2=0x7fb4a00708e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/3382831824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015ae50 0x7fb4a00708e0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7fb49000aa10 tx=0x7fb49002de40 comp rx=0 tx=0).stop 2024-03-04T07:27:44.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/3382831824 shutdown_connections 2024-03-04T07:27:44.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/3382831824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a0150a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/3382831824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015ae50 0x7fb4a00708e0 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/3382831824 >> 172.21.15.64:0/3382831824 conn(0x7fb4a0066d00 msgr2=0x7fb4a0067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:44.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/3382831824 shutdown_connections 2024-03-04T07:27:44.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.931+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/3382831824 wait complete. 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.932+0000 7fb4a4f0c640 1 Processor -- start 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.932+0000 7fb4a4f0c640 1 -- start start 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb4a4f0c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a013c880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb4a4f0c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4a015ae50 0x7fb4a01593b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb4a4f0c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015b230 0x7fb4a01598f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb4a4f0c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4a00b3950 con 0x7fb4a015b230 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb4a4f0c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4a00b3670 con 0x7fb4a015ae50 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb4a4f0c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4a00b4150 con 0x7fb4a0071870 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015b230 0x7fb4a01598f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015b230 0x7fb4a01598f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38824/0 (socket says 172.21.15.64:38824) 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 -- 172.21.15.64:0/1660174050 learned_addr learned my addr 172.21.15.64:0/1660174050 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:44.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49e59c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a013c880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49dd9b640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4a015ae50 0x7fb4a01593b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 -- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 msgr2=0x7fb4a013c880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a013c880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 -- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4a015ae50 msgr2=0x7fb4a01593b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4a015ae50 0x7fb4a01593b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 -- 172.21.15.64:0/1660174050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4a013cdc0 con 0x7fb4a015b230 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49e59c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a013c880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49dd9b640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4a015ae50 0x7fb4a01593b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb49ed9d640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015b230 0x7fb4a01598f0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fb49800b9a0 tx=0x7fb49800be60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb48effd640 1 -- 172.21.15.64:0/1660174050 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb498019070 con 0x7fb4a015b230 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.933+0000 7fb48effd640 1 -- 172.21.15.64:0/1660174050 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb49800cbc0 con 0x7fb4a015b230 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.934+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb4a014fc10 con 0x7fb4a015b230 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.934+0000 7fb48effd640 1 -- 172.21.15.64:0/1660174050 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb498014440 con 0x7fb4a015b230 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.934+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb4a01501d0 con 0x7fb4a015b230 2024-03-04T07:27:44.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.934+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb4a00733c0 con 0x7fb4a015b230 2024-03-04T07:27:44.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.935+0000 7fb48effd640 1 -- 172.21.15.64:0/1660174050 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb49801c070 con 0x7fb4a015b230 2024-03-04T07:27:44.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.935+0000 7fb48effd640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb48006b500 0x7fb48006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:44.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.935+0000 7fb48effd640 1 -- 172.21.15.64:0/1660174050 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fb498089af0 con 0x7fb4a015b230 2024-03-04T07:27:44.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.935+0000 7fb49e59c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb48006b500 0x7fb48006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:44.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.936+0000 7fb49e59c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb48006b500 0x7fb48006d9c0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fb490038000 tx=0x7fb490032000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:44.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:44.940+0000 7fb48effd640 1 -- 172.21.15.64:0/1660174050 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb498058d50 con 0x7fb4a015b230 2024-03-04T07:27:45.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.119+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fb4a004f410 con 0x7fb4a015b230 2024-03-04T07:27:45.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.120+0000 7fb48effd640 1 -- 172.21.15.64:0/1660174050 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v29) v1 ==== 93+0+10 (secure 0 0 0) 0x7fb498058650 con 0x7fb4a015b230 2024-03-04T07:27:45.119 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:27:45.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb48006b500 msgr2=0x7fb48006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb48006b500 0x7fb48006d9c0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fb490038000 tx=0x7fb490032000 comp rx=0 tx=0).stop 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015b230 msgr2=0x7fb4a01598f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015b230 0x7fb4a01598f0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fb49800b9a0 tx=0x7fb49800be60 comp rx=0 tx=0).stop 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 shutdown_connections 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb48006b500 0x7fb48006d9c0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4a015b230 0x7fb4a01598f0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4a015ae50 0x7fb4a01593b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 --2- 172.21.15.64:0/1660174050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4a0071870 0x7fb4a013c880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 >> 172.21.15.64:0/1660174050 conn(0x7fb4a0066d00 msgr2=0x7fb4a00b4400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 shutdown_connections 2024-03-04T07:27:45.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.122+0000 7fb4a4f0c640 1 -- 172.21.15.64:0/1660174050 wait complete. 2024-03-04T07:27:45.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:27:45.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.235+0000 7f0a1f7f6640 1 Processor -- start 2024-03-04T07:27:45.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.235+0000 7f0a1f7f6640 1 -- start start 2024-03-04T07:27:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.235+0000 7f0a1f7f6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0a180b1020 0x7f0a1806af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1f7f6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0a1806ed10 0x7f0a180cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1f7f6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0a180cc0e0 con 0x7f0a180cd6c0 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1f7f6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a180d28b0 con 0x7f0a1806ed10 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1f7f6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a180cf320 con 0x7f0a180b1020 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1cd91640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0a1806ed10 0x7f0a180cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1cd91640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0a1806ed10 0x7f0a180cd180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38832/0 (socket says 172.21.15.64:38832) 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1cd91640 1 -- 172.21.15.64:0/2666134178 learned_addr learned my addr 172.21.15.64:0/2666134178 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1cd91640 1 -- 172.21.15.64:0/2666134178 >> v2:172.21.15.66:3300/0 conn(0x7f0a180b1020 msgr2=0x7f0a1806af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1cd91640 1 --2- 172.21.15.64:0/2666134178 >> v2:172.21.15.66:3300/0 conn(0x7f0a180b1020 0x7f0a1806af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1cd91640 1 -- 172.21.15.64:0/2666134178 >> v1:172.21.15.66:6789/0 conn(0x7f0a180cd6c0 legacy=0x7f0a1815d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:45.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.236+0000 7f0a1cd91640 1 -- 172.21.15.64:0/2666134178 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a180cc3c0 con 0x7f0a1806ed10 2024-03-04T07:27:45.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a1cd91640 1 --2- 172.21.15.64:0/2666134178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a1806ed10 0x7f0a180cd180 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f0a18070d50 tx=0x7f0a0802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a0040eda54c88a57 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:45.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a0ffff640 1 -- 172.21.15.64:0/2666134178 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a08035020 con 0x7f0a1806ed10 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a0ffff640 1 -- 172.21.15.64:0/2666134178 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a08031870 con 0x7f0a1806ed10 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a0ffff640 1 -- 172.21.15.64:0/2666134178 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a08031c40 con 0x7f0a1806ed10 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/2666134178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a1806ed10 msgr2=0x7f0a180cd180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/2666134178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a1806ed10 0x7f0a180cd180 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f0a18070d50 tx=0x7f0a0802b910 comp rx=0 tx=0).stop 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/2666134178 shutdown_connections 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/2666134178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a1806ed10 0x7f0a180cd180 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/2666134178 >> v2:172.21.15.66:3300/0 conn(0x7f0a180b1020 0x7f0a1806af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.237+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/2666134178 >> 172.21.15.64:0/2666134178 conn(0x7f0a180d1a90 msgr2=0x7f0a180d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.238+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/2666134178 shutdown_connections 2024-03-04T07:27:45.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.238+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/2666134178 wait complete. 2024-03-04T07:27:45.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 Processor -- start 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 -- start start 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a1806ed10 0x7f0a180b2060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a180b1020 0x7f0a1815ddc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a180cd6c0 0x7f0a180b30c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a180ccc60 con 0x7f0a180cd6c0 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a180cc980 con 0x7f0a180b1020 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.239+0000 7f0a1f7f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0a180cdad0 con 0x7f0a1806ed10 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a180cd6c0 0x7f0a180b30c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a180cd6c0 0x7f0a180b30c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38840/0 (socket says 172.21.15.64:38840) 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 -- 172.21.15.64:0/534294444 learned_addr learned my addr 172.21.15.64:0/534294444 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1cd91640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a180b1020 0x7f0a1815ddc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1d592640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a1806ed10 0x7f0a180b2060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 -- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a1806ed10 msgr2=0x7f0a180b2060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a1806ed10 0x7f0a180b2060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 -- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a180b1020 msgr2=0x7f0a1815ddc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a180b1020 0x7f0a1815ddc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 -- 172.21.15.64:0/534294444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a180cbab0 con 0x7f0a180cd6c0 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1d592640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a1806ed10 0x7f0a180b2060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1dd93640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a180cd6c0 0x7f0a180b30c0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f0a1000b990 tx=0x7f0a1000be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a0dffb640 1 -- 172.21.15.64:0/534294444 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a10019070 con 0x7f0a180cd6c0 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a0dffb640 1 -- 172.21.15.64:0/534294444 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a1000cba0 con 0x7f0a180cd6c0 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a0dffb640 1 -- 172.21.15.64:0/534294444 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a10014440 con 0x7f0a180cd6c0 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0a180b3c30 con 0x7f0a180cd6c0 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.240+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a180b1860 con 0x7f0a180cd6c0 2024-03-04T07:27:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.241+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0a18067ff0 con 0x7f0a180cd6c0 2024-03-04T07:27:45.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.242+0000 7f0a0dffb640 1 -- 172.21.15.64:0/534294444 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0a1001c070 con 0x7f0a180cd6c0 2024-03-04T07:27:45.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.242+0000 7f0a0dffb640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09fc06b4d0 0x7f09fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.242+0000 7f0a0dffb640 1 -- 172.21.15.64:0/534294444 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f0a10089c10 con 0x7f0a180cd6c0 2024-03-04T07:27:45.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.242+0000 7f0a1d592640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09fc06b4d0 0x7f09fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.243+0000 7f0a1d592640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09fc06b4d0 0x7f09fc06d990 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f0a14009930 tx=0x7f0a14008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:45.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.246+0000 7f0a0dffb640 1 -- 172.21.15.64:0/534294444 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0a10058700 con 0x7f0a180cd6c0 2024-03-04T07:27:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.429+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f0a18068700 con 0x7f0a180cd6c0 2024-03-04T07:27:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.429+0000 7f0a0dffb640 1 -- 172.21.15.64:0/534294444 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v29) v1 ==== 84+0+65 (secure 0 0 0) 0x7f0a1005c7a0 con 0x7f0a180cd6c0 2024-03-04T07:27:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:45.429 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:27:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:27:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:27:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:27:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:45.430 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:27:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09fc06b4d0 msgr2=0x7f09fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09fc06b4d0 0x7f09fc06d990 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f0a14009930 tx=0x7f0a14008040 comp rx=0 tx=0).stop 2024-03-04T07:27:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a180cd6c0 msgr2=0x7f0a180b30c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a180cd6c0 0x7f0a180b30c0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f0a1000b990 tx=0x7f0a1000be50 comp rx=0 tx=0).stop 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 shutdown_connections 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09fc06b4d0 0x7f09fc06d990 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a180cd6c0 0x7f0a180b30c0 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a180b1020 0x7f0a1815ddc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 --2- 172.21.15.64:0/534294444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a1806ed10 0x7f0a180b2060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 >> 172.21.15.64:0/534294444 conn(0x7f0a180d1a90 msgr2=0x7f0a180d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 shutdown_connections 2024-03-04T07:27:45.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.431+0000 7f0a1f7f6640 1 -- 172.21.15.64:0/534294444 wait complete. 2024-03-04T07:27:45.441 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:27:45.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:27:45.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.547+0000 7f90d3b9a640 1 Processor -- start 2024-03-04T07:27:45.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d3b9a640 1 -- start start 2024-03-04T07:27:45.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d3b9a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f90cc0b2d30 0x7f90cc0b72e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d3b9a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f90cc0d4960 con 0x7f90cc0b7820 2024-03-04T07:27:45.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d3b9a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f90cc0d36c0 con 0x7f90cc071540 2024-03-04T07:27:45.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d3b9a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f90cc0d3230 con 0x7f90cc0b2d30 2024-03-04T07:27:45.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d1135640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f90cc0b2d30 0x7f90cc0b72e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d2137640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f90cc0b7820 0x7f90cc0b7cd0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54608/0 (socket says 172.21.15.64:54608) 2024-03-04T07:27:45.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.548+0000 7f90d2137640 1 -- 172.21.15.64:0/3630072860 learned_addr learned my addr 172.21.15.64:0/3630072860 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:45.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.549+0000 7f90d1135640 1 -- 172.21.15.64:0/3630072860 >> v1:172.21.15.66:6789/0 conn(0x7f90cc071540 legacy=0x7f90cc071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:45.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.549+0000 7f90d1135640 1 -- 172.21.15.64:0/3630072860 >> v1:172.21.15.64:6789/0 conn(0x7f90cc0b7820 legacy=0x7f90cc0b7cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.549+0000 7f90d1135640 1 -- 172.21.15.64:0/3630072860 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90cc0d2620 con 0x7f90cc0b2d30 2024-03-04T07:27:45.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.549+0000 7f90d1135640 1 --2- 172.21.15.64:0/3630072860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc0b2d30 0x7f90cc0b72e0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f90bc0097a0 tx=0x7f90bc02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2fcfcf2b293fa62 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:45.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d0934640 1 -- 172.21.15.64:0/3630072860 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90bc035020 con 0x7f90cc0b2d30 2024-03-04T07:27:45.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d0934640 1 -- 172.21.15.64:0/3630072860 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f90bc031830 con 0x7f90cc0b2d30 2024-03-04T07:27:45.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d0934640 1 -- 172.21.15.64:0/3630072860 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90bc031c00 con 0x7f90cc0b2d30 2024-03-04T07:27:45.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3630072860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc0b2d30 msgr2=0x7f90cc0b72e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3630072860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc0b2d30 0x7f90cc0b72e0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f90bc0097a0 tx=0x7f90bc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3630072860 shutdown_connections 2024-03-04T07:27:45.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3630072860 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc0b2d30 0x7f90cc0b72e0 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3630072860 >> 172.21.15.64:0/3630072860 conn(0x7f90cc0d3d60 msgr2=0x7f90cc0d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:45.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3630072860 shutdown_connections 2024-03-04T07:27:45.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.550+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3630072860 wait complete. 2024-03-04T07:27:45.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.551+0000 7f90d3b9a640 1 Processor -- start 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.551+0000 7f90d3b9a640 1 -- start start 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d3b9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc071540 0x7f90cc159230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d3b9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90cc0b2d30 0x7f90cc1f5870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d3b9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90cc0b7820 0x7f90cc15d020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d3b9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90cc0d2be0 con 0x7f90cc071540 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d3b9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90cc0b83e0 con 0x7f90cc0b2d30 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d3b9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f90cc0d2ec0 con 0x7f90cc0b7820 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d1936640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc071540 0x7f90cc159230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d1936640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc071540 0x7f90cc159230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38854/0 (socket says 172.21.15.64:38854) 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d1936640 1 -- 172.21.15.64:0/3613159471 learned_addr learned my addr 172.21.15.64:0/3613159471 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d1135640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90cc0b2d30 0x7f90cc1f5870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d2137640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90cc0b7820 0x7f90cc15d020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.552+0000 7f90d1936640 1 -- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90cc0b7820 msgr2=0x7f90cc15d020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d1936640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90cc0b7820 0x7f90cc15d020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d1936640 1 -- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90cc0b2d30 msgr2=0x7f90cc1f5870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d1936640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90cc0b2d30 0x7f90cc1f5870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d1936640 1 -- 172.21.15.64:0/3613159471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90cc1f5610 con 0x7f90cc071540 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d1135640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90cc0b2d30 0x7f90cc1f5870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d1936640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc071540 0x7f90cc159230 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f90c800b420 tx=0x7f90c800b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90c27fc640 1 -- 172.21.15.64:0/3613159471 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90c8011830 con 0x7f90cc071540 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90c27fc640 1 -- 172.21.15.64:0/3613159471 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f90c8016070 con 0x7f90cc071540 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90cc0c93b0 con 0x7f90cc071540 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90c27fc640 1 -- 172.21.15.64:0/3613159471 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90c8011c10 con 0x7f90cc071540 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.553+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90cc0c9970 con 0x7f90cc071540 2024-03-04T07:27:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.554+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f90cc0b6980 con 0x7f90cc071540 2024-03-04T07:27:45.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.554+0000 7f90c27fc640 1 -- 172.21.15.64:0/3613159471 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f90c801b460 con 0x7f90cc071540 2024-03-04T07:27:45.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.555+0000 7f90c27fc640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90b006b4d0 0x7f90b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:45.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.555+0000 7f90c27fc640 1 -- 172.21.15.64:0/3613159471 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(29..29 src has 1..29) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f90c8089c40 con 0x7f90cc071540 2024-03-04T07:27:45.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.555+0000 7f90d1135640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90b006b4d0 0x7f90b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:45.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.555+0000 7f90d1135640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90b006b4d0 0x7f90b006d990 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f90bc036000 tx=0x7f90bc008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:45.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.559+0000 7f90c27fc640 1 -- 172.21.15.64:0/3613159471 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f90c8058e10 con 0x7f90cc071540 2024-03-04T07:27:45.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.740+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f90cc072070 con 0x7f90cc071540 2024-03-04T07:27:45.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.981+0000 7f90c27fc640 1 -- 172.21.15.64:0/3613159471 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v30) v1 ==== 138+0+0 (secure 0 0 0) 0x7f90c8058710 con 0x7f90cc071540 2024-03-04T07:27:45.981 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:27:45.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.983+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90b006b4d0 msgr2=0x7f90b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.983+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90b006b4d0 0x7f90b006d990 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f90bc036000 tx=0x7f90bc008030 comp rx=0 tx=0).stop 2024-03-04T07:27:45.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.983+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc071540 msgr2=0x7f90cc159230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.983+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc071540 0x7f90cc159230 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f90c800b420 tx=0x7f90c800b8e0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 shutdown_connections 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90b006b4d0 0x7f90b006d990 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90cc0b7820 0x7f90cc15d020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90cc0b2d30 0x7f90cc1f5870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 --2- 172.21.15.64:0/3613159471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90cc071540 0x7f90cc159230 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 >> 172.21.15.64:0/3613159471 conn(0x7f90cc0d3d60 msgr2=0x7f90cc15c1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 shutdown_connections 2024-03-04T07:27:45.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:45.984+0000 7f90d3b9a640 1 -- 172.21.15.64:0/3613159471 wait complete. 2024-03-04T07:27:45.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:27:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.106+0000 7f0baf89a640 1 Processor -- start 2024-03-04T07:27:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.106+0000 7f0baf89a640 1 -- start start 2024-03-04T07:27:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0baf89a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba815d2c0 0x7f0ba815d9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0baf89a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0ba815fb60 0x7f0ba815ffd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0baf89a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0ba80d0a30 con 0x7f0ba80b4340 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0baf89a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0ba8160510 con 0x7f0ba815fb60 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0baf89a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ba80d01d0 con 0x7f0ba815d2c0 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bad636640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba815d2c0 0x7f0ba815d9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bace35640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f0ba80b4340 0x7f0ba80b2ac0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:35262/0 (socket says 172.21.15.64:35262) 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bade37640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0ba815fb60 0x7f0ba815ffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bace35640 1 -- 172.21.15.64:0/2817109343 learned_addr learned my addr 172.21.15.64:0/2817109343 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bad636640 1 -- 172.21.15.64:0/2817109343 >> v2:172.21.15.66:3300/0 conn(0x7f0ba815fb60 msgr2=0x7f0ba815ffd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bad636640 1 --2- 172.21.15.64:0/2817109343 >> v2:172.21.15.66:3300/0 conn(0x7f0ba815fb60 0x7f0ba815ffd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bad636640 1 -- 172.21.15.64:0/2817109343 >> v1:172.21.15.66:6789/0 conn(0x7f0ba80b4340 legacy=0x7f0ba80b2ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bad636640 1 -- 172.21.15.64:0/2817109343 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ba80cc5c0 con 0x7f0ba815d2c0 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.107+0000 7f0bade37640 1 --2- 172.21.15.64:0/2817109343 >> v2:172.21.15.66:3300/0 conn(0x7f0ba815fb60 0x7f0ba815ffd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.108+0000 7f0bad636640 1 --2- 172.21.15.64:0/2817109343 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba815d2c0 0x7f0ba815d9d0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f0ba40097a0 tx=0x7f0ba402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=13a08962d2b5b567 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.108+0000 7f0b9ffff640 1 -- 172.21.15.64:0/2817109343 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ba4035020 con 0x7f0ba815d2c0 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0b9ffff640 1 -- 172.21.15.64:0/2817109343 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ba4031830 con 0x7f0ba815d2c0 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0b9ffff640 1 -- 172.21.15.64:0/2817109343 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ba4031c00 con 0x7f0ba815d2c0 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 -- 172.21.15.64:0/2817109343 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba815d2c0 msgr2=0x7f0ba815d9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2817109343 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba815d2c0 0x7f0ba815d9d0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f0ba40097a0 tx=0x7f0ba402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 -- 172.21.15.64:0/2817109343 shutdown_connections 2024-03-04T07:27:46.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2817109343 >> v2:172.21.15.66:3300/0 conn(0x7f0ba815fb60 0x7f0ba815ffd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2817109343 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba815d2c0 0x7f0ba815d9d0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 -- 172.21.15.64:0/2817109343 >> 172.21.15.64:0/2817109343 conn(0x7f0ba80b3660 msgr2=0x7f0ba80b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:46.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 -- 172.21.15.64:0/2817109343 shutdown_connections 2024-03-04T07:27:46.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.109+0000 7f0baf89a640 1 -- 172.21.15.64:0/2817109343 wait complete. 2024-03-04T07:27:46.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.110+0000 7f0baf89a640 1 Processor -- start 2024-03-04T07:27:46.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.110+0000 7f0baf89a640 1 -- start start 2024-03-04T07:27:46.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0baf89a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba80b4340 0x7f0ba80c66d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0baf89a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ba815d2c0 0x7f0ba806fdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0baf89a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ba815fb60 0x7f0ba8070310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0baf89a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ba80cc860 con 0x7f0ba815d2c0 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0baf89a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ba80d0750 con 0x7f0ba815fb60 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0baf89a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ba80b31d0 con 0x7f0ba80b4340 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ba815d2c0 0x7f0ba806fdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ba815d2c0 0x7f0ba806fdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38866/0 (socket says 172.21.15.64:38866) 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 -- 172.21.15.64:0/2571995780 learned_addr learned my addr 172.21.15.64:0/2571995780 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bade37640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ba815fb60 0x7f0ba8070310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bad636640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba80b4340 0x7f0ba80c66d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 -- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba80b4340 msgr2=0x7f0ba80c66d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:46.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba80b4340 0x7f0ba80c66d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 -- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ba815fb60 msgr2=0x7f0ba8070310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ba815fb60 0x7f0ba8070310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bace35640 1 -- 172.21.15.64:0/2571995780 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ba80c7190 con 0x7f0ba815d2c0 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.111+0000 7f0bad636640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba80b4340 0x7f0ba80c66d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.112+0000 7f0bade37640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ba815fb60 0x7f0ba8070310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.112+0000 7f0bace35640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ba815d2c0 0x7f0ba806fdd0 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f0b9800c430 tx=0x7f0b9800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.112+0000 7f0b9dffb640 1 -- 172.21.15.64:0/2571995780 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b98017070 con 0x7f0ba815d2c0 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.112+0000 7f0b9dffb640 1 -- 172.21.15.64:0/2571995780 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b98005ce0 con 0x7f0ba815d2c0 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.112+0000 7f0b9dffb640 1 -- 172.21.15.64:0/2571995780 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b980072f0 con 0x7f0ba815d2c0 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.112+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0ba80c7470 con 0x7f0ba815d2c0 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.112+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0ba80c81a0 con 0x7f0ba815d2c0 2024-03-04T07:27:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.113+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0ba809fc50 con 0x7f0ba815d2c0 2024-03-04T07:27:46.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.113+0000 7f0b9dffb640 1 -- 172.21.15.64:0/2571995780 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0b9801a070 con 0x7f0ba815d2c0 2024-03-04T07:27:46.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.114+0000 7f0b9dffb640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b8c06b4d0 0x7f0b8c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:46.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.114+0000 7f0b9dffb640 1 -- 172.21.15.64:0/2571995780 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(30..30 src has 1..30) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f0b98089450 con 0x7f0ba815d2c0 2024-03-04T07:27:46.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.114+0000 7f0bad636640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b8c06b4d0 0x7f0b8c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:46.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.114+0000 7f0bad636640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b8c06b4d0 0x7f0b8c06d990 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f0ba4036000 tx=0x7f0ba4008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:46.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.118+0000 7f0b9dffb640 1 -- 172.21.15.64:0/2571995780 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0b980584a0 con 0x7f0ba815d2c0 2024-03-04T07:27:46.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.299+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f0ba804f410 con 0x7f0ba815d2c0 2024-03-04T07:27:46.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.984+0000 7f0b9dffb640 1 -- 172.21.15.64:0/2571995780 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v31) v1 ==== 170+0+0 (secure 0 0 0) 0x7f0b98057da0 con 0x7f0ba815d2c0 2024-03-04T07:27:46.984 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:27:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b8c06b4d0 msgr2=0x7f0b8c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b8c06b4d0 0x7f0b8c06d990 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f0ba4036000 tx=0x7f0ba4008030 comp rx=0 tx=0).stop 2024-03-04T07:27:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ba815d2c0 msgr2=0x7f0ba806fdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ba815d2c0 0x7f0ba806fdd0 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f0b9800c430 tx=0x7f0b9800c8f0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 shutdown_connections 2024-03-04T07:27:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b8c06b4d0 0x7f0b8c06d990 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ba815fb60 0x7f0ba8070310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ba815d2c0 0x7f0ba806fdd0 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 --2- 172.21.15.64:0/2571995780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ba80b4340 0x7f0ba80c66d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 >> 172.21.15.64:0/2571995780 conn(0x7f0ba80b3660 msgr2=0x7f0ba815c090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 shutdown_connections 2024-03-04T07:27:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:46.987+0000 7f0baf89a640 1 -- 172.21.15.64:0/2571995780 wait complete. 2024-03-04T07:27:47.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:27:47.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.121+0000 7f653c88f640 1 Processor -- start 2024-03-04T07:27:47.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.121+0000 7f653c88f640 1 -- start start 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f653c88f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f65380b3d50 0x7f6538168cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f653c88f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f653806cc20 con 0x7f653815e370 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f653c88f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f65380b4ae0 con 0x7f65381470b0 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f653c88f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6538147490 con 0x7f65380b3d50 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f6536d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f65380b3d50 0x7f6538168cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f6536d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f65380b3d50 0x7f6538168cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38870/0 (socket says 172.21.15.64:38870) 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f652ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f653815e370 0x7f65380b3640 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54620/0 (socket says 172.21.15.64:54620) 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f6536d9d640 1 -- 172.21.15.64:0/3536016885 learned_addr learned my addr 172.21.15.64:0/3536016885 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f6536d9d640 1 -- 172.21.15.64:0/3536016885 >> v1:172.21.15.66:6789/0 conn(0x7f65381470b0 legacy=0x7f6538162340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f6536d9d640 1 -- 172.21.15.64:0/3536016885 >> v1:172.21.15.64:6789/0 conn(0x7f653815e370 legacy=0x7f65380b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:47.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.122+0000 7f6536d9d640 1 -- 172.21.15.64:0/3536016885 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65380b4dc0 con 0x7f65380b3d50 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.123+0000 7f6536d9d640 1 --2- 172.21.15.64:0/3536016885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65380b3d50 0x7f6538168cf0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f6530009e80 tx=0x7f653002bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84c34f7e5224c825 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.123+0000 7f6535d9b640 1 -- 172.21.15.64:0/3536016885 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f653002caf0 con 0x7f65380b3d50 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.123+0000 7f6535d9b640 1 -- 172.21.15.64:0/3536016885 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f653002cc90 con 0x7f65380b3d50 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.123+0000 7f6535d9b640 1 -- 172.21.15.64:0/3536016885 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6530034610 con 0x7f65380b3d50 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.124+0000 7f653c88f640 1 -- 172.21.15.64:0/3536016885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65380b3d50 msgr2=0x7f6538168cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.124+0000 7f653c88f640 1 --2- 172.21.15.64:0/3536016885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65380b3d50 0x7f6538168cf0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f6530009e80 tx=0x7f653002bf50 comp rx=0 tx=0).stop 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.124+0000 7f653c88f640 1 -- 172.21.15.64:0/3536016885 shutdown_connections 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.124+0000 7f653c88f640 1 --2- 172.21.15.64:0/3536016885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65380b3d50 0x7f6538168cf0 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.124+0000 7f653c88f640 1 -- 172.21.15.64:0/3536016885 >> 172.21.15.64:0/3536016885 conn(0x7f65380b6ed0 msgr2=0x7f65380b72e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.124+0000 7f653c88f640 1 -- 172.21.15.64:0/3536016885 shutdown_connections 2024-03-04T07:27:47.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.124+0000 7f653c88f640 1 -- 172.21.15.64:0/3536016885 wait complete. 2024-03-04T07:27:47.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.125+0000 7f653c88f640 1 Processor -- start 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.125+0000 7f653c88f640 1 -- start start 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f653c88f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65380b3d50 0x7f65381f4ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f653c88f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65381470b0 0x7f653815d550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f653c88f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f653815e370 0x7f65380cf840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f653c88f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f653806cf00 con 0x7f653815e370 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f653c88f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65381477f0 con 0x7f65381470b0 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f653c88f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6538147970 con 0x7f65380b3d50 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f653815e370 0x7f65380cf840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f653815e370 0x7f65380cf840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38882/0 (socket says 172.21.15.64:38882) 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 -- 172.21.15.64:0/2933128117 learned_addr learned my addr 172.21.15.64:0/2933128117 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f653659c640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65380b3d50 0x7f65381f4ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f652ffff640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65381470b0 0x7f653815d550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 -- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65380b3d50 msgr2=0x7f65381f4ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:47.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65380b3d50 0x7f65381f4ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 -- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65381470b0 msgr2=0x7f653815d550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65381470b0 0x7f653815d550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 -- 172.21.15.64:0/2933128117 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65380cfd80 con 0x7f653815e370 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.126+0000 7f6536d9d640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f653815e370 0x7f65380cf840 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f6530036040 tx=0x7f6530004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.127+0000 7f652effd640 1 -- 172.21.15.64:0/2933128117 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6530043020 con 0x7f653815e370 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.127+0000 7f652effd640 1 -- 172.21.15.64:0/2933128117 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f653003e450 con 0x7f653815e370 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.127+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65380d0040 con 0x7f653815e370 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.127+0000 7f652effd640 1 -- 172.21.15.64:0/2933128117 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6530009510 con 0x7f653815e370 2024-03-04T07:27:47.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.127+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65380d0600 con 0x7f653815e370 2024-03-04T07:27:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.128+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f653806ffc0 con 0x7f653815e370 2024-03-04T07:27:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.128+0000 7f652effd640 1 -- 172.21.15.64:0/2933128117 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6530046020 con 0x7f653815e370 2024-03-04T07:27:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.129+0000 7f652effd640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f651806b4d0 0x7f651806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.129+0000 7f652effd640 1 -- 172.21.15.64:0/2933128117 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(31..31 src has 1..31) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f65300acd80 con 0x7f653815e370 2024-03-04T07:27:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.129+0000 7f653659c640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f651806b4d0 0x7f651806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.129+0000 7f653659c640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f651806b4d0 0x7f651806d990 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f6528001640 tx=0x7f6528009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:47.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.133+0000 7f652effd640 1 -- 172.21.15.64:0/2933128117 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f653004d020 con 0x7f653815e370 2024-03-04T07:27:47.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:27:47.336 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:27:47.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.354+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f6538161f90 con 0x7f653815e370 2024-03-04T07:27:47.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:47.999+0000 7f652effd640 1 -- 172.21.15.64:0/2933128117 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 4 and data pool 5 v12) v1 ==== 172+0+0 (secure 0 0 0) 0x7f653007bc60 con 0x7f653815e370 2024-03-04T07:27:47.999 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 4 and data pool 5 2024-03-04T07:27:48.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.011+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f651806b4d0 msgr2=0x7f651806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.011+0000 7f653c88f640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f651806b4d0 0x7f651806d990 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f6528001640 tx=0x7f6528009040 comp rx=0 tx=0).stop 2024-03-04T07:27:48.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.011+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f653815e370 msgr2=0x7f65380cf840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.011+0000 7f653c88f640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f653815e370 0x7f65380cf840 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f6530036040 tx=0x7f6530004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:48.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.014+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 shutdown_connections 2024-03-04T07:27:48.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.014+0000 7f653c88f640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f651806b4d0 0x7f651806d990 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.014+0000 7f653c88f640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f653815e370 0x7f65380cf840 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.014+0000 7f653c88f640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65381470b0 0x7f653815d550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.014+0000 7f653c88f640 1 --2- 172.21.15.64:0/2933128117 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65380b3d50 0x7f65381f4ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.014+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 >> 172.21.15.64:0/2933128117 conn(0x7f65380b6ed0 msgr2=0x7f6538162320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:48.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.020+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 shutdown_connections 2024-03-04T07:27:48.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.021+0000 7f653c88f640 1 -- 172.21.15.64:0/2933128117 wait complete. 2024-03-04T07:27:48.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:48.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.147+0000 7f42c1639640 1 Processor -- start 2024-03-04T07:27:48.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.147+0000 7f42c1639640 1 -- start start 2024-03-04T07:27:48.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.148+0000 7f42c1639640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f42bc0b0e90 0x7f42bc06b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.148+0000 7f42c1639640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0ae9b0 0x7f42bc0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.148+0000 7f42c1639640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f42bc0cee90 con 0x7f42bc06b5c0 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.148+0000 7f42c1639640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f42bc0cfa00 con 0x7f42bc0b0e90 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.148+0000 7f42c1639640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f42bc0cfc50 con 0x7f42bc0ae9b0 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.148+0000 7f42baffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f42bc06b5c0 0x7f42bc0b2e00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54628/0 (socket says 172.21.15.64:54628) 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.148+0000 7f42baffd640 1 -- 172.21.15.64:0/2180987487 learned_addr learned my addr 172.21.15.64:0/2180987487 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42ba7fc640 1 --2- 172.21.15.64:0/2180987487 >> v2:172.21.15.66:3300/0 conn(0x7f42bc0b0e90 0x7f42bc06b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42bb7fe640 1 --2- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0ae9b0 0x7f42bc0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42b9ffb640 1 -- 172.21.15.64:0/2180987487 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 746692016 0 0) 0x7f42bc0cee90 con 0x7f42bc06b5c0 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42b9ffb640 1 -- 172.21.15.64:0/2180987487 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f42a8003520 con 0x7f42bc06b5c0 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42ba7fc640 1 -- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0ae9b0 msgr2=0x7f42bc0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42ba7fc640 1 --2- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0ae9b0 0x7f42bc0d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42ba7fc640 1 -- 172.21.15.64:0/2180987487 >> v1:172.21.15.64:6789/0 conn(0x7f42bc06b5c0 legacy=0x7f42bc0b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42ba7fc640 1 -- 172.21.15.64:0/2180987487 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42bc0cff30 con 0x7f42bc0b0e90 2024-03-04T07:27:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.149+0000 7f42bb7fe640 1 --2- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0ae9b0 0x7f42bc0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:27:48.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42ba7fc640 1 --2- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0b0e90 0x7f42bc06b9a0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f42a40097a0 tx=0x7f42a402b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b535b9ec5fae8aee server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42b9ffb640 1 -- 172.21.15.64:0/2180987487 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42a4031780 con 0x7f42bc0b0e90 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42b9ffb640 1 -- 172.21.15.64:0/2180987487 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f42a4037070 con 0x7f42bc0b0e90 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42b9ffb640 1 -- 172.21.15.64:0/2180987487 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42a4031b20 con 0x7f42bc0b0e90 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 -- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0b0e90 msgr2=0x7f42bc06b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 --2- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0b0e90 0x7f42bc06b9a0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f42a40097a0 tx=0x7f42a402b820 comp rx=0 tx=0).stop 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 -- 172.21.15.64:0/2180987487 shutdown_connections 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 --2- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0ae9b0 0x7f42bc0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 --2- 172.21.15.64:0/2180987487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0b0e90 0x7f42bc06b9a0 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 -- 172.21.15.64:0/2180987487 >> 172.21.15.64:0/2180987487 conn(0x7f42bc0b36d0 msgr2=0x7f42bc0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 -- 172.21.15.64:0/2180987487 shutdown_connections 2024-03-04T07:27:48.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.150+0000 7f42c1639640 1 -- 172.21.15.64:0/2180987487 wait complete. 2024-03-04T07:27:48.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 Processor -- start 2024-03-04T07:27:48.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 -- start start 2024-03-04T07:27:48.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f42bc06b5c0 0x7f42bc0d08a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0ae9b0 0x7f42bc1dc070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0b0e90 0x7f42bc0c8bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42bc06c910 con 0x7f42bc06b5c0 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42bc06c790 con 0x7f42bc0ae9b0 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.152+0000 7f42c1639640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f42bc0aee00 con 0x7f42bc0b0e90 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f42bc06b5c0 0x7f42bc0d08a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f42bc06b5c0 0x7f42bc0d08a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38896/0 (socket says 172.21.15.64:38896) 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 -- 172.21.15.64:0/2709947456 learned_addr learned my addr 172.21.15.64:0/2709947456 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42ba7fc640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0ae9b0 0x7f42bc1dc070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42bb7fe640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0b0e90 0x7f42bc0c8bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 -- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0b0e90 msgr2=0x7f42bc0c8bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0b0e90 0x7f42bc0c8bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 -- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0ae9b0 msgr2=0x7f42bc1dc070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0ae9b0 0x7f42bc1dc070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 -- 172.21.15.64:0/2709947456 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42bc0c97c0 con 0x7f42bc06b5c0 2024-03-04T07:27:48.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42baffd640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f42bc06b5c0 0x7f42bc0d08a0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f42b400c460 tx=0x7f42b400c860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42b37fe640 1 -- 172.21.15.64:0/2709947456 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42b4017070 con 0x7f42bc06b5c0 2024-03-04T07:27:48.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42b37fe640 1 -- 172.21.15.64:0/2709947456 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f42b4005ce0 con 0x7f42bc06b5c0 2024-03-04T07:27:48.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42bc1dfd10 con 0x7f42bc06b5c0 2024-03-04T07:27:48.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42b37fe640 1 -- 172.21.15.64:0/2709947456 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42b40072f0 con 0x7f42bc06b5c0 2024-03-04T07:27:48.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.153+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42bc0c9470 con 0x7f42bc06b5c0 2024-03-04T07:27:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.154+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f42bc068700 con 0x7f42bc06b5c0 2024-03-04T07:27:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.155+0000 7f42b37fe640 1 -- 172.21.15.64:0/2709947456 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f42b401a070 con 0x7f42bc06b5c0 2024-03-04T07:27:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.155+0000 7f42b37fe640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f429c06b4d0 0x7f429c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.155+0000 7f42ba7fc640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f429c06b4d0 0x7f429c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.155+0000 7f42b37fe640 1 -- 172.21.15.64:0/2709947456 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(32..32 src has 1..32) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f42b40894b0 con 0x7f42bc06b5c0 2024-03-04T07:27:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.156+0000 7f42ba7fc640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f429c06b4d0 0x7f429c06d990 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f42a4036000 tx=0x7f42a403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.160+0000 7f42b37fe640 1 -- 172.21.15.64:0/2709947456 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f42b4057d70 con 0x7f42bc06b5c0 2024-03-04T07:27:48.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.343+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f42bc06fe10 con 0x7f42bc06b5c0 2024-03-04T07:27:48.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.344+0000 7f42b37fe640 1 -- 172.21.15.64:0/2709947456 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v32) v1 ==== 74+0+16634 (secure 0 0 0) 0x7f42b405be10 con 0x7f42bc06b5c0 2024-03-04T07:27:48.343 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:48.343 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":32,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:47.355600+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:27:45.759140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-04T07:27:46.318635+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:48.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.345+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f429c06b4d0 msgr2=0x7f429c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.345+0000 7f42c1639640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f429c06b4d0 0x7f429c06d990 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f42a4036000 tx=0x7f42a403d040 comp rx=0 tx=0).stop 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.345+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f42bc06b5c0 msgr2=0x7f42bc0d08a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.345+0000 7f42c1639640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f42bc06b5c0 0x7f42bc0d08a0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f42b400c460 tx=0x7f42b400c860 comp rx=0 tx=0).stop 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 shutdown_connections 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f429c06b4d0 0x7f429c06d990 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42bc0b0e90 0x7f42bc0c8bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f42bc0ae9b0 0x7f42bc1dc070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 --2- 172.21.15.64:0/2709947456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f42bc06b5c0 0x7f42bc0d08a0 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 >> 172.21.15.64:0/2709947456 conn(0x7f42bc0b36d0 msgr2=0x7f42bc0b2c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 shutdown_connections 2024-03-04T07:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.346+0000 7f42c1639640 1 -- 172.21.15.64:0/2709947456 wait complete. 2024-03-04T07:27:48.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:48.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.458+0000 7f2042179640 1 Processor -- start 2024-03-04T07:27:48.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.458+0000 7f2042179640 1 -- start start 2024-03-04T07:27:48.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f2042179640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 0x7f203c15b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f2042179640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f203c06c7b0 con 0x7f203c157f60 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f2042179640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f203c06ca90 con 0x7f203c0cbab0 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f2042179640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f203c06cd70 con 0x7f203c157b80 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203affd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f203c157f60 0x7f203c15b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54630/0 (socket says 172.21.15.64:54630) 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203affd640 1 -- 172.21.15.64:0/3408809655 learned_addr learned my addr 172.21.15.64:0/3408809655 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3831346497 0 0) 0x7f203c06c7b0 con 0x7f203c157f60 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203b7fe640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 0x7f203c15b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2024003440 con 0x7f203c157f60 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2947610863 0 0) 0x7f2024003440 con 0x7f203c157f60 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 msgr2=0x7f203c15b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203a7fc640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 0x7f203c15b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.459+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 >> v1:172.21.15.66:6789/0 conn(0x7f203c0cbab0 legacy=0x7f203c070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f203c06d050 con 0x7f203c157f60 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2030001670 con 0x7f203c157f60 2024-03-04T07:27:48.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 >> v1:172.21.15.64:6789/0 conn(0x7f203c157f60 legacy=0x7f203c15b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2024003ad0 0x7f2024003eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 0x7f2024008650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2024008d70 0x7f202400b1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f203c06cd70 con 0x7f2024008d70 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f202400b9c0 con 0x7f2024003ad0 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f202400bbc0 con 0x7f203c157b80 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203b7fe640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2024003ad0 0x7f2024003eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203affd640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 0x7f2024008650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203bfff640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2024008d70 0x7f202400b1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203b7fe640 1 -- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 msgr2=0x7f2024008650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203b7fe640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f203c157b80 0x7f2024008650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203b7fe640 1 -- 172.21.15.64:0/3408809655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2024008d70 msgr2=0x7f202400b1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203b7fe640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2024008d70 0x7f202400b1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.460+0000 7f203b7fe640 1 -- 172.21.15.64:0/3408809655 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2024003440 con 0x7f2024003ad0 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.461+0000 7f203b7fe640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2024003ad0 0x7f2024003eb0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f202c00bfc0 tx=0x7f2024003730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e117465d79bfa6e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.461+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f202c03b070 con 0x7f2024003ad0 2024-03-04T07:27:48.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.461+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f202c02e850 con 0x7f2024003ad0 2024-03-04T07:27:48.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.461+0000 7f203a7fc640 1 -- 172.21.15.64:0/3408809655 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f202c00bc10 con 0x7f2024003ad0 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.462+0000 7f2042179640 1 -- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2024003ad0 msgr2=0x7f2024003eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.462+0000 7f2042179640 1 --2- 172.21.15.64:0/3408809655 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2024003ad0 0x7f2024003eb0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f202c00bfc0 tx=0x7f2024003730 comp rx=0 tx=0).stop 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.462+0000 7f203bfff640 1 -- 172.21.15.64:0/3408809655 reap_dead start 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.462+0000 7f2042179640 1 -- 172.21.15.64:0/3408809655 shutdown_connections 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.462+0000 7f2042179640 1 -- 172.21.15.64:0/3408809655 >> 172.21.15.64:0/3408809655 conn(0x7f203c066d00 msgr2=0x7f203c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.462+0000 7f2042179640 1 -- 172.21.15.64:0/3408809655 shutdown_connections 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.462+0000 7f2042179640 1 -- 172.21.15.64:0/3408809655 wait complete. 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.463+0000 7f2042179640 1 Processor -- start 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.463+0000 7f2042179640 1 -- start start 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2024003ad0 0x7f203c14f5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f203c1f32e0 0x7f203c0ce850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f203c143ef0 0x7f203c144380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f203c071250 con 0x7f203c143ef0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f203c0cc260 con 0x7f203c1f32e0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f203c0713d0 con 0x7f2024003ad0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f203c143ef0 0x7f203c144380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f203c143ef0 0x7f203c144380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38906/0 (socket says 172.21.15.64:38906) 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 -- 172.21.15.64:0/2483789071 learned_addr learned my addr 172.21.15.64:0/2483789071 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203affd640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f203c1f32e0 0x7f203c0ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203b7fe640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2024003ad0 0x7f203c14f5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 -- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2024003ad0 msgr2=0x7f203c14f5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2024003ad0 0x7f203c14f5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 -- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f203c1f32e0 msgr2=0x7f203c0ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f203c1f32e0 0x7f203c0ce850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 -- 172.21.15.64:0/2483789071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f203c1f3770 con 0x7f203c143ef0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203affd640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f203c1f32e0 0x7f203c0ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f203bfff640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f203c143ef0 0x7f203c144380 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f2024004520 tx=0x7f203400ff70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2023fff640 1 -- 172.21.15.64:0/2483789071 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2034019070 con 0x7f203c143ef0 2024-03-04T07:27:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2023fff640 1 -- 172.21.15.64:0/2483789071 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2034006d60 con 0x7f203c143ef0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f203c1f3a50 con 0x7f203c143ef0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2023fff640 1 -- 172.21.15.64:0/2483789071 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2034003bb0 con 0x7f203c143ef0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.464+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f203c1f4010 con 0x7f203c143ef0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.465+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f203c1f4210 con 0x7f203c143ef0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.466+0000 7f2023fff640 1 -- 172.21.15.64:0/2483789071 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f203401c070 con 0x7f203c143ef0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.466+0000 7f2023fff640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f202806b4d0 0x7f202806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.466+0000 7f2023fff640 1 -- 172.21.15.64:0/2483789071 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(32..32 src has 1..32) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2034089630 con 0x7f203c143ef0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.466+0000 7f203b7fe640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f202806b4d0 0x7f202806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.467+0000 7f203b7fe640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f202806b4d0 0x7f202806d990 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f202c038000 tx=0x7f202c030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.471+0000 7f2023fff640 1 -- 172.21.15.64:0/2483789071 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f20340584b0 con 0x7f203c143ef0 2024-03-04T07:27:48.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.689+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f203c14de70 con 0x7f203c143ef0 2024-03-04T07:27:48.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.690+0000 7f2023fff640 1 -- 172.21.15.64:0/2483789071 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v32) v1 ==== 74+0+16634 (secure 0 0 0) 0x7f203405ce20 con 0x7f203c143ef0 2024-03-04T07:27:48.690 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:48.690 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":32,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:47.355600+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:27:45.759140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-04T07:27:46.318635+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f202806b4d0 msgr2=0x7f202806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f202806b4d0 0x7f202806d990 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f202c038000 tx=0x7f202c030000 comp rx=0 tx=0).stop 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f203c143ef0 msgr2=0x7f203c144380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f203c143ef0 0x7f203c144380 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f2024004520 tx=0x7f203400ff70 comp rx=0 tx=0).stop 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 shutdown_connections 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f202806b4d0 0x7f202806d990 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f203c143ef0 0x7f203c144380 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f203c1f32e0 0x7f203c0ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 --2- 172.21.15.64:0/2483789071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2024003ad0 0x7f203c14f5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 >> 172.21.15.64:0/2483789071 conn(0x7f203c066d00 msgr2=0x7f203c0af320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 shutdown_connections 2024-03-04T07:27:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.692+0000 7f2042179640 1 -- 172.21.15.64:0/2483789071 wait complete. 2024-03-04T07:27:48.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:27:48.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.810+0000 7f387bea4640 1 Processor -- start 2024-03-04T07:27:48.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.810+0000 7f387bea4640 1 -- start start 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387bea4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f387415dd90 0x7f38740ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387bea4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740b6d10 0x7f38740b7150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387bea4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f38740d3740 con 0x7f38740d29d0 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387bea4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38740d3400 con 0x7f38740b6d10 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387bea4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38740b7690 con 0x7f387415dd90 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387a441640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740b6d10 0x7f38740b7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387a441640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740b6d10 0x7f38740b7150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38916/0 (socket says 172.21.15.64:38916) 2024-03-04T07:27:48.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387a441640 1 -- 172.21.15.64:0/2593101491 learned_addr learned my addr 172.21.15.64:0/2593101491 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:48.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f387a441640 1 -- 172.21.15.64:0/2593101491 >> v2:172.21.15.66:3300/0 conn(0x7f387415dd90 msgr2=0x7f38740ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:48.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.811+0000 7f3879c40640 1 --2- 172.21.15.64:0/2593101491 >> v2:172.21.15.66:3300/0 conn(0x7f387415dd90 0x7f38740ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.812+0000 7f387a441640 1 --2- 172.21.15.64:0/2593101491 >> v2:172.21.15.66:3300/0 conn(0x7f387415dd90 0x7f38740ceea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.812+0000 7f387a441640 1 -- 172.21.15.64:0/2593101491 >> v1:172.21.15.66:6789/0 conn(0x7f38740d29d0 legacy=0x7f38740d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:48.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.812+0000 7f387a441640 1 -- 172.21.15.64:0/2593101491 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38740d3120 con 0x7f38740b6d10 2024-03-04T07:27:48.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387a441640 1 --2- 172.21.15.64:0/2593101491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b6d10 0x7f38740b7150 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f38740d1f90 tx=0x7f386c02bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6eaab4b5d7d3185f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f3878c3e640 1 -- 172.21.15.64:0/2593101491 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c02cb20 con 0x7f38740b6d10 2024-03-04T07:27:48.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f3878c3e640 1 -- 172.21.15.64:0/2593101491 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f386c02ccc0 con 0x7f38740b6d10 2024-03-04T07:27:48.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f3878c3e640 1 -- 172.21.15.64:0/2593101491 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c034610 con 0x7f38740b6d10 2024-03-04T07:27:48.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387bea4640 1 -- 172.21.15.64:0/2593101491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b6d10 msgr2=0x7f38740b7150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387bea4640 1 --2- 172.21.15.64:0/2593101491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b6d10 0x7f38740b7150 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f38740d1f90 tx=0x7f386c02bf80 comp rx=0 tx=0).stop 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387bea4640 1 -- 172.21.15.64:0/2593101491 shutdown_connections 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387bea4640 1 --2- 172.21.15.64:0/2593101491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b6d10 0x7f38740b7150 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387bea4640 1 --2- 172.21.15.64:0/2593101491 >> v2:172.21.15.66:3300/0 conn(0x7f387415dd90 0x7f38740ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387bea4640 1 -- 172.21.15.64:0/2593101491 >> 172.21.15.64:0/2593101491 conn(0x7f3874071960 msgr2=0x7f3874071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.813+0000 7f387bea4640 1 -- 172.21.15.64:0/2593101491 shutdown_connections 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.814+0000 7f387bea4640 1 -- 172.21.15.64:0/2593101491 wait complete. 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 Processor -- start 2024-03-04T07:27:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 -- start start 2024-03-04T07:27:48.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b6d10 0x7f38740ca070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740d29d0 0x7f38740ccff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415dd90 0x7f3874160790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f387415e170 con 0x7f38740d29d0 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38740d3cc0 con 0x7f38740b6d10 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387bea4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38740d2df0 con 0x7f387415dd90 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.815+0000 7f387943f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740d29d0 0x7f38740ccff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740d29d0 0x7f38740ccff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38918/0 (socket says 172.21.15.64:38918) 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 -- 172.21.15.64:0/2271702958 learned_addr learned my addr 172.21.15.64:0/2271702958 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f3879c40640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b6d10 0x7f38740ca070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387a441640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415dd90 0x7f3874160790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 -- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415dd90 msgr2=0x7f3874160790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415dd90 0x7f3874160790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 -- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b6d10 msgr2=0x7f38740ca070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b6d10 0x7f38740ca070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 -- 172.21.15.64:0/2271702958 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38740ca5b0 con 0x7f38740d29d0 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387943f640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740d29d0 0x7f38740ccff0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f386400c430 tx=0x7f386400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f386a7fc640 1 -- 172.21.15.64:0/2271702958 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3864011790 con 0x7f38740d29d0 2024-03-04T07:27:48.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f386a7fc640 1 -- 172.21.15.64:0/2271702958 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3864017070 con 0x7f38740d29d0 2024-03-04T07:27:48.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f38740c8b50 con 0x7f38740d29d0 2024-03-04T07:27:48.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f386a7fc640 1 -- 172.21.15.64:0/2271702958 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3864011bb0 con 0x7f38740d29d0 2024-03-04T07:27:48.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.816+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f38740c9110 con 0x7f38740d29d0 2024-03-04T07:27:48.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.817+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3874072a30 con 0x7f38740d29d0 2024-03-04T07:27:48.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.818+0000 7f386a7fc640 1 -- 172.21.15.64:0/2271702958 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f386401c460 con 0x7f38740d29d0 2024-03-04T07:27:48.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.819+0000 7f386a7fc640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f385806b500 0x7f385806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:48.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.819+0000 7f386a7fc640 1 -- 172.21.15.64:0/2271702958 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(32..32 src has 1..32) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3864089e70 con 0x7f38740d29d0 2024-03-04T07:27:48.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.819+0000 7f3879c40640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f385806b500 0x7f385806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:48.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.819+0000 7f3879c40640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f385806b500 0x7f385806d9c0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f387000aa20 tx=0x7f3870008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:48.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:48.823+0000 7f386a7fc640 1 -- 172.21.15.64:0/2271702958 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3864058730 con 0x7f38740d29d0 2024-03-04T07:27:49.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:49.044+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f387415e2f0 con 0x7f38740d29d0 2024-03-04T07:27:50.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.016+0000 7f386a7fc640 1 -- 172.21.15.64:0/2271702958 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v14) v1 ==== 118+0+0 (secure 0 0 0) 0x7f386405c7d0 con 0x7f38740d29d0 2024-03-04T07:27:50.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f385806b500 msgr2=0x7f385806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:50.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f385806b500 0x7f385806d9c0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f387000aa20 tx=0x7f3870008040 comp rx=0 tx=0).stop 2024-03-04T07:27:50.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740d29d0 msgr2=0x7f38740ccff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:50.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740d29d0 0x7f38740ccff0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f386400c430 tx=0x7f386400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 shutdown_connections 2024-03-04T07:27:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f385806b500 0x7f385806d9c0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415dd90 0x7f3874160790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740d29d0 0x7f38740ccff0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 --2- 172.21.15.64:0/2271702958 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b6d10 0x7f38740ca070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.018+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 >> 172.21.15.64:0/2271702958 conn(0x7f3874071960 msgr2=0x7f38740b36c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.019+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 shutdown_connections 2024-03-04T07:27:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.019+0000 7f387bea4640 1 -- 172.21.15.64:0/2271702958 wait complete. 2024-03-04T07:27:50.032 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:27:50.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:27:50.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.138+0000 7fba9673a640 1 Processor -- start 2024-03-04T07:27:50.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.138+0000 7fba9673a640 1 -- start start 2024-03-04T07:27:50.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba9673a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba90066d00 0x7fba90160180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:50.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba9673a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9015c180 0x7fba900670e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba9673a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fba901606c0 con 0x7fba9006def0 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba9673a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fba90067620 con 0x7fba90066d00 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba9673a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fba90145910 con 0x7fba9015c180 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba90066d00 0x7fba90160180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba90066d00 0x7fba90160180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38932/0 (socket says 172.21.15.64:38932) 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8ffff640 1 -- 172.21.15.64:0/1596660801 learned_addr learned my addr 172.21.15.64:0/1596660801 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8f7fe640 1 --2- 172.21.15.64:0/1596660801 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9015c180 0x7fba900670e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8ffff640 1 -- 172.21.15.64:0/1596660801 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9015c180 msgr2=0x7fba900670e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8ffff640 1 --2- 172.21.15.64:0/1596660801 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9015c180 0x7fba900670e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8ffff640 1 -- 172.21.15.64:0/1596660801 >> v1:172.21.15.66:6789/0 conn(0x7fba9006def0 legacy=0x7fba90071e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.139+0000 7fba8ffff640 1 -- 172.21.15.64:0/1596660801 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba9015c6f0 con 0x7fba90066d00 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.140+0000 7fba8ffff640 1 --2- 172.21.15.64:0/1596660801 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba90160180 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fba80009870 tx=0x7fba8002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b56aeaf527261119 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:50.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.140+0000 7fba8effd640 1 -- 172.21.15.64:0/1596660801 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba80034070 con 0x7fba90066d00 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.140+0000 7fba8effd640 1 -- 172.21.15.64:0/1596660801 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba80030840 con 0x7fba90066d00 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.140+0000 7fba8effd640 1 -- 172.21.15.64:0/1596660801 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba80030c10 con 0x7fba90066d00 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.140+0000 7fba9673a640 1 -- 172.21.15.64:0/1596660801 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 msgr2=0x7fba90160180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.140+0000 7fba9673a640 1 --2- 172.21.15.64:0/1596660801 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba90160180 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fba80009870 tx=0x7fba8002b980 comp rx=0 tx=0).stop 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.141+0000 7fba9673a640 1 -- 172.21.15.64:0/1596660801 shutdown_connections 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.141+0000 7fba9673a640 1 --2- 172.21.15.64:0/1596660801 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9015c180 0x7fba900670e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.141+0000 7fba9673a640 1 --2- 172.21.15.64:0/1596660801 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba90160180 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.141+0000 7fba9673a640 1 -- 172.21.15.64:0/1596660801 >> 172.21.15.64:0/1596660801 conn(0x7fba900d09c0 msgr2=0x7fba900d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.141+0000 7fba9673a640 1 -- 172.21.15.64:0/1596660801 shutdown_connections 2024-03-04T07:27:50.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.141+0000 7fba9673a640 1 -- 172.21.15.64:0/1596660801 wait complete. 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.142+0000 7fba9673a640 1 Processor -- start 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.142+0000 7fba9673a640 1 -- start start 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.143+0000 7fba9673a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba9015ab30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.143+0000 7fba9673a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9006def0 0x7fba9013dea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.143+0000 7fba9673a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba9015c180 0x7fba901f2330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.143+0000 7fba9673a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba9015cf90 con 0x7fba90066d00 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.143+0000 7fba9673a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba9015ccb0 con 0x7fba9015c180 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.143+0000 7fba9673a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fba9006e380 con 0x7fba9006def0 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba9015ab30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba9015ab30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38948/0 (socket says 172.21.15.64:38948) 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 -- 172.21.15.64:0/949516132 learned_addr learned my addr 172.21.15.64:0/949516132 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba94cd7640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba9015c180 0x7fba901f2330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:50.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8f7fe640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9006def0 0x7fba9013dea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 -- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9006def0 msgr2=0x7fba9013dea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9006def0 0x7fba9013dea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 -- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba9015c180 msgr2=0x7fba901f2330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba9015c180 0x7fba901f2330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 -- 172.21.15.64:0/949516132 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba901f1b50 con 0x7fba90066d00 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8ffff640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba9015ab30 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fba80000c00 tx=0x7fba80004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8cff9640 1 -- 172.21.15.64:0/949516132 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba80048070 con 0x7fba90066d00 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8cff9640 1 -- 172.21.15.64:0/949516132 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba80030640 con 0x7fba90066d00 2024-03-04T07:27:50.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fba9013e8f0 con 0x7fba90066d00 2024-03-04T07:27:50.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba8cff9640 1 -- 172.21.15.64:0/949516132 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba80034040 con 0x7fba90066d00 2024-03-04T07:27:50.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.144+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fba9013eeb0 con 0x7fba90066d00 2024-03-04T07:27:50.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.145+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fba900733d0 con 0x7fba90066d00 2024-03-04T07:27:50.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.146+0000 7fba8cff9640 1 -- 172.21.15.64:0/949516132 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fba8004b070 con 0x7fba90066d00 2024-03-04T07:27:50.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.146+0000 7fba8cff9640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba7c06b4d0 0x7fba7c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:50.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.146+0000 7fba8cff9640 1 -- 172.21.15.64:0/949516132 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fba800ac130 con 0x7fba90066d00 2024-03-04T07:27:50.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.146+0000 7fba8f7fe640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba7c06b4d0 0x7fba7c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:50.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.147+0000 7fba8f7fe640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba7c06b4d0 0x7fba7c06d990 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fba84006fd0 tx=0x7fba84008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:50.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.154+0000 7fba8cff9640 1 -- 172.21.15.64:0/949516132 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fba8007b040 con 0x7fba90066d00 2024-03-04T07:27:50.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:50.388+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7fba900d17c0 con 0x7fba90066d00 2024-03-04T07:27:51.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.024+0000 7fba8cff9640 1 -- 172.21.15.64:0/949516132 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v15) v1 ==== 122+0+0 (secure 0 0 0) 0x7fba8007a940 con 0x7fba90066d00 2024-03-04T07:27:51.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba7c06b4d0 msgr2=0x7fba7c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba7c06b4d0 0x7fba7c06d990 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fba84006fd0 tx=0x7fba84008040 comp rx=0 tx=0).stop 2024-03-04T07:27:51.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 msgr2=0x7fba9015ab30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba9015ab30 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fba80000c00 tx=0x7fba80004730 comp rx=0 tx=0).stop 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 shutdown_connections 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba7c06b4d0 0x7fba7c06d990 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba9015c180 0x7fba901f2330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba9006def0 0x7fba9013dea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 --2- 172.21.15.64:0/949516132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba90066d00 0x7fba9015ab30 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.026+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 >> 172.21.15.64:0/949516132 conn(0x7fba900d09c0 msgr2=0x7fba90071e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.027+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 shutdown_connections 2024-03-04T07:27:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.027+0000 7fba9673a640 1 -- 172.21.15.64:0/949516132 wait complete. 2024-03-04T07:27:51.037 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:27:51.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:51.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.140+0000 7f6ff2a57640 1 Processor -- start 2024-03-04T07:27:51.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.140+0000 7f6ff2a57640 1 -- start start 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.140+0000 7f6ff2a57640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6fec0b2b50 0x7f6fec0b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.140+0000 7f6ff2a57640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6fec0ce610 con 0x7f6fec0d3400 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.140+0000 7f6ff2a57640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6fec0d2920 con 0x7f6fec071540 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.140+0000 7f6ff2a57640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6fec0d2f70 con 0x7f6fec0b2b50 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6feb7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6fec0d3400 0x7f6fec0b6e20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54646/0 (socket says 172.21.15.64:54646) 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6feb7fe640 1 -- 172.21.15.64:0/3903426445 learned_addr learned my addr 172.21.15.64:0/3903426445 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6ff0ff4640 1 --2- 172.21.15.64:0/3903426445 >> v2:172.21.15.66:3300/0 conn(0x7f6fec0b2b50 0x7f6fec0b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6feaffd640 1 -- 172.21.15.64:0/3903426445 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2071380883 0 0) 0x7f6fec0ce610 con 0x7f6fec0d3400 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6feaffd640 1 -- 172.21.15.64:0/3903426445 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6fd4003520 con 0x7f6fec0d3400 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6ff0ff4640 1 -- 172.21.15.64:0/3903426445 >> v1:172.21.15.66:6789/0 conn(0x7f6fec071540 legacy=0x7f6fec071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6ff0ff4640 1 -- 172.21.15.64:0/3903426445 >> v1:172.21.15.64:6789/0 conn(0x7f6fec0d3400 legacy=0x7f6fec0b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6feaffd640 1 -- 172.21.15.64:0/3903426445 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 66240331 0 0) 0x7f6fd4003520 con 0x7f6fec0d3400 2024-03-04T07:27:51.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.141+0000 7f6ff0ff4640 1 -- 172.21.15.64:0/3903426445 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fec0d2c00 con 0x7f6fec0b2b50 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.142+0000 7f6ff0ff4640 1 --2- 172.21.15.64:0/3903426445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec0b30a0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f6fe4009d90 tx=0x7f6fe402be10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=de6439591254d4fb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.142+0000 7f6feaffd640 1 -- 172.21.15.64:0/3903426445 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fe4039070 con 0x7f6fec0b2b50 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.142+0000 7f6feaffd640 1 -- 172.21.15.64:0/3903426445 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fe402c940 con 0x7f6fec0b2b50 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6feaffd640 1 -- 172.21.15.64:0/3903426445 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fe0001670 con 0x7f6fec0b2b50 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6ff2a57640 1 -- 172.21.15.64:0/3903426445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 msgr2=0x7f6fec0b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/3903426445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec0b30a0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f6fe4009d90 tx=0x7f6fe402be10 comp rx=0 tx=0).stop 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6ff2a57640 1 -- 172.21.15.64:0/3903426445 shutdown_connections 2024-03-04T07:27:51.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/3903426445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec0b30a0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6ff2a57640 1 -- 172.21.15.64:0/3903426445 >> 172.21.15.64:0/3903426445 conn(0x7f6fec0d3ca0 msgr2=0x7f6fec0d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:51.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6ff2a57640 1 -- 172.21.15.64:0/3903426445 shutdown_connections 2024-03-04T07:27:51.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.143+0000 7f6ff2a57640 1 -- 172.21.15.64:0/3903426445 wait complete. 2024-03-04T07:27:51.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.144+0000 7f6ff2a57640 1 Processor -- start 2024-03-04T07:27:51.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.144+0000 7f6ff2a57640 1 -- start start 2024-03-04T07:27:51.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6ff2a57640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fec071540 0x7f6fec0b3080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6ff2a57640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec14d060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6ff2a57640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fec0d3400 0x7f6fec0cd4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6ff2a57640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fec15e370 con 0x7f6fec071540 2024-03-04T07:27:51.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6ff2a57640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fec0d4be0 con 0x7f6fec0b2b50 2024-03-04T07:27:51.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6ff2a57640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6fec0b3940 con 0x7f6fec0d3400 2024-03-04T07:27:51.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6febfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fec071540 0x7f6fec0b3080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6febfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fec071540 0x7f6fec0b3080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38958/0 (socket says 172.21.15.64:38958) 2024-03-04T07:27:51.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6febfff640 1 -- 172.21.15.64:0/177779281 learned_addr learned my addr 172.21.15.64:0/177779281 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:51.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6ff0ff4640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fec0d3400 0x7f6fec0cd4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6feb7fe640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec14d060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6febfff640 1 -- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fec0d3400 msgr2=0x7f6fec0cd4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.145+0000 7f6febfff640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fec0d3400 0x7f6fec0cd4e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6febfff640 1 -- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 msgr2=0x7f6fec14d060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6febfff640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec14d060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6febfff640 1 -- 172.21.15.64:0/177779281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fec0b3c20 con 0x7f6fec071540 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6ff0ff4640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fec0d3400 0x7f6fec0cd4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6feb7fe640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec14d060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6febfff640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fec071540 0x7f6fec0b3080 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f6fdc00c450 tx=0x7f6fdc00c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6fe8ff9640 1 -- 172.21.15.64:0/177779281 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fdc0052d0 con 0x7f6fec071540 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6fe8ff9640 1 -- 172.21.15.64:0/177779281 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fdc005470 con 0x7f6fec071540 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6fec0b3f00 con 0x7f6fec071540 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6fe8ff9640 1 -- 172.21.15.64:0/177779281 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fdc012870 con 0x7f6fec071540 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.146+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6fec14d5a0 con 0x7f6fec071540 2024-03-04T07:27:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.147+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6fec068700 con 0x7f6fec071540 2024-03-04T07:27:51.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.147+0000 7f6fe8ff9640 1 -- 172.21.15.64:0/177779281 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6fdc012a10 con 0x7f6fec071540 2024-03-04T07:27:51.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.148+0000 7f6fe8ff9640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6fd806b4d0 0x7f6fd806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.148+0000 7f6fe8ff9640 1 -- 172.21.15.64:0/177779281 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6fdc089e90 con 0x7f6fec071540 2024-03-04T07:27:51.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.148+0000 7f6feb7fe640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6fd806b4d0 0x7f6fd806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.148+0000 7f6feb7fe640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6fd806b4d0 0x7f6fd806d990 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f6fec0b8c90 tx=0x7f6fe0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.156+0000 7f6fe8ff9640 1 -- 172.21.15.64:0/177779281 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6fdc058c90 con 0x7f6fec071540 2024-03-04T07:27:51.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.376+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6fec0b89e0 con 0x7f6fec071540 2024-03-04T07:27:51.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.376+0000 7f6fe8ff9640 1 -- 172.21.15.64:0/177779281 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f6fdc058590 con 0x7f6fec071540 2024-03-04T07:27:51.376 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:51.376 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:27:51.376 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6fd806b4d0 msgr2=0x7f6fd806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6fd806b4d0 0x7f6fd806d990 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f6fec0b8c90 tx=0x7f6fe0009040 comp rx=0 tx=0).stop 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fec071540 msgr2=0x7f6fec0b3080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fec071540 0x7f6fec0b3080 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f6fdc00c450 tx=0x7f6fdc00c910 comp rx=0 tx=0).stop 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 shutdown_connections 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6fd806b4d0 0x7f6fd806d990 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fec0d3400 0x7f6fec0cd4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fec0b2b50 0x7f6fec14d060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 --2- 172.21.15.64:0/177779281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fec071540 0x7f6fec0b3080 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 >> 172.21.15.64:0/177779281 conn(0x7f6fec0d3ca0 msgr2=0x7f6fec15f490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:51.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 shutdown_connections 2024-03-04T07:27:51.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.378+0000 7f6ff2a57640 1 -- 172.21.15.64:0/177779281 wait complete. 2024-03-04T07:27:51.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:27:51.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.490+0000 7f654cfd7640 1 Processor -- start 2024-03-04T07:27:51.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.490+0000 7f654cfd7640 1 -- start start 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f654cfd7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f654815dca0 0x7f654806fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f654cfd7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 0x7f65480b3ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f654cfd7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6548161c20 con 0x7f654815e080 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f654cfd7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f65481619d0 con 0x7f654815dca0 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f654cfd7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6548161f00 con 0x7f65480b00c0 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f653dd9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f654815e080 0x7f6548066d00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54660/0 (socket says 172.21.15.64:54660) 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f653dd9b640 1 -- 172.21.15.64:0/70617177 learned_addr learned my addr 172.21.15.64:0/70617177 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f654659c640 1 --2- 172.21.15.64:0/70617177 >> v2:172.21.15.66:3300/0 conn(0x7f654815dca0 0x7f654806fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f6546d9d640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 0x7f65480b3ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3110668423 0 0) 0x7f6548161c20 con 0x7f654815e080 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.491+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6534003520 con 0x7f654815e080 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1542632601 0 0) 0x7f6534003520 con 0x7f654815e080 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 msgr2=0x7f65480b3ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 0x7f65480b3ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 >> v2:172.21.15.66:3300/0 conn(0x7f654815dca0 msgr2=0x7f654806fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 --2- 172.21.15.64:0/70617177 >> v2:172.21.15.66:3300/0 conn(0x7f654815dca0 0x7f654806fd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65481621e0 con 0x7f654815e080 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f6530001670 con 0x7f654815e080 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 >> v1:172.21.15.64:6789/0 conn(0x7f654815e080 legacy=0x7f6548066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65340039b0 0x7f6534003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 0x7f6534004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6534004d40 0x7f65340051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65481619d0 con 0x7f65340039b0 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6548161f00 con 0x7f6534004d40 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f653400b9c0 con 0x7f65480b00c0 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f654659c640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65340039b0 0x7f6534003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6546d9d640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6534004d40 0x7f65340051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f653dd9b640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 0x7f6534004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f654659c640 1 -- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 msgr2=0x7f6534004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f654659c640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f65480b00c0 0x7f6534004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f654659c640 1 -- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6534004d40 msgr2=0x7f65340051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f654659c640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6534004d40 0x7f65340051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f654659c640 1 -- 172.21.15.64:0/70617177 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6534003520 con 0x7f65340039b0 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.492+0000 7f6546d9d640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6534004d40 0x7f65340051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.493+0000 7f654659c640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65340039b0 0x7f6534003dd0 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f6538009ac0 tx=0x7f6534005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a1d562faf95e6ba server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.493+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6538039070 con 0x7f65340039b0 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.493+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f653802c6d0 con 0x7f65340039b0 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.493+0000 7f6545d9b640 1 -- 172.21.15.64:0/70617177 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6538009690 con 0x7f65340039b0 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.494+0000 7f654cfd7640 1 -- 172.21.15.64:0/70617177 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65340039b0 msgr2=0x7f6534003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.494+0000 7f654cfd7640 1 --2- 172.21.15.64:0/70617177 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f65340039b0 0x7f6534003dd0 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f6538009ac0 tx=0x7f6534005830 comp rx=0 tx=0).stop 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.494+0000 7f6546d9d640 1 -- 172.21.15.64:0/70617177 reap_dead start 2024-03-04T07:27:51.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.494+0000 7f654cfd7640 1 -- 172.21.15.64:0/70617177 shutdown_connections 2024-03-04T07:27:51.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.494+0000 7f654cfd7640 1 -- 172.21.15.64:0/70617177 >> 172.21.15.64:0/70617177 conn(0x7f65480d0990 msgr2=0x7f65480d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:51.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.494+0000 7f654cfd7640 1 -- 172.21.15.64:0/70617177 shutdown_connections 2024-03-04T07:27:51.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.494+0000 7f654cfd7640 1 -- 172.21.15.64:0/70617177 wait complete. 2024-03-04T07:27:51.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.496+0000 7f654cfd7640 1 Processor -- start 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.496+0000 7f654cfd7640 1 -- start start 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f654cfd7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65340039b0 0x7f6548160b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f654cfd7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f654814ca50 0x7f65480ae0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f654cfd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6548160230 0x7f65480ae600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f654cfd7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65480703d0 con 0x7f6548160230 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f654cfd7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6548070250 con 0x7f65340039b0 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f654cfd7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6548067410 con 0x7f654814ca50 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f6546d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6548160230 0x7f65480ae600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f6546d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6548160230 0x7f65480ae600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38968/0 (socket says 172.21.15.64:38968) 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f6546d9d640 1 -- 172.21.15.64:0/2198987547 learned_addr learned my addr 172.21.15.64:0/2198987547 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f654659c640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65340039b0 0x7f6548160b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.497+0000 7f653dd9b640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f654814ca50 0x7f65480ae0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f6546d9d640 1 -- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f654814ca50 msgr2=0x7f65480ae0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f6546d9d640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f654814ca50 0x7f65480ae0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f6546d9d640 1 -- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65340039b0 msgr2=0x7f6548160b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f6546d9d640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65340039b0 0x7f6548160b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f6546d9d640 1 -- 172.21.15.64:0/2198987547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65480cd5d0 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f6546d9d640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6548160230 0x7f65480ae600 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f65480aff50 tx=0x7f6540010590 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f653f7fe640 1 -- 172.21.15.64:0/2198987547 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f654001a030 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f653f7fe640 1 -- 172.21.15.64:0/2198987547 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6540010d80 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6548161070 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f653f7fe640 1 -- 172.21.15.64:0/2198987547 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6540003bd0 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.498+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f654815f290 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.499+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6548068700 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.499+0000 7f653f7fe640 1 -- 172.21.15.64:0/2198987547 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f654001d070 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.500+0000 7f653f7fe640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f652806b4d0 0x7f652806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.500+0000 7f653f7fe640 1 -- 172.21.15.64:0/2198987547 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f654008a800 con 0x7f6548160230 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.500+0000 7f654659c640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f652806b4d0 0x7f652806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.500+0000 7f654659c640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f652806b4d0 0x7f652806d990 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f6538036000 tx=0x7f6538030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.508+0000 7f653f7fe640 1 -- 172.21.15.64:0/2198987547 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6548161070 con 0x7f6548160230 2024-03-04T07:27:51.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.689+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6548160910 con 0x7f6548160230 2024-03-04T07:27:51.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.690+0000 7f653f7fe640 1 -- 172.21.15.64:0/2198987547 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) v1 ==== 74+0+16621 (secure 0 0 0) 0x7f6540059710 con 0x7f6548160230 2024-03-04T07:27:51.690 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:51.690 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":33,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:49.009109+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:27:45.759140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-04T07:27:46.318635+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f652806b4d0 msgr2=0x7f652806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f652806b4d0 0x7f652806d990 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f6538036000 tx=0x7f6538030000 comp rx=0 tx=0).stop 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6548160230 msgr2=0x7f65480ae600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6548160230 0x7f65480ae600 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f65480aff50 tx=0x7f6540010590 comp rx=0 tx=0).stop 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 shutdown_connections 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f652806b4d0 0x7f652806d990 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6548160230 0x7f65480ae600 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f654814ca50 0x7f65480ae0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 --2- 172.21.15.64:0/2198987547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f65340039b0 0x7f6548160b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 >> 172.21.15.64:0/2198987547 conn(0x7f65480d0990 msgr2=0x7f6548072240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:51.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 shutdown_connections 2024-03-04T07:27:51.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.692+0000 7f654cfd7640 1 -- 172.21.15.64:0/2198987547 wait complete. 2024-03-04T07:27:51.706 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:27:51.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:27:51.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.808+0000 7fa77bc66640 1 Processor -- start 2024-03-04T07:27:51.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.809+0000 7fa77bc66640 1 -- start start 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.809+0000 7fa77bc66640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774071250 0x7fa77415b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.809+0000 7fa77bc66640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa77406c7b0 con 0x7fa774157b80 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.809+0000 7fa77bc66640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa77406ca90 con 0x7fa774157f60 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.809+0000 7fa77bc66640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa77406cd70 con 0x7fa774071250 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa779a02640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa774157b80 0x7fa77415b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54672/0 (socket says 172.21.15.64:54672) 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa779a02640 1 -- 172.21.15.64:0/909969939 learned_addr learned my addr 172.21.15.64:0/909969939 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa77a203640 1 --2- 172.21.15.64:0/909969939 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774071250 0x7fa77415b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3141404701 0 0) 0x7fa77406c7b0 con 0x7fa774157b80 2024-03-04T07:27:51.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa760005730 con 0x7fa774157b80 2024-03-04T07:27:51.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 722831759 0 0) 0x7fa77406ca90 con 0x7fa774157f60 2024-03-04T07:27:51.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa77406c7b0 con 0x7fa774157f60 2024-03-04T07:27:51.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 75056913 0 0) 0x7fa760005730 con 0x7fa774157b80 2024-03-04T07:27:51.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa77a203640 1 -- 172.21.15.64:0/909969939 >> v1:172.21.15.66:6789/0 conn(0x7fa774157f60 legacy=0x7fa774070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa77a203640 1 -- 172.21.15.64:0/909969939 >> v1:172.21.15.64:6789/0 conn(0x7fa774157b80 legacy=0x7fa77415b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.810+0000 7fa77a203640 1 -- 172.21.15.64:0/909969939 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa77406d050 con 0x7fa774071250 2024-03-04T07:27:51.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.811+0000 7fa77a203640 1 --2- 172.21.15.64:0/909969939 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774071250 0x7fa77415b260 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fa76c00a970 tx=0x7fa76c02bbd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=280dcc4823869132 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.811+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7680019f0 con 0x7fa774071250 2024-03-04T07:27:51.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.811+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa76c02d710 con 0x7fa774071250 2024-03-04T07:27:51.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.811+0000 7fa778a00640 1 -- 172.21.15.64:0/909969939 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa76c00b040 con 0x7fa774071250 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.812+0000 7fa77bc66640 1 -- 172.21.15.64:0/909969939 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774071250 msgr2=0x7fa77415b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.812+0000 7fa77bc66640 1 --2- 172.21.15.64:0/909969939 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774071250 0x7fa77415b260 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fa76c00a970 tx=0x7fa76c02bbd0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.812+0000 7fa77bc66640 1 -- 172.21.15.64:0/909969939 shutdown_connections 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.812+0000 7fa77bc66640 1 --2- 172.21.15.64:0/909969939 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774071250 0x7fa77415b260 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.812+0000 7fa77bc66640 1 -- 172.21.15.64:0/909969939 >> 172.21.15.64:0/909969939 conn(0x7fa774066d00 msgr2=0x7fa7740670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.812+0000 7fa77bc66640 1 -- 172.21.15.64:0/909969939 shutdown_connections 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.812+0000 7fa77bc66640 1 -- 172.21.15.64:0/909969939 wait complete. 2024-03-04T07:27:51.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.814+0000 7fa77bc66640 1 Processor -- start 2024-03-04T07:27:51.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.814+0000 7fa77bc66640 1 -- start start 2024-03-04T07:27:51.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa77bc66640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa774071250 0x7fa7740ce6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa77bc66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa774157b80 0x7fa7740b3060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa77bc66640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774157f60 0x7fa7741ef740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa77bc66640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7740cbc30 con 0x7fa774157b80 2024-03-04T07:27:51.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa77bc66640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7740cbab0 con 0x7fa774071250 2024-03-04T07:27:51.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa77bc66640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa7740cbdb0 con 0x7fa774157f60 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa779201640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa774157b80 0x7fa7740b3060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa779201640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa774157b80 0x7fa7740b3060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38984/0 (socket says 172.21.15.64:38984) 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa779201640 1 -- 172.21.15.64:0/2556116316 learned_addr learned my addr 172.21.15.64:0/2556116316 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa779a02640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa774071250 0x7fa7740ce6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.815+0000 7fa77a203640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774157f60 0x7fa7741ef740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa779201640 1 -- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774157f60 msgr2=0x7fa7741ef740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa779201640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774157f60 0x7fa7741ef740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa779201640 1 -- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa774071250 msgr2=0x7fa7740ce6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa779201640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa774071250 0x7fa7740ce6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa779201640 1 -- 172.21.15.64:0/2556116316 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7740b35a0 con 0x7fa774157b80 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa779201640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa774157b80 0x7fa7740b3060 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7fa77000c3c0 tx=0x7fa77000c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa7667fc640 1 -- 172.21.15.64:0/2556116316 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa770017070 con 0x7fa774157b80 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa7667fc640 1 -- 172.21.15.64:0/2556116316 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa7700052f0 con 0x7fa774157b80 2024-03-04T07:27:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7741f0200 con 0x7fa774157b80 2024-03-04T07:27:51.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa7667fc640 1 -- 172.21.15.64:0/2556116316 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa770003830 con 0x7fa774157b80 2024-03-04T07:27:51.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.816+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7740b2860 con 0x7fa774157b80 2024-03-04T07:27:51.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.817+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa774067ff0 con 0x7fa774157b80 2024-03-04T07:27:51.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.818+0000 7fa7667fc640 1 -- 172.21.15.64:0/2556116316 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa77001a070 con 0x7fa774157b80 2024-03-04T07:27:51.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.818+0000 7fa7667fc640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa75806b4d0 0x7fa75806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:51.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.818+0000 7fa7667fc640 1 -- 172.21.15.64:0/2556116316 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa7700894f0 con 0x7fa774157b80 2024-03-04T07:27:51.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.818+0000 7fa779a02640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa75806b4d0 0x7fa75806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:51.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.819+0000 7fa779a02640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa75806b4d0 0x7fa75806d990 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fa768001560 tx=0x7fa768009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:51.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.823+0000 7fa7667fc640 1 -- 172.21.15.64:0/2556116316 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa770057df0 con 0x7fa774157b80 2024-03-04T07:27:51.826 INFO:tasks.ceph.mds.a.smithi064.stderr:2024-03-04T07:27:51.827+0000 7f7a06210640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:27:51.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:51.999+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fa7740af020 con 0x7fa75806b4d0 2024-03-04T07:27:52.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.005+0000 7fa7667fc640 1 -- 172.21.15.64:0/2556116316 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+239842 (secure 0 0 0) 0x7fa7740af020 con 0x7fa75806b4d0 2024-03-04T07:27:52.007 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:52.008 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:27:52.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa75806b4d0 msgr2=0x7fa75806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa75806b4d0 0x7fa75806d990 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fa768001560 tx=0x7fa768009040 comp rx=0 tx=0).stop 2024-03-04T07:27:52.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa774157b80 msgr2=0x7fa7740b3060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa774157b80 0x7fa7740b3060 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7fa77000c3c0 tx=0x7fa77000c880 comp rx=0 tx=0).stop 2024-03-04T07:27:52.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 shutdown_connections 2024-03-04T07:27:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa75806b4d0 0x7fa75806d990 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa774157f60 0x7fa7741ef740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa774157b80 0x7fa7740b3060 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 --2- 172.21.15.64:0/2556116316 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa774071250 0x7fa7740ce6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 >> 172.21.15.64:0/2556116316 conn(0x7fa774066d00 msgr2=0x7fa77415b980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 shutdown_connections 2024-03-04T07:27:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.008+0000 7fa77bc66640 1 -- 172.21.15.64:0/2556116316 wait complete. 2024-03-04T07:27:52.019 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":61,"stamp":"2024-03-04T07:27:51.152688+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":66,"acting":66,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":2,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219884,"kb_used_data":3688,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754754836,"statfs":{"total":773094113280,"available":772868952064,"internally_reserved":0,"allocated":3776512,"data_stored":2152328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.167526"},"pg_stats":[{"pgid":"5.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.31","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.25","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.988891+0000","last_change":"2024-03-04T07:27:45.988891+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.988891+0000","last_undegraded":"2024-03-04T07:27:45.988891+0000","last_fullsized":"2024-03-04T07:27:45.988891+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.993160+0000","last_change":"2024-03-04T07:27:45.993160+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.993160+0000","last_undegraded":"2024-03-04T07:27:45.993160+0000","last_fullsized":"2024-03-04T07:27:45.993160+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992251+0000","last_change":"2024-03-04T07:27:45.992251+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992251+0000","last_undegraded":"2024-03-04T07:27:45.992251+0000","last_fullsized":"2024-03-04T07:27:45.992251+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990657+0000","last_change":"2024-03-04T07:27:45.990657+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990657+0000","last_undegraded":"2024-03-04T07:27:45.990657+0000","last_fullsized":"2024-03-04T07:27:45.990657+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986339+0000","last_change":"2024-03-04T07:27:45.986339+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986339+0000","last_undegraded":"2024-03-04T07:27:45.986339+0000","last_fullsized":"2024-03-04T07:27:45.986339+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985443+0000","last_change":"2024-03-04T07:27:45.985443+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985443+0000","last_undegraded":"2024-03-04T07:27:45.985443+0000","last_fullsized":"2024-03-04T07:27:45.985443+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992885+0000","last_change":"2024-03-04T07:27:45.992885+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992885+0000","last_undegraded":"2024-03-04T07:27:45.992885+0000","last_fullsized":"2024-03-04T07:27:45.992885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986132+0000","last_change":"2024-03-04T07:27:45.986132+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986132+0000","last_undegraded":"2024-03-04T07:27:45.986132+0000","last_fullsized":"2024-03-04T07:27:45.986132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992198+0000","last_change":"2024-03-04T07:27:45.992198+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992198+0000","last_undegraded":"2024-03-04T07:27:45.992198+0000","last_fullsized":"2024-03-04T07:27:45.992198+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992700+0000","last_change":"2024-03-04T07:27:45.992700+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992700+0000","last_undegraded":"2024-03-04T07:27:45.992700+0000","last_fullsized":"2024-03-04T07:27:45.992700+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986647+0000","last_change":"2024-03-04T07:27:45.986647+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986647+0000","last_undegraded":"2024-03-04T07:27:45.986647+0000","last_fullsized":"2024-03-04T07:27:45.986647+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991205+0000","last_change":"2024-03-04T07:27:45.991205+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991205+0000","last_undegraded":"2024-03-04T07:27:45.991205+0000","last_fullsized":"2024-03-04T07:27:45.991205+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985450+0000","last_change":"2024-03-04T07:27:45.985450+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985450+0000","last_undegraded":"2024-03-04T07:27:45.985450+0000","last_fullsized":"2024-03-04T07:27:45.985450+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991409+0000","last_change":"2024-03-04T07:27:45.991409+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991409+0000","last_undegraded":"2024-03-04T07:27:45.991409+0000","last_fullsized":"2024-03-04T07:27:45.991409+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992493+0000","last_change":"2024-03-04T07:27:45.992493+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992493+0000","last_undegraded":"2024-03-04T07:27:45.992493+0000","last_fullsized":"2024-03-04T07:27:45.992493+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991194+0000","last_change":"2024-03-04T07:27:45.991194+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991194+0000","last_undegraded":"2024-03-04T07:27:45.991194+0000","last_fullsized":"2024-03-04T07:27:45.991194+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991531+0000","last_change":"2024-03-04T07:27:45.991531+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991531+0000","last_undegraded":"2024-03-04T07:27:45.991531+0000","last_fullsized":"2024-03-04T07:27:45.991531+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985596+0000","last_change":"2024-03-04T07:27:45.985596+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985596+0000","last_undegraded":"2024-03-04T07:27:45.985596+0000","last_fullsized":"2024-03-04T07:27:45.985596+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985840+0000","last_change":"2024-03-04T07:27:45.985840+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985840+0000","last_undegraded":"2024-03-04T07:27:45.985840+0000","last_fullsized":"2024-03-04T07:27:45.985840+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985710+0000","last_change":"2024-03-04T07:27:45.985710+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985710+0000","last_undegraded":"2024-03-04T07:27:45.985710+0000","last_fullsized":"2024-03-04T07:27:45.985710+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991976+0000","last_change":"2024-03-04T07:27:45.991976+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991976+0000","last_undegraded":"2024-03-04T07:27:45.991976+0000","last_fullsized":"2024-03-04T07:27:45.991976+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.988865+0000","last_change":"2024-03-04T07:27:45.988865+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.988865+0000","last_undegraded":"2024-03-04T07:27:45.988865+0000","last_fullsized":"2024-03-04T07:27:45.988865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986913+0000","last_change":"2024-03-04T07:27:45.986913+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986913+0000","last_undegraded":"2024-03-04T07:27:45.986913+0000","last_fullsized":"2024-03-04T07:27:45.986913+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986046+0000","last_change":"2024-03-04T07:27:45.986046+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986046+0000","last_undegraded":"2024-03-04T07:27:45.986046+0000","last_fullsized":"2024-03-04T07:27:45.986046+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991110+0000","last_change":"2024-03-04T07:27:45.991110+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991110+0000","last_undegraded":"2024-03-04T07:27:45.991110+0000","last_fullsized":"2024-03-04T07:27:45.991110+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991462+0000","last_change":"2024-03-04T07:27:45.991462+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991462+0000","last_undegraded":"2024-03-04T07:27:45.991462+0000","last_fullsized":"2024-03-04T07:27:45.991462+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990741+0000","last_change":"2024-03-04T07:27:45.990741+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990741+0000","last_undegraded":"2024-03-04T07:27:45.990741+0000","last_fullsized":"2024-03-04T07:27:45.990741+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990825+0000","last_change":"2024-03-04T07:27:45.990825+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990825+0000","last_undegraded":"2024-03-04T07:27:45.990825+0000","last_fullsized":"2024-03-04T07:27:45.990825+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991025+0000","last_change":"2024-03-04T07:27:45.991025+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991025+0000","last_undegraded":"2024-03-04T07:27:45.991025+0000","last_fullsized":"2024-03-04T07:27:45.991025+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985442+0000","last_change":"2024-03-04T07:27:45.985442+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985442+0000","last_undegraded":"2024-03-04T07:27:45.985442+0000","last_fullsized":"2024-03-04T07:27:45.985442+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990657+0000","last_change":"2024-03-04T07:27:45.990657+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990657+0000","last_undegraded":"2024-03-04T07:27:45.990657+0000","last_fullsized":"2024-03-04T07:27:45.990657+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.989107+0000","last_change":"2024-03-04T07:27:45.989107+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.989107+0000","last_undegraded":"2024-03-04T07:27:45.989107+0000","last_fullsized":"2024-03-04T07:27:45.989107+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":187,"reported_epoch":30,"state":"active+clean","last_fresh":"2024-03-04T07:27:45.989338+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:27:45.989338+0000","last_peered":"2024-03-04T07:27:45.989338+0000","last_clean":"2024-03-04T07:27:45.989338+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:27:45.989338+0000","last_undegraded":"2024-03-04T07:27:45.989338+0000","last_fullsized":"2024-03-04T07:27:45.989338+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574866,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27920,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343920,"statfs":{"total":96636764160,"available":96608174080,"internally_reserved":0,"allocated":917504,"data_stored":711817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27336,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344504,"statfs":{"total":96636764160,"available":96608772096,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574866,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574865,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27916,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343924,"statfs":{"total":96636764160,"available":96608178176,"internally_reserved":0,"allocated":917504,"data_stored":711817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:27:52.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:27:52.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.161+0000 7f06482a4640 1 Processor -- start 2024-03-04T07:27:52.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.162+0000 7f06482a4640 1 -- start start 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f06482a4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f064015df00 0x7f06400d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f06482a4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f06400b95a0 0x7f06400b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f06482a4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f06400cea30 con 0x7f06400b9a30 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f06482a4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f06400d2af0 con 0x7f064015df00 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f06482a4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f06400cf3c0 con 0x7f06400b95a0 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f0646040640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f064015df00 0x7f06400d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f0646841640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f06400b9a30 0x7f06400d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54678/0 (socket says 172.21.15.64:54678) 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f0646841640 1 -- 172.21.15.64:0/3388549564 learned_addr learned my addr 172.21.15.64:0/3388549564 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f064583f640 1 --2- 172.21.15.64:0/3388549564 >> v2:172.21.15.66:3300/0 conn(0x7f06400b95a0 0x7f06400b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f0646040640 1 -- 172.21.15.64:0/3388549564 >> v2:172.21.15.66:3300/0 conn(0x7f06400b95a0 msgr2=0x7f06400b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f0646040640 1 --2- 172.21.15.64:0/3388549564 >> v2:172.21.15.66:3300/0 conn(0x7f06400b95a0 0x7f06400b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f064503e640 1 -- 172.21.15.64:0/3388549564 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2414466248 0 0) 0x7f06400cea30 con 0x7f06400b9a30 2024-03-04T07:27:52.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.163+0000 7f0646040640 1 -- 172.21.15.64:0/3388549564 >> v1:172.21.15.64:6789/0 conn(0x7f06400b9a30 legacy=0x7f06400d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.164+0000 7f0646040640 1 -- 172.21.15.64:0/3388549564 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f06400cf0e0 con 0x7f064015df00 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.165+0000 7f0646040640 1 --2- 172.21.15.64:0/3388549564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015df00 0x7f06400d25b0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f06400d1760 tx=0x7f063c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bc6eea09b9d5935e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.165+0000 7f064503e640 1 -- 172.21.15.64:0/3388549564 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f063c035020 con 0x7f064015df00 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.165+0000 7f064503e640 1 -- 172.21.15.64:0/3388549564 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f063c031830 con 0x7f064015df00 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.165+0000 7f064503e640 1 -- 172.21.15.64:0/3388549564 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f063c031c00 con 0x7f064015df00 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 -- 172.21.15.64:0/3388549564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015df00 msgr2=0x7f06400d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 --2- 172.21.15.64:0/3388549564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015df00 0x7f06400d25b0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f06400d1760 tx=0x7f063c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 -- 172.21.15.64:0/3388549564 shutdown_connections 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 --2- 172.21.15.64:0/3388549564 >> v2:172.21.15.66:3300/0 conn(0x7f06400b95a0 0x7f06400b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 --2- 172.21.15.64:0/3388549564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015df00 0x7f06400d25b0 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 -- 172.21.15.64:0/3388549564 >> 172.21.15.64:0/3388549564 conn(0x7f0640071960 msgr2=0x7f0640071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 -- 172.21.15.64:0/3388549564 shutdown_connections 2024-03-04T07:27:52.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.166+0000 7f06482a4640 1 -- 172.21.15.64:0/3388549564 wait complete. 2024-03-04T07:27:52.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.167+0000 7f06482a4640 1 Processor -- start 2024-03-04T07:27:52.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.168+0000 7f06482a4640 1 -- start start 2024-03-04T07:27:52.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f06482a4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06400b95a0 0x7f06401f4290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f06482a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f064015df00 0x7f06401f0c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f06482a4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015e2e0 0x7f064015a190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f06482a4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06400cec70 con 0x7f064015e2e0 2024-03-04T07:27:52.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f06482a4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06400ce750 con 0x7f06400b95a0 2024-03-04T07:27:52.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f06482a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f06400cedf0 con 0x7f064015df00 2024-03-04T07:27:52.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015e2e0 0x7f064015a190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015e2e0 0x7f064015a190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39008/0 (socket says 172.21.15.64:39008) 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 -- 172.21.15.64:0/681009682 learned_addr learned my addr 172.21.15.64:0/681009682 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646040640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06400b95a0 0x7f06401f4290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 -- 172.21.15.64:0/681009682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f064015df00 msgr2=0x7f06401f0c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f064015df00 0x7f06401f0c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 -- 172.21.15.64:0/681009682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06400b95a0 msgr2=0x7f06401f4290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06400b95a0 0x7f06401f4290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 -- 172.21.15.64:0/681009682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f06401f47d0 con 0x7f064015e2e0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0646841640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015e2e0 0x7f064015a190 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f063800c7a0 tx=0x7f063800cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0636ffd640 1 -- 172.21.15.64:0/681009682 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0638018070 con 0x7f064015e2e0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f0636ffd640 1 -- 172.21.15.64:0/681009682 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f063800a640 con 0x7f064015e2e0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.169+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f064015e7e0 con 0x7f064015e2e0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.170+0000 7f0636ffd640 1 -- 172.21.15.64:0/681009682 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f063800aab0 con 0x7f064015e2e0 2024-03-04T07:27:52.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.170+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f064015eda0 con 0x7f064015e2e0 2024-03-04T07:27:52.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.170+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0640067ff0 con 0x7f064015e2e0 2024-03-04T07:27:52.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.171+0000 7f0636ffd640 1 -- 172.21.15.64:0/681009682 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f063801b070 con 0x7f064015e2e0 2024-03-04T07:27:52.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.171+0000 7f0636ffd640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f062406b4d0 0x7f062406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.172+0000 7f0646040640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f062406b4d0 0x7f062406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.172+0000 7f0636ffd640 1 -- 172.21.15.64:0/681009682 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0638089700 con 0x7f064015e2e0 2024-03-04T07:27:52.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.172+0000 7f0646040640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f062406b4d0 0x7f062406d990 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f063c036000 tx=0x7f063c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.176+0000 7f0636ffd640 1 -- 172.21.15.64:0/681009682 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f06401f47d0 con 0x7f064015e2e0 2024-03-04T07:27:52.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.364+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f0640068700 con 0x7f062406b4d0 2024-03-04T07:27:52.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.370+0000 7f0636ffd640 1 -- 172.21.15.64:0/681009682 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+239842 (secure 0 0 0) 0x7f0640068700 con 0x7f062406b4d0 2024-03-04T07:27:52.371 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:52.372 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:27:52.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f062406b4d0 msgr2=0x7f062406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f062406b4d0 0x7f062406d990 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f063c036000 tx=0x7f063c008030 comp rx=0 tx=0).stop 2024-03-04T07:27:52.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015e2e0 msgr2=0x7f064015a190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015e2e0 0x7f064015a190 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f063800c7a0 tx=0x7f063800cc60 comp rx=0 tx=0).stop 2024-03-04T07:27:52.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 shutdown_connections 2024-03-04T07:27:52.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f062406b4d0 0x7f062406d990 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f064015e2e0 0x7f064015a190 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f064015df00 0x7f06401f0c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 --2- 172.21.15.64:0/681009682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f06400b95a0 0x7f06401f4290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 >> 172.21.15.64:0/681009682 conn(0x7f0640071960 msgr2=0x7f06400b8ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:52.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.372+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 shutdown_connections 2024-03-04T07:27:52.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.373+0000 7f06482a4640 1 -- 172.21.15.64:0/681009682 wait complete. 2024-03-04T07:27:52.383 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":61,"stamp":"2024-03-04T07:27:51.152688+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":66,"acting":66,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":2,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219884,"kb_used_data":3688,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754754836,"statfs":{"total":773094113280,"available":772868952064,"internally_reserved":0,"allocated":3776512,"data_stored":2152328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.167526"},"pg_stats":[{"pgid":"5.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.31","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.25","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.988891+0000","last_change":"2024-03-04T07:27:45.988891+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.988891+0000","last_undegraded":"2024-03-04T07:27:45.988891+0000","last_fullsized":"2024-03-04T07:27:45.988891+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.993160+0000","last_change":"2024-03-04T07:27:45.993160+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.993160+0000","last_undegraded":"2024-03-04T07:27:45.993160+0000","last_fullsized":"2024-03-04T07:27:45.993160+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992251+0000","last_change":"2024-03-04T07:27:45.992251+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992251+0000","last_undegraded":"2024-03-04T07:27:45.992251+0000","last_fullsized":"2024-03-04T07:27:45.992251+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990657+0000","last_change":"2024-03-04T07:27:45.990657+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990657+0000","last_undegraded":"2024-03-04T07:27:45.990657+0000","last_fullsized":"2024-03-04T07:27:45.990657+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986339+0000","last_change":"2024-03-04T07:27:45.986339+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986339+0000","last_undegraded":"2024-03-04T07:27:45.986339+0000","last_fullsized":"2024-03-04T07:27:45.986339+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985443+0000","last_change":"2024-03-04T07:27:45.985443+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985443+0000","last_undegraded":"2024-03-04T07:27:45.985443+0000","last_fullsized":"2024-03-04T07:27:45.985443+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992885+0000","last_change":"2024-03-04T07:27:45.992885+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992885+0000","last_undegraded":"2024-03-04T07:27:45.992885+0000","last_fullsized":"2024-03-04T07:27:45.992885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986132+0000","last_change":"2024-03-04T07:27:45.986132+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986132+0000","last_undegraded":"2024-03-04T07:27:45.986132+0000","last_fullsized":"2024-03-04T07:27:45.986132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992198+0000","last_change":"2024-03-04T07:27:45.992198+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992198+0000","last_undegraded":"2024-03-04T07:27:45.992198+0000","last_fullsized":"2024-03-04T07:27:45.992198+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992700+0000","last_change":"2024-03-04T07:27:45.992700+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992700+0000","last_undegraded":"2024-03-04T07:27:45.992700+0000","last_fullsized":"2024-03-04T07:27:45.992700+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986647+0000","last_change":"2024-03-04T07:27:45.986647+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986647+0000","last_undegraded":"2024-03-04T07:27:45.986647+0000","last_fullsized":"2024-03-04T07:27:45.986647+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991205+0000","last_change":"2024-03-04T07:27:45.991205+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991205+0000","last_undegraded":"2024-03-04T07:27:45.991205+0000","last_fullsized":"2024-03-04T07:27:45.991205+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985450+0000","last_change":"2024-03-04T07:27:45.985450+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985450+0000","last_undegraded":"2024-03-04T07:27:45.985450+0000","last_fullsized":"2024-03-04T07:27:45.985450+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991409+0000","last_change":"2024-03-04T07:27:45.991409+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991409+0000","last_undegraded":"2024-03-04T07:27:45.991409+0000","last_fullsized":"2024-03-04T07:27:45.991409+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992493+0000","last_change":"2024-03-04T07:27:45.992493+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992493+0000","last_undegraded":"2024-03-04T07:27:45.992493+0000","last_fullsized":"2024-03-04T07:27:45.992493+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991194+0000","last_change":"2024-03-04T07:27:45.991194+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991194+0000","last_undegraded":"2024-03-04T07:27:45.991194+0000","last_fullsized":"2024-03-04T07:27:45.991194+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991531+0000","last_change":"2024-03-04T07:27:45.991531+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991531+0000","last_undegraded":"2024-03-04T07:27:45.991531+0000","last_fullsized":"2024-03-04T07:27:45.991531+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985596+0000","last_change":"2024-03-04T07:27:45.985596+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985596+0000","last_undegraded":"2024-03-04T07:27:45.985596+0000","last_fullsized":"2024-03-04T07:27:45.985596+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985840+0000","last_change":"2024-03-04T07:27:45.985840+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985840+0000","last_undegraded":"2024-03-04T07:27:45.985840+0000","last_fullsized":"2024-03-04T07:27:45.985840+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985710+0000","last_change":"2024-03-04T07:27:45.985710+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985710+0000","last_undegraded":"2024-03-04T07:27:45.985710+0000","last_fullsized":"2024-03-04T07:27:45.985710+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991976+0000","last_change":"2024-03-04T07:27:45.991976+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991976+0000","last_undegraded":"2024-03-04T07:27:45.991976+0000","last_fullsized":"2024-03-04T07:27:45.991976+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.988865+0000","last_change":"2024-03-04T07:27:45.988865+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.988865+0000","last_undegraded":"2024-03-04T07:27:45.988865+0000","last_fullsized":"2024-03-04T07:27:45.988865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986913+0000","last_change":"2024-03-04T07:27:45.986913+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986913+0000","last_undegraded":"2024-03-04T07:27:45.986913+0000","last_fullsized":"2024-03-04T07:27:45.986913+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986046+0000","last_change":"2024-03-04T07:27:45.986046+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986046+0000","last_undegraded":"2024-03-04T07:27:45.986046+0000","last_fullsized":"2024-03-04T07:27:45.986046+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991110+0000","last_change":"2024-03-04T07:27:45.991110+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991110+0000","last_undegraded":"2024-03-04T07:27:45.991110+0000","last_fullsized":"2024-03-04T07:27:45.991110+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991462+0000","last_change":"2024-03-04T07:27:45.991462+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991462+0000","last_undegraded":"2024-03-04T07:27:45.991462+0000","last_fullsized":"2024-03-04T07:27:45.991462+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990741+0000","last_change":"2024-03-04T07:27:45.990741+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990741+0000","last_undegraded":"2024-03-04T07:27:45.990741+0000","last_fullsized":"2024-03-04T07:27:45.990741+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990825+0000","last_change":"2024-03-04T07:27:45.990825+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990825+0000","last_undegraded":"2024-03-04T07:27:45.990825+0000","last_fullsized":"2024-03-04T07:27:45.990825+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991025+0000","last_change":"2024-03-04T07:27:45.991025+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991025+0000","last_undegraded":"2024-03-04T07:27:45.991025+0000","last_fullsized":"2024-03-04T07:27:45.991025+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985442+0000","last_change":"2024-03-04T07:27:45.985442+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985442+0000","last_undegraded":"2024-03-04T07:27:45.985442+0000","last_fullsized":"2024-03-04T07:27:45.985442+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990657+0000","last_change":"2024-03-04T07:27:45.990657+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990657+0000","last_undegraded":"2024-03-04T07:27:45.990657+0000","last_fullsized":"2024-03-04T07:27:45.990657+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.989107+0000","last_change":"2024-03-04T07:27:45.989107+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.989107+0000","last_undegraded":"2024-03-04T07:27:45.989107+0000","last_fullsized":"2024-03-04T07:27:45.989107+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":187,"reported_epoch":30,"state":"active+clean","last_fresh":"2024-03-04T07:27:45.989338+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:27:45.989338+0000","last_peered":"2024-03-04T07:27:45.989338+0000","last_clean":"2024-03-04T07:27:45.989338+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:27:45.989338+0000","last_undegraded":"2024-03-04T07:27:45.989338+0000","last_fullsized":"2024-03-04T07:27:45.989338+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574866,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27920,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343920,"statfs":{"total":96636764160,"available":96608174080,"internally_reserved":0,"allocated":917504,"data_stored":711817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27336,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344504,"statfs":{"total":96636764160,"available":96608772096,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574866,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574865,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27916,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343924,"statfs":{"total":96636764160,"available":96608178176,"internally_reserved":0,"allocated":917504,"data_stored":711817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:27:52.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:27:52.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.525+0000 7f5df709a640 1 Processor -- start 2024-03-04T07:27:52.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.525+0000 7f5df709a640 1 -- start start 2024-03-04T07:27:52.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.526+0000 7f5df709a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.526+0000 7f5df709a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5df00b9a30 0x7f5df00d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.526+0000 7f5df709a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5df00cea30 con 0x7f5df00b95a0 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.526+0000 7f5df709a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5df00d2af0 con 0x7f5df00b9a30 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.526+0000 7f5df709a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5df00cf3c0 con 0x7f5df015df00 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df5637640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5df00b9a30 0x7f5df00d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df4e36640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5deffff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f5df00b95a0 0x7f5df00b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:35314/0 (socket says 172.21.15.64:35314) 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5deffff640 1 -- 172.21.15.64:0/906891960 learned_addr learned my addr 172.21.15.64:0/906891960 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df4e36640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df00d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:37950/0 (socket says 172.21.15.64:37950) 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df5637640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5df00b9a30 0x7f5df00d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:54740/0 (socket says 172.21.15.64:54740) 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df4e36640 1 -- 172.21.15.64:0/906891960 >> v2:172.21.15.66:3300/0 conn(0x7f5df00b9a30 msgr2=0x7f5df00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df4e36640 1 --2- 172.21.15.64:0/906891960 >> v2:172.21.15.66:3300/0 conn(0x7f5df00b9a30 0x7f5df00d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df4e36640 1 -- 172.21.15.64:0/906891960 >> v1:172.21.15.66:6789/0 conn(0x7f5df00b95a0 legacy=0x7f5df00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.527+0000 7f5df4e36640 1 -- 172.21.15.64:0/906891960 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5df00cf0e0 con 0x7f5df015df00 2024-03-04T07:27:52.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.528+0000 7f5df4e36640 1 --2- 172.21.15.64:0/906891960 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df00d25b0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5de00097a0 tx=0x7f5de002b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d1159af5fe8e1411 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.528+0000 7f5def7fe640 1 -- 172.21.15.64:0/906891960 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de0035020 con 0x7f5df015df00 2024-03-04T07:27:52.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.528+0000 7f5def7fe640 1 -- 172.21.15.64:0/906891960 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5de0031830 con 0x7f5df015df00 2024-03-04T07:27:52.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.528+0000 7f5def7fe640 1 -- 172.21.15.64:0/906891960 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de0031c00 con 0x7f5df015df00 2024-03-04T07:27:52.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 -- 172.21.15.64:0/906891960 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 msgr2=0x7f5df00d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 --2- 172.21.15.64:0/906891960 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df00d25b0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5de00097a0 tx=0x7f5de002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 -- 172.21.15.64:0/906891960 shutdown_connections 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 --2- 172.21.15.64:0/906891960 >> v2:172.21.15.66:3300/0 conn(0x7f5df00b9a30 0x7f5df00d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 --2- 172.21.15.64:0/906891960 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df00d25b0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 -- 172.21.15.64:0/906891960 >> 172.21.15.64:0/906891960 conn(0x7f5df0071960 msgr2=0x7f5df0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 -- 172.21.15.64:0/906891960 shutdown_connections 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.529+0000 7f5df709a640 1 -- 172.21.15.64:0/906891960 wait complete. 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.530+0000 7f5df709a640 1 Processor -- start 2024-03-04T07:27:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.530+0000 7f5df709a640 1 -- start start 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df709a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5df00b95a0 0x7f5df0160ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df709a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df014ef10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df709a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5df00b3df0 0x7f5df00b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df709a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5df00cec70 con 0x7f5df00b3df0 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df709a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5df00ce750 con 0x7f5df00b95a0 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df709a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5df00cedf0 con 0x7f5df015df00 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5df00b3df0 0x7f5df00b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5df00b3df0 0x7f5df00b4eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39018/0 (socket says 172.21.15.64:39018) 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 -- 172.21.15.64:0/2262182209 learned_addr learned my addr 172.21.15.64:0/2262182209 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5deffff640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df014ef10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df4e36640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5df00b95a0 0x7f5df0160ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 -- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 msgr2=0x7f5df014ef10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df014ef10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 -- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5df00b95a0 msgr2=0x7f5df0160ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5df00b95a0 0x7f5df0160ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 -- 172.21.15.64:0/2262182209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5df015ed90 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df5637640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5df00b3df0 0x7f5df00b4eb0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f5de800efc0 tx=0x7f5de800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5ded7fa640 1 -- 172.21.15.64:0/2262182209 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de8018020 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5ded7fa640 1 -- 172.21.15.64:0/2262182209 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5de8014480 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5df015f070 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.531+0000 7f5ded7fa640 1 -- 172.21.15.64:0/2262182209 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5de8014870 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.532+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5df015f630 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.533+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5df0067ff0 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.533+0000 7f5ded7fa640 1 -- 172.21.15.64:0/2262182209 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5de8005d10 con 0x7f5df00b3df0 2024-03-04T07:27:52.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.533+0000 7f5ded7fa640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ddc06b500 0x7f5ddc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.533+0000 7f5ded7fa640 1 -- 172.21.15.64:0/2262182209 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5de808afa0 con 0x7f5df00b3df0 2024-03-04T07:27:52.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.533+0000 7f5df4e36640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ddc06b500 0x7f5ddc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.534+0000 7f5df4e36640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ddc06b500 0x7f5ddc06d9c0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f5de0036000 tx=0x7f5de003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.538+0000 7f5ded7fa640 1 -- 172.21.15.64:0/2262182209 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5de8022030 con 0x7f5df00b3df0 2024-03-04T07:27:52.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.714+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f5df015caf0 con 0x7f5ddc06b500 2024-03-04T07:27:52.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.720+0000 7f5ded7fa640 1 -- 172.21.15.64:0/2262182209 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+239842 (secure 0 0 0) 0x7f5de0082900 con 0x7f5ddc06b500 2024-03-04T07:27:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:52.723 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:27:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ddc06b500 msgr2=0x7f5ddc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ddc06b500 0x7f5ddc06d9c0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f5de0036000 tx=0x7f5de003c040 comp rx=0 tx=0).stop 2024-03-04T07:27:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5df00b3df0 msgr2=0x7f5df00b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5df00b3df0 0x7f5df00b4eb0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f5de800efc0 tx=0x7f5de800c490 comp rx=0 tx=0).stop 2024-03-04T07:27:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 shutdown_connections 2024-03-04T07:27:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ddc06b500 0x7f5ddc06d9c0 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5df00b3df0 0x7f5df00b4eb0 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5df015df00 0x7f5df014ef10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 --2- 172.21.15.64:0/2262182209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5df00b95a0 0x7f5df0160ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 >> 172.21.15.64:0/2262182209 conn(0x7f5df0071960 msgr2=0x7f5df00b8d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:52.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 shutdown_connections 2024-03-04T07:27:52.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.723+0000 7f5df709a640 1 -- 172.21.15.64:0/2262182209 wait complete. 2024-03-04T07:27:52.733 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":61,"stamp":"2024-03-04T07:27:51.152688+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":66,"acting":66,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":2,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":219884,"kb_used_data":3688,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754754836,"statfs":{"total":773094113280,"available":772868952064,"internally_reserved":0,"allocated":3776512,"data_stored":2152328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.167526"},"pg_stats":[{"pgid":"5.3f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.3a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.39","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.38","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.37","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.36","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.35","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.34","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.33","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.32","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.31","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.30","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.2a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.29","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.28","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.27","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.26","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.25","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.24","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.23","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.22","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.21","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.20","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"5.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.988891+0000","last_change":"2024-03-04T07:27:45.988891+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.988891+0000","last_undegraded":"2024-03-04T07:27:45.988891+0000","last_fullsized":"2024-03-04T07:27:45.988891+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.993160+0000","last_change":"2024-03-04T07:27:45.993160+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.993160+0000","last_undegraded":"2024-03-04T07:27:45.993160+0000","last_fullsized":"2024-03-04T07:27:45.993160+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992251+0000","last_change":"2024-03-04T07:27:45.992251+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992251+0000","last_undegraded":"2024-03-04T07:27:45.992251+0000","last_fullsized":"2024-03-04T07:27:45.992251+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990657+0000","last_change":"2024-03-04T07:27:45.990657+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990657+0000","last_undegraded":"2024-03-04T07:27:45.990657+0000","last_fullsized":"2024-03-04T07:27:45.990657+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986339+0000","last_change":"2024-03-04T07:27:45.986339+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986339+0000","last_undegraded":"2024-03-04T07:27:45.986339+0000","last_fullsized":"2024-03-04T07:27:45.986339+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985443+0000","last_change":"2024-03-04T07:27:45.985443+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985443+0000","last_undegraded":"2024-03-04T07:27:45.985443+0000","last_fullsized":"2024-03-04T07:27:45.985443+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992885+0000","last_change":"2024-03-04T07:27:45.992885+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992885+0000","last_undegraded":"2024-03-04T07:27:45.992885+0000","last_fullsized":"2024-03-04T07:27:45.992885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986132+0000","last_change":"2024-03-04T07:27:45.986132+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986132+0000","last_undegraded":"2024-03-04T07:27:45.986132+0000","last_fullsized":"2024-03-04T07:27:45.986132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992198+0000","last_change":"2024-03-04T07:27:45.992198+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992198+0000","last_undegraded":"2024-03-04T07:27:45.992198+0000","last_fullsized":"2024-03-04T07:27:45.992198+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992700+0000","last_change":"2024-03-04T07:27:45.992700+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992700+0000","last_undegraded":"2024-03-04T07:27:45.992700+0000","last_fullsized":"2024-03-04T07:27:45.992700+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986647+0000","last_change":"2024-03-04T07:27:45.986647+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986647+0000","last_undegraded":"2024-03-04T07:27:45.986647+0000","last_fullsized":"2024-03-04T07:27:45.986647+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991205+0000","last_change":"2024-03-04T07:27:45.991205+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991205+0000","last_undegraded":"2024-03-04T07:27:45.991205+0000","last_fullsized":"2024-03-04T07:27:45.991205+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985450+0000","last_change":"2024-03-04T07:27:45.985450+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985450+0000","last_undegraded":"2024-03-04T07:27:45.985450+0000","last_fullsized":"2024-03-04T07:27:45.985450+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991409+0000","last_change":"2024-03-04T07:27:45.991409+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991409+0000","last_undegraded":"2024-03-04T07:27:45.991409+0000","last_fullsized":"2024-03-04T07:27:45.991409+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.992493+0000","last_change":"2024-03-04T07:27:45.992493+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.992493+0000","last_undegraded":"2024-03-04T07:27:45.992493+0000","last_fullsized":"2024-03-04T07:27:45.992493+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991194+0000","last_change":"2024-03-04T07:27:45.991194+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991194+0000","last_undegraded":"2024-03-04T07:27:45.991194+0000","last_fullsized":"2024-03-04T07:27:45.991194+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991531+0000","last_change":"2024-03-04T07:27:45.991531+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991531+0000","last_undegraded":"2024-03-04T07:27:45.991531+0000","last_fullsized":"2024-03-04T07:27:45.991531+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985596+0000","last_change":"2024-03-04T07:27:45.985596+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985596+0000","last_undegraded":"2024-03-04T07:27:45.985596+0000","last_fullsized":"2024-03-04T07:27:45.985596+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985840+0000","last_change":"2024-03-04T07:27:45.985840+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985840+0000","last_undegraded":"2024-03-04T07:27:45.985840+0000","last_fullsized":"2024-03-04T07:27:45.985840+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985710+0000","last_change":"2024-03-04T07:27:45.985710+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985710+0000","last_undegraded":"2024-03-04T07:27:45.985710+0000","last_fullsized":"2024-03-04T07:27:45.985710+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991976+0000","last_change":"2024-03-04T07:27:45.991976+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991976+0000","last_undegraded":"2024-03-04T07:27:45.991976+0000","last_fullsized":"2024-03-04T07:27:45.991976+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.988865+0000","last_change":"2024-03-04T07:27:45.988865+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.988865+0000","last_undegraded":"2024-03-04T07:27:45.988865+0000","last_fullsized":"2024-03-04T07:27:45.988865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986913+0000","last_change":"2024-03-04T07:27:45.986913+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986913+0000","last_undegraded":"2024-03-04T07:27:45.986913+0000","last_fullsized":"2024-03-04T07:27:45.986913+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.986046+0000","last_change":"2024-03-04T07:27:45.986046+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.986046+0000","last_undegraded":"2024-03-04T07:27:45.986046+0000","last_fullsized":"2024-03-04T07:27:45.986046+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991110+0000","last_change":"2024-03-04T07:27:45.991110+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991110+0000","last_undegraded":"2024-03-04T07:27:45.991110+0000","last_fullsized":"2024-03-04T07:27:45.991110+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991462+0000","last_change":"2024-03-04T07:27:45.991462+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991462+0000","last_undegraded":"2024-03-04T07:27:45.991462+0000","last_fullsized":"2024-03-04T07:27:45.991462+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990741+0000","last_change":"2024-03-04T07:27:45.990741+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990741+0000","last_undegraded":"2024-03-04T07:27:45.990741+0000","last_fullsized":"2024-03-04T07:27:45.990741+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990825+0000","last_change":"2024-03-04T07:27:45.990825+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990825+0000","last_undegraded":"2024-03-04T07:27:45.990825+0000","last_fullsized":"2024-03-04T07:27:45.990825+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.991025+0000","last_change":"2024-03-04T07:27:45.991025+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.991025+0000","last_undegraded":"2024-03-04T07:27:45.991025+0000","last_fullsized":"2024-03-04T07:27:45.991025+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.985442+0000","last_change":"2024-03-04T07:27:45.985442+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.985442+0000","last_undegraded":"2024-03-04T07:27:45.985442+0000","last_fullsized":"2024-03-04T07:27:45.985442+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.990657+0000","last_change":"2024-03-04T07:27:45.990657+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.990657+0000","last_undegraded":"2024-03-04T07:27:45.990657+0000","last_fullsized":"2024-03-04T07:27:45.990657+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-03-04T07:27:46.982587+0000","last_change":"2024-03-04T07:27:46.982587+0000","last_active":"2024-03-04T07:27:46.982587+0000","last_peered":"2024-03-04T07:27:46.982587+0000","last_clean":"2024-03-04T07:27:46.982587+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:46.982587+0000","last_undegraded":"2024-03-04T07:27:46.982587+0000","last_fullsized":"2024-03-04T07:27:46.982587+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1,"reported_epoch":30,"state":"creating+peering","last_fresh":"2024-03-04T07:27:45.989107+0000","last_change":"2024-03-04T07:27:45.989107+0000","last_active":"2024-03-04T07:27:45.976127+0000","last_peered":"2024-03-04T07:27:45.976127+0000","last_clean":"2024-03-04T07:27:45.976127+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:27:45.989107+0000","last_undegraded":"2024-03-04T07:27:45.989107+0000","last_fullsized":"2024-03-04T07:27:45.989107+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":187,"reported_epoch":30,"state":"active+clean","last_fresh":"2024-03-04T07:27:45.989338+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:27:45.989338+0000","last_peered":"2024-03-04T07:27:45.989338+0000","last_clean":"2024-03-04T07:27:45.989338+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:27:45.989338+0000","last_undegraded":"2024-03-04T07:27:45.989338+0000","last_fullsized":"2024-03-04T07:27:45.989338+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574866,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27920,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343920,"statfs":{"total":96636764160,"available":96608174080,"internally_reserved":0,"allocated":917504,"data_stored":711817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27336,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344504,"statfs":{"total":96636764160,"available":96608772096,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574866,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":316,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":323584,"data_stored":121449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574865,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27916,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343924,"statfs":{"total":96636764160,"available":96608178176,"internally_reserved":0,"allocated":917504,"data_stored":711817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:27:52.737 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.3f is not active+clean 2024-03-04T07:27:52.737 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.3f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.737 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.3e is not active+clean 2024-03-04T07:27:52.737 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.3e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.737 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.3d is not active+clean 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.3d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.3c is not active+clean 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.3c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.3b is not active+clean 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.3b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.3a is not active+clean 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.3a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.39 is not active+clean 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.39', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.38 is not active+clean 2024-03-04T07:27:52.738 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.38', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.37 is not active+clean 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.37', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.36 is not active+clean 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.36', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.35 is not active+clean 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.35', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.34 is not active+clean 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.34', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.33 is not active+clean 2024-03-04T07:27:52.739 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.33', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.740 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.32 is not active+clean 2024-03-04T07:27:52.740 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.32', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.740 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.31 is not active+clean 2024-03-04T07:27:52.740 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.31', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.740 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.30 is not active+clean 2024-03-04T07:27:52.740 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.30', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.741 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.2f is not active+clean 2024-03-04T07:27:52.741 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.2f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.741 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.2e is not active+clean 2024-03-04T07:27:52.741 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.2e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.743 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.2d is not active+clean 2024-03-04T07:27:52.743 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.2d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.743 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.2c is not active+clean 2024-03-04T07:27:52.743 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.2c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.744 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.2b is not active+clean 2024-03-04T07:27:52.744 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.2b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.744 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.2a is not active+clean 2024-03-04T07:27:52.744 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.2a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.744 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.29 is not active+clean 2024-03-04T07:27:52.744 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.29', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.745 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.28 is not active+clean 2024-03-04T07:27:52.745 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.28', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.745 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.27 is not active+clean 2024-03-04T07:27:52.745 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.27', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.745 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.26 is not active+clean 2024-03-04T07:27:52.745 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.26', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.746 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.25 is not active+clean 2024-03-04T07:27:52.746 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.25', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.746 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.24 is not active+clean 2024-03-04T07:27:52.746 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.24', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.746 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.23 is not active+clean 2024-03-04T07:27:52.746 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.23', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.747 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.22 is not active+clean 2024-03-04T07:27:52.747 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.22', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.747 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.21 is not active+clean 2024-03-04T07:27:52.747 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.21', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.747 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.20 is not active+clean 2024-03-04T07:27:52.747 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.20', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.1e is not active+clean 2024-03-04T07:27:52.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.1e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.1f is not active+clean 2024-03-04T07:27:52.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.1f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.988891+0000', 'last_change': '2024-03-04T07:27:45.988891+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.988891+0000', 'last_undegraded': '2024-03-04T07:27:45.988891+0000', 'last_fullsized': '2024-03-04T07:27:45.988891+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 5], 'acting': [6, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-04T07:27:52.748 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.1f is not active+clean 2024-03-04T07:27:52.748 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.1f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.1e is not active+clean 2024-03-04T07:27:52.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.1e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.993160+0000', 'last_change': '2024-03-04T07:27:45.993160+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.993160+0000', 'last_undegraded': '2024-03-04T07:27:45.993160+0000', 'last_fullsized': '2024-03-04T07:27:45.993160+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 7], 'acting': [0, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:27:52.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.1c is not active+clean 2024-03-04T07:27:52.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.1c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.1d is not active+clean 2024-03-04T07:27:52.749 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.1d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.992251+0000', 'last_change': '2024-03-04T07:27:45.992251+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.992251+0000', 'last_undegraded': '2024-03-04T07:27:45.992251+0000', 'last_fullsized': '2024-03-04T07:27:45.992251+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 1], 'acting': [3, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:27:52.749 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.1d is not active+clean 2024-03-04T07:27:52.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.1d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.1c is not active+clean 2024-03-04T07:27:52.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.1c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.990657+0000', 'last_change': '2024-03-04T07:27:45.990657+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.990657+0000', 'last_undegraded': '2024-03-04T07:27:45.990657+0000', 'last_fullsized': '2024-03-04T07:27:45.990657+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-04T07:27:52.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.1a is not active+clean 2024-03-04T07:27:52.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.1a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.750 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.1b is not active+clean 2024-03-04T07:27:52.750 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.1b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.986339+0000', 'last_change': '2024-03-04T07:27:45.986339+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.986339+0000', 'last_undegraded': '2024-03-04T07:27:45.986339+0000', 'last_fullsized': '2024-03-04T07:27:45.986339+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 3], 'acting': [4, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.1b is not active+clean 2024-03-04T07:27:52.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.1b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.1a is not active+clean 2024-03-04T07:27:52.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.1a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.985443+0000', 'last_change': '2024-03-04T07:27:45.985443+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.985443+0000', 'last_undegraded': '2024-03-04T07:27:45.985443+0000', 'last_fullsized': '2024-03-04T07:27:45.985443+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 3], 'acting': [4, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.751 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.18 is not active+clean 2024-03-04T07:27:52.751 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.18', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.19 is not active+clean 2024-03-04T07:27:52.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.19', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.992885+0000', 'last_change': '2024-03-04T07:27:45.992885+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.992885+0000', 'last_undegraded': '2024-03-04T07:27:45.992885+0000', 'last_fullsized': '2024-03-04T07:27:45.992885+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 2], 'acting': [3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:27:52.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.19 is not active+clean 2024-03-04T07:27:52.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.19', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.752 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.18 is not active+clean 2024-03-04T07:27:52.752 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.18', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.986132+0000', 'last_change': '2024-03-04T07:27:45.986132+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.986132+0000', 'last_undegraded': '2024-03-04T07:27:45.986132+0000', 'last_fullsized': '2024-03-04T07:27:45.986132+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 6], 'acting': [4, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.16 is not active+clean 2024-03-04T07:27:52.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.16', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.17 is not active+clean 2024-03-04T07:27:52.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.17', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.992198+0000', 'last_change': '2024-03-04T07:27:45.992198+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.992198+0000', 'last_undegraded': '2024-03-04T07:27:45.992198+0000', 'last_fullsized': '2024-03-04T07:27:45.992198+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 7], 'acting': [3, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:27:52.753 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.17 is not active+clean 2024-03-04T07:27:52.753 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.17', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.16 is not active+clean 2024-03-04T07:27:52.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.16', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.992700+0000', 'last_change': '2024-03-04T07:27:45.992700+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.992700+0000', 'last_undegraded': '2024-03-04T07:27:45.992700+0000', 'last_fullsized': '2024-03-04T07:27:45.992700+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 3], 'acting': [0, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:27:52.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.14 is not active+clean 2024-03-04T07:27:52.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.14', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.15 is not active+clean 2024-03-04T07:27:52.754 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.15', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.986647+0000', 'last_change': '2024-03-04T07:27:45.986647+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.986647+0000', 'last_undegraded': '2024-03-04T07:27:45.986647+0000', 'last_fullsized': '2024-03-04T07:27:45.986647+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 7], 'acting': [5, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:27:52.754 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.15 is not active+clean 2024-03-04T07:27:52.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.15', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.14 is not active+clean 2024-03-04T07:27:52.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.14', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991205+0000', 'last_change': '2024-03-04T07:27:45.991205+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991205+0000', 'last_undegraded': '2024-03-04T07:27:45.991205+0000', 'last_fullsized': '2024-03-04T07:27:45.991205+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 1], 'acting': [3, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:27:52.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.12 is not active+clean 2024-03-04T07:27:52.755 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.12', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.755 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.13 is not active+clean 2024-03-04T07:27:52.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.13', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.985450+0000', 'last_change': '2024-03-04T07:27:45.985450+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.985450+0000', 'last_undegraded': '2024-03-04T07:27:45.985450+0000', 'last_fullsized': '2024-03-04T07:27:45.985450+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 6], 'acting': [4, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.13 is not active+clean 2024-03-04T07:27:52.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.13', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.12 is not active+clean 2024-03-04T07:27:52.756 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.12', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991409+0000', 'last_change': '2024-03-04T07:27:45.991409+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991409+0000', 'last_undegraded': '2024-03-04T07:27:45.991409+0000', 'last_fullsized': '2024-03-04T07:27:45.991409+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 6], 'acting': [1, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:27:52.756 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.10 is not active+clean 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.10', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.11 is not active+clean 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.11', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.992493+0000', 'last_change': '2024-03-04T07:27:45.992493+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.992493+0000', 'last_undegraded': '2024-03-04T07:27:45.992493+0000', 'last_fullsized': '2024-03-04T07:27:45.992493+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 7], 'acting': [3, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.11 is not active+clean 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.11', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.10 is not active+clean 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.10', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991194+0000', 'last_change': '2024-03-04T07:27:45.991194+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991194+0000', 'last_undegraded': '2024-03-04T07:27:45.991194+0000', 'last_fullsized': '2024-03-04T07:27:45.991194+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 1], 'acting': [3, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.e is not active+clean 2024-03-04T07:27:52.757 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.e', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.f is not active+clean 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991531+0000', 'last_change': '2024-03-04T07:27:45.991531+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991531+0000', 'last_undegraded': '2024-03-04T07:27:45.991531+0000', 'last_fullsized': '2024-03-04T07:27:45.991531+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 3], 'acting': [1, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.f is not active+clean 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.f', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.e is not active+clean 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.985596+0000', 'last_change': '2024-03-04T07:27:45.985596+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.985596+0000', 'last_undegraded': '2024-03-04T07:27:45.985596+0000', 'last_fullsized': '2024-03-04T07:27:45.985596+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 6], 'acting': [4, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.c is not active+clean 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.c', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.d is not active+clean 2024-03-04T07:27:52.758 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.985840+0000', 'last_change': '2024-03-04T07:27:45.985840+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.985840+0000', 'last_undegraded': '2024-03-04T07:27:45.985840+0000', 'last_fullsized': '2024-03-04T07:27:45.985840+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 2], 'acting': [4, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.d is not active+clean 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.d', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.c is not active+clean 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.985710+0000', 'last_change': '2024-03-04T07:27:45.985710+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.985710+0000', 'last_undegraded': '2024-03-04T07:27:45.985710+0000', 'last_fullsized': '2024-03-04T07:27:45.985710+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 3], 'acting': [4, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.a is not active+clean 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.a', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.b is not active+clean 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991976+0000', 'last_change': '2024-03-04T07:27:45.991976+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991976+0000', 'last_undegraded': '2024-03-04T07:27:45.991976+0000', 'last_fullsized': '2024-03-04T07:27:45.991976+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 1], 'acting': [0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.b is not active+clean 2024-03-04T07:27:52.759 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.b', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.a is not active+clean 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.988865+0000', 'last_change': '2024-03-04T07:27:45.988865+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.988865+0000', 'last_undegraded': '2024-03-04T07:27:45.988865+0000', 'last_fullsized': '2024-03-04T07:27:45.988865+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 1], 'acting': [6, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.8 is not active+clean 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.8', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.9 is not active+clean 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.9', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.986913+0000', 'last_change': '2024-03-04T07:27:45.986913+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.986913+0000', 'last_undegraded': '2024-03-04T07:27:45.986913+0000', 'last_fullsized': '2024-03-04T07:27:45.986913+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 1], 'acting': [4, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.9 is not active+clean 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.9', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.8 is not active+clean 2024-03-04T07:27:52.760 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.8', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.986046+0000', 'last_change': '2024-03-04T07:27:45.986046+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.986046+0000', 'last_undegraded': '2024-03-04T07:27:45.986046+0000', 'last_fullsized': '2024-03-04T07:27:45.986046+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 7], 'acting': [5, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.6 is not active+clean 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.6', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.7 is not active+clean 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.7', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991110+0000', 'last_change': '2024-03-04T07:27:45.991110+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991110+0000', 'last_undegraded': '2024-03-04T07:27:45.991110+0000', 'last_fullsized': '2024-03-04T07:27:45.991110+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 5], 'acting': [1, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.7 is not active+clean 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.7', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.6 is not active+clean 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.6', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991462+0000', 'last_change': '2024-03-04T07:27:45.991462+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991462+0000', 'last_undegraded': '2024-03-04T07:27:45.991462+0000', 'last_fullsized': '2024-03-04T07:27:45.991462+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 1], 'acting': [0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.5 is not active+clean 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.5', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.761 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.4 is not active+clean 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.4', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.990741+0000', 'last_change': '2024-03-04T07:27:45.990741+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.990741+0000', 'last_undegraded': '2024-03-04T07:27:45.990741+0000', 'last_fullsized': '2024-03-04T07:27:45.990741+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 2], 'acting': [1, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.2 is not active+clean 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.2', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.3 is not active+clean 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.3', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.990825+0000', 'last_change': '2024-03-04T07:27:45.990825+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.990825+0000', 'last_undegraded': '2024-03-04T07:27:45.990825+0000', 'last_fullsized': '2024-03-04T07:27:45.990825+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 5], 'acting': [0, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.3 is not active+clean 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.3', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.2 is not active+clean 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.2', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.991025+0000', 'last_change': '2024-03-04T07:27:45.991025+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.991025+0000', 'last_undegraded': '2024-03-04T07:27:45.991025+0000', 'last_fullsized': '2024-03-04T07:27:45.991025+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 5], 'acting': [1, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:27:52.762 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.0 is not active+clean 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.0', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.1 is not active+clean 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.1', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.985442+0000', 'last_change': '2024-03-04T07:27:45.985442+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.985442+0000', 'last_undegraded': '2024-03-04T07:27:45.985442+0000', 'last_fullsized': '2024-03-04T07:27:45.985442+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 5], 'acting': [4, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.1 is not active+clean 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.1', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.0 is not active+clean 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.0', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.990657+0000', 'last_change': '2024-03-04T07:27:45.990657+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.990657+0000', 'last_undegraded': '2024-03-04T07:27:45.990657+0000', 'last_fullsized': '2024-03-04T07:27:45.990657+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 7], 'acting': [3, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:PG 5.4 is not active+clean 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '5.4', 'version': "0'0", 'reported_seq': 0, 'reported_epoch': 0, 'state': 'unknown', 'last_fresh': '2024-03-04T07:27:46.982587+0000', 'last_change': '2024-03-04T07:27:46.982587+0000', 'last_active': '2024-03-04T07:27:46.982587+0000', 'last_peered': '2024-03-04T07:27:46.982587+0000', 'last_clean': '2024-03-04T07:27:46.982587+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:46.982587+0000', 'last_undegraded': '2024-03-04T07:27:46.982587+0000', 'last_fullsized': '2024-03-04T07:27:46.982587+0000', 'mapping_epoch': 0, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 0, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:46.982587+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': '--', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [], 'acting': [], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': -1, 'acting_primary': -1, 'purged_snaps': []} 2024-03-04T07:27:52.763 INFO:tasks.cephfs.filesystem.ceph_manager:PG 4.5 is not active+clean 2024-03-04T07:27:52.764 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '4.5', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 30, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:27:45.989107+0000', 'last_change': '2024-03-04T07:27:45.989107+0000', 'last_active': '2024-03-04T07:27:45.976127+0000', 'last_peered': '2024-03-04T07:27:45.976127+0000', 'last_clean': '2024-03-04T07:27:45.976127+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:27:45.989107+0000', 'last_undegraded': '2024-03-04T07:27:45.989107+0000', 'last_fullsized': '2024-03-04T07:27:45.989107+0000', 'mapping_epoch': 30, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 30, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'last_clean_scrub_stamp': '2024-03-04T07:27:45.976127+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 4], 'acting': [6, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-04T07:27:52.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json 2024-03-04T07:27:52.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.874+0000 7f5aea15c640 1 Processor -- start 2024-03-04T07:27:52.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.874+0000 7f5aea15c640 1 -- start start 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5aea15c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5ae40b95a0 0x7f5ae40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5aea15c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 0x7f5ae40b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5aea15c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5ae40cea30 con 0x7f5ae415df00 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5aea15c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5ae40cf3c0 con 0x7f5ae40b95a0 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5aea15c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5ae40cf0e0 con 0x7f5ae40b9a10 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5ae40b95a0 0x7f5ae40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5ae40b95a0 0x7f5ae40b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39022/0 (socket says 172.21.15.64:39022) 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae2ffd640 1 -- 172.21.15.64:0/4234012461 learned_addr learned my addr 172.21.15.64:0/4234012461 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:52.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae3fff640 1 --2- 172.21.15.64:0/4234012461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 0x7f5ae40b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae2ffd640 1 -- 172.21.15.64:0/4234012461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 msgr2=0x7f5ae40b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:52.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae2ffd640 1 --2- 172.21.15.64:0/4234012461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 0x7f5ae40b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae2ffd640 1 -- 172.21.15.64:0/4234012461 >> v1:172.21.15.66:6789/0 conn(0x7f5ae415df00 legacy=0x7f5ae40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:52.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.876+0000 7f5ae2ffd640 1 -- 172.21.15.64:0/4234012461 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ae40ce470 con 0x7f5ae40b95a0 2024-03-04T07:27:52.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.877+0000 7f5ae2ffd640 1 --2- 172.21.15.64:0/4234012461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae40b5700 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f5ad4009870 tx=0x7f5ad402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=463c49bca72a14a1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.877+0000 7f5ae27fc640 1 -- 172.21.15.64:0/4234012461 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad4034070 con 0x7f5ae40b95a0 2024-03-04T07:27:52.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.877+0000 7f5ae27fc640 1 -- 172.21.15.64:0/4234012461 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5ad4030840 con 0x7f5ae40b95a0 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.877+0000 7f5ae27fc640 1 -- 172.21.15.64:0/4234012461 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad4030c10 con 0x7f5ae40b95a0 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 -- 172.21.15.64:0/4234012461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 msgr2=0x7f5ae40b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 --2- 172.21.15.64:0/4234012461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae40b5700 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f5ad4009870 tx=0x7f5ad402b980 comp rx=0 tx=0).stop 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 -- 172.21.15.64:0/4234012461 shutdown_connections 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 --2- 172.21.15.64:0/4234012461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 0x7f5ae40b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 --2- 172.21.15.64:0/4234012461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae40b5700 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 -- 172.21.15.64:0/4234012461 >> 172.21.15.64:0/4234012461 conn(0x7f5ae4071960 msgr2=0x7f5ae4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 -- 172.21.15.64:0/4234012461 shutdown_connections 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.878+0000 7f5aea15c640 1 -- 172.21.15.64:0/4234012461 wait complete. 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.879+0000 7f5aea15c640 1 Processor -- start 2024-03-04T07:27:52.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.879+0000 7f5aea15c640 1 -- start start 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae415e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 0x7f5ae40cd5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ae415df00 0x7f5ae414b020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ae40cedf0 con 0x7f5ae40b95a0 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ae40cec70 con 0x7f5ae415df00 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5ae40d3220 con 0x7f5ae40b9a10 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae415e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae415e2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39026/0 (socket says 172.21.15.64:39026) 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 -- 172.21.15.64:0/1084209938 learned_addr learned my addr 172.21.15.64:0/1084209938 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:52.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae3fff640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ae415df00 0x7f5ae414b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 -- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 msgr2=0x7f5ae40cd5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 0x7f5ae40cd5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 -- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ae415df00 msgr2=0x7f5ae414b020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ae415df00 0x7f5ae414b020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 -- 172.21.15.64:0/1084209938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ae415ea30 con 0x7f5ae40b95a0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae3fff640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ae415df00 0x7f5ae414b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5ae37fe640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae415e2e0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f5ad800c3c0 tx=0x7f5ad800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5acbfff640 1 -- 172.21.15.64:0/1084209938 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad80052d0 con 0x7f5ae40b95a0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5acbfff640 1 -- 172.21.15.64:0/1084209938 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5ad8005470 con 0x7f5ae40b95a0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5ae40cc810 con 0x7f5ae40b95a0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5acbfff640 1 -- 172.21.15.64:0/1084209938 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad8007260 con 0x7f5ae40b95a0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.880+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ae40ccdd0 con 0x7f5ae40b95a0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.881+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5ae4067ff0 con 0x7f5ae40b95a0 2024-03-04T07:27:52.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.882+0000 7f5acbfff640 1 -- 172.21.15.64:0/1084209938 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5ad8007400 con 0x7f5ae40b95a0 2024-03-04T07:27:52.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.882+0000 7f5acbfff640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ad006b500 0x7f5ad006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:52.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.882+0000 7f5acbfff640 1 -- 172.21.15.64:0/1084209938 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5ad8089630 con 0x7f5ae40b95a0 2024-03-04T07:27:52.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.882+0000 7f5ae2ffd640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ad006b500 0x7f5ad006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:52.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.883+0000 7f5ae2ffd640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ad006b500 0x7f5ad006d9c0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f5ad4036000 tx=0x7f5ad403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:52.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:52.887+0000 7f5acbfff640 1 -- 172.21.15.64:0/1084209938 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5ad8058540 con 0x7f5ae40b95a0 2024-03-04T07:27:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.137+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "status", "format": "json"} v 0) v1 -- 0x7f5ae40c71e0 con 0x7f5ae40b95a0 2024-03-04T07:27:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.138+0000 7f5acbfff640 1 -- 172.21.15.64:0/1084209938 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json"}]=0 v0) v1 ==== 72+0+1656 (secure 0 0 0) 0x7f5ad8057e40 con 0x7f5ae40b95a0 2024-03-04T07:27:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:53.138 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":8,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":84,"monmap":{"epoch":1,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":33,"num_osds":8,"num_up_osds":8,"osd_up_since":1709537200,"num_in_osds":8,"osd_in_since":1709537191,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"unknown","count":64},{"state_name":"creating+peering","count":32},{"state_name":"active+clean","count":1}],"num_pgs":97,"num_pools":3,"num_objects":2,"data_bytes":590368,"bytes_used":225161216,"bytes_avail":772868952064,"bytes_total":773094113280,"unknown_pgs_ratio":0.65979379415512085,"inactive_pgs_ratio":0.32989689707756042},"fsmap":{"epoch":15,"id":2,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":2,"rank":0,"name":"a","status":"up:active","gid":5169}],"up:standby":3},"mgrmap":{"available":true,"num_standbys":1,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":3,"modified":"2024-03-04T07:27:43.151509+0000","services":{"mds":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"b":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"c":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"d":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-03-04T07:27:53.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ad006b500 msgr2=0x7f5ad006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ad006b500 0x7f5ad006d9c0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f5ad4036000 tx=0x7f5ad403b040 comp rx=0 tx=0).stop 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 msgr2=0x7f5ae415e2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae415e2e0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f5ad800c3c0 tx=0x7f5ad800c880 comp rx=0 tx=0).stop 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 shutdown_connections 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ad006b500 0x7f5ad006d9c0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ae415df00 0x7f5ae414b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ae40b9a10 0x7f5ae40cd5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 --2- 172.21.15.64:0/1084209938 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ae40b95a0 0x7f5ae415e2e0 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 >> 172.21.15.64:0/1084209938 conn(0x7f5ae4071960 msgr2=0x7f5ae40b6d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 shutdown_connections 2024-03-04T07:27:53.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.140+0000 7f5aea15c640 1 -- 172.21.15.64:0/1084209938 wait complete. 2024-03-04T07:27:53.153 INFO:tasks.cephfs.filesystem.ceph_manager:no progress seen, keeping timeout for now 2024-03-04T07:27:53.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:27:53.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.256+0000 7fc2b8c9a640 1 Processor -- start 2024-03-04T07:27:53.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.256+0000 7fc2b8c9a640 1 -- start start 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2b8c9a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc2b415df00 0x7fc2b40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2b8c9a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc2b40b95a0 0x7fc2b40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2b8c9a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc2b40cea30 con 0x7fc2b40b9a30 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2b8c9a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc2b40d2af0 con 0x7fc2b40b95a0 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2b8c9a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc2b40cf3c0 con 0x7fc2b415df00 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2a9d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc2b40b95a0 0x7fc2b40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2a9d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc2b40b95a0 0x7fc2b40b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39028/0 (socket says 172.21.15.64:39028) 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2a9d9b640 1 -- 172.21.15.64:0/2956890148 learned_addr learned my addr 172.21.15.64:0/2956890148 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2b259c640 1 --2- 172.21.15.64:0/2956890148 >> v2:172.21.15.66:3300/0 conn(0x7fc2b415df00 0x7fc2b40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2a9d9b640 1 -- 172.21.15.64:0/2956890148 >> v2:172.21.15.66:3300/0 conn(0x7fc2b415df00 msgr2=0x7fc2b40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2a9d9b640 1 --2- 172.21.15.64:0/2956890148 >> v2:172.21.15.66:3300/0 conn(0x7fc2b415df00 0x7fc2b40d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2a9d9b640 1 -- 172.21.15.64:0/2956890148 >> v1:172.21.15.66:6789/0 conn(0x7fc2b40b9a30 legacy=0x7fc2b40d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.257+0000 7fc2a9d9b640 1 -- 172.21.15.64:0/2956890148 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc2b40cf0e0 con 0x7fc2b40b95a0 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2a9d9b640 1 --2- 172.21.15.64:0/2956890148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b40b95a0 0x7fc2b40b5700 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7fc2b40d11b0 tx=0x7fc29c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=35707372fd95f25b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b1d9b640 1 -- 172.21.15.64:0/2956890148 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc29c035020 con 0x7fc2b40b95a0 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b1d9b640 1 -- 172.21.15.64:0/2956890148 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc29c031870 con 0x7fc2b40b95a0 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b1d9b640 1 -- 172.21.15.64:0/2956890148 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc29c031c40 con 0x7fc2b40b95a0 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2956890148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b40b95a0 msgr2=0x7fc2b40b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2956890148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b40b95a0 0x7fc2b40b5700 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7fc2b40d11b0 tx=0x7fc29c02b910 comp rx=0 tx=0).stop 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2956890148 shutdown_connections 2024-03-04T07:27:53.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2956890148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b40b95a0 0x7fc2b40b5700 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2956890148 >> v2:172.21.15.66:3300/0 conn(0x7fc2b415df00 0x7fc2b40d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.258+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2956890148 >> 172.21.15.64:0/2956890148 conn(0x7fc2b4071960 msgr2=0x7fc2b4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.259+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2956890148 shutdown_connections 2024-03-04T07:27:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.259+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2956890148 wait complete. 2024-03-04T07:27:53.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.260+0000 7fc2b8c9a640 1 Processor -- start 2024-03-04T07:27:53.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.260+0000 7fc2b8c9a640 1 -- start start 2024-03-04T07:27:53.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc2b40b95a0 0x7fc2b415e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:53.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc2b415df00 0x7fc2b415f420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:53.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b415f960 0x7fc2b40b8080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:53.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc2b40cec70 con 0x7fc2b415f960 2024-03-04T07:27:53.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc2b40ce750 con 0x7fc2b415df00 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc2b40cedf0 con 0x7fc2b40b95a0 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b415f960 0x7fc2b40b8080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b415f960 0x7fc2b40b8080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39030/0 (socket says 172.21.15.64:39030) 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 -- 172.21.15.64:0/2758693486 learned_addr learned my addr 172.21.15.64:0/2758693486 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2a9d9b640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc2b415df00 0x7fc2b415f420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 -- 172.21.15.64:0/2758693486 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc2b40b95a0 msgr2=0x7fc2b415e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc2b40b95a0 0x7fc2b415e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 -- 172.21.15.64:0/2758693486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc2b415df00 msgr2=0x7fc2b415f420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc2b415df00 0x7fc2b415f420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 -- 172.21.15.64:0/2758693486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc2b41f3c90 con 0x7fc2b415f960 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b2d9d640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b415f960 0x7fc2b40b8080 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7fc2ac00c780 tx=0x7fc2ac00cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2ab7fe640 1 -- 172.21.15.64:0/2758693486 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc2ac004b60 con 0x7fc2b415f960 2024-03-04T07:27:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2ab7fe640 1 -- 172.21.15.64:0/2758693486 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc2ac004d00 con 0x7fc2b415f960 2024-03-04T07:27:53.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc2b41f3f70 con 0x7fc2b415f960 2024-03-04T07:27:53.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2ab7fe640 1 -- 172.21.15.64:0/2758693486 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc2ac00aa90 con 0x7fc2b415f960 2024-03-04T07:27:53.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.261+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc2b41f4810 con 0x7fc2b415f960 2024-03-04T07:27:53.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.263+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc2b41f4140 con 0x7fc2b415f960 2024-03-04T07:27:53.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.263+0000 7fc2ab7fe640 1 -- 172.21.15.64:0/2758693486 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc2ac00ac30 con 0x7fc2b415f960 2024-03-04T07:27:53.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.263+0000 7fc2ab7fe640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc29406b4d0 0x7fc29406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:53.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.264+0000 7fc2ab7fe640 1 -- 172.21.15.64:0/2758693486 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc2ac08a4d0 con 0x7fc2b415f960 2024-03-04T07:27:53.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.264+0000 7fc2b259c640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc29406b4d0 0x7fc29406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:53.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.264+0000 7fc2b259c640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc29406b4d0 0x7fc29406d990 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fc2a4006fd0 tx=0x7fc2a4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:53.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.269+0000 7fc2ab7fe640 1 -- 172.21.15.64:0/2758693486 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc2ac020070 con 0x7fc2b415f960 2024-03-04T07:27:53.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.471+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fc2b41f4370 con 0x7fc29406b4d0 2024-03-04T07:27:53.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.477+0000 7fc2ab7fe640 1 -- 172.21.15.64:0/2758693486 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251950 (secure 0 0 0) 0x7fc2a400be20 con 0x7fc29406b4d0 2024-03-04T07:27:53.480 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:53.481 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:27:53.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc29406b4d0 msgr2=0x7fc29406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc29406b4d0 0x7fc29406d990 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fc2a4006fd0 tx=0x7fc2a4008040 comp rx=0 tx=0).stop 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b415f960 msgr2=0x7fc2b40b8080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b415f960 0x7fc2b40b8080 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7fc2ac00c780 tx=0x7fc2ac00cc40 comp rx=0 tx=0).stop 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 shutdown_connections 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc29406b4d0 0x7fc29406d990 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc2b415f960 0x7fc2b40b8080 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc2b415df00 0x7fc2b415f420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 --2- 172.21.15.64:0/2758693486 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc2b40b95a0 0x7fc2b415e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:53.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 >> 172.21.15.64:0/2758693486 conn(0x7fc2b4071960 msgr2=0x7fc2b40b3670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:53.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 shutdown_connections 2024-03-04T07:27:53.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:53.480+0000 7fc2b8c9a640 1 -- 172.21.15.64:0/2758693486 wait complete. 2024-03-04T07:27:53.491 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":62,"stamp":"2024-03-04T07:27:53.152951+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":167,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":220328,"kb_used_data":4136,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754754392,"statfs":{"total":773094113280,"available":772868497408,"internally_reserved":0,"allocated":4235264,"data_stored":2358420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":27,"apply_latency_ms":27,"commit_latency_ns":27000000,"apply_latency_ns":27000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.164960"},"pg_stats":[{"pgid":"5.3f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734952+0000","last_change":"2024-03-04T07:27:48.013291+0000","last_active":"2024-03-04T07:27:49.734952+0000","last_peered":"2024-03-04T07:27:49.734952+0000","last_clean":"2024-03-04T07:27:49.734952+0000","last_became_active":"2024-03-04T07:27:48.012990+0000","last_became_peered":"2024-03-04T07:27:48.012990+0000","last_unstale":"2024-03-04T07:27:49.734952+0000","last_undegraded":"2024-03-04T07:27:49.734952+0000","last_fullsized":"2024-03-04T07:27:49.734952+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:44.655515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.3e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733836+0000","last_change":"2024-03-04T07:27:48.019724+0000","last_active":"2024-03-04T07:27:49.733836+0000","last_peered":"2024-03-04T07:27:49.733836+0000","last_clean":"2024-03-04T07:27:49.733836+0000","last_became_active":"2024-03-04T07:27:48.019164+0000","last_became_peered":"2024-03-04T07:27:48.019164+0000","last_unstale":"2024-03-04T07:27:49.733836+0000","last_undegraded":"2024-03-04T07:27:49.733836+0000","last_fullsized":"2024-03-04T07:27:49.733836+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:21.613815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016685+0000","last_change":"2024-03-04T07:27:48.026227+0000","last_active":"2024-03-04T07:27:49.016685+0000","last_peered":"2024-03-04T07:27:49.016685+0000","last_clean":"2024-03-04T07:27:49.016685+0000","last_became_active":"2024-03-04T07:27:48.025372+0000","last_became_peered":"2024-03-04T07:27:48.025372+0000","last_unstale":"2024-03-04T07:27:49.016685+0000","last_undegraded":"2024-03-04T07:27:49.016685+0000","last_fullsized":"2024-03-04T07:27:49.016685+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:16.596129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.3c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734317+0000","last_change":"2024-03-04T07:27:48.029812+0000","last_active":"2024-03-04T07:27:49.734317+0000","last_peered":"2024-03-04T07:27:49.734317+0000","last_clean":"2024-03-04T07:27:49.734317+0000","last_became_active":"2024-03-04T07:27:48.029514+0000","last_became_peered":"2024-03-04T07:27:48.029514+0000","last_unstale":"2024-03-04T07:27:49.734317+0000","last_undegraded":"2024-03-04T07:27:49.734317+0000","last_fullsized":"2024-03-04T07:27:49.734317+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:35:56.668208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248979+0000","last_change":"2024-03-04T07:27:48.026234+0000","last_active":"2024-03-04T07:27:49.248979+0000","last_peered":"2024-03-04T07:27:49.248979+0000","last_clean":"2024-03-04T07:27:49.248979+0000","last_became_active":"2024-03-04T07:27:48.025978+0000","last_became_peered":"2024-03-04T07:27:48.025978+0000","last_unstale":"2024-03-04T07:27:49.248979+0000","last_undegraded":"2024-03-04T07:27:49.248979+0000","last_fullsized":"2024-03-04T07:27:49.248979+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:48.003788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733831+0000","last_change":"2024-03-04T07:27:48.012508+0000","last_active":"2024-03-04T07:27:49.733831+0000","last_peered":"2024-03-04T07:27:49.733831+0000","last_clean":"2024-03-04T07:27:49.733831+0000","last_became_active":"2024-03-04T07:27:48.012210+0000","last_became_peered":"2024-03-04T07:27:48.012210+0000","last_unstale":"2024-03-04T07:27:49.733831+0000","last_undegraded":"2024-03-04T07:27:49.733831+0000","last_fullsized":"2024-03-04T07:27:49.733831+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:43:41.689594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.39","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734226+0000","last_change":"2024-03-04T07:27:48.022347+0000","last_active":"2024-03-04T07:27:49.734226+0000","last_peered":"2024-03-04T07:27:49.734226+0000","last_clean":"2024-03-04T07:27:49.734226+0000","last_became_active":"2024-03-04T07:27:48.022049+0000","last_became_peered":"2024-03-04T07:27:48.022049+0000","last_unstale":"2024-03-04T07:27:49.734226+0000","last_undegraded":"2024-03-04T07:27:49.734226+0000","last_fullsized":"2024-03-04T07:27:49.734226+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:43.636899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.38","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733842+0000","last_change":"2024-03-04T07:27:48.018673+0000","last_active":"2024-03-04T07:27:49.733842+0000","last_peered":"2024-03-04T07:27:49.733842+0000","last_clean":"2024-03-04T07:27:49.733842+0000","last_became_active":"2024-03-04T07:27:48.018389+0000","last_became_peered":"2024-03-04T07:27:48.018389+0000","last_unstale":"2024-03-04T07:27:49.733842+0000","last_undegraded":"2024-03-04T07:27:49.733842+0000","last_fullsized":"2024-03-04T07:27:49.733842+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:32.542508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.37","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017447+0000","last_change":"2024-03-04T07:27:48.022817+0000","last_active":"2024-03-04T07:27:49.017447+0000","last_peered":"2024-03-04T07:27:49.017447+0000","last_clean":"2024-03-04T07:27:49.017447+0000","last_became_active":"2024-03-04T07:27:48.022521+0000","last_became_peered":"2024-03-04T07:27:48.022521+0000","last_unstale":"2024-03-04T07:27:49.017447+0000","last_undegraded":"2024-03-04T07:27:49.017447+0000","last_fullsized":"2024-03-04T07:27:49.017447+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:18:04.118075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.36","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733802+0000","last_change":"2024-03-04T07:27:48.032549+0000","last_active":"2024-03-04T07:27:49.733802+0000","last_peered":"2024-03-04T07:27:49.733802+0000","last_clean":"2024-03-04T07:27:49.733802+0000","last_became_active":"2024-03-04T07:27:48.032223+0000","last_became_peered":"2024-03-04T07:27:48.032223+0000","last_unstale":"2024-03-04T07:27:49.733802+0000","last_undegraded":"2024-03-04T07:27:49.733802+0000","last_fullsized":"2024-03-04T07:27:49.733802+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:28.399564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.35","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734043+0000","last_change":"2024-03-04T07:27:48.026868+0000","last_active":"2024-03-04T07:27:49.734043+0000","last_peered":"2024-03-04T07:27:49.734043+0000","last_clean":"2024-03-04T07:27:49.734043+0000","last_became_active":"2024-03-04T07:27:48.026602+0000","last_became_peered":"2024-03-04T07:27:48.026602+0000","last_unstale":"2024-03-04T07:27:49.734043+0000","last_undegraded":"2024-03-04T07:27:49.734043+0000","last_fullsized":"2024-03-04T07:27:49.734043+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:46:40.563493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.34","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017589+0000","last_change":"2024-03-04T07:27:48.016704+0000","last_active":"2024-03-04T07:27:49.017589+0000","last_peered":"2024-03-04T07:27:49.017589+0000","last_clean":"2024-03-04T07:27:49.017589+0000","last_became_active":"2024-03-04T07:27:48.015399+0000","last_became_peered":"2024-03-04T07:27:48.015399+0000","last_unstale":"2024-03-04T07:27:49.017589+0000","last_undegraded":"2024-03-04T07:27:49.017589+0000","last_fullsized":"2024-03-04T07:27:49.017589+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:51:19.707477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.33","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248987+0000","last_change":"2024-03-04T07:27:48.032216+0000","last_active":"2024-03-04T07:27:49.248987+0000","last_peered":"2024-03-04T07:27:49.248987+0000","last_clean":"2024-03-04T07:27:49.248987+0000","last_became_active":"2024-03-04T07:27:48.031918+0000","last_became_peered":"2024-03-04T07:27:48.031918+0000","last_unstale":"2024-03-04T07:27:49.248987+0000","last_undegraded":"2024-03-04T07:27:49.248987+0000","last_fullsized":"2024-03-04T07:27:49.248987+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:50.733472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.32","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734045+0000","last_change":"2024-03-04T07:27:48.017717+0000","last_active":"2024-03-04T07:27:49.734045+0000","last_peered":"2024-03-04T07:27:49.734045+0000","last_clean":"2024-03-04T07:27:49.734045+0000","last_became_active":"2024-03-04T07:27:48.017403+0000","last_became_peered":"2024-03-04T07:27:48.017403+0000","last_unstale":"2024-03-04T07:27:49.734045+0000","last_undegraded":"2024-03-04T07:27:49.734045+0000","last_fullsized":"2024-03-04T07:27:49.734045+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:45:46.331336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.31","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016685+0000","last_change":"2024-03-04T07:27:48.011842+0000","last_active":"2024-03-04T07:27:49.016685+0000","last_peered":"2024-03-04T07:27:49.016685+0000","last_clean":"2024-03-04T07:27:49.016685+0000","last_became_active":"2024-03-04T07:27:48.011427+0000","last_became_peered":"2024-03-04T07:27:48.011427+0000","last_unstale":"2024-03-04T07:27:49.016685+0000","last_undegraded":"2024-03-04T07:27:49.016685+0000","last_fullsized":"2024-03-04T07:27:49.016685+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:31:24.297134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.30","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734288+0000","last_change":"2024-03-04T07:27:48.024825+0000","last_active":"2024-03-04T07:27:49.734288+0000","last_peered":"2024-03-04T07:27:49.734288+0000","last_clean":"2024-03-04T07:27:49.734288+0000","last_became_active":"2024-03-04T07:27:48.024526+0000","last_became_peered":"2024-03-04T07:27:48.024526+0000","last_unstale":"2024-03-04T07:27:49.734288+0000","last_undegraded":"2024-03-04T07:27:49.734288+0000","last_fullsized":"2024-03-04T07:27:49.734288+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:51:19.707477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734255+0000","last_change":"2024-03-04T07:27:48.023704+0000","last_active":"2024-03-04T07:27:49.734255+0000","last_peered":"2024-03-04T07:27:49.734255+0000","last_clean":"2024-03-04T07:27:49.734255+0000","last_became_active":"2024-03-04T07:27:48.022614+0000","last_became_peered":"2024-03-04T07:27:48.022614+0000","last_unstale":"2024-03-04T07:27:49.734255+0000","last_undegraded":"2024-03-04T07:27:49.734255+0000","last_fullsized":"2024-03-04T07:27:49.734255+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:06:38.256223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733873+0000","last_change":"2024-03-04T07:27:48.025027+0000","last_active":"2024-03-04T07:27:49.733873+0000","last_peered":"2024-03-04T07:27:49.733873+0000","last_clean":"2024-03-04T07:27:49.733873+0000","last_became_active":"2024-03-04T07:27:48.024755+0000","last_became_peered":"2024-03-04T07:27:48.024755+0000","last_unstale":"2024-03-04T07:27:49.733873+0000","last_undegraded":"2024-03-04T07:27:49.733873+0000","last_fullsized":"2024-03-04T07:27:49.733873+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:34:38.900592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733731+0000","last_change":"2024-03-04T07:27:48.028256+0000","last_active":"2024-03-04T07:27:49.733731+0000","last_peered":"2024-03-04T07:27:49.733731+0000","last_clean":"2024-03-04T07:27:49.733731+0000","last_became_active":"2024-03-04T07:27:48.027932+0000","last_became_peered":"2024-03-04T07:27:48.027932+0000","last_unstale":"2024-03-04T07:27:49.733731+0000","last_undegraded":"2024-03-04T07:27:49.733731+0000","last_fullsized":"2024-03-04T07:27:49.733731+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:06:38.256223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248987+0000","last_change":"2024-03-04T07:27:48.029000+0000","last_active":"2024-03-04T07:27:49.248987+0000","last_peered":"2024-03-04T07:27:49.248987+0000","last_clean":"2024-03-04T07:27:49.248987+0000","last_became_active":"2024-03-04T07:27:48.028699+0000","last_became_peered":"2024-03-04T07:27:48.028699+0000","last_unstale":"2024-03-04T07:27:49.248987+0000","last_undegraded":"2024-03-04T07:27:49.248987+0000","last_fullsized":"2024-03-04T07:27:49.248987+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:08.964724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.2b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016622+0000","last_change":"2024-03-04T07:27:48.022182+0000","last_active":"2024-03-04T07:27:49.016622+0000","last_peered":"2024-03-04T07:27:49.016622+0000","last_clean":"2024-03-04T07:27:49.016622+0000","last_became_active":"2024-03-04T07:27:48.021884+0000","last_became_peered":"2024-03-04T07:27:48.021884+0000","last_unstale":"2024-03-04T07:27:49.016622+0000","last_undegraded":"2024-03-04T07:27:49.016622+0000","last_fullsized":"2024-03-04T07:27:49.016622+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:44.655515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.2a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017358+0000","last_change":"2024-03-04T07:27:48.016300+0000","last_active":"2024-03-04T07:27:49.017358+0000","last_peered":"2024-03-04T07:27:49.017358+0000","last_clean":"2024-03-04T07:27:49.017358+0000","last_became_active":"2024-03-04T07:27:48.016060+0000","last_became_peered":"2024-03-04T07:27:48.016060+0000","last_unstale":"2024-03-04T07:27:49.017358+0000","last_undegraded":"2024-03-04T07:27:49.017358+0000","last_fullsized":"2024-03-04T07:27:49.017358+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:35:56.668208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.29","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.014908+0000","last_change":"2024-03-04T07:27:48.016375+0000","last_active":"2024-03-04T07:27:49.014908+0000","last_peered":"2024-03-04T07:27:49.014908+0000","last_clean":"2024-03-04T07:27:49.014908+0000","last_became_active":"2024-03-04T07:27:48.016116+0000","last_became_peered":"2024-03-04T07:27:48.016116+0000","last_unstale":"2024-03-04T07:27:49.014908+0000","last_undegraded":"2024-03-04T07:27:49.014908+0000","last_fullsized":"2024-03-04T07:27:49.014908+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:13.778791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.28","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016157+0000","last_change":"2024-03-04T07:27:48.021118+0000","last_active":"2024-03-04T07:27:49.016157+0000","last_peered":"2024-03-04T07:27:49.016157+0000","last_clean":"2024-03-04T07:27:49.016157+0000","last_became_active":"2024-03-04T07:27:48.020762+0000","last_became_peered":"2024-03-04T07:27:48.020762+0000","last_unstale":"2024-03-04T07:27:49.016157+0000","last_undegraded":"2024-03-04T07:27:49.016157+0000","last_fullsized":"2024-03-04T07:27:49.016157+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:59:54.559611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.27","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734231+0000","last_change":"2024-03-04T07:27:48.034905+0000","last_active":"2024-03-04T07:27:49.734231+0000","last_peered":"2024-03-04T07:27:49.734231+0000","last_clean":"2024-03-04T07:27:49.734231+0000","last_became_active":"2024-03-04T07:27:48.034544+0000","last_became_peered":"2024-03-04T07:27:48.034544+0000","last_unstale":"2024-03-04T07:27:49.734231+0000","last_undegraded":"2024-03-04T07:27:49.734231+0000","last_fullsized":"2024-03-04T07:27:49.734231+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:16.191994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.26","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733981+0000","last_change":"2024-03-04T07:27:48.018767+0000","last_active":"2024-03-04T07:27:49.733981+0000","last_peered":"2024-03-04T07:27:49.733981+0000","last_clean":"2024-03-04T07:27:49.733981+0000","last_became_active":"2024-03-04T07:27:48.018332+0000","last_became_peered":"2024-03-04T07:27:48.018332+0000","last_unstale":"2024-03-04T07:27:49.733981+0000","last_undegraded":"2024-03-04T07:27:49.733981+0000","last_fullsized":"2024-03-04T07:27:49.733981+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:45:31.067674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.25","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734543+0000","last_change":"2024-03-04T07:27:48.029454+0000","last_active":"2024-03-04T07:27:49.734543+0000","last_peered":"2024-03-04T07:27:49.734543+0000","last_clean":"2024-03-04T07:27:49.734543+0000","last_became_active":"2024-03-04T07:27:48.029164+0000","last_became_peered":"2024-03-04T07:27:48.029164+0000","last_unstale":"2024-03-04T07:27:49.734543+0000","last_undegraded":"2024-03-04T07:27:49.734543+0000","last_fullsized":"2024-03-04T07:27:49.734543+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:35.111370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.24","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017652+0000","last_change":"2024-03-04T07:27:48.023330+0000","last_active":"2024-03-04T07:27:49.017652+0000","last_peered":"2024-03-04T07:27:49.017652+0000","last_clean":"2024-03-04T07:27:49.017652+0000","last_became_active":"2024-03-04T07:27:48.022994+0000","last_became_peered":"2024-03-04T07:27:48.022994+0000","last_unstale":"2024-03-04T07:27:49.017652+0000","last_undegraded":"2024-03-04T07:27:49.017652+0000","last_fullsized":"2024-03-04T07:27:49.017652+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:21:17.862929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.23","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016812+0000","last_change":"2024-03-04T07:27:48.013870+0000","last_active":"2024-03-04T07:27:49.016812+0000","last_peered":"2024-03-04T07:27:49.016812+0000","last_clean":"2024-03-04T07:27:49.016812+0000","last_became_active":"2024-03-04T07:27:48.012559+0000","last_became_peered":"2024-03-04T07:27:48.012559+0000","last_unstale":"2024-03-04T07:27:49.016812+0000","last_undegraded":"2024-03-04T07:27:49.016812+0000","last_fullsized":"2024-03-04T07:27:49.016812+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:19:32.594240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.22","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733779+0000","last_change":"2024-03-04T07:27:48.023490+0000","last_active":"2024-03-04T07:27:49.733779+0000","last_peered":"2024-03-04T07:27:49.733779+0000","last_clean":"2024-03-04T07:27:49.733779+0000","last_became_active":"2024-03-04T07:27:48.022703+0000","last_became_peered":"2024-03-04T07:27:48.022703+0000","last_unstale":"2024-03-04T07:27:49.733779+0000","last_undegraded":"2024-03-04T07:27:49.733779+0000","last_fullsized":"2024-03-04T07:27:49.733779+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:05:48.078538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.21","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734127+0000","last_change":"2024-03-04T07:27:48.017975+0000","last_active":"2024-03-04T07:27:49.734127+0000","last_peered":"2024-03-04T07:27:49.734127+0000","last_clean":"2024-03-04T07:27:49.734127+0000","last_became_active":"2024-03-04T07:27:48.017677+0000","last_became_peered":"2024-03-04T07:27:48.017677+0000","last_unstale":"2024-03-04T07:27:49.734127+0000","last_undegraded":"2024-03-04T07:27:49.734127+0000","last_fullsized":"2024-03-04T07:27:49.734127+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:48.068194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.20","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733699+0000","last_change":"2024-03-04T07:27:48.029518+0000","last_active":"2024-03-04T07:27:49.733699+0000","last_peered":"2024-03-04T07:27:49.733699+0000","last_clean":"2024-03-04T07:27:49.733699+0000","last_became_active":"2024-03-04T07:27:48.029207+0000","last_became_peered":"2024-03-04T07:27:48.029207+0000","last_unstale":"2024-03-04T07:27:49.733699+0000","last_undegraded":"2024-03-04T07:27:49.733699+0000","last_fullsized":"2024-03-04T07:27:49.733699+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:37.454873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015069+0000","last_change":"2024-03-04T07:27:48.022992+0000","last_active":"2024-03-04T07:27:49.015069+0000","last_peered":"2024-03-04T07:27:49.015069+0000","last_clean":"2024-03-04T07:27:49.015069+0000","last_became_active":"2024-03-04T07:27:48.022667+0000","last_became_peered":"2024-03-04T07:27:48.022667+0000","last_unstale":"2024-03-04T07:27:49.015069+0000","last_undegraded":"2024-03-04T07:27:49.015069+0000","last_fullsized":"2024-03-04T07:27:49.015069+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:16.677787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"32'4","reported_seq":18,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.735025+0000","last_change":"2024-03-04T07:27:47.005371+0000","last_active":"2024-03-04T07:27:49.735025+0000","last_peered":"2024-03-04T07:27:49.735025+0000","last_clean":"2024-03-04T07:27:49.735025+0000","last_became_active":"2024-03-04T07:27:47.005047+0000","last_became_peered":"2024-03-04T07:27:47.005047+0000","last_unstale":"2024-03-04T07:27:49.735025+0000","last_undegraded":"2024-03-04T07:27:49.735025+0000","last_fullsized":"2024-03-04T07:27:49.735025+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:45.599314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.1f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734962+0000","last_change":"2024-03-04T07:27:48.024968+0000","last_active":"2024-03-04T07:27:49.734962+0000","last_peered":"2024-03-04T07:27:49.734962+0000","last_clean":"2024-03-04T07:27:49.734962+0000","last_became_active":"2024-03-04T07:27:48.024698+0000","last_became_peered":"2024-03-04T07:27:48.024698+0000","last_unstale":"2024-03-04T07:27:49.734962+0000","last_undegraded":"2024-03-04T07:27:49.734962+0000","last_fullsized":"2024-03-04T07:27:49.734962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:28.399564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1e","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015011+0000","last_change":"2024-03-04T07:27:47.009041+0000","last_active":"2024-03-04T07:27:49.015011+0000","last_peered":"2024-03-04T07:27:49.015011+0000","last_clean":"2024-03-04T07:27:49.015011+0000","last_became_active":"2024-03-04T07:27:47.008719+0000","last_became_peered":"2024-03-04T07:27:47.008719+0000","last_unstale":"2024-03-04T07:27:49.015011+0000","last_undegraded":"2024-03-04T07:27:49.015011+0000","last_fullsized":"2024-03-04T07:27:49.015011+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:46.997329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017027+0000","last_change":"2024-03-04T07:27:48.016516+0000","last_active":"2024-03-04T07:27:49.017027+0000","last_peered":"2024-03-04T07:27:49.017027+0000","last_clean":"2024-03-04T07:27:49.017027+0000","last_became_active":"2024-03-04T07:27:48.016227+0000","last_became_peered":"2024-03-04T07:27:48.016227+0000","last_unstale":"2024-03-04T07:27:49.017027+0000","last_undegraded":"2024-03-04T07:27:49.017027+0000","last_fullsized":"2024-03-04T07:27:49.017027+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:16.191994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733411+0000","last_change":"2024-03-04T07:27:47.009369+0000","last_active":"2024-03-04T07:27:49.733411+0000","last_peered":"2024-03-04T07:27:49.733411+0000","last_clean":"2024-03-04T07:27:49.733411+0000","last_became_active":"2024-03-04T07:27:47.007428+0000","last_became_peered":"2024-03-04T07:27:47.007428+0000","last_unstale":"2024-03-04T07:27:49.733411+0000","last_undegraded":"2024-03-04T07:27:49.733411+0000","last_fullsized":"2024-03-04T07:27:49.733411+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:15.671328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734608+0000","last_change":"2024-03-04T07:27:48.025612+0000","last_active":"2024-03-04T07:27:49.734608+0000","last_peered":"2024-03-04T07:27:49.734608+0000","last_clean":"2024-03-04T07:27:49.734608+0000","last_became_active":"2024-03-04T07:27:48.025330+0000","last_became_peered":"2024-03-04T07:27:48.025330+0000","last_unstale":"2024-03-04T07:27:49.734608+0000","last_undegraded":"2024-03-04T07:27:49.734608+0000","last_fullsized":"2024-03-04T07:27:49.734608+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:37.454873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.249042+0000","last_change":"2024-03-04T07:27:47.009650+0000","last_active":"2024-03-04T07:27:49.249042+0000","last_peered":"2024-03-04T07:27:49.249042+0000","last_clean":"2024-03-04T07:27:49.249042+0000","last_became_active":"2024-03-04T07:27:47.009381+0000","last_became_peered":"2024-03-04T07:27:47.009381+0000","last_unstale":"2024-03-04T07:27:49.249042+0000","last_undegraded":"2024-03-04T07:27:49.249042+0000","last_fullsized":"2024-03-04T07:27:49.249042+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:26:17.499963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733952+0000","last_change":"2024-03-04T07:27:48.029496+0000","last_active":"2024-03-04T07:27:49.733952+0000","last_peered":"2024-03-04T07:27:49.733952+0000","last_clean":"2024-03-04T07:27:49.733952+0000","last_became_active":"2024-03-04T07:27:48.029194+0000","last_became_peered":"2024-03-04T07:27:48.029194+0000","last_unstale":"2024-03-04T07:27:49.733952+0000","last_undegraded":"2024-03-04T07:27:49.733952+0000","last_fullsized":"2024-03-04T07:27:49.733952+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:31:24.297134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016871+0000","last_change":"2024-03-04T07:27:47.005110+0000","last_active":"2024-03-04T07:27:49.016871+0000","last_peered":"2024-03-04T07:27:49.016871+0000","last_clean":"2024-03-04T07:27:49.016871+0000","last_became_active":"2024-03-04T07:27:47.004794+0000","last_became_peered":"2024-03-04T07:27:47.004794+0000","last_unstale":"2024-03-04T07:27:49.016871+0000","last_undegraded":"2024-03-04T07:27:49.016871+0000","last_fullsized":"2024-03-04T07:27:49.016871+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:19:20.202177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016339+0000","last_change":"2024-03-04T07:27:48.025831+0000","last_active":"2024-03-04T07:27:49.016339+0000","last_peered":"2024-03-04T07:27:49.016339+0000","last_clean":"2024-03-04T07:27:49.016339+0000","last_became_active":"2024-03-04T07:27:48.025513+0000","last_became_peered":"2024-03-04T07:27:48.025513+0000","last_unstale":"2024-03-04T07:27:49.016339+0000","last_undegraded":"2024-03-04T07:27:49.016339+0000","last_fullsized":"2024-03-04T07:27:49.016339+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:05:48.078538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017281+0000","last_change":"2024-03-04T07:27:47.005798+0000","last_active":"2024-03-04T07:27:49.017281+0000","last_peered":"2024-03-04T07:27:49.017281+0000","last_clean":"2024-03-04T07:27:49.017281+0000","last_became_active":"2024-03-04T07:27:47.005477+0000","last_became_peered":"2024-03-04T07:27:47.005477+0000","last_unstale":"2024-03-04T07:27:49.017281+0000","last_undegraded":"2024-03-04T07:27:49.017281+0000","last_fullsized":"2024-03-04T07:27:49.017281+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:53:22.194744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016315+0000","last_change":"2024-03-04T07:27:48.011650+0000","last_active":"2024-03-04T07:27:49.016315+0000","last_peered":"2024-03-04T07:27:49.016315+0000","last_clean":"2024-03-04T07:27:49.016315+0000","last_became_active":"2024-03-04T07:27:48.011360+0000","last_became_peered":"2024-03-04T07:27:48.011360+0000","last_unstale":"2024-03-04T07:27:49.016315+0000","last_undegraded":"2024-03-04T07:27:49.016315+0000","last_fullsized":"2024-03-04T07:27:49.016315+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:16.677787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733218+0000","last_change":"2024-03-04T07:27:47.009392+0000","last_active":"2024-03-04T07:27:49.733218+0000","last_peered":"2024-03-04T07:27:49.733218+0000","last_clean":"2024-03-04T07:27:49.733218+0000","last_became_active":"2024-03-04T07:27:47.008527+0000","last_became_peered":"2024-03-04T07:27:47.008527+0000","last_unstale":"2024-03-04T07:27:49.733218+0000","last_undegraded":"2024-03-04T07:27:49.733218+0000","last_fullsized":"2024-03-04T07:27:49.733218+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:12.772332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733549+0000","last_change":"2024-03-04T07:27:48.030958+0000","last_active":"2024-03-04T07:27:49.733549+0000","last_peered":"2024-03-04T07:27:49.733549+0000","last_clean":"2024-03-04T07:27:49.733549+0000","last_became_active":"2024-03-04T07:27:48.030659+0000","last_became_peered":"2024-03-04T07:27:48.030659+0000","last_unstale":"2024-03-04T07:27:49.733549+0000","last_undegraded":"2024-03-04T07:27:49.733549+0000","last_fullsized":"2024-03-04T07:27:49.733549+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:44.655515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015996+0000","last_change":"2024-03-04T07:27:46.997525+0000","last_active":"2024-03-04T07:27:49.015996+0000","last_peered":"2024-03-04T07:27:49.015996+0000","last_clean":"2024-03-04T07:27:49.015996+0000","last_became_active":"2024-03-04T07:27:46.997199+0000","last_became_peered":"2024-03-04T07:27:46.997199+0000","last_unstale":"2024-03-04T07:27:49.015996+0000","last_undegraded":"2024-03-04T07:27:49.015996+0000","last_fullsized":"2024-03-04T07:27:49.015996+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:13.016918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.16","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016786+0000","last_change":"2024-03-04T07:27:48.014703+0000","last_active":"2024-03-04T07:27:49.016786+0000","last_peered":"2024-03-04T07:27:49.016786+0000","last_clean":"2024-03-04T07:27:49.016786+0000","last_became_active":"2024-03-04T07:27:48.014408+0000","last_became_peered":"2024-03-04T07:27:48.014408+0000","last_unstale":"2024-03-04T07:27:49.016786+0000","last_undegraded":"2024-03-04T07:27:49.016786+0000","last_fullsized":"2024-03-04T07:27:49.016786+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:46:40.563493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733743+0000","last_change":"2024-03-04T07:27:47.009140+0000","last_active":"2024-03-04T07:27:49.733743+0000","last_peered":"2024-03-04T07:27:49.733743+0000","last_clean":"2024-03-04T07:27:49.733743+0000","last_became_active":"2024-03-04T07:27:47.008873+0000","last_became_peered":"2024-03-04T07:27:47.008873+0000","last_unstale":"2024-03-04T07:27:49.733743+0000","last_undegraded":"2024-03-04T07:27:49.733743+0000","last_fullsized":"2024-03-04T07:27:49.733743+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:43.649055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.17","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733783+0000","last_change":"2024-03-04T07:27:48.025212+0000","last_active":"2024-03-04T07:27:49.733783+0000","last_peered":"2024-03-04T07:27:49.733783+0000","last_clean":"2024-03-04T07:27:49.733783+0000","last_became_active":"2024-03-04T07:27:48.024941+0000","last_became_peered":"2024-03-04T07:27:48.024941+0000","last_unstale":"2024-03-04T07:27:49.733783+0000","last_undegraded":"2024-03-04T07:27:49.733783+0000","last_fullsized":"2024-03-04T07:27:49.733783+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:39.277425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015150+0000","last_change":"2024-03-04T07:27:47.002804+0000","last_active":"2024-03-04T07:27:49.015150+0000","last_peered":"2024-03-04T07:27:49.015150+0000","last_clean":"2024-03-04T07:27:49.015150+0000","last_became_active":"2024-03-04T07:27:47.002319+0000","last_became_peered":"2024-03-04T07:27:47.002319+0000","last_unstale":"2024-03-04T07:27:49.015150+0000","last_undegraded":"2024-03-04T07:27:49.015150+0000","last_fullsized":"2024-03-04T07:27:49.015150+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:07.958265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.14","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733760+0000","last_change":"2024-03-04T07:27:48.018348+0000","last_active":"2024-03-04T07:27:49.733760+0000","last_peered":"2024-03-04T07:27:49.733760+0000","last_clean":"2024-03-04T07:27:49.733760+0000","last_became_active":"2024-03-04T07:27:48.018094+0000","last_became_peered":"2024-03-04T07:27:48.018094+0000","last_unstale":"2024-03-04T07:27:49.733760+0000","last_undegraded":"2024-03-04T07:27:49.733760+0000","last_fullsized":"2024-03-04T07:27:49.733760+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:21:17.862929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"32'3","reported_seq":17,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016775+0000","last_change":"2024-03-04T07:27:47.008962+0000","last_active":"2024-03-04T07:27:49.016775+0000","last_peered":"2024-03-04T07:27:49.016775+0000","last_clean":"2024-03-04T07:27:49.016775+0000","last_became_active":"2024-03-04T07:27:47.008580+0000","last_became_peered":"2024-03-04T07:27:47.008580+0000","last_unstale":"2024-03-04T07:27:49.016775+0000","last_undegraded":"2024-03-04T07:27:49.016775+0000","last_fullsized":"2024-03-04T07:27:49.016775+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:47.061734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.15","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016467+0000","last_change":"2024-03-04T07:27:48.026475+0000","last_active":"2024-03-04T07:27:49.016467+0000","last_peered":"2024-03-04T07:27:49.016467+0000","last_clean":"2024-03-04T07:27:49.016467+0000","last_became_active":"2024-03-04T07:27:48.025361+0000","last_became_peered":"2024-03-04T07:27:48.025361+0000","last_unstale":"2024-03-04T07:27:49.016467+0000","last_undegraded":"2024-03-04T07:27:49.016467+0000","last_fullsized":"2024-03-04T07:27:49.016467+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:32.542508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"32'6","reported_seq":20,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733731+0000","last_change":"2024-03-04T07:27:47.008359+0000","last_active":"2024-03-04T07:27:49.733731+0000","last_peered":"2024-03-04T07:27:49.733731+0000","last_clean":"2024-03-04T07:27:49.733731+0000","last_became_active":"2024-03-04T07:27:47.008066+0000","last_became_peered":"2024-03-04T07:27:47.008066+0000","last_unstale":"2024-03-04T07:27:49.733731+0000","last_undegraded":"2024-03-04T07:27:49.733731+0000","last_fullsized":"2024-03-04T07:27:49.733731+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:58.050466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.12","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733839+0000","last_change":"2024-03-04T07:27:48.033929+0000","last_active":"2024-03-04T07:27:49.733839+0000","last_peered":"2024-03-04T07:27:49.733839+0000","last_clean":"2024-03-04T07:27:49.733839+0000","last_became_active":"2024-03-04T07:27:48.033514+0000","last_became_peered":"2024-03-04T07:27:48.033514+0000","last_unstale":"2024-03-04T07:27:49.733839+0000","last_undegraded":"2024-03-04T07:27:49.733839+0000","last_fullsized":"2024-03-04T07:27:49.733839+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:19:32.594240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016989+0000","last_change":"2024-03-04T07:27:46.998487+0000","last_active":"2024-03-04T07:27:49.016989+0000","last_peered":"2024-03-04T07:27:49.016989+0000","last_clean":"2024-03-04T07:27:49.016989+0000","last_became_active":"2024-03-04T07:27:46.997966+0000","last_became_peered":"2024-03-04T07:27:46.997966+0000","last_unstale":"2024-03-04T07:27:49.016989+0000","last_undegraded":"2024-03-04T07:27:49.016989+0000","last_fullsized":"2024-03-04T07:27:49.016989+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:46.997329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733734+0000","last_change":"2024-03-04T07:27:48.021887+0000","last_active":"2024-03-04T07:27:49.733734+0000","last_peered":"2024-03-04T07:27:49.733734+0000","last_clean":"2024-03-04T07:27:49.733734+0000","last_became_active":"2024-03-04T07:27:48.021529+0000","last_became_peered":"2024-03-04T07:27:48.021529+0000","last_unstale":"2024-03-04T07:27:49.733734+0000","last_undegraded":"2024-03-04T07:27:49.733734+0000","last_fullsized":"2024-03-04T07:27:49.733734+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:51:19.707477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733863+0000","last_change":"2024-03-04T07:27:47.001624+0000","last_active":"2024-03-04T07:27:49.733863+0000","last_peered":"2024-03-04T07:27:49.733863+0000","last_clean":"2024-03-04T07:27:49.733863+0000","last_became_active":"2024-03-04T07:27:47.001341+0000","last_became_peered":"2024-03-04T07:27:47.001341+0000","last_unstale":"2024-03-04T07:27:49.733863+0000","last_undegraded":"2024-03-04T07:27:49.733863+0000","last_fullsized":"2024-03-04T07:27:49.733863+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:45.599314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.10","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734601+0000","last_change":"2024-03-04T07:27:48.029489+0000","last_active":"2024-03-04T07:27:49.734601+0000","last_peered":"2024-03-04T07:27:49.734601+0000","last_clean":"2024-03-04T07:27:49.734601+0000","last_became_active":"2024-03-04T07:27:48.029189+0000","last_became_peered":"2024-03-04T07:27:48.029189+0000","last_unstale":"2024-03-04T07:27:49.734601+0000","last_undegraded":"2024-03-04T07:27:49.734601+0000","last_fullsized":"2024-03-04T07:27:49.734601+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:21:17.862929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733552+0000","last_change":"2024-03-04T07:27:47.009153+0000","last_active":"2024-03-04T07:27:49.733552+0000","last_peered":"2024-03-04T07:27:49.733552+0000","last_clean":"2024-03-04T07:27:49.733552+0000","last_became_active":"2024-03-04T07:27:47.008885+0000","last_became_peered":"2024-03-04T07:27:47.008885+0000","last_unstale":"2024-03-04T07:27:49.733552+0000","last_undegraded":"2024-03-04T07:27:49.733552+0000","last_fullsized":"2024-03-04T07:27:49.733552+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:47.061734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734300+0000","last_change":"2024-03-04T07:27:48.011225+0000","last_active":"2024-03-04T07:27:49.734300+0000","last_peered":"2024-03-04T07:27:49.734300+0000","last_clean":"2024-03-04T07:27:49.734300+0000","last_became_active":"2024-03-04T07:27:48.010909+0000","last_became_peered":"2024-03-04T07:27:48.010909+0000","last_unstale":"2024-03-04T07:27:49.734300+0000","last_undegraded":"2024-03-04T07:27:49.734300+0000","last_fullsized":"2024-03-04T07:27:49.734300+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:16.677787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"32'8","reported_seq":22,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733266+0000","last_change":"2024-03-04T07:27:47.007810+0000","last_active":"2024-03-04T07:27:49.733266+0000","last_peered":"2024-03-04T07:27:49.733266+0000","last_clean":"2024-03-04T07:27:49.733266+0000","last_became_active":"2024-03-04T07:27:47.007524+0000","last_became_peered":"2024-03-04T07:27:47.007524+0000","last_unstale":"2024-03-04T07:27:49.733266+0000","last_undegraded":"2024-03-04T07:27:49.733266+0000","last_fullsized":"2024-03-04T07:27:49.733266+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:27.393104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016651+0000","last_change":"2024-03-04T07:27:48.013051+0000","last_active":"2024-03-04T07:27:49.016651+0000","last_peered":"2024-03-04T07:27:49.016651+0000","last_clean":"2024-03-04T07:27:49.016651+0000","last_became_active":"2024-03-04T07:27:48.012763+0000","last_became_peered":"2024-03-04T07:27:48.012763+0000","last_unstale":"2024-03-04T07:27:49.016651+0000","last_undegraded":"2024-03-04T07:27:49.016651+0000","last_fullsized":"2024-03-04T07:27:49.016651+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:28.399564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734353+0000","last_change":"2024-03-04T07:27:47.007400+0000","last_active":"2024-03-04T07:27:49.734353+0000","last_peered":"2024-03-04T07:27:49.734353+0000","last_clean":"2024-03-04T07:27:49.734353+0000","last_became_active":"2024-03-04T07:27:47.007085+0000","last_became_peered":"2024-03-04T07:27:47.007085+0000","last_unstale":"2024-03-04T07:27:49.734353+0000","last_undegraded":"2024-03-04T07:27:49.734353+0000","last_fullsized":"2024-03-04T07:27:49.734353+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:07.958265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016995+0000","last_change":"2024-03-04T07:27:48.010695+0000","last_active":"2024-03-04T07:27:49.016995+0000","last_peered":"2024-03-04T07:27:49.016995+0000","last_clean":"2024-03-04T07:27:49.016995+0000","last_became_active":"2024-03-04T07:27:48.009424+0000","last_became_peered":"2024-03-04T07:27:48.009424+0000","last_unstale":"2024-03-04T07:27:49.016995+0000","last_undegraded":"2024-03-04T07:27:49.016995+0000","last_fullsized":"2024-03-04T07:27:49.016995+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:43.636899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017232+0000","last_change":"2024-03-04T07:27:46.997566+0000","last_active":"2024-03-04T07:27:49.017232+0000","last_peered":"2024-03-04T07:27:49.017232+0000","last_clean":"2024-03-04T07:27:49.017232+0000","last_became_active":"2024-03-04T07:27:46.997261+0000","last_became_peered":"2024-03-04T07:27:46.997261+0000","last_unstale":"2024-03-04T07:27:49.017232+0000","last_undegraded":"2024-03-04T07:27:49.017232+0000","last_fullsized":"2024-03-04T07:27:49.017232+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:07.958265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734442+0000","last_change":"2024-03-04T07:27:48.029455+0000","last_active":"2024-03-04T07:27:49.734442+0000","last_peered":"2024-03-04T07:27:49.734442+0000","last_clean":"2024-03-04T07:27:49.734442+0000","last_became_active":"2024-03-04T07:27:48.029135+0000","last_became_peered":"2024-03-04T07:27:48.029135+0000","last_unstale":"2024-03-04T07:27:49.734442+0000","last_undegraded":"2024-03-04T07:27:49.734442+0000","last_fullsized":"2024-03-04T07:27:49.734442+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:18:04.118075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016806+0000","last_change":"2024-03-04T07:27:47.005847+0000","last_active":"2024-03-04T07:27:49.016806+0000","last_peered":"2024-03-04T07:27:49.016806+0000","last_clean":"2024-03-04T07:27:49.016806+0000","last_became_active":"2024-03-04T07:27:47.005527+0000","last_became_peered":"2024-03-04T07:27:47.005527+0000","last_unstale":"2024-03-04T07:27:49.016806+0000","last_undegraded":"2024-03-04T07:27:49.016806+0000","last_fullsized":"2024-03-04T07:27:49.016806+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:53.997133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248979+0000","last_change":"2024-03-04T07:27:48.024600+0000","last_active":"2024-03-04T07:27:49.248979+0000","last_peered":"2024-03-04T07:27:49.248979+0000","last_clean":"2024-03-04T07:27:49.248979+0000","last_became_active":"2024-03-04T07:27:48.024320+0000","last_became_peered":"2024-03-04T07:27:48.024320+0000","last_unstale":"2024-03-04T07:27:49.248979+0000","last_undegraded":"2024-03-04T07:27:49.248979+0000","last_fullsized":"2024-03-04T07:27:49.248979+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:19:21.208636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017087+0000","last_change":"2024-03-04T07:27:47.004975+0000","last_active":"2024-03-04T07:27:49.017087+0000","last_peered":"2024-03-04T07:27:49.017087+0000","last_clean":"2024-03-04T07:27:49.017087+0000","last_became_active":"2024-03-04T07:27:47.004664+0000","last_became_peered":"2024-03-04T07:27:47.004664+0000","last_unstale":"2024-03-04T07:27:49.017087+0000","last_undegraded":"2024-03-04T07:27:49.017087+0000","last_fullsized":"2024-03-04T07:27:49.017087+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:05:08.012665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248882+0000","last_change":"2024-03-04T07:27:48.015787+0000","last_active":"2024-03-04T07:27:49.248882+0000","last_peered":"2024-03-04T07:27:49.248882+0000","last_clean":"2024-03-04T07:27:49.248882+0000","last_became_active":"2024-03-04T07:27:48.015481+0000","last_became_peered":"2024-03-04T07:27:48.015481+0000","last_unstale":"2024-03-04T07:27:49.248882+0000","last_undegraded":"2024-03-04T07:27:49.248882+0000","last_fullsized":"2024-03-04T07:27:49.248882+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:32:26.126570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015186+0000","last_change":"2024-03-04T07:27:47.005303+0000","last_active":"2024-03-04T07:27:49.015186+0000","last_peered":"2024-03-04T07:27:49.015186+0000","last_clean":"2024-03-04T07:27:49.015186+0000","last_became_active":"2024-03-04T07:27:47.005013+0000","last_became_peered":"2024-03-04T07:27:47.005013+0000","last_unstale":"2024-03-04T07:27:49.015186+0000","last_undegraded":"2024-03-04T07:27:49.015186+0000","last_fullsized":"2024-03-04T07:27:49.015186+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:45.599314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.249078+0000","last_change":"2024-03-04T07:27:48.020878+0000","last_active":"2024-03-04T07:27:49.249078+0000","last_peered":"2024-03-04T07:27:49.249078+0000","last_clean":"2024-03-04T07:27:49.249078+0000","last_became_active":"2024-03-04T07:27:48.020586+0000","last_became_peered":"2024-03-04T07:27:48.020586+0000","last_unstale":"2024-03-04T07:27:49.249078+0000","last_undegraded":"2024-03-04T07:27:49.249078+0000","last_fullsized":"2024-03-04T07:27:49.249078+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:36.360666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734662+0000","last_change":"2024-03-04T07:27:47.006202+0000","last_active":"2024-03-04T07:27:49.734662+0000","last_peered":"2024-03-04T07:27:49.734662+0000","last_clean":"2024-03-04T07:27:49.734662+0000","last_became_active":"2024-03-04T07:27:47.005900+0000","last_became_peered":"2024-03-04T07:27:47.005900+0000","last_unstale":"2024-03-04T07:27:49.734662+0000","last_undegraded":"2024-03-04T07:27:49.734662+0000","last_fullsized":"2024-03-04T07:27:49.734662+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:35.354207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248803+0000","last_change":"2024-03-04T07:27:48.019537+0000","last_active":"2024-03-04T07:27:49.248803+0000","last_peered":"2024-03-04T07:27:49.248803+0000","last_clean":"2024-03-04T07:27:49.248803+0000","last_became_active":"2024-03-04T07:27:48.019282+0000","last_became_peered":"2024-03-04T07:27:48.019282+0000","last_unstale":"2024-03-04T07:27:49.248803+0000","last_undegraded":"2024-03-04T07:27:49.248803+0000","last_fullsized":"2024-03-04T07:27:49.248803+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:03:00.941676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016562+0000","last_change":"2024-03-04T07:27:47.003400+0000","last_active":"2024-03-04T07:27:49.016562+0000","last_peered":"2024-03-04T07:27:49.016562+0000","last_clean":"2024-03-04T07:27:49.016562+0000","last_became_active":"2024-03-04T07:27:47.003071+0000","last_became_peered":"2024-03-04T07:27:47.003071+0000","last_unstale":"2024-03-04T07:27:49.016562+0000","last_undegraded":"2024-03-04T07:27:49.016562+0000","last_fullsized":"2024-03-04T07:27:49.016562+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:02:59.935216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.9","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733749+0000","last_change":"2024-03-04T07:27:48.020621+0000","last_active":"2024-03-04T07:27:49.733749+0000","last_peered":"2024-03-04T07:27:49.733749+0000","last_clean":"2024-03-04T07:27:49.733749+0000","last_became_active":"2024-03-04T07:27:48.020324+0000","last_became_peered":"2024-03-04T07:27:48.020324+0000","last_unstale":"2024-03-04T07:27:49.733749+0000","last_undegraded":"2024-03-04T07:27:49.733749+0000","last_fullsized":"2024-03-04T07:27:49.733749+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:59:54.559611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016223+0000","last_change":"2024-03-04T07:27:47.008049+0000","last_active":"2024-03-04T07:27:49.016223+0000","last_peered":"2024-03-04T07:27:49.016223+0000","last_clean":"2024-03-04T07:27:49.016223+0000","last_became_active":"2024-03-04T07:27:47.007495+0000","last_became_peered":"2024-03-04T07:27:47.007495+0000","last_unstale":"2024-03-04T07:27:49.016223+0000","last_undegraded":"2024-03-04T07:27:49.016223+0000","last_fullsized":"2024-03-04T07:27:49.016223+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:12.772332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.249000+0000","last_change":"2024-03-04T07:27:48.028254+0000","last_active":"2024-03-04T07:27:49.249000+0000","last_peered":"2024-03-04T07:27:49.249000+0000","last_clean":"2024-03-04T07:27:49.249000+0000","last_became_active":"2024-03-04T07:27:48.027969+0000","last_became_peered":"2024-03-04T07:27:48.027969+0000","last_unstale":"2024-03-04T07:27:49.249000+0000","last_undegraded":"2024-03-04T07:27:49.249000+0000","last_fullsized":"2024-03-04T07:27:49.249000+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:46.605774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734456+0000","last_change":"2024-03-04T07:27:47.002163+0000","last_active":"2024-03-04T07:27:49.734456+0000","last_peered":"2024-03-04T07:27:49.734456+0000","last_clean":"2024-03-04T07:27:49.734456+0000","last_became_active":"2024-03-04T07:27:47.001891+0000","last_became_peered":"2024-03-04T07:27:47.001891+0000","last_unstale":"2024-03-04T07:27:49.734456+0000","last_undegraded":"2024-03-04T07:27:49.734456+0000","last_fullsized":"2024-03-04T07:27:49.734456+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:49.727013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017076+0000","last_change":"2024-03-04T07:27:48.025663+0000","last_active":"2024-03-04T07:27:49.017076+0000","last_peered":"2024-03-04T07:27:49.017076+0000","last_clean":"2024-03-04T07:27:49.017076+0000","last_became_active":"2024-03-04T07:27:48.025271+0000","last_became_peered":"2024-03-04T07:27:48.025271+0000","last_unstale":"2024-03-04T07:27:49.017076+0000","last_undegraded":"2024-03-04T07:27:49.017076+0000","last_fullsized":"2024-03-04T07:27:49.017076+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:06:38.256223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015148+0000","last_change":"2024-03-04T07:27:46.997285+0000","last_active":"2024-03-04T07:27:49.015148+0000","last_peered":"2024-03-04T07:27:49.015148+0000","last_clean":"2024-03-04T07:27:49.015148+0000","last_became_active":"2024-03-04T07:27:46.997001+0000","last_became_peered":"2024-03-04T07:27:46.997001+0000","last_unstale":"2024-03-04T07:27:49.015148+0000","last_undegraded":"2024-03-04T07:27:49.015148+0000","last_fullsized":"2024-03-04T07:27:49.015148+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:49.727013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.014958+0000","last_change":"2024-03-04T07:27:48.026814+0000","last_active":"2024-03-04T07:27:49.014958+0000","last_peered":"2024-03-04T07:27:49.014958+0000","last_clean":"2024-03-04T07:27:49.014958+0000","last_became_active":"2024-03-04T07:27:48.026322+0000","last_became_peered":"2024-03-04T07:27:48.026322+0000","last_unstale":"2024-03-04T07:27:49.014958+0000","last_undegraded":"2024-03-04T07:27:49.014958+0000","last_fullsized":"2024-03-04T07:27:49.014958+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:48.068194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734485+0000","last_change":"2024-03-04T07:27:47.010602+0000","last_active":"2024-03-04T07:27:49.734485+0000","last_peered":"2024-03-04T07:27:49.734485+0000","last_clean":"2024-03-04T07:27:49.734485+0000","last_became_active":"2024-03-04T07:27:47.010286+0000","last_became_peered":"2024-03-04T07:27:47.010286+0000","last_unstale":"2024-03-04T07:27:49.734485+0000","last_undegraded":"2024-03-04T07:27:49.734485+0000","last_fullsized":"2024-03-04T07:27:49.734485+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:46.997329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734820+0000","last_change":"2024-03-04T07:27:48.019333+0000","last_active":"2024-03-04T07:27:49.734820+0000","last_peered":"2024-03-04T07:27:49.734820+0000","last_clean":"2024-03-04T07:27:49.734820+0000","last_became_active":"2024-03-04T07:27:48.019037+0000","last_became_peered":"2024-03-04T07:27:48.019037+0000","last_unstale":"2024-03-04T07:27:49.734820+0000","last_undegraded":"2024-03-04T07:27:49.734820+0000","last_fullsized":"2024-03-04T07:27:49.734820+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:59.056925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015253+0000","last_change":"2024-03-04T07:27:46.996003+0000","last_active":"2024-03-04T07:27:49.015253+0000","last_peered":"2024-03-04T07:27:49.015253+0000","last_clean":"2024-03-04T07:27:49.015253+0000","last_became_active":"2024-03-04T07:27:46.995495+0000","last_became_peered":"2024-03-04T07:27:46.995495+0000","last_unstale":"2024-03-04T07:27:49.015253+0000","last_undegraded":"2024-03-04T07:27:49.015253+0000","last_fullsized":"2024-03-04T07:27:49.015253+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:35.354207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015119+0000","last_change":"2024-03-04T07:27:48.020300+0000","last_active":"2024-03-04T07:27:49.015119+0000","last_peered":"2024-03-04T07:27:49.015119+0000","last_clean":"2024-03-04T07:27:49.015119+0000","last_became_active":"2024-03-04T07:27:48.020050+0000","last_became_peered":"2024-03-04T07:27:48.020050+0000","last_unstale":"2024-03-04T07:27:49.015119+0000","last_undegraded":"2024-03-04T07:27:49.015119+0000","last_fullsized":"2024-03-04T07:27:49.015119+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:59.056925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733888+0000","last_change":"2024-03-04T07:27:47.003837+0000","last_active":"2024-03-04T07:27:49.733888+0000","last_peered":"2024-03-04T07:27:49.733888+0000","last_clean":"2024-03-04T07:27:49.733888+0000","last_became_active":"2024-03-04T07:27:47.003534+0000","last_became_peered":"2024-03-04T07:27:47.003534+0000","last_unstale":"2024-03-04T07:27:49.733888+0000","last_undegraded":"2024-03-04T07:27:49.733888+0000","last_fullsized":"2024-03-04T07:27:49.733888+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:35.354207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733277+0000","last_change":"2024-03-04T07:27:48.027090+0000","last_active":"2024-03-04T07:27:49.733277+0000","last_peered":"2024-03-04T07:27:49.733277+0000","last_clean":"2024-03-04T07:27:49.733277+0000","last_became_active":"2024-03-04T07:27:48.026830+0000","last_became_peered":"2024-03-04T07:27:48.026830+0000","last_unstale":"2024-03-04T07:27:49.733277+0000","last_undegraded":"2024-03-04T07:27:49.733277+0000","last_fullsized":"2024-03-04T07:27:49.733277+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:43.636899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016689+0000","last_change":"2024-03-04T07:27:46.996917+0000","last_active":"2024-03-04T07:27:49.016689+0000","last_peered":"2024-03-04T07:27:49.016689+0000","last_clean":"2024-03-04T07:27:49.016689+0000","last_became_active":"2024-03-04T07:27:46.996595+0000","last_became_peered":"2024-03-04T07:27:46.996595+0000","last_unstale":"2024-03-04T07:27:49.016689+0000","last_undegraded":"2024-03-04T07:27:49.016689+0000","last_fullsized":"2024-03-04T07:27:49.016689+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:32:25.120111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016490+0000","last_change":"2024-03-04T07:27:48.013371+0000","last_active":"2024-03-04T07:27:49.016490+0000","last_peered":"2024-03-04T07:27:49.016490+0000","last_clean":"2024-03-04T07:27:49.016490+0000","last_became_active":"2024-03-04T07:27:48.012901+0000","last_became_peered":"2024-03-04T07:27:48.012901+0000","last_unstale":"2024-03-04T07:27:49.016490+0000","last_undegraded":"2024-03-04T07:27:49.016490+0000","last_fullsized":"2024-03-04T07:27:49.016490+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:59.056925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733327+0000","last_change":"2024-03-04T07:27:47.009841+0000","last_active":"2024-03-04T07:27:49.733327+0000","last_peered":"2024-03-04T07:27:49.733327+0000","last_clean":"2024-03-04T07:27:49.733327+0000","last_became_active":"2024-03-04T07:27:47.009498+0000","last_became_peered":"2024-03-04T07:27:47.009498+0000","last_unstale":"2024-03-04T07:27:49.733327+0000","last_undegraded":"2024-03-04T07:27:49.733327+0000","last_fullsized":"2024-03-04T07:27:49.733327+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:19:31.587781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733939+0000","last_change":"2024-03-04T07:27:48.021305+0000","last_active":"2024-03-04T07:27:49.733939+0000","last_peered":"2024-03-04T07:27:49.733939+0000","last_clean":"2024-03-04T07:27:49.733939+0000","last_became_active":"2024-03-04T07:27:48.021021+0000","last_became_peered":"2024-03-04T07:27:48.021021+0000","last_unstale":"2024-03-04T07:27:49.733939+0000","last_undegraded":"2024-03-04T07:27:49.733939+0000","last_fullsized":"2024-03-04T07:27:49.733939+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:16.596129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734692+0000","last_change":"2024-03-04T07:27:46.999584+0000","last_active":"2024-03-04T07:27:49.734692+0000","last_peered":"2024-03-04T07:27:49.734692+0000","last_clean":"2024-03-04T07:27:49.734692+0000","last_became_active":"2024-03-04T07:27:46.999273+0000","last_became_peered":"2024-03-04T07:27:46.999273+0000","last_unstale":"2024-03-04T07:27:49.734692+0000","last_undegraded":"2024-03-04T07:27:49.734692+0000","last_fullsized":"2024-03-04T07:27:49.734692+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:49.727013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":193,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734335+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:27:49.734335+0000","last_peered":"2024-03-04T07:27:49.734335+0000","last_clean":"2024-03-04T07:27:49.734335+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:27:49.734335+0000","last_undegraded":"2024-03-04T07:27:49.734335+0000","last_fullsized":"2024-03-04T07:27:49.734335+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574867,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27972,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343868,"statfs":{"total":96636764160,"available":96608120832,"internally_reserved":0,"allocated":970752,"data_stored":737066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574866,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":372,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":380928,"data_stored":147224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574866,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":372,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":380928,"data_stored":147280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574867,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":372,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":380928,"data_stored":146712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574866,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27404,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344436,"statfs":{"total":96636764160,"available":96608702464,"internally_reserved":0,"allocated":389120,"data_stored":148242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574866,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27392,"kb_used_data":368,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344448,"statfs":{"total":96636764160,"available":96608714752,"internally_reserved":0,"allocated":376832,"data_stored":146678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574866,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27404,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344436,"statfs":{"total":96636764160,"available":96608702464,"internally_reserved":0,"allocated":389120,"data_stored":148242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574866,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27968,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343872,"statfs":{"total":96636764160,"available":96608124928,"internally_reserved":0,"allocated":966656,"data_stored":736976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:27:56.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:27:56.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.598+0000 7f75b7ef4640 1 Processor -- start 2024-03-04T07:27:56.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.598+0000 7f75b7ef4640 1 -- start start 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b7ef4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f75b015df00 0x7f75b00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b7ef4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f75b00b95a0 0x7f75b00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b7ef4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75b00cea30 con 0x7f75b00b9a30 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b7ef4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75b00d2af0 con 0x7f75b00b95a0 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b7ef4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75b00cf3c0 con 0x7f75b015df00 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b548f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f75b00b95a0 0x7f75b00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b548f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f75b00b95a0 0x7f75b00b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49018/0 (socket says 172.21.15.64:49018) 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b548f640 1 -- 172.21.15.64:0/2879937250 learned_addr learned my addr 172.21.15.64:0/2879937250 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b6491640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f75b00b9a30 0x7f75b00d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54692/0 (socket says 172.21.15.64:54692) 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b5c90640 1 --2- 172.21.15.64:0/2879937250 >> v2:172.21.15.66:3300/0 conn(0x7f75b015df00 0x7f75b00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:56.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b548f640 1 -- 172.21.15.64:0/2879937250 >> v2:172.21.15.66:3300/0 conn(0x7f75b015df00 msgr2=0x7f75b00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b548f640 1 --2- 172.21.15.64:0/2879937250 >> v2:172.21.15.66:3300/0 conn(0x7f75b015df00 0x7f75b00d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b548f640 1 -- 172.21.15.64:0/2879937250 >> v1:172.21.15.64:6789/0 conn(0x7f75b00b9a30 legacy=0x7f75b00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b4c8e640 1 -- 172.21.15.64:0/2879937250 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 88633030 0 0) 0x7f75b00cea30 con 0x7f75b00b9a30 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.599+0000 7f75b548f640 1 -- 172.21.15.64:0/2879937250 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75b00cf0e0 con 0x7f75b00b95a0 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.600+0000 7f75b548f640 1 --2- 172.21.15.64:0/2879937250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b00b95a0 0x7f75b00b5700 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f75b00d11b0 tx=0x7f75a002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c2a364f55fe7e5f5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.600+0000 7f75b4c8e640 1 -- 172.21.15.64:0/2879937250 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75a0034070 con 0x7f75b00b95a0 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.600+0000 7f75b4c8e640 1 -- 172.21.15.64:0/2879937250 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75a0030860 con 0x7f75b00b95a0 2024-03-04T07:27:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.600+0000 7f75b4c8e640 1 -- 172.21.15.64:0/2879937250 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75a0030c30 con 0x7f75b00b95a0 2024-03-04T07:27:56.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2879937250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b00b95a0 msgr2=0x7f75b00b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:56.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2879937250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b00b95a0 0x7f75b00b5700 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f75b00d11b0 tx=0x7f75a002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2879937250 shutdown_connections 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2879937250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b00b95a0 0x7f75b00b5700 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2879937250 >> v2:172.21.15.66:3300/0 conn(0x7f75b015df00 0x7f75b00d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2879937250 >> 172.21.15.64:0/2879937250 conn(0x7f75b0071960 msgr2=0x7f75b0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2879937250 shutdown_connections 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.601+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2879937250 wait complete. 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.602+0000 7f75b7ef4640 1 Processor -- start 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.602+0000 7f75b7ef4640 1 -- start start 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b7ef4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75b00b95a0 0x7f75b015a860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b7ef4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75b015df00 0x7f75b01f78d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b7ef4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b015ae40 0x7f75b0161060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b7ef4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75b00cec70 con 0x7f75b015ae40 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b7ef4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75b00ce750 con 0x7f75b015df00 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b7ef4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75b00cedf0 con 0x7f75b00b95a0 2024-03-04T07:27:56.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b015ae40 0x7f75b0161060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b015ae40 0x7f75b0161060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49024/0 (socket says 172.21.15.64:49024) 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 -- 172.21.15.64:0/2382103636 learned_addr learned my addr 172.21.15.64:0/2382103636 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b548f640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75b015df00 0x7f75b01f78d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b5c90640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75b00b95a0 0x7f75b015a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 -- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75b00b95a0 msgr2=0x7f75b015a860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75b00b95a0 0x7f75b015a860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 -- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75b015df00 msgr2=0x7f75b01f78d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75b015df00 0x7f75b01f78d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 -- 172.21.15.64:0/2382103636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75b01f6e30 con 0x7f75b015ae40 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b548f640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75b015df00 0x7f75b01f78d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b5c90640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75b00b95a0 0x7f75b015a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75b6491640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b015ae40 0x7f75b0161060 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f75a800c7a0 tx=0x7f75a800cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.603+0000 7f75a67fc640 1 -- 172.21.15.64:0/2382103636 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75a8018070 con 0x7f75b015ae40 2024-03-04T07:27:56.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.604+0000 7f75a67fc640 1 -- 172.21.15.64:0/2382103636 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75a800a5e0 con 0x7f75b015ae40 2024-03-04T07:27:56.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.604+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f75b01e8db0 con 0x7f75b015ae40 2024-03-04T07:27:56.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.604+0000 7f75a67fc640 1 -- 172.21.15.64:0/2382103636 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75a800aa50 con 0x7f75b015ae40 2024-03-04T07:27:56.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.604+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f75b01f73b0 con 0x7f75b015ae40 2024-03-04T07:27:56.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.605+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f75b0067ff0 con 0x7f75b015ae40 2024-03-04T07:27:56.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.605+0000 7f75a67fc640 1 -- 172.21.15.64:0/2382103636 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f75a801b070 con 0x7f75b015ae40 2024-03-04T07:27:56.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.605+0000 7f75a67fc640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f759406b4d0 0x7f759406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:56.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.606+0000 7f75a67fc640 1 -- 172.21.15.64:0/2382103636 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f75a80897c0 con 0x7f75b015ae40 2024-03-04T07:27:56.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.606+0000 7f75b5c90640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f759406b4d0 0x7f759406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:56.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.606+0000 7f75b5c90640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f759406b4d0 0x7f759406d990 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f75ac006fd0 tx=0x7f75ac008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:56.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.610+0000 7f75a67fc640 1 -- 172.21.15.64:0/2382103636 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f75a8058640 con 0x7f75b015ae40 2024-03-04T07:27:56.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.911+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f75b00ce0a0 con 0x7f759406b4d0 2024-03-04T07:27:56.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.917+0000 7f75a67fc640 1 -- 172.21.15.64:0/2382103636 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251950 (secure 0 0 0) 0x7f75b00ce0a0 con 0x7f759406b4d0 2024-03-04T07:27:56.919 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:56.921 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f759406b4d0 msgr2=0x7f759406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f759406b4d0 0x7f759406d990 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f75ac006fd0 tx=0x7f75ac008040 comp rx=0 tx=0).stop 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b015ae40 msgr2=0x7f75b0161060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b015ae40 0x7f75b0161060 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f75a800c7a0 tx=0x7f75a800cc60 comp rx=0 tx=0).stop 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 shutdown_connections 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f759406b4d0 0x7f759406d990 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75b015ae40 0x7f75b0161060 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75b015df00 0x7f75b01f78d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 --2- 172.21.15.64:0/2382103636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75b00b95a0 0x7f75b015a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:56.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.920+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 >> 172.21.15.64:0/2382103636 conn(0x7f75b0071960 msgr2=0x7f75b00b8730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:56.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.921+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 shutdown_connections 2024-03-04T07:27:56.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:56.921+0000 7f75b7ef4640 1 -- 172.21.15.64:0/2382103636 wait complete. 2024-03-04T07:27:56.931 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":63,"stamp":"2024-03-04T07:27:55.153328+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":167,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":220328,"kb_used_data":4136,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":754754392,"statfs":{"total":773094113280,"available":772868497408,"internally_reserved":0,"allocated":4235264,"data_stored":2358420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":221236784},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":27,"apply_latency_ms":27,"commit_latency_ns":27000000,"apply_latency_ns":27000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001257"},"pg_stats":[{"pgid":"5.3f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734952+0000","last_change":"2024-03-04T07:27:48.013291+0000","last_active":"2024-03-04T07:27:49.734952+0000","last_peered":"2024-03-04T07:27:49.734952+0000","last_clean":"2024-03-04T07:27:49.734952+0000","last_became_active":"2024-03-04T07:27:48.012990+0000","last_became_peered":"2024-03-04T07:27:48.012990+0000","last_unstale":"2024-03-04T07:27:49.734952+0000","last_undegraded":"2024-03-04T07:27:49.734952+0000","last_fullsized":"2024-03-04T07:27:49.734952+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:44.655515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.3e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733836+0000","last_change":"2024-03-04T07:27:48.019724+0000","last_active":"2024-03-04T07:27:49.733836+0000","last_peered":"2024-03-04T07:27:49.733836+0000","last_clean":"2024-03-04T07:27:49.733836+0000","last_became_active":"2024-03-04T07:27:48.019164+0000","last_became_peered":"2024-03-04T07:27:48.019164+0000","last_unstale":"2024-03-04T07:27:49.733836+0000","last_undegraded":"2024-03-04T07:27:49.733836+0000","last_fullsized":"2024-03-04T07:27:49.733836+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:21.613815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.3d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016685+0000","last_change":"2024-03-04T07:27:48.026227+0000","last_active":"2024-03-04T07:27:49.016685+0000","last_peered":"2024-03-04T07:27:49.016685+0000","last_clean":"2024-03-04T07:27:49.016685+0000","last_became_active":"2024-03-04T07:27:48.025372+0000","last_became_peered":"2024-03-04T07:27:48.025372+0000","last_unstale":"2024-03-04T07:27:49.016685+0000","last_undegraded":"2024-03-04T07:27:49.016685+0000","last_fullsized":"2024-03-04T07:27:49.016685+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:16.596129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.3c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734317+0000","last_change":"2024-03-04T07:27:48.029812+0000","last_active":"2024-03-04T07:27:49.734317+0000","last_peered":"2024-03-04T07:27:49.734317+0000","last_clean":"2024-03-04T07:27:49.734317+0000","last_became_active":"2024-03-04T07:27:48.029514+0000","last_became_peered":"2024-03-04T07:27:48.029514+0000","last_unstale":"2024-03-04T07:27:49.734317+0000","last_undegraded":"2024-03-04T07:27:49.734317+0000","last_fullsized":"2024-03-04T07:27:49.734317+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:35:56.668208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248979+0000","last_change":"2024-03-04T07:27:48.026234+0000","last_active":"2024-03-04T07:27:49.248979+0000","last_peered":"2024-03-04T07:27:49.248979+0000","last_clean":"2024-03-04T07:27:49.248979+0000","last_became_active":"2024-03-04T07:27:48.025978+0000","last_became_peered":"2024-03-04T07:27:48.025978+0000","last_unstale":"2024-03-04T07:27:49.248979+0000","last_undegraded":"2024-03-04T07:27:49.248979+0000","last_fullsized":"2024-03-04T07:27:49.248979+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:48.003788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733831+0000","last_change":"2024-03-04T07:27:48.012508+0000","last_active":"2024-03-04T07:27:49.733831+0000","last_peered":"2024-03-04T07:27:49.733831+0000","last_clean":"2024-03-04T07:27:49.733831+0000","last_became_active":"2024-03-04T07:27:48.012210+0000","last_became_peered":"2024-03-04T07:27:48.012210+0000","last_unstale":"2024-03-04T07:27:49.733831+0000","last_undegraded":"2024-03-04T07:27:49.733831+0000","last_fullsized":"2024-03-04T07:27:49.733831+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:43:41.689594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.39","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734226+0000","last_change":"2024-03-04T07:27:48.022347+0000","last_active":"2024-03-04T07:27:49.734226+0000","last_peered":"2024-03-04T07:27:49.734226+0000","last_clean":"2024-03-04T07:27:49.734226+0000","last_became_active":"2024-03-04T07:27:48.022049+0000","last_became_peered":"2024-03-04T07:27:48.022049+0000","last_unstale":"2024-03-04T07:27:49.734226+0000","last_undegraded":"2024-03-04T07:27:49.734226+0000","last_fullsized":"2024-03-04T07:27:49.734226+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:43.636899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.38","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733842+0000","last_change":"2024-03-04T07:27:48.018673+0000","last_active":"2024-03-04T07:27:49.733842+0000","last_peered":"2024-03-04T07:27:49.733842+0000","last_clean":"2024-03-04T07:27:49.733842+0000","last_became_active":"2024-03-04T07:27:48.018389+0000","last_became_peered":"2024-03-04T07:27:48.018389+0000","last_unstale":"2024-03-04T07:27:49.733842+0000","last_undegraded":"2024-03-04T07:27:49.733842+0000","last_fullsized":"2024-03-04T07:27:49.733842+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:32.542508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.37","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017447+0000","last_change":"2024-03-04T07:27:48.022817+0000","last_active":"2024-03-04T07:27:49.017447+0000","last_peered":"2024-03-04T07:27:49.017447+0000","last_clean":"2024-03-04T07:27:49.017447+0000","last_became_active":"2024-03-04T07:27:48.022521+0000","last_became_peered":"2024-03-04T07:27:48.022521+0000","last_unstale":"2024-03-04T07:27:49.017447+0000","last_undegraded":"2024-03-04T07:27:49.017447+0000","last_fullsized":"2024-03-04T07:27:49.017447+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:18:04.118075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.36","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733802+0000","last_change":"2024-03-04T07:27:48.032549+0000","last_active":"2024-03-04T07:27:49.733802+0000","last_peered":"2024-03-04T07:27:49.733802+0000","last_clean":"2024-03-04T07:27:49.733802+0000","last_became_active":"2024-03-04T07:27:48.032223+0000","last_became_peered":"2024-03-04T07:27:48.032223+0000","last_unstale":"2024-03-04T07:27:49.733802+0000","last_undegraded":"2024-03-04T07:27:49.733802+0000","last_fullsized":"2024-03-04T07:27:49.733802+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:28.399564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.35","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734043+0000","last_change":"2024-03-04T07:27:48.026868+0000","last_active":"2024-03-04T07:27:49.734043+0000","last_peered":"2024-03-04T07:27:49.734043+0000","last_clean":"2024-03-04T07:27:49.734043+0000","last_became_active":"2024-03-04T07:27:48.026602+0000","last_became_peered":"2024-03-04T07:27:48.026602+0000","last_unstale":"2024-03-04T07:27:49.734043+0000","last_undegraded":"2024-03-04T07:27:49.734043+0000","last_fullsized":"2024-03-04T07:27:49.734043+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:46:40.563493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.34","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017589+0000","last_change":"2024-03-04T07:27:48.016704+0000","last_active":"2024-03-04T07:27:49.017589+0000","last_peered":"2024-03-04T07:27:49.017589+0000","last_clean":"2024-03-04T07:27:49.017589+0000","last_became_active":"2024-03-04T07:27:48.015399+0000","last_became_peered":"2024-03-04T07:27:48.015399+0000","last_unstale":"2024-03-04T07:27:49.017589+0000","last_undegraded":"2024-03-04T07:27:49.017589+0000","last_fullsized":"2024-03-04T07:27:49.017589+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:51:19.707477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.33","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248987+0000","last_change":"2024-03-04T07:27:48.032216+0000","last_active":"2024-03-04T07:27:49.248987+0000","last_peered":"2024-03-04T07:27:49.248987+0000","last_clean":"2024-03-04T07:27:49.248987+0000","last_became_active":"2024-03-04T07:27:48.031918+0000","last_became_peered":"2024-03-04T07:27:48.031918+0000","last_unstale":"2024-03-04T07:27:49.248987+0000","last_undegraded":"2024-03-04T07:27:49.248987+0000","last_fullsized":"2024-03-04T07:27:49.248987+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:50.733472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.32","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734045+0000","last_change":"2024-03-04T07:27:48.017717+0000","last_active":"2024-03-04T07:27:49.734045+0000","last_peered":"2024-03-04T07:27:49.734045+0000","last_clean":"2024-03-04T07:27:49.734045+0000","last_became_active":"2024-03-04T07:27:48.017403+0000","last_became_peered":"2024-03-04T07:27:48.017403+0000","last_unstale":"2024-03-04T07:27:49.734045+0000","last_undegraded":"2024-03-04T07:27:49.734045+0000","last_fullsized":"2024-03-04T07:27:49.734045+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:45:46.331336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.31","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016685+0000","last_change":"2024-03-04T07:27:48.011842+0000","last_active":"2024-03-04T07:27:49.016685+0000","last_peered":"2024-03-04T07:27:49.016685+0000","last_clean":"2024-03-04T07:27:49.016685+0000","last_became_active":"2024-03-04T07:27:48.011427+0000","last_became_peered":"2024-03-04T07:27:48.011427+0000","last_unstale":"2024-03-04T07:27:49.016685+0000","last_undegraded":"2024-03-04T07:27:49.016685+0000","last_fullsized":"2024-03-04T07:27:49.016685+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:31:24.297134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.30","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734288+0000","last_change":"2024-03-04T07:27:48.024825+0000","last_active":"2024-03-04T07:27:49.734288+0000","last_peered":"2024-03-04T07:27:49.734288+0000","last_clean":"2024-03-04T07:27:49.734288+0000","last_became_active":"2024-03-04T07:27:48.024526+0000","last_became_peered":"2024-03-04T07:27:48.024526+0000","last_unstale":"2024-03-04T07:27:49.734288+0000","last_undegraded":"2024-03-04T07:27:49.734288+0000","last_fullsized":"2024-03-04T07:27:49.734288+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:51:19.707477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734255+0000","last_change":"2024-03-04T07:27:48.023704+0000","last_active":"2024-03-04T07:27:49.734255+0000","last_peered":"2024-03-04T07:27:49.734255+0000","last_clean":"2024-03-04T07:27:49.734255+0000","last_became_active":"2024-03-04T07:27:48.022614+0000","last_became_peered":"2024-03-04T07:27:48.022614+0000","last_unstale":"2024-03-04T07:27:49.734255+0000","last_undegraded":"2024-03-04T07:27:49.734255+0000","last_fullsized":"2024-03-04T07:27:49.734255+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:06:38.256223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.2e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733873+0000","last_change":"2024-03-04T07:27:48.025027+0000","last_active":"2024-03-04T07:27:49.733873+0000","last_peered":"2024-03-04T07:27:49.733873+0000","last_clean":"2024-03-04T07:27:49.733873+0000","last_became_active":"2024-03-04T07:27:48.024755+0000","last_became_peered":"2024-03-04T07:27:48.024755+0000","last_unstale":"2024-03-04T07:27:49.733873+0000","last_undegraded":"2024-03-04T07:27:49.733873+0000","last_fullsized":"2024-03-04T07:27:49.733873+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:34:38.900592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733731+0000","last_change":"2024-03-04T07:27:48.028256+0000","last_active":"2024-03-04T07:27:49.733731+0000","last_peered":"2024-03-04T07:27:49.733731+0000","last_clean":"2024-03-04T07:27:49.733731+0000","last_became_active":"2024-03-04T07:27:48.027932+0000","last_became_peered":"2024-03-04T07:27:48.027932+0000","last_unstale":"2024-03-04T07:27:49.733731+0000","last_undegraded":"2024-03-04T07:27:49.733731+0000","last_fullsized":"2024-03-04T07:27:49.733731+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:06:38.256223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.2c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248987+0000","last_change":"2024-03-04T07:27:48.029000+0000","last_active":"2024-03-04T07:27:49.248987+0000","last_peered":"2024-03-04T07:27:49.248987+0000","last_clean":"2024-03-04T07:27:49.248987+0000","last_became_active":"2024-03-04T07:27:48.028699+0000","last_became_peered":"2024-03-04T07:27:48.028699+0000","last_unstale":"2024-03-04T07:27:49.248987+0000","last_undegraded":"2024-03-04T07:27:49.248987+0000","last_fullsized":"2024-03-04T07:27:49.248987+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:08.964724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.2b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016622+0000","last_change":"2024-03-04T07:27:48.022182+0000","last_active":"2024-03-04T07:27:49.016622+0000","last_peered":"2024-03-04T07:27:49.016622+0000","last_clean":"2024-03-04T07:27:49.016622+0000","last_became_active":"2024-03-04T07:27:48.021884+0000","last_became_peered":"2024-03-04T07:27:48.021884+0000","last_unstale":"2024-03-04T07:27:49.016622+0000","last_undegraded":"2024-03-04T07:27:49.016622+0000","last_fullsized":"2024-03-04T07:27:49.016622+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:44.655515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.2a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017358+0000","last_change":"2024-03-04T07:27:48.016300+0000","last_active":"2024-03-04T07:27:49.017358+0000","last_peered":"2024-03-04T07:27:49.017358+0000","last_clean":"2024-03-04T07:27:49.017358+0000","last_became_active":"2024-03-04T07:27:48.016060+0000","last_became_peered":"2024-03-04T07:27:48.016060+0000","last_unstale":"2024-03-04T07:27:49.017358+0000","last_undegraded":"2024-03-04T07:27:49.017358+0000","last_fullsized":"2024-03-04T07:27:49.017358+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:35:56.668208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.29","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.014908+0000","last_change":"2024-03-04T07:27:48.016375+0000","last_active":"2024-03-04T07:27:49.014908+0000","last_peered":"2024-03-04T07:27:49.014908+0000","last_clean":"2024-03-04T07:27:49.014908+0000","last_became_active":"2024-03-04T07:27:48.016116+0000","last_became_peered":"2024-03-04T07:27:48.016116+0000","last_unstale":"2024-03-04T07:27:49.014908+0000","last_undegraded":"2024-03-04T07:27:49.014908+0000","last_fullsized":"2024-03-04T07:27:49.014908+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:13.778791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.28","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016157+0000","last_change":"2024-03-04T07:27:48.021118+0000","last_active":"2024-03-04T07:27:49.016157+0000","last_peered":"2024-03-04T07:27:49.016157+0000","last_clean":"2024-03-04T07:27:49.016157+0000","last_became_active":"2024-03-04T07:27:48.020762+0000","last_became_peered":"2024-03-04T07:27:48.020762+0000","last_unstale":"2024-03-04T07:27:49.016157+0000","last_undegraded":"2024-03-04T07:27:49.016157+0000","last_fullsized":"2024-03-04T07:27:49.016157+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:59:54.559611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.27","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734231+0000","last_change":"2024-03-04T07:27:48.034905+0000","last_active":"2024-03-04T07:27:49.734231+0000","last_peered":"2024-03-04T07:27:49.734231+0000","last_clean":"2024-03-04T07:27:49.734231+0000","last_became_active":"2024-03-04T07:27:48.034544+0000","last_became_peered":"2024-03-04T07:27:48.034544+0000","last_unstale":"2024-03-04T07:27:49.734231+0000","last_undegraded":"2024-03-04T07:27:49.734231+0000","last_fullsized":"2024-03-04T07:27:49.734231+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:16.191994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.26","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733981+0000","last_change":"2024-03-04T07:27:48.018767+0000","last_active":"2024-03-04T07:27:49.733981+0000","last_peered":"2024-03-04T07:27:49.733981+0000","last_clean":"2024-03-04T07:27:49.733981+0000","last_became_active":"2024-03-04T07:27:48.018332+0000","last_became_peered":"2024-03-04T07:27:48.018332+0000","last_unstale":"2024-03-04T07:27:49.733981+0000","last_undegraded":"2024-03-04T07:27:49.733981+0000","last_fullsized":"2024-03-04T07:27:49.733981+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:45:31.067674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.25","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734543+0000","last_change":"2024-03-04T07:27:48.029454+0000","last_active":"2024-03-04T07:27:49.734543+0000","last_peered":"2024-03-04T07:27:49.734543+0000","last_clean":"2024-03-04T07:27:49.734543+0000","last_became_active":"2024-03-04T07:27:48.029164+0000","last_became_peered":"2024-03-04T07:27:48.029164+0000","last_unstale":"2024-03-04T07:27:49.734543+0000","last_undegraded":"2024-03-04T07:27:49.734543+0000","last_fullsized":"2024-03-04T07:27:49.734543+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:07:35.111370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.24","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017652+0000","last_change":"2024-03-04T07:27:48.023330+0000","last_active":"2024-03-04T07:27:49.017652+0000","last_peered":"2024-03-04T07:27:49.017652+0000","last_clean":"2024-03-04T07:27:49.017652+0000","last_became_active":"2024-03-04T07:27:48.022994+0000","last_became_peered":"2024-03-04T07:27:48.022994+0000","last_unstale":"2024-03-04T07:27:49.017652+0000","last_undegraded":"2024-03-04T07:27:49.017652+0000","last_fullsized":"2024-03-04T07:27:49.017652+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:21:17.862929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.23","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016812+0000","last_change":"2024-03-04T07:27:48.013870+0000","last_active":"2024-03-04T07:27:49.016812+0000","last_peered":"2024-03-04T07:27:49.016812+0000","last_clean":"2024-03-04T07:27:49.016812+0000","last_became_active":"2024-03-04T07:27:48.012559+0000","last_became_peered":"2024-03-04T07:27:48.012559+0000","last_unstale":"2024-03-04T07:27:49.016812+0000","last_undegraded":"2024-03-04T07:27:49.016812+0000","last_fullsized":"2024-03-04T07:27:49.016812+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:19:32.594240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.22","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733779+0000","last_change":"2024-03-04T07:27:48.023490+0000","last_active":"2024-03-04T07:27:49.733779+0000","last_peered":"2024-03-04T07:27:49.733779+0000","last_clean":"2024-03-04T07:27:49.733779+0000","last_became_active":"2024-03-04T07:27:48.022703+0000","last_became_peered":"2024-03-04T07:27:48.022703+0000","last_unstale":"2024-03-04T07:27:49.733779+0000","last_undegraded":"2024-03-04T07:27:49.733779+0000","last_fullsized":"2024-03-04T07:27:49.733779+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:05:48.078538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.21","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734127+0000","last_change":"2024-03-04T07:27:48.017975+0000","last_active":"2024-03-04T07:27:49.734127+0000","last_peered":"2024-03-04T07:27:49.734127+0000","last_clean":"2024-03-04T07:27:49.734127+0000","last_became_active":"2024-03-04T07:27:48.017677+0000","last_became_peered":"2024-03-04T07:27:48.017677+0000","last_unstale":"2024-03-04T07:27:49.734127+0000","last_undegraded":"2024-03-04T07:27:49.734127+0000","last_fullsized":"2024-03-04T07:27:49.734127+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:48.068194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.20","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733699+0000","last_change":"2024-03-04T07:27:48.029518+0000","last_active":"2024-03-04T07:27:49.733699+0000","last_peered":"2024-03-04T07:27:49.733699+0000","last_clean":"2024-03-04T07:27:49.733699+0000","last_became_active":"2024-03-04T07:27:48.029207+0000","last_became_peered":"2024-03-04T07:27:48.029207+0000","last_unstale":"2024-03-04T07:27:49.733699+0000","last_undegraded":"2024-03-04T07:27:49.733699+0000","last_fullsized":"2024-03-04T07:27:49.733699+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:37.454873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015069+0000","last_change":"2024-03-04T07:27:48.022992+0000","last_active":"2024-03-04T07:27:49.015069+0000","last_peered":"2024-03-04T07:27:49.015069+0000","last_clean":"2024-03-04T07:27:49.015069+0000","last_became_active":"2024-03-04T07:27:48.022667+0000","last_became_peered":"2024-03-04T07:27:48.022667+0000","last_unstale":"2024-03-04T07:27:49.015069+0000","last_undegraded":"2024-03-04T07:27:49.015069+0000","last_fullsized":"2024-03-04T07:27:49.015069+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:16.677787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"32'4","reported_seq":18,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.735025+0000","last_change":"2024-03-04T07:27:47.005371+0000","last_active":"2024-03-04T07:27:49.735025+0000","last_peered":"2024-03-04T07:27:49.735025+0000","last_clean":"2024-03-04T07:27:49.735025+0000","last_became_active":"2024-03-04T07:27:47.005047+0000","last_became_peered":"2024-03-04T07:27:47.005047+0000","last_unstale":"2024-03-04T07:27:49.735025+0000","last_undegraded":"2024-03-04T07:27:49.735025+0000","last_fullsized":"2024-03-04T07:27:49.735025+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:45.599314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.1f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734962+0000","last_change":"2024-03-04T07:27:48.024968+0000","last_active":"2024-03-04T07:27:49.734962+0000","last_peered":"2024-03-04T07:27:49.734962+0000","last_clean":"2024-03-04T07:27:49.734962+0000","last_became_active":"2024-03-04T07:27:48.024698+0000","last_became_peered":"2024-03-04T07:27:48.024698+0000","last_unstale":"2024-03-04T07:27:49.734962+0000","last_undegraded":"2024-03-04T07:27:49.734962+0000","last_fullsized":"2024-03-04T07:27:49.734962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:28.399564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1e","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015011+0000","last_change":"2024-03-04T07:27:47.009041+0000","last_active":"2024-03-04T07:27:49.015011+0000","last_peered":"2024-03-04T07:27:49.015011+0000","last_clean":"2024-03-04T07:27:49.015011+0000","last_became_active":"2024-03-04T07:27:47.008719+0000","last_became_peered":"2024-03-04T07:27:47.008719+0000","last_unstale":"2024-03-04T07:27:49.015011+0000","last_undegraded":"2024-03-04T07:27:49.015011+0000","last_fullsized":"2024-03-04T07:27:49.015011+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:46.997329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017027+0000","last_change":"2024-03-04T07:27:48.016516+0000","last_active":"2024-03-04T07:27:49.017027+0000","last_peered":"2024-03-04T07:27:49.017027+0000","last_clean":"2024-03-04T07:27:49.017027+0000","last_became_active":"2024-03-04T07:27:48.016227+0000","last_became_peered":"2024-03-04T07:27:48.016227+0000","last_unstale":"2024-03-04T07:27:49.017027+0000","last_undegraded":"2024-03-04T07:27:49.017027+0000","last_fullsized":"2024-03-04T07:27:49.017027+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:16.191994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733411+0000","last_change":"2024-03-04T07:27:47.009369+0000","last_active":"2024-03-04T07:27:49.733411+0000","last_peered":"2024-03-04T07:27:49.733411+0000","last_clean":"2024-03-04T07:27:49.733411+0000","last_became_active":"2024-03-04T07:27:47.007428+0000","last_became_peered":"2024-03-04T07:27:47.007428+0000","last_unstale":"2024-03-04T07:27:49.733411+0000","last_undegraded":"2024-03-04T07:27:49.733411+0000","last_fullsized":"2024-03-04T07:27:49.733411+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:15.671328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734608+0000","last_change":"2024-03-04T07:27:48.025612+0000","last_active":"2024-03-04T07:27:49.734608+0000","last_peered":"2024-03-04T07:27:49.734608+0000","last_clean":"2024-03-04T07:27:49.734608+0000","last_became_active":"2024-03-04T07:27:48.025330+0000","last_became_peered":"2024-03-04T07:27:48.025330+0000","last_unstale":"2024-03-04T07:27:49.734608+0000","last_undegraded":"2024-03-04T07:27:49.734608+0000","last_fullsized":"2024-03-04T07:27:49.734608+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:37.454873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.249042+0000","last_change":"2024-03-04T07:27:47.009650+0000","last_active":"2024-03-04T07:27:49.249042+0000","last_peered":"2024-03-04T07:27:49.249042+0000","last_clean":"2024-03-04T07:27:49.249042+0000","last_became_active":"2024-03-04T07:27:47.009381+0000","last_became_peered":"2024-03-04T07:27:47.009381+0000","last_unstale":"2024-03-04T07:27:49.249042+0000","last_undegraded":"2024-03-04T07:27:49.249042+0000","last_fullsized":"2024-03-04T07:27:49.249042+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:26:17.499963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733952+0000","last_change":"2024-03-04T07:27:48.029496+0000","last_active":"2024-03-04T07:27:49.733952+0000","last_peered":"2024-03-04T07:27:49.733952+0000","last_clean":"2024-03-04T07:27:49.733952+0000","last_became_active":"2024-03-04T07:27:48.029194+0000","last_became_peered":"2024-03-04T07:27:48.029194+0000","last_unstale":"2024-03-04T07:27:49.733952+0000","last_undegraded":"2024-03-04T07:27:49.733952+0000","last_fullsized":"2024-03-04T07:27:49.733952+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:31:24.297134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016871+0000","last_change":"2024-03-04T07:27:47.005110+0000","last_active":"2024-03-04T07:27:49.016871+0000","last_peered":"2024-03-04T07:27:49.016871+0000","last_clean":"2024-03-04T07:27:49.016871+0000","last_became_active":"2024-03-04T07:27:47.004794+0000","last_became_peered":"2024-03-04T07:27:47.004794+0000","last_unstale":"2024-03-04T07:27:49.016871+0000","last_undegraded":"2024-03-04T07:27:49.016871+0000","last_fullsized":"2024-03-04T07:27:49.016871+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:19:20.202177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016339+0000","last_change":"2024-03-04T07:27:48.025831+0000","last_active":"2024-03-04T07:27:49.016339+0000","last_peered":"2024-03-04T07:27:49.016339+0000","last_clean":"2024-03-04T07:27:49.016339+0000","last_became_active":"2024-03-04T07:27:48.025513+0000","last_became_peered":"2024-03-04T07:27:48.025513+0000","last_unstale":"2024-03-04T07:27:49.016339+0000","last_undegraded":"2024-03-04T07:27:49.016339+0000","last_fullsized":"2024-03-04T07:27:49.016339+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:05:48.078538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017281+0000","last_change":"2024-03-04T07:27:47.005798+0000","last_active":"2024-03-04T07:27:49.017281+0000","last_peered":"2024-03-04T07:27:49.017281+0000","last_clean":"2024-03-04T07:27:49.017281+0000","last_became_active":"2024-03-04T07:27:47.005477+0000","last_became_peered":"2024-03-04T07:27:47.005477+0000","last_unstale":"2024-03-04T07:27:49.017281+0000","last_undegraded":"2024-03-04T07:27:49.017281+0000","last_fullsized":"2024-03-04T07:27:49.017281+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:53:22.194744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016315+0000","last_change":"2024-03-04T07:27:48.011650+0000","last_active":"2024-03-04T07:27:49.016315+0000","last_peered":"2024-03-04T07:27:49.016315+0000","last_clean":"2024-03-04T07:27:49.016315+0000","last_became_active":"2024-03-04T07:27:48.011360+0000","last_became_peered":"2024-03-04T07:27:48.011360+0000","last_unstale":"2024-03-04T07:27:49.016315+0000","last_undegraded":"2024-03-04T07:27:49.016315+0000","last_fullsized":"2024-03-04T07:27:49.016315+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:16.677787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733218+0000","last_change":"2024-03-04T07:27:47.009392+0000","last_active":"2024-03-04T07:27:49.733218+0000","last_peered":"2024-03-04T07:27:49.733218+0000","last_clean":"2024-03-04T07:27:49.733218+0000","last_became_active":"2024-03-04T07:27:47.008527+0000","last_became_peered":"2024-03-04T07:27:47.008527+0000","last_unstale":"2024-03-04T07:27:49.733218+0000","last_undegraded":"2024-03-04T07:27:49.733218+0000","last_fullsized":"2024-03-04T07:27:49.733218+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:12.772332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733549+0000","last_change":"2024-03-04T07:27:48.030958+0000","last_active":"2024-03-04T07:27:49.733549+0000","last_peered":"2024-03-04T07:27:49.733549+0000","last_clean":"2024-03-04T07:27:49.733549+0000","last_became_active":"2024-03-04T07:27:48.030659+0000","last_became_peered":"2024-03-04T07:27:48.030659+0000","last_unstale":"2024-03-04T07:27:49.733549+0000","last_undegraded":"2024-03-04T07:27:49.733549+0000","last_fullsized":"2024-03-04T07:27:49.733549+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:44.655515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015996+0000","last_change":"2024-03-04T07:27:46.997525+0000","last_active":"2024-03-04T07:27:49.015996+0000","last_peered":"2024-03-04T07:27:49.015996+0000","last_clean":"2024-03-04T07:27:49.015996+0000","last_became_active":"2024-03-04T07:27:46.997199+0000","last_became_peered":"2024-03-04T07:27:46.997199+0000","last_unstale":"2024-03-04T07:27:49.015996+0000","last_undegraded":"2024-03-04T07:27:49.015996+0000","last_fullsized":"2024-03-04T07:27:49.015996+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:13.016918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.16","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016786+0000","last_change":"2024-03-04T07:27:48.014703+0000","last_active":"2024-03-04T07:27:49.016786+0000","last_peered":"2024-03-04T07:27:49.016786+0000","last_clean":"2024-03-04T07:27:49.016786+0000","last_became_active":"2024-03-04T07:27:48.014408+0000","last_became_peered":"2024-03-04T07:27:48.014408+0000","last_unstale":"2024-03-04T07:27:49.016786+0000","last_undegraded":"2024-03-04T07:27:49.016786+0000","last_fullsized":"2024-03-04T07:27:49.016786+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:46:40.563493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733743+0000","last_change":"2024-03-04T07:27:47.009140+0000","last_active":"2024-03-04T07:27:49.733743+0000","last_peered":"2024-03-04T07:27:49.733743+0000","last_clean":"2024-03-04T07:27:49.733743+0000","last_became_active":"2024-03-04T07:27:47.008873+0000","last_became_peered":"2024-03-04T07:27:47.008873+0000","last_unstale":"2024-03-04T07:27:49.733743+0000","last_undegraded":"2024-03-04T07:27:49.733743+0000","last_fullsized":"2024-03-04T07:27:49.733743+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:43.649055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.17","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733783+0000","last_change":"2024-03-04T07:27:48.025212+0000","last_active":"2024-03-04T07:27:49.733783+0000","last_peered":"2024-03-04T07:27:49.733783+0000","last_clean":"2024-03-04T07:27:49.733783+0000","last_became_active":"2024-03-04T07:27:48.024941+0000","last_became_peered":"2024-03-04T07:27:48.024941+0000","last_unstale":"2024-03-04T07:27:49.733783+0000","last_undegraded":"2024-03-04T07:27:49.733783+0000","last_fullsized":"2024-03-04T07:27:49.733783+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:39.277425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015150+0000","last_change":"2024-03-04T07:27:47.002804+0000","last_active":"2024-03-04T07:27:49.015150+0000","last_peered":"2024-03-04T07:27:49.015150+0000","last_clean":"2024-03-04T07:27:49.015150+0000","last_became_active":"2024-03-04T07:27:47.002319+0000","last_became_peered":"2024-03-04T07:27:47.002319+0000","last_unstale":"2024-03-04T07:27:49.015150+0000","last_undegraded":"2024-03-04T07:27:49.015150+0000","last_fullsized":"2024-03-04T07:27:49.015150+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:07.958265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.14","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733760+0000","last_change":"2024-03-04T07:27:48.018348+0000","last_active":"2024-03-04T07:27:49.733760+0000","last_peered":"2024-03-04T07:27:49.733760+0000","last_clean":"2024-03-04T07:27:49.733760+0000","last_became_active":"2024-03-04T07:27:48.018094+0000","last_became_peered":"2024-03-04T07:27:48.018094+0000","last_unstale":"2024-03-04T07:27:49.733760+0000","last_undegraded":"2024-03-04T07:27:49.733760+0000","last_fullsized":"2024-03-04T07:27:49.733760+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:21:17.862929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"32'3","reported_seq":17,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016775+0000","last_change":"2024-03-04T07:27:47.008962+0000","last_active":"2024-03-04T07:27:49.016775+0000","last_peered":"2024-03-04T07:27:49.016775+0000","last_clean":"2024-03-04T07:27:49.016775+0000","last_became_active":"2024-03-04T07:27:47.008580+0000","last_became_peered":"2024-03-04T07:27:47.008580+0000","last_unstale":"2024-03-04T07:27:49.016775+0000","last_undegraded":"2024-03-04T07:27:49.016775+0000","last_fullsized":"2024-03-04T07:27:49.016775+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:47.061734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.15","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016467+0000","last_change":"2024-03-04T07:27:48.026475+0000","last_active":"2024-03-04T07:27:49.016467+0000","last_peered":"2024-03-04T07:27:49.016467+0000","last_clean":"2024-03-04T07:27:49.016467+0000","last_became_active":"2024-03-04T07:27:48.025361+0000","last_became_peered":"2024-03-04T07:27:48.025361+0000","last_unstale":"2024-03-04T07:27:49.016467+0000","last_undegraded":"2024-03-04T07:27:49.016467+0000","last_fullsized":"2024-03-04T07:27:49.016467+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:32.542508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"32'6","reported_seq":20,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733731+0000","last_change":"2024-03-04T07:27:47.008359+0000","last_active":"2024-03-04T07:27:49.733731+0000","last_peered":"2024-03-04T07:27:49.733731+0000","last_clean":"2024-03-04T07:27:49.733731+0000","last_became_active":"2024-03-04T07:27:47.008066+0000","last_became_peered":"2024-03-04T07:27:47.008066+0000","last_unstale":"2024-03-04T07:27:49.733731+0000","last_undegraded":"2024-03-04T07:27:49.733731+0000","last_fullsized":"2024-03-04T07:27:49.733731+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:58.050466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.12","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733839+0000","last_change":"2024-03-04T07:27:48.033929+0000","last_active":"2024-03-04T07:27:49.733839+0000","last_peered":"2024-03-04T07:27:49.733839+0000","last_clean":"2024-03-04T07:27:49.733839+0000","last_became_active":"2024-03-04T07:27:48.033514+0000","last_became_peered":"2024-03-04T07:27:48.033514+0000","last_unstale":"2024-03-04T07:27:49.733839+0000","last_undegraded":"2024-03-04T07:27:49.733839+0000","last_fullsized":"2024-03-04T07:27:49.733839+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:19:32.594240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016989+0000","last_change":"2024-03-04T07:27:46.998487+0000","last_active":"2024-03-04T07:27:49.016989+0000","last_peered":"2024-03-04T07:27:49.016989+0000","last_clean":"2024-03-04T07:27:49.016989+0000","last_became_active":"2024-03-04T07:27:46.997966+0000","last_became_peered":"2024-03-04T07:27:46.997966+0000","last_unstale":"2024-03-04T07:27:49.016989+0000","last_undegraded":"2024-03-04T07:27:49.016989+0000","last_fullsized":"2024-03-04T07:27:49.016989+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:46.997329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733734+0000","last_change":"2024-03-04T07:27:48.021887+0000","last_active":"2024-03-04T07:27:49.733734+0000","last_peered":"2024-03-04T07:27:49.733734+0000","last_clean":"2024-03-04T07:27:49.733734+0000","last_became_active":"2024-03-04T07:27:48.021529+0000","last_became_peered":"2024-03-04T07:27:48.021529+0000","last_unstale":"2024-03-04T07:27:49.733734+0000","last_undegraded":"2024-03-04T07:27:49.733734+0000","last_fullsized":"2024-03-04T07:27:49.733734+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:51:19.707477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733863+0000","last_change":"2024-03-04T07:27:47.001624+0000","last_active":"2024-03-04T07:27:49.733863+0000","last_peered":"2024-03-04T07:27:49.733863+0000","last_clean":"2024-03-04T07:27:49.733863+0000","last_became_active":"2024-03-04T07:27:47.001341+0000","last_became_peered":"2024-03-04T07:27:47.001341+0000","last_unstale":"2024-03-04T07:27:49.733863+0000","last_undegraded":"2024-03-04T07:27:49.733863+0000","last_fullsized":"2024-03-04T07:27:49.733863+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:45.599314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.10","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734601+0000","last_change":"2024-03-04T07:27:48.029489+0000","last_active":"2024-03-04T07:27:49.734601+0000","last_peered":"2024-03-04T07:27:49.734601+0000","last_clean":"2024-03-04T07:27:49.734601+0000","last_became_active":"2024-03-04T07:27:48.029189+0000","last_became_peered":"2024-03-04T07:27:48.029189+0000","last_unstale":"2024-03-04T07:27:49.734601+0000","last_undegraded":"2024-03-04T07:27:49.734601+0000","last_fullsized":"2024-03-04T07:27:49.734601+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:21:17.862929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733552+0000","last_change":"2024-03-04T07:27:47.009153+0000","last_active":"2024-03-04T07:27:49.733552+0000","last_peered":"2024-03-04T07:27:49.733552+0000","last_clean":"2024-03-04T07:27:49.733552+0000","last_became_active":"2024-03-04T07:27:47.008885+0000","last_became_peered":"2024-03-04T07:27:47.008885+0000","last_unstale":"2024-03-04T07:27:49.733552+0000","last_undegraded":"2024-03-04T07:27:49.733552+0000","last_fullsized":"2024-03-04T07:27:49.733552+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:47.061734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734300+0000","last_change":"2024-03-04T07:27:48.011225+0000","last_active":"2024-03-04T07:27:49.734300+0000","last_peered":"2024-03-04T07:27:49.734300+0000","last_clean":"2024-03-04T07:27:49.734300+0000","last_became_active":"2024-03-04T07:27:48.010909+0000","last_became_peered":"2024-03-04T07:27:48.010909+0000","last_unstale":"2024-03-04T07:27:49.734300+0000","last_undegraded":"2024-03-04T07:27:49.734300+0000","last_fullsized":"2024-03-04T07:27:49.734300+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:09:16.677787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"32'8","reported_seq":22,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733266+0000","last_change":"2024-03-04T07:27:47.007810+0000","last_active":"2024-03-04T07:27:49.733266+0000","last_peered":"2024-03-04T07:27:49.733266+0000","last_clean":"2024-03-04T07:27:49.733266+0000","last_became_active":"2024-03-04T07:27:47.007524+0000","last_became_peered":"2024-03-04T07:27:47.007524+0000","last_unstale":"2024-03-04T07:27:49.733266+0000","last_undegraded":"2024-03-04T07:27:49.733266+0000","last_fullsized":"2024-03-04T07:27:49.733266+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:27.393104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.e","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016651+0000","last_change":"2024-03-04T07:27:48.013051+0000","last_active":"2024-03-04T07:27:49.016651+0000","last_peered":"2024-03-04T07:27:49.016651+0000","last_clean":"2024-03-04T07:27:49.016651+0000","last_became_active":"2024-03-04T07:27:48.012763+0000","last_became_peered":"2024-03-04T07:27:48.012763+0000","last_unstale":"2024-03-04T07:27:49.016651+0000","last_undegraded":"2024-03-04T07:27:49.016651+0000","last_fullsized":"2024-03-04T07:27:49.016651+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:27:28.399564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734353+0000","last_change":"2024-03-04T07:27:47.007400+0000","last_active":"2024-03-04T07:27:49.734353+0000","last_peered":"2024-03-04T07:27:49.734353+0000","last_clean":"2024-03-04T07:27:49.734353+0000","last_became_active":"2024-03-04T07:27:47.007085+0000","last_became_peered":"2024-03-04T07:27:47.007085+0000","last_unstale":"2024-03-04T07:27:49.734353+0000","last_undegraded":"2024-03-04T07:27:49.734353+0000","last_fullsized":"2024-03-04T07:27:49.734353+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:07.958265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.f","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016995+0000","last_change":"2024-03-04T07:27:48.010695+0000","last_active":"2024-03-04T07:27:49.016995+0000","last_peered":"2024-03-04T07:27:49.016995+0000","last_clean":"2024-03-04T07:27:49.016995+0000","last_became_active":"2024-03-04T07:27:48.009424+0000","last_became_peered":"2024-03-04T07:27:48.009424+0000","last_unstale":"2024-03-04T07:27:49.016995+0000","last_undegraded":"2024-03-04T07:27:49.016995+0000","last_fullsized":"2024-03-04T07:27:49.016995+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:43.636899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017232+0000","last_change":"2024-03-04T07:27:46.997566+0000","last_active":"2024-03-04T07:27:49.017232+0000","last_peered":"2024-03-04T07:27:49.017232+0000","last_clean":"2024-03-04T07:27:49.017232+0000","last_became_active":"2024-03-04T07:27:46.997261+0000","last_became_peered":"2024-03-04T07:27:46.997261+0000","last_unstale":"2024-03-04T07:27:49.017232+0000","last_undegraded":"2024-03-04T07:27:49.017232+0000","last_fullsized":"2024-03-04T07:27:49.017232+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:20:07.958265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734442+0000","last_change":"2024-03-04T07:27:48.029455+0000","last_active":"2024-03-04T07:27:49.734442+0000","last_peered":"2024-03-04T07:27:49.734442+0000","last_clean":"2024-03-04T07:27:49.734442+0000","last_became_active":"2024-03-04T07:27:48.029135+0000","last_became_peered":"2024-03-04T07:27:48.029135+0000","last_unstale":"2024-03-04T07:27:49.734442+0000","last_undegraded":"2024-03-04T07:27:49.734442+0000","last_fullsized":"2024-03-04T07:27:49.734442+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:18:04.118075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016806+0000","last_change":"2024-03-04T07:27:47.005847+0000","last_active":"2024-03-04T07:27:49.016806+0000","last_peered":"2024-03-04T07:27:49.016806+0000","last_clean":"2024-03-04T07:27:49.016806+0000","last_became_active":"2024-03-04T07:27:47.005527+0000","last_became_peered":"2024-03-04T07:27:47.005527+0000","last_unstale":"2024-03-04T07:27:49.016806+0000","last_undegraded":"2024-03-04T07:27:49.016806+0000","last_fullsized":"2024-03-04T07:27:49.016806+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:51:53.997133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248979+0000","last_change":"2024-03-04T07:27:48.024600+0000","last_active":"2024-03-04T07:27:49.248979+0000","last_peered":"2024-03-04T07:27:49.248979+0000","last_clean":"2024-03-04T07:27:49.248979+0000","last_became_active":"2024-03-04T07:27:48.024320+0000","last_became_peered":"2024-03-04T07:27:48.024320+0000","last_unstale":"2024-03-04T07:27:49.248979+0000","last_undegraded":"2024-03-04T07:27:49.248979+0000","last_fullsized":"2024-03-04T07:27:49.248979+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:19:21.208636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017087+0000","last_change":"2024-03-04T07:27:47.004975+0000","last_active":"2024-03-04T07:27:49.017087+0000","last_peered":"2024-03-04T07:27:49.017087+0000","last_clean":"2024-03-04T07:27:49.017087+0000","last_became_active":"2024-03-04T07:27:47.004664+0000","last_became_peered":"2024-03-04T07:27:47.004664+0000","last_unstale":"2024-03-04T07:27:49.017087+0000","last_undegraded":"2024-03-04T07:27:49.017087+0000","last_fullsized":"2024-03-04T07:27:49.017087+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:05:08.012665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248882+0000","last_change":"2024-03-04T07:27:48.015787+0000","last_active":"2024-03-04T07:27:49.248882+0000","last_peered":"2024-03-04T07:27:49.248882+0000","last_clean":"2024-03-04T07:27:49.248882+0000","last_became_active":"2024-03-04T07:27:48.015481+0000","last_became_peered":"2024-03-04T07:27:48.015481+0000","last_unstale":"2024-03-04T07:27:49.248882+0000","last_undegraded":"2024-03-04T07:27:49.248882+0000","last_fullsized":"2024-03-04T07:27:49.248882+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:32:26.126570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015186+0000","last_change":"2024-03-04T07:27:47.005303+0000","last_active":"2024-03-04T07:27:49.015186+0000","last_peered":"2024-03-04T07:27:49.015186+0000","last_clean":"2024-03-04T07:27:49.015186+0000","last_became_active":"2024-03-04T07:27:47.005013+0000","last_became_peered":"2024-03-04T07:27:47.005013+0000","last_unstale":"2024-03-04T07:27:49.015186+0000","last_undegraded":"2024-03-04T07:27:49.015186+0000","last_fullsized":"2024-03-04T07:27:49.015186+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:45.599314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.249078+0000","last_change":"2024-03-04T07:27:48.020878+0000","last_active":"2024-03-04T07:27:49.249078+0000","last_peered":"2024-03-04T07:27:49.249078+0000","last_clean":"2024-03-04T07:27:49.249078+0000","last_became_active":"2024-03-04T07:27:48.020586+0000","last_became_peered":"2024-03-04T07:27:48.020586+0000","last_unstale":"2024-03-04T07:27:49.249078+0000","last_undegraded":"2024-03-04T07:27:49.249078+0000","last_fullsized":"2024-03-04T07:27:49.249078+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:36.360666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734662+0000","last_change":"2024-03-04T07:27:47.006202+0000","last_active":"2024-03-04T07:27:49.734662+0000","last_peered":"2024-03-04T07:27:49.734662+0000","last_clean":"2024-03-04T07:27:49.734662+0000","last_became_active":"2024-03-04T07:27:47.005900+0000","last_became_peered":"2024-03-04T07:27:47.005900+0000","last_unstale":"2024-03-04T07:27:49.734662+0000","last_undegraded":"2024-03-04T07:27:49.734662+0000","last_fullsized":"2024-03-04T07:27:49.734662+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:35.354207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.248803+0000","last_change":"2024-03-04T07:27:48.019537+0000","last_active":"2024-03-04T07:27:49.248803+0000","last_peered":"2024-03-04T07:27:49.248803+0000","last_clean":"2024-03-04T07:27:49.248803+0000","last_became_active":"2024-03-04T07:27:48.019282+0000","last_became_peered":"2024-03-04T07:27:48.019282+0000","last_unstale":"2024-03-04T07:27:49.248803+0000","last_undegraded":"2024-03-04T07:27:49.248803+0000","last_fullsized":"2024-03-04T07:27:49.248803+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:03:00.941676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016562+0000","last_change":"2024-03-04T07:27:47.003400+0000","last_active":"2024-03-04T07:27:49.016562+0000","last_peered":"2024-03-04T07:27:49.016562+0000","last_clean":"2024-03-04T07:27:49.016562+0000","last_became_active":"2024-03-04T07:27:47.003071+0000","last_became_peered":"2024-03-04T07:27:47.003071+0000","last_unstale":"2024-03-04T07:27:49.016562+0000","last_undegraded":"2024-03-04T07:27:49.016562+0000","last_fullsized":"2024-03-04T07:27:49.016562+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:02:59.935216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.9","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733749+0000","last_change":"2024-03-04T07:27:48.020621+0000","last_active":"2024-03-04T07:27:49.733749+0000","last_peered":"2024-03-04T07:27:49.733749+0000","last_clean":"2024-03-04T07:27:49.733749+0000","last_became_active":"2024-03-04T07:27:48.020324+0000","last_became_peered":"2024-03-04T07:27:48.020324+0000","last_unstale":"2024-03-04T07:27:49.733749+0000","last_undegraded":"2024-03-04T07:27:49.733749+0000","last_fullsized":"2024-03-04T07:27:49.733749+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:59:54.559611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016223+0000","last_change":"2024-03-04T07:27:47.008049+0000","last_active":"2024-03-04T07:27:49.016223+0000","last_peered":"2024-03-04T07:27:49.016223+0000","last_clean":"2024-03-04T07:27:49.016223+0000","last_became_active":"2024-03-04T07:27:47.007495+0000","last_became_peered":"2024-03-04T07:27:47.007495+0000","last_unstale":"2024-03-04T07:27:49.016223+0000","last_undegraded":"2024-03-04T07:27:49.016223+0000","last_fullsized":"2024-03-04T07:27:49.016223+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:12.772332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.249000+0000","last_change":"2024-03-04T07:27:48.028254+0000","last_active":"2024-03-04T07:27:49.249000+0000","last_peered":"2024-03-04T07:27:49.249000+0000","last_clean":"2024-03-04T07:27:49.249000+0000","last_became_active":"2024-03-04T07:27:48.027969+0000","last_became_peered":"2024-03-04T07:27:48.027969+0000","last_unstale":"2024-03-04T07:27:49.249000+0000","last_undegraded":"2024-03-04T07:27:49.249000+0000","last_fullsized":"2024-03-04T07:27:49.249000+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:33:46.605774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"32'2","reported_seq":16,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734456+0000","last_change":"2024-03-04T07:27:47.002163+0000","last_active":"2024-03-04T07:27:49.734456+0000","last_peered":"2024-03-04T07:27:49.734456+0000","last_clean":"2024-03-04T07:27:49.734456+0000","last_became_active":"2024-03-04T07:27:47.001891+0000","last_became_peered":"2024-03-04T07:27:47.001891+0000","last_unstale":"2024-03-04T07:27:49.734456+0000","last_undegraded":"2024-03-04T07:27:49.734456+0000","last_fullsized":"2024-03-04T07:27:49.734456+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:49.727013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.7","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.017076+0000","last_change":"2024-03-04T07:27:48.025663+0000","last_active":"2024-03-04T07:27:49.017076+0000","last_peered":"2024-03-04T07:27:49.017076+0000","last_clean":"2024-03-04T07:27:49.017076+0000","last_became_active":"2024-03-04T07:27:48.025271+0000","last_became_peered":"2024-03-04T07:27:48.025271+0000","last_unstale":"2024-03-04T07:27:49.017076+0000","last_undegraded":"2024-03-04T07:27:49.017076+0000","last_fullsized":"2024-03-04T07:27:49.017076+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:06:38.256223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"32'1","reported_seq":15,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015148+0000","last_change":"2024-03-04T07:27:46.997285+0000","last_active":"2024-03-04T07:27:49.015148+0000","last_peered":"2024-03-04T07:27:49.015148+0000","last_clean":"2024-03-04T07:27:49.015148+0000","last_became_active":"2024-03-04T07:27:46.997001+0000","last_became_peered":"2024-03-04T07:27:46.997001+0000","last_unstale":"2024-03-04T07:27:49.015148+0000","last_undegraded":"2024-03-04T07:27:49.015148+0000","last_fullsized":"2024-03-04T07:27:49.015148+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:49.727013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.014958+0000","last_change":"2024-03-04T07:27:48.026814+0000","last_active":"2024-03-04T07:27:49.014958+0000","last_peered":"2024-03-04T07:27:49.014958+0000","last_clean":"2024-03-04T07:27:49.014958+0000","last_became_active":"2024-03-04T07:27:48.026322+0000","last_became_peered":"2024-03-04T07:27:48.026322+0000","last_unstale":"2024-03-04T07:27:49.014958+0000","last_undegraded":"2024-03-04T07:27:49.014958+0000","last_fullsized":"2024-03-04T07:27:49.014958+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:54:48.068194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734485+0000","last_change":"2024-03-04T07:27:47.010602+0000","last_active":"2024-03-04T07:27:49.734485+0000","last_peered":"2024-03-04T07:27:49.734485+0000","last_clean":"2024-03-04T07:27:49.734485+0000","last_became_active":"2024-03-04T07:27:47.010286+0000","last_became_peered":"2024-03-04T07:27:47.010286+0000","last_unstale":"2024-03-04T07:27:49.734485+0000","last_undegraded":"2024-03-04T07:27:49.734485+0000","last_fullsized":"2024-03-04T07:27:49.734485+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:49:46.997329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734820+0000","last_change":"2024-03-04T07:27:48.019333+0000","last_active":"2024-03-04T07:27:49.734820+0000","last_peered":"2024-03-04T07:27:49.734820+0000","last_clean":"2024-03-04T07:27:49.734820+0000","last_became_active":"2024-03-04T07:27:48.019037+0000","last_became_peered":"2024-03-04T07:27:48.019037+0000","last_unstale":"2024-03-04T07:27:49.734820+0000","last_undegraded":"2024-03-04T07:27:49.734820+0000","last_fullsized":"2024-03-04T07:27:49.734820+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:59.056925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015253+0000","last_change":"2024-03-04T07:27:46.996003+0000","last_active":"2024-03-04T07:27:49.015253+0000","last_peered":"2024-03-04T07:27:49.015253+0000","last_clean":"2024-03-04T07:27:49.015253+0000","last_became_active":"2024-03-04T07:27:46.995495+0000","last_became_peered":"2024-03-04T07:27:46.995495+0000","last_unstale":"2024-03-04T07:27:49.015253+0000","last_undegraded":"2024-03-04T07:27:49.015253+0000","last_fullsized":"2024-03-04T07:27:49.015253+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:35.354207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.015119+0000","last_change":"2024-03-04T07:27:48.020300+0000","last_active":"2024-03-04T07:27:49.015119+0000","last_peered":"2024-03-04T07:27:49.015119+0000","last_clean":"2024-03-04T07:27:49.015119+0000","last_became_active":"2024-03-04T07:27:48.020050+0000","last_became_peered":"2024-03-04T07:27:48.020050+0000","last_unstale":"2024-03-04T07:27:49.015119+0000","last_undegraded":"2024-03-04T07:27:49.015119+0000","last_fullsized":"2024-03-04T07:27:49.015119+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:59.056925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733888+0000","last_change":"2024-03-04T07:27:47.003837+0000","last_active":"2024-03-04T07:27:49.733888+0000","last_peered":"2024-03-04T07:27:49.733888+0000","last_clean":"2024-03-04T07:27:49.733888+0000","last_became_active":"2024-03-04T07:27:47.003534+0000","last_became_peered":"2024-03-04T07:27:47.003534+0000","last_unstale":"2024-03-04T07:27:49.733888+0000","last_undegraded":"2024-03-04T07:27:49.733888+0000","last_fullsized":"2024-03-04T07:27:49.733888+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:35.354207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733277+0000","last_change":"2024-03-04T07:27:48.027090+0000","last_active":"2024-03-04T07:27:49.733277+0000","last_peered":"2024-03-04T07:27:49.733277+0000","last_clean":"2024-03-04T07:27:49.733277+0000","last_became_active":"2024-03-04T07:27:48.026830+0000","last_became_peered":"2024-03-04T07:27:48.026830+0000","last_unstale":"2024-03-04T07:27:49.733277+0000","last_undegraded":"2024-03-04T07:27:49.733277+0000","last_fullsized":"2024-03-04T07:27:49.733277+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:43.636899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016689+0000","last_change":"2024-03-04T07:27:46.996917+0000","last_active":"2024-03-04T07:27:49.016689+0000","last_peered":"2024-03-04T07:27:49.016689+0000","last_clean":"2024-03-04T07:27:49.016689+0000","last_became_active":"2024-03-04T07:27:46.996595+0000","last_became_peered":"2024-03-04T07:27:46.996595+0000","last_unstale":"2024-03-04T07:27:49.016689+0000","last_undegraded":"2024-03-04T07:27:49.016689+0000","last_fullsized":"2024-03-04T07:27:49.016689+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:32:25.120111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.016490+0000","last_change":"2024-03-04T07:27:48.013371+0000","last_active":"2024-03-04T07:27:49.016490+0000","last_peered":"2024-03-04T07:27:49.016490+0000","last_clean":"2024-03-04T07:27:49.016490+0000","last_became_active":"2024-03-04T07:27:48.012901+0000","last_became_peered":"2024-03-04T07:27:48.012901+0000","last_unstale":"2024-03-04T07:27:49.016490+0000","last_undegraded":"2024-03-04T07:27:49.016490+0000","last_fullsized":"2024-03-04T07:27:49.016490+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:30:59.056925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733327+0000","last_change":"2024-03-04T07:27:47.009841+0000","last_active":"2024-03-04T07:27:49.733327+0000","last_peered":"2024-03-04T07:27:49.733327+0000","last_clean":"2024-03-04T07:27:49.733327+0000","last_became_active":"2024-03-04T07:27:47.009498+0000","last_became_peered":"2024-03-04T07:27:47.009498+0000","last_unstale":"2024-03-04T07:27:49.733327+0000","last_undegraded":"2024-03-04T07:27:49.733327+0000","last_fullsized":"2024-03-04T07:27:49.733327+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:19:31.587781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.4","version":"0'0","reported_seq":12,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.733939+0000","last_change":"2024-03-04T07:27:48.021305+0000","last_active":"2024-03-04T07:27:49.733939+0000","last_peered":"2024-03-04T07:27:49.733939+0000","last_clean":"2024-03-04T07:27:49.733939+0000","last_became_active":"2024-03-04T07:27:48.021021+0000","last_became_peered":"2024-03-04T07:27:48.021021+0000","last_unstale":"2024-03-04T07:27:49.733939+0000","last_undegraded":"2024-03-04T07:27:49.733939+0000","last_fullsized":"2024-03-04T07:27:49.733939+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":31,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:46.982587+0000","last_clean_scrub_stamp":"2024-03-04T07:27:46.982587+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:16.596129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734692+0000","last_change":"2024-03-04T07:27:46.999584+0000","last_active":"2024-03-04T07:27:49.734692+0000","last_peered":"2024-03-04T07:27:49.734692+0000","last_clean":"2024-03-04T07:27:49.734692+0000","last_became_active":"2024-03-04T07:27:46.999273+0000","last_became_peered":"2024-03-04T07:27:46.999273+0000","last_unstale":"2024-03-04T07:27:49.734692+0000","last_undegraded":"2024-03-04T07:27:49.734692+0000","last_fullsized":"2024-03-04T07:27:49.734692+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":30,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:27:45.976127+0000","last_clean_scrub_stamp":"2024-03-04T07:27:45.976127+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:22:49.727013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":193,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-03-04T07:27:49.734335+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:27:49.734335+0000","last_peered":"2024-03-04T07:27:49.734335+0000","last_clean":"2024-03-04T07:27:49.734335+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:27:49.734335+0000","last_undegraded":"2024-03-04T07:27:49.734335+0000","last_fullsized":"2024-03-04T07:27:49.734335+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574867,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27972,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343868,"statfs":{"total":96636764160,"available":96608120832,"internally_reserved":0,"allocated":970752,"data_stored":737066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":27654602},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574866,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":372,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":380928,"data_stored":147224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574866,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":372,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":380928,"data_stored":147280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574867,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27396,"kb_used_data":372,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344444,"statfs":{"total":96636764160,"available":96608710656,"internally_reserved":0,"allocated":380928,"data_stored":146712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574866,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27404,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344436,"statfs":{"total":96636764160,"available":96608702464,"internally_reserved":0,"allocated":389120,"data_stored":148242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574866,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27392,"kb_used_data":368,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344448,"statfs":{"total":96636764160,"available":96608714752,"internally_reserved":0,"allocated":376832,"data_stored":146678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574866,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27404,"kb_used_data":380,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344436,"statfs":{"total":96636764160,"available":96608702464,"internally_reserved":0,"allocated":389120,"data_stored":148242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574866,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27968,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343872,"statfs":{"total":96636764160,"available":96608124928,"internally_reserved":0,"allocated":966656,"data_stored":736976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:27:56.934 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:27:56.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:27:57.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.080+0000 7f45224eb640 1 Processor -- start 2024-03-04T07:27:57.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.080+0000 7f45224eb640 1 -- start start 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f45224eb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f451c0ceea0 0x7f451c0d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f45224eb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f451c0d3740 con 0x7f451c0b6d10 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f45224eb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f451c0d3120 con 0x7f451c15dd90 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f45224eb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f451c0d3400 con 0x7f451c0ceea0 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f451b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f451c0ceea0 0x7f451c0d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f451b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f451c0ceea0 0x7f451c0d29d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49028/0 (socket says 172.21.15.64:49028) 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f451b7fe640 1 -- 172.21.15.64:0/791544845 learned_addr learned my addr 172.21.15.64:0/791544845 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f4520a88640 1 --1- 172.21.15.64:0/791544845 >> v1:172.21.15.64:6789/0 conn(0x7f451c0b6d10 0x7f451c0d0c30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54704/0 (socket says 172.21.15.64:54704) 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f451b7fe640 1 -- 172.21.15.64:0/791544845 >> v1:172.21.15.66:6789/0 conn(0x7f451c15dd90 legacy=0x7f451c0d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f451b7fe640 1 -- 172.21.15.64:0/791544845 >> v1:172.21.15.64:6789/0 conn(0x7f451c0b6d10 legacy=0x7f451c0d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.081+0000 7f451b7fe640 1 -- 172.21.15.64:0/791544845 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f451c0d39e0 con 0x7f451c0ceea0 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.082+0000 7f451b7fe640 1 --2- 172.21.15.64:0/791544845 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0ceea0 0x7f451c0d29d0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f4504009870 tx=0x7f450402ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=550bd4cba3510328 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.082+0000 7f451affd640 1 -- 172.21.15.64:0/791544845 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f450402f840 con 0x7f451c0ceea0 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.082+0000 7f451affd640 1 -- 172.21.15.64:0/791544845 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f450402f9e0 con 0x7f451c0ceea0 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.082+0000 7f451affd640 1 -- 172.21.15.64:0/791544845 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f450402fdb0 con 0x7f451c0ceea0 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.083+0000 7f45224eb640 1 -- 172.21.15.64:0/791544845 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0ceea0 msgr2=0x7f451c0d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.083+0000 7f45224eb640 1 --2- 172.21.15.64:0/791544845 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0ceea0 0x7f451c0d29d0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f4504009870 tx=0x7f450402ba20 comp rx=0 tx=0).stop 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.083+0000 7f45224eb640 1 -- 172.21.15.64:0/791544845 shutdown_connections 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.083+0000 7f45224eb640 1 --2- 172.21.15.64:0/791544845 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0ceea0 0x7f451c0d29d0 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.083+0000 7f45224eb640 1 -- 172.21.15.64:0/791544845 >> 172.21.15.64:0/791544845 conn(0x7f451c071960 msgr2=0x7f451c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.083+0000 7f45224eb640 1 -- 172.21.15.64:0/791544845 shutdown_connections 2024-03-04T07:27:57.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.083+0000 7f45224eb640 1 -- 172.21.15.64:0/791544845 wait complete. 2024-03-04T07:27:57.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.084+0000 7f45224eb640 1 Processor -- start 2024-03-04T07:27:57.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.084+0000 7f45224eb640 1 -- start start 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f45224eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0b6d10 0x7f451c1495e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f45224eb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f451c0ceea0 0x7f451c1f7510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f45224eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f451c15dd90 0x7f451c160e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f45224eb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f451c0d48f0 con 0x7f451c0b6d10 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f45224eb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f451c15e170 con 0x7f451c0ceea0 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f45224eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f451c0d4a70 con 0x7f451c15dd90 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0b6d10 0x7f451c1495e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0b6d10 0x7f451c1495e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49044/0 (socket says 172.21.15.64:49044) 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451bfff640 1 -- 172.21.15.64:0/272369628 learned_addr learned my addr 172.21.15.64:0/272369628 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451bfff640 1 -- 172.21.15.64:0/272369628 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f451c15dd90 msgr2=0x7f451c160e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451bfff640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f451c15dd90 0x7f451c160e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451bfff640 1 -- 172.21.15.64:0/272369628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f451c0ceea0 msgr2=0x7f451c1f7510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451b7fe640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f451c0ceea0 0x7f451c1f7510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.085+0000 7f451bfff640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f451c0ceea0 0x7f451c1f7510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.086+0000 7f451bfff640 1 -- 172.21.15.64:0/272369628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f451c149e50 con 0x7f451c0b6d10 2024-03-04T07:27:57.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.086+0000 7f451bfff640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0b6d10 0x7f451c1495e0 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f450c00b3f0 tx=0x7f450c00b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:57.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.086+0000 7f4518ff9640 1 -- 172.21.15.64:0/272369628 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f450c015020 con 0x7f451c0b6d10 2024-03-04T07:27:57.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.086+0000 7f4518ff9640 1 -- 172.21.15.64:0/272369628 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f450c011830 con 0x7f451c0b6d10 2024-03-04T07:27:57.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.086+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f451c1f7ba0 con 0x7f451c0b6d10 2024-03-04T07:27:57.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.086+0000 7f4518ff9640 1 -- 172.21.15.64:0/272369628 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f450c011c20 con 0x7f451c0b6d10 2024-03-04T07:27:57.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.086+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f451c1f8160 con 0x7f451c0b6d10 2024-03-04T07:27:57.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.087+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f451c072a30 con 0x7f451c0b6d10 2024-03-04T07:27:57.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.087+0000 7f4518ff9640 1 -- 172.21.15.64:0/272369628 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f450c018020 con 0x7f451c0b6d10 2024-03-04T07:27:57.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.088+0000 7f4518ff9640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f44fc06b4d0 0x7f44fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.088+0000 7f451b7fe640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f44fc06b4d0 0x7f44fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.088+0000 7f4518ff9640 1 -- 172.21.15.64:0/272369628 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f450c089e90 con 0x7f451c0b6d10 2024-03-04T07:27:57.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.088+0000 7f451b7fe640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f44fc06b4d0 0x7f44fc06d990 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f4504036000 tx=0x7f4504039040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:57.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.092+0000 7f4518ff9640 1 -- 172.21.15.64:0/272369628 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f450c01f030 con 0x7f451c0b6d10 2024-03-04T07:27:57.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.350+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f451c04f410 con 0x7f451c0b6d10 2024-03-04T07:27:57.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.352+0000 7f4518ff9640 1 -- 172.21.15.64:0/272369628 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v15) v1 ==== 190+0+147 (secure 0 0 0) 0x7f450c0172c0 con 0x7f451c0b6d10 2024-03-04T07:27:57.352 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:27:57.352 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:27:57.352 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:27:57.352 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:27:57.352 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:27:57.352 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:27:57.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f44fc06b4d0 msgr2=0x7f44fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f44fc06b4d0 0x7f44fc06d990 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f4504036000 tx=0x7f4504039040 comp rx=0 tx=0).stop 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0b6d10 msgr2=0x7f451c1495e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0b6d10 0x7f451c1495e0 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f450c00b3f0 tx=0x7f450c00b8b0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 shutdown_connections 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f44fc06b4d0 0x7f44fc06d990 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f451c15dd90 0x7f451c160e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f451c0ceea0 0x7f451c1f7510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 --2- 172.21.15.64:0/272369628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f451c0b6d10 0x7f451c1495e0 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 >> 172.21.15.64:0/272369628 conn(0x7f451c071960 msgr2=0x7f451c0d0e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 shutdown_connections 2024-03-04T07:27:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.355+0000 7f45224eb640 1 -- 172.21.15.64:0/272369628 wait complete. 2024-03-04T07:27:57.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:27:57.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.467+0000 7f0d93e4e640 1 Processor -- start 2024-03-04T07:27:57.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d93e4e640 1 -- start start 2024-03-04T07:27:57.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d93e4e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0d8c157b80 0x7f0d8c15b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d93e4e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0d8c157f60 0x7f0d8c15b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d93e4e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0d8c06c7b0 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d93e4e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d8c06ca90 con 0x7f0d8c157b80 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d93e4e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d8c06cd70 con 0x7f0d8c157f60 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d91bea640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0d8c157b80 0x7f0d8c15b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d923eb640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0d8c0cbab0 0x7f0d8c070b40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54710/0 (socket says 172.21.15.64:54710) 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.468+0000 7f0d923eb640 1 -- 172.21.15.64:0/2372913233 learned_addr learned my addr 172.21.15.64:0/2372913233 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.469+0000 7f0d913e9640 1 --2- 172.21.15.64:0/2372913233 >> v2:172.21.15.66:3300/0 conn(0x7f0d8c157f60 0x7f0d8c15b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.469+0000 7f0d91bea640 1 -- 172.21.15.64:0/2372913233 >> v2:172.21.15.66:3300/0 conn(0x7f0d8c157f60 msgr2=0x7f0d8c15b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.469+0000 7f0d91bea640 1 --2- 172.21.15.64:0/2372913233 >> v2:172.21.15.66:3300/0 conn(0x7f0d8c157f60 0x7f0d8c15b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.469+0000 7f0d91bea640 1 -- 172.21.15.64:0/2372913233 >> v1:172.21.15.64:6789/0 conn(0x7f0d8c0cbab0 legacy=0x7f0d8c070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.469+0000 7f0d91bea640 1 -- 172.21.15.64:0/2372913233 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d8c06d050 con 0x7f0d8c157b80 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.469+0000 7f0d90be8640 1 -- 172.21.15.64:0/2372913233 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2891001957 0 0) 0x7f0d8c06c7b0 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.469+0000 7f0d913e9640 1 --2- 172.21.15.64:0/2372913233 >> v2:172.21.15.66:3300/0 conn(0x7f0d8c157f60 0x7f0d8c15b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.470+0000 7f0d91bea640 1 --2- 172.21.15.64:0/2372913233 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c157b80 0x7f0d8c15b260 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f0d8c15a7e0 tx=0x7f0d7c02de80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=437fa85e56101101 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.470+0000 7f0d90be8640 1 -- 172.21.15.64:0/2372913233 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d7c034070 con 0x7f0d8c157b80 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.470+0000 7f0d90be8640 1 -- 172.21.15.64:0/2372913233 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d7c02ec30 con 0x7f0d8c157b80 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.470+0000 7f0d90be8640 1 -- 172.21.15.64:0/2372913233 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d7c038400 con 0x7f0d8c157b80 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 -- 172.21.15.64:0/2372913233 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c157b80 msgr2=0x7f0d8c15b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/2372913233 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c157b80 0x7f0d8c15b260 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f0d8c15a7e0 tx=0x7f0d7c02de80 comp rx=0 tx=0).stop 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 -- 172.21.15.64:0/2372913233 shutdown_connections 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/2372913233 >> v2:172.21.15.66:3300/0 conn(0x7f0d8c157f60 0x7f0d8c15b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/2372913233 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c157b80 0x7f0d8c15b260 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 -- 172.21.15.64:0/2372913233 >> 172.21.15.64:0/2372913233 conn(0x7f0d8c066d00 msgr2=0x7f0d8c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 -- 172.21.15.64:0/2372913233 shutdown_connections 2024-03-04T07:27:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.471+0000 7f0d93e4e640 1 -- 172.21.15.64:0/2372913233 wait complete. 2024-03-04T07:27:57.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.472+0000 7f0d93e4e640 1 Processor -- start 2024-03-04T07:27:57.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.472+0000 7f0d93e4e640 1 -- start start 2024-03-04T07:27:57.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d93e4e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c0cbab0 0x7f0d8c1440d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d93e4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d8c157b80 0x7f0d8c1e40b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d93e4e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d8c157f60 0x7f0d8c1485a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d93e4e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d8c0cc1d0 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d93e4e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d8c0cc050 con 0x7f0d8c157f60 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d93e4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d8c071250 con 0x7f0d8c157b80 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c0cbab0 0x7f0d8c1440d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c0cbab0 0x7f0d8c1440d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49050/0 (socket says 172.21.15.64:49050) 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 -- 172.21.15.64:0/4088270731 learned_addr learned my addr 172.21.15.64:0/4088270731 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d923eb640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d8c157f60 0x7f0d8c1485a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d913e9640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d8c157b80 0x7f0d8c1e40b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 -- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d8c157b80 msgr2=0x7f0d8c1e40b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d8c157b80 0x7f0d8c1e40b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 -- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d8c157f60 msgr2=0x7f0d8c1485a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d8c157f60 0x7f0d8c1485a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 -- 172.21.15.64:0/4088270731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d8c1454d0 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d91bea640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c0cbab0 0x7f0d8c1440d0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f0d7c02e490 tx=0x7f0d7c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:57.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.473+0000 7f0d8a7fc640 1 -- 172.21.15.64:0/4088270731 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d7c047020 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.474+0000 7f0d8a7fc640 1 -- 172.21.15.64:0/4088270731 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d7c043880 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.474+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d8c1457b0 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.474+0000 7f0d8a7fc640 1 -- 172.21.15.64:0/4088270731 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d7c043c30 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.474+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d8c145d70 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.474+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d8c1e3070 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.475+0000 7f0d8a7fc640 1 -- 172.21.15.64:0/4088270731 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0d7c04a020 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.475+0000 7f0d8a7fc640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d7806b4d0 0x7f0d7806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:57.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.475+0000 7f0d8a7fc640 1 -- 172.21.15.64:0/4088270731 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0d7c0af300 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.476+0000 7f0d913e9640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d7806b4d0 0x7f0d7806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:57.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.476+0000 7f0d913e9640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d7806b4d0 0x7f0d7806d990 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f0d8c0d04b0 tx=0x7f0d80008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:57.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.480+0000 7f0d8a7fc640 1 -- 172.21.15.64:0/4088270731 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0d7c051030 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.697+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0d8c0d07a0 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.698+0000 7f0d8a7fc640 1 -- 172.21.15.64:0/4088270731 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f0d7c07e190 con 0x7f0d8c0cbab0 2024-03-04T07:27:57.698 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:27:57.698 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:27:57.698 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d7806b4d0 msgr2=0x7f0d7806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d7806b4d0 0x7f0d7806d990 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f0d8c0d04b0 tx=0x7f0d80008040 comp rx=0 tx=0).stop 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c0cbab0 msgr2=0x7f0d8c1440d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c0cbab0 0x7f0d8c1440d0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f0d7c02e490 tx=0x7f0d7c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 shutdown_connections 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d7806b4d0 0x7f0d7806d990 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d8c157f60 0x7f0d8c1485a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d8c157b80 0x7f0d8c1e40b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 --2- 172.21.15.64:0/4088270731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d8c0cbab0 0x7f0d8c1440d0 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 >> 172.21.15.64:0/4088270731 conn(0x7f0d8c066d00 msgr2=0x7f0d8c159d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 shutdown_connections 2024-03-04T07:27:57.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:57.700+0000 7f0d93e4e640 1 -- 172.21.15.64:0/4088270731 wait complete. 2024-03-04T07:27:58.112 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 15, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:27:47.355925+0000', 'modified': '2024-03-04T07:27:50.389687+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 5169}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_5169': {'gid': 5169, 'name': 'a', 'rank': 0, 'incarnation': 12, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.64:6836/2136250526', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 2136250526}, {'type': 'v1', 'addr': '172.21.15.64:6836', 'nonce': 2136250526}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [5], 'metadata_pool': 4, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 5169, 'qdb_cluster': [5169]} 2024-03-04T07:27:58.112 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:27:58.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.a status 2024-03-04T07:27:58.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.217+0000 7fb0ba488640 1 Processor -- start 2024-03-04T07:27:58.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.217+0000 7fb0ba488640 1 -- start start 2024-03-04T07:27:58.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0ba488640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415c190 0x7fb0b40d0730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0ba488640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb0b415fa80 0x7fb0b415fed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0ba488640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb0b40cf690 con 0x7fb0b415bdb0 2024-03-04T07:27:58.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0ba488640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0b40d0c70 con 0x7fb0b415fa80 2024-03-04T07:27:58.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0ba488640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b4160410 con 0x7fb0b415c190 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0b8a25640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb0b415fa80 0x7fb0b415fed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0b8a25640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb0b415fa80 0x7fb0b415fed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49060/0 (socket says 172.21.15.64:49060) 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0b8a25640 1 -- 172.21.15.64:0/1470263582 learned_addr learned my addr 172.21.15.64:0/1470263582 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.218+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/1470263582 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415c190 0x7fb0b40d0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.219+0000 7fb0b8a25640 1 -- 172.21.15.64:0/1470263582 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415c190 msgr2=0x7fb0b40d0730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.219+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/1470263582 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415c190 0x7fb0b40d0730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.219+0000 7fb0b8a25640 1 -- 172.21.15.64:0/1470263582 >> v1:172.21.15.66:6789/0 conn(0x7fb0b415bdb0 legacy=0x7fb0b40cd290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.219+0000 7fb0b8a25640 1 -- 172.21.15.64:0/1470263582 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0b40cf970 con 0x7fb0b415fa80 2024-03-04T07:27:58.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/1470263582 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b415fa80 0x7fb0b415fed0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac009e50 tx=0x7fb0ac02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44f7818716d4d35d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0b2ffd640 1 -- 172.21.15.64:0/1470263582 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac02ca70 con 0x7fb0b415fa80 2024-03-04T07:27:58.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0b2ffd640 1 -- 172.21.15.64:0/1470263582 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0ac02cc10 con 0x7fb0b415fa80 2024-03-04T07:27:58.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0b2ffd640 1 -- 172.21.15.64:0/1470263582 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac0343f0 con 0x7fb0b415fa80 2024-03-04T07:27:58.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 -- 172.21.15.64:0/1470263582 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b415fa80 msgr2=0x7fb0b415fed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 --2- 172.21.15.64:0/1470263582 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b415fa80 0x7fb0b415fed0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac009e50 tx=0x7fb0ac02bed0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 -- 172.21.15.64:0/1470263582 shutdown_connections 2024-03-04T07:27:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 --2- 172.21.15.64:0/1470263582 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b415fa80 0x7fb0b415fed0 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 --2- 172.21.15.64:0/1470263582 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415c190 0x7fb0b40d0730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 -- 172.21.15.64:0/1470263582 >> 172.21.15.64:0/1470263582 conn(0x7fb0b40b35d0 msgr2=0x7fb0b40b39e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 -- 172.21.15.64:0/1470263582 shutdown_connections 2024-03-04T07:27:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.220+0000 7fb0ba488640 1 -- 172.21.15.64:0/1470263582 wait complete. 2024-03-04T07:27:58.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 Processor -- start 2024-03-04T07:27:58.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 -- start start 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0b415bdb0 0x7fb0b40c6f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415fa80 0x7fb0b4153f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b40c60d0 0x7fb0b40b6db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b40d0210 con 0x7fb0b40c60d0 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b40cff30 con 0x7fb0b415bdb0 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.222+0000 7fb0ba488640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b40b4320 con 0x7fb0b415fa80 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b40c60d0 0x7fb0b40b6db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b40c60d0 0x7fb0b40b6db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49062/0 (socket says 172.21.15.64:49062) 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 -- 172.21.15.64:0/839921680 learned_addr learned my addr 172.21.15.64:0/839921680 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0b415bdb0 0x7fb0b40c6f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415fa80 0x7fb0b4153f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 -- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415fa80 msgr2=0x7fb0b4153f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415fa80 0x7fb0b4153f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 -- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0b415bdb0 msgr2=0x7fb0b40c6f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0b415bdb0 0x7fb0b40c6f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 -- 172.21.15.64:0/839921680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0b40c5310 con 0x7fb0b40c60d0 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b40c60d0 0x7fb0b40b6db0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac036040 tx=0x7fb0ac004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b0ff9640 1 -- 172.21.15.64:0/839921680 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac043020 con 0x7fb0b40c60d0 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b0ff9640 1 -- 172.21.15.64:0/839921680 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0ac03e450 con 0x7fb0b40c60d0 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0b0ff9640 1 -- 172.21.15.64:0/839921680 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac009510 con 0x7fb0b40c60d0 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0b40c55f0 con 0x7fb0b40c60d0 2024-03-04T07:27:58.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.223+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0b40c5bb0 con 0x7fb0b40c60d0 2024-03-04T07:27:58.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.225+0000 7fb0b0ff9640 1 -- 172.21.15.64:0/839921680 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb0ac046020 con 0x7fb0b40c60d0 2024-03-04T07:27:58.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.225+0000 7fb0b0ff9640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb0a006b4d0 0x7fb0a006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.225+0000 7fb0b0ff9640 1 -- 172.21.15.64:0/839921680 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb0ac0ac7d0 con 0x7fb0b40c60d0 2024-03-04T07:27:58.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.225+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb0a006b4d0 0x7fb0a006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.226+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb0a006b4d0 0x7fb0a006d990 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fb0a80016e0 tx=0x7fb0a8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.226+0000 7fb0bd073740 1 Processor -- start 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.226+0000 7fb0bd073740 1 -- start start 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674da870 0x562e674dac50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e674de8c0 0x562e674e0360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750aa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0bd073740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x562e674446b0 con 0x562e6750a600 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x562e673eda60 con 0x562e674de8c0 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x562e67440750 con 0x562e674da870 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750aa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750aa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49064/0 (socket says 172.21.15.64:49064) 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 -- 172.21.15.64:0/3152061981 learned_addr learned my addr 172.21.15.64:0/3152061981 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e674de8c0 0x562e674e0360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 -- 172.21.15.64:0/3152061981 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674da870 msgr2=0x562e674dac50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674da870 0x562e674dac50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 -- 172.21.15.64:0/3152061981 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e674de8c0 msgr2=0x562e674e0360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e674de8c0 0x562e674e0360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.227+0000 7fb0b8a25640 1 -- 172.21.15.64:0/3152061981 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562e6740bf30 con 0x562e6750a600 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750aa80 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac0aadb0 tx=0x7fb0ac07aab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d172dafc514a45e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb099ffb640 1 -- 172.21.15.64:0/3152061981 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac092020 con 0x562e6750a600 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb099ffb640 1 -- 172.21.15.64:0/3152061981 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0ac0836f0 con 0x562e6750a600 2024-03-04T07:27:58.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb099ffb640 1 -- 172.21.15.64:0/3152061981 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac077700 con 0x562e6750a600 2024-03-04T07:27:58.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 -- 172.21.15.64:0/3152061981 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 msgr2=0x562e6750aa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750aa80 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac0aadb0 tx=0x7fb0ac07aab0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 -- 172.21.15.64:0/3152061981 shutdown_connections 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750aa80 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e674de8c0 0x562e674e0360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 --2- 172.21.15.64:0/3152061981 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674da870 0x562e674dac50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 -- 172.21.15.64:0/3152061981 >> 172.21.15.64:0/3152061981 conn(0x562e674dc8f0 msgr2=0x562e674de1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 -- 172.21.15.64:0/3152061981 shutdown_connections 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.228+0000 7fb0bd073740 1 -- 172.21.15.64:0/3152061981 wait complete. 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.229+0000 7fb0bd073740 1 -- start start 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.230+0000 7fb0bd073740 1 Processor -- start 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750bcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e674efac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675276b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0bd073740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x562e674446b0 con 0x562e6750a600 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x562e67400ea0 con 0x562e67523790 2024-03-04T07:27:58.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x562e674ded50 con 0x562e674f0000 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0bd073740 10 client.0 _ll_register_callbacks cb 0x562e67400870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750bcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750bcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49072/0 (socket says 172.21.15.64:49072) 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 -- 172.21.15.64:0/474613341 learned_addr learned my addr 172.21.15.64:0/474613341 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 -- 172.21.15.64:0/474613341 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 msgr2=0x562e675276b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675276b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 -- 172.21.15.64:0/474613341 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 msgr2=0x562e674efac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e674efac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 -- 172.21.15.64:0/474613341 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562e67401270 con 0x562e6750a600 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750bcb0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7fb0a80072b0 tx=0x7fb0a8007e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d4efd966499b0b8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.231+0000 7fb092ffd640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb092ffd640 1 -- 172.21.15.64:0/474613341 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0a8006cd0 con 0x562e6750a600 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb092ffd640 1 -- 172.21.15.64:0/474613341 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb0a8023ad0 con 0x562e6750a600 2024-03-04T07:27:58.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb0ac0836f0 con 0x562e6750a600 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb092ffd640 1 -- 172.21.15.64:0/474613341 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=15) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb0a8023cd0 con 0x562e6750a600 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 10 client.5385 fetch_fsmap learned FSMap version 15 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=15,monmap=2+}) v3 -- 0x562e671592d0 con 0x562e6750a600 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb092ffd640 1 -- 172.21.15.64:0/474613341 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0ac0836f0 con 0x562e6750a600 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb092ffd640 1 -- 172.21.15.64:0/474613341 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 15) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fb0a802a050 con 0x562e6750a600 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 10 client.5385 fetch_fsmap finished waiting for FSMap version 15 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 10 client.5385 resolve_mds: resolved name 'a' to daemon mds.a 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 20 client.5385 populate_metadata read hostname 'smithi064' 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e675023a0 0x562e67543db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 4 client.5385 mds_command: new command op to 5169 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 --> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562e67451bf0 con 0x562e675023a0 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.232+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e675023a0 0x562e67543db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb092ffd640 10 client.5385 ms_handle_connect on v2:172.21.15.64:6834/2136250526 2024-03-04T07:27:58.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e675023a0 0x562e67543db0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9b5a16730189c123 server_cookie=742eb17f7a554cc6 in_seq=0 out_seq=0 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb092ffd640 1 -- 172.21.15.64:0/474613341 <== mds.0 v2:172.21.15.64:6834/2136250526 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x562e67451bf0 con 0x562e675023a0 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb092ffd640 10 client.5385 handle_command_reply: tid=0 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 client.5385 shutdown 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 msgr2=0x562e6750bcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750bcb0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7fb0a80072b0 tx=0x7fb0a8007e50 comp rx=0 tx=0).stop 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 shutdown_connections 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e675023a0 0x562e67543db0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675276b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e674efac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb092ffd640 0 client.5385 ms_handle_reset on v2:172.21.15.64:6834/2136250526 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 --2- 172.21.15.64:0/474613341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e6750bcb0 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.233+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 >> 172.21.15.64:0/474613341 conn(0x562e674da870 msgr2=0x562e67526b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.234+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 shutdown_connections 2024-03-04T07:27:58.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.234+0000 7fb0bd073740 1 -- 172.21.15.64:0/474613341 wait complete. 2024-03-04T07:27:58.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.234+0000 7fb0bd073740 20 client.5385 trim_cache size 0 max 16384 2024-03-04T07:27:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.266+0000 7fb0bd073740 1 Processor -- start 2024-03-04T07:27:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.266+0000 7fb0bd073740 1 -- start start 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675048f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e67504e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e6750da40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0bd073740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b40b5510 con 0x562e6750a600 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b40b4910 con 0x562e67523790 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0b40b4f10 con 0x562e674f0000 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e67504e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e67504e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49084/0 (socket says 172.21.15.64:49084) 2024-03-04T07:27:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 -- 172.21.15.64:0/722598593 learned_addr learned my addr 172.21.15.64:0/722598593 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675048f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e6750da40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 -- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 msgr2=0x562e675048f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675048f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 -- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 msgr2=0x562e6750da40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e6750da40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 -- 172.21.15.64:0/722598593 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562e672236b0 con 0x562e6750a600 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e6750da40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e67504e30 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fb0a4006e80 tx=0x7fb0a4007de0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b44d7c39c4fa5ce7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0997fa640 1 -- 172.21.15.64:0/722598593 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0a40040f0 con 0x562e6750a600 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0997fa640 1 -- 172.21.15.64:0/722598593 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0a4004290 con 0x562e6750a600 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.268+0000 7fb0997fa640 1 -- 172.21.15.64:0/722598593 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0a4016940 con 0x562e6750a600 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 -- 172.21.15.64:0/722598593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 msgr2=0x562e67504e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e67504e30 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fb0a4006e80 tx=0x7fb0a4007de0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 -- 172.21.15.64:0/722598593 shutdown_connections 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e67523790 0x562e6750da40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e6750a600 0x562e67504e30 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 --2- 172.21.15.64:0/722598593 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675048f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 -- 172.21.15.64:0/722598593 >> 172.21.15.64:0/722598593 conn(0x562e674da870 msgr2=0x562e674de080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 -- 172.21.15.64:0/722598593 shutdown_connections 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 -- 172.21.15.64:0/722598593 wait complete. 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.269+0000 7fb0bd073740 1 -- start start 2024-03-04T07:27:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 1 Processor -- start 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675249e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e6750a600 0x562e6753e0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e67523790 0x562e6759e180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x562e67525820 con 0x562e67523790 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x562e67502e00 con 0x562e6750a600 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x562e675260c0 con 0x562e674f0000 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.270+0000 7fb0bd073740 10 client.0 _ll_register_callbacks cb 0x7fb0a800fe60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e67523790 0x562e6759e180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e67523790 0x562e6759e180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49092/0 (socket says 172.21.15.64:49092) 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 -- 172.21.15.64:0/2893439691 learned_addr learned my addr 172.21.15.64:0/2893439691 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b37fe640 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e6750a600 0x562e6753e0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675249e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 -- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 msgr2=0x562e675249e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675249e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 -- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e6750a600 msgr2=0x562e6753e0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e6750a600 0x562e6753e0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 -- 172.21.15.64:0/2893439691 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562e674efcb0 con 0x562e67523790 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0b8a25640 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e67523790 0x562e6759e180 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac07aca0 tx=0x7fb0ac009810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d859fb0e9581c19 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb092ffd640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb092ffd640 1 -- 172.21.15.64:0/2893439691 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0ac092040 con 0x562e67523790 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb092ffd640 1 -- 172.21.15.64:0/2893439691 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb0ac083260 con 0x562e67523790 2024-03-04T07:27:58.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb0a4004290 con 0x562e67523790 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb092ffd640 1 -- 172.21.15.64:0/2893439691 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=15) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb0ac0836f0 con 0x562e67523790 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0bd073740 10 client.5391 fetch_fsmap learned FSMap version 15 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.271+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=15,monmap=2+}) v3 -- 0x562e6753e890 con 0x562e67523790 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb092ffd640 1 -- 172.21.15.64:0/2893439691 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb0a4004290 con 0x562e67523790 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb092ffd640 1 -- 172.21.15.64:0/2893439691 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 15) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fb0ac07a4e0 con 0x562e67523790 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0bd073740 10 client.5391 fetch_fsmap finished waiting for FSMap version 15 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0bd073740 10 client.5391 resolve_mds: resolved name 'a' to daemon mds.a 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0bd073740 20 client.5391 populate_metadata read hostname 'smithi064' 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0bd073740 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e67500e80 0x562e67501260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0bd073740 4 client.5391 mds_command: new command op to 5169 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 --> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] -- command(tid 0: {"prefix": "status"}) v1 -- 0x562e674db630 con 0x562e67500e80 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e67500e80 0x562e67501260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0b3fff640 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e67500e80 0x562e67501260 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=659d8d39fed1313e server_cookie=168f8bf8fa1f39c5 in_seq=0 out_seq=0 2024-03-04T07:27:58.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb092ffd640 10 client.5391 ms_handle_connect on v2:172.21.15.64:6834/2136250526 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb092ffd640 1 -- 172.21.15.64:0/2893439691 <== mds.0 v2:172.21.15.64:6834/2136250526 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+325 (crc 0 0 0) 0x562e674db630 con 0x562e67500e80 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb092ffd640 10 client.5391 handle_command_reply: tid=0 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.272+0000 7fb0bd073740 1 client.5391 shutdown 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e67523790 msgr2=0x562e6759e180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e67523790 0x562e6759e180 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac07aca0 tx=0x7fb0ac009810 comp rx=0 tx=0).stop 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 shutdown_connections 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:6834/2136250526,v1:172.21.15.64:6836/2136250526] conn(0x562e67500e80 0x562e67501260 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x562e67523790 0x562e6759e180 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x562e6750a600 0x562e6753e0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb092ffd640 0 client.5391 ms_handle_reset on v2:172.21.15.64:6834/2136250526 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 --2- 172.21.15.64:0/2893439691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x562e674f0000 0x562e675249e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 >> 172.21.15.64:0/2893439691 conn(0x562e674da870 msgr2=0x562e675048f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 shutdown_connections 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 1 -- 172.21.15.64:0/2893439691 wait complete. 2024-03-04T07:27:58.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0bd073740 20 client.5391 trim_cache size 0 max 16384 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 5169, 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 10.272293149999999, 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 15, 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 33, 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 33, 2024-03-04T07:27:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 17.299668418 2024-03-04T07:27:58.280 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:27:58.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb0a006b4d0 msgr2=0x7fb0a006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb0a006b4d0 0x7fb0a006d990 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fb0a80016e0 tx=0x7fb0a8009040 comp rx=0 tx=0).stop 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b40c60d0 msgr2=0x7fb0b40b6db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b40c60d0 0x7fb0b40b6db0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fb0ac036040 tx=0x7fb0ac004a40 comp rx=0 tx=0).stop 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 shutdown_connections 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb0a006b4d0 0x7fb0a006d990 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0b40c60d0 0x7fb0b40b6db0 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0b415fa80 0x7fb0b4153f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 --2- 172.21.15.64:0/839921680 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0b415bdb0 0x7fb0b40c6f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.273+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 >> 172.21.15.64:0/839921680 conn(0x7fb0b40b35d0 msgr2=0x7fb0b40cd060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.274+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 shutdown_connections 2024-03-04T07:27:58.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.274+0000 7fb0ba488640 1 -- 172.21.15.64:0/839921680 wait complete. 2024-03-04T07:27:58.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:27:58.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.388+0000 7f86f4c88640 1 Processor -- start 2024-03-04T07:27:58.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.388+0000 7f86f4c88640 1 -- start start 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86f4c88640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f86f015f740 0x7f86f015be60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86f4c88640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f86f006fdc0 0x7f86f00719b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86f4c88640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f86f015fcc0 con 0x7f86f0066d00 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86f4c88640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f86f015ffa0 con 0x7f86f015f740 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86f4c88640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f86f0160280 con 0x7f86f006fdc0 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86e7fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f86f015f740 0x7f86f015be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86e7fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f86f015f740 0x7f86f015be60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49106/0 (socket says 172.21.15.64:49106) 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86e7fff640 1 -- 172.21.15.64:0/3316191513 learned_addr learned my addr 172.21.15.64:0/3316191513 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86eed9d640 1 --2- 172.21.15.64:0/3316191513 >> v2:172.21.15.66:3300/0 conn(0x7f86f006fdc0 0x7f86f00719b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86e7fff640 1 -- 172.21.15.64:0/3316191513 >> v2:172.21.15.66:3300/0 conn(0x7f86f006fdc0 msgr2=0x7f86f00719b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86e7fff640 1 --2- 172.21.15.64:0/3316191513 >> v2:172.21.15.66:3300/0 conn(0x7f86f006fdc0 0x7f86f00719b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.389+0000 7f86e7fff640 1 -- 172.21.15.64:0/3316191513 >> v1:172.21.15.66:6789/0 conn(0x7f86f0066d00 legacy=0x7f86f00670e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.390+0000 7f86e7fff640 1 -- 172.21.15.64:0/3316191513 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f86f0160560 con 0x7f86f015f740 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.390+0000 7f86e7fff640 1 --2- 172.21.15.64:0/3316191513 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f015be60 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f86f00b39f0 tx=0x7f86d802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=54e5d24d250d8ef0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.390+0000 7f86edd9b640 1 -- 172.21.15.64:0/3316191513 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86d8035020 con 0x7f86f015f740 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.390+0000 7f86edd9b640 1 -- 172.21.15.64:0/3316191513 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f86d8031870 con 0x7f86f015f740 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.390+0000 7f86edd9b640 1 -- 172.21.15.64:0/3316191513 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86d8031c40 con 0x7f86f015f740 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 -- 172.21.15.64:0/3316191513 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 msgr2=0x7f86f015be60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 --2- 172.21.15.64:0/3316191513 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f015be60 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f86f00b39f0 tx=0x7f86d802b910 comp rx=0 tx=0).stop 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 -- 172.21.15.64:0/3316191513 shutdown_connections 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 --2- 172.21.15.64:0/3316191513 >> v2:172.21.15.66:3300/0 conn(0x7f86f006fdc0 0x7f86f00719b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 --2- 172.21.15.64:0/3316191513 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f015be60 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 -- 172.21.15.64:0/3316191513 >> 172.21.15.64:0/3316191513 conn(0x7f86f00d0990 msgr2=0x7f86f00d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 -- 172.21.15.64:0/3316191513 shutdown_connections 2024-03-04T07:27:58.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.391+0000 7f86f4c88640 1 -- 172.21.15.64:0/3316191513 wait complete. 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.392+0000 7f86f4c88640 1 Processor -- start 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86f4c88640 1 -- start start 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86f4c88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86f0066d00 0x7f86f0141e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86f4c88640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86f006fdc0 0x7f86f0159450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86f4c88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f0159990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86f4c88640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86f0072070 con 0x7f86f015f740 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86f4c88640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86f0071ef0 con 0x7f86f006fdc0 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86f4c88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f86f015c6e0 con 0x7f86f0066d00 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86eed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f0159990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86eed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f0159990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49116/0 (socket says 172.21.15.64:49116) 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.393+0000 7f86eed9d640 1 -- 172.21.15.64:0/2281326018 learned_addr learned my addr 172.21.15.64:0/2281326018 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:27:58.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86eed9d640 1 -- 172.21.15.64:0/2281326018 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86f0066d00 msgr2=0x7f86f0141e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86eed9d640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86f0066d00 0x7f86f0141e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86e7fff640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86f006fdc0 0x7f86f0159450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86eed9d640 1 -- 172.21.15.64:0/2281326018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86f006fdc0 msgr2=0x7f86f0159450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86eed9d640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86f006fdc0 0x7f86f0159450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86eed9d640 1 -- 172.21.15.64:0/2281326018 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f86f0141b10 con 0x7f86f015f740 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86eed9d640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f0159990 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f86e8007e70 tx=0x7f86e800e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86e6ffd640 1 -- 172.21.15.64:0/2281326018 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86e8014620 con 0x7f86f015f740 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86e6ffd640 1 -- 172.21.15.64:0/2281326018 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f86e8019070 con 0x7f86f015f740 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86e6ffd640 1 -- 172.21.15.64:0/2281326018 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86e8014a20 con 0x7f86f015f740 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f86f0142710 con 0x7f86f015f740 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.394+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f86f0142cd0 con 0x7f86f015f740 2024-03-04T07:27:58.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.395+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f86f0068700 con 0x7f86f015f740 2024-03-04T07:27:58.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.396+0000 7f86e6ffd640 1 -- 172.21.15.64:0/2281326018 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f86e8014bc0 con 0x7f86f015f740 2024-03-04T07:27:58.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.396+0000 7f86e6ffd640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f86d006b500 0x7f86d006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:27:58.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.396+0000 7f86e6ffd640 1 -- 172.21.15.64:0/2281326018 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f86e808aa50 con 0x7f86f015f740 2024-03-04T07:27:58.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.396+0000 7f86ee59c640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f86d006b500 0x7f86d006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:27:58.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.397+0000 7f86ee59c640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f86d006b500 0x7f86d006d9c0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f86e0003cc0 tx=0x7f86e00079b0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:27:58.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.401+0000 7f86e6ffd640 1 -- 172.21.15.64:0/2281326018 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f86e80598d0 con 0x7f86f015f740 2024-03-04T07:27:58.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.620+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f86f006dc60 con 0x7f86f015f740 2024-03-04T07:27:58.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.621+0000 7f86e6ffd640 1 -- 172.21.15.64:0/2281326018 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v15) v1 ==== 53+0+73 (secure 0 0 0) 0x7f86e80591d0 con 0x7f86f015f740 2024-03-04T07:27:58.620 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:27:58.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f86d006b500 msgr2=0x7f86d006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f86d006b500 0x7f86d006d9c0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f86e0003cc0 tx=0x7f86e00079b0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 msgr2=0x7f86f0159990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f0159990 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f86e8007e70 tx=0x7f86e800e4b0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 shutdown_connections 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f86d006b500 0x7f86d006d9c0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f86f015f740 0x7f86f0159990 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86f006fdc0 0x7f86f0159450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 --2- 172.21.15.64:0/2281326018 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86f0066d00 0x7f86f0141e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.622+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 >> 172.21.15.64:0/2281326018 conn(0x7f86f00d0990 msgr2=0x7f86f00cfd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.623+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 shutdown_connections 2024-03-04T07:27:58.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:58.623+0000 7f86f4c88640 1 -- 172.21.15.64:0/2281326018 wait complete. 2024-03-04T07:27:58.633 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:27:58.633 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:27:58.633 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:27:58.633 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:27:58.633 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:58.634 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:27:58.634 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:27:58.634 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:27:58.634 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:27:58.634 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:58.634 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:27:58.648 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:27:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42634sec preferred_lft 42634sec 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:27:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:27:58.651 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:58.651 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:27:58.715 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:27:58.716 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:58.716 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:27:58.772 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:27:58.772 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:27:58.772 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:27:58.772 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:27:58.772 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:27:58.772 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:58.773 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:27:58.773 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:27:58.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:27:58.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:27:58.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:27:58.773 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:27:58.868 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:58.868 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:27:58.882 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:27:58.882 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:27:58.883 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:27:58.883 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:58.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:27:58.949 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:27:58.949 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:58.949 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:59.004 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:27:59.005 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:27:59.005 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:59.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:27:59.126 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:59.182 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:27:59.182 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:27:59.182 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:27:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:27:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:59.185 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:59.185 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:59.185 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:59.185 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:59.185 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:27:59.237 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:27:59.237 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:27:59.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:27:59.311 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:27:59.367 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:27:59.367+0000 7f2c2cbfd480 -1 init, newargv = 0x559248602bc0 newargc=15 2024-03-04T07:27:59.367 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[52830]: starting ceph client 2024-03-04T07:27:59.368 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:27:59.368 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:27:59.368 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:27:59.368 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:27:59.368 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:27:59.368 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:27:59.368 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:27:59.369 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:59.370 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:27:59.371 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:59.371 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:27:59.371 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[52830]: starting fuse 2024-03-04T07:27:59.385 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:27:59.385 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:27:59.386 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:27:59.387 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:27:59.506 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.52830.asok 2024-03-04T07:27:59.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:27:59 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:27:59.517 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.52830.asok 2024-03-04T07:27:59.517 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:59.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.52830.asok status 2024-03-04T07:27:59.654 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "52830", 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 5406, 2024-03-04T07:27:59.655 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 5406 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 1226266241 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:27:59.656 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 1226266241 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.5406 192.168.0.1:0/1226266241", 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/1226266241", 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 15, 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 33, 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:27:59.657 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:27:59.663 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:59.663 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:27:59.663 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:59.678 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:27:59.678 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:59.678 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:27:59.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:27:59.745 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:27:59.745 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:27:59.745 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:27:59.745 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> subdir = "delete_me" 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> size = 1024 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> file_count = 1000 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> os.mkdir(os.path.join(mountpoint, subdir)) 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> for i in range(0, file_count): 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> filename = "{0}_{1}.bin".format(i, size) 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> with open(os.path.join(mountpoint, subdir, filename), '"'"'w'"'"') as f: 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> f.write(size * '"'"'x'"'"') 2024-03-04T07:27:59.746 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:28:01.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:28:01.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.811+0000 7f0de1242640 1 Processor -- start 2024-03-04T07:28:01.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.811+0000 7f0de1242640 1 -- start start 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.811+0000 7f0de1242640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc06b5c0 0x7f0ddc0b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.811+0000 7f0de1242640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0ddc0ae9b0 0x7f0ddc0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.811+0000 7f0de1242640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0ddc0b3340 con 0x7f0ddc0b0e90 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.811+0000 7f0de1242640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0ddc0cfa00 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.811+0000 7f0de1242640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc0cee90 con 0x7f0ddc06b5c0 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0dda59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0ddc0b0e90 0x7f0ddc06b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54714/0 (socket says 172.21.15.64:54714) 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0dda59c640 1 -- 172.21.15.64:0/311672041 learned_addr learned my addr 172.21.15.64:0/311672041 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0ddb59e640 1 --2- 172.21.15.64:0/311672041 >> v2:172.21.15.66:3300/0 conn(0x7f0ddc0ae9b0 0x7f0ddc0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0ddad9d640 1 --2- 172.21.15.64:0/311672041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc06b5c0 0x7f0ddc0b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0dd9d9b640 1 -- 172.21.15.64:0/311672041 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1973311876 0 0) 0x7f0ddc0b3340 con 0x7f0ddc0b0e90 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0dd9d9b640 1 -- 172.21.15.64:0/311672041 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0dc8003520 con 0x7f0ddc0b0e90 2024-03-04T07:28:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0ddad9d640 1 -- 172.21.15.64:0/311672041 >> v2:172.21.15.66:3300/0 conn(0x7f0ddc0ae9b0 msgr2=0x7f0ddc0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:01.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0ddad9d640 1 --2- 172.21.15.64:0/311672041 >> v2:172.21.15.66:3300/0 conn(0x7f0ddc0ae9b0 0x7f0ddc0d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:01.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0ddad9d640 1 -- 172.21.15.64:0/311672041 >> v1:172.21.15.64:6789/0 conn(0x7f0ddc0b0e90 legacy=0x7f0ddc06b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:01.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.812+0000 7f0ddad9d640 1 -- 172.21.15.64:0/311672041 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ddc0cfc50 con 0x7f0ddc06b5c0 2024-03-04T07:28:01.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0ddad9d640 1 --2- 172.21.15.64:0/311672041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc06b5c0 0x7f0ddc0b2e00 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f0dd40097a0 tx=0x7f0dd402b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ebb37a4479762935 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:01.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0dd9d9b640 1 -- 172.21.15.64:0/311672041 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dd40315a0 con 0x7f0ddc06b5c0 2024-03-04T07:28:01.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0dd9d9b640 1 -- 172.21.15.64:0/311672041 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0dd4037070 con 0x7f0ddc06b5c0 2024-03-04T07:28:01.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0dd9d9b640 1 -- 172.21.15.64:0/311672041 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dd4031940 con 0x7f0ddc06b5c0 2024-03-04T07:28:01.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0de1242640 1 -- 172.21.15.64:0/311672041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc06b5c0 msgr2=0x7f0ddc0b2e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0de1242640 1 --2- 172.21.15.64:0/311672041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc06b5c0 0x7f0ddc0b2e00 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f0dd40097a0 tx=0x7f0dd402b840 comp rx=0 tx=0).stop 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0de1242640 1 -- 172.21.15.64:0/311672041 shutdown_connections 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0de1242640 1 --2- 172.21.15.64:0/311672041 >> v2:172.21.15.66:3300/0 conn(0x7f0ddc0ae9b0 0x7f0ddc0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0de1242640 1 --2- 172.21.15.64:0/311672041 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc06b5c0 0x7f0ddc0b2e00 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.813+0000 7f0de1242640 1 -- 172.21.15.64:0/311672041 >> 172.21.15.64:0/311672041 conn(0x7f0ddc0b36d0 msgr2=0x7f0ddc0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.814+0000 7f0de1242640 1 -- 172.21.15.64:0/311672041 shutdown_connections 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.814+0000 7f0de1242640 1 -- 172.21.15.64:0/311672041 wait complete. 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.815+0000 7f0de1242640 1 Processor -- start 2024-03-04T07:28:01.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.815+0000 7f0de1242640 1 -- start start 2024-03-04T07:28:01.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0de1242640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc06b5c0 0x7f0ddc1e73d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:01.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0de1242640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0ae9b0 0x7f0ddc1dfe20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0de1242640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc0b0e90 0x7f0ddc06f7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0de1242640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc06c910 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0de1242640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc06c790 con 0x7f0ddc06b5c0 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0de1242640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc0aee00 con 0x7f0ddc0b0e90 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0ae9b0 0x7f0ddc1dfe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0ae9b0 0x7f0ddc1dfe20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49126/0 (socket says 172.21.15.64:49126) 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 -- 172.21.15.64:0/2487947224 learned_addr learned my addr 172.21.15.64:0/2487947224 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0ddad9d640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc06b5c0 0x7f0ddc1e73d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0ddb59e640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc0b0e90 0x7f0ddc06f7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 -- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc0b0e90 msgr2=0x7f0ddc06f7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc0b0e90 0x7f0ddc06f7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 -- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc06b5c0 msgr2=0x7f0ddc1e73d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc06b5c0 0x7f0ddc1e73d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 -- 172.21.15.64:0/2487947224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ddc1e7b40 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dda59c640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0ae9b0 0x7f0ddc1dfe20 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7f0dc400c430 tx=0x7f0dc400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dd37fe640 1 -- 172.21.15.64:0/2487947224 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dc4017070 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0dd37fe640 1 -- 172.21.15.64:0/2487947224 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0dc4005ce0 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.816+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0ddc1e7e20 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.817+0000 7f0dd37fe640 1 -- 172.21.15.64:0/2487947224 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dc40072f0 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.817+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0ddc1e83e0 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.818+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0ddc068700 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.818+0000 7f0dd37fe640 1 -- 172.21.15.64:0/2487947224 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0dc401a070 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.818+0000 7f0dd37fe640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dbc06b500 0x7f0dbc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:01.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.819+0000 7f0ddad9d640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dbc06b500 0x7f0dbc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:01.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.819+0000 7f0dd37fe640 1 -- 172.21.15.64:0/2487947224 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0dc4089680 con 0x7f0ddc0ae9b0 2024-03-04T07:28:01.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.819+0000 7f0ddad9d640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dbc06b500 0x7f0dbc06d9c0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f0dd4036000 tx=0x7f0dd403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:01.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:01.823+0000 7f0dd37fe640 1 -- 172.21.15.64:0/2487947224 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0dc4057e20 con 0x7f0ddc0ae9b0 2024-03-04T07:28:02.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.043+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0ddc0d08d0 con 0x7f0ddc0ae9b0 2024-03-04T07:28:02.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.043+0000 7f0dd37fe640 1 -- 172.21.15.64:0/2487947224 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f0dc405bec0 con 0x7f0ddc0ae9b0 2024-03-04T07:28:02.043 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:02.043 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:02.043 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dbc06b500 msgr2=0x7f0dbc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dbc06b500 0x7f0dbc06d9c0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f0dd4036000 tx=0x7f0dd403d040 comp rx=0 tx=0).stop 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0ae9b0 msgr2=0x7f0ddc1dfe20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0ae9b0 0x7f0ddc1dfe20 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7f0dc400c430 tx=0x7f0dc400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 shutdown_connections 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dbc06b500 0x7f0dbc06d9c0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc0b0e90 0x7f0ddc06f7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0ae9b0 0x7f0ddc1dfe20 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 --2- 172.21.15.64:0/2487947224 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc06b5c0 0x7f0ddc1e73d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 >> 172.21.15.64:0/2487947224 conn(0x7f0ddc0b36d0 msgr2=0x7f0ddc1e50f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:02.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 shutdown_connections 2024-03-04T07:28:02.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.045+0000 7f0de1242640 1 -- 172.21.15.64:0/2487947224 wait complete. 2024-03-04T07:28:02.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json flush journal 2024-03-04T07:28:02.398 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:28:02.399 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:28:02.399 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:28:02.399 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:28:02.399 DEBUG:teuthology.orchestra.run.smithi064:> import stat 2024-03-04T07:28:02.399 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:02.400 DEBUG:teuthology.orchestra.run.smithi064:> print(os.stat("/home/ubuntu/cephtest/mnt.0/delete_me").st_ino) 2024-03-04T07:28:02.400 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:28:02.465 INFO:teuthology.orchestra.run.smithi064.stdout:1099511627776 2024-03-04T07:28:02.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:28:02.473 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_metadata stat 10000000000.00000000 2024-03-04T07:28:02.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 Processor -- start 2024-03-04T07:28:02.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 -- start start 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3940 0x55a66a448c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c3d20 0x55a66a37ecf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c51e0 0x55a66a442fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a66a3c42c0 con 0x55a66a3c51e0 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a66a44b970 con 0x55a66a3c3940 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcd447f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55a66a3c4560 con 0x55a66a3c3d20 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.554+0000 7fcdcafa9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c51e0 0x55a66a442fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c51e0 0x55a66a442fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49146/0 (socket says 172.21.15.64:49146) 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 -- 172.21.15.64:0/3644514493 learned_addr learned my addr 172.21.15.64:0/3644514493 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdca7a8640 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3940 0x55a66a448c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdc9fa7640 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c3d20 0x55a66a37ecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 -- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c3d20 msgr2=0x55a66a37ecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c3d20 0x55a66a37ecf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 -- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3940 msgr2=0x55a66a448c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3940 0x55a66a448c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 -- 172.21.15.64:0/3644514493 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a66a2a9c50 con 0x55a66a3c51e0 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdc9fa7640 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c3d20 0x55a66a37ecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdcafa9640 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c51e0 0x55a66a442fd0 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fcdc400c4b0 tx=0x7fcdc400c970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b0625c25d198d71a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdc8fa5640 1 -- 172.21.15.64:0/3644514493 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcdc40116b0 con 0x55a66a3c51e0 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdc8fa5640 1 -- 172.21.15.64:0/3644514493 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcdc4017070 con 0x55a66a3c51e0 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.555+0000 7fcdc8fa5640 1 -- 172.21.15.64:0/3644514493 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcdc4011a90 con 0x55a66a3c51e0 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 -- 172.21.15.64:0/3644514493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c51e0 msgr2=0x55a66a442fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c51e0 0x55a66a442fd0 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fcdc400c4b0 tx=0x7fcdc400c970 comp rx=0 tx=0).stop 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 -- 172.21.15.64:0/3644514493 shutdown_connections 2024-03-04T07:28:02.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c51e0 0x55a66a442fd0 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c3d20 0x55a66a37ecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/3644514493 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3940 0x55a66a448c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 -- 172.21.15.64:0/3644514493 >> 172.21.15.64:0/3644514493 conn(0x55a66a37c5e0 msgr2=0x55a66a3ab350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:02.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 -- 172.21.15.64:0/3644514493 shutdown_connections 2024-03-04T07:28:02.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 -- 172.21.15.64:0/3644514493 wait complete. 2024-03-04T07:28:02.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 Processor -- start 2024-03-04T07:28:02.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.556+0000 7fcdcd447f40 1 -- start start 2024-03-04T07:28:02.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdcd447f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c3940 0x55a66a44d8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdcd447f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3d20 0x55a66a453b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdcd447f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c51e0 0x55a66a4540c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdcd447f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a66a449320 con 0x55a66a3c3940 2024-03-04T07:28:02.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdcd447f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a66a445ff0 con 0x55a66a3c3d20 2024-03-04T07:28:02.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdcd447f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55a66a446250 con 0x55a66a3c51e0 2024-03-04T07:28:02.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c3940 0x55a66a44d8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c3940 0x55a66a44d8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49158/0 (socket says 172.21.15.64:49158) 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 -- 172.21.15.64:0/609419205 learned_addr learned my addr 172.21.15.64:0/609419205 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdc9fa7640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3d20 0x55a66a453b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdcafa9640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c51e0 0x55a66a4540c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 -- 172.21.15.64:0/609419205 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c51e0 msgr2=0x55a66a4540c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55a66a3c51e0 0x55a66a4540c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 -- 172.21.15.64:0/609419205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3d20 msgr2=0x55a66a453b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3d20 0x55a66a453b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdca7a8640 1 -- 172.21.15.64:0/609419205 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a66a44e1e0 con 0x55a66a3c3940 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.557+0000 7fcdc9fa7640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55a66a3c3d20 0x55a66a453b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.558+0000 7fcdca7a8640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c3940 0x55a66a44d8a0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7fcdbc058040 tx=0x7fcdbc002fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.558+0000 7fcdbb7fe640 1 -- 172.21.15.64:0/609419205 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcdbc06b480 con 0x55a66a3c3940 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.558+0000 7fcdbb7fe640 1 -- 172.21.15.64:0/609419205 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcdbc06b640 con 0x55a66a3c3940 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.558+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x55a66a44e4c0 con 0x55a66a3c3940 2024-03-04T07:28:02.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.558+0000 7fcdbb7fe640 1 -- 172.21.15.64:0/609419205 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcdbc06ba30 con 0x55a66a3c3940 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.558+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55a66a44ea80 con 0x55a66a3c3940 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.559+0000 7fcdbb7fe640 1 -- 172.21.15.64:0/609419205 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcdbc056420 con 0x55a66a3c3940 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdbb7fe640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcdac069eb0 0x7fcdac06c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdbb7fe640 1 -- 172.21.15.64:0/609419205 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcdbc0ce410 con 0x55a66a3c3940 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdc9fa7640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcdac069eb0 0x7fcdac06c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55a66a37ba60 0x55a66a44c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:1 4.10 4:0cf6ad0f:::10000000000.00000000:head [stat] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e33) v9 -- 0x55a66a44cb20 con 0x55a66a37ba60 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdcafa9640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55a66a37ba60 0x55a66a44c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdc9fa7640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcdac069eb0 0x7fcdac06c370 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x55a66a4e99f0 tx=0x7fcdc0005980 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.560+0000 7fcdcafa9640 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55a66a37ba60 0x55a66a44c5e0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcafa9640 1 -- 172.21.15.64:0/609419205 <== osd.3 v2:172.21.15.64:6816/1402142346 1 ==== osd_op_reply(1 10000000000.00000000 [stat out=16b] v0'0 uv10 ondisk = 0) v8 ==== 164+0+16 (crc 0 0 0) 0x7fcdc4011850 con 0x55a66a37ba60 2024-03-04T07:28:02.563 INFO:teuthology.orchestra.run.smithi064.stdout:cephfs_metadata/10000000000.00000000 mtime 2024-03-04T07:28:02.000000+0000, size 0 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55a66a37ba60 msgr2=0x55a66a44c5e0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55a66a37ba60 0x55a66a44c5e0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcdac069eb0 msgr2=0x7fcdac06c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcdac069eb0 0x7fcdac06c370 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x55a66a4e99f0 tx=0x7fcdc0005980 comp rx=0 tx=0).stop 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c3940 msgr2=0x55a66a44d8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 --2- 172.21.15.64:0/609419205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55a66a3c3940 0x55a66a44d8a0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7fcdbc058040 tx=0x7fcdbc002fb0 comp rx=0 tx=0).stop 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcafa9640 1 -- 172.21.15.64:0/609419205 reap_dead start 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 shutdown_connections 2024-03-04T07:28:02.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 >> 172.21.15.64:0/609419205 conn(0x55a66a37c5e0 msgr2=0x55a66a449db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:02.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 shutdown_connections 2024-03-04T07:28:02.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:02.561+0000 7fcdcd447f40 1 -- 172.21.15.64:0/609419205 wait complete. 2024-03-04T07:28:02.566 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:02.566 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -rf delete_me) 2024-03-04T07:28:04.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:05.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.058+0000 7fe48ee88640 1 Processor -- start 2024-03-04T07:28:05.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.058+0000 7fe48ee88640 1 -- start start 2024-03-04T07:28:05.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe48ee88640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe4880b4260 0x7fe4880cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe48ee88640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4880b2320 con 0x7fe4880d06c0 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe48ee88640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4880b0dd0 con 0x7fe4880d02e0 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe48ee88640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe4880b0b80 con 0x7fe4880b4260 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe48d425640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe4880d06c0 0x7fe4880b1c10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57146/0 (socket says 172.21.15.64:57146) 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe48d425640 1 -- 172.21.15.64:0/2854286323 learned_addr learned my addr 172.21.15.64:0/2854286323 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1698563113 0 0) 0x7fe4880b2320 con 0x7fe4880d06c0 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe470003520 con 0x7fe4880d06c0 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.059+0000 7fe487fff640 1 --2- 172.21.15.64:0/2854286323 >> v2:172.21.15.66:3300/0 conn(0x7fe4880b4260 0x7fe4880cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3365938526 0 0) 0x7fe470003520 con 0x7fe4880d06c0 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 >> v2:172.21.15.66:3300/0 conn(0x7fe4880b4260 msgr2=0x7fe4880cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 --2- 172.21.15.64:0/2854286323 >> v2:172.21.15.66:3300/0 conn(0x7fe4880b4260 0x7fe4880cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 >> v1:172.21.15.66:6789/0 conn(0x7fe4880d02e0 legacy=0x7fe4880b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4880b10b0 con 0x7fe4880d06c0 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fe478001670 con 0x7fe4880d06c0 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe487fff640 1 --2- 172.21.15.64:0/2854286323 >> v2:172.21.15.66:3300/0 conn(0x7fe4880b4260 0x7fe4880cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 >> v1:172.21.15.64:6789/0 conn(0x7fe4880d06c0 legacy=0x7fe4880b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4700039b0 0x7fe470003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe470004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4700051c0 0x7fe47000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4880b0b80 con 0x7fe470004800 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe47000c0f0 con 0x7fe4700039b0 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe47000c2f0 con 0x7fe4700051c0 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe487fff640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe470004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe48cc24640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4700039b0 0x7fe470003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe48d425640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4700051c0 0x7fe47000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe487fff640 1 -- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4700051c0 msgr2=0x7fe47000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe487fff640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4700051c0 0x7fe47000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe487fff640 1 -- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4700039b0 msgr2=0x7fe470003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe487fff640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4700039b0 0x7fe470003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe487fff640 1 -- 172.21.15.64:0/2854286323 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe470003520 con 0x7fe470004800 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe48cc24640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4700039b0 0x7fe470003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.060+0000 7fe48d425640 1 -- 172.21.15.64:0/2854286323 reap_dead start 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.061+0000 7fe487fff640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe470004c80 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7fe47c0098a0 tx=0x7fe470005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e62019c5727b41c7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.061+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe47c00f040 con 0x7fe470004800 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.061+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe47c004230 con 0x7fe470004800 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.061+0000 7fe4877fe640 1 -- 172.21.15.64:0/2854286323 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe47c003610 con 0x7fe470004800 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.062+0000 7fe48ee88640 1 -- 172.21.15.64:0/2854286323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 msgr2=0x7fe470004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.062+0000 7fe48ee88640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe470004c80 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7fe47c0098a0 tx=0x7fe470005830 comp rx=0 tx=0).stop 2024-03-04T07:28:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.062+0000 7fe48ee88640 1 -- 172.21.15.64:0/2854286323 shutdown_connections 2024-03-04T07:28:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.062+0000 7fe48ee88640 1 --2- 172.21.15.64:0/2854286323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe470004c80 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.062+0000 7fe48ee88640 1 -- 172.21.15.64:0/2854286323 >> 172.21.15.64:0/2854286323 conn(0x7fe4880b36d0 msgr2=0x7fe4880b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.062+0000 7fe48ee88640 1 -- 172.21.15.64:0/2854286323 shutdown_connections 2024-03-04T07:28:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.062+0000 7fe48ee88640 1 -- 172.21.15.64:0/2854286323 wait complete. 2024-03-04T07:28:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.063+0000 7fe48ee88640 1 Processor -- start 2024-03-04T07:28:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.064+0000 7fe48ee88640 1 -- start start 2024-03-04T07:28:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48ee88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe4881532d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48ee88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe488153b90 0x7fe48806fcb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48ee88640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe488154010 0x7fe4880701f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48ee88640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4880cfa00 con 0x7fe470004800 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48ee88640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4880aeec0 con 0x7fe488154010 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48ee88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4880cfb80 con 0x7fe488153b90 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe4881532d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe4881532d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42400/0 (socket says 172.21.15.64:42400) 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 -- 172.21.15.64:0/988864076 learned_addr learned my addr 172.21.15.64:0/988864076 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 -- 172.21.15.64:0/988864076 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe488153b90 msgr2=0x7fe48806fcb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe488153b90 0x7fe48806fcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 -- 172.21.15.64:0/988864076 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe488154010 msgr2=0x7fe4880701f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe488154010 0x7fe4880701f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 -- 172.21.15.64:0/988864076 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4880c8c70 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.065+0000 7fe48cc24640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe4881532d0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7fe48000e7b0 tx=0x7fe48000ec70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.066+0000 7fe4857fa640 1 -- 172.21.15.64:0/988864076 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe480003b80 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.066+0000 7fe4857fa640 1 -- 172.21.15.64:0/988864076 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe480003d20 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.066+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4880c8f50 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.066+0000 7fe4857fa640 1 -- 172.21.15.64:0/988864076 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe480006b50 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.066+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4880c9510 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.067+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe488068700 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.067+0000 7fe4857fa640 1 -- 172.21.15.64:0/988864076 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe48001a020 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.068+0000 7fe4857fa640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe47406b500 0x7fe47406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.068+0000 7fe487fff640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe47406b500 0x7fe47406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.068+0000 7fe4857fa640 1 -- 172.21.15.64:0/988864076 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe480089980 con 0x7fe470004800 2024-03-04T07:28:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.068+0000 7fe487fff640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe47406b500 0x7fe47406d9c0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fe47c035000 tx=0x7fe47c030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.072+0000 7fe4857fa640 1 -- 172.21.15.64:0/988864076 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe480058890 con 0x7fe470004800 2024-03-04T07:28:05.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.295+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe48806f790 con 0x7fe470004800 2024-03-04T07:28:05.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.296+0000 7fe4857fa640 1 -- 172.21.15.64:0/988864076 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7fe480058190 con 0x7fe470004800 2024-03-04T07:28:05.296 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:05.296 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:05.296 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe47406b500 msgr2=0x7fe47406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe47406b500 0x7fe47406d9c0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fe47c035000 tx=0x7fe47c030000 comp rx=0 tx=0).stop 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 msgr2=0x7fe4881532d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe4881532d0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7fe48000e7b0 tx=0x7fe48000ec70 comp rx=0 tx=0).stop 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 shutdown_connections 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe47406b500 0x7fe47406d9c0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe488154010 0x7fe4880701f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe488153b90 0x7fe48806fcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 --2- 172.21.15.64:0/988864076 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe470004800 0x7fe4881532d0 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 >> 172.21.15.64:0/988864076 conn(0x7fe4880b36d0 msgr2=0x7fe48806b130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:05.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 shutdown_connections 2024-03-04T07:28:05.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.298+0000 7fe48ee88640 1 -- 172.21.15.64:0/988864076 wait complete. 2024-03-04T07:28:05.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json flush journal 2024-03-04T07:28:05.604 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:28:05.605 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:28:05.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:05.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.708+0000 7fbbc2e6b640 1 Processor -- start 2024-03-04T07:28:05.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.708+0000 7fbbc2e6b640 1 -- start start 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc2e6b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fbbbc06b5c0 0x7fbbbc0b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc2e6b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbbbc0ae9b0 0x7fbbbc0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc2e6b640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbbbc0b3340 con 0x7fbbbc0b0e90 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc2e6b640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbbbc0cfa00 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc2e6b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbbbc0cee90 con 0x7fbbbc06b5c0 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc1408640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbbbc0ae9b0 0x7fbbbc0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbb3fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbbbc0b0e90 0x7fbbbc06b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57152/0 (socket says 172.21.15.64:57152) 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc1408640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbbbc0ae9b0 0x7fbbbc0d03a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42408/0 (socket says 172.21.15.64:42408) 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbb3fff640 1 -- 172.21.15.64:0/2291864148 learned_addr learned my addr 172.21.15.64:0/2291864148 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.709+0000 7fbbc0c07640 1 --2- 172.21.15.64:0/2291864148 >> v2:172.21.15.66:3300/0 conn(0x7fbbbc06b5c0 0x7fbbbc0b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.710+0000 7fbbc1408640 1 -- 172.21.15.64:0/2291864148 >> v2:172.21.15.66:3300/0 conn(0x7fbbbc06b5c0 msgr2=0x7fbbbc0b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.710+0000 7fbbc1408640 1 --2- 172.21.15.64:0/2291864148 >> v2:172.21.15.66:3300/0 conn(0x7fbbbc06b5c0 0x7fbbbc0b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.710+0000 7fbbc1408640 1 -- 172.21.15.64:0/2291864148 >> v1:172.21.15.64:6789/0 conn(0x7fbbbc0b0e90 legacy=0x7fbbbc06b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.710+0000 7fbbc1408640 1 -- 172.21.15.64:0/2291864148 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbbbc0cfc50 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.710+0000 7fbbc0c07640 1 --2- 172.21.15.64:0/2291864148 >> v2:172.21.15.66:3300/0 conn(0x7fbbbc06b5c0 0x7fbbbc0b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc1408640 1 --2- 172.21.15.64:0/2291864148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0d03a0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7fbbbc071b00 tx=0x7fbbb402be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=13fe0f43c9408bd1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbb37fe640 1 -- 172.21.15.64:0/2291864148 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbbb4039070 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbb37fe640 1 -- 172.21.15.64:0/2291864148 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbbb402cbf0 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbb37fe640 1 -- 172.21.15.64:0/2291864148 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbbb4034440 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/2291864148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 msgr2=0x7fbbbc0d03a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/2291864148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0d03a0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7fbbbc071b00 tx=0x7fbbb402be40 comp rx=0 tx=0).stop 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/2291864148 shutdown_connections 2024-03-04T07:28:05.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/2291864148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0d03a0 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/2291864148 >> v2:172.21.15.66:3300/0 conn(0x7fbbbc06b5c0 0x7fbbbc0b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/2291864148 >> 172.21.15.64:0/2291864148 conn(0x7fbbbc0b36d0 msgr2=0x7fbbbc0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/2291864148 shutdown_connections 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.711+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/2291864148 wait complete. 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 Processor -- start 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 -- start start 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbbbc06b5c0 0x7fbbbc1e6f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0c6040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbbbc0b0e90 0x7fbbbc1dd270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbbbc06c910 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbbbc06c790 con 0x7fbbbc0b0e90 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.713+0000 7fbbc2e6b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbbbc0aedc0 con 0x7fbbbc06b5c0 2024-03-04T07:28:05.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0c6040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0c6040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42422/0 (socket says 172.21.15.64:42422) 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 -- 172.21.15.64:0/999814001 learned_addr learned my addr 172.21.15.64:0/999814001 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbc1408640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbbbc0b0e90 0x7fbbbc1dd270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbc0c07640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbbbc06b5c0 0x7fbbbc1e6f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 -- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbbbc06b5c0 msgr2=0x7fbbbc1e6f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbbbc06b5c0 0x7fbbbc1e6f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 -- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbbbc0b0e90 msgr2=0x7fbbbc1dd270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbbbc0b0e90 0x7fbbbc1dd270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 -- 172.21.15.64:0/999814001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbbbc0c7c40 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbc0c07640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbbbc06b5c0 0x7fbbbc1e6f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb3fff640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0c6040 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7fbbac00c3c0 tx=0x7fbbac00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb17fa640 1 -- 172.21.15.64:0/999814001 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbbac017070 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb17fa640 1 -- 172.21.15.64:0/999814001 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbbac005ce0 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbbbc0c7ee0 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbb17fa640 1 -- 172.21.15.64:0/999814001 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbbac007310 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.714+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbbbc1e41e0 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.715+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbbbc068700 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.716+0000 7fbbb17fa640 1 -- 172.21.15.64:0/999814001 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbbac01a070 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.716+0000 7fbbb17fa640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbba806b4d0 0x7fbba806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:05.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.716+0000 7fbbb17fa640 1 -- 172.21.15.64:0/999814001 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbbac0894f0 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.716+0000 7fbbc0c07640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbba806b4d0 0x7fbba806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:05.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.717+0000 7fbbc0c07640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbba806b4d0 0x7fbba806d990 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fbbb8006fd0 tx=0x7fbbb8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:05.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.721+0000 7fbbb17fa640 1 -- 172.21.15.64:0/999814001 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbbac058400 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.939+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbbbc1dd7b0 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.940+0000 7fbbb17fa640 1 -- 172.21.15.64:0/999814001 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7fbbac057d00 con 0x7fbbbc0ae9b0 2024-03-04T07:28:05.940 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:05.940 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:05.940 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbba806b4d0 msgr2=0x7fbba806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbba806b4d0 0x7fbba806d990 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fbbb8006fd0 tx=0x7fbbb8008040 comp rx=0 tx=0).stop 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 msgr2=0x7fbbbc0c6040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0c6040 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7fbbac00c3c0 tx=0x7fbbac00c880 comp rx=0 tx=0).stop 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 shutdown_connections 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbba806b4d0 0x7fbba806d990 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbbbc0b0e90 0x7fbbbc1dd270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbbbc0ae9b0 0x7fbbbc0c6040 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 --2- 172.21.15.64:0/999814001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbbbc06b5c0 0x7fbbbc1e6f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 >> 172.21.15.64:0/999814001 conn(0x7fbbbc0b36d0 msgr2=0x7fbbbc1df9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 shutdown_connections 2024-03-04T07:28:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:05.943+0000 7fbbc2e6b640 1 -- 172.21.15.64:0/999814001 wait complete. 2024-03-04T07:28:05.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_created 2024-03-04T07:28:06.100 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_created":1001}} 2024-03-04T07:28:06.101 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 1001 } } 2024-03-04T07:28:06.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:06.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.234+0000 7fe6f2eb8640 1 Processor -- start 2024-03-04T07:28:06.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.234+0000 7fe6f2eb8640 1 -- start start 2024-03-04T07:28:06.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6f2eb8640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe6ec15ac30 0x7fe6ec0b3940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:06.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6f2eb8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe6ec0720c0 0x7fe6ec0cdf90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6f2eb8640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe6ec0b3e80 con 0x7fe6ec0afb30 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6f2eb8640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe6ec0b4120 con 0x7fe6ec0720c0 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6f2eb8640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe6ec0b4400 con 0x7fe6ec15ac30 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6f1455640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe6ec0720c0 0x7fe6ec0cdf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6ebfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe6ec0afb30 0x7fe6ec0719b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57154/0 (socket says 172.21.15.64:57154) 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.235+0000 7fe6ebfff640 1 -- 172.21.15.64:0/3539797727 learned_addr learned my addr 172.21.15.64:0/3539797727 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6f0c54640 1 --2- 172.21.15.64:0/3539797727 >> v2:172.21.15.66:3300/0 conn(0x7fe6ec15ac30 0x7fe6ec0b3940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6f1455640 1 -- 172.21.15.64:0/3539797727 >> v2:172.21.15.66:3300/0 conn(0x7fe6ec15ac30 msgr2=0x7fe6ec0b3940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6f1455640 1 --2- 172.21.15.64:0/3539797727 >> v2:172.21.15.66:3300/0 conn(0x7fe6ec15ac30 0x7fe6ec0b3940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6f1455640 1 -- 172.21.15.64:0/3539797727 >> v1:172.21.15.64:6789/0 conn(0x7fe6ec0afb30 legacy=0x7fe6ec0719b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6f1455640 1 -- 172.21.15.64:0/3539797727 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6ec0b46e0 con 0x7fe6ec0720c0 2024-03-04T07:28:06.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6eb7fe640 1 -- 172.21.15.64:0/3539797727 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2148473723 0 0) 0x7fe6ec0b3e80 con 0x7fe6ec0afb30 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6f1455640 1 --2- 172.21.15.64:0/3539797727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec0cdf90 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fe6ec0d1a70 tx=0x7fe6dc02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=db92ce3597f8af54 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6eb7fe640 1 -- 172.21.15.64:0/3539797727 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6dc035020 con 0x7fe6ec0720c0 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6eb7fe640 1 -- 172.21.15.64:0/3539797727 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6dc031830 con 0x7fe6ec0720c0 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.236+0000 7fe6eb7fe640 1 -- 172.21.15.64:0/3539797727 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6dc031c00 con 0x7fe6ec0720c0 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/3539797727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 msgr2=0x7fe6ec0cdf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/3539797727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec0cdf90 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fe6ec0d1a70 tx=0x7fe6dc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/3539797727 shutdown_connections 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/3539797727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec0cdf90 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/3539797727 >> v2:172.21.15.66:3300/0 conn(0x7fe6ec15ac30 0x7fe6ec0b3940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/3539797727 >> 172.21.15.64:0/3539797727 conn(0x7fe6ec0d0990 msgr2=0x7fe6ec0d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/3539797727 shutdown_connections 2024-03-04T07:28:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.237+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/3539797727 wait complete. 2024-03-04T07:28:06.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.239+0000 7fe6f2eb8640 1 Processor -- start 2024-03-04T07:28:06.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.239+0000 7fe6f2eb8640 1 -- start start 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f2eb8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec150290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f2eb8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6ec0afb30 0x7fe6ec14faf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f2eb8640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6ec15ac30 0x7fe6ec066d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f2eb8640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6ec0ce4d0 con 0x7fe6ec0720c0 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f2eb8640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6ec0b4ca0 con 0x7fe6ec15ac30 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f2eb8640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe6ec0ce650 con 0x7fe6ec0afb30 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f0c54640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec150290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f0c54640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec150290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42442/0 (socket says 172.21.15.64:42442) 2024-03-04T07:28:06.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.240+0000 7fe6f0c54640 1 -- 172.21.15.64:0/1698469471 learned_addr learned my addr 172.21.15.64:0/1698469471 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f0c54640 1 -- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6ec0afb30 msgr2=0x7fe6ec14faf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f0c54640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6ec0afb30 0x7fe6ec14faf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f1455640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6ec15ac30 0x7fe6ec066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f0c54640 1 -- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6ec15ac30 msgr2=0x7fe6ec066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f0c54640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6ec15ac30 0x7fe6ec066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f0c54640 1 -- 172.21.15.64:0/1698469471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6ec1f5100 con 0x7fe6ec0720c0 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f1455640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6ec15ac30 0x7fe6ec066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f0c54640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec150290 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7fe6e400cd00 tx=0x7fe6e400a5c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6e97fa640 1 -- 172.21.15.64:0/1698469471 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6e4003990 con 0x7fe6ec0720c0 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6e97fa640 1 -- 172.21.15.64:0/1698469471 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6e4003b30 con 0x7fe6ec0720c0 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe6ec1f53e0 con 0x7fe6ec0720c0 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6e97fa640 1 -- 172.21.15.64:0/1698469471 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6e4004020 con 0x7fe6ec0720c0 2024-03-04T07:28:06.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.241+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe6ec150030 con 0x7fe6ec0720c0 2024-03-04T07:28:06.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.242+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe6ec072b50 con 0x7fe6ec0720c0 2024-03-04T07:28:06.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.243+0000 7fe6e97fa640 1 -- 172.21.15.64:0/1698469471 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe6e40041c0 con 0x7fe6ec0720c0 2024-03-04T07:28:06.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.243+0000 7fe6e97fa640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6cc06b4d0 0x7fe6cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:06.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.243+0000 7fe6e97fa640 1 -- 172.21.15.64:0/1698469471 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe6e408a670 con 0x7fe6ec0720c0 2024-03-04T07:28:06.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.243+0000 7fe6ebfff640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6cc06b4d0 0x7fe6cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:06.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.244+0000 7fe6ebfff640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6cc06b4d0 0x7fe6cc06d990 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7fe6d4005880 tx=0x7fe6d4007950 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:06.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.248+0000 7fe6e97fa640 1 -- 172.21.15.64:0/1698469471 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe6ec072b50 con 0x7fe6ec0720c0 2024-03-04T07:28:06.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.465+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe6ec068700 con 0x7fe6ec0720c0 2024-03-04T07:28:06.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.466+0000 7fe6e97fa640 1 -- 172.21.15.64:0/1698469471 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7fe6e4014030 con 0x7fe6ec0720c0 2024-03-04T07:28:06.562 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:06.562 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6cc06b4d0 msgr2=0x7fe6cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6cc06b4d0 0x7fe6cc06d990 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7fe6d4005880 tx=0x7fe6d4007950 comp rx=0 tx=0).stop 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 msgr2=0x7fe6ec150290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec150290 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7fe6e400cd00 tx=0x7fe6e400a5c0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 shutdown_connections 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6cc06b4d0 0x7fe6cc06d990 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6ec15ac30 0x7fe6ec066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6ec0afb30 0x7fe6ec14faf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 --2- 172.21.15.64:0/1698469471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6ec0720c0 0x7fe6ec150290 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 >> 172.21.15.64:0/1698469471 conn(0x7fe6ec0d0990 msgr2=0x7fe6ec06f020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 shutdown_connections 2024-03-04T07:28:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:06.468+0000 7fe6f2eb8640 1 -- 172.21.15.64:0/1698469471 wait complete. 2024-03-04T07:28:06.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:28:06.684 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":427}} 2024-03-04T07:28:06.684 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 427 } } 2024-03-04T07:28:06.684 DEBUG:tasks.ceph_test_case:wait_until_equal: 427 != 1001, waiting (timeout=600)... 2024-03-04T07:28:11.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:11.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.788+0000 7f6931266640 1 Processor -- start 2024-03-04T07:28:11.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.788+0000 7f6931266640 1 -- start start 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f6931266640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f692c06b5c0 0x7f692c0b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f6931266640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 0x7f692c0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f6931266640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f692c0b3340 con 0x7f692c0b0e90 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f6931266640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f692c0cfa00 con 0x7f692c06b5c0 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f6931266640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f692c0cee90 con 0x7f692c0ae9b0 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f692c06b5c0 0x7f692c0b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692a7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f692c0b0e90 0x7f692c06b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57170/0 (socket says 172.21.15.64:57170) 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f692c06b5c0 0x7f692c0b2e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42452/0 (socket says 172.21.15.64:42452) 2024-03-04T07:28:11.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692a7fc640 1 -- 172.21.15.64:0/583205131 learned_addr learned my addr 172.21.15.64:0/583205131 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:11.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692affd640 1 -- 172.21.15.64:0/583205131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 msgr2=0x7f692c0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:11.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692affd640 1 --2- 172.21.15.64:0/583205131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 0x7f692c0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:11.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692affd640 1 -- 172.21.15.64:0/583205131 >> v1:172.21.15.64:6789/0 conn(0x7f692c0b0e90 legacy=0x7f692c06b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:11.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f692affd640 1 -- 172.21.15.64:0/583205131 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f692c0cfc50 con 0x7f692c06b5c0 2024-03-04T07:28:11.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.790+0000 7f6929ffb640 1 -- 172.21.15.64:0/583205131 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2452106866 0 0) 0x7f692c0b3340 con 0x7f692c0b0e90 2024-03-04T07:28:11.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.791+0000 7f692affd640 1 --2- 172.21.15.64:0/583205131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c0b2e00 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f691c0097a0 tx=0x7f691c02b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aeee0aa760458b8b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.791+0000 7f6929ffb640 1 -- 172.21.15.64:0/583205131 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f691c0315a0 con 0x7f692c06b5c0 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.791+0000 7f6929ffb640 1 -- 172.21.15.64:0/583205131 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f691c037070 con 0x7f692c06b5c0 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.791+0000 7f6929ffb640 1 -- 172.21.15.64:0/583205131 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f691c031960 con 0x7f692c06b5c0 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 -- 172.21.15.64:0/583205131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 msgr2=0x7f692c0b2e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 --2- 172.21.15.64:0/583205131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c0b2e00 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f691c0097a0 tx=0x7f691c02b840 comp rx=0 tx=0).stop 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 -- 172.21.15.64:0/583205131 shutdown_connections 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 --2- 172.21.15.64:0/583205131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 0x7f692c0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 --2- 172.21.15.64:0/583205131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c0b2e00 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 -- 172.21.15.64:0/583205131 >> 172.21.15.64:0/583205131 conn(0x7f692c0b36d0 msgr2=0x7f692c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 -- 172.21.15.64:0/583205131 shutdown_connections 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.792+0000 7f6931266640 1 -- 172.21.15.64:0/583205131 wait complete. 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.793+0000 7f6931266640 1 Processor -- start 2024-03-04T07:28:11.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.793+0000 7f6931266640 1 -- start start 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f6931266640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c1d1070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f6931266640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 0x7f692c0ccfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f6931266640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f692c0b0e90 0x7f692c0cd510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f6931266640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f692c06c910 con 0x7f692c06b5c0 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f6931266640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f692c06c790 con 0x7f692c0b0e90 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f6931266640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f692c0aee00 con 0x7f692c0ae9b0 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f692affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c1d1070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:11.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f692affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c1d1070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42454/0 (socket says 172.21.15.64:42454) 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.794+0000 7f692affd640 1 -- 172.21.15.64:0/722793713 learned_addr learned my addr 172.21.15.64:0/722793713 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692a7fc640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 0x7f692c0ccfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692b7fe640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f692c0b0e90 0x7f692c0cd510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692affd640 1 -- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 msgr2=0x7f692c0ccfd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692affd640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 0x7f692c0ccfd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692affd640 1 -- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f692c0b0e90 msgr2=0x7f692c0cd510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692affd640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f692c0b0e90 0x7f692c0cd510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692affd640 1 -- 172.21.15.64:0/722793713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f692c1e5380 con 0x7f692c06b5c0 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f692affd640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c1d1070 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f691c0097a0 tx=0x7f691c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f69237fe640 1 -- 172.21.15.64:0/722793713 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f691c004230 con 0x7f692c06b5c0 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f69237fe640 1 -- 172.21.15.64:0/722793713 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f691c0043f0 con 0x7f692c06b5c0 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f692c1e4dc0 con 0x7f692c06b5c0 2024-03-04T07:28:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f69237fe640 1 -- 172.21.15.64:0/722793713 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f691c032880 con 0x7f692c06b5c0 2024-03-04T07:28:11.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.795+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f692c0c6af0 con 0x7f692c06b5c0 2024-03-04T07:28:11.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.796+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f692c068700 con 0x7f692c06b5c0 2024-03-04T07:28:11.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.797+0000 7f69237fe640 1 -- 172.21.15.64:0/722793713 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f691c032a20 con 0x7f692c06b5c0 2024-03-04T07:28:11.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.797+0000 7f69237fe640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f690c06b4d0 0x7f690c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:11.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.797+0000 7f69237fe640 1 -- 172.21.15.64:0/722793713 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f691c02f030 con 0x7f692c06b5c0 2024-03-04T07:28:11.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.797+0000 7f692a7fc640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f690c06b4d0 0x7f690c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:11.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.798+0000 7f692a7fc640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f690c06b4d0 0x7f690c06d990 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f692c0706c0 tx=0x7f6914007c00 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:11.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:11.801+0000 7f69237fe640 1 -- 172.21.15.64:0/722793713 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f691c041070 con 0x7f692c06b5c0 2024-03-04T07:28:12.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.020+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f692c1e66a0 con 0x7f692c06b5c0 2024-03-04T07:28:12.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.020+0000 7f69237fe640 1 -- 172.21.15.64:0/722793713 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f691c07b390 con 0x7f692c06b5c0 2024-03-04T07:28:12.020 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:12.020 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:12.020 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:12.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.022+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f690c06b4d0 msgr2=0x7f690c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f690c06b4d0 0x7f690c06d990 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f692c0706c0 tx=0x7f6914007c00 comp rx=0 tx=0).stop 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 msgr2=0x7f692c1d1070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c1d1070 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f691c0097a0 tx=0x7f691c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 shutdown_connections 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f690c06b4d0 0x7f690c06d990 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f692c0b0e90 0x7f692c0cd510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f692c0ae9b0 0x7f692c0ccfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 --2- 172.21.15.64:0/722793713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f692c06b5c0 0x7f692c1d1070 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 >> 172.21.15.64:0/722793713 conn(0x7f692c0b36d0 msgr2=0x7f692c1e2bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 shutdown_connections 2024-03-04T07:28:12.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.023+0000 7f6931266640 1 -- 172.21.15.64:0/722793713 wait complete. 2024-03-04T07:28:12.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:28:12.180 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":1001}} 2024-03-04T07:28:12.181 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 1001 } } 2024-03-04T07:28:12.181 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:12.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.285+0000 7eff423f7640 1 Processor -- start 2024-03-04T07:28:12.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.285+0000 7eff423f7640 1 -- start start 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff423f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7eff3c0b0e90 0x7eff3c06b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff423f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff423f7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7eff3c0cee90 con 0x7eff3c06b5c0 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff423f7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7eff3c0cfa00 con 0x7eff3c0b0e90 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff423f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff3c0cfc50 con 0x7eff3c0ae9b0 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff3b7fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7eff3c0b0e90 0x7eff3c06b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff40994640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff3bfff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7eff3c06b5c0 0x7eff3c0b2e00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:44264/0 (socket says 172.21.15.64:44264) 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff3bfff640 1 -- 172.21.15.64:0/608464912 learned_addr learned my addr 172.21.15.64:0/608464912 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:12.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff3b7fe640 1 -- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 msgr2=0x7eff3c0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:12.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff3b7fe640 1 --2- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c0d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff3b7fe640 1 -- 172.21.15.64:0/608464912 >> v1:172.21.15.66:6789/0 conn(0x7eff3c06b5c0 legacy=0x7eff3c0b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff3b7fe640 1 -- 172.21.15.64:0/608464912 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff3c0cff30 con 0x7eff3c0b0e90 2024-03-04T07:28:12.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.286+0000 7eff40994640 1 --2- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:28:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.287+0000 7eff3b7fe640 1 --2- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 0x7eff3c06b9a0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7eff2c0097a0 tx=0x7eff2c02b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c0740303ff17410e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.287+0000 7eff3affd640 1 -- 172.21.15.64:0/608464912 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff2c031780 con 0x7eff3c0b0e90 2024-03-04T07:28:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.287+0000 7eff3affd640 1 -- 172.21.15.64:0/608464912 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff2c037070 con 0x7eff3c0b0e90 2024-03-04T07:28:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.287+0000 7eff3affd640 1 -- 172.21.15.64:0/608464912 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff2c031b40 con 0x7eff3c0b0e90 2024-03-04T07:28:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 -- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 msgr2=0x7eff3c06b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 --2- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 0x7eff3c06b9a0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7eff2c0097a0 tx=0x7eff2c02b820 comp rx=0 tx=0).stop 2024-03-04T07:28:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 -- 172.21.15.64:0/608464912 shutdown_connections 2024-03-04T07:28:12.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 --2- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 --2- 172.21.15.64:0/608464912 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 0x7eff3c06b9a0 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 -- 172.21.15.64:0/608464912 >> 172.21.15.64:0/608464912 conn(0x7eff3c0b36d0 msgr2=0x7eff3c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:12.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 -- 172.21.15.64:0/608464912 shutdown_connections 2024-03-04T07:28:12.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.288+0000 7eff423f7640 1 -- 172.21.15.64:0/608464912 wait complete. 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.289+0000 7eff423f7640 1 Processor -- start 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.289+0000 7eff423f7640 1 -- start start 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff423f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff3c06b5c0 0x7eff3c1e7430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff423f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c1d1370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff423f7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 0x7eff3c0c7cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff423f7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff3c06c910 con 0x7eff3c06b5c0 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff423f7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff3c06c790 con 0x7eff3c0b0e90 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff423f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff3c0aee00 con 0x7eff3c0ae9b0 2024-03-04T07:28:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff3bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff3c06b5c0 0x7eff3c1e7430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff3bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff3c06b5c0 0x7eff3c1e7430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42456/0 (socket says 172.21.15.64:42456) 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff3bfff640 1 -- 172.21.15.64:0/4081950411 learned_addr learned my addr 172.21.15.64:0/4081950411 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff3b7fe640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c1d1370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff40994640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 0x7eff3c0c7cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff3bfff640 1 -- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 msgr2=0x7eff3c1d1370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.290+0000 7eff3bfff640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c1d1370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff3bfff640 1 -- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 msgr2=0x7eff3c0c7cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff3bfff640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 0x7eff3c0c7cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff3bfff640 1 -- 172.21.15.64:0/4081950411 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff3c1d3e60 con 0x7eff3c06b5c0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff3bfff640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff3c06b5c0 0x7eff3c1e7430 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7eff3000c430 tx=0x7eff3000c830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff38ff9640 1 -- 172.21.15.64:0/4081950411 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff30017070 con 0x7eff3c06b5c0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff38ff9640 1 -- 172.21.15.64:0/4081950411 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff30005ce0 con 0x7eff3c06b5c0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff3c1d4140 con 0x7eff3c06b5c0 2024-03-04T07:28:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff38ff9640 1 -- 172.21.15.64:0/4081950411 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff300072f0 con 0x7eff3c06b5c0 2024-03-04T07:28:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.291+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff3c1d49e0 con 0x7eff3c06b5c0 2024-03-04T07:28:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.292+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7eff3c068700 con 0x7eff3c06b5c0 2024-03-04T07:28:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.292+0000 7eff38ff9640 1 -- 172.21.15.64:0/4081950411 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7eff3001a070 con 0x7eff3c06b5c0 2024-03-04T07:28:12.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.293+0000 7eff38ff9640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff2806b4d0 0x7eff2806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.293+0000 7eff38ff9640 1 -- 172.21.15.64:0/4081950411 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7eff30089440 con 0x7eff3c06b5c0 2024-03-04T07:28:12.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.293+0000 7eff3b7fe640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff2806b4d0 0x7eff2806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.293+0000 7eff3b7fe640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff2806b4d0 0x7eff2806d990 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7eff2c036000 tx=0x7eff2c03d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.298+0000 7eff38ff9640 1 -- 172.21.15.64:0/4081950411 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7eff30057d00 con 0x7eff3c06b5c0 2024-03-04T07:28:12.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.519+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7eff3c070010 con 0x7eff3c06b5c0 2024-03-04T07:28:12.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.520+0000 7eff38ff9640 1 -- 172.21.15.64:0/4081950411 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7eff3005bda0 con 0x7eff3c06b5c0 2024-03-04T07:28:12.520 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:12.520 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:12.520 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff2806b4d0 msgr2=0x7eff2806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff2806b4d0 0x7eff2806d990 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7eff2c036000 tx=0x7eff2c03d040 comp rx=0 tx=0).stop 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff3c06b5c0 msgr2=0x7eff3c1e7430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff3c06b5c0 0x7eff3c1e7430 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7eff3000c430 tx=0x7eff3000c830 comp rx=0 tx=0).stop 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 shutdown_connections 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff2806b4d0 0x7eff2806d990 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff3c0b0e90 0x7eff3c0c7cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff3c0ae9b0 0x7eff3c1d1370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 --2- 172.21.15.64:0/4081950411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff3c06b5c0 0x7eff3c1e7430 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 >> 172.21.15.64:0/4081950411 conn(0x7eff3c0b36d0 msgr2=0x7eff3c0cdf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:12.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 shutdown_connections 2024-03-04T07:28:12.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.522+0000 7eff423f7640 1 -- 172.21.15.64:0/4081950411 wait complete. 2024-03-04T07:28:12.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump purge_queue pq_executed 2024-03-04T07:28:12.650 INFO:teuthology.orchestra.run.smithi064.stdout:{"purge_queue":{"pq_executed":1001}} 2024-03-04T07:28:12.650 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 1001 } } 2024-03-04T07:28:12.650 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_metadata stat 10000000000.00000000 2024-03-04T07:28:12.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 Processor -- start 2024-03-04T07:28:12.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 -- start start 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d641a10 0x55675d710ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 0x55675d71dee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d714650 0x55675d669190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55675d686c60 con 0x55675d714650 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55675d5e5b70 con 0x55675d641a10 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.693+0000 7f820a63bf40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55675d686de0 con 0x55675d713d30 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d714650 0x55675d669190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d714650 0x55675d669190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42462/0 (socket says 172.21.15.64:42462) 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 -- 172.21.15.64:0/55029789 learned_addr learned my addr 172.21.15.64:0/55029789 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:12.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 -- 172.21.15.64:0/55029789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 msgr2=0x55675d71dee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 0x55675d71dee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 -- 172.21.15.64:0/55029789 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d641a10 msgr2=0x55675d710ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f820789f640 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d641a10 0x55675d710ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d641a10 0x55675d710ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 -- 172.21.15.64:0/55029789 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55675d7113e0 con 0x55675d714650 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f82080a0640 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d714650 0x55675d669190 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f8200044720 tx=0x7f820004c000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=15ee8a2067a2e70b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f820609c640 1 -- 172.21.15.64:0/55029789 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8200063070 con 0x55675d714650 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f820609c640 1 -- 172.21.15.64:0/55029789 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8200052a40 con 0x55675d714650 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.694+0000 7f820609c640 1 -- 172.21.15.64:0/55029789 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8200052e10 con 0x55675d714650 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 -- 172.21.15.64:0/55029789 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d714650 msgr2=0x55675d669190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d714650 0x55675d669190 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f8200044720 tx=0x7f820004c000 comp rx=0 tx=0).stop 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 -- 172.21.15.64:0/55029789 shutdown_connections 2024-03-04T07:28:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d714650 0x55675d669190 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 0x55675d71dee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 --2- 172.21.15.64:0/55029789 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d641a10 0x55675d710ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 -- 172.21.15.64:0/55029789 >> 172.21.15.64:0/55029789 conn(0x55675d63e5e0 msgr2=0x55675d687350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 -- 172.21.15.64:0/55029789 shutdown_connections 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 -- 172.21.15.64:0/55029789 wait complete. 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 Processor -- start 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.695+0000 7f820a63bf40 1 -- start start 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820a63bf40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d641a10 0x55675d66a1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820a63bf40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 0x55675d7039a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820a63bf40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d714650 0x55675d703ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820a63bf40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55675d71e420 con 0x55675d641a10 2024-03-04T07:28:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820a63bf40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55675d7188b0 con 0x55675d714650 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820a63bf40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55675d668bd0 con 0x55675d713d30 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820789f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d641a10 0x55675d66a1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820789f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d641a10 0x55675d66a1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42478/0 (socket says 172.21.15.64:42478) 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820789f640 1 -- 172.21.15.64:0/1866582451 learned_addr learned my addr 172.21.15.64:0/1866582451 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820709e640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 0x55675d7039a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f82080a0640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d714650 0x55675d703ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.696+0000 7f820789f640 1 -- 172.21.15.64:0/1866582451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 msgr2=0x55675d7039a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820789f640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55675d713d30 0x55675d7039a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820789f640 1 -- 172.21.15.64:0/1866582451 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d714650 msgr2=0x55675d703ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820789f640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55675d714650 0x55675d703ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820789f640 1 -- 172.21.15.64:0/1866582451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55675d718ed0 con 0x55675d641a10 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820789f640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d641a10 0x55675d66a1b0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f81f800be30 tx=0x7f81f800bf20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820509a640 1 -- 172.21.15.64:0/1866582451 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81f8019070 con 0x55675d641a10 2024-03-04T07:28:12.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820509a640 1 -- 172.21.15.64:0/1866582451 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81f8007af0 con 0x55675d641a10 2024-03-04T07:28:12.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x55675d7191b0 con 0x55675d641a10 2024-03-04T07:28:12.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820509a640 1 -- 172.21.15.64:0/1866582451 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81f8014410 con 0x55675d641a10 2024-03-04T07:28:12.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.697+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55675d715ae0 con 0x55675d641a10 2024-03-04T07:28:12.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.698+0000 7f820509a640 1 -- 172.21.15.64:0/1866582451 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f81f801c070 con 0x55675d641a10 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.699+0000 7f820509a640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81f4069ee0 0x7f81f406c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.699+0000 7f820709e640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81f4069ee0 0x7f81f406c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.699+0000 7f820509a640 1 -- 172.21.15.64:0/1866582451 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f81f8087df0 con 0x55675d641a10 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.699+0000 7f820a63bf40 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55675d705050 0x55675d714f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.699+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:1 4.10 4:0cf6ad0f:::10000000000.00000000:head [stat] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e33) v9 -- 0x55675d66a6f0 con 0x55675d705050 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.699+0000 7f82080a0640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55675d705050 0x55675d714f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.699+0000 7f820709e640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81f4069ee0 0x7f81f406c3a0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x55675d718880 tx=0x7f81fc0073e0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f82080a0640 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55675d705050 0x55675d714f40 crc :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f82080a0640 1 -- 172.21.15.64:0/1866582451 <== osd.3 v2:172.21.15.64:6816/1402142346 1 ==== osd_op_reply(1 10000000000.00000000 [stat] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 ==== 164+0+0 (crc 0 0 0) 0x7f820005e4f0 con 0x55675d705050 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr: error stat-ing cephfs_metadata/10000000000.00000000: (2) No such file or directory 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55675d705050 msgr2=0x55675d714f40 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x55675d705050 0x55675d714f40 crc :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81f4069ee0 msgr2=0x7f81f406c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81f4069ee0 0x7f81f406c3a0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x55675d718880 tx=0x7f81fc0073e0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d641a10 msgr2=0x55675d66a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 --2- 172.21.15.64:0/1866582451 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55675d641a10 0x55675d66a1b0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f81f800be30 tx=0x7f81f800bf20 comp rx=0 tx=0).stop 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f82080a0640 1 -- 172.21.15.64:0/1866582451 reap_dead start 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 shutdown_connections 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 >> 172.21.15.64:0/1866582451 conn(0x55675d63e5e0 msgr2=0x55675d70fec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 shutdown_connections 2024-03-04T07:28:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.700+0000 7f820a63bf40 1 -- 172.21.15.64:0/1866582451 wait complete. 2024-03-04T07:28:12.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph df --format=json-pretty 2024-03-04T07:28:12.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.845+0000 7fdb61be2640 1 Processor -- start 2024-03-04T07:28:12.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.845+0000 7fdb61be2640 1 -- start start 2024-03-04T07:28:12.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb61be2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fdb5c15df80 0x7fdb5c06f530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb61be2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdb5c0cf130 con 0x7fdb5c0cda90 2024-03-04T07:28:12.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb61be2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdb5c0d0af0 con 0x7fdb5c0d2620 2024-03-04T07:28:12.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb61be2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdb5c0cc5f0 con 0x7fdb5c15df80 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fdb5c0cda90 0x7fdb5c0cea20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57186/0 (socket says 172.21.15.64:57186) 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5bfff640 1 -- 172.21.15.64:0/1339344990 learned_addr learned my addr 172.21.15.64:0/1339344990 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5b7fe640 1 --2- 172.21.15.64:0/1339344990 >> v2:172.21.15.66:3300/0 conn(0x7fdb5c15df80 0x7fdb5c06f530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5a7fc640 1 -- 172.21.15.64:0/1339344990 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1488230990 0 0) 0x7fdb5c0cf130 con 0x7fdb5c0cda90 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5a7fc640 1 -- 172.21.15.64:0/1339344990 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fdb44003520 con 0x7fdb5c0cda90 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5a7fc640 1 -- 172.21.15.64:0/1339344990 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3310136376 0 0) 0x7fdb5c0d0af0 con 0x7fdb5c0d2620 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5a7fc640 1 -- 172.21.15.64:0/1339344990 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fdb5c0cf130 con 0x7fdb5c0d2620 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5b7fe640 1 -- 172.21.15.64:0/1339344990 >> v1:172.21.15.66:6789/0 conn(0x7fdb5c0d2620 legacy=0x7fdb5c0ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5b7fe640 1 -- 172.21.15.64:0/1339344990 >> v1:172.21.15.64:6789/0 conn(0x7fdb5c0cda90 legacy=0x7fdb5c0cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.846+0000 7fdb5b7fe640 1 -- 172.21.15.64:0/1339344990 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb5c0d0dd0 con 0x7fdb5c15df80 2024-03-04T07:28:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.847+0000 7fdb5b7fe640 1 --2- 172.21.15.64:0/1339344990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06f530 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fdb4c0097a0 tx=0x7fdb4c02b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3336e35a6435bf7e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.847+0000 7fdb5a7fc640 1 -- 172.21.15.64:0/1339344990 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb4c034070 con 0x7fdb5c15df80 2024-03-04T07:28:12.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.847+0000 7fdb5a7fc640 1 -- 172.21.15.64:0/1339344990 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdb4c030610 con 0x7fdb5c15df80 2024-03-04T07:28:12.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.847+0000 7fdb5a7fc640 1 -- 172.21.15.64:0/1339344990 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb4c0309c0 con 0x7fdb5c15df80 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.848+0000 7fdb61be2640 1 -- 172.21.15.64:0/1339344990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c15df80 msgr2=0x7fdb5c06f530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.848+0000 7fdb61be2640 1 --2- 172.21.15.64:0/1339344990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06f530 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fdb4c0097a0 tx=0x7fdb4c02b960 comp rx=0 tx=0).stop 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.848+0000 7fdb61be2640 1 -- 172.21.15.64:0/1339344990 shutdown_connections 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.848+0000 7fdb61be2640 1 --2- 172.21.15.64:0/1339344990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06f530 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.848+0000 7fdb61be2640 1 -- 172.21.15.64:0/1339344990 >> 172.21.15.64:0/1339344990 conn(0x7fdb5c0d1a90 msgr2=0x7fdb5c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.848+0000 7fdb61be2640 1 -- 172.21.15.64:0/1339344990 shutdown_connections 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.848+0000 7fdb61be2640 1 -- 172.21.15.64:0/1339344990 wait complete. 2024-03-04T07:28:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.850+0000 7fdb61be2640 1 Processor -- start 2024-03-04T07:28:12.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.850+0000 7fdb61be2640 1 -- start start 2024-03-04T07:28:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb61be2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c0cda90 0x7fdb5c14cf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb61be2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb5c0d2620 0x7fdb5c0b2ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb61be2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb61be2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb5c0d0700 con 0x7fdb5c15df80 2024-03-04T07:28:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb61be2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb5c15e360 con 0x7fdb5c0cda90 2024-03-04T07:28:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb61be2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdb5c0d0880 con 0x7fdb5c0d2620 2024-03-04T07:28:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb5bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb5bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06c3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42488/0 (socket says 172.21.15.64:42488) 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb5bfff640 1 -- 172.21.15.64:0/2551302734 learned_addr learned my addr 172.21.15.64:0/2551302734 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb5b7fe640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c0cda90 0x7fdb5c14cf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.851+0000 7fdb5affd640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb5c0d2620 0x7fdb5c0b2ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb5bfff640 1 -- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb5c0d2620 msgr2=0x7fdb5c0b2ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb5bfff640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb5c0d2620 0x7fdb5c0b2ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb5bfff640 1 -- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c0cda90 msgr2=0x7fdb5c14cf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb5bfff640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c0cda90 0x7fdb5c14cf50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb5bfff640 1 -- 172.21.15.64:0/2551302734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb5c0b4370 con 0x7fdb5c15df80 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb5bfff640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06c3f0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fdb5400cbd0 tx=0x7fdb5400a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb43fff640 1 -- 172.21.15.64:0/2551302734 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb54017070 con 0x7fdb5c15df80 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb43fff640 1 -- 172.21.15.64:0/2551302734 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdb5400ace0 con 0x7fdb5c15df80 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdb5c1565a0 con 0x7fdb5c15df80 2024-03-04T07:28:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb43fff640 1 -- 172.21.15.64:0/2551302734 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb54012620 con 0x7fdb5c15df80 2024-03-04T07:28:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.852+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdb5c0b32d0 con 0x7fdb5c15df80 2024-03-04T07:28:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.853+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdb5c068700 con 0x7fdb5c15df80 2024-03-04T07:28:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.853+0000 7fdb43fff640 1 -- 172.21.15.64:0/2551302734 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdb5401a070 con 0x7fdb5c15df80 2024-03-04T07:28:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.854+0000 7fdb43fff640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb4806b4d0 0x7fdb4806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.854+0000 7fdb43fff640 1 -- 172.21.15.64:0/2551302734 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fdb5408a100 con 0x7fdb5c15df80 2024-03-04T07:28:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.854+0000 7fdb5b7fe640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb4806b4d0 0x7fdb4806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.854+0000 7fdb5b7fe640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb4806b4d0 0x7fdb4806d990 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fdb4c036000 tx=0x7fdb4c03b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:12.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:12.858+0000 7fdb43fff640 1 -- 172.21.15.64:0/2551302734 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdb540589c0 con 0x7fdb5c15df80 2024-03-04T07:28:13.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.131+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "df", "format": "json-pretty"} v 0) v1 -- 0x7fdb5c06af20 con 0x7fdb5c15df80 2024-03-04T07:28:13.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.132+0000 7fdb43fff640 1 -- 172.21.15.64:0/2551302734 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "df", "format": "json-pretty"}]=0 v0) v1 ==== 75+0+1695 (secure 0 0 0) 0x7fdb5405ca60 con 0x7fdb5c15df80 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "total_bytes": 773094113280, 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "total_avail_bytes": 772824891392, 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_bytes": 269221888, 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_bytes": 269221888, 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_ratio": 0.00034823946771211922, 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "num_osds": 8, 2024-03-04T07:28:13.132 INFO:teuthology.orchestra.run.smithi064.stdout: "num_per_pool_osds": 8, 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "num_per_pool_omap_osds": 8 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "stats_by_class": { 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "ssd": { 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "total_bytes": 773094113280, 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "total_avail_bytes": 772824891392, 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_bytes": 269221888, 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_bytes": 269221888, 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_ratio": 0.00034823946771211922 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:13.133 INFO:teuthology.orchestra.run.smithi064.stdout: "pools": [ 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "name": ".mgr", 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 1, 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 590368, 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 2, 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 1160, 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 1187840, 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 1.6180570128199179e-06, 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367056945152 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:13.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "cephfs_metadata", 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 4, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 3030381, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 24, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 5976, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 6119424, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 8.3357272160355933e-06, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367056945152 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "cephfs_data", 2024-03-04T07:28:13.135 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 5, 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 586752, 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 573, 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 4584, 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 4694016, 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 6.3940838117559906e-06, 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367056945152 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:28:13.136 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:28:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb4806b4d0 msgr2=0x7fdb4806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb4806b4d0 0x7fdb4806d990 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fdb4c036000 tx=0x7fdb4c03b040 comp rx=0 tx=0).stop 2024-03-04T07:28:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb5c15df80 msgr2=0x7fdb5c06c3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06c3f0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fdb5400cbd0 tx=0x7fdb5400a430 comp rx=0 tx=0).stop 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 shutdown_connections 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb4806b4d0 0x7fdb4806d990 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb5c15df80 0x7fdb5c06c3f0 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb5c0d2620 0x7fdb5c0b2ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 --2- 172.21.15.64:0/2551302734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb5c0cda90 0x7fdb5c14cf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 >> 172.21.15.64:0/2551302734 conn(0x7fdb5c0d1a90 msgr2=0x7fdb5c06ba00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 shutdown_connections 2024-03-04T07:28:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:13.134+0000 7fdb61be2640 1 -- 172.21.15.64:0/2551302734 wait complete. 2024-03-04T07:28:13.144 DEBUG:tasks.ceph_test_case:wait_until_true: waiting (timeout=60 retry_count=0)... 2024-03-04T07:28:17.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:28:17.402 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:28:18.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph df --format=json-pretty 2024-03-04T07:28:18.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.246+0000 7f20ba6ed640 1 Processor -- start 2024-03-04T07:28:18.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.246+0000 7f20ba6ed640 1 -- start start 2024-03-04T07:28:18.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20ba6ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b406fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20ba6ed640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f20b40b00c0 0x7f20b40b3ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20ba6ed640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f20b4161c20 con 0x7f20b415e080 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20ba6ed640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f20b41619d0 con 0x7f20b40b00c0 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20ba6ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20b4161f00 con 0x7f20b415dca0 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20b37fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f20b415e080 0x7f20b4066d00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59372/0 (socket says 172.21.15.64:59372) 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20b37fe640 1 -- 172.21.15.64:0/1710794400 learned_addr learned my addr 172.21.15.64:0/1710794400 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20b3fff640 1 --2- 172.21.15.64:0/1710794400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b406fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.247+0000 7f20b8c8a640 1 --2- 172.21.15.64:0/1710794400 >> v2:172.21.15.66:3300/0 conn(0x7f20b40b00c0 0x7f20b40b3ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.248+0000 7f20b2ffd640 1 -- 172.21.15.64:0/1710794400 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1367548442 0 0) 0x7f20b4161c20 con 0x7f20b415e080 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.248+0000 7f20b2ffd640 1 -- 172.21.15.64:0/1710794400 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f209c003520 con 0x7f20b415e080 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.248+0000 7f20b3fff640 1 -- 172.21.15.64:0/1710794400 >> v2:172.21.15.66:3300/0 conn(0x7f20b40b00c0 msgr2=0x7f20b40b3ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:18.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.248+0000 7f20b3fff640 1 --2- 172.21.15.64:0/1710794400 >> v2:172.21.15.66:3300/0 conn(0x7f20b40b00c0 0x7f20b40b3ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.248+0000 7f20b3fff640 1 -- 172.21.15.64:0/1710794400 >> v1:172.21.15.64:6789/0 conn(0x7f20b415e080 legacy=0x7f20b4066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.248+0000 7f20b3fff640 1 -- 172.21.15.64:0/1710794400 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20b41621e0 con 0x7f20b415dca0 2024-03-04T07:28:18.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.248+0000 7f20b8c8a640 1 --2- 172.21.15.64:0/1710794400 >> v2:172.21.15.66:3300/0 conn(0x7f20b40b00c0 0x7f20b40b3ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.249+0000 7f20b3fff640 1 --2- 172.21.15.64:0/1710794400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b406fd10 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f20a40097a0 tx=0x7f20a402b8b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=77626e8ae106d7e2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.249+0000 7f20b2ffd640 1 -- 172.21.15.64:0/1710794400 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20a4035020 con 0x7f20b415dca0 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.249+0000 7f20b2ffd640 1 -- 172.21.15.64:0/1710794400 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20a4031810 con 0x7f20b415dca0 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.249+0000 7f20b2ffd640 1 -- 172.21.15.64:0/1710794400 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20a4031be0 con 0x7f20b415dca0 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 -- 172.21.15.64:0/1710794400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 msgr2=0x7f20b406fd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/1710794400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b406fd10 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f20a40097a0 tx=0x7f20a402b8b0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 -- 172.21.15.64:0/1710794400 shutdown_connections 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/1710794400 >> v2:172.21.15.66:3300/0 conn(0x7f20b40b00c0 0x7f20b40b3ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/1710794400 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b406fd10 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 -- 172.21.15.64:0/1710794400 >> 172.21.15.64:0/1710794400 conn(0x7f20b40d0990 msgr2=0x7f20b40d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:18.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 -- 172.21.15.64:0/1710794400 shutdown_connections 2024-03-04T07:28:18.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.250+0000 7f20ba6ed640 1 -- 172.21.15.64:0/1710794400 wait complete. 2024-03-04T07:28:18.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.251+0000 7f20ba6ed640 1 Processor -- start 2024-03-04T07:28:18.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.251+0000 7f20ba6ed640 1 -- start start 2024-03-04T07:28:18.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20ba6ed640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20b40b00c0 0x7f20b40cfe80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20ba6ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b415fb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20ba6ed640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20b4160070 0x7f20b415e930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20ba6ed640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20b40703d0 con 0x7f20b4160070 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20ba6ed640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20b4070250 con 0x7f20b40b00c0 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20ba6ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20b4067410 con 0x7f20b415dca0 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20b4160070 0x7f20b415e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20b4160070 0x7f20b415e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57600/0 (socket says 172.21.15.64:57600) 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 -- 172.21.15.64:0/569953617 learned_addr learned my addr 172.21.15.64:0/569953617 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b37fe640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b415fb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b3fff640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20b40b00c0 0x7f20b40cfe80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 -- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 msgr2=0x7f20b415fb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b415fb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 -- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20b40b00c0 msgr2=0x7f20b40cfe80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20b40b00c0 0x7f20b40cfe80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b8c8a640 1 -- 172.21.15.64:0/569953617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20b41574b0 con 0x7f20b4160070 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b37fe640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b415fb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.252+0000 7f20b3fff640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20b40b00c0 0x7f20b40cfe80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.253+0000 7f20b8c8a640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20b4160070 0x7f20b415e930 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f20ac00ef90 tx=0x7f20ac009f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.253+0000 7f20b0ff9640 1 -- 172.21.15.64:0/569953617 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20ac018020 con 0x7f20b4160070 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.253+0000 7f20b0ff9640 1 -- 172.21.15.64:0/569953617 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20ac014480 con 0x7f20b4160070 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.253+0000 7f20b0ff9640 1 -- 172.21.15.64:0/569953617 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20ac014870 con 0x7f20b4160070 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.253+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20b4157770 con 0x7f20b4160070 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.253+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20b4159b90 con 0x7f20b4160070 2024-03-04T07:28:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.254+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f20b40b5890 con 0x7f20b4160070 2024-03-04T07:28:18.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.254+0000 7f20b0ff9640 1 -- 172.21.15.64:0/569953617 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f20ac005d10 con 0x7f20b4160070 2024-03-04T07:28:18.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.255+0000 7f20b0ff9640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20a006b500 0x7f20a006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.255+0000 7f20b0ff9640 1 -- 172.21.15.64:0/569953617 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f20ac08b080 con 0x7f20b4160070 2024-03-04T07:28:18.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.255+0000 7f20b3fff640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20a006b500 0x7f20a006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.255+0000 7f20b3fff640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20a006b500 0x7f20a006d9c0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f20a4036000 tx=0x7f20a4008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:18.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.263+0000 7f20b0ff9640 1 -- 172.21.15.64:0/569953617 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f20b40b5890 con 0x7f20b4160070 2024-03-04T07:28:18.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.515+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "df", "format": "json-pretty"} v 0) v1 -- 0x7f20b40adb90 con 0x7f20b4160070 2024-03-04T07:28:18.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.516+0000 7f20b0ff9640 1 -- 172.21.15.64:0/569953617 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "df", "format": "json-pretty"}]=0 v0) v1 ==== 75+0+1658 (secure 0 0 0) 0x7f20ac022030 con 0x7f20b4160070 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: "total_bytes": 773094113280, 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: "total_avail_bytes": 772822941696, 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_bytes": 271171584, 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_bytes": 271171584, 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_ratio": 0.00035076140193268657, 2024-03-04T07:28:18.516 INFO:teuthology.orchestra.run.smithi064.stdout: "num_osds": 8, 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "num_per_pool_osds": 8, 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "num_per_pool_omap_osds": 8 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "stats_by_class": { 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "ssd": { 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "total_bytes": 773094113280, 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "total_avail_bytes": 772822941696, 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_bytes": 271171584, 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_bytes": 271171584, 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_ratio": 0.00035076140193268657 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:18.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "pools": [ 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "name": ".mgr", 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 1, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 590368, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 2, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 1160, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 1187840, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 1.6179831163753988e-06, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367073689600 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:18.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "cephfs_metadata", 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 4, 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 3677172, 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 23, 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 7240, 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 7413760, 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 1.0098360689880792e-05, 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367073689600 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:18.519 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "cephfs_data", 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 5, 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 0, 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 0, 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 0, 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 0, 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 0, 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367073689600 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:18.520 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:28:18.521 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20a006b500 msgr2=0x7f20a006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20a006b500 0x7f20a006d9c0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f20a4036000 tx=0x7f20a4008030 comp rx=0 tx=0).stop 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20b4160070 msgr2=0x7f20b415e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20b4160070 0x7f20b415e930 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f20ac00ef90 tx=0x7f20ac009f90 comp rx=0 tx=0).stop 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 shutdown_connections 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20a006b500 0x7f20a006d9c0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20b4160070 0x7f20b415e930 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20b415dca0 0x7f20b415fb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 --2- 172.21.15.64:0/569953617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20b40b00c0 0x7f20b40cfe80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 >> 172.21.15.64:0/569953617 conn(0x7f20b40d0990 msgr2=0x7f20b40719b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:18.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 shutdown_connections 2024-03-04T07:28:18.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.518+0000 7f20ba6ed640 1 -- 172.21.15.64:0/569953617 wait complete. 2024-03-04T07:28:18.532 DEBUG:tasks.ceph_test_case:wait_until_true: success in 5s and 0 retries 2024-03-04T07:28:18.532 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:28:18.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:28:18.570 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:28:18.618 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:28:18.618 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:28:18.618 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:18.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:18.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.773+0000 7f5d5b06f640 1 Processor -- start 2024-03-04T07:28:18.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d5b06f640 1 -- start start 2024-03-04T07:28:18.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d5b06f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 0x7f5d540cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d5b06f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5d540d06c0 0x7f5d540b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d5b06f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5d540b0dd0 con 0x7f5d540b4260 2024-03-04T07:28:18.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d5b06f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5d540b0b80 con 0x7f5d540d06c0 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d5b06f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5d540aeec0 con 0x7f5d540d02e0 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d53fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5d540b4260 0x7f5d540b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59382/0 (socket says 172.21.15.64:59382) 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.774+0000 7f5d53fff640 1 -- 172.21.15.64:0/2292380264 learned_addr learned my addr 172.21.15.64:0/2292380264 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1997163904 0 0) 0x7f5d540b0dd0 con 0x7f5d540b4260 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5d44003520 con 0x7f5d540b4260 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d58e0b640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 0x7f5d540cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d5960c640 1 --2- 172.21.15.64:0/2292380264 >> v2:172.21.15.66:3300/0 conn(0x7f5d540d06c0 0x7f5d540b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1816234619 0 0) 0x7f5d44003520 con 0x7f5d540b4260 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 msgr2=0x7f5d540cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 0x7f5d540cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 >> v2:172.21.15.66:3300/0 conn(0x7f5d540d06c0 msgr2=0x7f5d540b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 --2- 172.21.15.64:0/2292380264 >> v2:172.21.15.66:3300/0 conn(0x7f5d540d06c0 0x7f5d540b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5d540b10b0 con 0x7f5d540b4260 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f5d40001670 con 0x7f5d540b4260 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 >> v1:172.21.15.64:6789/0 conn(0x7f5d540b4260 legacy=0x7f5d540b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d440039b0 0x7f5d44003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d44004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 0x7f5d440051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d540b0b80 con 0x7f5d44004800 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d540aeec0 con 0x7f5d440039b0 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5d4400ba70 con 0x7f5d540d02e0 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d5960c640 1 --2- 172.21.15.64:0/2292380264 >> v2:172.21.15.66:3300/0 conn(0x7f5d540d06c0 0x7f5d540b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d53fff640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d44004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d58e0b640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d440039b0 0x7f5d44003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d5960c640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 0x7f5d440051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d53fff640 1 -- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 msgr2=0x7f5d440051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d53fff640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d540d02e0 0x7f5d440051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d53fff640 1 -- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d440039b0 msgr2=0x7f5d44003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.775+0000 7f5d53fff640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d440039b0 0x7f5d44003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.776+0000 7f5d53fff640 1 -- 172.21.15.64:0/2292380264 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5d44003520 con 0x7f5d44004800 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.776+0000 7f5d58e0b640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d440039b0 0x7f5d44003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d53fff640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d44004c80 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f5d40001050 tx=0x7f5d44005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=34b196899b5e9998 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d40007d70 con 0x7f5d44004800 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5d40030070 con 0x7f5d44004800 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d537fe640 1 -- 172.21.15.64:0/2292380264 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d4002cd40 con 0x7f5d44004800 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d5b06f640 1 -- 172.21.15.64:0/2292380264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 msgr2=0x7f5d44004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d5b06f640 1 --2- 172.21.15.64:0/2292380264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d44004c80 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f5d40001050 tx=0x7f5d44005830 comp rx=0 tx=0).stop 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d5960c640 1 -- 172.21.15.64:0/2292380264 reap_dead start 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d5b06f640 1 -- 172.21.15.64:0/2292380264 shutdown_connections 2024-03-04T07:28:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d5b06f640 1 -- 172.21.15.64:0/2292380264 >> 172.21.15.64:0/2292380264 conn(0x7f5d540b36d0 msgr2=0x7f5d540b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:18.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d5b06f640 1 -- 172.21.15.64:0/2292380264 shutdown_connections 2024-03-04T07:28:18.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.777+0000 7f5d5b06f640 1 -- 172.21.15.64:0/2292380264 wait complete. 2024-03-04T07:28:18.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 Processor -- start 2024-03-04T07:28:18.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 -- start start 2024-03-04T07:28:18.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d541553a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d54153f00 0x7f5d541f19e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d541f1f20 0x7f5d541f23a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d540b22d0 con 0x7f5d44004800 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d540b2150 con 0x7f5d54153f00 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.779+0000 7f5d5b06f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5d540cfa00 con 0x7f5d541f1f20 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d541553a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d541553a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57624/0 (socket says 172.21.15.64:57624) 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 -- 172.21.15.64:0/895934711 learned_addr learned my addr 172.21.15.64:0/895934711 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d5960c640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d541f1f20 0x7f5d541f23a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d53fff640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d54153f00 0x7f5d541f19e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 -- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d541f1f20 msgr2=0x7f5d541f23a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d541f1f20 0x7f5d541f23a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 -- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d54153f00 msgr2=0x7f5d541f19e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d54153f00 0x7f5d541f19e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 -- 172.21.15.64:0/895934711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5d54155f50 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d58e0b640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d541553a0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f5d48011540 tx=0x7f5d48011a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d517fa640 1 -- 172.21.15.64:0/895934711 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d48015870 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d517fa640 1 -- 172.21.15.64:0/895934711 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5d48015a10 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5d540ca880 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d517fa640 1 -- 172.21.15.64:0/895934711 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d48015dc0 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.780+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5d540cae40 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.782+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5d54068700 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.782+0000 7f5d517fa640 1 -- 172.21.15.64:0/895934711 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5d4801d400 con 0x7f5d44004800 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.782+0000 7f5d517fa640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d3806b4d0 0x7f5d3806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.782+0000 7f5d53fff640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d3806b4d0 0x7f5d3806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:18.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.782+0000 7f5d517fa640 1 -- 172.21.15.64:0/895934711 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5d4808a780 con 0x7f5d44004800 2024-03-04T07:28:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.783+0000 7f5d53fff640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d3806b4d0 0x7f5d3806d990 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f5d40034000 tx=0x7f5d40007000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.787+0000 7f5d517fa640 1 -- 172.21.15.64:0/895934711 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5d48019030 con 0x7f5d44004800 2024-03-04T07:28:18.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.970+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f5d5415b120 con 0x7f5d44004800 2024-03-04T07:28:18.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.971+0000 7f5d517fa640 1 -- 172.21.15.64:0/895934711 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) v1 ==== 74+0+16621 (secure 0 0 0) 0x7f5d48059040 con 0x7f5d44004800 2024-03-04T07:28:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:18.971 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":33,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:49.009109+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:27:45.759140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-04T07:27:46.318635+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:18.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.972+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d3806b4d0 msgr2=0x7f5d3806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.972+0000 7f5d5b06f640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d3806b4d0 0x7f5d3806d990 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f5d40034000 tx=0x7f5d40007000 comp rx=0 tx=0).stop 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.972+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 msgr2=0x7f5d541553a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.972+0000 7f5d5b06f640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d541553a0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f5d48011540 tx=0x7f5d48011a00 comp rx=0 tx=0).stop 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 shutdown_connections 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5d3806b4d0 0x7f5d3806d990 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5d541f1f20 0x7f5d541f23a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5d54153f00 0x7f5d541f19e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 --2- 172.21.15.64:0/895934711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5d44004800 0x7f5d541553a0 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 >> 172.21.15.64:0/895934711 conn(0x7f5d540b36d0 msgr2=0x7f5d5406f5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 shutdown_connections 2024-03-04T07:28:18.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:18.973+0000 7f5d5b06f640 1 -- 172.21.15.64:0/895934711 wait complete. 2024-03-04T07:28:18.983 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:27:45.759140+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '32', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 5, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:27:46.318635+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '33', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:28:18.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:28:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.090+0000 7f50b6c8f640 1 Processor -- start 2024-03-04T07:28:19.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.090+0000 7f50b6c8f640 1 -- start start 2024-03-04T07:28:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50b6c8f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f50b00ceea0 0x7f50b00d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50b6c8f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f50b00b6d10 0x7f50b00b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50b6c8f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f50b00d3740 con 0x7f50b015dd90 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50b6c8f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f50b00d3400 con 0x7f50b00ceea0 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50b6c8f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f50b00d3120 con 0x7f50b00b6d10 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50affff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f50b00ceea0 0x7f50b00d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50affff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f50b00ceea0 0x7f50b00d29d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57640/0 (socket says 172.21.15.64:57640) 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50affff640 1 -- 172.21.15.64:0/1846019131 learned_addr learned my addr 172.21.15.64:0/1846019131 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50b522c640 1 --2- 172.21.15.64:0/1846019131 >> v2:172.21.15.66:3300/0 conn(0x7f50b00b6d10 0x7f50b00b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50affff640 1 -- 172.21.15.64:0/1846019131 >> v2:172.21.15.66:3300/0 conn(0x7f50b00b6d10 msgr2=0x7f50b00b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.091+0000 7f50affff640 1 --2- 172.21.15.64:0/1846019131 >> v2:172.21.15.66:3300/0 conn(0x7f50b00b6d10 0x7f50b00b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.092+0000 7f50affff640 1 -- 172.21.15.64:0/1846019131 >> v1:172.21.15.66:6789/0 conn(0x7f50b015dd90 legacy=0x7f50b00d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.092+0000 7f50affff640 1 -- 172.21.15.64:0/1846019131 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50b00d39e0 con 0x7f50b00ceea0 2024-03-04T07:28:19.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.092+0000 7f50b522c640 1 --2- 172.21.15.64:0/1846019131 >> v2:172.21.15.66:3300/0 conn(0x7f50b00b6d10 0x7f50b00b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.092+0000 7f50affff640 1 --2- 172.21.15.64:0/1846019131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00d29d0 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f50b00b5850 tx=0x7f509802ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=12cffed55f873f2a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.092+0000 7f50af7fe640 1 -- 172.21.15.64:0/1846019131 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f509802f850 con 0x7f50b00ceea0 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.092+0000 7f50af7fe640 1 -- 172.21.15.64:0/1846019131 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f509802f9f0 con 0x7f50b00ceea0 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.092+0000 7f50af7fe640 1 -- 172.21.15.64:0/1846019131 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f509802fdc0 con 0x7f50b00ceea0 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 -- 172.21.15.64:0/1846019131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 msgr2=0x7f50b00d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/1846019131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00d29d0 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f50b00b5850 tx=0x7f509802ba30 comp rx=0 tx=0).stop 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 -- 172.21.15.64:0/1846019131 shutdown_connections 2024-03-04T07:28:19.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/1846019131 >> v2:172.21.15.66:3300/0 conn(0x7f50b00b6d10 0x7f50b00b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/1846019131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00d29d0 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 -- 172.21.15.64:0/1846019131 >> 172.21.15.64:0/1846019131 conn(0x7f50b0071960 msgr2=0x7f50b0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 -- 172.21.15.64:0/1846019131 shutdown_connections 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.093+0000 7f50b6c8f640 1 -- 172.21.15.64:0/1846019131 wait complete. 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.094+0000 7f50b6c8f640 1 Processor -- start 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.094+0000 7f50b6c8f640 1 -- start start 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b6c8f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50b00b6d10 0x7f50b00c8c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b6c8f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00cbb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b6c8f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f50b015dd90 0x7f50b01f6d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b6c8f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50b00d48f0 con 0x7f50b00ceea0 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b6c8f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50b015e170 con 0x7f50b015dd90 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b6c8f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f50b00d4a70 con 0x7f50b00b6d10 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00cbb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00cbb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57644/0 (socket says 172.21.15.64:57644) 2024-03-04T07:28:19.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 -- 172.21.15.64:0/4226705225 learned_addr learned my addr 172.21.15.64:0/4226705225 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b4a2b640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50b00b6d10 0x7f50b00c8c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50b522c640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f50b015dd90 0x7f50b01f6d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 -- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50b00b6d10 msgr2=0x7f50b00c8c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50b00b6d10 0x7f50b00c8c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 -- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f50b015dd90 msgr2=0x7f50b01f6d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f50b015dd90 0x7f50b01f6d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.095+0000 7f50affff640 1 -- 172.21.15.64:0/4226705225 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50b00cc0c0 con 0x7f50b00ceea0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.096+0000 7f50affff640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00cbb80 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f509802ba70 tx=0x7f5098004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.096+0000 7f50ad7fa640 1 -- 172.21.15.64:0/4226705225 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5098037c90 con 0x7f50b00ceea0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.096+0000 7f50ad7fa640 1 -- 172.21.15.64:0/4226705225 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5098046070 con 0x7f50b00ceea0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.096+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f50b00ccae0 con 0x7f50b00ceea0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.096+0000 7f50ad7fa640 1 -- 172.21.15.64:0/4226705225 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5098041600 con 0x7f50b00ceea0 2024-03-04T07:28:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.096+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f50b00cd320 con 0x7f50b00ceea0 2024-03-04T07:28:19.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.097+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f50b0072a30 con 0x7f50b00ceea0 2024-03-04T07:28:19.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.097+0000 7f50ad7fa640 1 -- 172.21.15.64:0/4226705225 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5098040070 con 0x7f50b00ceea0 2024-03-04T07:28:19.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.098+0000 7f50ad7fa640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f509006b4d0 0x7f509006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.098+0000 7f50ad7fa640 1 -- 172.21.15.64:0/4226705225 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f50980ac8b0 con 0x7f50b00ceea0 2024-03-04T07:28:19.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.098+0000 7f50b4a2b640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f509006b4d0 0x7f509006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.098+0000 7f50b4a2b640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f509006b4d0 0x7f509006d990 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f50b0160f60 tx=0x7f50a0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.102+0000 7f50ad7fa640 1 -- 172.21.15.64:0/4226705225 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f50b00ccae0 con 0x7f50b00ceea0 2024-03-04T07:28:19.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.285+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f50b004f410 con 0x7f50b00ceea0 2024-03-04T07:28:19.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.286+0000 7f50ad7fa640 1 -- 172.21.15.64:0/4226705225 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v33) v1 ==== 93+0+10 (secure 0 0 0) 0x7f5098033030 con 0x7f50b00ceea0 2024-03-04T07:28:19.286 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:28:19.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f509006b4d0 msgr2=0x7f509006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f509006b4d0 0x7f509006d990 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f50b0160f60 tx=0x7f50a0009040 comp rx=0 tx=0).stop 2024-03-04T07:28:19.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 msgr2=0x7f50b00cbb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00cbb80 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f509802ba70 tx=0x7f5098004730 comp rx=0 tx=0).stop 2024-03-04T07:28:19.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 shutdown_connections 2024-03-04T07:28:19.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f509006b4d0 0x7f509006d990 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f50b015dd90 0x7f50b01f6d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50b00ceea0 0x7f50b00cbb80 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 --2- 172.21.15.64:0/4226705225 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50b00b6d10 0x7f50b00c8c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 >> 172.21.15.64:0/4226705225 conn(0x7f50b0071960 msgr2=0x7f50b00b71a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:19.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 shutdown_connections 2024-03-04T07:28:19.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.288+0000 7f50b6c8f640 1 -- 172.21.15.64:0/4226705225 wait complete. 2024-03-04T07:28:19.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:28:19.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.406+0000 7ff0c8881640 1 Processor -- start 2024-03-04T07:28:19.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.406+0000 7ff0c8881640 1 -- start start 2024-03-04T07:28:19.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.406+0000 7ff0c8881640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0c415f500 0x7ff0c40ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c8881640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0c40cd310 con 0x7ff0c40cf740 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c8881640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0c40cd5b0 con 0x7ff0c40c5e30 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c8881640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0c415f8e0 con 0x7ff0c415f500 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c259c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0c415f500 0x7ff0c40ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c1d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff0c40cf740 0x7ff0c40ce840 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59390/0 (socket says 172.21.15.64:59390) 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c1d9b640 1 -- 172.21.15.64:0/3552664666 learned_addr learned my addr 172.21.15.64:0/3552664666 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c259c640 1 -- 172.21.15.64:0/3552664666 >> v1:172.21.15.66:6789/0 conn(0x7ff0c40c5e30 legacy=0x7ff0c40c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c259c640 1 -- 172.21.15.64:0/3552664666 >> v1:172.21.15.64:6789/0 conn(0x7ff0c40cf740 legacy=0x7ff0c40ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c259c640 1 -- 172.21.15.64:0/3552664666 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0c40cd890 con 0x7ff0c415f500 2024-03-04T07:28:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.407+0000 7ff0c159a640 1 -- 172.21.15.64:0/3552664666 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3043273536 0 0) 0x7ff0c40cd310 con 0x7ff0c40cf740 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.408+0000 7ff0c259c640 1 --2- 172.21.15.64:0/3552664666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c415f500 0x7ff0c40ce300 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7ff0b40097a0 tx=0x7ff0b402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=17aa89699b9d0eec server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.408+0000 7ff0c159a640 1 -- 172.21.15.64:0/3552664666 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0b4035020 con 0x7ff0c415f500 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.408+0000 7ff0c159a640 1 -- 172.21.15.64:0/3552664666 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0b4031830 con 0x7ff0c415f500 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.408+0000 7ff0c159a640 1 -- 172.21.15.64:0/3552664666 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0b4031c00 con 0x7ff0c415f500 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.408+0000 7ff0c8881640 1 -- 172.21.15.64:0/3552664666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c415f500 msgr2=0x7ff0c40ce300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.408+0000 7ff0c8881640 1 --2- 172.21.15.64:0/3552664666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c415f500 0x7ff0c40ce300 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7ff0b40097a0 tx=0x7ff0b402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.409+0000 7ff0c8881640 1 -- 172.21.15.64:0/3552664666 shutdown_connections 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.409+0000 7ff0c8881640 1 --2- 172.21.15.64:0/3552664666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c415f500 0x7ff0c40ce300 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.409+0000 7ff0c8881640 1 -- 172.21.15.64:0/3552664666 >> 172.21.15.64:0/3552664666 conn(0x7ff0c40b3700 msgr2=0x7ff0c40b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.409+0000 7ff0c8881640 1 -- 172.21.15.64:0/3552664666 shutdown_connections 2024-03-04T07:28:19.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.409+0000 7ff0c8881640 1 -- 172.21.15.64:0/3552664666 wait complete. 2024-03-04T07:28:19.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 Processor -- start 2024-03-04T07:28:19.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 -- start start 2024-03-04T07:28:19.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c40c5e30 0x7ff0c4151cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0c40cf740 0x7ff0c40ca4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0c415f500 0x7ff0c40caa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0c40c59a0 con 0x7ff0c40c5e30 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0c40c56c0 con 0x7ff0c415f500 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c8881640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0c40cef50 con 0x7ff0c40cf740 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c40c5e30 0x7ff0c4151cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.411+0000 7ff0c259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c40c5e30 0x7ff0c4151cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57662/0 (socket says 172.21.15.64:57662) 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c259c640 1 -- 172.21.15.64:0/1200438067 learned_addr learned my addr 172.21.15.64:0/1200438067 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c1d9b640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0c40cf740 0x7ff0c40ca4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c2d9d640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0c415f500 0x7ff0c40caa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c259c640 1 -- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0c40cf740 msgr2=0x7ff0c40ca4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c259c640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0c40cf740 0x7ff0c40ca4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c259c640 1 -- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0c415f500 msgr2=0x7ff0c40caa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c259c640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0c415f500 0x7ff0c40caa10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c259c640 1 -- 172.21.15.64:0/1200438067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0c40b54e0 con 0x7ff0c40c5e30 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c1d9b640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0c40cf740 0x7ff0c40ca4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c259c640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c40c5e30 0x7ff0c4151cc0 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7ff0b40097a0 tx=0x7ff0b40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0b2ffd640 1 -- 172.21.15.64:0/1200438067 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0b4041070 con 0x7ff0c40c5e30 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0b2ffd640 1 -- 172.21.15.64:0/1200438067 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0b4004230 con 0x7ff0c40c5e30 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff0c40b57c0 con 0x7ff0c40c5e30 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0b2ffd640 1 -- 172.21.15.64:0/1200438067 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0b4035040 con 0x7ff0c40c5e30 2024-03-04T07:28:19.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.412+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff0c40b5d80 con 0x7ff0c40c5e30 2024-03-04T07:28:19.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.413+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff0c409fcf0 con 0x7ff0c40c5e30 2024-03-04T07:28:19.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.414+0000 7ff0b2ffd640 1 -- 172.21.15.64:0/1200438067 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff0b4038020 con 0x7ff0c40c5e30 2024-03-04T07:28:19.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.414+0000 7ff0b2ffd640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0a406b4d0 0x7ff0a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.414+0000 7ff0b2ffd640 1 -- 172.21.15.64:0/1200438067 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff0b40abc20 con 0x7ff0c40c5e30 2024-03-04T07:28:19.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.414+0000 7ff0c1d9b640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0a406b4d0 0x7ff0a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.415+0000 7ff0c1d9b640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0a406b4d0 0x7ff0a406d990 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7ff0c40c7240 tx=0x7ff0b8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.419+0000 7ff0b2ffd640 1 -- 172.21.15.64:0/1200438067 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff0b407ab30 con 0x7ff0c40c5e30 2024-03-04T07:28:19.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.597+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7ff0c404f410 con 0x7ff0c40c5e30 2024-03-04T07:28:19.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.598+0000 7ff0b2ffd640 1 -- 172.21.15.64:0/1200438067 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v33) v1 ==== 104+0+11 (secure 0 0 0) 0x7ff0b407a430 con 0x7ff0c40c5e30 2024-03-04T07:28:19.598 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0a406b4d0 msgr2=0x7ff0a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0a406b4d0 0x7ff0a406d990 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7ff0c40c7240 tx=0x7ff0b8009040 comp rx=0 tx=0).stop 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c40c5e30 msgr2=0x7ff0c4151cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c40c5e30 0x7ff0c4151cc0 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7ff0b40097a0 tx=0x7ff0b40046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 shutdown_connections 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0a406b4d0 0x7ff0a406d990 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0c415f500 0x7ff0c40caa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0c40cf740 0x7ff0c40ca4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 --2- 172.21.15.64:0/1200438067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0c40c5e30 0x7ff0c4151cc0 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 >> 172.21.15.64:0/1200438067 conn(0x7ff0c40b3700 msgr2=0x7ff0c4070160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 shutdown_connections 2024-03-04T07:28:19.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.600+0000 7ff0c8881640 1 -- 172.21.15.64:0/1200438067 wait complete. 2024-03-04T07:28:19.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:28:19.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.715+0000 7f35ec239640 1 Processor -- start 2024-03-04T07:28:19.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.715+0000 7f35ec239640 1 -- start start 2024-03-04T07:28:19.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35ec239640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f35e40cdcd0 0x7f35e40b1550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35ec239640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f35e406b2c0 0x7f35e406b710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35ec239640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f35e4161a40 con 0x7f35e40b2cd0 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35ec239640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35e40d11e0 con 0x7f35e40cdcd0 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35ec239640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35e40b2a20 con 0x7f35e406b2c0 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35e9fd5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f35e40cdcd0 0x7f35e40b1550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35e97d4640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f35e40b2cd0 0x7f35e40b1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59400/0 (socket says 172.21.15.64:59400) 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35e97d4640 1 -- 172.21.15.64:0/149722214 learned_addr learned my addr 172.21.15.64:0/149722214 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.716+0000 7f35ea7d6640 1 --2- 172.21.15.64:0/149722214 >> v2:172.21.15.66:3300/0 conn(0x7f35e406b2c0 0x7f35e406b710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.717+0000 7f35e8fd3640 1 -- 172.21.15.64:0/149722214 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1949105703 0 0) 0x7f35e4161a40 con 0x7f35e40b2cd0 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.717+0000 7f35e8fd3640 1 -- 172.21.15.64:0/149722214 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f35d0003520 con 0x7f35e40b2cd0 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.717+0000 7f35e9fd5640 1 -- 172.21.15.64:0/149722214 >> v2:172.21.15.66:3300/0 conn(0x7f35e406b2c0 msgr2=0x7f35e406b710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.717+0000 7f35e9fd5640 1 --2- 172.21.15.64:0/149722214 >> v2:172.21.15.66:3300/0 conn(0x7f35e406b2c0 0x7f35e406b710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.717+0000 7f35e9fd5640 1 -- 172.21.15.64:0/149722214 >> v1:172.21.15.64:6789/0 conn(0x7f35e40b2cd0 legacy=0x7f35e40b1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.717+0000 7f35e9fd5640 1 -- 172.21.15.64:0/149722214 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35e40d0940 con 0x7f35e40cdcd0 2024-03-04T07:28:19.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.717+0000 7f35e9fd5640 1 --2- 172.21.15.64:0/149722214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40cdcd0 0x7f35e40b1550 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f35e40b3570 tx=0x7f35d402b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=531476787c8b4426 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35e8fd3640 1 -- 172.21.15.64:0/149722214 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35d40315a0 con 0x7f35e40cdcd0 2024-03-04T07:28:19.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35e8fd3640 1 -- 172.21.15.64:0/149722214 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35d4037070 con 0x7f35e40cdcd0 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35e8fd3640 1 -- 172.21.15.64:0/149722214 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35d4031940 con 0x7f35e40cdcd0 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 -- 172.21.15.64:0/149722214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40cdcd0 msgr2=0x7f35e40b1550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 --2- 172.21.15.64:0/149722214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40cdcd0 0x7f35e40b1550 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f35e40b3570 tx=0x7f35d402b840 comp rx=0 tx=0).stop 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 -- 172.21.15.64:0/149722214 shutdown_connections 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 --2- 172.21.15.64:0/149722214 >> v2:172.21.15.66:3300/0 conn(0x7f35e406b2c0 0x7f35e406b710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 --2- 172.21.15.64:0/149722214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40cdcd0 0x7f35e40b1550 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 -- 172.21.15.64:0/149722214 >> 172.21.15.64:0/149722214 conn(0x7f35e40b3700 msgr2=0x7f35e40b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 -- 172.21.15.64:0/149722214 shutdown_connections 2024-03-04T07:28:19.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.718+0000 7f35ec239640 1 -- 172.21.15.64:0/149722214 wait complete. 2024-03-04T07:28:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.719+0000 7f35ec239640 1 Processor -- start 2024-03-04T07:28:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.719+0000 7f35ec239640 1 -- start start 2024-03-04T07:28:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35ec239640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35e406b2c0 0x7f35e40c6180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35ec239640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40b2cd0 0x7f35e4158630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35ec239640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35e40cdcd0 0x7f35e415b730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35ec239640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35e40cc1c0 con 0x7f35e40b2cd0 2024-03-04T07:28:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35ec239640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35e40d0f00 con 0x7f35e40cdcd0 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35ec239640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35e40cc340 con 0x7f35e406b2c0 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35e97d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40b2cd0 0x7f35e4158630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35e97d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40b2cd0 0x7f35e4158630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57686/0 (socket says 172.21.15.64:57686) 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35e97d4640 1 -- 172.21.15.64:0/2133779251 learned_addr learned my addr 172.21.15.64:0/2133779251 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35e9fd5640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35e406b2c0 0x7f35e40c6180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.720+0000 7f35ea7d6640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35e40cdcd0 0x7f35e415b730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35e97d4640 1 -- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35e406b2c0 msgr2=0x7f35e40c6180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35e97d4640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35e406b2c0 0x7f35e40c6180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35e97d4640 1 -- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35e40cdcd0 msgr2=0x7f35e415b730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35e97d4640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35e40cdcd0 0x7f35e415b730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35e97d4640 1 -- 172.21.15.64:0/2133779251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35e40c69d0 con 0x7f35e40b2cd0 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35e9fd5640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35e406b2c0 0x7f35e40c6180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35ea7d6640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35e40cdcd0 0x7f35e415b730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35e97d4640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40b2cd0 0x7f35e4158630 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f35e000c400 tx=0x7f35e000c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35da7fc640 1 -- 172.21.15.64:0/2133779251 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35e0017070 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35da7fc640 1 -- 172.21.15.64:0/2133779251 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35e0005d00 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35e40c6cb0 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35da7fc640 1 -- 172.21.15.64:0/2133779251 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35e0007310 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.721+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35e40c7270 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.722+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f35e40c7750 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.722+0000 7f35da7fc640 1 -- 172.21.15.64:0/2133779251 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f35e001a070 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.723+0000 7f35da7fc640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35c806b4d0 0x7f35c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.723+0000 7f35da7fc640 1 -- 172.21.15.64:0/2133779251 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f35e0089690 con 0x7f35e40b2cd0 2024-03-04T07:28:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.723+0000 7f35e9fd5640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35c806b4d0 0x7f35c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.723+0000 7f35e9fd5640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35c806b4d0 0x7f35c806d990 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f35d4036000 tx=0x7f35d403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:19.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.731+0000 7f35da7fc640 1 -- 172.21.15.64:0/2133779251 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f35e40c7750 con 0x7f35e40b2cd0 2024-03-04T07:28:19.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.910+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f35e409fcf0 con 0x7f35e40b2cd0 2024-03-04T07:28:19.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.910+0000 7f35da7fc640 1 -- 172.21.15.64:0/2133779251 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v33) v1 ==== 100+0+11 (secure 0 0 0) 0x7f35e0058510 con 0x7f35e40b2cd0 2024-03-04T07:28:19.910 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35c806b4d0 msgr2=0x7f35c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35c806b4d0 0x7f35c806d990 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f35d4036000 tx=0x7f35d403d040 comp rx=0 tx=0).stop 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40b2cd0 msgr2=0x7f35e4158630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40b2cd0 0x7f35e4158630 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f35e000c400 tx=0x7f35e000c8c0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 shutdown_connections 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35c806b4d0 0x7f35c806d990 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35e40cdcd0 0x7f35e415b730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35e40b2cd0 0x7f35e4158630 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 --2- 172.21.15.64:0/2133779251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35e406b2c0 0x7f35e40c6180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 >> 172.21.15.64:0/2133779251 conn(0x7f35e40b3700 msgr2=0x7f35e40ce3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 shutdown_connections 2024-03-04T07:28:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:19.913+0000 7f35ec239640 1 -- 172.21.15.64:0/2133779251 wait complete. 2024-03-04T07:28:19.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:28:20.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.028+0000 7f8c66179640 1 Processor -- start 2024-03-04T07:28:20.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.028+0000 7f8c66179640 1 -- start start 2024-03-04T07:28:20.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c66179640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c600d0730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c66179640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8c6015c190 0x7f8c600cd290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c66179640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8c600cf690 con 0x7f8c600cd7d0 2024-03-04T07:28:20.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c66179640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8c600d0c70 con 0x7f8c6015c190 2024-03-04T07:28:20.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c66179640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c600cf970 con 0x7f8c6015bdb0 2024-03-04T07:28:20.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5f7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c600d0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5effd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8c6015c190 0x7f8c600cd290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5effd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8c6015c190 0x7f8c600cd290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:57588/0 (socket says 172.21.15.64:57588) 2024-03-04T07:28:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5effd640 1 -- 172.21.15.64:0/3454914786 learned_addr learned my addr 172.21.15.64:0/3454914786 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5ffff640 1 --1- 172.21.15.64:0/3454914786 >> v1:172.21.15.66:6789/0 conn(0x7f8c600cd7d0 0x7f8c6015fa80 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:46854/0 (socket says 172.21.15.64:46854) 2024-03-04T07:28:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5effd640 1 -- 172.21.15.64:0/3454914786 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 msgr2=0x7f8c600d0730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5effd640 1 --2- 172.21.15.64:0/3454914786 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c600d0730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5effd640 1 -- 172.21.15.64:0/3454914786 >> v1:172.21.15.66:6789/0 conn(0x7f8c600cd7d0 legacy=0x7f8c6015fa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.030+0000 7f8c5effd640 1 -- 172.21.15.64:0/3454914786 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c600cfc50 con 0x7f8c6015c190 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.031+0000 7f8c5effd640 1 --2- 172.21.15.64:0/3454914786 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6015c190 0x7f8c600cd290 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f8c54009870 tx=0x7f8c5402b860 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5d71e1afc969b353 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.031+0000 7f8c5e7fc640 1 -- 172.21.15.64:0/3454914786 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c540317c0 con 0x7f8c6015c190 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.031+0000 7f8c5e7fc640 1 -- 172.21.15.64:0/3454914786 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c54037070 con 0x7f8c6015c190 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.031+0000 7f8c5e7fc640 1 -- 172.21.15.64:0/3454914786 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c54031b80 con 0x7f8c6015c190 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 -- 172.21.15.64:0/3454914786 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6015c190 msgr2=0x7f8c600cd290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 --2- 172.21.15.64:0/3454914786 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6015c190 0x7f8c600cd290 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f8c54009870 tx=0x7f8c5402b860 comp rx=0 tx=0).stop 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 -- 172.21.15.64:0/3454914786 shutdown_connections 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 --2- 172.21.15.64:0/3454914786 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6015c190 0x7f8c600cd290 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 --2- 172.21.15.64:0/3454914786 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c600d0730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 -- 172.21.15.64:0/3454914786 >> 172.21.15.64:0/3454914786 conn(0x7f8c600b35d0 msgr2=0x7f8c600b39e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 -- 172.21.15.64:0/3454914786 shutdown_connections 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.032+0000 7f8c66179640 1 -- 172.21.15.64:0/3454914786 wait complete. 2024-03-04T07:28:20.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.033+0000 7f8c66179640 1 Processor -- start 2024-03-04T07:28:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.033+0000 7f8c66179640 1 -- start start 2024-03-04T07:28:20.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c66179640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c600cd7d0 0x7f8c6015f280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c66179640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c6015a970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c66179640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c600cac60 0x7f8c600cb0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c66179640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c600b4320 con 0x7f8c600cac60 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c66179640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c600d0210 con 0x7f8c600cd7d0 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c66179640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c60160190 con 0x7f8c6015bdb0 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c600cac60 0x7f8c600cb0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c600cac60 0x7f8c600cb0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57696/0 (socket says 172.21.15.64:57696) 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 -- 172.21.15.64:0/2023421628 learned_addr learned my addr 172.21.15.64:0/2023421628 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5f7fe640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c600cd7d0 0x7f8c6015f280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5effd640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c6015a970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 -- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 msgr2=0x7f8c6015a970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c6015a970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 -- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c600cd7d0 msgr2=0x7f8c6015f280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c600cd7d0 0x7f8c6015f280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.034+0000 7f8c5ffff640 1 -- 172.21.15.64:0/2023421628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c6015aeb0 con 0x7f8c600cac60 2024-03-04T07:28:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.035+0000 7f8c5ffff640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c600cac60 0x7f8c600cb0f0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f8c5800c760 tx=0x7f8c5800cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.035+0000 7f8c47fff640 1 -- 172.21.15.64:0/2023421628 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c58017070 con 0x7f8c600cac60 2024-03-04T07:28:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.035+0000 7f8c47fff640 1 -- 172.21.15.64:0/2023421628 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c5800a640 con 0x7f8c600cac60 2024-03-04T07:28:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.035+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c6015d0c0 con 0x7f8c600cac60 2024-03-04T07:28:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.035+0000 7f8c47fff640 1 -- 172.21.15.64:0/2023421628 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c5800a9f0 con 0x7f8c600cac60 2024-03-04T07:28:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.035+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c6015d680 con 0x7f8c600cac60 2024-03-04T07:28:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.036+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c60068700 con 0x7f8c600cac60 2024-03-04T07:28:20.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.036+0000 7f8c47fff640 1 -- 172.21.15.64:0/2023421628 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8c5801a070 con 0x7f8c600cac60 2024-03-04T07:28:20.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.037+0000 7f8c47fff640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c4c06b4d0 0x7f8c4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.037+0000 7f8c47fff640 1 -- 172.21.15.64:0/2023421628 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8c580895d0 con 0x7f8c600cac60 2024-03-04T07:28:20.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.037+0000 7f8c5f7fe640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c4c06b4d0 0x7f8c4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.037+0000 7f8c5f7fe640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c4c06b4d0 0x7f8c4c06d990 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f8c50009b40 tx=0x7f8c50009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.041+0000 7f8c47fff640 1 -- 172.21.15.64:0/2023421628 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8c58057e90 con 0x7f8c600cac60 2024-03-04T07:28:20.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.232+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f8c600b7a40 con 0x7f8c600cac60 2024-03-04T07:28:20.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.233+0000 7f8c47fff640 1 -- 172.21.15.64:0/2023421628 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v33) v1 ==== 80+0+0 (secure 0 0 0) 0x7f8c5805bf30 con 0x7f8c600cac60 2024-03-04T07:28:20.233 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c4c06b4d0 msgr2=0x7f8c4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c4c06b4d0 0x7f8c4c06d990 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f8c50009b40 tx=0x7f8c50009190 comp rx=0 tx=0).stop 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c600cac60 msgr2=0x7f8c600cb0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c600cac60 0x7f8c600cb0f0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f8c5800c760 tx=0x7f8c5800cc20 comp rx=0 tx=0).stop 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 shutdown_connections 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c4c06b4d0 0x7f8c4c06d990 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c600cac60 0x7f8c600cb0f0 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6015bdb0 0x7f8c6015a970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 --2- 172.21.15.64:0/2023421628 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c600cd7d0 0x7f8c6015f280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 >> 172.21.15.64:0/2023421628 conn(0x7f8c600b35d0 msgr2=0x7f8c60070430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 shutdown_connections 2024-03-04T07:28:20.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.235+0000 7f8c66179640 1 -- 172.21.15.64:0/2023421628 wait complete. 2024-03-04T07:28:20.245 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:28:20.246 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:20.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:20.270 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[52830]: fuse finished with error 0 and tester_r 0 2024-03-04T07:28:20.272 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:28:20.272 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:20.272 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:28:20.377 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:28:20.377 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:20.395 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:28:20.395 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:28:20.395 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:20.395 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:28:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:20.451 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:28:20.452 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:28:20.452 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:20.467 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:28:20.467 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:28:20.467 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:20.467 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:28:20.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:20.521 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:28:20.521 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:28:20.521 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:20.538 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:28:20.538 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:28:20.538 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:20.538 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:28:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:20.595 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:28:20.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.696+0000 7f397e815640 1 Processor -- start 2024-03-04T07:28:20.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.696+0000 7f397e815640 1 -- start start 2024-03-04T07:28:20.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f397e815640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f39780b95a0 0x7f39780b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f397e815640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f39780b9a10 0x7f39780b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f397e815640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39780cea30 con 0x7f397815df00 2024-03-04T07:28:20.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f397e815640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f39780cf3c0 con 0x7f39780b95a0 2024-03-04T07:28:20.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f397e815640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f39780cf0e0 con 0x7f39780b9a10 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f39777fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f39780b95a0 0x7f39780b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f39777fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f39780b95a0 0x7f39780b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57698/0 (socket says 172.21.15.64:57698) 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f39777fe640 1 -- 172.21.15.64:0/2567271495 learned_addr learned my addr 172.21.15.64:0/2567271495 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f397cdb2640 1 --2- 172.21.15.64:0/2567271495 >> v2:172.21.15.66:3300/0 conn(0x7f39780b9a10 0x7f39780b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f39777fe640 1 -- 172.21.15.64:0/2567271495 >> v2:172.21.15.66:3300/0 conn(0x7f39780b9a10 msgr2=0x7f39780b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f39777fe640 1 --2- 172.21.15.64:0/2567271495 >> v2:172.21.15.66:3300/0 conn(0x7f39780b9a10 0x7f39780b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f39777fe640 1 -- 172.21.15.64:0/2567271495 >> v1:172.21.15.66:6789/0 conn(0x7f397815df00 legacy=0x7f39780d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.697+0000 7f39777fe640 1 -- 172.21.15.64:0/2567271495 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39780ce470 con 0x7f39780b95a0 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.698+0000 7f39777fe640 1 --2- 172.21.15.64:0/2567271495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39780b95a0 0x7f39780b5700 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f39780d4d20 tx=0x7f396c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=500348bdcf2d9998 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.698+0000 7f3976ffd640 1 -- 172.21.15.64:0/2567271495 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f396c034070 con 0x7f39780b95a0 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.698+0000 7f3976ffd640 1 -- 172.21.15.64:0/2567271495 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f396c030860 con 0x7f39780b95a0 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.698+0000 7f3976ffd640 1 -- 172.21.15.64:0/2567271495 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f396c030c30 con 0x7f39780b95a0 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 -- 172.21.15.64:0/2567271495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39780b95a0 msgr2=0x7f39780b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 --2- 172.21.15.64:0/2567271495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39780b95a0 0x7f39780b5700 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f39780d4d20 tx=0x7f396c02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 -- 172.21.15.64:0/2567271495 shutdown_connections 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 --2- 172.21.15.64:0/2567271495 >> v2:172.21.15.66:3300/0 conn(0x7f39780b9a10 0x7f39780b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 --2- 172.21.15.64:0/2567271495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39780b95a0 0x7f39780b5700 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 -- 172.21.15.64:0/2567271495 >> 172.21.15.64:0/2567271495 conn(0x7f3978071960 msgr2=0x7f3978071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 -- 172.21.15.64:0/2567271495 shutdown_connections 2024-03-04T07:28:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.699+0000 7f397e815640 1 -- 172.21.15.64:0/2567271495 wait complete. 2024-03-04T07:28:20.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.700+0000 7f397e815640 1 Processor -- start 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.700+0000 7f397e815640 1 -- start start 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397e815640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39780b95a0 0x7f3978155230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397e815640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39780b9a10 0x7f39780cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397e815640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f397815df00 0x7f39781618b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397e815640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39780cedf0 con 0x7f397815df00 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397e815640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39780cec70 con 0x7f39780b9a10 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397e815640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39780d3220 con 0x7f39780b95a0 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f397815df00 0x7f39781618b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f397815df00 0x7f39781618b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57704/0 (socket says 172.21.15.64:57704) 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 -- 172.21.15.64:0/3477674285 learned_addr learned my addr 172.21.15.64:0/3477674285 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f39777fe640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39780b9a10 0x7f39780cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f3977fff640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39780b95a0 0x7f3978155230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 -- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39780b95a0 msgr2=0x7f3978155230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39780b95a0 0x7f3978155230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 -- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39780b9a10 msgr2=0x7f39780cad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39780b9a10 0x7f39780cad50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.701+0000 7f397cdb2640 1 -- 172.21.15.64:0/3477674285 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3978155770 con 0x7f397815df00 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.702+0000 7f397cdb2640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f397815df00 0x7f39781618b0 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f397000efc0 tx=0x7f397000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.702+0000 7f3974ff9640 1 -- 172.21.15.64:0/3477674285 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3970018020 con 0x7f397815df00 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.702+0000 7f3974ff9640 1 -- 172.21.15.64:0/3477674285 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3970014480 con 0x7f397815df00 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.702+0000 7f3974ff9640 1 -- 172.21.15.64:0/3477674285 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3970014870 con 0x7f397815df00 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.702+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39780cb7f0 con 0x7f397815df00 2024-03-04T07:28:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.702+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39780cc0d0 con 0x7f397815df00 2024-03-04T07:28:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.703+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3978067ff0 con 0x7f397815df00 2024-03-04T07:28:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.703+0000 7f3974ff9640 1 -- 172.21.15.64:0/3477674285 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3970005d10 con 0x7f397815df00 2024-03-04T07:28:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.704+0000 7f3974ff9640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b500 0x7f396406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.704+0000 7f3977fff640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b500 0x7f396406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.704+0000 7f3974ff9640 1 -- 172.21.15.64:0/3477674285 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f397008aaa0 con 0x7f397815df00 2024-03-04T07:28:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.704+0000 7f3977fff640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b500 0x7f396406d9c0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f39780c7930 tx=0x7f3968009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:20.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.708+0000 7f3974ff9640 1 -- 172.21.15.64:0/3477674285 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3970022070 con 0x7f397815df00 2024-03-04T07:28:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.926+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f39780b7d40 con 0x7f397815df00 2024-03-04T07:28:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.927+0000 7f3974ff9640 1 -- 172.21.15.64:0/3477674285 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f3978155770 con 0x7f397815df00 2024-03-04T07:28:20.927 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:20.927 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:20.927 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b500 msgr2=0x7f396406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b500 0x7f396406d9c0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f39780c7930 tx=0x7f3968009040 comp rx=0 tx=0).stop 2024-03-04T07:28:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f397815df00 msgr2=0x7f39781618b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f397815df00 0x7f39781618b0 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f397000efc0 tx=0x7f397000c490 comp rx=0 tx=0).stop 2024-03-04T07:28:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 shutdown_connections 2024-03-04T07:28:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b500 0x7f396406d9c0 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f397815df00 0x7f39781618b0 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39780b9a10 0x7f39780cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 --2- 172.21.15.64:0/3477674285 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39780b95a0 0x7f3978155230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 >> 172.21.15.64:0/3477674285 conn(0x7f3978071960 msgr2=0x7f397815c070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 shutdown_connections 2024-03-04T07:28:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:20.929+0000 7f397e815640 1 -- 172.21.15.64:0/3477674285 wait complete. 2024-03-04T07:28:20.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.059+0000 7f2d26b79640 1 Processor -- start 2024-03-04T07:28:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.059+0000 7f2d26b79640 1 -- start start 2024-03-04T07:28:21.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.060+0000 7f2d26b79640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2d20158520 0x7f2d2006d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.060+0000 7f2d26b79640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d20071240 con 0x7f2d200aee10 2024-03-04T07:28:21.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.060+0000 7f2d26b79640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d20071520 con 0x7f2d2006dd60 2024-03-04T07:28:21.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.060+0000 7f2d26b79640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d20071b70 con 0x7f2d20158520 2024-03-04T07:28:21.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.060+0000 7f2d25116640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2d200aee10 0x7f2d200b3640 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59412/0 (socket says 172.21.15.64:59412) 2024-03-04T07:28:21.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.060+0000 7f2d25116640 1 -- 172.21.15.64:0/1450632326 learned_addr learned my addr 172.21.15.64:0/1450632326 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:21.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 686970799 0 0) 0x7f2d20071240 con 0x7f2d200aee10 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d24915640 1 --2- 172.21.15.64:0/1450632326 >> v2:172.21.15.66:3300/0 conn(0x7f2d20158520 0x7f2d2006d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d10003520 con 0x7f2d200aee10 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4055033547 0 0) 0x7f2d10003520 con 0x7f2d200aee10 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 >> v2:172.21.15.66:3300/0 conn(0x7f2d20158520 msgr2=0x7f2d2006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 --2- 172.21.15.64:0/1450632326 >> v2:172.21.15.66:3300/0 conn(0x7f2d20158520 0x7f2d2006d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 >> v1:172.21.15.66:6789/0 conn(0x7f2d2006dd60 legacy=0x7f2d200ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d20071800 con 0x7f2d200aee10 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2d180019f0 con 0x7f2d200aee10 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 >> v1:172.21.15.64:6789/0 conn(0x7f2d200aee10 legacy=0x7f2d200b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d10003df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d10004800 0x7f2d10004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d100051c0 0x7f2d1000dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d20071b70 con 0x7f2d100039b0 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d1000e2a0 con 0x7f2d100051c0 2024-03-04T07:28:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d1000e4a0 con 0x7f2d10004800 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d24915640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d10003df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d1ffff640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d10004800 0x7f2d10004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.061+0000 7f2d25116640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d100051c0 0x7f2d1000dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.062+0000 7f2d24915640 1 -- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d10004800 msgr2=0x7f2d10004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.062+0000 7f2d24915640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d10004800 0x7f2d10004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.062+0000 7f2d24915640 1 -- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d100051c0 msgr2=0x7f2d1000dbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.062+0000 7f2d24915640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d100051c0 0x7f2d1000dbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.062+0000 7f2d24915640 1 -- 172.21.15.64:0/1450632326 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d10003520 con 0x7f2d100039b0 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.062+0000 7f2d25116640 1 -- 172.21.15.64:0/1450632326 reap_dead start 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d24915640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d10003df0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f2d14009ac0 tx=0x7f2d10005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=475a70d1e7e1627e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d14039070 con 0x7f2d100039b0 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d1402c6b0 con 0x7f2d100039b0 2024-03-04T07:28:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d1f7fe640 1 -- 172.21.15.64:0/1450632326 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d14009690 con 0x7f2d100039b0 2024-03-04T07:28:21.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d26b79640 1 -- 172.21.15.64:0/1450632326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 msgr2=0x7f2d10003df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d26b79640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d10003df0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f2d14009ac0 tx=0x7f2d10005830 comp rx=0 tx=0).stop 2024-03-04T07:28:21.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d26b79640 1 -- 172.21.15.64:0/1450632326 shutdown_connections 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d26b79640 1 --2- 172.21.15.64:0/1450632326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d10003df0 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d26b79640 1 -- 172.21.15.64:0/1450632326 >> 172.21.15.64:0/1450632326 conn(0x7f2d20066d00 msgr2=0x7f2d200670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d26b79640 1 -- 172.21.15.64:0/1450632326 shutdown_connections 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.063+0000 7f2d26b79640 1 -- 172.21.15.64:0/1450632326 wait complete. 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.064+0000 7f2d26b79640 1 Processor -- start 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d26b79640 1 -- start start 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d26b79640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d2014d740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d26b79640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d2014ef50 0x7f2d200cd010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d26b79640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d2015cd10 0x7f2d200ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d26b79640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d20158a80 con 0x7f2d100039b0 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d26b79640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d20158900 con 0x7f2d2014ef50 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d26b79640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d200ae700 con 0x7f2d2015cd10 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.065+0000 7f2d24915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d2014d740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d2014d740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57718/0 (socket says 172.21.15.64:57718) 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 -- 172.21.15.64:0/3735935743 learned_addr learned my addr 172.21.15.64:0/3735935743 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d25116640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d2015cd10 0x7f2d200ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d1ffff640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d2014ef50 0x7f2d200cd010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 -- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d2015cd10 msgr2=0x7f2d200ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d2015cd10 0x7f2d200ad8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 -- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d2014ef50 msgr2=0x7f2d200cd010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d2014ef50 0x7f2d200cd010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 -- 172.21.15.64:0/3735935743 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d2015d190 con 0x7f2d100039b0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d25116640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d2015cd10 0x7f2d200ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d24915640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d2014d740 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f2d14009be0 tx=0x7f2d14004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d1d7fa640 1 -- 172.21.15.64:0/3735935743 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d14042020 con 0x7f2d100039b0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d1d7fa640 1 -- 172.21.15.64:0/3735935743 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d1403a040 con 0x7f2d100039b0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d200cd710 con 0x7f2d100039b0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d1d7fa640 1 -- 172.21.15.64:0/3735935743 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d14039070 con 0x7f2d100039b0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.066+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d200cdcd0 con 0x7f2d100039b0 2024-03-04T07:28:21.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.067+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d20067ff0 con 0x7f2d100039b0 2024-03-04T07:28:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.068+0000 7f2d1d7fa640 1 -- 172.21.15.64:0/3735935743 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d14045020 con 0x7f2d100039b0 2024-03-04T07:28:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.069+0000 7f2d1d7fa640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b500 0x7f2d0406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.069+0000 7f2d1d7fa640 1 -- 172.21.15.64:0/3735935743 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2d140ab500 con 0x7f2d100039b0 2024-03-04T07:28:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.069+0000 7f2d1ffff640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b500 0x7f2d0406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.069+0000 7f2d1ffff640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b500 0x7f2d0406d9c0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f2d200b4c90 tx=0x7f2d0c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.073+0000 7f2d1d7fa640 1 -- 172.21.15.64:0/3735935743 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d1404c070 con 0x7f2d100039b0 2024-03-04T07:28:21.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.258+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2d200b1060 con 0x7f2d100039b0 2024-03-04T07:28:21.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.259+0000 7f2d1d7fa640 1 -- 172.21.15.64:0/3735935743 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) v1 ==== 74+0+16621 (secure 0 0 0) 0x7f2d1407a380 con 0x7f2d100039b0 2024-03-04T07:28:21.259 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:21.259 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":33,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:49.009109+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:27:45.759140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-04T07:27:46.318635+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b500 msgr2=0x7f2d0406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b500 0x7f2d0406d9c0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f2d200b4c90 tx=0x7f2d0c009040 comp rx=0 tx=0).stop 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 msgr2=0x7f2d2014d740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d2014d740 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f2d14009be0 tx=0x7f2d14004730 comp rx=0 tx=0).stop 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 shutdown_connections 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b500 0x7f2d0406d9c0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d2015cd10 0x7f2d200ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d2014ef50 0x7f2d200cd010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 --2- 172.21.15.64:0/3735935743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d100039b0 0x7f2d2014d740 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 >> 172.21.15.64:0/3735935743 conn(0x7f2d20066d00 msgr2=0x7f2d2006c6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:21.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 shutdown_connections 2024-03-04T07:28:21.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.261+0000 7f2d26b79640 1 -- 172.21.15.64:0/3735935743 wait complete. 2024-03-04T07:28:21.272 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:27:45.759140+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '32', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 5, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:27:46.318635+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '33', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:28:21.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:28:21.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.378+0000 7fb6d8f93640 1 Processor -- start 2024-03-04T07:28:21.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.378+0000 7fb6d8f93640 1 -- start start 2024-03-04T07:28:21.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.379+0000 7fb6d8f93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 0x7fb6d4168e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.379+0000 7fb6d8f93640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6d415e6d0 con 0x7fb6d40b41f0 2024-03-04T07:28:21.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.379+0000 7fb6d8f93640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6d40ccd00 con 0x7fb6d415e2f0 2024-03-04T07:28:21.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.379+0000 7fb6d8f93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6d40d0a10 con 0x7fb6d40cead0 2024-03-04T07:28:21.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.379+0000 7fb6c9d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb6d40b41f0 0x7fb6d406bbf0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59426/0 (socket says 172.21.15.64:59426) 2024-03-04T07:28:21.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.379+0000 7fb6c9d9b640 1 -- 172.21.15.64:0/285184359 learned_addr learned my addr 172.21.15.64:0/285184359 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d2d9d640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 0x7fb6d4168e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2967242428 0 0) 0x7fb6d415e6d0 con 0x7fb6d40b41f0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb6c0003520 con 0x7fb6d40b41f0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2531846140 0 0) 0x7fb6d40ccd00 con 0x7fb6d415e2f0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb6d415e6d0 con 0x7fb6d415e2f0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 192611949 0 0) 0x7fb6c0003520 con 0x7fb6d40b41f0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 msgr2=0x7fb6d4168e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 0x7fb6d4168e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 >> v1:172.21.15.66:6789/0 conn(0x7fb6d415e2f0 legacy=0x7fb6d406b4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6d40ccfe0 con 0x7fb6d40b41f0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fb6bc001670 con 0x7fb6d40b41f0 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 >> v1:172.21.15.64:6789/0 conn(0x7fb6d40b41f0 legacy=0x7fb6d406bbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 0x7fb6c0003b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6c0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6c0004d40 0x7fb6c00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6d40d0a10 con 0x7fb6c0004120 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6c000bb80 con 0x7fb6c0004d40 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6c000bd80 con 0x7fb6d40cead0 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6c9d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6c0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d2d9d640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6c0004d40 0x7fb6c00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d259c640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 0x7fb6c0003b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6c9d9b640 1 -- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 msgr2=0x7fb6c0003b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6c9d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 0x7fb6c0003b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6c9d9b640 1 -- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6c0004d40 msgr2=0x7fb6c00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6c9d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6c0004d40 0x7fb6c00051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6c9d9b640 1 -- 172.21.15.64:0/285184359 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6c0003520 con 0x7fb6c0004120 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d259c640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d40cead0 0x7fb6c0003b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:21.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.380+0000 7fb6d2d9d640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6c0004d40 0x7fb6c00051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:21.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6c9d9b640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6c0004800 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7fb6bc00c0e0 tx=0x7fb6c0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9256a836a5583bd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6bc007c40 con 0x7fb6c0004120 2024-03-04T07:28:21.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6bc030070 con 0x7fb6c0004120 2024-03-04T07:28:21.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d1d9b640 1 -- 172.21.15.64:0/285184359 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6bc005d90 con 0x7fb6c0004120 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d8f93640 1 -- 172.21.15.64:0/285184359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 msgr2=0x7fb6c0004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d8f93640 1 --2- 172.21.15.64:0/285184359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6c0004800 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7fb6bc00c0e0 tx=0x7fb6c0005830 comp rx=0 tx=0).stop 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d2d9d640 1 -- 172.21.15.64:0/285184359 reap_dead start 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d8f93640 1 -- 172.21.15.64:0/285184359 shutdown_connections 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d8f93640 1 -- 172.21.15.64:0/285184359 >> 172.21.15.64:0/285184359 conn(0x7fb6d40b36d0 msgr2=0x7fb6d40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d8f93640 1 -- 172.21.15.64:0/285184359 shutdown_connections 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.382+0000 7fb6d8f93640 1 -- 172.21.15.64:0/285184359 wait complete. 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.384+0000 7fb6d8f93640 1 Processor -- start 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.384+0000 7fb6d8f93640 1 -- start start 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6d4154f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6d406f570 0x7fb6d406f9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d4160c90 0x7fb6d4161110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6d406c300 con 0x7fb6c0004120 2024-03-04T07:28:21.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6d40cd5a0 con 0x7fb6d406f570 2024-03-04T07:28:21.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6d4166e20 con 0x7fb6d4160c90 2024-03-04T07:28:21.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6d4154f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6d4154f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57726/0 (socket says 172.21.15.64:57726) 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 -- 172.21.15.64:0/3579277847 learned_addr learned my addr 172.21.15.64:0/3579277847 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d2d9d640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d4160c90 0x7fb6d4161110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6c9d9b640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6d406f570 0x7fb6d406f9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 -- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d4160c90 msgr2=0x7fb6d4161110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d4160c90 0x7fb6d4161110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 -- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6d406f570 msgr2=0x7fb6d406f9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6d406f570 0x7fb6d406f9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 -- 172.21.15.64:0/3579277847 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6d415f2a0 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d2d9d640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d4160c90 0x7fb6d4161110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d259c640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6d4154f40 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fb6c400e860 tx=0x7fb6c400ed20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6cb7fe640 1 -- 172.21.15.64:0/3579277847 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6c400ee50 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6cb7fe640 1 -- 172.21.15.64:0/3579277847 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb6c40052e0 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6d4161650 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6cb7fe640 1 -- 172.21.15.64:0/3579277847 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6c4007020 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.385+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6d40b4840 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.386+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb6d409fd30 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.387+0000 7fb6cb7fe640 1 -- 172.21.15.64:0/3579277847 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb6c401a020 con 0x7fb6c0004120 2024-03-04T07:28:21.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.388+0000 7fb6cb7fe640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6b406b500 0x7fb6b406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.388+0000 7fb6c9d9b640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6b406b500 0x7fb6b406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.388+0000 7fb6cb7fe640 1 -- 172.21.15.64:0/3579277847 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb6c4089b80 con 0x7fb6c0004120 2024-03-04T07:28:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.388+0000 7fb6c9d9b640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6b406b500 0x7fb6b406d9c0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fb6bc034000 tx=0x7fb6bc009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.392+0000 7fb6cb7fe640 1 -- 172.21.15.64:0/3579277847 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb6c4058a00 con 0x7fb6c0004120 2024-03-04T07:28:21.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.574+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fb6d4155540 con 0x7fb6c0004120 2024-03-04T07:28:21.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.574+0000 7fb6cb7fe640 1 -- 172.21.15.64:0/3579277847 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v33) v1 ==== 93+0+10 (secure 0 0 0) 0x7fb6c4058300 con 0x7fb6c0004120 2024-03-04T07:28:21.574 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:28:21.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6b406b500 msgr2=0x7fb6b406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6b406b500 0x7fb6b406d9c0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fb6bc034000 tx=0x7fb6bc009030 comp rx=0 tx=0).stop 2024-03-04T07:28:21.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 msgr2=0x7fb6d4154f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6d4154f40 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fb6c400e860 tx=0x7fb6c400ed20 comp rx=0 tx=0).stop 2024-03-04T07:28:21.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 shutdown_connections 2024-03-04T07:28:21.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb6b406b500 0x7fb6b406d9c0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6d4160c90 0x7fb6d4161110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6d406f570 0x7fb6d406f9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 --2- 172.21.15.64:0/3579277847 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6c0004120 0x7fb6d4154f40 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 >> 172.21.15.64:0/3579277847 conn(0x7fb6d40b36d0 msgr2=0x7fb6d40cef90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:21.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 shutdown_connections 2024-03-04T07:28:21.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.576+0000 7fb6d8f93640 1 -- 172.21.15.64:0/3579277847 wait complete. 2024-03-04T07:28:21.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:28:21.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.693+0000 7fb75ebc9640 1 Processor -- start 2024-03-04T07:28:21.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.693+0000 7fb75ebc9640 1 -- start start 2024-03-04T07:28:21.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75ebc9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb75815f500 0x7fb7580ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75ebc9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7580c5e30 0x7fb7580c6280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75ebc9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb7580cd310 con 0x7fb7580cf740 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75ebc9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb75815f8e0 con 0x7fb7580c5e30 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75ebc9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb7580c56c0 con 0x7fb75815f500 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75d166640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7580c5e30 0x7fb7580c6280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75d166640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb7580c5e30 0x7fb7580c6280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57736/0 (socket says 172.21.15.64:57736) 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb757fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb7580cf740 0x7fb7580ce840 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59430/0 (socket says 172.21.15.64:59430) 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.694+0000 7fb75d166640 1 -- 172.21.15.64:0/574167285 learned_addr learned my addr 172.21.15.64:0/574167285 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.695+0000 7fb75c965640 1 --2- 172.21.15.64:0/574167285 >> v2:172.21.15.66:3300/0 conn(0x7fb75815f500 0x7fb7580ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.695+0000 7fb75d166640 1 -- 172.21.15.64:0/574167285 >> v2:172.21.15.66:3300/0 conn(0x7fb75815f500 msgr2=0x7fb7580ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.695+0000 7fb75d166640 1 --2- 172.21.15.64:0/574167285 >> v2:172.21.15.66:3300/0 conn(0x7fb75815f500 0x7fb7580ce300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.695+0000 7fb75d166640 1 -- 172.21.15.64:0/574167285 >> v1:172.21.15.64:6789/0 conn(0x7fb7580cf740 legacy=0x7fb7580ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.695+0000 7fb75d166640 1 -- 172.21.15.64:0/574167285 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7580cd5b0 con 0x7fb7580c5e30 2024-03-04T07:28:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.695+0000 7fb7577fe640 1 -- 172.21.15.64:0/574167285 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1299012430 0 0) 0x7fb7580cd310 con 0x7fb7580cf740 2024-03-04T07:28:21.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.695+0000 7fb75d166640 1 --2- 172.21.15.64:0/574167285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580c5e30 0x7fb7580c6280 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7fb758071ca0 tx=0x7fb75002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=518c892c7cd2c69c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb7577fe640 1 -- 172.21.15.64:0/574167285 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb75002ca80 con 0x7fb7580c5e30 2024-03-04T07:28:21.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb7577fe640 1 -- 172.21.15.64:0/574167285 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb75002cc20 con 0x7fb7580c5e30 2024-03-04T07:28:21.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb7577fe640 1 -- 172.21.15.64:0/574167285 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7500343f0 con 0x7fb7580c5e30 2024-03-04T07:28:21.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 -- 172.21.15.64:0/574167285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580c5e30 msgr2=0x7fb7580c6280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/574167285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580c5e30 0x7fb7580c6280 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7fb758071ca0 tx=0x7fb75002bee0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 -- 172.21.15.64:0/574167285 shutdown_connections 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/574167285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580c5e30 0x7fb7580c6280 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/574167285 >> v2:172.21.15.66:3300/0 conn(0x7fb75815f500 0x7fb7580ce300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 -- 172.21.15.64:0/574167285 >> 172.21.15.64:0/574167285 conn(0x7fb7580b3700 msgr2=0x7fb7580b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 -- 172.21.15.64:0/574167285 shutdown_connections 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.696+0000 7fb75ebc9640 1 -- 172.21.15.64:0/574167285 wait complete. 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.697+0000 7fb75ebc9640 1 Processor -- start 2024-03-04T07:28:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb75ebc9640 1 -- start start 2024-03-04T07:28:21.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb75ebc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580c5e30 0x7fb758150ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb75ebc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580cf740 0x7fb75815a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb75ebc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75815f500 0x7fb75815a710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb75ebc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7580b43d0 con 0x7fb7580cf740 2024-03-04T07:28:21.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb75ebc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7580c59a0 con 0x7fb75815f500 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb75ebc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb7580cef50 con 0x7fb7580c5e30 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.698+0000 7fb757fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580cf740 0x7fb75815a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580cf740 0x7fb75815a1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57744/0 (socket says 172.21.15.64:57744) 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 -- 172.21.15.64:0/1809556781 learned_addr learned my addr 172.21.15.64:0/1809556781 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb75c965640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580c5e30 0x7fb758150ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb75d166640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75815f500 0x7fb75815a710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 -- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580c5e30 msgr2=0x7fb758150ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580c5e30 0x7fb758150ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 -- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75815f500 msgr2=0x7fb75815a710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75815f500 0x7fb75815a710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 -- 172.21.15.64:0/1809556781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb758151410 con 0x7fb7580cf740 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb757fff640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580cf740 0x7fb75815a1d0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fb74400c4b0 tx=0x7fb74400c970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb7557fa640 1 -- 172.21.15.64:0/1809556781 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb744018070 con 0x7fb7580cf740 2024-03-04T07:28:21.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb7557fa640 1 -- 172.21.15.64:0/1809556781 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb744007480 con 0x7fb7580cf740 2024-03-04T07:28:21.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb7580c9be0 con 0x7fb7580cf740 2024-03-04T07:28:21.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb7557fa640 1 -- 172.21.15.64:0/1809556781 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb744004820 con 0x7fb7580cf740 2024-03-04T07:28:21.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.699+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7580ca1a0 con 0x7fb7580cf740 2024-03-04T07:28:21.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.700+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb75809fcf0 con 0x7fb7580cf740 2024-03-04T07:28:21.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.701+0000 7fb7557fa640 1 -- 172.21.15.64:0/1809556781 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb74401b070 con 0x7fb7580cf740 2024-03-04T07:28:21.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.702+0000 7fb7557fa640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b4d0 0x7fb73c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:21.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.702+0000 7fb7557fa640 1 -- 172.21.15.64:0/1809556781 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb744089360 con 0x7fb7580cf740 2024-03-04T07:28:21.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.702+0000 7fb75c965640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b4d0 0x7fb73c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:21.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.702+0000 7fb75c965640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b4d0 0x7fb73c06d990 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fb74c00aa20 tx=0x7fb74c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:21.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.709+0000 7fb7557fa640 1 -- 172.21.15.64:0/1809556781 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb744057ce0 con 0x7fb7580cf740 2024-03-04T07:28:21.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.888+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fb7580d0cf0 con 0x7fb7580cf740 2024-03-04T07:28:21.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.888+0000 7fb7557fa640 1 -- 172.21.15.64:0/1809556781 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v33) v1 ==== 104+0+11 (secure 0 0 0) 0x7fb74405bd80 con 0x7fb7580cf740 2024-03-04T07:28:21.889 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:28:21.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b4d0 msgr2=0x7fb73c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b4d0 0x7fb73c06d990 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fb74c00aa20 tx=0x7fb74c008040 comp rx=0 tx=0).stop 2024-03-04T07:28:21.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580cf740 msgr2=0x7fb75815a1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:21.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580cf740 0x7fb75815a1d0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fb74400c4b0 tx=0x7fb74400c970 comp rx=0 tx=0).stop 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 shutdown_connections 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb73c06b4d0 0x7fb73c06d990 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb75815f500 0x7fb75815a710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.890+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb7580cf740 0x7fb75815a1d0 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.891+0000 7fb75ebc9640 1 --2- 172.21.15.64:0/1809556781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb7580c5e30 0x7fb758150ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.891+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 >> 172.21.15.64:0/1809556781 conn(0x7fb7580b3700 msgr2=0x7fb7580cdf30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.891+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 shutdown_connections 2024-03-04T07:28:21.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:21.891+0000 7fb75ebc9640 1 -- 172.21.15.64:0/1809556781 wait complete. 2024-03-04T07:28:21.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:28:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.007+0000 7f14197b6640 1 Processor -- start 2024-03-04T07:28:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14197b6640 1 -- start start 2024-03-04T07:28:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14197b6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14141614d0 0x7f14140cec80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14197b6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14141618b0 0x7f14140b6de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14197b6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f14140cab70 con 0x7f14140d1260 2024-03-04T07:28:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14197b6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14140d23c0 con 0x7f14141614d0 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14197b6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14140d26a0 con 0x7f14141618b0 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f1412ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14141614d0 0x7f14140cec80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14137fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f14140d1260 0x7f14140d3180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59432/0 (socket says 172.21.15.64:59432) 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f14137fe640 1 -- 172.21.15.64:0/81144773 learned_addr learned my addr 172.21.15.64:0/81144773 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.008+0000 7f1412ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14141614d0 0x7f14140cec80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57752/0 (socket says 172.21.15.64:57752) 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.009+0000 7f140bfff640 1 --2- 172.21.15.64:0/81144773 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14141618b0 0x7f14140b6de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.009+0000 7f1412ffd640 1 -- 172.21.15.64:0/81144773 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14141618b0 msgr2=0x7f14140b6de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.009+0000 7f1412ffd640 1 --2- 172.21.15.64:0/81144773 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14141618b0 0x7f14140b6de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.009+0000 7f1412ffd640 1 -- 172.21.15.64:0/81144773 >> v1:172.21.15.64:6789/0 conn(0x7f14140d1260 legacy=0x7f14140d3180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.009+0000 7f1412ffd640 1 -- 172.21.15.64:0/81144773 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14140d2980 con 0x7f14141614d0 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.009+0000 7f14127fc640 1 -- 172.21.15.64:0/81144773 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 179678288 0 0) 0x7f14140cab70 con 0x7f14140d1260 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.009+0000 7f140bfff640 1 --2- 172.21.15.64:0/81144773 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14141618b0 0x7f14140b6de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.010+0000 7f1412ffd640 1 --2- 172.21.15.64:0/81144773 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f14140cec80 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f14040097a0 tx=0x7f140402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=edceb145827c253 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.010+0000 7f14127fc640 1 -- 172.21.15.64:0/81144773 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1404035020 con 0x7f14141614d0 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.010+0000 7f14127fc640 1 -- 172.21.15.64:0/81144773 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1404031830 con 0x7f14141614d0 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.010+0000 7f14127fc640 1 -- 172.21.15.64:0/81144773 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1404031c00 con 0x7f14141614d0 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.010+0000 7f14197b6640 1 -- 172.21.15.64:0/81144773 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 msgr2=0x7f14140cec80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.010+0000 7f14197b6640 1 --2- 172.21.15.64:0/81144773 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f14140cec80 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f14040097a0 tx=0x7f140402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.011+0000 7f14197b6640 1 -- 172.21.15.64:0/81144773 shutdown_connections 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.011+0000 7f14197b6640 1 --2- 172.21.15.64:0/81144773 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14141618b0 0x7f14140b6de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.011+0000 7f14197b6640 1 --2- 172.21.15.64:0/81144773 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f14140cec80 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.011+0000 7f14197b6640 1 -- 172.21.15.64:0/81144773 >> 172.21.15.64:0/81144773 conn(0x7f1414067120 msgr2=0x7f1414067530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.011+0000 7f14197b6640 1 -- 172.21.15.64:0/81144773 shutdown_connections 2024-03-04T07:28:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.011+0000 7f14197b6640 1 -- 172.21.15.64:0/81144773 wait complete. 2024-03-04T07:28:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 Processor -- start 2024-03-04T07:28:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 -- start start 2024-03-04T07:28:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14140d1260 0x7f14141f78d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f141415d620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14141618b0 0x7f1414147e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14140d3890 con 0x7f14141614d0 2024-03-04T07:28:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14140cf380 con 0x7f14141618b0 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f14197b6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14140d3a10 con 0x7f14140d1260 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.012+0000 7f140bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f141415d620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f141415d620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57766/0 (socket says 172.21.15.64:57766) 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 -- 172.21.15.64:0/3321426921 learned_addr learned my addr 172.21.15.64:0/3321426921 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f1412ffd640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14140d1260 0x7f14141f78d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f14137fe640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14141618b0 0x7f1414147e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 -- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14140d1260 msgr2=0x7f14141f78d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14140d1260 0x7f14141f78d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 -- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14141618b0 msgr2=0x7f1414147e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14141618b0 0x7f1414147e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 -- 172.21.15.64:0/3321426921 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14141f8800 con 0x7f14141614d0 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140bfff640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f141415d620 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f13fc00d830 tx=0x7f13fc00dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140b7fe640 1 -- 172.21.15.64:0/3321426921 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13fc004030 con 0x7f14141614d0 2024-03-04T07:28:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140b7fe640 1 -- 172.21.15.64:0/3321426921 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f13fc0041d0 con 0x7f14141614d0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f14141f8240 con 0x7f14141614d0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f140b7fe640 1 -- 172.21.15.64:0/3321426921 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13fc0096a0 con 0x7f14141614d0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.013+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f141415a3d0 con 0x7f14141614d0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.014+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f141415d270 con 0x7f14141614d0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.015+0000 7f140b7fe640 1 -- 172.21.15.64:0/3321426921 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f13fc009840 con 0x7f14141614d0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.015+0000 7f140b7fe640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f13f406b4d0 0x7f13f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.015+0000 7f140b7fe640 1 -- 172.21.15.64:0/3321426921 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f13fc08a2f0 con 0x7f14141614d0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.015+0000 7f1412ffd640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f13f406b4d0 0x7f13f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.015+0000 7f1412ffd640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f13f406b4d0 0x7f13f406d990 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f1404036000 tx=0x7f140403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.019+0000 7f140b7fe640 1 -- 172.21.15.64:0/3321426921 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f13fc058bb0 con 0x7f14141614d0 2024-03-04T07:28:22.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.199+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f141404f410 con 0x7f14141614d0 2024-03-04T07:28:22.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.200+0000 7f140b7fe640 1 -- 172.21.15.64:0/3321426921 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v33) v1 ==== 100+0+11 (secure 0 0 0) 0x7f13fc05cc50 con 0x7f14141614d0 2024-03-04T07:28:22.200 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:28:22.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.201+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f13f406b4d0 msgr2=0x7f13f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.201+0000 7f14197b6640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f13f406b4d0 0x7f13f406d990 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f1404036000 tx=0x7f140403d040 comp rx=0 tx=0).stop 2024-03-04T07:28:22.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.201+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 msgr2=0x7f141415d620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.201+0000 7f14197b6640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f141415d620 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f13fc00d830 tx=0x7f13fc00dcf0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 shutdown_connections 2024-03-04T07:28:22.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f13f406b4d0 0x7f13f406d990 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14141618b0 0x7f1414147e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14141614d0 0x7f141415d620 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 --2- 172.21.15.64:0/3321426921 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14140d1260 0x7f14141f78d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 >> 172.21.15.64:0/3321426921 conn(0x7f1414067120 msgr2=0x7f14140cbd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:22.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 shutdown_connections 2024-03-04T07:28:22.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.202+0000 7f14197b6640 1 -- 172.21.15.64:0/3321426921 wait complete. 2024-03-04T07:28:22.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:22.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.314+0000 7f51e3a0e640 1 Processor -- start 2024-03-04T07:28:22.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.314+0000 7f51e3a0e640 1 -- start start 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e3a0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc15df00 0x7f51dc0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e3a0e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f51dc0b9a30 0x7f51dc0d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e3a0e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f51dc0cea30 con 0x7f51dc0b95a0 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e3a0e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f51dc0d2af0 con 0x7f51dc0b9a30 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e3a0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f51dc0cf3c0 con 0x7f51dc15df00 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e1fab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f51dc0b9a30 0x7f51dc0d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e1fab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f51dc0b9a30 0x7f51dc0d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57772/0 (socket says 172.21.15.64:57772) 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.315+0000 7f51e1fab640 1 -- 172.21.15.64:0/2063476448 learned_addr learned my addr 172.21.15.64:0/2063476448 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.316+0000 7f51e17aa640 1 --2- 172.21.15.64:0/2063476448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc15df00 0x7f51dc0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.316+0000 7f51e1fab640 1 -- 172.21.15.64:0/2063476448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc15df00 msgr2=0x7f51dc0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:22.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.316+0000 7f51e1fab640 1 --2- 172.21.15.64:0/2063476448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc15df00 0x7f51dc0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.316+0000 7f51e1fab640 1 -- 172.21.15.64:0/2063476448 >> v1:172.21.15.66:6789/0 conn(0x7f51dc0b95a0 legacy=0x7f51dc0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.316+0000 7f51e1fab640 1 -- 172.21.15.64:0/2063476448 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51dc0cf0e0 con 0x7f51dc0b9a30 2024-03-04T07:28:22.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.316+0000 7f51e1fab640 1 --2- 172.21.15.64:0/2063476448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0b9a30 0x7f51dc0d3220 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7f51d4009e50 tx=0x7f51d402bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cb8c1ea98a34fac6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51d3fff640 1 -- 172.21.15.64:0/2063476448 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51d402cb00 con 0x7f51dc0b9a30 2024-03-04T07:28:22.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51d3fff640 1 -- 172.21.15.64:0/2063476448 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f51d402cca0 con 0x7f51dc0b9a30 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51d3fff640 1 -- 172.21.15.64:0/2063476448 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51d4034610 con 0x7f51dc0b9a30 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 -- 172.21.15.64:0/2063476448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0b9a30 msgr2=0x7f51dc0d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/2063476448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0b9a30 0x7f51dc0d3220 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7f51d4009e50 tx=0x7f51d402bf60 comp rx=0 tx=0).stop 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 -- 172.21.15.64:0/2063476448 shutdown_connections 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/2063476448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0b9a30 0x7f51dc0d3220 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/2063476448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc15df00 0x7f51dc0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 -- 172.21.15.64:0/2063476448 >> 172.21.15.64:0/2063476448 conn(0x7f51dc071960 msgr2=0x7f51dc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 -- 172.21.15.64:0/2063476448 shutdown_connections 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.317+0000 7f51e3a0e640 1 -- 172.21.15.64:0/2063476448 wait complete. 2024-03-04T07:28:22.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.318+0000 7f51e3a0e640 1 Processor -- start 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.318+0000 7f51e3a0e640 1 -- start start 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e3a0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc0b95a0 0x7f51dc157b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e3a0e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f51dc15df00 0x7f51dc0cc510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e3a0e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0cca50 0x7f51dc15f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e3a0e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51dc0cec70 con 0x7f51dc0cca50 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e3a0e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51dc0ce750 con 0x7f51dc15df00 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e3a0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f51dc0cedf0 con 0x7f51dc0b95a0 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e1fab640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0cca50 0x7f51dc15f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e1fab640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0cca50 0x7f51dc15f1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57774/0 (socket says 172.21.15.64:57774) 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e1fab640 1 -- 172.21.15.64:0/45621549 learned_addr learned my addr 172.21.15.64:0/45621549 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e17aa640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc0b95a0 0x7f51dc157b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.319+0000 7f51e0fa9640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f51dc15df00 0x7f51dc0cc510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e1fab640 1 -- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc0b95a0 msgr2=0x7f51dc157b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e1fab640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc0b95a0 0x7f51dc157b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e1fab640 1 -- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f51dc15df00 msgr2=0x7f51dc0cc510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e1fab640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f51dc15df00 0x7f51dc0cc510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e1fab640 1 -- 172.21.15.64:0/45621549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51dc15e380 con 0x7f51dc0cca50 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e0fa9640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f51dc15df00 0x7f51dc0cc510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e1fab640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0cca50 0x7f51dc15f1d0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f51d4009e50 tx=0x7f51d4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51d1ffb640 1 -- 172.21.15.64:0/45621549 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51d4043020 con 0x7f51dc0cca50 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51d1ffb640 1 -- 172.21.15.64:0/45621549 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f51d402ce50 con 0x7f51dc0cca50 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f51dc15e620 con 0x7f51dc0cca50 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51d1ffb640 1 -- 172.21.15.64:0/45621549 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51d4009510 con 0x7f51dc0cca50 2024-03-04T07:28:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.320+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f51dc15ebe0 con 0x7f51dc0cca50 2024-03-04T07:28:22.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.321+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f51dc067ff0 con 0x7f51dc0cca50 2024-03-04T07:28:22.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.322+0000 7f51d1ffb640 1 -- 172.21.15.64:0/45621549 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f51d4046020 con 0x7f51dc0cca50 2024-03-04T07:28:22.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.322+0000 7f51d1ffb640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51c006b4d0 0x7f51c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.322+0000 7f51d1ffb640 1 -- 172.21.15.64:0/45621549 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f51d40ac870 con 0x7f51dc0cca50 2024-03-04T07:28:22.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.322+0000 7f51e17aa640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51c006b4d0 0x7f51c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.323+0000 7f51e17aa640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51c006b4d0 0x7f51c006d990 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f51d8006fd0 tx=0x7f51d8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.327+0000 7f51d1ffb640 1 -- 172.21.15.64:0/45621549 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f51d404d030 con 0x7f51dc0cca50 2024-03-04T07:28:22.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.544+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f51dc15ee20 con 0x7f51dc0cca50 2024-03-04T07:28:22.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.545+0000 7f51d1ffb640 1 -- 172.21.15.64:0/45621549 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f51d407b6f0 con 0x7f51dc0cca50 2024-03-04T07:28:22.546 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:22.546 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:22.546 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51c006b4d0 msgr2=0x7f51c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51c006b4d0 0x7f51c006d990 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f51d8006fd0 tx=0x7f51d8008040 comp rx=0 tx=0).stop 2024-03-04T07:28:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0cca50 msgr2=0x7f51dc15f1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0cca50 0x7f51dc15f1d0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f51d4009e50 tx=0x7f51d4004a40 comp rx=0 tx=0).stop 2024-03-04T07:28:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 shutdown_connections 2024-03-04T07:28:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51c006b4d0 0x7f51c006d990 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f51dc0cca50 0x7f51dc15f1d0 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f51dc15df00 0x7f51dc0cc510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 --2- 172.21.15.64:0/45621549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f51dc0b95a0 0x7f51dc157b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 >> 172.21.15.64:0/45621549 conn(0x7f51dc071960 msgr2=0x7f51dc15bd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 shutdown_connections 2024-03-04T07:28:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.547+0000 7f51e3a0e640 1 -- 172.21.15.64:0/45621549 wait complete. 2024-03-04T07:28:22.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:22.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.667+0000 7f20e36c9640 1 Processor -- start 2024-03-04T07:28:22.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.667+0000 7f20e36c9640 1 -- start start 2024-03-04T07:28:22.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e36c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f20dc15e060 0x7f20dc0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e36c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc0d0880 0x7f20dc0ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e36c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f20dc0d0dd0 con 0x7f20dc06fe20 2024-03-04T07:28:22.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e36c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f20dc0d10b0 con 0x7f20dc15e060 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e36c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20dc0cc0e0 con 0x7f20dc0d0880 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e1465640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f20dc15e060 0x7f20dc0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e1465640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f20dc15e060 0x7f20dc0ce310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57784/0 (socket says 172.21.15.64:57784) 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e1465640 1 -- 172.21.15.64:0/3253285861 learned_addr learned my addr 172.21.15.64:0/3253285861 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.668+0000 7f20e1c66640 1 --2- 172.21.15.64:0/3253285861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc0d0880 0x7f20dc0ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.669+0000 7f20e1465640 1 -- 172.21.15.64:0/3253285861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc0d0880 msgr2=0x7f20dc0ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.669+0000 7f20e1465640 1 --2- 172.21.15.64:0/3253285861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc0d0880 0x7f20dc0ccbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.669+0000 7f20e1465640 1 -- 172.21.15.64:0/3253285861 >> v1:172.21.15.66:6789/0 conn(0x7f20dc06fe20 legacy=0x7f20dc0ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.669+0000 7f20e1465640 1 -- 172.21.15.64:0/3253285861 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20dc0cc3c0 con 0x7f20dc15e060 2024-03-04T07:28:22.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.669+0000 7f20e1c66640 1 --2- 172.21.15.64:0/3253285861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc0d0880 0x7f20dc0ccbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:22.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.669+0000 7f20e1465640 1 --2- 172.21.15.64:0/3253285861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc15e060 0x7f20dc0ce310 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f20d0009870 tx=0x7f20d002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a3fb240c6a0d44dc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20cffff640 1 -- 172.21.15.64:0/3253285861 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20d0034070 con 0x7f20dc15e060 2024-03-04T07:28:22.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20cffff640 1 -- 172.21.15.64:0/3253285861 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20d0030840 con 0x7f20dc15e060 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20cffff640 1 -- 172.21.15.64:0/3253285861 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20d0030c10 con 0x7f20dc15e060 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 -- 172.21.15.64:0/3253285861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc15e060 msgr2=0x7f20dc0ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 --2- 172.21.15.64:0/3253285861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc15e060 0x7f20dc0ce310 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f20d0009870 tx=0x7f20d002b980 comp rx=0 tx=0).stop 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 -- 172.21.15.64:0/3253285861 shutdown_connections 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 --2- 172.21.15.64:0/3253285861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc0d0880 0x7f20dc0ccbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 --2- 172.21.15.64:0/3253285861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc15e060 0x7f20dc0ce310 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 -- 172.21.15.64:0/3253285861 >> 172.21.15.64:0/3253285861 conn(0x7f20dc0d1a90 msgr2=0x7f20dc0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 -- 172.21.15.64:0/3253285861 shutdown_connections 2024-03-04T07:28:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.670+0000 7f20e36c9640 1 -- 172.21.15.64:0/3253285861 wait complete. 2024-03-04T07:28:22.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.672+0000 7f20e36c9640 1 Processor -- start 2024-03-04T07:28:22.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.672+0000 7f20e36c9640 1 -- start start 2024-03-04T07:28:22.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.673+0000 7f20e36c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc06fe20 0x7f20dc14cc40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.673+0000 7f20e36c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc0d0880 0x7f20dc0c7f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.673+0000 7f20e36c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20dc15e060 0x7f20dc14d4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.673+0000 7f20e36c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20dc0cef60 con 0x7f20dc0d0880 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.673+0000 7f20e36c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20dc15e440 con 0x7f20dc15e060 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.673+0000 7f20e36c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20dc0cf0e0 con 0x7f20dc06fe20 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc0d0880 0x7f20dc0c7f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc0d0880 0x7f20dc0c7f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57800/0 (socket says 172.21.15.64:57800) 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 -- 172.21.15.64:0/4041118706 learned_addr learned my addr 172.21.15.64:0/4041118706 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e1465640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc06fe20 0x7f20dc14cc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e1c66640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20dc15e060 0x7f20dc14d4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 -- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc06fe20 msgr2=0x7f20dc14cc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc06fe20 0x7f20dc14cc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 -- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20dc15e060 msgr2=0x7f20dc14d4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20dc15e060 0x7f20dc14d4e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 -- 172.21.15.64:0/4041118706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20dc0b52e0 con 0x7f20dc0d0880 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e0c64640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc0d0880 0x7f20dc0c7f40 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f20d800c470 tx=0x7f20d800c930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20cdffb640 1 -- 172.21.15.64:0/4041118706 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20d80052d0 con 0x7f20dc0d0880 2024-03-04T07:28:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20cdffb640 1 -- 172.21.15.64:0/4041118706 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20d8005490 con 0x7f20dc0d0880 2024-03-04T07:28:22.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20dc0b55c0 con 0x7f20dc0d0880 2024-03-04T07:28:22.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20cdffb640 1 -- 172.21.15.64:0/4041118706 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20d8007260 con 0x7f20dc0d0880 2024-03-04T07:28:22.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.674+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20dc0b5b80 con 0x7f20dc0d0880 2024-03-04T07:28:22.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.676+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f20dc067ff0 con 0x7f20dc0d0880 2024-03-04T07:28:22.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.676+0000 7f20cdffb640 1 -- 172.21.15.64:0/4041118706 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f20d8007400 con 0x7f20dc0d0880 2024-03-04T07:28:22.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.676+0000 7f20cdffb640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20c006b4d0 0x7f20c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.677+0000 7f20e1465640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20c006b4d0 0x7f20c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.677+0000 7f20cdffb640 1 -- 172.21.15.64:0/4041118706 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f20d8089540 con 0x7f20dc0d0880 2024-03-04T07:28:22.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.677+0000 7f20e1465640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20c006b4d0 0x7f20c006d990 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f20d0036000 tx=0x7f20d0008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.681+0000 7f20cdffb640 1 -- 172.21.15.64:0/4041118706 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f20d8057e00 con 0x7f20dc0d0880 2024-03-04T07:28:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.862+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f20dc0b5d50 con 0x7f20dc0d0880 2024-03-04T07:28:22.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.863+0000 7f20cdffb640 1 -- 172.21.15.64:0/4041118706 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) v1 ==== 74+0+16621 (secure 0 0 0) 0x7f20d805bea0 con 0x7f20dc0d0880 2024-03-04T07:28:22.864 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:22.864 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":33,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:49.009109+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:27:45.759140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-04T07:27:46.318635+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20c006b4d0 msgr2=0x7f20c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20c006b4d0 0x7f20c006d990 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f20d0036000 tx=0x7f20d0008030 comp rx=0 tx=0).stop 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc0d0880 msgr2=0x7f20dc0c7f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc0d0880 0x7f20dc0c7f40 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f20d800c470 tx=0x7f20d800c930 comp rx=0 tx=0).stop 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 shutdown_connections 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20c006b4d0 0x7f20c006d990 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20dc15e060 0x7f20dc14d4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20dc0d0880 0x7f20dc0c7f40 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 --2- 172.21.15.64:0/4041118706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20dc06fe20 0x7f20dc14cc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 >> 172.21.15.64:0/4041118706 conn(0x7f20dc0d1a90 msgr2=0x7f20dc06eb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 shutdown_connections 2024-03-04T07:28:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.866+0000 7f20e36c9640 1 -- 172.21.15.64:0/4041118706 wait complete. 2024-03-04T07:28:22.881 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:28:22.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:22.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.982+0000 7f80f6581640 1 Processor -- start 2024-03-04T07:28:22.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.982+0000 7f80f6581640 1 -- start start 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.982+0000 7f80f6581640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f00ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.982+0000 7f80f6581640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f80f006fe20 0x7f80f00ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80f6581640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f80f00d0dd0 con 0x7f80f00d0880 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80f6581640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f80f00d10b0 con 0x7f80f006fe20 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80f6581640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80f00cc0e0 con 0x7f80f015e060 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80ef7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f80f006fe20 0x7f80f00ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80f4b1e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f80f00d0880 0x7f80f006ed30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59444/0 (socket says 172.21.15.64:59444) 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80f4b1e640 1 -- 172.21.15.64:0/130315250 learned_addr learned my addr 172.21.15.64:0/130315250 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80effff640 1 --2- 172.21.15.64:0/130315250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f00ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80ef7fe640 1 -- 172.21.15.64:0/130315250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 msgr2=0x7f80f00ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80ef7fe640 1 --2- 172.21.15.64:0/130315250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f00ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80ef7fe640 1 -- 172.21.15.64:0/130315250 >> v1:172.21.15.64:6789/0 conn(0x7f80f00d0880 legacy=0x7f80f006ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80eeffd640 1 -- 172.21.15.64:0/130315250 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3985379221 0 0) 0x7f80f00d0dd0 con 0x7f80f00d0880 2024-03-04T07:28:22.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.983+0000 7f80ef7fe640 1 -- 172.21.15.64:0/130315250 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80f00cc3c0 con 0x7f80f006fe20 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80ef7fe640 1 --2- 172.21.15.64:0/130315250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00ce850 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7f80e40097a0 tx=0x7f80e402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6ba26ff8bc423dfb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80eeffd640 1 -- 172.21.15.64:0/130315250 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80e4035020 con 0x7f80f006fe20 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80eeffd640 1 -- 172.21.15.64:0/130315250 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f80e4031830 con 0x7f80f006fe20 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80eeffd640 1 -- 172.21.15.64:0/130315250 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80e4031c00 con 0x7f80f006fe20 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80f6581640 1 -- 172.21.15.64:0/130315250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 msgr2=0x7f80f00ce850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80f6581640 1 --2- 172.21.15.64:0/130315250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00ce850 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7f80e40097a0 tx=0x7f80e402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80f6581640 1 -- 172.21.15.64:0/130315250 shutdown_connections 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80f6581640 1 --2- 172.21.15.64:0/130315250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00ce850 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80f6581640 1 --2- 172.21.15.64:0/130315250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f00ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.984+0000 7f80f6581640 1 -- 172.21.15.64:0/130315250 >> 172.21.15.64:0/130315250 conn(0x7f80f00d1a90 msgr2=0x7f80f00d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:22.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.985+0000 7f80f6581640 1 -- 172.21.15.64:0/130315250 shutdown_connections 2024-03-04T07:28:22.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.985+0000 7f80f6581640 1 -- 172.21.15.64:0/130315250 wait complete. 2024-03-04T07:28:22.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.986+0000 7f80f6581640 1 Processor -- start 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.986+0000 7f80f6581640 1 -- start start 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00b1af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80f00d0880 0x7f80f0160140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f014e110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80f00cdbd0 con 0x7f80f006fe20 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80f015e440 con 0x7f80f00d0880 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80f00cdd50 con 0x7f80f015e060 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00b1af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00b1af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57808/0 (socket says 172.21.15.64:57808) 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 -- 172.21.15.64:0/2044330696 learned_addr learned my addr 172.21.15.64:0/2044330696 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80ef7fe640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80f00d0880 0x7f80f0160140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f4b1e640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f014e110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 -- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 msgr2=0x7f80f014e110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f014e110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 -- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80f00d0880 msgr2=0x7f80f0160140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80f00d0880 0x7f80f0160140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 -- 172.21.15.64:0/2044330696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80f00b2140 con 0x7f80f006fe20 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80effff640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00b1af0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7f80e000e9b0 tx=0x7f80e000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80ecff9640 1 -- 172.21.15.64:0/2044330696 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80e0017070 con 0x7f80f006fe20 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80ecff9640 1 -- 172.21.15.64:0/2044330696 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f80e000c9a0 con 0x7f80f006fe20 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80f00b2420 con 0x7f80f006fe20 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80ecff9640 1 -- 172.21.15.64:0/2044330696 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80e000cd90 con 0x7f80f006fe20 2024-03-04T07:28:22.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.987+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80f00b2cc0 con 0x7f80f006fe20 2024-03-04T07:28:22.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.988+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f80f00b25f0 con 0x7f80f006fe20 2024-03-04T07:28:22.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.989+0000 7f80ecff9640 1 -- 172.21.15.64:0/2044330696 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f80e001a070 con 0x7f80f006fe20 2024-03-04T07:28:22.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.989+0000 7f80ecff9640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f80dc06b4d0 0x7f80dc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:22.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.989+0000 7f80ecff9640 1 -- 172.21.15.64:0/2044330696 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f80e008a5e0 con 0x7f80f006fe20 2024-03-04T07:28:22.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.989+0000 7f80ef7fe640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f80dc06b4d0 0x7f80dc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:22.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.990+0000 7f80ef7fe640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f80dc06b4d0 0x7f80dc06d990 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f80e4036000 tx=0x7f80e4008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:22.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:22.998+0000 7f80ecff9640 1 -- 172.21.15.64:0/2044330696 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f80e0059460 con 0x7f80f006fe20 2024-03-04T07:28:23.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.216+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f80f0067ff0 con 0x7f80f006fe20 2024-03-04T07:28:23.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.217+0000 7f80ecff9640 1 -- 172.21.15.64:0/2044330696 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f80e0058d60 con 0x7f80f006fe20 2024-03-04T07:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:23.218 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:23.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f80dc06b4d0 msgr2=0x7f80dc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f80dc06b4d0 0x7f80dc06d990 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f80e4036000 tx=0x7f80e4008030 comp rx=0 tx=0).stop 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 msgr2=0x7f80f00b1af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00b1af0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7f80e000e9b0 tx=0x7f80e000ee70 comp rx=0 tx=0).stop 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 shutdown_connections 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f80dc06b4d0 0x7f80dc06d990 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80f015e060 0x7f80f014e110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80f00d0880 0x7f80f0160140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 --2- 172.21.15.64:0/2044330696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80f006fe20 0x7f80f00b1af0 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 >> 172.21.15.64:0/2044330696 conn(0x7f80f00d1a90 msgr2=0x7f80f00d12f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 shutdown_connections 2024-03-04T07:28:23.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.219+0000 7f80f6581640 1 -- 172.21.15.64:0/2044330696 wait complete. 2024-03-04T07:28:23.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:23.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.332+0000 7f93d6d56640 1 Processor -- start 2024-03-04T07:28:23.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.332+0000 7f93d6d56640 1 -- start start 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.333+0000 7f93d6d56640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f93d015df00 0x7f93d00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.333+0000 7f93d6d56640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 0x7f93d00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.333+0000 7f93d6d56640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f93d00cea30 con 0x7f93d00b9a30 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.333+0000 7f93d6d56640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f93d00d2af0 con 0x7f93d015df00 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.333+0000 7f93d6d56640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f93d00cf3c0 con 0x7f93d00b95a0 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.333+0000 7f93d52f3640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f93d00b9a30 0x7f93d00d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59458/0 (socket says 172.21.15.64:59458) 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93d52f3640 1 -- 172.21.15.64:0/3015586835 learned_addr learned my addr 172.21.15.64:0/3015586835 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cffff640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 0x7f93d00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3015586835 >> v2:172.21.15.66:3300/0 conn(0x7f93d015df00 0x7f93d00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3153152171 0 0) 0x7f93d00cea30 con 0x7f93d00b9a30 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f93c0003520 con 0x7f93d00b9a30 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1203173375 0 0) 0x7f93c0003520 con 0x7f93d00b9a30 2024-03-04T07:28:23.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 msgr2=0x7f93d00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 0x7f93d00b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 >> v2:172.21.15.66:3300/0 conn(0x7f93d015df00 msgr2=0x7f93d00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 --2- 172.21.15.64:0/3015586835 >> v2:172.21.15.66:3300/0 conn(0x7f93d015df00 0x7f93d00d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f93d00cf0e0 con 0x7f93d00b9a30 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f93c80019f0 con 0x7f93d00b9a30 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 >> v1:172.21.15.64:6789/0 conn(0x7f93d00b9a30 legacy=0x7f93d00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93c0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93c0004800 0x7f93c0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 0x7f93c00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93d00d2af0 con 0x7f93c00039b0 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93d00cf3c0 con 0x7f93c0004800 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f93c000b9c0 con 0x7f93d00b95a0 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93c0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93cffff640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93c0004800 0x7f93c0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.334+0000 7f93d52f3640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 0x7f93c00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93d4af2640 1 -- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 msgr2=0x7f93c00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d00b95a0 0x7f93c00051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93d4af2640 1 -- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93c0004800 msgr2=0x7f93c0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93c0004800 0x7f93c0004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93d4af2640 1 -- 172.21.15.64:0/3015586835 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f93c0003520 con 0x7f93c00039b0 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93c0003dd0 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f93c4009ac0 tx=0x7f93c0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f56ba9cc8906887d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f93c4039070 con 0x7f93c00039b0 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.335+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f93c402c710 con 0x7f93c00039b0 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93cf7fe640 1 -- 172.21.15.64:0/3015586835 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f93c4009690 con 0x7f93c00039b0 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93d6d56640 1 -- 172.21.15.64:0/3015586835 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 msgr2=0x7f93c0003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93d6d56640 1 --2- 172.21.15.64:0/3015586835 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93c0003dd0 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f93c4009ac0 tx=0x7f93c0005830 comp rx=0 tx=0).stop 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93d52f3640 1 -- 172.21.15.64:0/3015586835 reap_dead start 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93d6d56640 1 -- 172.21.15.64:0/3015586835 shutdown_connections 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93d6d56640 1 -- 172.21.15.64:0/3015586835 >> 172.21.15.64:0/3015586835 conn(0x7f93d0071960 msgr2=0x7f93d0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93d6d56640 1 -- 172.21.15.64:0/3015586835 shutdown_connections 2024-03-04T07:28:23.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.336+0000 7f93d6d56640 1 -- 172.21.15.64:0/3015586835 wait complete. 2024-03-04T07:28:23.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.337+0000 7f93d6d56640 1 Processor -- start 2024-03-04T07:28:23.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.337+0000 7f93d6d56640 1 -- start start 2024-03-04T07:28:23.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d6d56640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93d00cd000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d6d56640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93d01551b0 0x7f93d00b2c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d6d56640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d0155630 0x7f93d00b3180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d6d56640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93d00cec70 con 0x7f93c00039b0 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d6d56640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93d00ce750 con 0x7f93d01551b0 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d6d56640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f93d00cedf0 con 0x7f93d0155630 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d4af2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93d00cd000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d4af2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93d00cd000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:57826/0 (socket says 172.21.15.64:57826) 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d4af2640 1 -- 172.21.15.64:0/3770158712 learned_addr learned my addr 172.21.15.64:0/3770158712 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93cffff640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93d01551b0 0x7f93d00b2c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.338+0000 7f93d52f3640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d0155630 0x7f93d00b3180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d4af2640 1 -- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d0155630 msgr2=0x7f93d00b3180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d0155630 0x7f93d00b3180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d4af2640 1 -- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93d01551b0 msgr2=0x7f93d00b2c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93d01551b0 0x7f93d00b2c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d4af2640 1 -- 172.21.15.64:0/3770158712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f93d0154150 con 0x7f93c00039b0 2024-03-04T07:28:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93cffff640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93d01551b0 0x7f93d00b2c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d4af2640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93d00cd000 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f93c4009be0 tx=0x7f93c4004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93cd7fa640 1 -- 172.21.15.64:0/3770158712 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f93c4042020 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93cd7fa640 1 -- 172.21.15.64:0/3770158712 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f93c403a040 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f93d01543f0 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93cd7fa640 1 -- 172.21.15.64:0/3770158712 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f93c4039070 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.339+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f93d01549b0 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.340+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f93d0067ff0 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.341+0000 7f93cd7fa640 1 -- 172.21.15.64:0/3770158712 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f93c4045020 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.341+0000 7f93cd7fa640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f93b406b4d0 0x7f93b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.341+0000 7f93cd7fa640 1 -- 172.21.15.64:0/3770158712 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f93c40ab9a0 con 0x7f93c00039b0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.341+0000 7f93cffff640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f93b406b4d0 0x7f93b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.341+0000 7f93cffff640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f93b406b4d0 0x7f93b406d990 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f93d0152860 tx=0x7f93bc008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:23.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.345+0000 7f93cd7fa640 1 -- 172.21.15.64:0/3770158712 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f93c404c030 con 0x7f93c00039b0 2024-03-04T07:28:23.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.604+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f93d015c810 con 0x7f93c00039b0 2024-03-04T07:28:23.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.605+0000 7f93cd7fa640 1 -- 172.21.15.64:0/3770158712 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 15 v15) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f93c407a820 con 0x7f93c00039b0 2024-03-04T07:28:23.606 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:23.606 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":15,"default_fscid":2,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":15,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:27:47.355925+0000","modified":"2024-03-04T07:27:50.389687+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5169},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5169":{"gid":5169,"name":"a","rank":0,"incarnation":12,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/2136250526","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2136250526},{"type":"v1","addr":"172.21.15.64:6836","nonce":2136250526}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[5],"metadata_pool":4,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5169,"qdb_cluster":[5169]},"id":2}]} 2024-03-04T07:28:23.606 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 15 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f93b406b4d0 msgr2=0x7f93b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f93b406b4d0 0x7f93b406d990 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f93d0152860 tx=0x7f93bc008040 comp rx=0 tx=0).stop 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 msgr2=0x7f93d00cd000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93d00cd000 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f93c4009be0 tx=0x7f93c4004730 comp rx=0 tx=0).stop 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 shutdown_connections 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f93b406b4d0 0x7f93b406d990 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f93d0155630 0x7f93d00b3180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93d01551b0 0x7f93d00b2c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 --2- 172.21.15.64:0/3770158712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93c00039b0 0x7f93d00cd000 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 >> 172.21.15.64:0/3770158712 conn(0x7f93d0071960 msgr2=0x7f93d015ba00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:23.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 shutdown_connections 2024-03-04T07:28:23.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.607+0000 7f93d6d56640 1 -- 172.21.15.64:0/3770158712 wait complete. 2024-03-04T07:28:23.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:23.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.719+0000 7f9e3bc6b640 1 Processor -- start 2024-03-04T07:28:23.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.719+0000 7f9e3bc6b640 1 -- start start 2024-03-04T07:28:23.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.720+0000 7f9e3bc6b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9e340b9a10 0x7f9e340d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.720+0000 7f9e3bc6b640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9e340cea30 con 0x7f9e3415df00 2024-03-04T07:28:23.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.720+0000 7f9e3bc6b640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9e340cf3c0 con 0x7f9e340b95a0 2024-03-04T07:28:23.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.720+0000 7f9e3bc6b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9e340cf0e0 con 0x7f9e340b9a10 2024-03-04T07:28:23.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.720+0000 7f9e39a07640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9e3415df00 0x7f9e340d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47130/0 (socket says 172.21.15.64:47130) 2024-03-04T07:28:23.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.720+0000 7f9e39a07640 1 -- 172.21.15.64:0/1319802122 learned_addr learned my addr 172.21.15.64:0/1319802122 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:23.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.721+0000 7f9e3a208640 1 --2- 172.21.15.64:0/1319802122 >> v2:172.21.15.66:3300/0 conn(0x7f9e340b9a10 0x7f9e340d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.721+0000 7f9e38a05640 1 -- 172.21.15.64:0/1319802122 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 991971047 0 0) 0x7f9e340cea30 con 0x7f9e3415df00 2024-03-04T07:28:23.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.721+0000 7f9e38a05640 1 -- 172.21.15.64:0/1319802122 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9e20003520 con 0x7f9e3415df00 2024-03-04T07:28:23.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.721+0000 7f9e3a208640 1 -- 172.21.15.64:0/1319802122 >> v1:172.21.15.66:6789/0 conn(0x7f9e340b95a0 legacy=0x7f9e340b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.721+0000 7f9e3a208640 1 -- 172.21.15.64:0/1319802122 >> v1:172.21.15.64:6789/0 conn(0x7f9e3415df00 legacy=0x7f9e340d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.721+0000 7f9e3a208640 1 -- 172.21.15.64:0/1319802122 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e340ce470 con 0x7f9e340b9a10 2024-03-04T07:28:23.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e3a208640 1 --2- 172.21.15.64:0/1319802122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b9a10 0x7f9e340d3220 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f9e340cf590 tx=0x7f9e2c02bf30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=89356959fe5bda4d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e38a05640 1 -- 172.21.15.64:0/1319802122 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e2c039070 con 0x7f9e340b9a10 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e38a05640 1 -- 172.21.15.64:0/1319802122 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e2c02cce0 con 0x7f9e340b9a10 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e38a05640 1 -- 172.21.15.64:0/1319802122 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e2c034640 con 0x7f9e340b9a10 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/1319802122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b9a10 msgr2=0x7f9e340d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/1319802122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b9a10 0x7f9e340d3220 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f9e340cf590 tx=0x7f9e2c02bf30 comp rx=0 tx=0).stop 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/1319802122 shutdown_connections 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/1319802122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b9a10 0x7f9e340d3220 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.722+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/1319802122 >> 172.21.15.64:0/1319802122 conn(0x7f9e34071960 msgr2=0x7f9e34071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.723+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/1319802122 shutdown_connections 2024-03-04T07:28:23.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.723+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/1319802122 wait complete. 2024-03-04T07:28:23.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 Processor -- start 2024-03-04T07:28:23.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 -- start start 2024-03-04T07:28:23.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b95a0 0x7f9e3415aeb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e340b9a10 0x7f9e340c9880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e3415df00 0x7f9e340c7890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e340cedf0 con 0x7f9e3415df00 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e340cec70 con 0x7f9e340b95a0 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3bc6b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9e340b5e10 con 0x7f9e340b9a10 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3a208640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e3415df00 0x7f9e340c7890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3a208640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e3415df00 0x7f9e340c7890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33488/0 (socket says 172.21.15.64:33488) 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.724+0000 7f9e3a208640 1 -- 172.21.15.64:0/678261263 learned_addr learned my addr 172.21.15.64:0/678261263 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e39a07640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b95a0 0x7f9e3415aeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e39206640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e340b9a10 0x7f9e340c9880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3a208640 1 -- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e340b9a10 msgr2=0x7f9e340c9880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3a208640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e340b9a10 0x7f9e340c9880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3a208640 1 -- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b95a0 msgr2=0x7f9e3415aeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3a208640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b95a0 0x7f9e3415aeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3a208640 1 -- 172.21.15.64:0/678261263 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e340ca2d0 con 0x7f9e3415df00 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3a208640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e3415df00 0x7f9e340c7890 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7f9e2c02c540 tx=0x7f9e2c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e267fc640 1 -- 172.21.15.64:0/678261263 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e2c042020 con 0x7f9e3415df00 2024-03-04T07:28:23.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e267fc640 1 -- 172.21.15.64:0/678261263 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9e2c03a040 con 0x7f9e3415df00 2024-03-04T07:28:23.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e340ca5b0 con 0x7f9e3415df00 2024-03-04T07:28:23.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e267fc640 1 -- 172.21.15.64:0/678261263 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e2c039070 con 0x7f9e3415df00 2024-03-04T07:28:23.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.725+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e340cb910 con 0x7f9e3415df00 2024-03-04T07:28:23.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.726+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9e34067ff0 con 0x7f9e3415df00 2024-03-04T07:28:23.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.727+0000 7f9e267fc640 1 -- 172.21.15.64:0/678261263 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9e2c045020 con 0x7f9e3415df00 2024-03-04T07:28:23.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.727+0000 7f9e267fc640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e1806b4d0 0x7f9e1806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:23.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.727+0000 7f9e39a07640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e1806b4d0 0x7f9e1806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:23.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.727+0000 7f9e267fc640 1 -- 172.21.15.64:0/678261263 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9e2c0ac460 con 0x7f9e3415df00 2024-03-04T07:28:23.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.728+0000 7f9e39a07640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e1806b4d0 0x7f9e1806d990 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f9e300011e0 tx=0x7f9e30009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:23.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.732+0000 7f9e267fc640 1 -- 172.21.15.64:0/678261263 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9e2c04c030 con 0x7f9e3415df00 2024-03-04T07:28:23.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.918+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f9e340b5830 con 0x7f9e3415df00 2024-03-04T07:28:23.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.919+0000 7f9e267fc640 1 -- 172.21.15.64:0/678261263 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) v1 ==== 74+0+16621 (secure 0 0 0) 0x7f9e2c07b430 con 0x7f9e3415df00 2024-03-04T07:28:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:23.920 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":33,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:27:49.009109+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:27:45.759140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"cephfs_data","create_time":"2024-03-04T07:27:46.318635+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:23.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e1806b4d0 msgr2=0x7f9e1806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e1806b4d0 0x7f9e1806d990 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f9e300011e0 tx=0x7f9e30009040 comp rx=0 tx=0).stop 2024-03-04T07:28:23.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e3415df00 msgr2=0x7f9e340c7890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e3415df00 0x7f9e340c7890 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7f9e2c02c540 tx=0x7f9e2c004a40 comp rx=0 tx=0).stop 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 shutdown_connections 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e1806b4d0 0x7f9e1806d990 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9e3415df00 0x7f9e340c7890 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9e340b9a10 0x7f9e340c9880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 --2- 172.21.15.64:0/678261263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9e340b95a0 0x7f9e3415aeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 >> 172.21.15.64:0/678261263 conn(0x7f9e34071960 msgr2=0x7f9e3415d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 shutdown_connections 2024-03-04T07:28:23.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:23.921+0000 7f9e3bc6b640 1 -- 172.21.15.64:0/678261263 wait complete. 2024-03-04T07:28:23.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:28:24.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.038+0000 7fbe4d815640 1 Processor -- start 2024-03-04T07:28:24.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.038+0000 7fbe4d815640 1 -- start start 2024-03-04T07:28:24.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.038+0000 7fbe4d815640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 0x7fbe480ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.038+0000 7fbe4d815640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe480d0dd0 con 0x7fbe480d0880 2024-03-04T07:28:24.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.038+0000 7fbe4d815640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe480d10b0 con 0x7fbe4806fe20 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.038+0000 7fbe4d815640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe480cc0e0 con 0x7fbe4815e060 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe477fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbe480d0880 0x7fbe480ccbc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47140/0 (socket says 172.21.15.64:47140) 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe477fe640 1 -- 172.21.15.64:0/710089270 learned_addr learned my addr 172.21.15.64:0/710089270 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe46ffd640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 0x7fbe480ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2286132197 0 0) 0x7fbe480d0dd0 con 0x7fbe480d0880 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbe30003520 con 0x7fbe480d0880 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1666345340 0 0) 0x7fbe30003520 con 0x7fbe480d0880 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 msgr2=0x7fbe480ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 0x7fbe480ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 >> v1:172.21.15.66:6789/0 conn(0x7fbe4806fe20 legacy=0x7fbe480ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe480cc3c0 con 0x7fbe480d0880 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fbe400019f0 con 0x7fbe480d0880 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 >> v1:172.21.15.64:6789/0 conn(0x7fbe480d0880 legacy=0x7fbe480ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe300039e0 0x7fbe30003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe30004390 0x7fbe30004860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 0x7fbe300050c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe480cc0e0 con 0x7fbe30004390 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe3000b9c0 con 0x7fbe300039e0 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe3000bbc0 con 0x7fbe4815e060 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe467fc640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe30004390 0x7fbe30004860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe46ffd640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe300039e0 0x7fbe30003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.039+0000 7fbe477fe640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 0x7fbe300050c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.040+0000 7fbe467fc640 1 -- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 msgr2=0x7fbe300050c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.040+0000 7fbe467fc640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe4815e060 0x7fbe300050c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.040+0000 7fbe467fc640 1 -- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe300039e0 msgr2=0x7fbe30003dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.040+0000 7fbe467fc640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe300039e0 0x7fbe30003dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.040+0000 7fbe467fc640 1 -- 172.21.15.64:0/710089270 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe30003520 con 0x7fbe30004390 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe467fc640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe30004390 0x7fbe30004860 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fbe3c001860 tx=0x7fbe30005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a53d6ee033a54a4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe3c038070 con 0x7fbe30004390 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe3c033410 con 0x7fbe30004390 2024-03-04T07:28:24.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe45ffb640 1 -- 172.21.15.64:0/710089270 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe3c001430 con 0x7fbe30004390 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe4d815640 1 -- 172.21.15.64:0/710089270 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe30004390 msgr2=0x7fbe30004860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe4d815640 1 --2- 172.21.15.64:0/710089270 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe30004390 0x7fbe30004860 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fbe3c001860 tx=0x7fbe30005830 comp rx=0 tx=0).stop 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe477fe640 1 -- 172.21.15.64:0/710089270 reap_dead start 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe4d815640 1 -- 172.21.15.64:0/710089270 shutdown_connections 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.041+0000 7fbe4d815640 1 -- 172.21.15.64:0/710089270 >> 172.21.15.64:0/710089270 conn(0x7fbe480d1a90 msgr2=0x7fbe480d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.042+0000 7fbe4d815640 1 -- 172.21.15.64:0/710089270 shutdown_connections 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.042+0000 7fbe4d815640 1 -- 172.21.15.64:0/710089270 wait complete. 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.043+0000 7fbe4d815640 1 Processor -- start 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.044+0000 7fbe4d815640 1 -- start start 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe4d815640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe30004390 0x7fbe481520a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe4d815640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe480cdff0 0x7fbe480ce470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe4d815640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe480c5f00 0x7fbe480c63b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe4d815640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe480cef60 con 0x7fbe480c5f00 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe4d815640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe4815e440 con 0x7fbe480cdff0 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe4d815640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe480cf0e0 con 0x7fbe30004390 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe480c5f00 0x7fbe480c63b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe480c5f00 0x7fbe480c63b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33506/0 (socket says 172.21.15.64:33506) 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 -- 172.21.15.64:0/668954276 learned_addr learned my addr 172.21.15.64:0/668954276 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe467fc640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe480cdff0 0x7fbe480ce470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe46ffd640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe30004390 0x7fbe481520a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 -- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe30004390 msgr2=0x7fbe481520a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe30004390 0x7fbe481520a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 -- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe480cdff0 msgr2=0x7fbe480ce470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe480cdff0 0x7fbe480ce470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 -- 172.21.15.64:0/668954276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe480c68f0 con 0x7fbe480c5f00 2024-03-04T07:28:24.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.045+0000 7fbe477fe640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe480c5f00 0x7fbe480c63b0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7fbe48067b00 tx=0x7fbe40007b10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.046+0000 7fbe2ffff640 1 -- 172.21.15.64:0/668954276 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe4000ab80 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.046+0000 7fbe2ffff640 1 -- 172.21.15.64:0/668954276 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe40018070 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.046+0000 7fbe2ffff640 1 -- 172.21.15.64:0/668954276 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe400133f0 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.046+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbe480c6bc0 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.046+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe480c7180 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.047+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbe48067ff0 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.047+0000 7fbe2ffff640 1 -- 172.21.15.64:0/668954276 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbe40013590 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.047+0000 7fbe2ffff640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe3406b500 0x7fbe3406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.047+0000 7fbe2ffff640 1 -- 172.21.15.64:0/668954276 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(33..33 src has 1..33) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbe4008a660 con 0x7fbe480c5f00 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.047+0000 7fbe46ffd640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe3406b500 0x7fbe3406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.048+0000 7fbe46ffd640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe3406b500 0x7fbe3406d9c0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fbe4815d1b0 tx=0x7fbe38009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:24.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.052+0000 7fbe2ffff640 1 -- 172.21.15.64:0/668954276 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbe40059570 con 0x7fbe480c5f00 2024-03-04T07:28:24.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.312+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7fbe4815ca90 con 0x7fbe480c5f00 2024-03-04T07:28:24.313 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:28:24.312+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:28:24.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.314+0000 7fbe2ffff640 1 -- 172.21.15.64:0/668954276 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v16) v1 ==== 161+0+0 (secure 0 0 0) 0x7fbe40058e70 con 0x7fbe480c5f00 2024-03-04T07:28:24.314 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe3406b500 msgr2=0x7fbe3406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe3406b500 0x7fbe3406d9c0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fbe4815d1b0 tx=0x7fbe38009190 comp rx=0 tx=0).stop 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe480c5f00 msgr2=0x7fbe480c63b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe480c5f00 0x7fbe480c63b0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7fbe48067b00 tx=0x7fbe40007b10 comp rx=0 tx=0).stop 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 shutdown_connections 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe3406b500 0x7fbe3406d9c0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe480c5f00 0x7fbe480c63b0 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe480cdff0 0x7fbe480ce470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 --2- 172.21.15.64:0/668954276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe30004390 0x7fbe481520a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 >> 172.21.15.64:0/668954276 conn(0x7fbe480d1a90 msgr2=0x7fbe4815d290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:24.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 shutdown_connections 2024-03-04T07:28:24.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.317+0000 7fbe4d815640 1 -- 172.21.15.64:0/668954276 wait complete. 2024-03-04T07:28:24.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:28:24.356 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:28:24.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.438+0000 7fab6421c640 1 Processor -- start 2024-03-04T07:28:24.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.438+0000 7fab6421c640 1 -- start start 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab6421c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c15b130 0x7fab5c15b590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab6421c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fab5c0ce900 0x7fab5c0ced90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab6421c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fab5c15bc40 con 0x7fab5c144640 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab6421c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fab5c06df70 con 0x7fab5c0ce900 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab6421c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab5c15bf20 con 0x7fab5c15b130 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab627b9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fab5c0ce900 0x7fab5c0ced90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab627b9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fab5c0ce900 0x7fab5c0ced90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33532/0 (socket says 172.21.15.64:33532) 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab627b9640 1 -- 172.21.15.64:0/2052269159 learned_addr learned my addr 172.21.15.64:0/2052269159 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab61fb8640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fab5c144640 0x7fab5c06e400 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47158/0 (socket says 172.21.15.64:47158) 2024-03-04T07:28:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab617b7640 1 --2- 172.21.15.64:0/2052269159 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c15b130 0x7fab5c15b590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab627b9640 1 -- 172.21.15.64:0/2052269159 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c15b130 msgr2=0x7fab5c15b590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab627b9640 1 --2- 172.21.15.64:0/2052269159 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c15b130 0x7fab5c15b590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab627b9640 1 -- 172.21.15.64:0/2052269159 >> v1:172.21.15.64:6789/0 conn(0x7fab5c144640 legacy=0x7fab5c06e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab60fb6640 1 -- 172.21.15.64:0/2052269159 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 200356730 0 0) 0x7fab5c15bc40 con 0x7fab5c144640 2024-03-04T07:28:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab627b9640 1 -- 172.21.15.64:0/2052269159 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab5c06dc00 con 0x7fab5c0ce900 2024-03-04T07:28:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.439+0000 7fab617b7640 1 --2- 172.21.15.64:0/2052269159 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c15b130 0x7fab5c15b590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:24.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab627b9640 1 --2- 172.21.15.64:0/2052269159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c0ce900 0x7fab5c0ced90 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fab54009e50 tx=0x7fab5402bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c17120284e2b1a79 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:24.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab60fb6640 1 -- 172.21.15.64:0/2052269159 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab5402cb00 con 0x7fab5c0ce900 2024-03-04T07:28:24.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab60fb6640 1 -- 172.21.15.64:0/2052269159 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab5402cca0 con 0x7fab5c0ce900 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab60fb6640 1 -- 172.21.15.64:0/2052269159 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab54034610 con 0x7fab5c0ce900 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab6421c640 1 -- 172.21.15.64:0/2052269159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c0ce900 msgr2=0x7fab5c0ced90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab6421c640 1 --2- 172.21.15.64:0/2052269159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c0ce900 0x7fab5c0ced90 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fab54009e50 tx=0x7fab5402bf60 comp rx=0 tx=0).stop 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab6421c640 1 -- 172.21.15.64:0/2052269159 shutdown_connections 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab6421c640 1 --2- 172.21.15.64:0/2052269159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c0ce900 0x7fab5c0ced90 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab6421c640 1 --2- 172.21.15.64:0/2052269159 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c15b130 0x7fab5c15b590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.440+0000 7fab6421c640 1 -- 172.21.15.64:0/2052269159 >> 172.21.15.64:0/2052269159 conn(0x7fab5c066d00 msgr2=0x7fab5c067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.441+0000 7fab6421c640 1 -- 172.21.15.64:0/2052269159 shutdown_connections 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.441+0000 7fab6421c640 1 -- 172.21.15.64:0/2052269159 wait complete. 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 Processor -- start 2024-03-04T07:28:24.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 -- start start 2024-03-04T07:28:24.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c0ce900 0x7fab5c15a350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab5c144640 0x7fab5c159b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c15b130 0x7fab5c1f0280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab5c070c20 con 0x7fab5c15b130 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab5c070aa0 con 0x7fab5c144640 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab6421c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab5c0b36d0 con 0x7fab5c0ce900 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.442+0000 7fab627b9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c15b130 0x7fab5c1f0280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c15b130 0x7fab5c1f0280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33546/0 (socket says 172.21.15.64:33546) 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 -- 172.21.15.64:0/1681576429 learned_addr learned my addr 172.21.15.64:0/1681576429 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab61fb8640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c0ce900 0x7fab5c15a350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab617b7640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab5c144640 0x7fab5c159b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 -- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c0ce900 msgr2=0x7fab5c15a350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c0ce900 0x7fab5c15a350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 -- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab5c144640 msgr2=0x7fab5c159b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab5c144640 0x7fab5c159b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 -- 172.21.15.64:0/1681576429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab5c139210 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab627b9640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c15b130 0x7fab5c1f0280 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fab54009e50 tx=0x7fab54004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab527fc640 1 -- 172.21.15.64:0/1681576429 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab54043030 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab527fc640 1 -- 172.21.15.64:0/1681576429 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab5402ce50 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab5c1394f0 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab527fc640 1 -- 172.21.15.64:0/1681576429 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab54009510 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.443+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab5c1efa40 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.444+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab5c0733c0 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.445+0000 7fab527fc640 1 -- 172.21.15.64:0/1681576429 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fab54046070 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.445+0000 7fab527fc640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab4006b4d0 0x7fab4006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.445+0000 7fab527fc640 1 -- 172.21.15.64:0/1681576429 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(34..34 src has 1..34) v4 ==== 5795+0+0 (secure 0 0 0) 0x7fab540ac460 con 0x7fab5c15b130 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.445+0000 7fab61fb8640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab4006b4d0 0x7fab4006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:24.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.445+0000 7fab61fb8640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab4006b4d0 0x7fab4006d990 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fab58000f60 tx=0x7fab58007b70 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:24.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.449+0000 7fab527fc640 1 -- 172.21.15.64:0/1681576429 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fab5404d070 con 0x7fab5c15b130 2024-03-04T07:28:24.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:24.667+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7fab5c04f410 con 0x7fab5c15b130 2024-03-04T07:28:25.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.320+0000 7fab527fc640 1 -- 172.21.15.64:0/1681576429 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v17) v1 ==== 104+0+0 (secure 0 0 0) 0x7fab5407b310 con 0x7fab5c15b130 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab4006b4d0 msgr2=0x7fab4006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab4006b4d0 0x7fab4006d990 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fab58000f60 tx=0x7fab58007b70 comp rx=0 tx=0).stop 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c15b130 msgr2=0x7fab5c1f0280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c15b130 0x7fab5c1f0280 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fab54009e50 tx=0x7fab54004a40 comp rx=0 tx=0).stop 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 shutdown_connections 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab4006b4d0 0x7fab4006d990 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab5c15b130 0x7fab5c1f0280 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab5c144640 0x7fab5c159b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 --2- 172.21.15.64:0/1681576429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab5c0ce900 0x7fab5c15a350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 >> 172.21.15.64:0/1681576429 conn(0x7fab5c066d00 msgr2=0x7fab5c15c590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 shutdown_connections 2024-03-04T07:28:25.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.323+0000 7fab6421c640 1 -- 172.21.15.64:0/1681576429 wait complete. 2024-03-04T07:28:25.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:28:25.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.485+0000 7ff142720640 1 Processor -- start 2024-03-04T07:28:25.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.485+0000 7ff142720640 1 -- start start 2024-03-04T07:28:25.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.485+0000 7ff142720640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff13c071540 0x7ff13c071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.485+0000 7ff142720640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff13c071e60 con 0x7ff13c0b2d30 2024-03-04T07:28:25.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.485+0000 7ff142720640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff13c0d4960 con 0x7ff13c0cee80 2024-03-04T07:28:25.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.485+0000 7ff142720640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff13c0d3230 con 0x7ff13c071540 2024-03-04T07:28:25.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13b7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff13c0b2d30 0x7ff13c0ce770 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47168/0 (socket says 172.21.15.64:47168) 2024-03-04T07:28:25.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13b7fe640 1 -- 172.21.15.64:0/1390309627 learned_addr learned my addr 172.21.15.64:0/1390309627 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:25.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1941974952 0 0) 0x7ff13c071e60 con 0x7ff13c0b2d30 2024-03-04T07:28:25.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff124003520 con 0x7ff13c0b2d30 2024-03-04T07:28:25.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13bfff640 1 --2- 172.21.15.64:0/1390309627 >> v2:172.21.15.66:3300/0 conn(0x7ff13c071540 0x7ff13c071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 956778838 0 0) 0x7ff124003520 con 0x7ff13c0b2d30 2024-03-04T07:28:25.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 >> v2:172.21.15.66:3300/0 conn(0x7ff13c071540 msgr2=0x7ff13c071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:25.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 --2- 172.21.15.64:0/1390309627 >> v2:172.21.15.66:3300/0 conn(0x7ff13c071540 0x7ff13c071920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 >> v1:172.21.15.66:6789/0 conn(0x7ff13c0cee80 legacy=0x7ff13c168900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:25.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff13c0d36c0 con 0x7ff13c0b2d30 2024-03-04T07:28:25.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7ff130001670 con 0x7ff13c0b2d30 2024-03-04T07:28:25.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 >> v1:172.21.15.64:6789/0 conn(0x7ff13c0b2d30 legacy=0x7ff13c0ce770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:25.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff1240039b0 0x7ff124003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff124004800 0x7ff124004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff1240051c0 0x7ff12400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff13c0d3230 con 0x7ff124004800 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff12400c060 con 0x7ff1240051c0 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff12400c260 con 0x7ff1240039b0 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13b7fe640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff124004800 0x7ff124004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff13bfff640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff1240039b0 0x7ff124003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.486+0000 7ff140cbd640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff1240051c0 0x7ff12400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13b7fe640 1 -- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff1240039b0 msgr2=0x7ff124003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13b7fe640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff1240039b0 0x7ff124003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13b7fe640 1 -- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff1240051c0 msgr2=0x7ff12400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13b7fe640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff1240051c0 0x7ff12400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13b7fe640 1 -- 172.21.15.64:0/1390309627 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff124003520 con 0x7ff124004800 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff140cbd640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff1240051c0 0x7ff12400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:25.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13bfff640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff1240039b0 0x7ff124003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff140cbd640 1 -- 172.21.15.64:0/1390309627 reap_dead start 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13b7fe640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff124004800 0x7ff124004c80 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7ff13c08d1a0 tx=0x7ff124005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d39ffb1d09c1d6f2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff130007d30 con 0x7ff124004800 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.487+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff130030070 con 0x7ff124004800 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff13affd640 1 -- 172.21.15.64:0/1390309627 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff130005d70 con 0x7ff124004800 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff142720640 1 -- 172.21.15.64:0/1390309627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff124004800 msgr2=0x7ff124004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff142720640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff124004800 0x7ff124004c80 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7ff13c08d1a0 tx=0x7ff124005830 comp rx=0 tx=0).stop 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff142720640 1 -- 172.21.15.64:0/1390309627 shutdown_connections 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff142720640 1 --2- 172.21.15.64:0/1390309627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff124004800 0x7ff124004c80 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff142720640 1 -- 172.21.15.64:0/1390309627 >> 172.21.15.64:0/1390309627 conn(0x7ff13c0d3d60 msgr2=0x7ff13c0d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff142720640 1 -- 172.21.15.64:0/1390309627 shutdown_connections 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.488+0000 7ff142720640 1 -- 172.21.15.64:0/1390309627 wait complete. 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.489+0000 7ff142720640 1 Processor -- start 2024-03-04T07:28:25.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.489+0000 7ff142720640 1 -- start start 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff142720640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff124004800 0x7ff13c14b1e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff142720640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff13c1e7090 0x7ff13c1e74f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff142720640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff13c0c9b50 0x7ff13c0c9fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff142720640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff13c0d2be0 con 0x7ff13c1e7090 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff142720640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff13c0d2900 con 0x7ff124004800 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff142720640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff13c0d2ec0 con 0x7ff13c0c9b50 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff13c1e7090 0x7ff13c1e74f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff13c1e7090 0x7ff13c1e74f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33574/0 (socket says 172.21.15.64:33574) 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13b7fe640 1 -- 172.21.15.64:0/271784544 learned_addr learned my addr 172.21.15.64:0/271784544 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13bfff640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff124004800 0x7ff13c14b1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff140cbd640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff13c0c9b50 0x7ff13c0c9fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13b7fe640 1 -- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff13c0c9b50 msgr2=0x7ff13c0c9fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13b7fe640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff13c0c9b50 0x7ff13c0c9fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13b7fe640 1 -- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff124004800 msgr2=0x7ff13c14b1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:25.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.490+0000 7ff13b7fe640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff124004800 0x7ff13c14b1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff13b7fe640 1 -- 172.21.15.64:0/271784544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff13c1e4860 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff13bfff640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff124004800 0x7ff13c14b1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff140cbd640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff13c0c9b50 0x7ff13c0c9fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff13b7fe640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff13c1e7090 0x7ff13c1e74f0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7ff13000c2d0 tx=0x7ff13002e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff138ff9640 1 -- 172.21.15.64:0/271784544 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff130046070 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff138ff9640 1 -- 172.21.15.64:0/271784544 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff130004bf0 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff13c1e4b40 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff138ff9640 1 -- 172.21.15.64:0/271784544 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff130007d10 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.491+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff13c1e5100 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.492+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff13c0ca900 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.492+0000 7ff138ff9640 1 -- 172.21.15.64:0/271784544 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff130007070 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.493+0000 7ff138ff9640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff12806b4d0 0x7ff12806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.493+0000 7ff138ff9640 1 -- 172.21.15.64:0/271784544 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(34..34 src has 1..34) v4 ==== 5795+0+0 (secure 0 0 0) 0x7ff1300ab660 con 0x7ff13c1e7090 2024-03-04T07:28:25.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.493+0000 7ff13bfff640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff12806b4d0 0x7ff12806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:25.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.493+0000 7ff13bfff640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff12806b4d0 0x7ff12806d990 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7ff12c009b00 tx=0x7ff12c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:25.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.497+0000 7ff138ff9640 1 -- 172.21.15.64:0/271784544 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff13007a480 con 0x7ff13c1e7090 2024-03-04T07:28:25.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:25.677+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7ff13c072070 con 0x7ff13c1e7090 2024-03-04T07:28:26.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.324+0000 7ff138ff9640 1 -- 172.21.15.64:0/271784544 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v35) v1 ==== 181+0+0 (secure 0 0 0) 0x7ff130079d80 con 0x7ff13c1e7090 2024-03-04T07:28:26.324 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff12806b4d0 msgr2=0x7ff12806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff12806b4d0 0x7ff12806d990 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7ff12c009b00 tx=0x7ff12c008040 comp rx=0 tx=0).stop 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff13c1e7090 msgr2=0x7ff13c1e74f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff13c1e7090 0x7ff13c1e74f0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7ff13000c2d0 tx=0x7ff13002e070 comp rx=0 tx=0).stop 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 shutdown_connections 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff12806b4d0 0x7ff12806d990 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff13c0c9b50 0x7ff13c0c9fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff13c1e7090 0x7ff13c1e74f0 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 --2- 172.21.15.64:0/271784544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff124004800 0x7ff13c14b1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 >> 172.21.15.64:0/271784544 conn(0x7ff13c0d3d60 msgr2=0x7ff13c0b5580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:26.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 shutdown_connections 2024-03-04T07:28:26.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.326+0000 7ff142720640 1 -- 172.21.15.64:0/271784544 wait complete. 2024-03-04T07:28:26.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:28:26.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.452+0000 7efe87047640 1 Processor -- start 2024-03-04T07:28:26.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.452+0000 7efe87047640 1 -- start start 2024-03-04T07:28:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe87047640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efe8015dd90 0x7efe800ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe87047640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efe800b6d10 0x7efe800b7150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe87047640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efe800d3740 con 0x7efe800d29d0 2024-03-04T07:28:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe87047640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efe800d3400 con 0x7efe8015dd90 2024-03-04T07:28:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe87047640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efe800b7690 con 0x7efe800b6d10 2024-03-04T07:28:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe84de3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efe8015dd90 0x7efe800ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe7ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7efe800d29d0 0x7efe800d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47180/0 (socket says 172.21.15.64:47180) 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.453+0000 7efe7ffff640 1 -- 172.21.15.64:0/3222109688 learned_addr learned my addr 172.21.15.64:0/3222109688 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe855e4640 1 --2- 172.21.15.64:0/3222109688 >> v2:172.21.15.66:3300/0 conn(0x7efe800b6d10 0x7efe800b7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe84de3640 1 -- 172.21.15.64:0/3222109688 >> v2:172.21.15.66:3300/0 conn(0x7efe800b6d10 msgr2=0x7efe800b7150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe84de3640 1 --2- 172.21.15.64:0/3222109688 >> v2:172.21.15.66:3300/0 conn(0x7efe800b6d10 0x7efe800b7150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe84de3640 1 -- 172.21.15.64:0/3222109688 >> v1:172.21.15.64:6789/0 conn(0x7efe800d29d0 legacy=0x7efe800d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe7f7fe640 1 -- 172.21.15.64:0/3222109688 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 60113148 0 0) 0x7efe800d3740 con 0x7efe800d29d0 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe84de3640 1 -- 172.21.15.64:0/3222109688 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe800d3120 con 0x7efe8015dd90 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe855e4640 1 --2- 172.21.15.64:0/3222109688 >> v2:172.21.15.66:3300/0 conn(0x7efe800b6d10 0x7efe800b7150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.454+0000 7efe84de3640 1 --2- 172.21.15.64:0/3222109688 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800ceea0 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7efe800b5820 tx=0x7efe7802bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cec39256d86e36ba server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe7f7fe640 1 -- 172.21.15.64:0/3222109688 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe78039070 con 0x7efe8015dd90 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe7f7fe640 1 -- 172.21.15.64:0/3222109688 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efe7802cd00 con 0x7efe8015dd90 2024-03-04T07:28:26.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe7f7fe640 1 -- 172.21.15.64:0/3222109688 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe78034660 con 0x7efe8015dd90 2024-03-04T07:28:26.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 -- 172.21.15.64:0/3222109688 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 msgr2=0x7efe800ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:26.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 --2- 172.21.15.64:0/3222109688 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800ceea0 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7efe800b5820 tx=0x7efe7802bf50 comp rx=0 tx=0).stop 2024-03-04T07:28:26.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 -- 172.21.15.64:0/3222109688 shutdown_connections 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 --2- 172.21.15.64:0/3222109688 >> v2:172.21.15.66:3300/0 conn(0x7efe800b6d10 0x7efe800b7150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 --2- 172.21.15.64:0/3222109688 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800ceea0 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 -- 172.21.15.64:0/3222109688 >> 172.21.15.64:0/3222109688 conn(0x7efe80071960 msgr2=0x7efe80071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 -- 172.21.15.64:0/3222109688 shutdown_connections 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.455+0000 7efe87047640 1 -- 172.21.15.64:0/3222109688 wait complete. 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.456+0000 7efe87047640 1 Processor -- start 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.456+0000 7efe87047640 1 -- start start 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe87047640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efe800b6d10 0x7efe80150070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe87047640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efe800d29d0 0x7efe800cb910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe87047640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800cbe50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe87047640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe8015e170 con 0x7efe8015dd90 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe87047640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe800d3cc0 con 0x7efe800d29d0 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe87047640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efe800d2df0 con 0x7efe800b6d10 2024-03-04T07:28:26.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe855e4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800cbe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe855e4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800cbe50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33582/0 (socket says 172.21.15.64:33582) 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.457+0000 7efe855e4640 1 -- 172.21.15.64:0/4137249007 learned_addr learned my addr 172.21.15.64:0/4137249007 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe7ffff640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efe800d29d0 0x7efe800cb910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe84de3640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efe800b6d10 0x7efe80150070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe855e4640 1 -- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efe800b6d10 msgr2=0x7efe80150070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe855e4640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efe800b6d10 0x7efe80150070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe855e4640 1 -- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efe800d29d0 msgr2=0x7efe800cb910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe855e4640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efe800d29d0 0x7efe800cb910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe855e4640 1 -- 172.21.15.64:0/4137249007 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe8015f9f0 con 0x7efe8015dd90 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe84de3640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efe800b6d10 0x7efe80150070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe7ffff640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efe800d29d0 0x7efe800cb910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe855e4640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800cbe50 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7efe7000c820 tx=0x7efe7000cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:26.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe7d7fa640 1 -- 172.21.15.64:0/4137249007 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe70015070 con 0x7efe8015dd90 2024-03-04T07:28:26.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe7d7fa640 1 -- 172.21.15.64:0/4137249007 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efe70007a30 con 0x7efe8015dd90 2024-03-04T07:28:26.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efe8015fcd0 con 0x7efe8015dd90 2024-03-04T07:28:26.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe7d7fa640 1 -- 172.21.15.64:0/4137249007 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe70007e20 con 0x7efe8015dd90 2024-03-04T07:28:26.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.458+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe80160290 con 0x7efe8015dd90 2024-03-04T07:28:26.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.459+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efe80072a30 con 0x7efe8015dd90 2024-03-04T07:28:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.459+0000 7efe7d7fa640 1 -- 172.21.15.64:0/4137249007 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efe70018070 con 0x7efe8015dd90 2024-03-04T07:28:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.460+0000 7efe7d7fa640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efe6c06b4d0 0x7efe6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.460+0000 7efe7d7fa640 1 -- 172.21.15.64:0/4137249007 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 5357+0+0 (secure 0 0 0) 0x7efe7008a020 con 0x7efe8015dd90 2024-03-04T07:28:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.460+0000 7efe84de3640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efe6c06b4d0 0x7efe6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:26.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.460+0000 7efe84de3640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efe6c06b4d0 0x7efe6c06d990 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7efe78036000 tx=0x7efe78030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:26.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.464+0000 7efe7d7fa640 1 -- 172.21.15.64:0/4137249007 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efe70059000 con 0x7efe8015dd90 2024-03-04T07:28:26.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:26.645+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7efe800ce6c0 con 0x7efe8015dd90 2024-03-04T07:28:27.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.328+0000 7efe7d7fa640 1 -- 172.21.15.64:0/4137249007 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v36) v1 ==== 169+0+0 (secure 0 0 0) 0x7efe70058900 con 0x7efe8015dd90 2024-03-04T07:28:27.329 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:28:27.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.331+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efe6c06b4d0 msgr2=0x7efe6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.331+0000 7efe87047640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efe6c06b4d0 0x7efe6c06d990 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7efe78036000 tx=0x7efe78030000 comp rx=0 tx=0).stop 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.331+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 msgr2=0x7efe800cbe50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.331+0000 7efe87047640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800cbe50 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7efe7000c820 tx=0x7efe7000cce0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 shutdown_connections 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efe6c06b4d0 0x7efe6c06d990 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efe8015dd90 0x7efe800cbe50 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efe800d29d0 0x7efe800cb910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 --2- 172.21.15.64:0/4137249007 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efe800b6d10 0x7efe80150070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 >> 172.21.15.64:0/4137249007 conn(0x7efe80071960 msgr2=0x7efe800b36c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 shutdown_connections 2024-03-04T07:28:27.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.332+0000 7efe87047640 1 -- 172.21.15.64:0/4137249007 wait complete. 2024-03-04T07:28:27.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:28:27.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.454+0000 7f4552381640 1 Processor -- start 2024-03-04T07:28:27.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.454+0000 7f4552381640 1 -- start start 2024-03-04T07:28:27.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f4552381640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f454c0ce4f0 0x7f454c0ce960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f4552381640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f454c15f730 con 0x7f454c1618e0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f4552381640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f454c15ebb0 con 0x7f454c0d0150 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f4552381640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f454c0cfce0 con 0x7f454c0ce4f0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f454c1618e0 0x7f454c06b130 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47190/0 (socket says 172.21.15.64:47190) 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454bfff640 1 -- 172.21.15.64:0/2312707886 learned_addr learned my addr 172.21.15.64:0/2312707886 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2654929648 0 0) 0x7f454c15f730 con 0x7f454c1618e0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454b7fe640 1 --2- 172.21.15.64:0/2312707886 >> v2:172.21.15.66:3300/0 conn(0x7f454c0ce4f0 0x7f454c0ce960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4534003520 con 0x7f454c1618e0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1861747249 0 0) 0x7f4534003520 con 0x7f454c1618e0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 >> v2:172.21.15.66:3300/0 conn(0x7f454c0ce4f0 msgr2=0x7f454c0ce960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454affd640 1 --2- 172.21.15.64:0/2312707886 >> v2:172.21.15.66:3300/0 conn(0x7f454c0ce4f0 0x7f454c0ce960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 >> v1:172.21.15.66:6789/0 conn(0x7f454c0d0150 legacy=0x7f454c0d05a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.455+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f454c15ee90 con 0x7f454c1618e0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f4540001670 con 0x7f454c1618e0 2024-03-04T07:28:27.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 >> v1:172.21.15.64:6789/0 conn(0x7f454c1618e0 legacy=0x7f454c06b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f45340039b0 0x7f4534003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4534004800 0x7f4534004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f453400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f453400c040 con 0x7f45340051c0 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f453400c1f0 con 0x7f45340039b0 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f453400c3f0 con 0x7f4534004800 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f455091e640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f453400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454bfff640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f45340039b0 0x7f4534003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f454b7fe640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4534004800 0x7f4534004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f455091e640 1 -- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4534004800 msgr2=0x7f4534004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f455091e640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4534004800 0x7f4534004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f455091e640 1 -- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f45340039b0 msgr2=0x7f4534003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:27.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f455091e640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f45340039b0 0x7f4534003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.456+0000 7f455091e640 1 -- 172.21.15.64:0/2312707886 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4534003520 con 0x7f45340051c0 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f455091e640 1 -- 172.21.15.64:0/2312707886 reap_dead start 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f455091e640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f453400b9c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f4544001e50 tx=0x7f4534005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=448f55ca2a567750 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4544001a70 con 0x7f45340051c0 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4544003bb0 con 0x7f45340051c0 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f454affd640 1 -- 172.21.15.64:0/2312707886 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45440050d0 con 0x7f45340051c0 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f4552381640 1 -- 172.21.15.64:0/2312707886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 msgr2=0x7f453400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f4552381640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f453400b9c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f4544001e50 tx=0x7f4534005830 comp rx=0 tx=0).stop 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f4552381640 1 -- 172.21.15.64:0/2312707886 shutdown_connections 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f4552381640 1 --2- 172.21.15.64:0/2312707886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f453400b9c0 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f4552381640 1 -- 172.21.15.64:0/2312707886 >> 172.21.15.64:0/2312707886 conn(0x7f454c0b3660 msgr2=0x7f454c0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.457+0000 7f4552381640 1 -- 172.21.15.64:0/2312707886 shutdown_connections 2024-03-04T07:28:27.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.458+0000 7f4552381640 1 -- 172.21.15.64:0/2312707886 wait complete. 2024-03-04T07:28:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.459+0000 7f4552381640 1 Processor -- start 2024-03-04T07:28:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.459+0000 7f4552381640 1 -- start start 2024-03-04T07:28:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.459+0000 7f4552381640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f454c0b74b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.459+0000 7f4552381640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f454c148d20 0x7f454c1458b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.459+0000 7f4552381640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f454c0b4770 0x7f454c0d1c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.459+0000 7f4552381640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f454c15e7d0 con 0x7f45340051c0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f4552381640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f454c15f450 con 0x7f454c148d20 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f4552381640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f454c15e950 con 0x7f454c0b4770 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f454c0b74b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f454c0b74b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33592/0 (socket says 172.21.15.64:33592) 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 -- 172.21.15.64:0/2265137675 learned_addr learned my addr 172.21.15.64:0/2265137675 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454b7fe640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f454c148d20 0x7f454c1458b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f455091e640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f454c0b4770 0x7f454c0d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 -- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f454c0b4770 msgr2=0x7f454c0d1c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f454c0b4770 0x7f454c0d1c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 -- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f454c148d20 msgr2=0x7f454c1458b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f454c148d20 0x7f454c1458b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 -- 172.21.15.64:0/2265137675 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f454c0b6450 con 0x7f45340051c0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f454bfff640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f454c0b74b0 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f4540001050 tx=0x7f4540007720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f4548ff9640 1 -- 172.21.15.64:0/2265137675 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f454000f960 con 0x7f45340051c0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f4548ff9640 1 -- 172.21.15.64:0/2265137675 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4540018070 con 0x7f45340051c0 2024-03-04T07:28:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f454c0b66f0 con 0x7f45340051c0 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f4548ff9640 1 -- 172.21.15.64:0/2265137675 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f454000fd80 con 0x7f45340051c0 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.460+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f454c0b6cb0 con 0x7f45340051c0 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.461+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f454c0741c0 con 0x7f45340051c0 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.462+0000 7f4548ff9640 1 -- 172.21.15.64:0/2265137675 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f454001d460 con 0x7f45340051c0 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.462+0000 7f4548ff9640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f453806b500 0x7f453806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.462+0000 7f4548ff9640 1 -- 172.21.15.64:0/2265137675 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f4540089f70 con 0x7f45340051c0 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.462+0000 7f454b7fe640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f453806b500 0x7f453806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.462+0000 7f454b7fe640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f453806b500 0x7f453806d9c0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f454c0ce120 tx=0x7f453c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:27.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.466+0000 7f4548ff9640 1 -- 172.21.15.64:0/2265137675 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f454c0741c0 con 0x7f45340051c0 2024-03-04T07:28:27.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.648+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f454c06af20 con 0x7f45340051c0 2024-03-04T07:28:27.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.652+0000 7f4548ff9640 1 -- 172.21.15.64:0/2265137675 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v11) v1 ==== 104+0+0 (secure 0 0 0) 0x7f454c06af20 con 0x7f45340051c0 2024-03-04T07:28:27.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f453806b500 msgr2=0x7f453806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f453806b500 0x7f453806d9c0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f454c0ce120 tx=0x7f453c008040 comp rx=0 tx=0).stop 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 msgr2=0x7f454c0b74b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f454c0b74b0 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f4540001050 tx=0x7f4540007720 comp rx=0 tx=0).stop 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 shutdown_connections 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f453806b500 0x7f453806d9c0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f454c0b4770 0x7f454c0d1c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f454c148d20 0x7f454c1458b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 --2- 172.21.15.64:0/2265137675 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f45340051c0 0x7f454c0b74b0 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 >> 172.21.15.64:0/2265137675 conn(0x7f454c0b3660 msgr2=0x7f454c0d1390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 shutdown_connections 2024-03-04T07:28:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.654+0000 7f4552381640 1 -- 172.21.15.64:0/2265137675 wait complete. 2024-03-04T07:28:27.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dir_deletion' 2024-03-04T07:28:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.766+0000 7fd84ceb5640 1 Processor -- start 2024-03-04T07:28:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.766+0000 7fd84ceb5640 1 -- start start 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd84ceb5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd848071c50 0x7fd8480b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd84ceb5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd8480b6f20 con 0x7fd84815df70 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd84ceb5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd8480d3030 con 0x7fd8480d18c0 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd84ceb5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd8480d3310 con 0x7fd848071c50 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd846d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd848071c50 0x7fd8480b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd846d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd848071c50 0x7fd8480b2700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33608/0 (socket says 172.21.15.64:33608) 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd846d9d640 1 -- 172.21.15.64:0/2325806295 learned_addr learned my addr 172.21.15.64:0/2325806295 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd84659c640 1 --1- 172.21.15.64:0/2325806295 >> v1:172.21.15.64:6789/0 conn(0x7fd84815df70 0x7fd8480d27b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47196/0 (socket says 172.21.15.64:47196) 2024-03-04T07:28:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd846d9d640 1 -- 172.21.15.64:0/2325806295 >> v1:172.21.15.66:6789/0 conn(0x7fd8480d18c0 legacy=0x7fd848071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd846d9d640 1 -- 172.21.15.64:0/2325806295 >> v1:172.21.15.64:6789/0 conn(0x7fd84815df70 legacy=0x7fd8480d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.767+0000 7fd846d9d640 1 -- 172.21.15.64:0/2325806295 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8480d35f0 con 0x7fd848071c50 2024-03-04T07:28:27.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.768+0000 7fd846d9d640 1 --2- 172.21.15.64:0/2325806295 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd848071c50 0x7fd8480b2700 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7fd840009870 tx=0x7fd84002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2f360d1b4e55902d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:27.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.768+0000 7fd84559a640 1 -- 172.21.15.64:0/2325806295 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd84002f840 con 0x7fd848071c50 2024-03-04T07:28:27.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.768+0000 7fd84559a640 1 -- 172.21.15.64:0/2325806295 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd84002f9e0 con 0x7fd848071c50 2024-03-04T07:28:27.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.768+0000 7fd84559a640 1 -- 172.21.15.64:0/2325806295 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd84002fdb0 con 0x7fd848071c50 2024-03-04T07:28:27.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.769+0000 7fd84ceb5640 1 -- 172.21.15.64:0/2325806295 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd848071c50 msgr2=0x7fd8480b2700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.769+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/2325806295 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd848071c50 0x7fd8480b2700 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7fd840009870 tx=0x7fd84002ba20 comp rx=0 tx=0).stop 2024-03-04T07:28:27.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.769+0000 7fd84ceb5640 1 -- 172.21.15.64:0/2325806295 shutdown_connections 2024-03-04T07:28:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.769+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/2325806295 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd848071c50 0x7fd8480b2700 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.769+0000 7fd84ceb5640 1 -- 172.21.15.64:0/2325806295 >> 172.21.15.64:0/2325806295 conn(0x7fd8480d3cf0 msgr2=0x7fd8480d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.769+0000 7fd84ceb5640 1 -- 172.21.15.64:0/2325806295 shutdown_connections 2024-03-04T07:28:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.769+0000 7fd84ceb5640 1 -- 172.21.15.64:0/2325806295 wait complete. 2024-03-04T07:28:27.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.770+0000 7fd84ceb5640 1 Processor -- start 2024-03-04T07:28:27.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.770+0000 7fd84ceb5640 1 -- start start 2024-03-04T07:28:27.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd84ceb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd848071c50 0x7fd8481531f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd84ceb5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8480d18c0 0x7fd8480cd5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd84ceb5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd84815df70 0x7fd8480cbad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd84ceb5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd84815e4d0 con 0x7fd8480d18c0 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd84ceb5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd84815e350 con 0x7fd84815df70 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd84ceb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd8480b8410 con 0x7fd848071c50 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8480d18c0 0x7fd8480cd5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8480d18c0 0x7fd8480cd5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33610/0 (socket says 172.21.15.64:33610) 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 -- 172.21.15.64:0/1781225238 learned_addr learned my addr 172.21.15.64:0/1781225238 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd84659c640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd848071c50 0x7fd8481531f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 -- 172.21.15.64:0/1781225238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd848071c50 msgr2=0x7fd8481531f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd848071c50 0x7fd8481531f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 -- 172.21.15.64:0/1781225238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd84815df70 msgr2=0x7fd8480cbad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd84815df70 0x7fd8480cbad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.771+0000 7fd845d9b640 1 -- 172.21.15.64:0/1781225238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8480cc520 con 0x7fd8480d18c0 2024-03-04T07:28:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.772+0000 7fd845d9b640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8480d18c0 0x7fd8480cd5c0 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7fd83000c430 tx=0x7fd83000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.772+0000 7fd83effd640 1 -- 172.21.15.64:0/1781225238 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd830011790 con 0x7fd8480d18c0 2024-03-04T07:28:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.772+0000 7fd83effd640 1 -- 172.21.15.64:0/1781225238 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd830017070 con 0x7fd8480d18c0 2024-03-04T07:28:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.772+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd8480cc800 con 0x7fd8480d18c0 2024-03-04T07:28:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.772+0000 7fd83effd640 1 -- 172.21.15.64:0/1781225238 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd830011bb0 con 0x7fd8480d18c0 2024-03-04T07:28:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.772+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd8480ccdc0 con 0x7fd8480d18c0 2024-03-04T07:28:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.773+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd848067ff0 con 0x7fd8480d18c0 2024-03-04T07:28:27.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.773+0000 7fd83effd640 1 -- 172.21.15.64:0/1781225238 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd83001c460 con 0x7fd8480d18c0 2024-03-04T07:28:27.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.773+0000 7fd83effd640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd82806b4d0 0x7fd82806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:27.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.773+0000 7fd83effd640 1 -- 172.21.15.64:0/1781225238 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fd830089d50 con 0x7fd8480d18c0 2024-03-04T07:28:27.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.774+0000 7fd84659c640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd82806b4d0 0x7fd82806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:27.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.774+0000 7fd84659c640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd82806b4d0 0x7fd82806d990 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fd848071890 tx=0x7fd838009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:27.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:27.778+0000 7fd83effd640 1 -- 172.21.15.64:0/1781225238 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd83008d030 con 0x7fd8480d18c0 2024-03-04T07:28:28.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.034+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]} v 0) v1 -- 0x7fd8480b6100 con 0x7fd8480d18c0 2024-03-04T07:28:28.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.329+0000 7fd83effd640 1 -- 172.21.15.64:0/1781225238 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dir_deletion"]}]=0 v103) v1 ==== 132+0+0 (secure 0 0 0) 0x7fd830058f60 con 0x7fd8480d18c0 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd82806b4d0 msgr2=0x7fd82806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd82806b4d0 0x7fd82806d990 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fd848071890 tx=0x7fd838009040 comp rx=0 tx=0).stop 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8480d18c0 msgr2=0x7fd8480cd5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8480d18c0 0x7fd8480cd5c0 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7fd83000c430 tx=0x7fd83000c8f0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 shutdown_connections 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd82806b4d0 0x7fd82806d990 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd84815df70 0x7fd8480cbad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8480d18c0 0x7fd8480cd5c0 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 --2- 172.21.15.64:0/1781225238 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd848071c50 0x7fd8481531f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 >> 172.21.15.64:0/1781225238 conn(0x7fd8480d3cf0 msgr2=0x7fd8480b5860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:28.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 shutdown_connections 2024-03-04T07:28:28.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.332+0000 7fd84ceb5640 1 -- 172.21.15.64:0/1781225238 wait complete. 2024-03-04T07:28:28.344 INFO:tasks.cephfs_test_runner:That when deleting a bunch of dentries and the containing ... ok 2024-03-04T07:28:28.344 INFO:tasks.cephfs_test_runner:Starting test: test_dirfrag_limit (tasks.cephfs.test_strays.TestStrays) That the directory fragment size cannot exceed mds_bal_fragment_size_max (using a limit of 50 in all configurations). 2024-03-04T07:28:28.344 INFO:tasks.cephfs_test_runner:test_dirfrag_limit (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:28:28.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:28.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.447+0000 7fbb8e356640 1 Processor -- start 2024-03-04T07:28:28.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.447+0000 7fbb8e356640 1 -- start start 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb8e356640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb8806af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb8e356640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbb8806ed10 0x7fbb880cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb8e356640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbb880cc0e0 con 0x7fbb880cd6c0 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb8e356640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbb880d28b0 con 0x7fbb8806ed10 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb8e356640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbb880cf320 con 0x7fbb880b1020 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb877fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbb8806ed10 0x7fbb880cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb877fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbb8806ed10 0x7fbb880cd180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33612/0 (socket says 172.21.15.64:33612) 2024-03-04T07:28:28.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb877fe640 1 -- 172.21.15.64:0/2474683717 learned_addr learned my addr 172.21.15.64:0/2474683717 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:28.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb87fff640 1 --2- 172.21.15.64:0/2474683717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb8806af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb877fe640 1 -- 172.21.15.64:0/2474683717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 msgr2=0x7fbb8806af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:28.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb877fe640 1 --2- 172.21.15.64:0/2474683717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb8806af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb877fe640 1 -- 172.21.15.64:0/2474683717 >> v1:172.21.15.66:6789/0 conn(0x7fbb880cd6c0 legacy=0x7fbb8815d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:28.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.448+0000 7fbb877fe640 1 -- 172.21.15.64:0/2474683717 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb880cc3c0 con 0x7fbb8806ed10 2024-03-04T07:28:28.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.449+0000 7fbb877fe640 1 --2- 172.21.15.64:0/2474683717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb880cd180 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7fbb78009870 tx=0x7fbb7802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=38e732d841da39c0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:28.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.449+0000 7fbb86ffd640 1 -- 172.21.15.64:0/2474683717 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb78034070 con 0x7fbb8806ed10 2024-03-04T07:28:28.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.449+0000 7fbb86ffd640 1 -- 172.21.15.64:0/2474683717 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbb78030840 con 0x7fbb8806ed10 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.449+0000 7fbb86ffd640 1 -- 172.21.15.64:0/2474683717 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb78030c10 con 0x7fbb8806ed10 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 -- 172.21.15.64:0/2474683717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 msgr2=0x7fbb880cd180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 --2- 172.21.15.64:0/2474683717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb880cd180 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7fbb78009870 tx=0x7fbb7802b980 comp rx=0 tx=0).stop 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 -- 172.21.15.64:0/2474683717 shutdown_connections 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 --2- 172.21.15.64:0/2474683717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb880cd180 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 --2- 172.21.15.64:0/2474683717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb8806af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 -- 172.21.15.64:0/2474683717 >> 172.21.15.64:0/2474683717 conn(0x7fbb880d1a90 msgr2=0x7fbb880d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 -- 172.21.15.64:0/2474683717 shutdown_connections 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.450+0000 7fbb8e356640 1 -- 172.21.15.64:0/2474683717 wait complete. 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.451+0000 7fbb8e356640 1 Processor -- start 2024-03-04T07:28:28.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.451+0000 7fbb8e356640 1 -- start start 2024-03-04T07:28:28.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8e356640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb88158e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8e356640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb880b6cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8e356640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb880cd6c0 0x7fbb880b7200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8e356640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb880ccc60 con 0x7fbb8806ed10 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8e356640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb880cc980 con 0x7fbb880cd6c0 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8e356640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbb880cdaf0 con 0x7fbb880b1020 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb88158e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb88158e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33622/0 (socket says 172.21.15.64:33622) 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 -- 172.21.15.64:0/4255800509 learned_addr learned my addr 172.21.15.64:0/4255800509 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb877fe640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb880b6cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8c8f3640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb880cd6c0 0x7fbb880b7200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 -- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 msgr2=0x7fbb880b6cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb880b6cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 -- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb880cd6c0 msgr2=0x7fbb880b7200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb880cd6c0 0x7fbb880b7200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 -- 172.21.15.64:0/4255800509 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb880b5840 con 0x7fbb8806ed10 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb8c8f3640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb880cd6c0 0x7fbb880b7200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.452+0000 7fbb87fff640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb88158e00 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7fbb7c00d830 tx=0x7fbb7c00dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.453+0000 7fbb84ff9640 1 -- 172.21.15.64:0/4255800509 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb7c00b6c0 con 0x7fbb8806ed10 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.453+0000 7fbb84ff9640 1 -- 172.21.15.64:0/4255800509 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbb7c019070 con 0x7fbb8806ed10 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.453+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb880cb9b0 con 0x7fbb8806ed10 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.453+0000 7fbb84ff9640 1 -- 172.21.15.64:0/4255800509 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb7c00ba80 con 0x7fbb8806ed10 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.453+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb880c9d50 con 0x7fbb8806ed10 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.454+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbb88067ff0 con 0x7fbb8806ed10 2024-03-04T07:28:28.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.454+0000 7fbb84ff9640 1 -- 172.21.15.64:0/4255800509 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbb7c01e460 con 0x7fbb8806ed10 2024-03-04T07:28:28.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.455+0000 7fbb84ff9640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbb7406b500 0x7fbb7406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.455+0000 7fbb877fe640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbb7406b500 0x7fbb7406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.455+0000 7fbb84ff9640 1 -- 172.21.15.64:0/4255800509 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fbb7c08a500 con 0x7fbb8806ed10 2024-03-04T07:28:28.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.455+0000 7fbb877fe640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbb7406b500 0x7fbb7406d9c0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fbb78036000 tx=0x7fbb78008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:28.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.459+0000 7fbb84ff9640 1 -- 172.21.15.64:0/4255800509 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbb7c059680 con 0x7fbb8806ed10 2024-03-04T07:28:28.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.640+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fbb8806f2f0 con 0x7fbb8806ed10 2024-03-04T07:28:28.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.641+0000 7fbb84ff9640 1 -- 172.21.15.64:0/4255800509 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v36) v1 ==== 74+0+12972 (secure 0 0 0) 0x7fbb7c058f80 con 0x7fbb8806ed10 2024-03-04T07:28:28.642 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:28.642 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:27.325942+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6836/2136250526":"2024-03-05T07:28:24.314203+0000","172.21.15.64:6834/2136250526":"2024-03-05T07:28:24.314203+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbb7406b500 msgr2=0x7fbb7406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbb7406b500 0x7fbb7406d9c0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fbb78036000 tx=0x7fbb78008030 comp rx=0 tx=0).stop 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 msgr2=0x7fbb88158e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb88158e00 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7fbb7c00d830 tx=0x7fbb7c00dcf0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 shutdown_connections 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbb7406b500 0x7fbb7406d9c0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb880cd6c0 0x7fbb880b7200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb880b1020 0x7fbb880b6cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 --2- 172.21.15.64:0/4255800509 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb8806ed10 0x7fbb88158e00 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 >> 172.21.15.64:0/4255800509 conn(0x7fbb880d1a90 msgr2=0x7fbb880d1310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 shutdown_connections 2024-03-04T07:28:28.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.643+0000 7fbb8e356640 1 -- 172.21.15.64:0/4255800509 wait complete. 2024-03-04T07:28:28.654 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:28:28.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:28:28.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.758+0000 7f6678808640 1 Processor -- start 2024-03-04T07:28:28.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.758+0000 7f6678808640 1 -- start start 2024-03-04T07:28:28.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.759+0000 7f6678808640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f66740ceea0 0x7f66740d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.759+0000 7f6678808640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66740b6d10 0x7f66740b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.759+0000 7f6678808640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66740d3740 con 0x7f667415dd90 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.759+0000 7f6678808640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f66740d3400 con 0x7f66740ceea0 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.759+0000 7f6678808640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66740d3120 con 0x7f66740b6d10 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.759+0000 7f6671d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f66740ceea0 0x7f66740d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.760+0000 7f6671d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f66740ceea0 0x7f66740d29d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33630/0 (socket says 172.21.15.64:33630) 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.760+0000 7f6671d9b640 1 -- 172.21.15.64:0/4169222311 learned_addr learned my addr 172.21.15.64:0/4169222311 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.760+0000 7f6672d9d640 1 --2- 172.21.15.64:0/4169222311 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66740b6d10 0x7f66740b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.760+0000 7f6671d9b640 1 -- 172.21.15.64:0/4169222311 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66740b6d10 msgr2=0x7f66740b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.760+0000 7f6671d9b640 1 --2- 172.21.15.64:0/4169222311 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66740b6d10 0x7f66740b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.760+0000 7f6671d9b640 1 -- 172.21.15.64:0/4169222311 >> v1:172.21.15.66:6789/0 conn(0x7f667415dd90 legacy=0x7f66740d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.760+0000 7f6671d9b640 1 -- 172.21.15.64:0/4169222311 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66740d39e0 con 0x7f66740ceea0 2024-03-04T07:28:28.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6671d9b640 1 --2- 172.21.15.64:0/4169222311 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740ceea0 0x7f66740d29d0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f66740724d0 tx=0x7f665c02b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d8cac96a6fc9f9e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:28.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f667159a640 1 -- 172.21.15.64:0/4169222311 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f665c034070 con 0x7f66740ceea0 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f667159a640 1 -- 172.21.15.64:0/4169222311 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f665c030610 con 0x7f66740ceea0 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f667159a640 1 -- 172.21.15.64:0/4169222311 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f665c0309e0 con 0x7f66740ceea0 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 -- 172.21.15.64:0/4169222311 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740ceea0 msgr2=0x7f66740d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 --2- 172.21.15.64:0/4169222311 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740ceea0 0x7f66740d29d0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f66740724d0 tx=0x7f665c02b960 comp rx=0 tx=0).stop 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 -- 172.21.15.64:0/4169222311 shutdown_connections 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 --2- 172.21.15.64:0/4169222311 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66740b6d10 0x7f66740b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 --2- 172.21.15.64:0/4169222311 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740ceea0 0x7f66740d29d0 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 -- 172.21.15.64:0/4169222311 >> 172.21.15.64:0/4169222311 conn(0x7f6674071960 msgr2=0x7f6674071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 -- 172.21.15.64:0/4169222311 shutdown_connections 2024-03-04T07:28:28.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.761+0000 7f6678808640 1 -- 172.21.15.64:0/4169222311 wait complete. 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 Processor -- start 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 -- start start 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740b6d10 0x7f6674156f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66740ceea0 0x7f667415e650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f667415dd90 0x7f66740c9a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66740d48f0 con 0x7f66740b6d10 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f667415e170 con 0x7f66740ceea0 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f6678808640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66740d4a70 con 0x7f667415dd90 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.763+0000 7f667259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740b6d10 0x7f6674156f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740b6d10 0x7f6674156f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33632/0 (socket says 172.21.15.64:33632) 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 -- 172.21.15.64:0/2081625744 learned_addr learned my addr 172.21.15.64:0/2081625744 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:28.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f6672d9d640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f667415dd90 0x7f66740c9a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f6671d9b640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66740ceea0 0x7f667415e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 -- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f667415dd90 msgr2=0x7f66740c9a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f667415dd90 0x7f66740c9a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 -- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66740ceea0 msgr2=0x7f667415e650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66740ceea0 0x7f667415e650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 -- 172.21.15.64:0/2081625744 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66740cae40 con 0x7f66740b6d10 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f6671d9b640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66740ceea0 0x7f667415e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f6672d9d640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f667415dd90 0x7f66740c9a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f667259c640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740b6d10 0x7f6674156f10 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f666400b420 tx=0x7f666400b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f666affd640 1 -- 172.21.15.64:0/2081625744 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6664011830 con 0x7f66740b6d10 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f666affd640 1 -- 172.21.15.64:0/2081625744 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6664016070 con 0x7f66740b6d10 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66740cb3d0 con 0x7f66740b6d10 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f666affd640 1 -- 172.21.15.64:0/2081625744 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6664011c10 con 0x7f66740b6d10 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.764+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6674157450 con 0x7f66740b6d10 2024-03-04T07:28:28.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.765+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6674072a30 con 0x7f66740b6d10 2024-03-04T07:28:28.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.766+0000 7f666affd640 1 -- 172.21.15.64:0/2081625744 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f666401b460 con 0x7f66740b6d10 2024-03-04T07:28:28.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.766+0000 7f666affd640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f665406b4d0 0x7f665406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:28.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.766+0000 7f666affd640 1 -- 172.21.15.64:0/2081625744 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f66640899f0 con 0x7f66740b6d10 2024-03-04T07:28:28.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.766+0000 7f6671d9b640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f665406b4d0 0x7f665406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:28.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.767+0000 7f6671d9b640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f665406b4d0 0x7f665406d990 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f665c036000 tx=0x7f665c03b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:28.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.771+0000 7f666affd640 1 -- 172.21.15.64:0/2081625744 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6664020020 con 0x7f66740b6d10 2024-03-04T07:28:28.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.949+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f667404f410 con 0x7f66740b6d10 2024-03-04T07:28:28.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.950+0000 7f666affd640 1 -- 172.21.15.64:0/2081625744 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v36) v1 ==== 93+0+10 (secure 0 0 0) 0x7f6664058b70 con 0x7f66740b6d10 2024-03-04T07:28:28.950 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:28:28.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.951+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f665406b4d0 msgr2=0x7f665406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.951+0000 7f6678808640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f665406b4d0 0x7f665406d990 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f665c036000 tx=0x7f665c03b040 comp rx=0 tx=0).stop 2024-03-04T07:28:28.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.951+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740b6d10 msgr2=0x7f6674156f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.951+0000 7f6678808640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740b6d10 0x7f6674156f10 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f666400b420 tx=0x7f666400b8e0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.951+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 shutdown_connections 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.951+0000 7f6678808640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f665406b4d0 0x7f665406d990 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.952+0000 7f6678808640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f667415dd90 0x7f66740c9a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.952+0000 7f6678808640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66740ceea0 0x7f667415e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.952+0000 7f6678808640 1 --2- 172.21.15.64:0/2081625744 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66740b6d10 0x7f6674156f10 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.952+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 >> 172.21.15.64:0/2081625744 conn(0x7f6674071960 msgr2=0x7f66740b71a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.952+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 shutdown_connections 2024-03-04T07:28:28.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:28.952+0000 7f6678808640 1 -- 172.21.15.64:0/2081625744 wait complete. 2024-03-04T07:28:28.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit' 2024-03-04T07:28:29.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.064+0000 7f8adc084640 1 Processor -- start 2024-03-04T07:28:29.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.065+0000 7f8adc084640 1 -- start start 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.065+0000 7f8adc084640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8ad40d3360 0x7f8ad4071c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.065+0000 7f8adc084640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8ad40ce950 con 0x7f8ad40b3020 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.065+0000 7f8adc084640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8ad4162960 con 0x7f8ad4161a50 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.065+0000 7f8adc084640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8ad40ce670 con 0x7f8ad40d3360 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.065+0000 7f8ad961f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8ad40b3020 0x7f8ad4071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47200/0 (socket says 172.21.15.64:47200) 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ad961f640 1 -- 172.21.15.64:0/1459817331 learned_addr learned my addr 172.21.15.64:0/1459817331 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ada621640 1 --2- 172.21.15.64:0/1459817331 >> v2:172.21.15.66:3300/0 conn(0x7f8ad40d3360 0x7f8ad4071c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 613773812 0 0) 0x7f8ad40ce950 con 0x7f8ad40b3020 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8ac0003520 con 0x7f8ad40b3020 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4138997135 0 0) 0x7f8ad0001110 con 0x7f8ad4161a50 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8ad40ce950 con 0x7f8ad4161a50 2024-03-04T07:28:29.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ada621640 1 -- 172.21.15.64:0/1459817331 >> v1:172.21.15.66:6789/0 conn(0x7f8ad4161a50 legacy=0x7f8ad4161e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4220013871 0 0) 0x7f8ac0003520 con 0x7f8ad40b3020 2024-03-04T07:28:29.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ada621640 1 -- 172.21.15.64:0/1459817331 >> v1:172.21.15.64:6789/0 conn(0x7f8ad40b3020 legacy=0x7f8ad4071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.066+0000 7f8ada621640 1 -- 172.21.15.64:0/1459817331 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8ad40cee50 con 0x7f8ad40d3360 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.067+0000 7f8ada621640 1 --2- 172.21.15.64:0/1459817331 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad40d3360 0x7f8ad4071c50 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f8acc009d90 tx=0x7f8acc02bd80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b36b6eead6d31629 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.067+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8acc039070 con 0x7f8ad40d3360 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.067+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8acc02c8b0 con 0x7f8ad40d3360 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.067+0000 7f8ad8e1e640 1 -- 172.21.15.64:0/1459817331 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ac8001670 con 0x7f8ad40d3360 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.068+0000 7f8adc084640 1 -- 172.21.15.64:0/1459817331 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad40d3360 msgr2=0x7f8ad4071c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.068+0000 7f8adc084640 1 --2- 172.21.15.64:0/1459817331 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad40d3360 0x7f8ad4071c50 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f8acc009d90 tx=0x7f8acc02bd80 comp rx=0 tx=0).stop 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.068+0000 7f8adc084640 1 -- 172.21.15.64:0/1459817331 shutdown_connections 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.068+0000 7f8adc084640 1 --2- 172.21.15.64:0/1459817331 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad40d3360 0x7f8ad4071c50 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.068+0000 7f8adc084640 1 -- 172.21.15.64:0/1459817331 >> 172.21.15.64:0/1459817331 conn(0x7f8ad40d3ce0 msgr2=0x7f8ad40d40f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.068+0000 7f8adc084640 1 -- 172.21.15.64:0/1459817331 shutdown_connections 2024-03-04T07:28:29.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.068+0000 7f8adc084640 1 -- 172.21.15.64:0/1459817331 wait complete. 2024-03-04T07:28:29.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 Processor -- start 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 -- start start 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8ad40b3020 0x7f8ad41500c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8ad40d3360 0x7f8ad415eef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad4161a50 0x7f8ad414f850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ad41626c0 con 0x7f8ad40d3360 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ad4162540 con 0x7f8ad4161a50 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.069+0000 7f8adc084640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8ad415e1b0 con 0x7f8ad40b3020 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8ad40d3360 0x7f8ad415eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8ad40d3360 0x7f8ad415eef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33648/0 (socket says 172.21.15.64:33648) 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 -- 172.21.15.64:0/434900946 learned_addr learned my addr 172.21.15.64:0/434900946 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad9e20640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8ad40b3020 0x7f8ad41500c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ada621640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad4161a50 0x7f8ad414f850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 -- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8ad40b3020 msgr2=0x7f8ad41500c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8ad40b3020 0x7f8ad41500c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 -- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad4161a50 msgr2=0x7f8ad414f850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad4161a50 0x7f8ad414f850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 -- 172.21.15.64:0/434900946 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8ad415f430 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ad961f640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8ad40d3360 0x7f8ad415eef0 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f8ac8000fd0 tx=0x7f8ac800cbe0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ac67fc640 1 -- 172.21.15.64:0/434900946 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ac8017070 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ac67fc640 1 -- 172.21.15.64:0/434900946 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ac8005ad0 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8ad40c9da0 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8ac67fc640 1 -- 172.21.15.64:0/434900946 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ac8007070 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.070+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8ad40b3b30 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.071+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8ad4067ff0 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.072+0000 7f8ac67fc640 1 -- 172.21.15.64:0/434900946 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8ac801a070 con 0x7f8ad40d3360 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.072+0000 7f8ac67fc640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ab806b4d0 0x7f8ab806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.072+0000 7f8ac67fc640 1 -- 172.21.15.64:0/434900946 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f8ac80892d0 con 0x7f8ad40d3360 2024-03-04T07:28:29.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.072+0000 7f8ad9e20640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ab806b4d0 0x7f8ab806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.073+0000 7f8ad9e20640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ab806b4d0 0x7f8ab806d990 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f8ad00013b0 tx=0x7f8ad0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.077+0000 7f8ac67fc640 1 -- 172.21.15.64:0/434900946 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8ac8057e90 con 0x7f8ad40d3360 2024-03-04T07:28:29.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.325+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]} v 0) v1 -- 0x7f8ad40d1cf0 con 0x7f8ad40d3360 2024-03-04T07:28:29.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.337+0000 7f8ac67fc640 1 -- 172.21.15.64:0/434900946 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]}]=0 v104) v1 ==== 136+0+0 (secure 0 0 0) 0x7f8ac805bf30 con 0x7f8ad40d3360 2024-03-04T07:28:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ab806b4d0 msgr2=0x7f8ab806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ab806b4d0 0x7f8ab806d990 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f8ad00013b0 tx=0x7f8ad0009040 comp rx=0 tx=0).stop 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8ad40d3360 msgr2=0x7f8ad415eef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8ad40d3360 0x7f8ad415eef0 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f8ac8000fd0 tx=0x7f8ac800cbe0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 shutdown_connections 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ab806b4d0 0x7f8ab806d990 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8ad4161a50 0x7f8ad414f850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8ad40d3360 0x7f8ad415eef0 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 --2- 172.21.15.64:0/434900946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8ad40b3020 0x7f8ad41500c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 >> 172.21.15.64:0/434900946 conn(0x7f8ad40d3ce0 msgr2=0x7f8ad40b8810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 shutdown_connections 2024-03-04T07:28:29.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.339+0000 7f8adc084640 1 -- 172.21.15.64:0/434900946 wait complete. 2024-03-04T07:28:29.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:28:29.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.458+0000 7fe6ba24e640 1 Processor -- start 2024-03-04T07:28:29.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.458+0000 7fe6ba24e640 1 -- start start 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6ba24e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b4830 0x7fe6b4160280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6ba24e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe6b40b29a0 0x7fe6b40b2dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6ba24e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe6b4070980 con 0x7fe6b4142170 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6ba24e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe6b41458e0 con 0x7fe6b40b29a0 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6ba24e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe6b40b3310 con 0x7fe6b40b4830 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe6b40b29a0 0x7fe6b40b2dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b37fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe6b4142170 0x7fe6b4142a20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47204/0 (socket says 172.21.15.64:47204) 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe6b40b29a0 0x7fe6b40b2dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33656/0 (socket says 172.21.15.64:33656) 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b37fe640 1 -- 172.21.15.64:0/584325477 learned_addr learned my addr 172.21.15.64:0/584325477 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:29.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b3fff640 1 -- 172.21.15.64:0/584325477 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b4830 msgr2=0x7fe6b4160280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:29.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b3fff640 1 --2- 172.21.15.64:0/584325477 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b4830 0x7fe6b4160280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b3fff640 1 -- 172.21.15.64:0/584325477 >> v1:172.21.15.64:6789/0 conn(0x7fe6b4142170 legacy=0x7fe6b4142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b3fff640 1 -- 172.21.15.64:0/584325477 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6b41607c0 con 0x7fe6b40b29a0 2024-03-04T07:28:29.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.459+0000 7fe6b27fc640 1 -- 172.21.15.64:0/584325477 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 198509103 0 0) 0x7fe6b4070980 con 0x7fe6b4142170 2024-03-04T07:28:29.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.460+0000 7fe6b3fff640 1 --2- 172.21.15.64:0/584325477 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b29a0 0x7fe6b40b2dd0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7fe6ac009dc0 tx=0x7fe6ac02be90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3d5206fd334a7ffa server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.460+0000 7fe6b27fc640 1 -- 172.21.15.64:0/584325477 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6ac039070 con 0x7fe6b40b29a0 2024-03-04T07:28:29.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.460+0000 7fe6b27fc640 1 -- 172.21.15.64:0/584325477 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6ac02cc40 con 0x7fe6b40b29a0 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.460+0000 7fe6b27fc640 1 -- 172.21.15.64:0/584325477 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6ac034440 con 0x7fe6b40b29a0 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 -- 172.21.15.64:0/584325477 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b29a0 msgr2=0x7fe6b40b2dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/584325477 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b29a0 0x7fe6b40b2dd0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7fe6ac009dc0 tx=0x7fe6ac02be90 comp rx=0 tx=0).stop 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 -- 172.21.15.64:0/584325477 shutdown_connections 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/584325477 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b29a0 0x7fe6b40b2dd0 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/584325477 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b4830 0x7fe6b4160280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 -- 172.21.15.64:0/584325477 >> 172.21.15.64:0/584325477 conn(0x7fe6b4066d00 msgr2=0x7fe6b4067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 -- 172.21.15.64:0/584325477 shutdown_connections 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.461+0000 7fe6ba24e640 1 -- 172.21.15.64:0/584325477 wait complete. 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.462+0000 7fe6ba24e640 1 Processor -- start 2024-03-04T07:28:29.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.462+0000 7fe6ba24e640 1 -- start start 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b29a0 0x7fe6b40adee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b4830 0x7fe6b40afe60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6b4142170 0x7fe6b413ebd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6b40711b0 con 0x7fe6b40b4830 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6b4070ed0 con 0x7fe6b4142170 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe6b40b4df0 con 0x7fe6b40b29a0 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b4830 0x7fe6b40afe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b4830 0x7fe6b40afe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33660/0 (socket says 172.21.15.64:33660) 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 -- 172.21.15.64:0/3253230666 learned_addr learned my addr 172.21.15.64:0/3253230666 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b37fe640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b29a0 0x7fe6b40adee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 -- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b29a0 msgr2=0x7fe6b40adee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b29a0 0x7fe6b40adee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b3fff640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6b4142170 0x7fe6b413ebd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 -- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6b4142170 msgr2=0x7fe6b413ebd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6b4142170 0x7fe6b413ebd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 -- 172.21.15.64:0/3253230666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6b40ae420 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6b2ffd640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b4830 0x7fe6b40afe60 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7fe6a400b4a0 tx=0x7fe6a400b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe69bfff640 1 -- 172.21.15.64:0/3253230666 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a4014070 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe69bfff640 1 -- 172.21.15.64:0/3253230666 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6a4010610 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe6b40b0970 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe69bfff640 1 -- 172.21.15.64:0/3253230666 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a4010a00 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.463+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe6b412aad0 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.464+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe6b40733c0 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.465+0000 7fe69bfff640 1 -- 172.21.15.64:0/3253230666 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe6a4017070 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.465+0000 7fe69bfff640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6a006b4d0 0x7fe6a006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.465+0000 7fe69bfff640 1 -- 172.21.15.64:0/3253230666 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fe6a4089920 con 0x7fe6b40b4830 2024-03-04T07:28:29.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.465+0000 7fe6b37fe640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6a006b4d0 0x7fe6a006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.466+0000 7fe6b37fe640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6a006b4d0 0x7fe6a006d990 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fe6a8000f60 tx=0x7fe6a8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.470+0000 7fe69bfff640 1 -- 172.21.15.64:0/3253230666 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe6a4058aa0 con 0x7fe6b40b4830 2024-03-04T07:28:29.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.649+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7fe6b404f410 con 0x7fe6b40b4830 2024-03-04T07:28:29.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.653+0000 7fe69bfff640 1 -- 172.21.15.64:0/3253230666 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v12)=0 v12) v1 ==== 122+0+0 (secure 0 0 0) 0x7fe6a40583a0 con 0x7fe6b40b4830 2024-03-04T07:28:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6a006b4d0 msgr2=0x7fe6a006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6a006b4d0 0x7fe6a006d990 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fe6a8000f60 tx=0x7fe6a8009040 comp rx=0 tx=0).stop 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b4830 msgr2=0x7fe6b40afe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b4830 0x7fe6b40afe60 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7fe6a400b4a0 tx=0x7fe6a400b960 comp rx=0 tx=0).stop 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 shutdown_connections 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6a006b4d0 0x7fe6a006d990 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6b4142170 0x7fe6b413ebd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6b40b4830 0x7fe6b40afe60 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 --2- 172.21.15.64:0/3253230666 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6b40b29a0 0x7fe6b40adee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 >> 172.21.15.64:0/3253230666 conn(0x7fe6b4066d00 msgr2=0x7fe6b40716d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 shutdown_connections 2024-03-04T07:28:29.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.655+0000 7fe6ba24e640 1 -- 172.21.15.64:0/3253230666 wait complete. 2024-03-04T07:28:29.666 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:28:29.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:28:29.706 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:28:29.757 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:28:29.757 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:28:29.757 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:29.772 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:28:29.773 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:28:29.773 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:29.790 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:28:29.791 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:28:29.791 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:29.827 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:28:29.827 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:28:29.827 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:29.849 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:28:29.849 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:28:29.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:29.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.968+0000 7fe0f4808640 1 Processor -- start 2024-03-04T07:28:29.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.968+0000 7fe0f4808640 1 -- start start 2024-03-04T07:28:29.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0f4808640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe0f00b95a0 0x7fe0f00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0f4808640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe0f00cea30 con 0x7fe0f00b9a10 2024-03-04T07:28:29.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0f4808640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe0f00cf3c0 con 0x7fe0f015df00 2024-03-04T07:28:29.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0f4808640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe0f00cf0e0 con 0x7fe0f00b95a0 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0e7fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe0f00b95a0 0x7fe0f00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0eed9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe0f00b9a10 0x7fe0f00d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47220/0 (socket says 172.21.15.64:47220) 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0e7fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe0f00b95a0 0x7fe0f00b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33664/0 (socket says 172.21.15.64:33664) 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0eed9d640 1 -- 172.21.15.64:0/3340321278 learned_addr learned my addr 172.21.15.64:0/3340321278 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0e7fff640 1 -- 172.21.15.64:0/3340321278 >> v1:172.21.15.66:6789/0 conn(0x7fe0f015df00 legacy=0x7fe0f00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0e7fff640 1 -- 172.21.15.64:0/3340321278 >> v1:172.21.15.64:6789/0 conn(0x7fe0f00b9a10 legacy=0x7fe0f00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.969+0000 7fe0e7fff640 1 -- 172.21.15.64:0/3340321278 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe0f00ce470 con 0x7fe0f00b95a0 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0e7fff640 1 --2- 172.21.15.64:0/3340321278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b95a0 0x7fe0f00b5700 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7fe0f00cf590 tx=0x7fe0d802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0eca3927c1ea6a4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0edd9b640 1 -- 172.21.15.64:0/3340321278 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d8034070 con 0x7fe0f00b95a0 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0edd9b640 1 -- 172.21.15.64:0/3340321278 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe0d8030610 con 0x7fe0f00b95a0 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0edd9b640 1 -- 172.21.15.64:0/3340321278 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d80309e0 con 0x7fe0f00b95a0 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0f4808640 1 -- 172.21.15.64:0/3340321278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b95a0 msgr2=0x7fe0f00b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0f4808640 1 --2- 172.21.15.64:0/3340321278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b95a0 0x7fe0f00b5700 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7fe0f00cf590 tx=0x7fe0d802b960 comp rx=0 tx=0).stop 2024-03-04T07:28:29.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0f4808640 1 -- 172.21.15.64:0/3340321278 shutdown_connections 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0f4808640 1 --2- 172.21.15.64:0/3340321278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b95a0 0x7fe0f00b5700 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.970+0000 7fe0f4808640 1 -- 172.21.15.64:0/3340321278 >> 172.21.15.64:0/3340321278 conn(0x7fe0f0071960 msgr2=0x7fe0f0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.971+0000 7fe0f4808640 1 -- 172.21.15.64:0/3340321278 shutdown_connections 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.971+0000 7fe0f4808640 1 -- 172.21.15.64:0/3340321278 wait complete. 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.972+0000 7fe0f4808640 1 Processor -- start 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.972+0000 7fe0f4808640 1 -- start start 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe0f00b95a0 0x7fe0f014b940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b9a10 0x7fe0f0149170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe0f015df00 0x7fe0f01f77d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0f00cedf0 con 0x7fe0f00b9a10 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0f00cec70 con 0x7fe0f015df00 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe0f00b5c40 con 0x7fe0f00b95a0 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b9a10 0x7fe0f0149170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b9a10 0x7fe0f0149170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33670/0 (socket says 172.21.15.64:33670) 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 -- 172.21.15.64:0/448561888 learned_addr learned my addr 172.21.15.64:0/448561888 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0eed9d640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe0f015df00 0x7fe0f01f77d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0ee59c640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe0f00b95a0 0x7fe0f014b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 -- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe0f00b95a0 msgr2=0x7fe0f014b940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe0f00b95a0 0x7fe0f014b940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 -- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe0f015df00 msgr2=0x7fe0f01f77d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe0f015df00 0x7fe0f01f77d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 -- 172.21.15.64:0/448561888 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe0f014bf20 con 0x7fe0f00b9a10 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0ee59c640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe0f00b95a0 0x7fe0f014b940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e7fff640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b9a10 0x7fe0f0149170 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7fe0d8009890 tx=0x7fe0d80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e6ffd640 1 -- 172.21.15.64:0/448561888 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d8046020 con 0x7fe0f00b9a10 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e6ffd640 1 -- 172.21.15.64:0/448561888 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe0d8039a10 con 0x7fe0f00b9a10 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe0f014c200 con 0x7fe0f00b9a10 2024-03-04T07:28:29.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0e6ffd640 1 -- 172.21.15.64:0/448561888 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d8034040 con 0x7fe0f00b9a10 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.973+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe0f014c7c0 con 0x7fe0f00b9a10 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.974+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe0f0067ff0 con 0x7fe0f00b9a10 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.975+0000 7fe0e6ffd640 1 -- 172.21.15.64:0/448561888 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe0d8037070 con 0x7fe0f00b9a10 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.975+0000 7fe0e6ffd640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe0d006b4d0 0x7fe0d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.975+0000 7fe0ee59c640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe0d006b4d0 0x7fe0d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.975+0000 7fe0e6ffd640 1 -- 172.21.15.64:0/448561888 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fe0d80abd00 con 0x7fe0f00b9a10 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.976+0000 7fe0ee59c640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe0d006b4d0 0x7fe0d006d990 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fe0e0001660 tx=0x7fe0e0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:29.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:29.980+0000 7fe0e6ffd640 1 -- 172.21.15.64:0/448561888 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe0d8050020 con 0x7fe0f00b9a10 2024-03-04T07:28:30.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.198+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe0f006b780 con 0x7fe0f00b9a10 2024-03-04T07:28:30.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.198+0000 7fe0e6ffd640 1 -- 172.21.15.64:0/448561888 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+3696 (secure 0 0 0) 0x7fe0d807af10 con 0x7fe0f00b9a10 2024-03-04T07:28:30.200 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:30.200 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":17,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":4907,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/968948062","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":968948062},{"type":"v1","addr":"172.21.15.66:6835","nonce":968948062}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5128,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2588727104","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2588727104},{"type":"v1","addr":"172.21.15.64:6837","nonce":2588727104}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5140,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1750663789","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1750663789},{"type":"v1","addr":"172.21.15.66:6834","nonce":1750663789}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":11},{"gid":5598,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/2867806702","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2867806702},{"type":"v1","addr":"172.21.15.64:6836","nonce":2867806702}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":17}],"filesystems":[]} 2024-03-04T07:28:30.200 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 17 2024-03-04T07:28:30.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe0d006b4d0 msgr2=0x7fe0d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe0d006b4d0 0x7fe0d006d990 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fe0e0001660 tx=0x7fe0e0009040 comp rx=0 tx=0).stop 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b9a10 msgr2=0x7fe0f0149170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b9a10 0x7fe0f0149170 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7fe0d8009890 tx=0x7fe0d80046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 shutdown_connections 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe0d006b4d0 0x7fe0d006d990 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe0f015df00 0x7fe0f01f77d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe0f00b9a10 0x7fe0f0149170 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.200+0000 7fe0f4808640 1 --2- 172.21.15.64:0/448561888 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe0f00b95a0 0x7fe0f014b940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.201+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 >> 172.21.15.64:0/448561888 conn(0x7fe0f0071960 msgr2=0x7fe0f00b7ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.201+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 shutdown_connections 2024-03-04T07:28:30.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.201+0000 7fe0f4808640 1 -- 172.21.15.64:0/448561888 wait complete. 2024-03-04T07:28:30.212 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:28:30.213 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:28:30.213 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:28:30.213 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:28:30.213 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:28:30.213 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:28:30.214 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:28:30.214 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:28:30.214 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:28:30.214 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:28:30.215 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:28:30.215 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:28:30.215 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:28:30.215 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:28:30.216 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:28:30.216 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:28:30.225 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:28:30.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:28:30.226 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:28:30.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:28:30.228 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:28:30.228 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:28:30.229 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:28:30.229 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:28:30.268 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:28:30.268 DEBUG:teuthology.parallel:result is None 2024-03-04T07:28:30.269 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:28:30.269 DEBUG:teuthology.parallel:result is None 2024-03-04T07:28:30.271 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:28:30.271 DEBUG:teuthology.parallel:result is None 2024-03-04T07:28:30.272 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:28:30.272 DEBUG:teuthology.parallel:result is None 2024-03-04T07:28:30.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:28:30.344 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:28:30.345 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:28:30.361 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:28:30.361 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:28:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.379+0000 7f05c5c88640 1 Processor -- start 2024-03-04T07:28:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.379+0000 7f05c5c88640 1 -- start start 2024-03-04T07:28:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05c5c88640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f05c00d02e0 0x7f05c00cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05c5c88640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f05c00d06c0 0x7f05c00b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05c5c88640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f05c00b0dd0 con 0x7f05c00b4260 2024-03-04T07:28:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05c5c88640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f05c00b0b80 con 0x7f05c00d06c0 2024-03-04T07:28:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05c5c88640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f05c00aeec0 con 0x7f05c00d02e0 2024-03-04T07:28:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f05c00d06c0 0x7f05c00b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f05c00d06c0 0x7f05c00b1c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33708/0 (socket says 172.21.15.64:33708) 2024-03-04T07:28:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bffff640 1 -- 172.21.15.64:0/2374815753 learned_addr learned my addr 172.21.15.64:0/2374815753 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bffff640 1 -- 172.21.15.64:0/2374815753 >> v2:172.21.15.66:3300/0 conn(0x7f05c00d02e0 msgr2=0x7f05c00cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bf7fe640 1 --2- 172.21.15.64:0/2374815753 >> v2:172.21.15.66:3300/0 conn(0x7f05c00d02e0 0x7f05c00cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bffff640 1 --2- 172.21.15.64:0/2374815753 >> v2:172.21.15.66:3300/0 conn(0x7f05c00d02e0 0x7f05c00cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bffff640 1 -- 172.21.15.64:0/2374815753 >> v1:172.21.15.66:6789/0 conn(0x7f05c00b4260 legacy=0x7f05c00b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.380+0000 7f05bffff640 1 -- 172.21.15.64:0/2374815753 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05c00b10b0 con 0x7f05c00d06c0 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.381+0000 7f05bffff640 1 --2- 172.21.15.64:0/2374815753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00d06c0 0x7f05c00b1c10 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f05c0071b00 tx=0x7f05b802bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2f126a0e8798c746 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.381+0000 7f05be7fc640 1 -- 172.21.15.64:0/2374815753 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05b802cb20 con 0x7f05c00d06c0 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.381+0000 7f05be7fc640 1 -- 172.21.15.64:0/2374815753 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f05b802ccc0 con 0x7f05c00d06c0 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.381+0000 7f05be7fc640 1 -- 172.21.15.64:0/2374815753 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05b8034610 con 0x7f05c00d06c0 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 -- 172.21.15.64:0/2374815753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00d06c0 msgr2=0x7f05c00b1c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2374815753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00d06c0 0x7f05c00b1c10 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f05c0071b00 tx=0x7f05b802bf80 comp rx=0 tx=0).stop 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 -- 172.21.15.64:0/2374815753 shutdown_connections 2024-03-04T07:28:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2374815753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00d06c0 0x7f05c00b1c10 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2374815753 >> v2:172.21.15.66:3300/0 conn(0x7f05c00d02e0 0x7f05c00cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 -- 172.21.15.64:0/2374815753 >> 172.21.15.64:0/2374815753 conn(0x7f05c00b36d0 msgr2=0x7f05c00b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 -- 172.21.15.64:0/2374815753 shutdown_connections 2024-03-04T07:28:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.382+0000 7f05c5c88640 1 -- 172.21.15.64:0/2374815753 wait complete. 2024-03-04T07:28:30.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.383+0000 7f05c5c88640 1 Processor -- start 2024-03-04T07:28:30.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.383+0000 7f05c5c88640 1 -- start start 2024-03-04T07:28:30.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05c5c88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00b4260 0x7f05c00bcf70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:30.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05c5c88640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05c00d02e0 0x7f05c0152650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:30.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05c5c88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05c00d06c0 0x7f05c0152b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05c5c88640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05c00b22d0 con 0x7f05c00b4260 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05c5c88640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05c00b2150 con 0x7f05c00d02e0 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05c5c88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05c00af350 con 0x7f05c00d06c0 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00b4260 0x7f05c00bcf70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00b4260 0x7f05c00bcf70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33720/0 (socket says 172.21.15.64:33720) 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 -- 172.21.15.64:0/2197253618 learned_addr learned my addr 172.21.15.64:0/2197253618 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05beffd640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05c00d02e0 0x7f05c0152650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bffff640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05c00d06c0 0x7f05c0152b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 -- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05c00d06c0 msgr2=0x7f05c0152b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05c00d06c0 0x7f05c0152b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 -- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05c00d02e0 msgr2=0x7f05c0152650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05c00d02e0 0x7f05c0152650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 -- 172.21.15.64:0/2197253618 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05c01560a0 con 0x7f05c00b4260 2024-03-04T07:28:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.384+0000 7f05bf7fe640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00b4260 0x7f05c00bcf70 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f05b400c7f0 tx=0x7f05b400ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.385+0000 7f05a7fff640 1 -- 172.21.15.64:0/2197253618 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05b4015070 con 0x7f05c00b4260 2024-03-04T07:28:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.385+0000 7f05a7fff640 1 -- 172.21.15.64:0/2197253618 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f05b4004240 con 0x7f05c00b4260 2024-03-04T07:28:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.385+0000 7f05a7fff640 1 -- 172.21.15.64:0/2197253618 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05b4007620 con 0x7f05c00b4260 2024-03-04T07:28:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.385+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f05c0156360 con 0x7f05c00b4260 2024-03-04T07:28:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.385+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f05c0156920 con 0x7f05c00b4260 2024-03-04T07:28:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.386+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f05c0068700 con 0x7f05c00b4260 2024-03-04T07:28:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.386+0000 7f05a7fff640 1 -- 172.21.15.64:0/2197253618 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f05b40079d0 con 0x7f05c00b4260 2024-03-04T07:28:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.386+0000 7f05a7fff640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f05ac06b4d0 0x7f05ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.386+0000 7f05a7fff640 1 -- 172.21.15.64:0/2197253618 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(36..36 src has 1..36) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f05b408a260 con 0x7f05c00b4260 2024-03-04T07:28:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.387+0000 7f05beffd640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f05ac06b4d0 0x7f05ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.387+0000 7f05beffd640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f05ac06b4d0 0x7f05ac06d990 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f05c015b500 tx=0x7f05b0007e30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:30.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.391+0000 7f05a7fff640 1 -- 172.21.15.64:0/2197253618 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f05b4058d90 con 0x7f05c00b4260 2024-03-04T07:28:30.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:30.586+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f05c00ccae0 con 0x7f05c00b4260 2024-03-04T07:28:31.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.346+0000 7f05a7fff640 1 -- 172.21.15.64:0/2197253618 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v38) v1 ==== 97+0+0 (secure 0 0 0) 0x7f05b408a080 con 0x7f05c00b4260 2024-03-04T07:28:31.347 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:28:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f05ac06b4d0 msgr2=0x7f05ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f05ac06b4d0 0x7f05ac06d990 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f05c015b500 tx=0x7f05b0007e30 comp rx=0 tx=0).stop 2024-03-04T07:28:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00b4260 msgr2=0x7f05c00bcf70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00b4260 0x7f05c00bcf70 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f05b400c7f0 tx=0x7f05b400ccb0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 shutdown_connections 2024-03-04T07:28:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f05ac06b4d0 0x7f05ac06d990 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05c00d06c0 0x7f05c0152b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05c00d02e0 0x7f05c0152650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 --2- 172.21.15.64:0/2197253618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05c00b4260 0x7f05c00bcf70 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.349+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 >> 172.21.15.64:0/2197253618 conn(0x7f05c00b36d0 msgr2=0x7f05c015bc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.350+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 shutdown_connections 2024-03-04T07:28:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.350+0000 7f05c5c88640 1 -- 172.21.15.64:0/2197253618 wait complete. 2024-03-04T07:28:31.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:28:31.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.466+0000 7fd62a5b5640 1 Processor -- start 2024-03-04T07:28:31.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.466+0000 7fd62a5b5640 1 -- start start 2024-03-04T07:28:31.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd62a5b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd62415e060 0x7fd6240ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd62a5b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd6240d0dd0 con 0x7fd6240d0880 2024-03-04T07:28:31.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd62a5b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd6240d10b0 con 0x7fd62406fe20 2024-03-04T07:28:31.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd62a5b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6240cc0e0 con 0x7fd62415e060 2024-03-04T07:28:31.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd628b52640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd6240d0880 0x7fd6240ccbc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47242/0 (socket says 172.21.15.64:47242) 2024-03-04T07:28:31.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd628b52640 1 -- 172.21.15.64:0/1880070931 learned_addr learned my addr 172.21.15.64:0/1880070931 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:31.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd623fff640 1 --2- 172.21.15.64:0/1880070931 >> v2:172.21.15.66:3300/0 conn(0x7fd62415e060 0x7fd6240ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1251798089 0 0) 0x7fd6240d0dd0 con 0x7fd6240d0880 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd60c003520 con 0x7fd6240d0880 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2214920660 0 0) 0x7fd60c003520 con 0x7fd6240d0880 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 >> v2:172.21.15.66:3300/0 conn(0x7fd62415e060 msgr2=0x7fd6240ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 --2- 172.21.15.64:0/1880070931 >> v2:172.21.15.66:3300/0 conn(0x7fd62415e060 0x7fd6240ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 >> v1:172.21.15.66:6789/0 conn(0x7fd62406fe20 legacy=0x7fd6240ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6240cc3c0 con 0x7fd6240d0880 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd61c0019f0 con 0x7fd6240d0880 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 >> v1:172.21.15.64:6789/0 conn(0x7fd6240d0880 legacy=0x7fd6240ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd60c0039b0 0x7fd60c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd60c004800 0x7fd60c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c0051c0 0x7fd60c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6240cc0e0 con 0x7fd60c004800 2024-03-04T07:28:31.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd60c00c060 con 0x7fd60c0039b0 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd60c00c260 con 0x7fd60c0051c0 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.467+0000 7fd6237fe640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd60c004800 0x7fd60c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd623fff640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd60c0039b0 0x7fd60c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd628b52640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c0051c0 0x7fd60c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd6237fe640 1 -- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c0051c0 msgr2=0x7fd60c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd6237fe640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c0051c0 0x7fd60c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd6237fe640 1 -- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd60c0039b0 msgr2=0x7fd60c003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd6237fe640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd60c0039b0 0x7fd60c003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd6237fe640 1 -- 172.21.15.64:0/1880070931 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd60c003520 con 0x7fd60c004800 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd628b52640 1 -- 172.21.15.64:0/1880070931 reap_dead start 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.468+0000 7fd6237fe640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd60c004800 0x7fd60c004c80 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7fd618001720 tx=0x7fd60c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8b7bebf03b905dfe server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd618001370 con 0x7fd60c004800 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd6180052d0 con 0x7fd60c004800 2024-03-04T07:28:31.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd622ffd640 1 -- 172.21.15.64:0/1880070931 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd618003830 con 0x7fd60c004800 2024-03-04T07:28:31.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd62a5b5640 1 -- 172.21.15.64:0/1880070931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd60c004800 msgr2=0x7fd60c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd60c004800 0x7fd60c004c80 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7fd618001720 tx=0x7fd60c005830 comp rx=0 tx=0).stop 2024-03-04T07:28:31.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd62a5b5640 1 -- 172.21.15.64:0/1880070931 shutdown_connections 2024-03-04T07:28:31.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/1880070931 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd60c004800 0x7fd60c004c80 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd62a5b5640 1 -- 172.21.15.64:0/1880070931 >> 172.21.15.64:0/1880070931 conn(0x7fd6240d1a90 msgr2=0x7fd6240d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:31.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd62a5b5640 1 -- 172.21.15.64:0/1880070931 shutdown_connections 2024-03-04T07:28:31.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.469+0000 7fd62a5b5640 1 -- 172.21.15.64:0/1880070931 wait complete. 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.470+0000 7fd62a5b5640 1 Processor -- start 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.470+0000 7fd62a5b5640 1 -- start start 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd62a5b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c004800 0x7fd6240c6c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd62a5b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240c7460 0x7fd62414ef00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd62a5b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6240c7df0 0x7fd6240ca360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd62a5b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6240ce310 con 0x7fd6240c7460 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd62a5b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd62415e440 con 0x7fd6240c7df0 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd62a5b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd6240ce490 con 0x7fd60c004800 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd6237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240c7460 0x7fd62414ef00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd6237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240c7460 0x7fd62414ef00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33742/0 (socket says 172.21.15.64:33742) 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.471+0000 7fd6237fe640 1 -- 172.21.15.64:0/121462622 learned_addr learned my addr 172.21.15.64:0/121462622 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd628b52640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6240c7df0 0x7fd6240ca360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd6237fe640 1 -- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c004800 msgr2=0x7fd6240c6c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd623fff640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c004800 0x7fd6240c6c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd6237fe640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c004800 0x7fd6240c6c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd6237fe640 1 -- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6240c7df0 msgr2=0x7fd6240ca360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd6237fe640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6240c7df0 0x7fd6240ca360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd6237fe640 1 -- 172.21.15.64:0/121462622 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd62414f440 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd6237fe640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240c7460 0x7fd62414ef00 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7fd618000c00 tx=0x7fd618001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd620ff9640 1 -- 172.21.15.64:0/121462622 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd618041020 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd620ff9640 1 -- 172.21.15.64:0/121462622 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd618005480 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd6240c78f0 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd620ff9640 1 -- 172.21.15.64:0/121462622 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd618007a30 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.472+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6240c6430 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.473+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd624067ff0 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.473+0000 7fd620ff9640 1 -- 172.21.15.64:0/121462622 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd618044020 con 0x7fd6240c7460 2024-03-04T07:28:31.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.474+0000 7fd620ff9640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd61006b4d0 0x7fd61006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.474+0000 7fd620ff9640 1 -- 172.21.15.64:0/121462622 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fd6180aba30 con 0x7fd6240c7460 2024-03-04T07:28:31.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.474+0000 7fd623fff640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd61006b4d0 0x7fd61006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.474+0000 7fd623fff640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd61006b4d0 0x7fd61006d990 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7fd614009a00 tx=0x7fd614008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:31.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.478+0000 7fd620ff9640 1 -- 172.21.15.64:0/121462622 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd61804b030 con 0x7fd6240c7460 2024-03-04T07:28:31.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.727+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7fd6240c7bd0 con 0x7fd6240c7460 2024-03-04T07:28:31.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.727+0000 7fd620ff9640 1 -- 172.21.15.64:0/121462622 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v15) v1 ==== 80+0+5304 (secure 0 0 0) 0x7fd62414f440 con 0x7fd6240c7460 2024-03-04T07:28:31.728 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:31.728 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:28:31.728 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:28:31.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.728 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:28:31.728 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:28:31.729 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.731 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.733 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.734 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:28:31.736 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow r", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:28:31.737 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:28:31.738 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:28:31.739 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:28:31.740 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:28:31.741 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd61006b4d0 msgr2=0x7fd61006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd61006b4d0 0x7fd61006d990 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7fd614009a00 tx=0x7fd614008040 comp rx=0 tx=0).stop 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240c7460 msgr2=0x7fd62414ef00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240c7460 0x7fd62414ef00 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7fd618000c00 tx=0x7fd618001120 comp rx=0 tx=0).stop 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 shutdown_connections 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd61006b4d0 0x7fd61006d990 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6240c7df0 0x7fd6240ca360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240c7460 0x7fd62414ef00 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 --2- 172.21.15.64:0/121462622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd60c004800 0x7fd6240c6c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 >> 172.21.15.64:0/121462622 conn(0x7fd6240d1a90 msgr2=0x7fd62406eb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 shutdown_connections 2024-03-04T07:28:31.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.729+0000 7fd62a5b5640 1 -- 172.21.15.64:0/121462622 wait complete. 2024-03-04T07:28:31.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:31.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.845+0000 7fb899b79640 1 Processor -- start 2024-03-04T07:28:31.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.845+0000 7fb899b79640 1 -- start start 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.845+0000 7fb899b79640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb89415bac0 0x7fb89415cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.845+0000 7fb899b79640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb8940729b0 con 0x7fb8940aee10 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.845+0000 7fb899b79640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb89406c920 con 0x7fb89406d400 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.845+0000 7fb899b79640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb89406cf70 con 0x7fb89415bac0 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb893fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb8940aee10 0x7fb894070870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47256/0 (socket says 172.21.15.64:47256) 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb893fff640 1 -- 172.21.15.64:0/4141303160 learned_addr learned my addr 172.21.15.64:0/4141303160 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 194143483 0 0) 0x7fb8940729b0 con 0x7fb8940aee10 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb87c003520 con 0x7fb8940aee10 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8937fe640 1 --2- 172.21.15.64:0/4141303160 >> v2:172.21.15.66:3300/0 conn(0x7fb89415bac0 0x7fb89415cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3317592627 0 0) 0x7fb87c003520 con 0x7fb8940aee10 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 >> v2:172.21.15.66:3300/0 conn(0x7fb89415bac0 msgr2=0x7fb89415cbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 --2- 172.21.15.64:0/4141303160 >> v2:172.21.15.66:3300/0 conn(0x7fb89415bac0 0x7fb89415cbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 >> v1:172.21.15.66:6789/0 conn(0x7fb89406d400 legacy=0x7fb8940ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb89406cc00 con 0x7fb8940aee10 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fb88c0019f0 con 0x7fb8940aee10 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 >> v1:172.21.15.64:6789/0 conn(0x7fb8940aee10 legacy=0x7fb894070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb87c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb87c004800 0x7fb87c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb87c0051c0 0x7fb87c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb89406cf70 con 0x7fb87c0039b0 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb87c00c110 con 0x7fb87c0051c0 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb87c00c310 con 0x7fb87c004800 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8937fe640 1 --2- 172.21.15.64:0/4141303160 >> v2:172.21.15.66:3300/0 conn(0x7fb89415bac0 0x7fb89415cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb8937fe640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb87c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb893fff640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb87c0051c0 0x7fb87c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.846+0000 7fb892ffd640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb87c004800 0x7fb87c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8937fe640 1 -- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb87c004800 msgr2=0x7fb87c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8937fe640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb87c004800 0x7fb87c004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8937fe640 1 -- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb87c0051c0 msgr2=0x7fb87c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8937fe640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb87c0051c0 0x7fb87c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8937fe640 1 -- 172.21.15.64:0/4141303160 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb87c003520 con 0x7fb87c0039b0 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb893fff640 1 -- 172.21.15.64:0/4141303160 reap_dead start 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb892ffd640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb87c004800 0x7fb87c004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8937fe640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb87c003dd0 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fb8840098a0 tx=0x7fb87c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d2e84e39573171f3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb884034070 con 0x7fb87c0039b0 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb884030630 con 0x7fb87c0039b0 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.847+0000 7fb8927fc640 1 -- 172.21.15.64:0/4141303160 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb88400f040 con 0x7fb87c0039b0 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.848+0000 7fb899b79640 1 -- 172.21.15.64:0/4141303160 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 msgr2=0x7fb87c003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.848+0000 7fb899b79640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb87c003dd0 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fb8840098a0 tx=0x7fb87c005830 comp rx=0 tx=0).stop 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.848+0000 7fb899b79640 1 -- 172.21.15.64:0/4141303160 shutdown_connections 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.848+0000 7fb899b79640 1 --2- 172.21.15.64:0/4141303160 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb87c003dd0 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.848+0000 7fb899b79640 1 -- 172.21.15.64:0/4141303160 >> 172.21.15.64:0/4141303160 conn(0x7fb894066d00 msgr2=0x7fb8940670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:31.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.848+0000 7fb899b79640 1 -- 172.21.15.64:0/4141303160 shutdown_connections 2024-03-04T07:28:31.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.848+0000 7fb899b79640 1 -- 172.21.15.64:0/4141303160 wait complete. 2024-03-04T07:28:31.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.849+0000 7fb899b79640 1 Processor -- start 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.849+0000 7fb899b79640 1 -- start start 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb899b79640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb8941f1230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb899b79640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb894154770 0x7fb894159da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb899b79640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb894154bf0 0x7fb89415a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb899b79640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb89415c020 con 0x7fb87c0039b0 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb899b79640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb89415bea0 con 0x7fb894154770 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb899b79640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb894072580 con 0x7fb894154bf0 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb8941f1230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb8941f1230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33760/0 (socket says 172.21.15.64:33760) 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 -- 172.21.15.64:0/541795681 learned_addr learned my addr 172.21.15.64:0/541795681 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb892ffd640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb894154770 0x7fb894159da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 -- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb894154bf0 msgr2=0x7fb89415a2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb894154bf0 0x7fb89415a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 -- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb894154770 msgr2=0x7fb894159da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb894154770 0x7fb894159da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb8937fe640 1 -- 172.21.15.64:0/541795681 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb894152a90 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.850+0000 7fb892ffd640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb894154770 0x7fb894159da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.851+0000 7fb8937fe640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb8941f1230 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7fb884009990 tx=0x7fb884004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.851+0000 7fb87bfff640 1 -- 172.21.15.64:0/541795681 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb884034040 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.851+0000 7fb87bfff640 1 -- 172.21.15.64:0/541795681 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb88400f000 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.851+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb894152d70 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.851+0000 7fb87bfff640 1 -- 172.21.15.64:0/541795681 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb88403a9d0 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.851+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb894153330 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.852+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb894067ff0 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.852+0000 7fb87bfff640 1 -- 172.21.15.64:0/541795681 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb884037070 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.853+0000 7fb87bfff640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb88006b4d0 0x7fb88006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.853+0000 7fb87bfff640 1 -- 172.21.15.64:0/541795681 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fb8840abe20 con 0x7fb87c0039b0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.853+0000 7fb892ffd640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb88006b4d0 0x7fb88006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:31.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.853+0000 7fb892ffd640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb88006b4d0 0x7fb88006d990 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fb894156a90 tx=0x7fb888009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:31.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:31.857+0000 7fb87bfff640 1 -- 172.21.15.64:0/541795681 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb88407af60 con 0x7fb87c0039b0 2024-03-04T07:28:32.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.038+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fb89406de50 con 0x7fb87c0039b0 2024-03-04T07:28:32.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.039+0000 7fb87bfff640 1 -- 172.21.15.64:0/541795681 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v40) v1 ==== 74+0+12843 (secure 0 0 0) 0x7fb88407a860 con 0x7fb87c0039b0 2024-03-04T07:28:32.040 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:32.040 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":40,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:31.355422+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb88006b4d0 msgr2=0x7fb88006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb88006b4d0 0x7fb88006d990 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fb894156a90 tx=0x7fb888009040 comp rx=0 tx=0).stop 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 msgr2=0x7fb8941f1230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb8941f1230 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7fb884009990 tx=0x7fb884004730 comp rx=0 tx=0).stop 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 shutdown_connections 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb88006b4d0 0x7fb88006d990 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb894154bf0 0x7fb89415a2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb894154770 0x7fb894159da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 --2- 172.21.15.64:0/541795681 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb87c0039b0 0x7fb8941f1230 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 >> 172.21.15.64:0/541795681 conn(0x7fb894066d00 msgr2=0x7fb8940ced90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 shutdown_connections 2024-03-04T07:28:32.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.041+0000 7fb899b79640 1 -- 172.21.15.64:0/541795681 wait complete. 2024-03-04T07:28:32.053 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:28:32.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:28:32.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.157+0000 7fba2fb25640 1 Processor -- start 2024-03-04T07:28:32.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.157+0000 7fba2fb25640 1 -- start start 2024-03-04T07:28:32.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2fb25640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba2815dd90 0x7fba280ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2fb25640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fba280d3740 con 0x7fba280d29d0 2024-03-04T07:28:32.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2fb25640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fba280d3120 con 0x7fba280b6d10 2024-03-04T07:28:32.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2fb25640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fba280d3400 con 0x7fba2815dd90 2024-03-04T07:28:32.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2d8c1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba2815dd90 0x7fba280ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2d8c1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba2815dd90 0x7fba280ceea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33774/0 (socket says 172.21.15.64:33774) 2024-03-04T07:28:32.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2d8c1640 1 -- 172.21.15.64:0/1336593870 learned_addr learned my addr 172.21.15.64:0/1336593870 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:32.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.158+0000 7fba2d0c0640 1 --1- 172.21.15.64:0/1336593870 >> v1:172.21.15.64:6789/0 conn(0x7fba280d29d0 0x7fba280d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47264/0 (socket says 172.21.15.64:47264) 2024-03-04T07:28:32.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.159+0000 7fba2d8c1640 1 -- 172.21.15.64:0/1336593870 >> v1:172.21.15.66:6789/0 conn(0x7fba280b6d10 legacy=0x7fba280d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:32.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.159+0000 7fba2d8c1640 1 -- 172.21.15.64:0/1336593870 >> v1:172.21.15.64:6789/0 conn(0x7fba280d29d0 legacy=0x7fba280d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.159+0000 7fba2d8c1640 1 -- 172.21.15.64:0/1336593870 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba280d39e0 con 0x7fba2815dd90 2024-03-04T07:28:32.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.159+0000 7fba2d8c1640 1 --2- 172.21.15.64:0/1336593870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280ceea0 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fba20009870 tx=0x7fba2002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2afdb0223ea190e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2c8bf640 1 -- 172.21.15.64:0/1336593870 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba2002f840 con 0x7fba2815dd90 2024-03-04T07:28:32.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2c8bf640 1 -- 172.21.15.64:0/1336593870 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba2002f9e0 con 0x7fba2815dd90 2024-03-04T07:28:32.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2c8bf640 1 -- 172.21.15.64:0/1336593870 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba2002fdb0 con 0x7fba2815dd90 2024-03-04T07:28:32.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2fb25640 1 -- 172.21.15.64:0/1336593870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 msgr2=0x7fba280ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2fb25640 1 --2- 172.21.15.64:0/1336593870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280ceea0 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fba20009870 tx=0x7fba2002ba20 comp rx=0 tx=0).stop 2024-03-04T07:28:32.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2fb25640 1 -- 172.21.15.64:0/1336593870 shutdown_connections 2024-03-04T07:28:32.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2fb25640 1 --2- 172.21.15.64:0/1336593870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280ceea0 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2fb25640 1 -- 172.21.15.64:0/1336593870 >> 172.21.15.64:0/1336593870 conn(0x7fba28071960 msgr2=0x7fba28071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:32.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2fb25640 1 -- 172.21.15.64:0/1336593870 shutdown_connections 2024-03-04T07:28:32.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.160+0000 7fba2fb25640 1 -- 172.21.15.64:0/1336593870 wait complete. 2024-03-04T07:28:32.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.161+0000 7fba2fb25640 1 Processor -- start 2024-03-04T07:28:32.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.161+0000 7fba2fb25640 1 -- start start 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2fb25640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba280b6d10 0x7fba28159590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2fb25640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba280d29d0 0x7fba281e7760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2fb25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280c94e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2fb25640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba280d2df0 con 0x7fba2815dd90 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2fb25640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba2815e170 con 0x7fba280b6d10 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2fb25640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fba280d48f0 con 0x7fba280d29d0 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2e0c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280c94e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2e0c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280c94e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33776/0 (socket says 172.21.15.64:33776) 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2e0c2640 1 -- 172.21.15.64:0/3164527217 learned_addr learned my addr 172.21.15.64:0/3164527217 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2d8c1640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba280b6d10 0x7fba28159590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.162+0000 7fba2d0c0640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba280d29d0 0x7fba281e7760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2e0c2640 1 -- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba280d29d0 msgr2=0x7fba281e7760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2e0c2640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba280d29d0 0x7fba281e7760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2e0c2640 1 -- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba280b6d10 msgr2=0x7fba28159590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2e0c2640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba280b6d10 0x7fba28159590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2e0c2640 1 -- 172.21.15.64:0/3164527217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba281e9250 con 0x7fba2815dd90 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2d8c1640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba280b6d10 0x7fba28159590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2e0c2640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280c94e0 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fba2400c780 tx=0x7fba2400cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba1e7fc640 1 -- 172.21.15.64:0/3164527217 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba24004b60 con 0x7fba2815dd90 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba1e7fc640 1 -- 172.21.15.64:0/3164527217 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba24004d00 con 0x7fba2815dd90 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fba281e81b0 con 0x7fba2815dd90 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba1e7fc640 1 -- 172.21.15.64:0/3164527217 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba2400aa90 con 0x7fba2815dd90 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.163+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fba281e8770 con 0x7fba2815dd90 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.164+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fba28072a30 con 0x7fba2815dd90 2024-03-04T07:28:32.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.164+0000 7fba1e7fc640 1 -- 172.21.15.64:0/3164527217 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fba2400ac30 con 0x7fba2815dd90 2024-03-04T07:28:32.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.165+0000 7fba1e7fc640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba0c06b4d0 0x7fba0c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.165+0000 7fba1e7fc640 1 -- 172.21.15.64:0/3164527217 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fba2408a080 con 0x7fba2815dd90 2024-03-04T07:28:32.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.165+0000 7fba2d8c1640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba0c06b4d0 0x7fba0c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.165+0000 7fba2d8c1640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba0c06b4d0 0x7fba0c06d990 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fba20036000 tx=0x7fba20008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.169+0000 7fba1e7fc640 1 -- 172.21.15.64:0/3164527217 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fba24020070 con 0x7fba2815dd90 2024-03-04T07:28:32.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.351+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fba28068ac0 con 0x7fba2815dd90 2024-03-04T07:28:32.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.351+0000 7fba1e7fc640 1 -- 172.21.15.64:0/3164527217 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v40) v1 ==== 93+0+10 (secure 0 0 0) 0x7fba240592e0 con 0x7fba2815dd90 2024-03-04T07:28:32.352 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:28:32.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba0c06b4d0 msgr2=0x7fba0c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba0c06b4d0 0x7fba0c06d990 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fba20036000 tx=0x7fba20008030 comp rx=0 tx=0).stop 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 msgr2=0x7fba280c94e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280c94e0 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fba2400c780 tx=0x7fba2400cc40 comp rx=0 tx=0).stop 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 shutdown_connections 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba0c06b4d0 0x7fba0c06d990 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba2815dd90 0x7fba280c94e0 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba280d29d0 0x7fba281e7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 --2- 172.21.15.64:0/3164527217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba280b6d10 0x7fba28159590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 >> 172.21.15.64:0/3164527217 conn(0x7fba28071960 msgr2=0x7fba2815c0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 shutdown_connections 2024-03-04T07:28:32.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.353+0000 7fba2fb25640 1 -- 172.21.15.64:0/3164527217 wait complete. 2024-03-04T07:28:32.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:28:32.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.465+0000 7f6869c5c640 1 Processor -- start 2024-03-04T07:28:32.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.465+0000 7f6869c5c640 1 -- start start 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.466+0000 7f6869c5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68640d3400 0x7f68640b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.466+0000 7f6869c5c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f68640b2b50 0x7f68640b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.466+0000 7f6869c5c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f68640ce610 con 0x7f6864071540 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.466+0000 7f6869c5c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f68640d2f70 con 0x7f68640b2b50 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.466+0000 7f6869c5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f68640d4be0 con 0x7f68640d3400 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f6863fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f68640b2b50 0x7f68640b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f6862ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68640d3400 0x7f68640b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f68637fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f6864071540 0x7f6864071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:33046/0 (socket says 172.21.15.64:33046) 2024-03-04T07:28:32.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f68637fe640 1 -- 172.21.15.64:0/1602492972 learned_addr learned my addr 172.21.15.64:0/1602492972 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:32.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f6863fff640 1 -- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68640d3400 msgr2=0x7f68640b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:32.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f6863fff640 1 --2- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68640d3400 0x7f68640b30a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f6863fff640 1 -- 172.21.15.64:0/1602492972 >> v1:172.21.15.66:6789/0 conn(0x7f6864071540 legacy=0x7f6864071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f6863fff640 1 -- 172.21.15.64:0/1602492972 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68640d2920 con 0x7f68640b2b50 2024-03-04T07:28:32.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.467+0000 7f6862ffd640 1 --2- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68640d3400 0x7f68640b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f6863fff640 1 --2- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f68640b6e20 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f685c0097a0 tx=0x7f685c02b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7d63978ab557b80b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f68627fc640 1 -- 172.21.15.64:0/1602492972 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c035020 con 0x7f68640b2b50 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f68627fc640 1 -- 172.21.15.64:0/1602492972 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f685c031810 con 0x7f68640b2b50 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f68627fc640 1 -- 172.21.15.64:0/1602492972 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c031be0 con 0x7f68640b2b50 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f6869c5c640 1 -- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 msgr2=0x7f68640b6e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f6869c5c640 1 --2- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f68640b6e20 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f685c0097a0 tx=0x7f685c02b8b0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f6869c5c640 1 -- 172.21.15.64:0/1602492972 shutdown_connections 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f6869c5c640 1 --2- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f68640b6e20 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.468+0000 7f6869c5c640 1 --2- 172.21.15.64:0/1602492972 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68640d3400 0x7f68640b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.469+0000 7f6869c5c640 1 -- 172.21.15.64:0/1602492972 >> 172.21.15.64:0/1602492972 conn(0x7f68640d3ca0 msgr2=0x7f68640d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.469+0000 7f6869c5c640 1 -- 172.21.15.64:0/1602492972 shutdown_connections 2024-03-04T07:28:32.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.469+0000 7f6869c5c640 1 -- 172.21.15.64:0/1602492972 wait complete. 2024-03-04T07:28:32.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.470+0000 7f6869c5c640 1 Processor -- start 2024-03-04T07:28:32.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.470+0000 7f6869c5c640 1 -- start start 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6869c5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6864071540 0x7f68640b58d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6869c5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f686414d140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6869c5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68640d3400 0x7f68641f3f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6869c5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f686415e370 con 0x7f68640d3400 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6869c5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68640ce8f0 con 0x7f68640b2b50 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6869c5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f68640cbb20 con 0x7f6864071540 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68640d3400 0x7f68641f3f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68640d3400 0x7f68641f3f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33784/0 (socket says 172.21.15.64:33784) 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 -- 172.21.15.64:0/4106258712 learned_addr learned my addr 172.21.15.64:0/4106258712 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6862ffd640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f686414d140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f68637fe640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6864071540 0x7f68640b58d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 -- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6864071540 msgr2=0x7f68640b58d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6864071540 0x7f68640b58d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 -- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 msgr2=0x7f686414d140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f686414d140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 -- 172.21.15.64:0/4106258712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68640caf10 con 0x7f68640d3400 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6862ffd640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f686414d140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f68637fe640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6864071540 0x7f68640b58d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6863fff640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68640d3400 0x7f68641f3f90 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f685c02b8f0 tx=0x7f685c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f684bfff640 1 -- 172.21.15.64:0/4106258712 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c041070 con 0x7f68640d3400 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f684bfff640 1 -- 172.21.15.64:0/4106258712 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f685c03ad00 con 0x7f68640d3400 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f68640cb1f0 con 0x7f68640d3400 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.471+0000 7f684bfff640 1 -- 172.21.15.64:0/4106258712 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f685c035040 con 0x7f68640d3400 2024-03-04T07:28:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.472+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f68640cb7b0 con 0x7f68640d3400 2024-03-04T07:28:32.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.472+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6864068700 con 0x7f68640d3400 2024-03-04T07:28:32.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.473+0000 7f684bfff640 1 -- 172.21.15.64:0/4106258712 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f685c038020 con 0x7f68640d3400 2024-03-04T07:28:32.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.473+0000 7f684bfff640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685006b4d0 0x7f685006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.473+0000 7f684bfff640 1 -- 172.21.15.64:0/4106258712 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f685c0abb00 con 0x7f68640d3400 2024-03-04T07:28:32.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.473+0000 7f68637fe640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685006b4d0 0x7f685006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.474+0000 7f68637fe640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685006b4d0 0x7f685006d990 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f68540012e0 tx=0x7f6854009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.478+0000 7f684bfff640 1 -- 172.21.15.64:0/4106258712 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f685c07ac40 con 0x7f68640d3400 2024-03-04T07:28:32.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.658+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f686415e960 con 0x7f68640d3400 2024-03-04T07:28:32.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.659+0000 7f684bfff640 1 -- 172.21.15.64:0/4106258712 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v40) v1 ==== 84+0+65 (secure 0 0 0) 0x7f685c07a540 con 0x7f68640d3400 2024-03-04T07:28:32.659 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:32.659 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:28:32.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:28:32.660 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:28:32.660 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:28:32.660 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:32.660 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:28:32.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685006b4d0 msgr2=0x7f685006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685006b4d0 0x7f685006d990 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f68540012e0 tx=0x7f6854009040 comp rx=0 tx=0).stop 2024-03-04T07:28:32.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68640d3400 msgr2=0x7f68641f3f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68640d3400 0x7f68641f3f90 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f685c02b8f0 tx=0x7f685c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 shutdown_connections 2024-03-04T07:28:32.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f685006b4d0 0x7f685006d990 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68640d3400 0x7f68641f3f90 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68640b2b50 0x7f686414d140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 --2- 172.21.15.64:0/4106258712 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6864071540 0x7f68640b58d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.660+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 >> 172.21.15.64:0/4106258712 conn(0x7f68640d3ca0 msgr2=0x7f686415f270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:32.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.661+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 shutdown_connections 2024-03-04T07:28:32.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.661+0000 7f6869c5c640 1 -- 172.21.15.64:0/4106258712 wait complete. 2024-03-04T07:28:32.672 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:28:32.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:28:32.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.777+0000 7f5ab3b3e640 1 Processor -- start 2024-03-04T07:28:32.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.777+0000 7f5ab3b3e640 1 -- start start 2024-03-04T07:28:32.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab3b3e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15b230 0x7f5aac071160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab3b3e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5aac15b690 0x7f5aac0716a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab3b3e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5aac0b2e30 con 0x7f5aac15ae50 2024-03-04T07:28:32.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab3b3e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5aac06dfe0 con 0x7f5aac15b690 2024-03-04T07:28:32.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab3b3e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5aac0b30b0 con 0x7f5aac15b230 2024-03-04T07:28:32.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab20db640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5aac15b690 0x7f5aac0716a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab18da640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5aac15ae50 0x7f5aac0708e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47268/0 (socket says 172.21.15.64:47268) 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab18da640 1 -- 172.21.15.64:0/672872226 learned_addr learned my addr 172.21.15.64:0/672872226 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab10d9640 1 --2- 172.21.15.64:0/672872226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15b230 0x7f5aac071160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab20db640 1 -- 172.21.15.64:0/672872226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15b230 msgr2=0x7f5aac071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab20db640 1 --2- 172.21.15.64:0/672872226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15b230 0x7f5aac071160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab20db640 1 -- 172.21.15.64:0/672872226 >> v1:172.21.15.64:6789/0 conn(0x7f5aac15ae50 legacy=0x7f5aac0708e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab08d8640 1 -- 172.21.15.64:0/672872226 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2698209847 0 0) 0x7f5aa4001110 con 0x7f5aac15ae50 2024-03-04T07:28:32.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.778+0000 7f5ab20db640 1 -- 172.21.15.64:0/672872226 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5aac0b3390 con 0x7f5aac15b690 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.779+0000 7f5ab20db640 1 --2- 172.21.15.64:0/672872226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac0716a0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f5a9c009d90 tx=0x7f5a9c02be10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dab17ed10b370c3a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.779+0000 7f5ab08d8640 1 -- 172.21.15.64:0/672872226 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a9c039070 con 0x7f5aac15b690 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.779+0000 7f5ab08d8640 1 -- 172.21.15.64:0/672872226 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a9c02cbc0 con 0x7f5aac15b690 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.779+0000 7f5ab08d8640 1 -- 172.21.15.64:0/672872226 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a9c034440 con 0x7f5aac15b690 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/672872226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 msgr2=0x7f5aac0716a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/672872226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac0716a0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f5a9c009d90 tx=0x7f5a9c02be10 comp rx=0 tx=0).stop 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/672872226 shutdown_connections 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/672872226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac0716a0 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/672872226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15b230 0x7f5aac071160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/672872226 >> 172.21.15.64:0/672872226 conn(0x7f5aac066d00 msgr2=0x7f5aac067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:32.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/672872226 shutdown_connections 2024-03-04T07:28:32.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.780+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/672872226 wait complete. 2024-03-04T07:28:32.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.781+0000 7f5ab3b3e640 1 Processor -- start 2024-03-04T07:28:32.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.781+0000 7f5ab3b3e640 1 -- start start 2024-03-04T07:28:32.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab3b3e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15ae50 0x7f5aac148c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab3b3e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5aac15b230 0x7f5aac149140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab3b3e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac137d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab3b3e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5aac0b4150 con 0x7f5aac15b690 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab3b3e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5aac0b3950 con 0x7f5aac15b230 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab3b3e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5aac150a00 con 0x7f5aac15ae50 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac137d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac137d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33798/0 (socket says 172.21.15.64:33798) 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 -- 172.21.15.64:0/1168971363 learned_addr learned my addr 172.21.15.64:0/1168971363 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 -- 172.21.15.64:0/1168971363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15ae50 msgr2=0x7f5aac148c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15ae50 0x7f5aac148c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 -- 172.21.15.64:0/1168971363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5aac15b230 msgr2=0x7f5aac149140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5aac15b230 0x7f5aac149140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 -- 172.21.15.64:0/1168971363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5aac1f22e0 con 0x7f5aac15b690 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.782+0000 7f5ab20db640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac137d10 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f5a9c009eb0 tx=0x7f5a9c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.783+0000 7f5aaa7fc640 1 -- 172.21.15.64:0/1168971363 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a9c039050 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.783+0000 7f5aaa7fc640 1 -- 172.21.15.64:0/1168971363 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a9c045070 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.783+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5aac1f25c0 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.783+0000 7f5aaa7fc640 1 -- 172.21.15.64:0/1168971363 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a9c03e850 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.783+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5aac1f2b80 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.784+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5aac0733c0 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.784+0000 7f5aaa7fc640 1 -- 172.21.15.64:0/1168971363 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5a9c03e9f0 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.784+0000 7f5aaa7fc640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a9806b500 0x7f5a9806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.785+0000 7f5aaa7fc640 1 -- 172.21.15.64:0/1168971363 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f5a9c0acf70 con 0x7f5aac15b690 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.785+0000 7f5ab18da640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a9806b500 0x7f5a9806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.785+0000 7f5ab18da640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a9806b500 0x7f5a9806d9c0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f5aa4001620 tx=0x7f5aa4001310 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:32.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.789+0000 7f5aaa7fc640 1 -- 172.21.15.64:0/1168971363 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5a9c07c260 con 0x7f5aac15b690 2024-03-04T07:28:32.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:32.969+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f5aac06cdb0 con 0x7f5aac15b690 2024-03-04T07:28:33.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.352+0000 7f5aaa7fc640 1 -- 172.21.15.64:0/1168971363 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v41) v1 ==== 138+0+0 (secure 0 0 0) 0x7f5a9c07bb60 con 0x7f5aac15b690 2024-03-04T07:28:33.353 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:28:33.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a9806b500 msgr2=0x7f5a9806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:33.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a9806b500 0x7f5a9806d9c0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f5aa4001620 tx=0x7f5aa4001310 comp rx=0 tx=0).stop 2024-03-04T07:28:33.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 msgr2=0x7f5aac137d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:33.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac137d10 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f5a9c009eb0 tx=0x7f5a9c004a40 comp rx=0 tx=0).stop 2024-03-04T07:28:33.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 shutdown_connections 2024-03-04T07:28:33.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a9806b500 0x7f5a9806d9c0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5aac15b690 0x7f5aac137d10 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5aac15b230 0x7f5aac149140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 --2- 172.21.15.64:0/1168971363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5aac15ae50 0x7f5aac148c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 >> 172.21.15.64:0/1168971363 conn(0x7f5aac066d00 msgr2=0x7f5aac14ef90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:33.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 shutdown_connections 2024-03-04T07:28:33.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.354+0000 7f5ab3b3e640 1 -- 172.21.15.64:0/1168971363 wait complete. 2024-03-04T07:28:33.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:28:33.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.479+0000 7fd9ce20e640 1 Processor -- start 2024-03-04T07:28:33.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.479+0000 7fd9ce20e640 1 -- start start 2024-03-04T07:28:33.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9ce20e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd9c815ae50 0x7fd9c80708e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9ce20e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 0x7fd9c8071160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9ce20e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd9c8070e20 con 0x7fd9c815b690 2024-03-04T07:28:33.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9ce20e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd9c80b2e30 con 0x7fd9c815ae50 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9ce20e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd9c806dfe0 con 0x7fd9c815b230 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd9c815b690 0x7fd9c8150a00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40884/0 (socket says 172.21.15.64:40884) 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c7fff640 1 -- 172.21.15.64:0/655259783 learned_addr learned my addr 172.21.15.64:0/655259783 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c6ffd640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 0x7fd9c8071160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/655259783 >> v2:172.21.15.66:3300/0 conn(0x7fd9c815ae50 0x7fd9c80708e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2687138124 0 0) 0x7fd9c8070e20 con 0x7fd9c815b690 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd9b0003520 con 0x7fd9c815b690 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 324182185 0 0) 0x7fd9b0003520 con 0x7fd9c815b690 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 msgr2=0x7fd9c8071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.480+0000 7fd9c67fc640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 0x7fd9c8071160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 >> v2:172.21.15.66:3300/0 conn(0x7fd9c815ae50 msgr2=0x7fd9c80708e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 --2- 172.21.15.64:0/655259783 >> v2:172.21.15.66:3300/0 conn(0x7fd9c815ae50 0x7fd9c80708e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9c80b30b0 con 0x7fd9c815b690 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd9c00019f0 con 0x7fd9c815b690 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 >> v1:172.21.15.64:6789/0 conn(0x7fd9c815b690 legacy=0x7fd9c8150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9b0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9b0004800 0x7fd9b0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 0x7fd9b00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9c80b2e30 con 0x7fd9b00039b0 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9c806dfe0 con 0x7fd9b0004800 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd9b000b9f0 con 0x7fd9c815b230 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9b0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c7fff640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 0x7fd9b00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c6ffd640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9b0004800 0x7fd9b0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c77fe640 1 -- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 msgr2=0x7fd9b00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:33.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 0x7fd9b00051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c77fe640 1 -- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9b0004800 msgr2=0x7fd9b0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9b0004800 0x7fd9b0004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c77fe640 1 -- 172.21.15.64:0/655259783 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9b0003520 con 0x7fd9b00039b0 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.481+0000 7fd9c7fff640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c815b230 0x7fd9b00051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9b0003dd0 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7fd9b8009ac0 tx=0x7fd9b0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1930cebbc1e651d6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9b8039070 con 0x7fd9b00039b0 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd9b802c6d0 con 0x7fd9b00039b0 2024-03-04T07:28:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9c67fc640 1 -- 172.21.15.64:0/655259783 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9b8009690 con 0x7fd9b00039b0 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9ce20e640 1 -- 172.21.15.64:0/655259783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 msgr2=0x7fd9b0003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9ce20e640 1 --2- 172.21.15.64:0/655259783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9b0003dd0 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7fd9b8009ac0 tx=0x7fd9b0005830 comp rx=0 tx=0).stop 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9c7fff640 1 -- 172.21.15.64:0/655259783 reap_dead start 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9ce20e640 1 -- 172.21.15.64:0/655259783 shutdown_connections 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.482+0000 7fd9ce20e640 1 -- 172.21.15.64:0/655259783 >> 172.21.15.64:0/655259783 conn(0x7fd9c8066d00 msgr2=0x7fd9c8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.483+0000 7fd9ce20e640 1 -- 172.21.15.64:0/655259783 shutdown_connections 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.483+0000 7fd9ce20e640 1 -- 172.21.15.64:0/655259783 wait complete. 2024-03-04T07:28:33.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.484+0000 7fd9ce20e640 1 Processor -- start 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.484+0000 7fd9ce20e640 1 -- start start 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9c814f440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c80b12c0 0x7fd9c81589f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9c8158f30 0x7fd9c8153000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9c80b3950 con 0x7fd9b00039b0 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9c80b3670 con 0x7fd9c8158f30 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd9c80b4150 con 0x7fd9c80b12c0 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9c814f440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9c814f440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34868/0 (socket says 172.21.15.64:34868) 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 -- 172.21.15.64:0/1930822594 learned_addr learned my addr 172.21.15.64:0/1930822594 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c6ffd640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c80b12c0 0x7fd9c81589f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c7fff640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9c8158f30 0x7fd9c8153000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 -- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c80b12c0 msgr2=0x7fd9c81589f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:33.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c80b12c0 0x7fd9c81589f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 -- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9c8158f30 msgr2=0x7fd9c8153000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9c8158f30 0x7fd9c8153000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 -- 172.21.15.64:0/1930822594 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9c80b0220 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c6ffd640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c80b12c0 0x7fd9c81589f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c7fff640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9c8158f30 0x7fd9c8153000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9c77fe640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9c814f440 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7fd9b8000c00 tx=0x7fd9b8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9affff640 1 -- 172.21.15.64:0/1930822594 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9b8039050 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9affff640 1 -- 172.21.15.64:0/1930822594 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd9b8045070 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd9c80b0500 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9affff640 1 -- 172.21.15.64:0/1930822594 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9b80044e0 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.485+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd9c80b0ac0 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.486+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd9c80733c0 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.487+0000 7fd9affff640 1 -- 172.21.15.64:0/1930822594 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd9b8003a50 con 0x7fd9b00039b0 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.487+0000 7fd9affff640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9b406b500 0x7fd9b406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:33.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.487+0000 7fd9affff640 1 -- 172.21.15.64:0/1930822594 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(41..41 src has 1..41) v4 ==== 5227+0+0 (secure 0 0 0) 0x7fd9b80ac1c0 con 0x7fd9b00039b0 2024-03-04T07:28:33.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.487+0000 7fd9c6ffd640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9b406b500 0x7fd9b406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:33.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.488+0000 7fd9c6ffd640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9b406b500 0x7fd9b406d9c0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fd9c8154410 tx=0x7fd9bc008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:33.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.492+0000 7fd9affff640 1 -- 172.21.15.64:0/1930822594 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd9b807b340 con 0x7fd9b00039b0 2024-03-04T07:28:33.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:33.671+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fd9c804f410 con 0x7fd9b00039b0 2024-03-04T07:28:34.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.354+0000 7fd9affff640 1 -- 172.21.15.64:0/1930822594 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v42) v1 ==== 170+0+0 (secure 0 0 0) 0x7fd9b807ac40 con 0x7fd9b00039b0 2024-03-04T07:28:34.355 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:28:34.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.358+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9b406b500 msgr2=0x7fd9b406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:34.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.358+0000 7fd9ce20e640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9b406b500 0x7fd9b406d9c0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fd9c8154410 tx=0x7fd9bc008040 comp rx=0 tx=0).stop 2024-03-04T07:28:34.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.358+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 msgr2=0x7fd9c814f440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:34.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.358+0000 7fd9ce20e640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9c814f440 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7fd9b8000c00 tx=0x7fd9b8004730 comp rx=0 tx=0).stop 2024-03-04T07:28:34.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.359+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 shutdown_connections 2024-03-04T07:28:34.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.359+0000 7fd9ce20e640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9b406b500 0x7fd9b406d9c0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:34.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.359+0000 7fd9ce20e640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd9c8158f30 0x7fd9c8153000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:34.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.359+0000 7fd9ce20e640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd9c80b12c0 0x7fd9c81589f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:34.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.359+0000 7fd9ce20e640 1 --2- 172.21.15.64:0/1930822594 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd9b00039b0 0x7fd9c814f440 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:34.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.359+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 >> 172.21.15.64:0/1930822594 conn(0x7fd9c8066d00 msgr2=0x7fd9c8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:34.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.361+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 shutdown_connections 2024-03-04T07:28:34.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.363+0000 7fd9ce20e640 1 -- 172.21.15.64:0/1930822594 wait complete. 2024-03-04T07:28:34.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:28:34.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.494+0000 7f8229a5c640 1 Processor -- start 2024-03-04T07:28:34.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.494+0000 7f8229a5c640 1 -- start start 2024-03-04T07:28:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f8229a5c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f82240b79f0 0x7f82240b7ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f8229a5c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f82240d4960 con 0x7f8224071540 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f8229a5c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f82240d36c0 con 0x7f82240b2d30 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f8229a5c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f82240d3230 con 0x7f82240b79f0 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f82237fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f82240b79f0 0x7f82240b7ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f82237fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f82240b79f0 0x7f82240b7ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34880/0 (socket says 172.21.15.64:34880) 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f8222ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8224071540 0x7f8224071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40894/0 (socket says 172.21.15.64:40894) 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.495+0000 7f82237fe640 1 -- 172.21.15.64:0/162474808 learned_addr learned my addr 172.21.15.64:0/162474808 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.496+0000 7f82237fe640 1 -- 172.21.15.64:0/162474808 >> v1:172.21.15.66:6789/0 conn(0x7f82240b2d30 legacy=0x7f82240b72e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.496+0000 7f82237fe640 1 -- 172.21.15.64:0/162474808 >> v1:172.21.15.64:6789/0 conn(0x7f8224071540 legacy=0x7f8224071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.496+0000 7f82237fe640 1 -- 172.21.15.64:0/162474808 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82240d2620 con 0x7f82240b79f0 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.496+0000 7f82237fe640 1 --2- 172.21.15.64:0/162474808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b7ea0 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f821c009870 tx=0x7f821c02ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1faa4e0d4d2bda89 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.496+0000 7f8221ffb640 1 -- 172.21.15.64:0/162474808 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f821c02f840 con 0x7f82240b79f0 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8221ffb640 1 -- 172.21.15.64:0/162474808 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f821c02f9e0 con 0x7f82240b79f0 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8221ffb640 1 -- 172.21.15.64:0/162474808 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f821c02fdb0 con 0x7f82240b79f0 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8229a5c640 1 -- 172.21.15.64:0/162474808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 msgr2=0x7f82240b7ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8229a5c640 1 --2- 172.21.15.64:0/162474808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b7ea0 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f821c009870 tx=0x7f821c02ba20 comp rx=0 tx=0).stop 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8229a5c640 1 -- 172.21.15.64:0/162474808 shutdown_connections 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8229a5c640 1 --2- 172.21.15.64:0/162474808 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b7ea0 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8229a5c640 1 -- 172.21.15.64:0/162474808 >> 172.21.15.64:0/162474808 conn(0x7f82240d3d60 msgr2=0x7f82240d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8229a5c640 1 -- 172.21.15.64:0/162474808 shutdown_connections 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.497+0000 7f8229a5c640 1 -- 172.21.15.64:0/162474808 wait complete. 2024-03-04T07:28:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.498+0000 7f8229a5c640 1 Processor -- start 2024-03-04T07:28:34.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.498+0000 7f8229a5c640 1 -- start start 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f8229a5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8224071540 0x7f82240cd9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f8229a5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82240b2d30 0x7f82240b4450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f8229a5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b4990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f8229a5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82240d2be0 con 0x7f82240b79f0 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f8229a5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82240b83e0 con 0x7f82240b2d30 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f8229a5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f82240d2ec0 con 0x7f8224071540 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f82237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b4990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f82237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b4990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34886/0 (socket says 172.21.15.64:34886) 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f82237fe640 1 -- 172.21.15.64:0/1866999854 learned_addr learned my addr 172.21.15.64:0/1866999854 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:34.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f82227fc640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82240b2d30 0x7f82240b4450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.499+0000 7f8222ffd640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8224071540 0x7f82240cd9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f82237fe640 1 -- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8224071540 msgr2=0x7f82240cd9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f82237fe640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8224071540 0x7f82240cd9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f82237fe640 1 -- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82240b2d30 msgr2=0x7f82240b4450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f82237fe640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82240b2d30 0x7f82240b4450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f82237fe640 1 -- 172.21.15.64:0/1866999854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82240c7cc0 con 0x7f82240b79f0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f82237fe640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b4990 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f821c009870 tx=0x7f821c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f820bfff640 1 -- 172.21.15.64:0/1866999854 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f821c041400 con 0x7f82240b79f0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f820bfff640 1 -- 172.21.15.64:0/1866999854 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f821c046070 con 0x7f82240b79f0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f82240c7fa0 con 0x7f82240b79f0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f820bfff640 1 -- 172.21.15.64:0/1866999854 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f821c041800 con 0x7f82240b79f0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.500+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f82240c8560 con 0x7f82240b79f0 2024-03-04T07:28:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.501+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f82240b2a10 con 0x7f82240b79f0 2024-03-04T07:28:34.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.501+0000 7f820bfff640 1 -- 172.21.15.64:0/1866999854 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f821c040070 con 0x7f82240b79f0 2024-03-04T07:28:34.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.502+0000 7f820bfff640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f821006b4d0 0x7f821006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:34.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.502+0000 7f820bfff640 1 -- 172.21.15.64:0/1866999854 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(42..42 src has 1..42) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f821c0ac8c0 con 0x7f82240b79f0 2024-03-04T07:28:34.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.502+0000 7f8222ffd640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f821006b4d0 0x7f821006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:34.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.502+0000 7f8222ffd640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f821006b4d0 0x7f821006d990 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f82140012e0 tx=0x7f8214001650 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:34.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.507+0000 7f820bfff640 1 -- 172.21.15.64:0/1866999854 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f821c033030 con 0x7f82240b79f0 2024-03-04T07:28:34.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:34.723+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f82240c6fc0 con 0x7f82240b79f0 2024-03-04T07:28:35.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.378+0000 7f820bfff640 1 -- 172.21.15.64:0/1866999854 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 6 and data pool 7 v20) v1 ==== 172+0+0 (secure 0 0 0) 0x7f821c07b270 con 0x7f82240b79f0 2024-03-04T07:28:35.380 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 6 and data pool 7 2024-03-04T07:28:35.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.387+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f821006b4d0 msgr2=0x7f821006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.387+0000 7f8229a5c640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f821006b4d0 0x7f821006d990 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f82140012e0 tx=0x7f8214001650 comp rx=0 tx=0).stop 2024-03-04T07:28:35.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.387+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 msgr2=0x7f82240b4990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.387+0000 7f8229a5c640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b4990 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f821c009870 tx=0x7f821c004730 comp rx=0 tx=0).stop 2024-03-04T07:28:35.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.389+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 shutdown_connections 2024-03-04T07:28:35.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.389+0000 7f8229a5c640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f821006b4d0 0x7f821006d990 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.389+0000 7f8229a5c640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82240b79f0 0x7f82240b4990 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.389+0000 7f8229a5c640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82240b2d30 0x7f82240b4450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.389+0000 7f8229a5c640 1 --2- 172.21.15.64:0/1866999854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8224071540 0x7f82240cd9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.389+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 >> 172.21.15.64:0/1866999854 conn(0x7f82240d3d60 msgr2=0x7f822415b160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:35.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.391+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 shutdown_connections 2024-03-04T07:28:35.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.391+0000 7f8229a5c640 1 -- 172.21.15.64:0/1866999854 wait complete. 2024-03-04T07:28:35.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:35.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.514+0000 7f630738a640 1 Processor -- start 2024-03-04T07:28:35.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.514+0000 7f630738a640 1 -- start start 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f630738a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 0x7f63000d0470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f630738a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6300066d00 0x7f63000671b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f630738a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63000cf3d0 con 0x7f63000ceed0 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f630738a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6300072c30 con 0x7f6300066d00 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f630738a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63000cf6b0 con 0x7f6300145780 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f6304925640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f63000ceed0 0x7f6300072520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40900/0 (socket says 172.21.15.64:40900) 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f6304925640 1 -- 172.21.15.64:0/422473368 learned_addr learned my addr 172.21.15.64:0/422473368 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2138750025 0 0) 0x7f63000cf3d0 con 0x7f63000ceed0 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f6305927640 1 --2- 172.21.15.64:0/422473368 >> v2:172.21.15.66:3300/0 conn(0x7f6300066d00 0x7f63000671b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f62e8003520 con 0x7f63000ceed0 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.515+0000 7f6305126640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 0x7f63000d0470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 244316881 0 0) 0x7f62e8003520 con 0x7f63000ceed0 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 msgr2=0x7f63000d0470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 0x7f63000d0470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 >> v2:172.21.15.66:3300/0 conn(0x7f6300066d00 msgr2=0x7f63000671b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 --2- 172.21.15.64:0/422473368 >> v2:172.21.15.66:3300/0 conn(0x7f6300066d00 0x7f63000671b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63000cf990 con 0x7f63000ceed0 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f62f4001670 con 0x7f63000ceed0 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 >> v1:172.21.15.64:6789/0 conn(0x7f63000ceed0 legacy=0x7f6300072520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e80039b0 0x7f62e8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62e8004800 0x7f62e8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 0x7f62e80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6300072c30 con 0x7f62e8004800 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63000cf6b0 con 0x7f62e80039b0 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62e800b9c0 con 0x7f6300145780 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6304925640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62e8004800 0x7f62e8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6305126640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e80039b0 0x7f62e8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6305927640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 0x7f62e80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6304925640 1 -- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 msgr2=0x7f62e80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6304925640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6300145780 0x7f62e80051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6304925640 1 -- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e80039b0 msgr2=0x7f62e8003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6304925640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e80039b0 0x7f62e8003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6304925640 1 -- 172.21.15.64:0/422473368 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62e8003520 con 0x7f62e8004800 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.516+0000 7f6305126640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e80039b0 0x7f62e8003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.517+0000 7f6304925640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62e8004800 0x7f62e8004c80 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f62f400c210 tx=0x7f62e8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=79c88d7a54605a13 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.517+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62f4007d70 con 0x7f62e8004800 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.517+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62f4030070 con 0x7f62e8004800 2024-03-04T07:28:35.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.517+0000 7f62fffff640 1 -- 172.21.15.64:0/422473368 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62f402cd60 con 0x7f62e8004800 2024-03-04T07:28:35.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.518+0000 7f630738a640 1 -- 172.21.15.64:0/422473368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62e8004800 msgr2=0x7f62e8004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.518+0000 7f630738a640 1 --2- 172.21.15.64:0/422473368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62e8004800 0x7f62e8004c80 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f62f400c210 tx=0x7f62e8005830 comp rx=0 tx=0).stop 2024-03-04T07:28:35.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.518+0000 7f6305927640 1 -- 172.21.15.64:0/422473368 reap_dead start 2024-03-04T07:28:35.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.518+0000 7f630738a640 1 -- 172.21.15.64:0/422473368 shutdown_connections 2024-03-04T07:28:35.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.518+0000 7f630738a640 1 -- 172.21.15.64:0/422473368 >> 172.21.15.64:0/422473368 conn(0x7f63000b2720 msgr2=0x7f63000b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:35.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.518+0000 7f630738a640 1 -- 172.21.15.64:0/422473368 shutdown_connections 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.518+0000 7f630738a640 1 -- 172.21.15.64:0/422473368 wait complete. 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.519+0000 7f630738a640 1 Processor -- start 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.519+0000 7f630738a640 1 -- start start 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f630738a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e8004800 0x7f63000d0c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f630738a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63000caa00 0x7f63001f3cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f630738a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63001f4200 0x7f6300157240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f630738a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63000d05f0 con 0x7f63001f4200 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f630738a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63000d0470 con 0x7f62e8004800 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f630738a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63000d0770 con 0x7f63000caa00 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63001f4200 0x7f6300157240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63001f4200 0x7f6300157240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34892/0 (socket says 172.21.15.64:34892) 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 -- 172.21.15.64:0/1220703217 learned_addr learned my addr 172.21.15.64:0/1220703217 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305126640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e8004800 0x7f63000d0c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6304925640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63000caa00 0x7f63001f3cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 -- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63000caa00 msgr2=0x7f63001f3cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63000caa00 0x7f63001f3cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 -- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e8004800 msgr2=0x7f63000d0c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e8004800 0x7f63000d0c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 -- 172.21.15.64:0/1220703217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63001f49d0 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f6305927640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63001f4200 0x7f6300157240 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f630006a0c0 tx=0x7f62f000f580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f62fdffb640 1 -- 172.21.15.64:0/1220703217 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62f8009690 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f62fdffb640 1 -- 172.21.15.64:0/1220703217 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62f001d070 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63000c9f20 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.520+0000 7f62fdffb640 1 -- 172.21.15.64:0/1220703217 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62f0015670 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.521+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63001f4680 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.522+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6300067ff0 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.522+0000 7f62fdffb640 1 -- 172.21.15.64:0/1220703217 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f62f0015810 con 0x7f63001f4200 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.522+0000 7f62fdffb640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62ec06b4d0 0x7f62ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.522+0000 7f62fdffb640 1 -- 172.21.15.64:0/1220703217 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f62f008a9c0 con 0x7f63001f4200 2024-03-04T07:28:35.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.522+0000 7f6305126640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62ec06b4d0 0x7f62ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.523+0000 7f6305126640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62ec06b4d0 0x7f62ec06d990 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f62f8009c00 tx=0x7f62f8009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:35.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.526+0000 7f62fdffb640 1 -- 172.21.15.64:0/1220703217 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f62f008ce20 con 0x7f63001f4200 2024-03-04T07:28:35.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.708+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f63000b14f0 con 0x7f63001f4200 2024-03-04T07:28:35.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.709+0000 7f62fdffb640 1 -- 172.21.15.64:0/1220703217 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v43) v1 ==== 74+0+16578 (secure 0 0 0) 0x7f62f005f020 con 0x7f63001f4200 2024-03-04T07:28:35.710 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:35.711 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:34.725978+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:28:32.989901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-04T07:28:33.692080+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:35.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.711+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62ec06b4d0 msgr2=0x7f62ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.711+0000 7f630738a640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62ec06b4d0 0x7f62ec06d990 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f62f8009c00 tx=0x7f62f8009190 comp rx=0 tx=0).stop 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63001f4200 msgr2=0x7f6300157240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63001f4200 0x7f6300157240 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f630006a0c0 tx=0x7f62f000f580 comp rx=0 tx=0).stop 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 shutdown_connections 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62ec06b4d0 0x7f62ec06d990 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63001f4200 0x7f6300157240 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63000caa00 0x7f63001f3cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 --2- 172.21.15.64:0/1220703217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62e8004800 0x7f63000d0c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 >> 172.21.15.64:0/1220703217 conn(0x7f63000b2720 msgr2=0x7f630006dfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 shutdown_connections 2024-03-04T07:28:35.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.712+0000 7f630738a640 1 -- 172.21.15.64:0/1220703217 wait complete. 2024-03-04T07:28:35.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:35.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.824+0000 7f5b70377640 1 Processor -- start 2024-03-04T07:28:35.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.824+0000 7f5b70377640 1 -- start start 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b70377640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5b68157b80 0x7f5b6815b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b70377640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5b68157f60 0x7f5b6815b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b70377640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68070b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b70377640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5b6806c7b0 con 0x7f5b68157f60 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b70377640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5b6806ca90 con 0x7f5b68157b80 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b70377640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b6806cd70 con 0x7f5b680cbab0 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b6d912640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5b68157f60 0x7f5b6815b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b6d912640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5b68157f60 0x7f5b6815b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34906/0 (socket says 172.21.15.64:34906) 2024-03-04T07:28:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b6d912640 1 -- 172.21.15.64:0/3022202870 learned_addr learned my addr 172.21.15.64:0/3022202870 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b6e113640 1 --2- 172.21.15.64:0/3022202870 >> v2:172.21.15.66:3300/0 conn(0x7f5b68157b80 0x7f5b6815b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.825+0000 7f5b6e914640 1 --2- 172.21.15.64:0/3022202870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68070b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d912640 1 -- 172.21.15.64:0/3022202870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 msgr2=0x7f5b68070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d912640 1 --2- 172.21.15.64:0/3022202870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68070b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d912640 1 -- 172.21.15.64:0/3022202870 >> v2:172.21.15.66:3300/0 conn(0x7f5b68157b80 msgr2=0x7f5b6815b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d912640 1 --2- 172.21.15.64:0/3022202870 >> v2:172.21.15.66:3300/0 conn(0x7f5b68157b80 0x7f5b6815b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d912640 1 -- 172.21.15.64:0/3022202870 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b6806d050 con 0x7f5b68157f60 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6e113640 1 --2- 172.21.15.64:0/3022202870 >> v2:172.21.15.66:3300/0 conn(0x7f5b68157b80 0x7f5b6815b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d912640 1 --2- 172.21.15.64:0/3022202870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b6815b9a0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f5b6806ed60 tx=0x7f5b5802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=58492ced7725e64f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d111640 1 -- 172.21.15.64:0/3022202870 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b58035020 con 0x7f5b68157f60 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d111640 1 -- 172.21.15.64:0/3022202870 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b58031870 con 0x7f5b68157f60 2024-03-04T07:28:35.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.826+0000 7f5b6d111640 1 -- 172.21.15.64:0/3022202870 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b58031c40 con 0x7f5b68157f60 2024-03-04T07:28:35.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 -- 172.21.15.64:0/3022202870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 msgr2=0x7f5b6815b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 --2- 172.21.15.64:0/3022202870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b6815b9a0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f5b6806ed60 tx=0x7f5b5802b910 comp rx=0 tx=0).stop 2024-03-04T07:28:35.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 -- 172.21.15.64:0/3022202870 shutdown_connections 2024-03-04T07:28:35.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 --2- 172.21.15.64:0/3022202870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68070b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 --2- 172.21.15.64:0/3022202870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b6815b9a0 unknown :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 --2- 172.21.15.64:0/3022202870 >> v2:172.21.15.66:3300/0 conn(0x7f5b68157b80 0x7f5b6815b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 -- 172.21.15.64:0/3022202870 >> 172.21.15.64:0/3022202870 conn(0x7f5b68066d00 msgr2=0x7f5b680670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 -- 172.21.15.64:0/3022202870 shutdown_connections 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.827+0000 7f5b70377640 1 -- 172.21.15.64:0/3022202870 wait complete. 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.828+0000 7f5b70377640 1 Processor -- start 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.828+0000 7f5b70377640 1 -- start start 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b70377640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68154690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b70377640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b68157b80 0x7f5b6814e750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b70377640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b68155170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b70377640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b680cc1d0 con 0x7f5b68157f60 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b70377640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b680cc050 con 0x7f5b68157b80 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b70377640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5b68071080 con 0x7f5b680cbab0 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b68155170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b68155170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34912/0 (socket says 172.21.15.64:34912) 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 -- 172.21.15.64:0/650006433 learned_addr learned my addr 172.21.15.64:0/650006433 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:35.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6d912640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b68157b80 0x7f5b6814e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e113640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68154690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 -- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 msgr2=0x7f5b68154690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68154690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 -- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b68157b80 msgr2=0x7f5b6814e750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b68157b80 0x7f5b6814e750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.829+0000 7f5b6e914640 1 -- 172.21.15.64:0/650006433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b6814ec90 con 0x7f5b68157f60 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.830+0000 7f5b6e914640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b68155170 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f5b64007e60 tx=0x7f5b6400e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.830+0000 7f5b5effd640 1 -- 172.21.15.64:0/650006433 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b64014620 con 0x7f5b68157f60 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.830+0000 7f5b5effd640 1 -- 172.21.15.64:0/650006433 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5b64019070 con 0x7f5b68157f60 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.830+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b681556b0 con 0x7f5b68157f60 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.830+0000 7f5b5effd640 1 -- 172.21.15.64:0/650006433 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b64014a20 con 0x7f5b68157f60 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.830+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b68155c30 con 0x7f5b68157f60 2024-03-04T07:28:35.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.831+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5b68067ff0 con 0x7f5b68157f60 2024-03-04T07:28:35.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.831+0000 7f5b5effd640 1 -- 172.21.15.64:0/650006433 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5b64014bc0 con 0x7f5b68157f60 2024-03-04T07:28:35.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.832+0000 7f5b5effd640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b4c06b4d0 0x7f5b4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:35.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.832+0000 7f5b6e113640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b4c06b4d0 0x7f5b4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:35.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.832+0000 7f5b5effd640 1 -- 172.21.15.64:0/650006433 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5b6408a980 con 0x7f5b68157f60 2024-03-04T07:28:35.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.832+0000 7f5b6e113640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b4c06b4d0 0x7f5b4c06d990 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f5b60006fd0 tx=0x7f5b60008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:35.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:35.836+0000 7f5b5effd640 1 -- 172.21.15.64:0/650006433 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5b64059800 con 0x7f5b68157f60 2024-03-04T07:28:36.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.018+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f5b6815a220 con 0x7f5b68157f60 2024-03-04T07:28:36.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.020+0000 7f5b5effd640 1 -- 172.21.15.64:0/650006433 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v43) v1 ==== 74+0+16578 (secure 0 0 0) 0x7f5b64059100 con 0x7f5b68157f60 2024-03-04T07:28:36.020 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:36.021 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:34.725978+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:28:32.989901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-04T07:28:33.692080+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:36.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.021+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b4c06b4d0 msgr2=0x7f5b4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.021+0000 7f5b70377640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b4c06b4d0 0x7f5b4c06d990 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f5b60006fd0 tx=0x7f5b60008040 comp rx=0 tx=0).stop 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.021+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 msgr2=0x7f5b68155170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.021+0000 7f5b70377640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b68155170 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f5b64007e60 tx=0x7f5b6400e4b0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 shutdown_connections 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5b4c06b4d0 0x7f5b4c06d990 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5b68157f60 0x7f5b68155170 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5b68157b80 0x7f5b6814e750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 --2- 172.21.15.64:0/650006433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5b680cbab0 0x7f5b68154690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 >> 172.21.15.64:0/650006433 conn(0x7f5b68066d00 msgr2=0x7f5b68158da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 shutdown_connections 2024-03-04T07:28:36.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.022+0000 7f5b70377640 1 -- 172.21.15.64:0/650006433 wait complete. 2024-03-04T07:28:36.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:28:36.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.137+0000 7fc7da865640 1 Processor -- start 2024-03-04T07:28:36.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.137+0000 7fc7da865640 1 -- start start 2024-03-04T07:28:36.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.137+0000 7fc7da865640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d4071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.137+0000 7fc7da865640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc7d40b3060 0x7fc7d40d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.137+0000 7fc7da865640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc7d415e4b0 con 0x7fc7d40b2b10 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.137+0000 7fc7da865640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc7d40d2e10 con 0x7fc7d40b3060 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.137+0000 7fc7da865640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc7d40b97e0 con 0x7fc7d40717b0 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d37fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc7d40b3060 0x7fc7d40d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d37fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc7d40b3060 0x7fc7d40d32a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34922/0 (socket says 172.21.15.64:34922) 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d37fe640 1 -- 172.21.15.64:0/963709331 learned_addr learned my addr 172.21.15.64:0/963709331 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d3fff640 1 --2- 172.21.15.64:0/963709331 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d4071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d37fe640 1 -- 172.21.15.64:0/963709331 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 msgr2=0x7fc7d4071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d37fe640 1 --2- 172.21.15.64:0/963709331 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d4071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d37fe640 1 -- 172.21.15.64:0/963709331 >> v1:172.21.15.64:6789/0 conn(0x7fc7d40b2b10 legacy=0x7fc7d40b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.138+0000 7fc7d37fe640 1 -- 172.21.15.64:0/963709331 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7d40b9a80 con 0x7fc7d40b3060 2024-03-04T07:28:36.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7d37fe640 1 --2- 172.21.15.64:0/963709331 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40d32a0 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fc7c8009870 tx=0x7fc7c802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e64d8f2c42d2b33 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7d2ffd640 1 -- 172.21.15.64:0/963709331 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7c8034070 con 0x7fc7d40b3060 2024-03-04T07:28:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7d2ffd640 1 -- 172.21.15.64:0/963709331 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc7c8030840 con 0x7fc7d40b3060 2024-03-04T07:28:36.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7d2ffd640 1 -- 172.21.15.64:0/963709331 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7c8030c10 con 0x7fc7d40b3060 2024-03-04T07:28:36.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 -- 172.21.15.64:0/963709331 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 msgr2=0x7fc7d40d32a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 --2- 172.21.15.64:0/963709331 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40d32a0 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fc7c8009870 tx=0x7fc7c802b980 comp rx=0 tx=0).stop 2024-03-04T07:28:36.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 -- 172.21.15.64:0/963709331 shutdown_connections 2024-03-04T07:28:36.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 --2- 172.21.15.64:0/963709331 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40d32a0 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 --2- 172.21.15.64:0/963709331 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d4071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 -- 172.21.15.64:0/963709331 >> 172.21.15.64:0/963709331 conn(0x7fc7d40d3d20 msgr2=0x7fc7d40d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:36.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 -- 172.21.15.64:0/963709331 shutdown_connections 2024-03-04T07:28:36.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.139+0000 7fc7da865640 1 -- 172.21.15.64:0/963709331 wait complete. 2024-03-04T07:28:36.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 Processor -- start 2024-03-04T07:28:36.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 -- start start 2024-03-04T07:28:36.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d40c9cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7d40b2b10 0x7fc7d40cc850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40ccd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7d40b7560 con 0x7fc7d40b3060 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7d40d4bf0 con 0x7fc7d40b2b10 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.141+0000 7fc7da865640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc7d40b76e0 con 0x7fc7d40717b0 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40ccd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40ccd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34934/0 (socket says 172.21.15.64:34934) 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 -- 172.21.15.64:0/3810899383 learned_addr learned my addr 172.21.15.64:0/3810899383 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d37fe640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7d40b2b10 0x7fc7d40cc850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d3fff640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d40c9cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 -- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 msgr2=0x7fc7d40c9cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d40c9cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 -- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7d40b2b10 msgr2=0x7fc7d40cc850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7d40b2b10 0x7fc7d40cc850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 -- 172.21.15.64:0/3810899383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7d4156740 con 0x7fc7d40b3060 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d37fe640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7d40b2b10 0x7fc7d40cc850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d3fff640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d40c9cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d8e02640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40ccd90 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7fc7cc00c7a0 tx=0x7fc7cc00cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d0ff9640 1 -- 172.21.15.64:0/3810899383 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7cc003b70 con 0x7fc7d40b3060 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d0ff9640 1 -- 172.21.15.64:0/3810899383 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc7cc003d30 con 0x7fc7d40b3060 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc7d40cd3c0 con 0x7fc7d40b3060 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7d0ff9640 1 -- 172.21.15.64:0/3810899383 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7cc00aa90 con 0x7fc7d40b3060 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.142+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc7d4148920 con 0x7fc7d40b3060 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.143+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc7d406ce40 con 0x7fc7d40b3060 2024-03-04T07:28:36.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.144+0000 7fc7d0ff9640 1 -- 172.21.15.64:0/3810899383 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc7cc00ac30 con 0x7fc7d40b3060 2024-03-04T07:28:36.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.144+0000 7fc7d0ff9640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7c006b4d0 0x7fc7c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.144+0000 7fc7d0ff9640 1 -- 172.21.15.64:0/3810899383 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(43..43 src has 1..43) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc7cc08a1d0 con 0x7fc7d40b3060 2024-03-04T07:28:36.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.144+0000 7fc7d3fff640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7c006b4d0 0x7fc7c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.144+0000 7fc7d3fff640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7c006b4d0 0x7fc7c006d990 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fc7c4008890 tx=0x7fc7c4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:36.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.148+0000 7fc7d0ff9640 1 -- 172.21.15.64:0/3810899383 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc7d415e4b0 con 0x7fc7d40b3060 2024-03-04T07:28:36.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.371+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7fc7d404f410 con 0x7fc7d40b3060 2024-03-04T07:28:36.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.379+0000 7fc7d0ff9640 1 -- 172.21.15.64:0/3810899383 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v21) v1 ==== 118+0+0 (secure 0 0 0) 0x7fc7cc058fd0 con 0x7fc7d40b3060 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7c006b4d0 msgr2=0x7fc7c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7c006b4d0 0x7fc7c006d990 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fc7c4008890 tx=0x7fc7c4008040 comp rx=0 tx=0).stop 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 msgr2=0x7fc7d40ccd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40ccd90 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7fc7cc00c7a0 tx=0x7fc7cc00cc60 comp rx=0 tx=0).stop 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 shutdown_connections 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7c006b4d0 0x7fc7c006d990 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7d40b3060 0x7fc7d40ccd90 unknown :-1 s=CLOSED pgs=527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7d40b2b10 0x7fc7d40cc850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 --2- 172.21.15.64:0/3810899383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7d40717b0 0x7fc7d40c9cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 >> 172.21.15.64:0/3810899383 conn(0x7fc7d40d3d20 msgr2=0x7fc7d40b59d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:36.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 shutdown_connections 2024-03-04T07:28:36.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.382+0000 7fc7da865640 1 -- 172.21.15.64:0/3810899383 wait complete. 2024-03-04T07:28:36.396 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:28:36.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:28:36.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.500+0000 7f70b8325640 1 Processor -- start 2024-03-04T07:28:36.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.500+0000 7f70b8325640 1 -- start start 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b8325640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f70b00b2cd0 0x7f70b00b1c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b8325640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f70b006b2c0 0x7f70b006b710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b8325640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f70b0161a40 con 0x7f70b00cdcd0 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b8325640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f70b00d11e0 con 0x7f70b00b2cd0 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b8325640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f70b00b2a20 con 0x7f70b006b2c0 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b58c0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f70b00b2cd0 0x7f70b00b1c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b58c0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f70b00b2cd0 0x7f70b00b1c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34938/0 (socket says 172.21.15.64:34938) 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b58c0640 1 -- 172.21.15.64:0/3247437317 learned_addr learned my addr 172.21.15.64:0/3247437317 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b68c2640 1 --2- 172.21.15.64:0/3247437317 >> v2:172.21.15.66:3300/0 conn(0x7f70b006b2c0 0x7f70b006b710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b58c0640 1 -- 172.21.15.64:0/3247437317 >> v2:172.21.15.66:3300/0 conn(0x7f70b006b2c0 msgr2=0x7f70b006b710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:36.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b58c0640 1 --2- 172.21.15.64:0/3247437317 >> v2:172.21.15.66:3300/0 conn(0x7f70b006b2c0 0x7f70b006b710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b58c0640 1 -- 172.21.15.64:0/3247437317 >> v1:172.21.15.66:6789/0 conn(0x7f70b00cdcd0 legacy=0x7f70b00b1550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:36.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.501+0000 7f70b58c0640 1 -- 172.21.15.64:0/3247437317 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70b00d0940 con 0x7f70b00b2cd0 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.502+0000 7f70b58c0640 1 --2- 172.21.15.64:0/3247437317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00b2cd0 0x7f70b00b1c60 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f70b00b3570 tx=0x7f70a002b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7c73e60b483a29ba server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.502+0000 7f70b50bf640 1 -- 172.21.15.64:0/3247437317 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70a002e070 con 0x7f70b00b2cd0 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.502+0000 7f70b50bf640 1 -- 172.21.15.64:0/3247437317 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f70a0037070 con 0x7f70b00b2cd0 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.502+0000 7f70b50bf640 1 -- 172.21.15.64:0/3247437317 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70a0031a00 con 0x7f70b00b2cd0 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 -- 172.21.15.64:0/3247437317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00b2cd0 msgr2=0x7f70b00b1c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 --2- 172.21.15.64:0/3247437317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00b2cd0 0x7f70b00b1c60 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f70b00b3570 tx=0x7f70a002b880 comp rx=0 tx=0).stop 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 -- 172.21.15.64:0/3247437317 shutdown_connections 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 --2- 172.21.15.64:0/3247437317 >> v2:172.21.15.66:3300/0 conn(0x7f70b006b2c0 0x7f70b006b710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 --2- 172.21.15.64:0/3247437317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00b2cd0 0x7f70b00b1c60 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 -- 172.21.15.64:0/3247437317 >> 172.21.15.64:0/3247437317 conn(0x7f70b00b3700 msgr2=0x7f70b00b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:36.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 -- 172.21.15.64:0/3247437317 shutdown_connections 2024-03-04T07:28:36.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.503+0000 7f70b8325640 1 -- 172.21.15.64:0/3247437317 wait complete. 2024-03-04T07:28:36.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.504+0000 7f70b8325640 1 Processor -- start 2024-03-04T07:28:36.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.504+0000 7f70b8325640 1 -- start start 2024-03-04T07:28:36.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b8325640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70b006b2c0 0x7f70b015adf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b8325640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70b00b2cd0 0x7f70b01f49b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b8325640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00cdcd0 0x7f70b00b5fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b8325640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70b00cc1c0 con 0x7f70b00cdcd0 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b8325640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70b00d0f00 con 0x7f70b006b2c0 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b8325640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70b00cc340 con 0x7f70b00b2cd0 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00cdcd0 0x7f70b00b5fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00cdcd0 0x7f70b00b5fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34944/0 (socket says 172.21.15.64:34944) 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 -- 172.21.15.64:0/2571693212 learned_addr learned my addr 172.21.15.64:0/2571693212 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b60c1640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70b006b2c0 0x7f70b015adf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b58c0640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70b00b2cd0 0x7f70b01f49b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 -- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70b00b2cd0 msgr2=0x7f70b01f49b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70b00b2cd0 0x7f70b01f49b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 -- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70b006b2c0 msgr2=0x7f70b015adf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70b006b2c0 0x7f70b015adf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 -- 172.21.15.64:0/2571693212 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70b00b4990 con 0x7f70b00cdcd0 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b58c0640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70b00b2cd0 0x7f70b01f49b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:36.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.505+0000 7f70b68c2640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00cdcd0 0x7f70b00b5fa0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f70a8007e70 tx=0x7f70a800e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:36.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.506+0000 7f70a6ffd640 1 -- 172.21.15.64:0/2571693212 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70a8014620 con 0x7f70b00cdcd0 2024-03-04T07:28:36.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.506+0000 7f70a6ffd640 1 -- 172.21.15.64:0/2571693212 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f70a8019070 con 0x7f70b00cdcd0 2024-03-04T07:28:36.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.506+0000 7f70a6ffd640 1 -- 172.21.15.64:0/2571693212 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70a8014a20 con 0x7f70b00cdcd0 2024-03-04T07:28:36.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.506+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70b00b4c70 con 0x7f70b00cdcd0 2024-03-04T07:28:36.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.506+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70b00b5230 con 0x7f70b00cdcd0 2024-03-04T07:28:36.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.507+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f70b009fcf0 con 0x7f70b00cdcd0 2024-03-04T07:28:36.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.507+0000 7f70a6ffd640 1 -- 172.21.15.64:0/2571693212 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f70a8014bc0 con 0x7f70b00cdcd0 2024-03-04T07:28:36.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.507+0000 7f70a6ffd640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f709406b4d0 0x7f709406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:36.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.508+0000 7f70a6ffd640 1 -- 172.21.15.64:0/2571693212 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f70a808aa70 con 0x7f70b00cdcd0 2024-03-04T07:28:36.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.508+0000 7f70b60c1640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f709406b4d0 0x7f709406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:36.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.508+0000 7f70b60c1640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f709406b4d0 0x7f709406d990 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f70ac00a750 tx=0x7f70ac008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:36.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.512+0000 7f70a6ffd640 1 -- 172.21.15.64:0/2571693212 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f70a8059980 con 0x7f70b00cdcd0 2024-03-04T07:28:36.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:36.734+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f70b004f410 con 0x7f70b00cdcd0 2024-03-04T07:28:37.350 INFO:tasks.ceph.mds.a.smithi064.stderr:2024-03-04T07:28:37.351+0000 7f4b61b48640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:28:37.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.390+0000 7f70a6ffd640 1 -- 172.21.15.64:0/2571693212 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v22) v1 ==== 122+0+0 (secure 0 0 0) 0x7f70a8059280 con 0x7f70b00cdcd0 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f709406b4d0 msgr2=0x7f709406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f709406b4d0 0x7f709406d990 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f70ac00a750 tx=0x7f70ac008040 comp rx=0 tx=0).stop 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00cdcd0 msgr2=0x7f70b00b5fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00cdcd0 0x7f70b00b5fa0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f70a8007e70 tx=0x7f70a800e4b0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 shutdown_connections 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f709406b4d0 0x7f709406d990 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70b00cdcd0 0x7f70b00b5fa0 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70b00b2cd0 0x7f70b01f49b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 --2- 172.21.15.64:0/2571693212 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70b006b2c0 0x7f70b015adf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 >> 172.21.15.64:0/2571693212 conn(0x7f70b00b3700 msgr2=0x7f70b00cc4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:37.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 shutdown_connections 2024-03-04T07:28:37.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.392+0000 7f70b8325640 1 -- 172.21.15.64:0/2571693212 wait complete. 2024-03-04T07:28:37.406 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:28:37.406 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:37.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.508+0000 7f84ab9bc640 1 Processor -- start 2024-03-04T07:28:37.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.508+0000 7f84ab9bc640 1 -- start start 2024-03-04T07:28:37.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.509+0000 7f84ab9bc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f84a40b95a0 0x7f84a40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.509+0000 7f84ab9bc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f84a40cea30 con 0x7f84a40b9a10 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.509+0000 7f84ab9bc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f84a40cf3c0 con 0x7f84a415df00 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.509+0000 7f84ab9bc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f84a40cf0e0 con 0x7f84a40b95a0 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.509+0000 7f84a9f59640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f84a40b9a10 0x7f84a40d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40916/0 (socket says 172.21.15.64:40916) 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.509+0000 7f84a9f59640 1 -- 172.21.15.64:0/1806394260 learned_addr learned my addr 172.21.15.64:0/1806394260 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.509+0000 7f84a8f57640 1 --2- 172.21.15.64:0/1806394260 >> v2:172.21.15.66:3300/0 conn(0x7f84a40b95a0 0x7f84a40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3439755126 0 0) 0x7f84a40cea30 con 0x7f84a40b9a10 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8490003520 con 0x7f84a40b9a10 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2153866862 0 0) 0x7f84a40cf3c0 con 0x7f84a415df00 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f84a40cea30 con 0x7f84a415df00 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 519955007 0 0) 0x7f8490003520 con 0x7f84a40b9a10 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 >> v2:172.21.15.66:3300/0 conn(0x7f84a40b95a0 msgr2=0x7f84a40b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 --2- 172.21.15.64:0/1806394260 >> v2:172.21.15.66:3300/0 conn(0x7f84a40b95a0 0x7f84a40b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 >> v1:172.21.15.66:6789/0 conn(0x7f84a415df00 legacy=0x7f84a40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84a40ce470 con 0x7f84a40b9a10 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f849c0019f0 con 0x7f84a40b9a10 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 >> v1:172.21.15.64:6789/0 conn(0x7f84a40b9a10 legacy=0x7f84a40d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8490003b90 0x7f8490003fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490004800 0x7f8490004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84900051c0 0x7f849000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84a40cf0e0 con 0x7f8490003b90 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f849000c0c0 con 0x7f84900051c0 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f849000c2c0 con 0x7f8490004800 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9758640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8490003b90 0x7f8490003fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a8f57640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490004800 0x7f8490004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9f59640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84900051c0 0x7f849000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9758640 1 -- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490004800 msgr2=0x7f8490004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9758640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490004800 0x7f8490004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9758640 1 -- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84900051c0 msgr2=0x7f849000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9758640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84900051c0 0x7f849000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9758640 1 -- 172.21.15.64:0/1806394260 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8490003520 con 0x7f8490003b90 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9f59640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84900051c0 0x7f849000b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.510+0000 7f84a9f59640 1 -- 172.21.15.64:0/1806394260 reap_dead start 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.511+0000 7f84a9758640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8490003b90 0x7f8490003fb0 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f84a00018d0 tx=0x7f8490005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be8adcffa33db2a4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.511+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84a0037070 con 0x7f8490003b90 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.511+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f84a00052d0 con 0x7f8490003b90 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.511+0000 7f849bfff640 1 -- 172.21.15.64:0/1806394260 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84a00014f0 con 0x7f8490003b90 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.512+0000 7f84ab9bc640 1 -- 172.21.15.64:0/1806394260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8490003b90 msgr2=0x7f8490003fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.512+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8490003b90 0x7f8490003fb0 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f84a00018d0 tx=0x7f8490005830 comp rx=0 tx=0).stop 2024-03-04T07:28:37.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.512+0000 7f84ab9bc640 1 -- 172.21.15.64:0/1806394260 shutdown_connections 2024-03-04T07:28:37.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.512+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/1806394260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8490003b90 0x7f8490003fb0 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.512+0000 7f84ab9bc640 1 -- 172.21.15.64:0/1806394260 >> 172.21.15.64:0/1806394260 conn(0x7f84a4071960 msgr2=0x7f84a4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:37.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.512+0000 7f84ab9bc640 1 -- 172.21.15.64:0/1806394260 shutdown_connections 2024-03-04T07:28:37.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.512+0000 7f84ab9bc640 1 -- 172.21.15.64:0/1806394260 wait complete. 2024-03-04T07:28:37.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.513+0000 7f84ab9bc640 1 Processor -- start 2024-03-04T07:28:37.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.513+0000 7f84ab9bc640 1 -- start start 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84ab9bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490003b90 0x7f84a40d3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84ab9bc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f84a40caee0 0x7f84a40cbaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84ab9bc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84a415f740 0x7f84a415fbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84ab9bc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84a40cedf0 con 0x7f84a40caee0 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84ab9bc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84a40cec70 con 0x7f84a415f740 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84ab9bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f84a40d25b0 con 0x7f8490003b90 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f84a40caee0 0x7f84a40cbaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f84a40caee0 0x7f84a40cbaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34962/0 (socket says 172.21.15.64:34962) 2024-03-04T07:28:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 -- 172.21.15.64:0/702974443 learned_addr learned my addr 172.21.15.64:0/702974443 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a9758640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490003b90 0x7f84a40d3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a9f59640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84a415f740 0x7f84a415fbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 -- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490003b90 msgr2=0x7f84a40d3870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490003b90 0x7f84a40d3870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 -- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84a415f740 msgr2=0x7f84a415fbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84a415f740 0x7f84a415fbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 -- 172.21.15.64:0/702974443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84a415e6a0 con 0x7f84a40caee0 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a9f59640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84a415f740 0x7f84a415fbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f84a8f57640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f84a40caee0 0x7f84a40cbaf0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f8494011aa0 tx=0x7f8494011f60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f8499ffb640 1 -- 172.21.15.64:0/702974443 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f849400fb80 con 0x7f84a40caee0 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.514+0000 7f8499ffb640 1 -- 172.21.15.64:0/702974443 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f849400fd20 con 0x7f84a40caee0 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.515+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84a415e980 con 0x7f84a40caee0 2024-03-04T07:28:37.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.515+0000 7f8499ffb640 1 -- 172.21.15.64:0/702974443 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8494014660 con 0x7f84a40caee0 2024-03-04T07:28:37.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.515+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84a415ef40 con 0x7f84a40caee0 2024-03-04T07:28:37.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.516+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f84a4067ff0 con 0x7f84a40caee0 2024-03-04T07:28:37.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.517+0000 7f8499ffb640 1 -- 172.21.15.64:0/702974443 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8494012040 con 0x7f84a40caee0 2024-03-04T07:28:37.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.517+0000 7f8499ffb640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f848806b4d0 0x7f848806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.517+0000 7f8499ffb640 1 -- 172.21.15.64:0/702974443 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f849408a900 con 0x7f84a40caee0 2024-03-04T07:28:37.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.517+0000 7f84a9758640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f848806b4d0 0x7f848806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.518+0000 7f84a9758640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f848806b4d0 0x7f848806d990 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f84a0034000 tx=0x7f84a0031000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:37.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.521+0000 7f8499ffb640 1 -- 172.21.15.64:0/702974443 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8494019030 con 0x7f84a40caee0 2024-03-04T07:28:37.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.739+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f84a40b2920 con 0x7f84a40caee0 2024-03-04T07:28:37.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.740+0000 7f8499ffb640 1 -- 172.21.15.64:0/702974443 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 22 v22) v1 ==== 94+0+4971 (secure 0 0 0) 0x7f8494059810 con 0x7f84a40caee0 2024-03-04T07:28:37.739 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:37.740 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5138,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/627869633","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":627869633},{"type":"v1","addr":"172.21.15.66:6834","nonce":627869633}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5141,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2387393562","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2387393562},{"type":"v1","addr":"172.21.15.66:6835","nonce":2387393562}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5685,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/817733642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":817733642},{"type":"v1","addr":"172.21.15.64:6837","nonce":817733642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":18}],"filesystems":[{"mdsmap":{"epoch":22,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:28:34.726288+0000","modified":"2024-03-04T07:28:36.736175+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5688},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5688":{"gid":5688,"name":"a","rank":0,"incarnation":20,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/3981471748","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3981471748},{"type":"v1","addr":"172.21.15.64:6836","nonce":3981471748}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5688,"qdb_cluster":[5688]},"id":3}]} 2024-03-04T07:28:37.740 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 22 2024-03-04T07:28:37.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.742+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f848806b4d0 msgr2=0x7f848806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.742+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f848806b4d0 0x7f848806d990 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f84a0034000 tx=0x7f84a0031000 comp rx=0 tx=0).stop 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.742+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f84a40caee0 msgr2=0x7f84a40cbaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.742+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f84a40caee0 0x7f84a40cbaf0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f8494011aa0 tx=0x7f8494011f60 comp rx=0 tx=0).stop 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.742+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 shutdown_connections 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.742+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f848806b4d0 0x7f848806d990 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.743+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f84a415f740 0x7f84a415fbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.743+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f84a40caee0 0x7f84a40cbaf0 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.743+0000 7f84ab9bc640 1 --2- 172.21.15.64:0/702974443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8490003b90 0x7f84a40d3870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.743+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 >> 172.21.15.64:0/702974443 conn(0x7f84a4071960 msgr2=0x7f84a415d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.743+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 shutdown_connections 2024-03-04T07:28:37.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.743+0000 7f84ab9bc640 1 -- 172.21.15.64:0/702974443 wait complete. 2024-03-04T07:28:37.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:37.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.856+0000 7f6c6e9ee640 1 Processor -- start 2024-03-04T07:28:37.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.856+0000 7f6c6e9ee640 1 -- start start 2024-03-04T07:28:37.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c6e9ee640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6c68071540 0x7f6c680b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c6e9ee640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6c680d3400 0x7f6c68071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c6e9ee640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6c68071e60 con 0x7f6c680b2b50 2024-03-04T07:28:37.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c6e9ee640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c680d2f70 con 0x7f6c680d3400 2024-03-04T07:28:37.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c6e9ee640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6c680d4be0 con 0x7f6c68071540 2024-03-04T07:28:37.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c677fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6c680d3400 0x7f6c68071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c6cf8b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6c680b2b50 0x7f6c680d1870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40920/0 (socket says 172.21.15.64:40920) 2024-03-04T07:28:37.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c6cf8b640 1 -- 172.21.15.64:0/1542593474 learned_addr learned my addr 172.21.15.64:0/1542593474 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:37.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c67fff640 1 --2- 172.21.15.64:0/1542593474 >> v2:172.21.15.66:3300/0 conn(0x7f6c68071540 0x7f6c680b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c677fe640 1 -- 172.21.15.64:0/1542593474 >> v2:172.21.15.66:3300/0 conn(0x7f6c68071540 msgr2=0x7f6c680b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:37.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c677fe640 1 --2- 172.21.15.64:0/1542593474 >> v2:172.21.15.66:3300/0 conn(0x7f6c68071540 0x7f6c680b30a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c677fe640 1 -- 172.21.15.64:0/1542593474 >> v1:172.21.15.64:6789/0 conn(0x7f6c680b2b50 legacy=0x7f6c680d1870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c66ffd640 1 -- 172.21.15.64:0/1542593474 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2564767826 0 0) 0x7f6c68071e60 con 0x7f6c680b2b50 2024-03-04T07:28:37.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.857+0000 7f6c677fe640 1 -- 172.21.15.64:0/1542593474 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c680ce610 con 0x7f6c680d3400 2024-03-04T07:28:37.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.858+0000 7f6c677fe640 1 --2- 172.21.15.64:0/1542593474 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c68071920 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f6c680d4da0 tx=0x7f6c5c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c34e727b994a826 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:37.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.858+0000 7f6c66ffd640 1 -- 172.21.15.64:0/1542593474 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c5c035020 con 0x7f6c680d3400 2024-03-04T07:28:37.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.858+0000 7f6c66ffd640 1 -- 172.21.15.64:0/1542593474 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c5c031830 con 0x7f6c680d3400 2024-03-04T07:28:37.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.858+0000 7f6c66ffd640 1 -- 172.21.15.64:0/1542593474 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c5c031c00 con 0x7f6c680d3400 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/1542593474 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 msgr2=0x7f6c68071920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/1542593474 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c68071920 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f6c680d4da0 tx=0x7f6c5c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/1542593474 shutdown_connections 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/1542593474 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c68071920 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/1542593474 >> v2:172.21.15.66:3300/0 conn(0x7f6c68071540 0x7f6c680b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/1542593474 >> 172.21.15.64:0/1542593474 conn(0x7f6c680d3ca0 msgr2=0x7f6c680d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/1542593474 shutdown_connections 2024-03-04T07:28:37.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.859+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/1542593474 wait complete. 2024-03-04T07:28:37.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.860+0000 7f6c6e9ee640 1 Processor -- start 2024-03-04T07:28:37.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.861+0000 7f6c6e9ee640 1 -- start start 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6e9ee640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6c68071540 0x7f6c680b22a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6e9ee640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6c680b2b50 0x7f6c680c7440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6e9ee640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c680c7980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6e9ee640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c680ce8f0 con 0x7f6c680d3400 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6e9ee640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c680d2c00 con 0x7f6c68071540 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6e9ee640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c6815e370 con 0x7f6c680b2b50 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c680c7980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c680c7980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34990/0 (socket says 172.21.15.64:34990) 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 -- 172.21.15.64:0/3015268227 learned_addr learned my addr 172.21.15.64:0/3015268227 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c677fe640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6c680b2b50 0x7f6c680c7440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c67fff640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6c68071540 0x7f6c680b22a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 -- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6c680b2b50 msgr2=0x7f6c680c7440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6c680b2b50 0x7f6c680c7440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 -- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6c68071540 msgr2=0x7f6c680b22a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6c68071540 0x7f6c680b22a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c6cf8b640 1 -- 172.21.15.64:0/3015268227 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c680c7fb0 con 0x7f6c680d3400 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c677fe640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6c680b2b50 0x7f6c680c7440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.862+0000 7f6c67fff640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6c68071540 0x7f6c680b22a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.863+0000 7f6c6cf8b640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c680c7980 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f6c6000c7d0 tx=0x7f6c6000cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.863+0000 7f6c64ff9640 1 -- 172.21.15.64:0/3015268227 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c60018070 con 0x7f6c680d3400 2024-03-04T07:28:37.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.863+0000 7f6c64ff9640 1 -- 172.21.15.64:0/3015268227 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6c6000a640 con 0x7f6c680d3400 2024-03-04T07:28:37.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.863+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6c680c8290 con 0x7f6c680d3400 2024-03-04T07:28:37.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.863+0000 7f6c64ff9640 1 -- 172.21.15.64:0/3015268227 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c6000aa70 con 0x7f6c680d3400 2024-03-04T07:28:37.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.863+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c680c8850 con 0x7f6c680d3400 2024-03-04T07:28:37.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.864+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6c68068700 con 0x7f6c680d3400 2024-03-04T07:28:37.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.864+0000 7f6c64ff9640 1 -- 172.21.15.64:0/3015268227 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6c6001b070 con 0x7f6c680d3400 2024-03-04T07:28:37.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.865+0000 7f6c64ff9640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6c5406b4d0 0x7f6c5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:37.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.865+0000 7f6c64ff9640 1 -- 172.21.15.64:0/3015268227 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6c60089820 con 0x7f6c680d3400 2024-03-04T07:28:37.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.865+0000 7f6c67fff640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6c5406b4d0 0x7f6c5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:37.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.865+0000 7f6c67fff640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6c5406b4d0 0x7f6c5406d990 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f6c58008890 tx=0x7f6c58008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:37.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:37.869+0000 7f6c64ff9640 1 -- 172.21.15.64:0/3015268227 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6c600586a0 con 0x7f6c680d3400 2024-03-04T07:28:38.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.052+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6c6806b8e0 con 0x7f6c680d3400 2024-03-04T07:28:38.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.054+0000 7f6c64ff9640 1 -- 172.21.15.64:0/3015268227 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v44) v1 ==== 74+0+16565 (secure 0 0 0) 0x7f6c60057fa0 con 0x7f6c680d3400 2024-03-04T07:28:38.054 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:38.054 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":44,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:36.369097+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:28:32.989901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-04T07:28:33.692080+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"44","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:38.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6c5406b4d0 msgr2=0x7f6c5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6c5406b4d0 0x7f6c5406d990 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f6c58008890 tx=0x7f6c58008040 comp rx=0 tx=0).stop 2024-03-04T07:28:38.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 msgr2=0x7f6c680c7980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c680c7980 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f6c6000c7d0 tx=0x7f6c6000cc90 comp rx=0 tx=0).stop 2024-03-04T07:28:38.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 shutdown_connections 2024-03-04T07:28:38.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6c5406b4d0 0x7f6c5406d990 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6c680d3400 0x7f6c680c7980 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6c680b2b50 0x7f6c680c7440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 --2- 172.21.15.64:0/3015268227 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6c68071540 0x7f6c680b22a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 >> 172.21.15.64:0/3015268227 conn(0x7f6c680d3ca0 msgr2=0x7f6c68160b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:38.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.056+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 shutdown_connections 2024-03-04T07:28:38.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.057+0000 7f6c6e9ee640 1 -- 172.21.15.64:0/3015268227 wait complete. 2024-03-04T07:28:38.067 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:28:38.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:28:38.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.169+0000 7fb53a0df640 1 Processor -- start 2024-03-04T07:28:38.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.169+0000 7fb53a0df640 1 -- start start 2024-03-04T07:28:38.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb53a0df640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb534071540 0x7fb5340b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb53a0df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340d3400 0x7fb534071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb53a0df640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb534071e60 con 0x7fb5340b2b50 2024-03-04T07:28:38.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb53a0df640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5340d2f70 con 0x7fb534071540 2024-03-04T07:28:38.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb53a0df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5340d4be0 con 0x7fb5340d3400 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb532ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340d3400 0x7fb534071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb532ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340d3400 0x7fb534071920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:40340/0 (socket says 172.21.15.64:40340) 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb532ffd640 1 -- 172.21.15.64:0/1083133868 learned_addr learned my addr 172.21.15.64:0/1083133868 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.170+0000 7fb5337fe640 1 --2- 172.21.15.64:0/1083133868 >> v2:172.21.15.66:3300/0 conn(0x7fb534071540 0x7fb5340b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.171+0000 7fb5327fc640 1 -- 172.21.15.64:0/1083133868 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2844297228 0 0) 0x7fb534071e60 con 0x7fb5340b2b50 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.171+0000 7fb5327fc640 1 -- 172.21.15.64:0/1083133868 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb514003520 con 0x7fb5340b2b50 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.171+0000 7fb532ffd640 1 -- 172.21.15.64:0/1083133868 >> v2:172.21.15.66:3300/0 conn(0x7fb534071540 msgr2=0x7fb5340b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.171+0000 7fb532ffd640 1 --2- 172.21.15.64:0/1083133868 >> v2:172.21.15.66:3300/0 conn(0x7fb534071540 0x7fb5340b30a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.171+0000 7fb532ffd640 1 -- 172.21.15.64:0/1083133868 >> v1:172.21.15.66:6789/0 conn(0x7fb5340b2b50 legacy=0x7fb5340d1870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.171+0000 7fb532ffd640 1 -- 172.21.15.64:0/1083133868 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5340ce610 con 0x7fb5340d3400 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.171+0000 7fb532ffd640 1 --2- 172.21.15.64:0/1083133868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340d3400 0x7fb534071920 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fb51c009870 tx=0x7fb51c02b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2f4d8b00da8c651c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb5327fc640 1 -- 172.21.15.64:0/1083133868 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb51c034070 con 0x7fb5340d3400 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb5327fc640 1 -- 172.21.15.64:0/1083133868 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb51c030840 con 0x7fb5340d3400 2024-03-04T07:28:38.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb5327fc640 1 -- 172.21.15.64:0/1083133868 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb51c030c10 con 0x7fb5340d3400 2024-03-04T07:28:38.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 -- 172.21.15.64:0/1083133868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340d3400 msgr2=0x7fb534071920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1083133868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340d3400 0x7fb534071920 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fb51c009870 tx=0x7fb51c02b980 comp rx=0 tx=0).stop 2024-03-04T07:28:38.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 -- 172.21.15.64:0/1083133868 shutdown_connections 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1083133868 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340d3400 0x7fb534071920 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1083133868 >> v2:172.21.15.66:3300/0 conn(0x7fb534071540 0x7fb5340b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 -- 172.21.15.64:0/1083133868 >> 172.21.15.64:0/1083133868 conn(0x7fb5340d3ca0 msgr2=0x7fb5340d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 -- 172.21.15.64:0/1083133868 shutdown_connections 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.172+0000 7fb53a0df640 1 -- 172.21.15.64:0/1083133868 wait complete. 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.173+0000 7fb53a0df640 1 Processor -- start 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb53a0df640 1 -- start start 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb53a0df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb534071540 0x7fb53415a310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb53a0df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340b2b50 0x7fb5340c7440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb53a0df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5340d3400 0x7fb5340cc7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb53a0df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5340ce8f0 con 0x7fb5340d3400 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb53a0df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5340d2c00 con 0x7fb534071540 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb53a0df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb53415e370 con 0x7fb5340b2b50 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb533fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5340d3400 0x7fb5340cc7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb533fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5340d3400 0x7fb5340cc7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35000/0 (socket says 172.21.15.64:35000) 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.174+0000 7fb533fff640 1 -- 172.21.15.64:0/1143461985 learned_addr learned my addr 172.21.15.64:0/1143461985 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb532ffd640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340b2b50 0x7fb5340c7440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb5337fe640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb534071540 0x7fb53415a310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb533fff640 1 -- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340b2b50 msgr2=0x7fb5340c7440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb533fff640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340b2b50 0x7fb5340c7440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb533fff640 1 -- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb534071540 msgr2=0x7fb53415a310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb533fff640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb534071540 0x7fb53415a310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb533fff640 1 -- 172.21.15.64:0/1143461985 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5340b48b0 con 0x7fb5340d3400 2024-03-04T07:28:38.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb533fff640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5340d3400 0x7fb5340cc7a0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7fb52c00c790 tx=0x7fb52c00cc50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb52bfff640 1 -- 172.21.15.64:0/1143461985 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb52c003b70 con 0x7fb5340d3400 2024-03-04T07:28:38.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb52bfff640 1 -- 172.21.15.64:0/1143461985 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb52c003d10 con 0x7fb5340d3400 2024-03-04T07:28:38.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5340ca0e0 con 0x7fb5340d3400 2024-03-04T07:28:38.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb52bfff640 1 -- 172.21.15.64:0/1143461985 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb52c00aa50 con 0x7fb5340d3400 2024-03-04T07:28:38.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.175+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5340ca6a0 con 0x7fb5340d3400 2024-03-04T07:28:38.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.176+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb534068700 con 0x7fb5340d3400 2024-03-04T07:28:38.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.177+0000 7fb52bfff640 1 -- 172.21.15.64:0/1143461985 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb52c00abf0 con 0x7fb5340d3400 2024-03-04T07:28:38.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.177+0000 7fb52bfff640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb51806b500 0x7fb51806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.177+0000 7fb52bfff640 1 -- 172.21.15.64:0/1143461985 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb52c08a2c0 con 0x7fb5340d3400 2024-03-04T07:28:38.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.177+0000 7fb5337fe640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb51806b500 0x7fb51806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.178+0000 7fb5337fe640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb51806b500 0x7fb51806d9c0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fb524009ad0 tx=0x7fb524008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.186+0000 7fb52bfff640 1 -- 172.21.15.64:0/1143461985 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb52c059030 con 0x7fb5340d3400 2024-03-04T07:28:38.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.362+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fb53415da90 con 0x7fb51806b500 2024-03-04T07:28:38.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.374+0000 7fb52bfff640 1 -- 172.21.15.64:0/1143461985 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251888 (secure 0 0 0) 0x7fb53415da90 con 0x7fb51806b500 2024-03-04T07:28:38.377 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:38.378 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:28:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb51806b500 msgr2=0x7fb51806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb51806b500 0x7fb51806d9c0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fb524009ad0 tx=0x7fb524008040 comp rx=0 tx=0).stop 2024-03-04T07:28:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5340d3400 msgr2=0x7fb5340cc7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5340d3400 0x7fb5340cc7a0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7fb52c00c790 tx=0x7fb52c00cc50 comp rx=0 tx=0).stop 2024-03-04T07:28:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 shutdown_connections 2024-03-04T07:28:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb51806b500 0x7fb51806d9c0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5340d3400 0x7fb5340cc7a0 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5340b2b50 0x7fb5340c7440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 --2- 172.21.15.64:0/1143461985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb534071540 0x7fb53415a310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 >> 172.21.15.64:0/1143461985 conn(0x7fb5340d3ca0 msgr2=0x7fb53415f5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:38.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 shutdown_connections 2024-03-04T07:28:38.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.377+0000 7fb53a0df640 1 -- 172.21.15.64:0/1143461985 wait complete. 2024-03-04T07:28:38.388 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":95,"stamp":"2024-03-04T07:28:37.160910+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":67,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":254624,"kb_used_data":5128,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754720096,"statfs":{"total":773094113280,"available":772833378304,"internally_reserved":0,"allocated":5251072,"data_stored":2799902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000617"},"pg_stats":[{"pgid":"7.3f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390954+0000","last_change":"2024-03-04T07:28:35.386301+0000","last_active":"2024-03-04T07:28:35.390954+0000","last_peered":"2024-03-04T07:28:35.390954+0000","last_clean":"2024-03-04T07:28:35.390954+0000","last_became_active":"2024-03-04T07:28:35.385731+0000","last_became_peered":"2024-03-04T07:28:35.385731+0000","last_unstale":"2024-03-04T07:28:35.390954+0000","last_undegraded":"2024-03-04T07:28:35.390954+0000","last_fullsized":"2024-03-04T07:28:35.390954+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:20.808035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391273+0000","last_change":"2024-03-04T07:28:35.387855+0000","last_active":"2024-03-04T07:28:35.391273+0000","last_peered":"2024-03-04T07:28:35.391273+0000","last_clean":"2024-03-04T07:28:35.391273+0000","last_became_active":"2024-03-04T07:28:35.387557+0000","last_became_peered":"2024-03-04T07:28:35.387557+0000","last_unstale":"2024-03-04T07:28:35.391273+0000","last_undegraded":"2024-03-04T07:28:35.391273+0000","last_fullsized":"2024-03-04T07:28:35.391273+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:20.808035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.388400+0000","last_change":"2024-03-04T07:28:35.383936+0000","last_active":"2024-03-04T07:28:35.388400+0000","last_peered":"2024-03-04T07:28:35.388400+0000","last_clean":"2024-03-04T07:28:35.388400+0000","last_became_active":"2024-03-04T07:28:35.383665+0000","last_became_peered":"2024-03-04T07:28:35.383665+0000","last_unstale":"2024-03-04T07:28:35.388400+0000","last_undegraded":"2024-03-04T07:28:35.388400+0000","last_fullsized":"2024-03-04T07:28:35.388400+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:27:54.268773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398366+0000","last_change":"2024-03-04T07:28:35.393479+0000","last_active":"2024-03-04T07:28:35.398366+0000","last_peered":"2024-03-04T07:28:35.398366+0000","last_clean":"2024-03-04T07:28:35.398366+0000","last_became_active":"2024-03-04T07:28:35.393191+0000","last_became_peered":"2024-03-04T07:28:35.393191+0000","last_unstale":"2024-03-04T07:28:35.398366+0000","last_undegraded":"2024-03-04T07:28:35.398366+0000","last_fullsized":"2024-03-04T07:28:35.398366+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:27:54.268773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391019+0000","last_change":"2024-03-04T07:28:35.387862+0000","last_active":"2024-03-04T07:28:35.391019+0000","last_peered":"2024-03-04T07:28:35.391019+0000","last_clean":"2024-03-04T07:28:35.391019+0000","last_became_active":"2024-03-04T07:28:35.387563+0000","last_became_peered":"2024-03-04T07:28:35.387563+0000","last_unstale":"2024-03-04T07:28:35.391019+0000","last_undegraded":"2024-03-04T07:28:35.391019+0000","last_fullsized":"2024-03-04T07:28:35.391019+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:56:17.616203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396855+0000","last_change":"2024-03-04T07:28:35.390018+0000","last_active":"2024-03-04T07:28:35.396855+0000","last_peered":"2024-03-04T07:28:35.396855+0000","last_clean":"2024-03-04T07:28:35.396855+0000","last_became_active":"2024-03-04T07:28:35.389744+0000","last_became_peered":"2024-03-04T07:28:35.389744+0000","last_unstale":"2024-03-04T07:28:35.396855+0000","last_undegraded":"2024-03-04T07:28:35.396855+0000","last_fullsized":"2024-03-04T07:28:35.396855+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:21.080275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.39","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391039+0000","last_change":"2024-03-04T07:28:35.386005+0000","last_active":"2024-03-04T07:28:35.391039+0000","last_peered":"2024-03-04T07:28:35.391039+0000","last_clean":"2024-03-04T07:28:35.391039+0000","last_became_active":"2024-03-04T07:28:35.385676+0000","last_became_peered":"2024-03-04T07:28:35.385676+0000","last_unstale":"2024-03-04T07:28:35.391039+0000","last_undegraded":"2024-03-04T07:28:35.391039+0000","last_fullsized":"2024-03-04T07:28:35.391039+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:08:07.352168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.38","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398413+0000","last_change":"2024-03-04T07:28:35.393414+0000","last_active":"2024-03-04T07:28:35.398413+0000","last_peered":"2024-03-04T07:28:35.398413+0000","last_clean":"2024-03-04T07:28:35.398413+0000","last_became_active":"2024-03-04T07:28:35.392748+0000","last_became_peered":"2024-03-04T07:28:35.392748+0000","last_unstale":"2024-03-04T07:28:35.398413+0000","last_undegraded":"2024-03-04T07:28:35.398413+0000","last_fullsized":"2024-03-04T07:28:35.398413+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:48.003109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.37","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.395009+0000","last_change":"2024-03-04T07:28:35.388485+0000","last_active":"2024-03-04T07:28:35.395009+0000","last_peered":"2024-03-04T07:28:35.395009+0000","last_clean":"2024-03-04T07:28:35.395009+0000","last_became_active":"2024-03-04T07:28:35.387264+0000","last_became_peered":"2024-03-04T07:28:35.387264+0000","last_unstale":"2024-03-04T07:28:35.395009+0000","last_undegraded":"2024-03-04T07:28:35.395009+0000","last_fullsized":"2024-03-04T07:28:35.395009+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:15.340221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.36","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398013+0000","last_change":"2024-03-04T07:28:35.392528+0000","last_active":"2024-03-04T07:28:35.398013+0000","last_peered":"2024-03-04T07:28:35.398013+0000","last_clean":"2024-03-04T07:28:35.398013+0000","last_became_active":"2024-03-04T07:28:35.392051+0000","last_became_peered":"2024-03-04T07:28:35.392051+0000","last_unstale":"2024-03-04T07:28:35.398013+0000","last_undegraded":"2024-03-04T07:28:35.398013+0000","last_fullsized":"2024-03-04T07:28:35.398013+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:20:25.059386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.35","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396037+0000","last_change":"2024-03-04T07:28:35.390987+0000","last_active":"2024-03-04T07:28:35.396037+0000","last_peered":"2024-03-04T07:28:35.396037+0000","last_clean":"2024-03-04T07:28:35.396037+0000","last_became_active":"2024-03-04T07:28:35.390557+0000","last_became_peered":"2024-03-04T07:28:35.390557+0000","last_unstale":"2024-03-04T07:28:35.396037+0000","last_undegraded":"2024-03-04T07:28:35.396037+0000","last_fullsized":"2024-03-04T07:28:35.396037+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:39:56.900531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.34","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390932+0000","last_change":"2024-03-04T07:28:35.384783+0000","last_active":"2024-03-04T07:28:35.390932+0000","last_peered":"2024-03-04T07:28:35.390932+0000","last_clean":"2024-03-04T07:28:35.390932+0000","last_became_active":"2024-03-04T07:28:35.384490+0000","last_became_peered":"2024-03-04T07:28:35.384490+0000","last_unstale":"2024-03-04T07:28:35.390932+0000","last_undegraded":"2024-03-04T07:28:35.390932+0000","last_fullsized":"2024-03-04T07:28:35.390932+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:08:07.352168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.33","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390843+0000","last_change":"2024-03-04T07:28:35.389993+0000","last_active":"2024-03-04T07:28:35.390843+0000","last_peered":"2024-03-04T07:28:35.390843+0000","last_clean":"2024-03-04T07:28:35.390843+0000","last_became_active":"2024-03-04T07:28:35.388419+0000","last_became_peered":"2024-03-04T07:28:35.388419+0000","last_unstale":"2024-03-04T07:28:35.390843+0000","last_undegraded":"2024-03-04T07:28:35.390843+0000","last_fullsized":"2024-03-04T07:28:35.390843+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:07:25.627401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.32","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394895+0000","last_change":"2024-03-04T07:28:35.386955+0000","last_active":"2024-03-04T07:28:35.394895+0000","last_peered":"2024-03-04T07:28:35.394895+0000","last_clean":"2024-03-04T07:28:35.394895+0000","last_became_active":"2024-03-04T07:28:35.386539+0000","last_became_peered":"2024-03-04T07:28:35.386539+0000","last_unstale":"2024-03-04T07:28:35.394895+0000","last_undegraded":"2024-03-04T07:28:35.394895+0000","last_fullsized":"2024-03-04T07:28:35.394895+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:27.205310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.31","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393070+0000","last_change":"2024-03-04T07:28:35.387067+0000","last_active":"2024-03-04T07:28:35.393070+0000","last_peered":"2024-03-04T07:28:35.393070+0000","last_clean":"2024-03-04T07:28:35.393070+0000","last_became_active":"2024-03-04T07:28:35.386715+0000","last_became_peered":"2024-03-04T07:28:35.386715+0000","last_unstale":"2024-03-04T07:28:35.393070+0000","last_undegraded":"2024-03-04T07:28:35.393070+0000","last_fullsized":"2024-03-04T07:28:35.393070+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:35:56.294515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.30","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393024+0000","last_change":"2024-03-04T07:28:35.389480+0000","last_active":"2024-03-04T07:28:35.393024+0000","last_peered":"2024-03-04T07:28:35.393024+0000","last_clean":"2024-03-04T07:28:35.393024+0000","last_became_active":"2024-03-04T07:28:35.388674+0000","last_became_peered":"2024-03-04T07:28:35.388674+0000","last_unstale":"2024-03-04T07:28:35.393024+0000","last_undegraded":"2024-03-04T07:28:35.393024+0000","last_fullsized":"2024-03-04T07:28:35.393024+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:46:33.702514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.385746+0000","last_change":"2024-03-04T07:28:35.380670+0000","last_active":"2024-03-04T07:28:35.385746+0000","last_peered":"2024-03-04T07:28:35.385746+0000","last_clean":"2024-03-04T07:28:35.385746+0000","last_became_active":"2024-03-04T07:28:35.380384+0000","last_became_peered":"2024-03-04T07:28:35.380384+0000","last_unstale":"2024-03-04T07:28:35.385746+0000","last_undegraded":"2024-03-04T07:28:35.385746+0000","last_fullsized":"2024-03-04T07:28:35.385746+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:39:38.317488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394565+0000","last_change":"2024-03-04T07:28:35.391021+0000","last_active":"2024-03-04T07:28:35.394565+0000","last_peered":"2024-03-04T07:28:35.394565+0000","last_clean":"2024-03-04T07:28:35.394565+0000","last_became_active":"2024-03-04T07:28:35.390704+0000","last_became_peered":"2024-03-04T07:28:35.390704+0000","last_unstale":"2024-03-04T07:28:35.394565+0000","last_undegraded":"2024-03-04T07:28:35.394565+0000","last_fullsized":"2024-03-04T07:28:35.394565+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:42.397556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391884+0000","last_change":"2024-03-04T07:28:35.388426+0000","last_active":"2024-03-04T07:28:35.391884+0000","last_peered":"2024-03-04T07:28:35.391884+0000","last_clean":"2024-03-04T07:28:35.391884+0000","last_became_active":"2024-03-04T07:28:35.388044+0000","last_became_peered":"2024-03-04T07:28:35.388044+0000","last_unstale":"2024-03-04T07:28:35.391884+0000","last_undegraded":"2024-03-04T07:28:35.391884+0000","last_fullsized":"2024-03-04T07:28:35.391884+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:44:29.060773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390884+0000","last_change":"2024-03-04T07:28:35.387875+0000","last_active":"2024-03-04T07:28:35.390884+0000","last_peered":"2024-03-04T07:28:35.390884+0000","last_clean":"2024-03-04T07:28:35.390884+0000","last_became_active":"2024-03-04T07:28:35.387578+0000","last_became_peered":"2024-03-04T07:28:35.387578+0000","last_unstale":"2024-03-04T07:28:35.390884+0000","last_undegraded":"2024-03-04T07:28:35.390884+0000","last_fullsized":"2024-03-04T07:28:35.390884+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:47:27.934672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398513+0000","last_change":"2024-03-04T07:28:35.394789+0000","last_active":"2024-03-04T07:28:35.398513+0000","last_peered":"2024-03-04T07:28:35.398513+0000","last_clean":"2024-03-04T07:28:35.398513+0000","last_became_active":"2024-03-04T07:28:35.394517+0000","last_became_peered":"2024-03-04T07:28:35.394517+0000","last_unstale":"2024-03-04T07:28:35.398513+0000","last_undegraded":"2024-03-04T07:28:35.398513+0000","last_fullsized":"2024-03-04T07:28:35.398513+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:41:51.471223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398051+0000","last_change":"2024-03-04T07:28:35.392919+0000","last_active":"2024-03-04T07:28:35.398051+0000","last_peered":"2024-03-04T07:28:35.398051+0000","last_clean":"2024-03-04T07:28:35.398051+0000","last_became_active":"2024-03-04T07:28:35.392648+0000","last_became_peered":"2024-03-04T07:28:35.392648+0000","last_unstale":"2024-03-04T07:28:35.398051+0000","last_undegraded":"2024-03-04T07:28:35.398051+0000","last_fullsized":"2024-03-04T07:28:35.398051+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:39:38.317488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.29","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391912+0000","last_change":"2024-03-04T07:28:35.389395+0000","last_active":"2024-03-04T07:28:35.391912+0000","last_peered":"2024-03-04T07:28:35.391912+0000","last_clean":"2024-03-04T07:28:35.391912+0000","last_became_active":"2024-03-04T07:28:35.389040+0000","last_became_peered":"2024-03-04T07:28:35.389040+0000","last_unstale":"2024-03-04T07:28:35.391912+0000","last_undegraded":"2024-03-04T07:28:35.391912+0000","last_fullsized":"2024-03-04T07:28:35.391912+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:46:18.438852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.28","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393041+0000","last_change":"2024-03-04T07:28:35.391034+0000","last_active":"2024-03-04T07:28:35.393041+0000","last_peered":"2024-03-04T07:28:35.393041+0000","last_clean":"2024-03-04T07:28:35.393041+0000","last_became_active":"2024-03-04T07:28:35.390440+0000","last_became_peered":"2024-03-04T07:28:35.390440+0000","last_unstale":"2024-03-04T07:28:35.393041+0000","last_undegraded":"2024-03-04T07:28:35.393041+0000","last_fullsized":"2024-03-04T07:28:35.393041+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:27.205310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.27","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391590+0000","last_change":"2024-03-04T07:28:35.387632+0000","last_active":"2024-03-04T07:28:35.391590+0000","last_peered":"2024-03-04T07:28:35.391590+0000","last_clean":"2024-03-04T07:28:35.391590+0000","last_became_active":"2024-03-04T07:28:35.387329+0000","last_became_peered":"2024-03-04T07:28:35.387329+0000","last_unstale":"2024-03-04T07:28:35.391590+0000","last_undegraded":"2024-03-04T07:28:35.391590+0000","last_fullsized":"2024-03-04T07:28:35.391590+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:26.648603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.26","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390965+0000","last_change":"2024-03-04T07:28:35.388767+0000","last_active":"2024-03-04T07:28:35.390965+0000","last_peered":"2024-03-04T07:28:35.390965+0000","last_clean":"2024-03-04T07:28:35.390965+0000","last_became_active":"2024-03-04T07:28:35.388245+0000","last_became_peered":"2024-03-04T07:28:35.388245+0000","last_unstale":"2024-03-04T07:28:35.390965+0000","last_undegraded":"2024-03-04T07:28:35.390965+0000","last_fullsized":"2024-03-04T07:28:35.390965+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:30:31.008078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.25","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.388376+0000","last_change":"2024-03-04T07:28:35.382911+0000","last_active":"2024-03-04T07:28:35.388376+0000","last_peered":"2024-03-04T07:28:35.388376+0000","last_clean":"2024-03-04T07:28:35.388376+0000","last_became_active":"2024-03-04T07:28:35.382508+0000","last_became_peered":"2024-03-04T07:28:35.382508+0000","last_unstale":"2024-03-04T07:28:35.388376+0000","last_undegraded":"2024-03-04T07:28:35.388376+0000","last_fullsized":"2024-03-04T07:28:35.388376+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:36:13.026173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.24","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.388594+0000","last_change":"2024-03-04T07:28:35.382446+0000","last_active":"2024-03-04T07:28:35.388594+0000","last_peered":"2024-03-04T07:28:35.388594+0000","last_clean":"2024-03-04T07:28:35.388594+0000","last_became_active":"2024-03-04T07:28:35.382130+0000","last_became_peered":"2024-03-04T07:28:35.382130+0000","last_unstale":"2024-03-04T07:28:35.388594+0000","last_undegraded":"2024-03-04T07:28:35.388594+0000","last_fullsized":"2024-03-04T07:28:35.388594+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:24.035297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.23","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391963+0000","last_change":"2024-03-04T07:28:35.389333+0000","last_active":"2024-03-04T07:28:35.391963+0000","last_peered":"2024-03-04T07:28:35.391963+0000","last_clean":"2024-03-04T07:28:35.391963+0000","last_became_active":"2024-03-04T07:28:35.388996+0000","last_became_peered":"2024-03-04T07:28:35.388996+0000","last_unstale":"2024-03-04T07:28:35.391963+0000","last_undegraded":"2024-03-04T07:28:35.391963+0000","last_fullsized":"2024-03-04T07:28:35.391963+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:35:26.271770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.22","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391241+0000","last_change":"2024-03-04T07:28:35.386665+0000","last_active":"2024-03-04T07:28:35.391241+0000","last_peered":"2024-03-04T07:28:35.391241+0000","last_clean":"2024-03-04T07:28:35.391241+0000","last_became_active":"2024-03-04T07:28:35.386189+0000","last_became_peered":"2024-03-04T07:28:35.386189+0000","last_unstale":"2024-03-04T07:28:35.391241+0000","last_undegraded":"2024-03-04T07:28:35.391241+0000","last_fullsized":"2024-03-04T07:28:35.391241+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:48.003109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.21","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396818+0000","last_change":"2024-03-04T07:28:35.391088+0000","last_active":"2024-03-04T07:28:35.396818+0000","last_peered":"2024-03-04T07:28:35.396818+0000","last_clean":"2024-03-04T07:28:35.396818+0000","last_became_active":"2024-03-04T07:28:35.390835+0000","last_became_peered":"2024-03-04T07:28:35.390835+0000","last_unstale":"2024-03-04T07:28:35.396818+0000","last_undegraded":"2024-03-04T07:28:35.396818+0000","last_fullsized":"2024-03-04T07:28:35.396818+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:40:01.718326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.20","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.384670+0000","last_change":"2024-03-04T07:28:35.378799+0000","last_active":"2024-03-04T07:28:35.384670+0000","last_peered":"2024-03-04T07:28:35.384670+0000","last_clean":"2024-03-04T07:28:35.384670+0000","last_became_active":"2024-03-04T07:28:35.378549+0000","last_became_peered":"2024-03-04T07:28:35.378549+0000","last_unstale":"2024-03-04T07:28:35.384670+0000","last_undegraded":"2024-03-04T07:28:35.384670+0000","last_fullsized":"2024-03-04T07:28:35.384670+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:38:03.967307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394331+0000","last_change":"2024-03-04T07:28:35.389778+0000","last_active":"2024-03-04T07:28:35.394331+0000","last_peered":"2024-03-04T07:28:35.394331+0000","last_clean":"2024-03-04T07:28:35.394331+0000","last_became_active":"2024-03-04T07:28:35.389483+0000","last_became_peered":"2024-03-04T07:28:35.389483+0000","last_unstale":"2024-03-04T07:28:35.394331+0000","last_undegraded":"2024-03-04T07:28:35.394331+0000","last_fullsized":"2024-03-04T07:28:35.394331+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:27:54.268773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1f","version":"43'4","reported_seq":16,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.406497+0000","last_change":"2024-03-04T07:28:34.375428+0000","last_active":"2024-03-04T07:28:35.406497+0000","last_peered":"2024-03-04T07:28:35.406497+0000","last_clean":"2024-03-04T07:28:35.406497+0000","last_became_active":"2024-03-04T07:28:34.375154+0000","last_became_peered":"2024-03-04T07:28:34.375154+0000","last_unstale":"2024-03-04T07:28:35.406497+0000","last_undegraded":"2024-03-04T07:28:35.406497+0000","last_fullsized":"2024-03-04T07:28:35.406497+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:02.999903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391576+0000","last_change":"2024-03-04T07:28:35.387444+0000","last_active":"2024-03-04T07:28:35.391576+0000","last_peered":"2024-03-04T07:28:35.391576+0000","last_clean":"2024-03-04T07:28:35.391576+0000","last_became_active":"2024-03-04T07:28:35.387122+0000","last_became_peered":"2024-03-04T07:28:35.387122+0000","last_unstale":"2024-03-04T07:28:35.391576+0000","last_undegraded":"2024-03-04T07:28:35.391576+0000","last_fullsized":"2024-03-04T07:28:35.391576+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:05.234107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1e","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.405762+0000","last_change":"2024-03-04T07:28:34.372736+0000","last_active":"2024-03-04T07:28:35.405762+0000","last_peered":"2024-03-04T07:28:35.405762+0000","last_clean":"2024-03-04T07:28:35.405762+0000","last_became_active":"2024-03-04T07:28:34.372341+0000","last_became_peered":"2024-03-04T07:28:34.372341+0000","last_unstale":"2024-03-04T07:28:35.405762+0000","last_undegraded":"2024-03-04T07:28:35.405762+0000","last_fullsized":"2024-03-04T07:28:35.405762+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:44:26.407962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.1c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394797+0000","last_change":"2024-03-04T07:28:35.392431+0000","last_active":"2024-03-04T07:28:35.394797+0000","last_peered":"2024-03-04T07:28:35.394797+0000","last_clean":"2024-03-04T07:28:35.394797+0000","last_became_active":"2024-03-04T07:28:35.391875+0000","last_became_peered":"2024-03-04T07:28:35.391875+0000","last_unstale":"2024-03-04T07:28:35.394797+0000","last_undegraded":"2024-03-04T07:28:35.394797+0000","last_fullsized":"2024-03-04T07:28:35.394797+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:04.006481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1d","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403441+0000","last_change":"2024-03-04T07:28:34.374183+0000","last_active":"2024-03-04T07:28:35.403441+0000","last_peered":"2024-03-04T07:28:35.403441+0000","last_clean":"2024-03-04T07:28:35.403441+0000","last_became_active":"2024-03-04T07:28:34.373834+0000","last_became_peered":"2024-03-04T07:28:34.373834+0000","last_unstale":"2024-03-04T07:28:35.403441+0000","last_undegraded":"2024-03-04T07:28:35.403441+0000","last_fullsized":"2024-03-04T07:28:35.403441+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:39:37.310910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.400829+0000","last_change":"2024-03-04T07:28:35.394680+0000","last_active":"2024-03-04T07:28:35.400829+0000","last_peered":"2024-03-04T07:28:35.400829+0000","last_clean":"2024-03-04T07:28:35.400829+0000","last_became_active":"2024-03-04T07:28:35.393884+0000","last_became_peered":"2024-03-04T07:28:35.393884+0000","last_unstale":"2024-03-04T07:28:35.400829+0000","last_undegraded":"2024-03-04T07:28:35.400829+0000","last_fullsized":"2024-03-04T07:28:35.400829+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:48.003109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403230+0000","last_change":"2024-03-04T07:28:34.373187+0000","last_active":"2024-03-04T07:28:35.403230+0000","last_peered":"2024-03-04T07:28:35.403230+0000","last_clean":"2024-03-04T07:28:35.403230+0000","last_became_active":"2024-03-04T07:28:34.372783+0000","last_became_peered":"2024-03-04T07:28:34.372783+0000","last_unstale":"2024-03-04T07:28:35.403230+0000","last_undegraded":"2024-03-04T07:28:35.403230+0000","last_fullsized":"2024-03-04T07:28:35.403230+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:29.155064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.1a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394548+0000","last_change":"2024-03-04T07:28:35.392054+0000","last_active":"2024-03-04T07:28:35.394548+0000","last_peered":"2024-03-04T07:28:35.394548+0000","last_clean":"2024-03-04T07:28:35.394548+0000","last_became_active":"2024-03-04T07:28:35.391748+0000","last_became_peered":"2024-03-04T07:28:35.391748+0000","last_unstale":"2024-03-04T07:28:35.394548+0000","last_undegraded":"2024-03-04T07:28:35.394548+0000","last_fullsized":"2024-03-04T07:28:35.394548+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:37.738646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403106+0000","last_change":"2024-03-04T07:28:34.373176+0000","last_active":"2024-03-04T07:28:35.403106+0000","last_peered":"2024-03-04T07:28:35.403106+0000","last_clean":"2024-03-04T07:28:35.403106+0000","last_became_active":"2024-03-04T07:28:34.372905+0000","last_became_peered":"2024-03-04T07:28:34.372905+0000","last_unstale":"2024-03-04T07:28:35.403106+0000","last_undegraded":"2024-03-04T07:28:35.403106+0000","last_fullsized":"2024-03-04T07:28:35.403106+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:28:11.075094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.392053+0000","last_change":"2024-03-04T07:28:35.386741+0000","last_active":"2024-03-04T07:28:35.392053+0000","last_peered":"2024-03-04T07:28:35.392053+0000","last_clean":"2024-03-04T07:28:35.392053+0000","last_became_active":"2024-03-04T07:28:35.386444+0000","last_became_peered":"2024-03-04T07:28:35.386444+0000","last_unstale":"2024-03-04T07:28:35.392053+0000","last_undegraded":"2024-03-04T07:28:35.392053+0000","last_fullsized":"2024-03-04T07:28:35.392053+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:57:26.300871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1a","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.397711+0000","last_change":"2024-03-04T07:28:34.374405+0000","last_active":"2024-03-04T07:28:35.397711+0000","last_peered":"2024-03-04T07:28:35.397711+0000","last_clean":"2024-03-04T07:28:35.397711+0000","last_became_active":"2024-03-04T07:28:34.373892+0000","last_became_peered":"2024-03-04T07:28:34.373892+0000","last_unstale":"2024-03-04T07:28:35.397711+0000","last_undegraded":"2024-03-04T07:28:35.397711+0000","last_fullsized":"2024-03-04T07:28:35.397711+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:14.333643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398435+0000","last_change":"2024-03-04T07:28:35.394047+0000","last_active":"2024-03-04T07:28:35.398435+0000","last_peered":"2024-03-04T07:28:35.398435+0000","last_clean":"2024-03-04T07:28:35.398435+0000","last_became_active":"2024-03-04T07:28:35.393677+0000","last_became_peered":"2024-03-04T07:28:35.393677+0000","last_unstale":"2024-03-04T07:28:35.398435+0000","last_undegraded":"2024-03-04T07:28:35.398435+0000","last_fullsized":"2024-03-04T07:28:35.398435+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:30.161642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.373689+0000","last_change":"2024-03-04T07:28:34.372986+0000","last_active":"2024-03-04T07:28:35.373689+0000","last_peered":"2024-03-04T07:28:35.373689+0000","last_clean":"2024-03-04T07:28:35.373689+0000","last_became_active":"2024-03-04T07:28:34.372642+0000","last_became_peered":"2024-03-04T07:28:34.372642+0000","last_unstale":"2024-03-04T07:28:35.373689+0000","last_undegraded":"2024-03-04T07:28:35.373689+0000","last_fullsized":"2024-03-04T07:28:35.373689+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:29.155064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394999+0000","last_change":"2024-03-04T07:28:35.389959+0000","last_active":"2024-03-04T07:28:35.394999+0000","last_peered":"2024-03-04T07:28:35.394999+0000","last_clean":"2024-03-04T07:28:35.394999+0000","last_became_active":"2024-03-04T07:28:35.389454+0000","last_became_peered":"2024-03-04T07:28:35.389454+0000","last_unstale":"2024-03-04T07:28:35.394999+0000","last_undegraded":"2024-03-04T07:28:35.394999+0000","last_fullsized":"2024-03-04T07:28:35.394999+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:57:26.300871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.18","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377550+0000","last_change":"2024-03-04T07:28:34.378843+0000","last_active":"2024-03-04T07:28:35.377550+0000","last_peered":"2024-03-04T07:28:35.377550+0000","last_clean":"2024-03-04T07:28:35.377550+0000","last_became_active":"2024-03-04T07:28:34.378564+0000","last_became_peered":"2024-03-04T07:28:34.378564+0000","last_unstale":"2024-03-04T07:28:35.377550+0000","last_undegraded":"2024-03-04T07:28:35.377550+0000","last_fullsized":"2024-03-04T07:28:35.377550+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:07.978415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.16","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393836+0000","last_change":"2024-03-04T07:28:35.389893+0000","last_active":"2024-03-04T07:28:35.393836+0000","last_peered":"2024-03-04T07:28:35.393836+0000","last_clean":"2024-03-04T07:28:35.393836+0000","last_became_active":"2024-03-04T07:28:35.389585+0000","last_became_peered":"2024-03-04T07:28:35.389585+0000","last_unstale":"2024-03-04T07:28:35.393836+0000","last_undegraded":"2024-03-04T07:28:35.393836+0000","last_fullsized":"2024-03-04T07:28:35.393836+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:30.161642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.17","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.374293+0000","last_change":"2024-03-04T07:28:34.376739+0000","last_active":"2024-03-04T07:28:35.374293+0000","last_peered":"2024-03-04T07:28:35.374293+0000","last_clean":"2024-03-04T07:28:35.374293+0000","last_became_active":"2024-03-04T07:28:34.376306+0000","last_became_peered":"2024-03-04T07:28:34.376306+0000","last_unstale":"2024-03-04T07:28:35.374293+0000","last_undegraded":"2024-03-04T07:28:35.374293+0000","last_fullsized":"2024-03-04T07:28:35.374293+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:26.379900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390976+0000","last_change":"2024-03-04T07:28:35.386548+0000","last_active":"2024-03-04T07:28:35.390976+0000","last_peered":"2024-03-04T07:28:35.390976+0000","last_clean":"2024-03-04T07:28:35.390976+0000","last_became_active":"2024-03-04T07:28:35.386008+0000","last_became_peered":"2024-03-04T07:28:35.386008+0000","last_unstale":"2024-03-04T07:28:35.390976+0000","last_undegraded":"2024-03-04T07:28:35.390976+0000","last_fullsized":"2024-03-04T07:28:35.390976+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:56:17.616203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.16","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.376961+0000","last_change":"2024-03-04T07:28:34.378266+0000","last_active":"2024-03-04T07:28:35.376961+0000","last_peered":"2024-03-04T07:28:35.376961+0000","last_clean":"2024-03-04T07:28:35.376961+0000","last_became_active":"2024-03-04T07:28:34.377944+0000","last_became_peered":"2024-03-04T07:28:34.377944+0000","last_unstale":"2024-03-04T07:28:35.376961+0000","last_undegraded":"2024-03-04T07:28:35.376961+0000","last_fullsized":"2024-03-04T07:28:35.376961+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:44:28.054195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396692+0000","last_change":"2024-03-04T07:28:35.388083+0000","last_active":"2024-03-04T07:28:35.396692+0000","last_peered":"2024-03-04T07:28:35.396692+0000","last_clean":"2024-03-04T07:28:35.396692+0000","last_became_active":"2024-03-04T07:28:35.387825+0000","last_became_peered":"2024-03-04T07:28:35.387825+0000","last_unstale":"2024-03-04T07:28:35.396692+0000","last_undegraded":"2024-03-04T07:28:35.396692+0000","last_fullsized":"2024-03-04T07:28:35.396692+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:53:29.133169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"43'3","reported_seq":15,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.405951+0000","last_change":"2024-03-04T07:28:34.378354+0000","last_active":"2024-03-04T07:28:35.405951+0000","last_peered":"2024-03-04T07:28:35.405951+0000","last_clean":"2024-03-04T07:28:35.405951+0000","last_became_active":"2024-03-04T07:28:34.378086+0000","last_became_peered":"2024-03-04T07:28:34.378086+0000","last_unstale":"2024-03-04T07:28:35.405951+0000","last_undegraded":"2024-03-04T07:28:35.405951+0000","last_fullsized":"2024-03-04T07:28:35.405951+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:36.732068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.395121+0000","last_change":"2024-03-04T07:28:35.392827+0000","last_active":"2024-03-04T07:28:35.395121+0000","last_peered":"2024-03-04T07:28:35.395121+0000","last_clean":"2024-03-04T07:28:35.395121+0000","last_became_active":"2024-03-04T07:28:35.392517+0000","last_became_peered":"2024-03-04T07:28:35.392517+0000","last_unstale":"2024-03-04T07:28:35.395121+0000","last_undegraded":"2024-03-04T07:28:35.395121+0000","last_fullsized":"2024-03-04T07:28:35.395121+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:28:12.081672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"43'6","reported_seq":18,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.406163+0000","last_change":"2024-03-04T07:28:34.372976+0000","last_active":"2024-03-04T07:28:35.406163+0000","last_peered":"2024-03-04T07:28:35.406163+0000","last_clean":"2024-03-04T07:28:35.406163+0000","last_became_active":"2024-03-04T07:28:34.372677+0000","last_became_peered":"2024-03-04T07:28:34.372677+0000","last_unstale":"2024-03-04T07:28:35.406163+0000","last_undegraded":"2024-03-04T07:28:35.406163+0000","last_fullsized":"2024-03-04T07:28:35.406163+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:00:40.924212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.12","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.392105+0000","last_change":"2024-03-04T07:28:35.389045+0000","last_active":"2024-03-04T07:28:35.392105+0000","last_peered":"2024-03-04T07:28:35.392105+0000","last_clean":"2024-03-04T07:28:35.392105+0000","last_became_active":"2024-03-04T07:28:35.388749+0000","last_became_peered":"2024-03-04T07:28:35.388749+0000","last_unstale":"2024-03-04T07:28:35.392105+0000","last_undegraded":"2024-03-04T07:28:35.392105+0000","last_fullsized":"2024-03-04T07:28:35.392105+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:08.984993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.402682+0000","last_change":"2024-03-04T07:28:34.378924+0000","last_active":"2024-03-04T07:28:35.402682+0000","last_peered":"2024-03-04T07:28:35.402682+0000","last_clean":"2024-03-04T07:28:35.402682+0000","last_became_active":"2024-03-04T07:28:34.378611+0000","last_became_peered":"2024-03-04T07:28:34.378611+0000","last_unstale":"2024-03-04T07:28:35.402682+0000","last_undegraded":"2024-03-04T07:28:35.402682+0000","last_fullsized":"2024-03-04T07:28:35.402682+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:19:49.334496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.397903+0000","last_change":"2024-03-04T07:28:35.391894+0000","last_active":"2024-03-04T07:28:35.397903+0000","last_peered":"2024-03-04T07:28:35.397903+0000","last_clean":"2024-03-04T07:28:35.397903+0000","last_became_active":"2024-03-04T07:28:35.391627+0000","last_became_peered":"2024-03-04T07:28:35.391627+0000","last_unstale":"2024-03-04T07:28:35.397903+0000","last_undegraded":"2024-03-04T07:28:35.397903+0000","last_fullsized":"2024-03-04T07:28:35.397903+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:44:27.414540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.12","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377915+0000","last_change":"2024-03-04T07:28:34.370402+0000","last_active":"2024-03-04T07:28:35.377915+0000","last_peered":"2024-03-04T07:28:35.377915+0000","last_clean":"2024-03-04T07:28:35.377915+0000","last_became_active":"2024-03-04T07:28:34.370062+0000","last_became_peered":"2024-03-04T07:28:34.370062+0000","last_unstale":"2024-03-04T07:28:35.377915+0000","last_undegraded":"2024-03-04T07:28:35.377915+0000","last_fullsized":"2024-03-04T07:28:35.377915+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:41.390978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.10","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393042+0000","last_change":"2024-03-04T07:28:35.386850+0000","last_active":"2024-03-04T07:28:35.393042+0000","last_peered":"2024-03-04T07:28:35.393042+0000","last_clean":"2024-03-04T07:28:35.393042+0000","last_became_active":"2024-03-04T07:28:35.386512+0000","last_became_peered":"2024-03-04T07:28:35.386512+0000","last_unstale":"2024-03-04T07:28:35.393042+0000","last_undegraded":"2024-03-04T07:28:35.393042+0000","last_fullsized":"2024-03-04T07:28:35.393042+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:09.518032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.11","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.380159+0000","last_change":"2024-03-04T07:28:34.379003+0000","last_active":"2024-03-04T07:28:35.380159+0000","last_peered":"2024-03-04T07:28:35.380159+0000","last_clean":"2024-03-04T07:28:35.380159+0000","last_became_active":"2024-03-04T07:28:34.378746+0000","last_became_peered":"2024-03-04T07:28:34.378746+0000","last_unstale":"2024-03-04T07:28:35.380159+0000","last_undegraded":"2024-03-04T07:28:35.380159+0000","last_fullsized":"2024-03-04T07:28:35.380159+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:19.801457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.399810+0000","last_change":"2024-03-04T07:28:35.393367+0000","last_active":"2024-03-04T07:28:35.399810+0000","last_peered":"2024-03-04T07:28:35.399810+0000","last_clean":"2024-03-04T07:28:35.399810+0000","last_became_active":"2024-03-04T07:28:35.393071+0000","last_became_peered":"2024-03-04T07:28:35.393071+0000","last_unstale":"2024-03-04T07:28:35.399810+0000","last_undegraded":"2024-03-04T07:28:35.399810+0000","last_fullsized":"2024-03-04T07:28:35.399810+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:19:50.341073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"43'8","reported_seq":20,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.409398+0000","last_change":"2024-03-04T07:28:34.380700+0000","last_active":"2024-03-04T07:28:35.409398+0000","last_peered":"2024-03-04T07:28:35.409398+0000","last_clean":"2024-03-04T07:28:35.409398+0000","last_became_active":"2024-03-04T07:28:34.380395+0000","last_became_peered":"2024-03-04T07:28:34.380395+0000","last_unstale":"2024-03-04T07:28:35.409398+0000","last_undegraded":"2024-03-04T07:28:35.409398+0000","last_fullsized":"2024-03-04T07:28:35.409398+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:35:25.265192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.383585+0000","last_change":"2024-03-04T07:28:35.377087+0000","last_active":"2024-03-04T07:28:35.383585+0000","last_peered":"2024-03-04T07:28:35.383585+0000","last_clean":"2024-03-04T07:28:35.383585+0000","last_became_active":"2024-03-04T07:28:35.376813+0000","last_became_peered":"2024-03-04T07:28:35.376813+0000","last_unstale":"2024-03-04T07:28:35.383585+0000","last_undegraded":"2024-03-04T07:28:35.383585+0000","last_fullsized":"2024-03-04T07:28:35.383585+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:19.913686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.402398+0000","last_change":"2024-03-04T07:28:34.373601+0000","last_active":"2024-03-04T07:28:35.402398+0000","last_peered":"2024-03-04T07:28:35.402398+0000","last_clean":"2024-03-04T07:28:35.402398+0000","last_became_active":"2024-03-04T07:28:34.373332+0000","last_became_peered":"2024-03-04T07:28:34.373332+0000","last_unstale":"2024-03-04T07:28:35.402398+0000","last_undegraded":"2024-03-04T07:28:35.402398+0000","last_fullsized":"2024-03-04T07:28:35.402398+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:06:34.443138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391821+0000","last_change":"2024-03-04T07:28:35.386018+0000","last_active":"2024-03-04T07:28:35.391821+0000","last_peered":"2024-03-04T07:28:35.391821+0000","last_clean":"2024-03-04T07:28:35.391821+0000","last_became_active":"2024-03-04T07:28:35.385596+0000","last_became_peered":"2024-03-04T07:28:35.385596+0000","last_unstale":"2024-03-04T07:28:35.391821+0000","last_undegraded":"2024-03-04T07:28:35.391821+0000","last_fullsized":"2024-03-04T07:28:35.391821+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:21:48.544037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.400759+0000","last_change":"2024-03-04T07:28:34.369493+0000","last_active":"2024-03-04T07:28:35.400759+0000","last_peered":"2024-03-04T07:28:35.400759+0000","last_clean":"2024-03-04T07:28:35.400759+0000","last_became_active":"2024-03-04T07:28:34.369199+0000","last_became_peered":"2024-03-04T07:28:34.369199+0000","last_unstale":"2024-03-04T07:28:35.400759+0000","last_undegraded":"2024-03-04T07:28:35.400759+0000","last_fullsized":"2024-03-04T07:28:35.400759+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:20:24.052808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.392988+0000","last_change":"2024-03-04T07:28:35.389946+0000","last_active":"2024-03-04T07:28:35.392988+0000","last_peered":"2024-03-04T07:28:35.392988+0000","last_clean":"2024-03-04T07:28:35.392988+0000","last_became_active":"2024-03-04T07:28:35.389263+0000","last_became_peered":"2024-03-04T07:28:35.389263+0000","last_unstale":"2024-03-04T07:28:35.392988+0000","last_undegraded":"2024-03-04T07:28:35.392988+0000","last_fullsized":"2024-03-04T07:28:35.392988+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:21:48.544037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403052+0000","last_change":"2024-03-04T07:28:34.369186+0000","last_active":"2024-03-04T07:28:35.403052+0000","last_peered":"2024-03-04T07:28:35.403052+0000","last_clean":"2024-03-04T07:28:35.403052+0000","last_became_active":"2024-03-04T07:28:34.368869+0000","last_became_peered":"2024-03-04T07:28:34.368869+0000","last_unstale":"2024-03-04T07:28:35.403052+0000","last_undegraded":"2024-03-04T07:28:35.403052+0000","last_fullsized":"2024-03-04T07:28:35.403052+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:42:37.468797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.397646+0000","last_change":"2024-03-04T07:28:35.393868+0000","last_active":"2024-03-04T07:28:35.397646+0000","last_peered":"2024-03-04T07:28:35.397646+0000","last_clean":"2024-03-04T07:28:35.397646+0000","last_became_active":"2024-03-04T07:28:35.393247+0000","last_became_peered":"2024-03-04T07:28:35.393247+0000","last_unstale":"2024-03-04T07:28:35.397646+0000","last_undegraded":"2024-03-04T07:28:35.397646+0000","last_fullsized":"2024-03-04T07:28:35.397646+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:27.386478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.381095+0000","last_change":"2024-03-04T07:28:34.376484+0000","last_active":"2024-03-04T07:28:35.381095+0000","last_peered":"2024-03-04T07:28:35.381095+0000","last_clean":"2024-03-04T07:28:35.381095+0000","last_became_active":"2024-03-04T07:28:34.376178+0000","last_became_peered":"2024-03-04T07:28:34.376178+0000","last_unstale":"2024-03-04T07:28:35.381095+0000","last_undegraded":"2024-03-04T07:28:35.381095+0000","last_fullsized":"2024-03-04T07:28:35.381095+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:56:16.609626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396798+0000","last_change":"2024-03-04T07:28:35.388379+0000","last_active":"2024-03-04T07:28:35.396798+0000","last_peered":"2024-03-04T07:28:35.396798+0000","last_clean":"2024-03-04T07:28:35.396798+0000","last_became_active":"2024-03-04T07:28:35.387541+0000","last_became_peered":"2024-03-04T07:28:35.387541+0000","last_unstale":"2024-03-04T07:28:35.396798+0000","last_undegraded":"2024-03-04T07:28:35.396798+0000","last_fullsized":"2024-03-04T07:28:35.396798+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:08.866801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.405845+0000","last_change":"2024-03-04T07:28:34.373024+0000","last_active":"2024-03-04T07:28:35.405845+0000","last_peered":"2024-03-04T07:28:35.405845+0000","last_clean":"2024-03-04T07:28:35.405845+0000","last_became_active":"2024-03-04T07:28:34.372705+0000","last_became_peered":"2024-03-04T07:28:34.372705+0000","last_unstale":"2024-03-04T07:28:35.405845+0000","last_undegraded":"2024-03-04T07:28:35.405845+0000","last_fullsized":"2024-03-04T07:28:35.405845+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:29:03.369932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391229+0000","last_change":"2024-03-04T07:28:35.383641+0000","last_active":"2024-03-04T07:28:35.391229+0000","last_peered":"2024-03-04T07:28:35.391229+0000","last_clean":"2024-03-04T07:28:35.391229+0000","last_became_active":"2024-03-04T07:28:35.383379+0000","last_became_peered":"2024-03-04T07:28:35.383379+0000","last_unstale":"2024-03-04T07:28:35.391229+0000","last_undegraded":"2024-03-04T07:28:35.391229+0000","last_fullsized":"2024-03-04T07:28:35.391229+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:36:13.026173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.375891+0000","last_change":"2024-03-04T07:28:34.376230+0000","last_active":"2024-03-04T07:28:35.375891+0000","last_peered":"2024-03-04T07:28:35.375891+0000","last_clean":"2024-03-04T07:28:35.375891+0000","last_became_active":"2024-03-04T07:28:34.375928+0000","last_became_peered":"2024-03-04T07:28:34.375928+0000","last_unstale":"2024-03-04T07:28:35.375891+0000","last_undegraded":"2024-03-04T07:28:35.375891+0000","last_fullsized":"2024-03-04T07:28:35.375891+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:41.390978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390911+0000","last_change":"2024-03-04T07:28:35.387702+0000","last_active":"2024-03-04T07:28:35.390911+0000","last_peered":"2024-03-04T07:28:35.390911+0000","last_clean":"2024-03-04T07:28:35.390911+0000","last_became_active":"2024-03-04T07:28:35.387355+0000","last_became_peered":"2024-03-04T07:28:35.387355+0000","last_unstale":"2024-03-04T07:28:35.390911+0000","last_undegraded":"2024-03-04T07:28:35.390911+0000","last_fullsized":"2024-03-04T07:28:35.390911+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:11.668312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.379870+0000","last_change":"2024-03-04T07:28:34.380203+0000","last_active":"2024-03-04T07:28:35.379870+0000","last_peered":"2024-03-04T07:28:35.379870+0000","last_clean":"2024-03-04T07:28:35.379870+0000","last_became_active":"2024-03-04T07:28:34.379880+0000","last_became_peered":"2024-03-04T07:28:34.379880+0000","last_unstale":"2024-03-04T07:28:35.379870+0000","last_undegraded":"2024-03-04T07:28:35.379870+0000","last_fullsized":"2024-03-04T07:28:35.379870+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:46:32.695936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.400211+0000","last_change":"2024-03-04T07:28:35.394843+0000","last_active":"2024-03-04T07:28:35.400211+0000","last_peered":"2024-03-04T07:28:35.400211+0000","last_clean":"2024-03-04T07:28:35.400211+0000","last_became_active":"2024-03-04T07:28:35.394538+0000","last_became_peered":"2024-03-04T07:28:35.394538+0000","last_unstale":"2024-03-04T07:28:35.400211+0000","last_undegraded":"2024-03-04T07:28:35.400211+0000","last_fullsized":"2024-03-04T07:28:35.400211+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:42.397556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377996+0000","last_change":"2024-03-04T07:28:34.374763+0000","last_active":"2024-03-04T07:28:35.377996+0000","last_peered":"2024-03-04T07:28:35.377996+0000","last_clean":"2024-03-04T07:28:35.377996+0000","last_became_active":"2024-03-04T07:28:34.374443+0000","last_became_peered":"2024-03-04T07:28:34.374443+0000","last_unstale":"2024-03-04T07:28:35.377996+0000","last_undegraded":"2024-03-04T07:28:35.377996+0000","last_fullsized":"2024-03-04T07:28:35.377996+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:42:37.468797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.395065+0000","last_change":"2024-03-04T07:28:35.390316+0000","last_active":"2024-03-04T07:28:35.395065+0000","last_peered":"2024-03-04T07:28:35.395065+0000","last_clean":"2024-03-04T07:28:35.395065+0000","last_became_active":"2024-03-04T07:28:35.387957+0000","last_became_peered":"2024-03-04T07:28:35.387957+0000","last_unstale":"2024-03-04T07:28:35.395065+0000","last_undegraded":"2024-03-04T07:28:35.395065+0000","last_fullsized":"2024-03-04T07:28:35.395065+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:15.340221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.381791+0000","last_change":"2024-03-04T07:28:34.377932+0000","last_active":"2024-03-04T07:28:35.381791+0000","last_peered":"2024-03-04T07:28:35.381791+0000","last_clean":"2024-03-04T07:28:35.381791+0000","last_became_active":"2024-03-04T07:28:34.377652+0000","last_became_peered":"2024-03-04T07:28:34.377652+0000","last_unstale":"2024-03-04T07:28:35.381791+0000","last_undegraded":"2024-03-04T07:28:35.381791+0000","last_fullsized":"2024-03-04T07:28:35.381791+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:03:33.880246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398340+0000","last_change":"2024-03-04T07:28:35.393147+0000","last_active":"2024-03-04T07:28:35.398340+0000","last_peered":"2024-03-04T07:28:35.398340+0000","last_clean":"2024-03-04T07:28:35.398340+0000","last_became_active":"2024-03-04T07:28:35.392224+0000","last_became_peered":"2024-03-04T07:28:35.392224+0000","last_unstale":"2024-03-04T07:28:35.398340+0000","last_undegraded":"2024-03-04T07:28:35.398340+0000","last_fullsized":"2024-03-04T07:28:35.398340+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:36:13.026173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.378280+0000","last_change":"2024-03-04T07:28:34.378349+0000","last_active":"2024-03-04T07:28:35.378280+0000","last_peered":"2024-03-04T07:28:35.378280+0000","last_clean":"2024-03-04T07:28:35.378280+0000","last_became_active":"2024-03-04T07:28:34.378057+0000","last_became_peered":"2024-03-04T07:28:34.378057+0000","last_unstale":"2024-03-04T07:28:35.378280+0000","last_undegraded":"2024-03-04T07:28:35.378280+0000","last_fullsized":"2024-03-04T07:28:35.378280+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:45:42.486910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391013+0000","last_change":"2024-03-04T07:28:35.387189+0000","last_active":"2024-03-04T07:28:35.391013+0000","last_peered":"2024-03-04T07:28:35.391013+0000","last_clean":"2024-03-04T07:28:35.391013+0000","last_became_active":"2024-03-04T07:28:35.386897+0000","last_became_peered":"2024-03-04T07:28:35.386897+0000","last_unstale":"2024-03-04T07:28:35.391013+0000","last_undegraded":"2024-03-04T07:28:35.391013+0000","last_fullsized":"2024-03-04T07:28:35.391013+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:04.006481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.375353+0000","last_change":"2024-03-04T07:28:34.371108+0000","last_active":"2024-03-04T07:28:35.375353+0000","last_peered":"2024-03-04T07:28:35.375353+0000","last_clean":"2024-03-04T07:28:35.375353+0000","last_became_active":"2024-03-04T07:28:34.370806+0000","last_became_peered":"2024-03-04T07:28:34.370806+0000","last_unstale":"2024-03-04T07:28:35.375353+0000","last_undegraded":"2024-03-04T07:28:35.375353+0000","last_fullsized":"2024-03-04T07:28:35.375353+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:44:26.407962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393016+0000","last_change":"2024-03-04T07:28:35.386310+0000","last_active":"2024-03-04T07:28:35.393016+0000","last_peered":"2024-03-04T07:28:35.393016+0000","last_clean":"2024-03-04T07:28:35.393016+0000","last_became_active":"2024-03-04T07:28:35.386020+0000","last_became_peered":"2024-03-04T07:28:35.386020+0000","last_unstale":"2024-03-04T07:28:35.393016+0000","last_undegraded":"2024-03-04T07:28:35.393016+0000","last_fullsized":"2024-03-04T07:28:35.393016+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:35:45.048101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.378888+0000","last_change":"2024-03-04T07:28:34.378354+0000","last_active":"2024-03-04T07:28:35.378888+0000","last_peered":"2024-03-04T07:28:35.378888+0000","last_clean":"2024-03-04T07:28:35.378888+0000","last_became_active":"2024-03-04T07:28:34.378034+0000","last_became_peered":"2024-03-04T07:28:34.378034+0000","last_unstale":"2024-03-04T07:28:35.378888+0000","last_undegraded":"2024-03-04T07:28:35.378888+0000","last_fullsized":"2024-03-04T07:28:35.378888+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:23.028719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396746+0000","last_change":"2024-03-04T07:28:35.388926+0000","last_active":"2024-03-04T07:28:35.396746+0000","last_peered":"2024-03-04T07:28:35.396746+0000","last_clean":"2024-03-04T07:28:35.396746+0000","last_became_active":"2024-03-04T07:28:35.388659+0000","last_became_peered":"2024-03-04T07:28:35.388659+0000","last_unstale":"2024-03-04T07:28:35.396746+0000","last_undegraded":"2024-03-04T07:28:35.396746+0000","last_fullsized":"2024-03-04T07:28:35.396746+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:10.484767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.374385+0000","last_change":"2024-03-04T07:28:34.379297+0000","last_active":"2024-03-04T07:28:35.374385+0000","last_peered":"2024-03-04T07:28:35.374385+0000","last_clean":"2024-03-04T07:28:35.374385+0000","last_became_active":"2024-03-04T07:28:34.379041+0000","last_became_peered":"2024-03-04T07:28:34.379041+0000","last_unstale":"2024-03-04T07:28:35.374385+0000","last_undegraded":"2024-03-04T07:28:35.374385+0000","last_fullsized":"2024-03-04T07:28:35.374385+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:39:55.893953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.383050+0000","last_change":"2024-03-04T07:28:35.378306+0000","last_active":"2024-03-04T07:28:35.383050+0000","last_peered":"2024-03-04T07:28:35.383050+0000","last_clean":"2024-03-04T07:28:35.383050+0000","last_became_active":"2024-03-04T07:28:35.378008+0000","last_became_peered":"2024-03-04T07:28:35.378008+0000","last_unstale":"2024-03-04T07:28:35.383050+0000","last_undegraded":"2024-03-04T07:28:35.383050+0000","last_fullsized":"2024-03-04T07:28:35.383050+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:45:43.493488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.375606+0000","last_change":"2024-03-04T07:28:34.367749+0000","last_active":"2024-03-04T07:28:35.375606+0000","last_peered":"2024-03-04T07:28:35.375606+0000","last_clean":"2024-03-04T07:28:35.375606+0000","last_became_active":"2024-03-04T07:28:34.367417+0000","last_became_peered":"2024-03-04T07:28:34.367417+0000","last_unstale":"2024-03-04T07:28:35.375606+0000","last_undegraded":"2024-03-04T07:28:35.375606+0000","last_fullsized":"2024-03-04T07:28:35.375606+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:20:24.052808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394568+0000","last_change":"2024-03-04T07:28:35.391088+0000","last_active":"2024-03-04T07:28:35.394568+0000","last_peered":"2024-03-04T07:28:35.394568+0000","last_clean":"2024-03-04T07:28:35.394568+0000","last_became_active":"2024-03-04T07:28:35.390812+0000","last_became_peered":"2024-03-04T07:28:35.390812+0000","last_unstale":"2024-03-04T07:28:35.394568+0000","last_undegraded":"2024-03-04T07:28:35.394568+0000","last_fullsized":"2024-03-04T07:28:35.394568+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:42:38.475375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377883+0000","last_change":"2024-03-04T07:28:34.378476+0000","last_active":"2024-03-04T07:28:35.377883+0000","last_peered":"2024-03-04T07:28:35.377883+0000","last_clean":"2024-03-04T07:28:35.377883+0000","last_became_active":"2024-03-04T07:28:34.378170+0000","last_became_peered":"2024-03-04T07:28:34.378170+0000","last_unstale":"2024-03-04T07:28:35.377883+0000","last_undegraded":"2024-03-04T07:28:35.377883+0000","last_fullsized":"2024-03-04T07:28:35.377883+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:46:17.432274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394994+0000","last_change":"2024-03-04T07:28:35.390240+0000","last_active":"2024-03-04T07:28:35.394994+0000","last_peered":"2024-03-04T07:28:35.394994+0000","last_clean":"2024-03-04T07:28:35.394994+0000","last_became_active":"2024-03-04T07:28:35.389978+0000","last_became_peered":"2024-03-04T07:28:35.389978+0000","last_unstale":"2024-03-04T07:28:35.394994+0000","last_undegraded":"2024-03-04T07:28:35.394994+0000","last_fullsized":"2024-03-04T07:28:35.394994+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:27.386478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403169+0000","last_change":"2024-03-04T07:28:34.377128+0000","last_active":"2024-03-04T07:28:35.403169+0000","last_peered":"2024-03-04T07:28:35.403169+0000","last_clean":"2024-03-04T07:28:35.403169+0000","last_became_active":"2024-03-04T07:28:34.376617+0000","last_became_peered":"2024-03-04T07:28:34.376617+0000","last_unstale":"2024-03-04T07:28:35.403169+0000","last_undegraded":"2024-03-04T07:28:35.403169+0000","last_fullsized":"2024-03-04T07:28:35.403169+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:36.732068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":211,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.378107+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:28:35.378107+0000","last_peered":"2024-03-04T07:28:35.378107+0000","last_clean":"2024-03-04T07:28:35.378107+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:28:35.378107+0000","last_undegraded":"2024-03-04T07:28:35.378107+0000","last_fullsized":"2024-03-04T07:28:35.378107+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":6,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574876,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32264,"kb_used_data":1076,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339576,"statfs":{"total":96636764160,"available":96603725824,"internally_reserved":0,"allocated":1101824,"data_stored":793958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574875,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31688,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340152,"statfs":{"total":96636764160,"available":96604315648,"internally_reserved":0,"allocated":512000,"data_stored":204172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574875,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31684,"kb_used_data":496,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340156,"statfs":{"total":96636764160,"available":96604319744,"internally_reserved":0,"allocated":507904,"data_stored":203570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574876,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":516096,"data_stored":205112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574875,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":516096,"data_stored":204138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574875,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":508,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":520192,"data_stored":205134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574875,"num_pgs":10,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31664,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340176,"statfs":{"total":96636764160,"available":96604340224,"internally_reserved":0,"allocated":491520,"data_stored":196725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574875,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32244,"kb_used_data":1060,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339596,"statfs":{"total":96636764160,"available":96603746304,"internally_reserved":0,"allocated":1085440,"data_stored":787093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:28:38.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:28:38.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.530+0000 7f3730d20640 1 Processor -- start 2024-03-04T07:28:38.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.530+0000 7f3730d20640 1 -- start start 2024-03-04T07:28:38.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f3730d20640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f372c158520 0x7f372c06d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f3730d20640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f372c0aee10 0x7f372c0b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f3730d20640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f372c071240 con 0x7f372c06dd60 2024-03-04T07:28:38.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f3730d20640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f372c071b70 con 0x7f372c0aee10 2024-03-04T07:28:38.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f3730d20640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f372c071520 con 0x7f372c158520 2024-03-04T07:28:38.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f372c0aee10 0x7f372c0b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f372c0aee10 0x7f372c0b3640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35006/0 (socket says 172.21.15.64:35006) 2024-03-04T07:28:38.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372ad9d640 1 -- 172.21.15.64:0/1154366850 learned_addr learned my addr 172.21.15.64:0/1154366850 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:38.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372a59c640 1 --2- 172.21.15.64:0/1154366850 >> v2:172.21.15.66:3300/0 conn(0x7f372c158520 0x7f372c06d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372ad9d640 1 -- 172.21.15.64:0/1154366850 >> v2:172.21.15.66:3300/0 conn(0x7f372c158520 msgr2=0x7f372c06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:38.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372ad9d640 1 --2- 172.21.15.64:0/1154366850 >> v2:172.21.15.66:3300/0 conn(0x7f372c158520 0x7f372c06d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372ad9d640 1 -- 172.21.15.64:0/1154366850 >> v1:172.21.15.66:6789/0 conn(0x7f372c06dd60 legacy=0x7f372c0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:38.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.531+0000 7f372ad9d640 1 -- 172.21.15.64:0/1154366850 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f372c071800 con 0x7f372c0aee10 2024-03-04T07:28:38.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f372ad9d640 1 --2- 172.21.15.64:0/1154366850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c0b3640 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f372c06e900 tx=0x7f371c02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf0bd6b2e84d0d9c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f372959a640 1 -- 172.21.15.64:0/1154366850 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f371c02ca80 con 0x7f372c0aee10 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f372959a640 1 -- 172.21.15.64:0/1154366850 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f371c02cc20 con 0x7f372c0aee10 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f372959a640 1 -- 172.21.15.64:0/1154366850 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f371c0343f0 con 0x7f372c0aee10 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 -- 172.21.15.64:0/1154366850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 msgr2=0x7f372c0b3640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 --2- 172.21.15.64:0/1154366850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c0b3640 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f372c06e900 tx=0x7f371c02bee0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 -- 172.21.15.64:0/1154366850 shutdown_connections 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 --2- 172.21.15.64:0/1154366850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c0b3640 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 --2- 172.21.15.64:0/1154366850 >> v2:172.21.15.66:3300/0 conn(0x7f372c158520 0x7f372c06d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 -- 172.21.15.64:0/1154366850 >> 172.21.15.64:0/1154366850 conn(0x7f372c066d00 msgr2=0x7f372c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 -- 172.21.15.64:0/1154366850 shutdown_connections 2024-03-04T07:28:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.533+0000 7f3730d20640 1 -- 172.21.15.64:0/1154366850 wait complete. 2024-03-04T07:28:38.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 Processor -- start 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 -- start start 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f372c06dd60 0x7f372c14cdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c1463b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f372c158520 0x7f372c0ad960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f372c158a80 con 0x7f372c0aee10 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f372c158900 con 0x7f372c158520 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3730d20640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f372c0b3b80 con 0x7f372c06dd60 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3729d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c1463b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3729d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c1463b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35012/0 (socket says 172.21.15.64:35012) 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.535+0000 7f3729d9b640 1 -- 172.21.15.64:0/149169309 learned_addr learned my addr 172.21.15.64:0/149169309 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f372ad9d640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f372c158520 0x7f372c0ad960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f372a59c640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f372c06dd60 0x7f372c14cdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3729d9b640 1 -- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f372c06dd60 msgr2=0x7f372c14cdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3729d9b640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f372c06dd60 0x7f372c14cdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3729d9b640 1 -- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f372c158520 msgr2=0x7f372c0ad960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3729d9b640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f372c158520 0x7f372c0ad960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3729d9b640 1 -- 172.21.15.64:0/149169309 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f372c14d310 con 0x7f372c0aee10 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3729d9b640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c1463b0 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f371400c430 tx=0x7f371400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3722ffd640 1 -- 172.21.15.64:0/149169309 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3714011790 con 0x7f372c0aee10 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3722ffd640 1 -- 172.21.15.64:0/149169309 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3714017070 con 0x7f372c0aee10 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f372c14bd30 con 0x7f372c0aee10 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3722ffd640 1 -- 172.21.15.64:0/149169309 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3714011bb0 con 0x7f372c0aee10 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.536+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f372c14c2f0 con 0x7f372c0aee10 2024-03-04T07:28:38.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.537+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f372c067ff0 con 0x7f372c0aee10 2024-03-04T07:28:38.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.538+0000 7f3722ffd640 1 -- 172.21.15.64:0/149169309 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f371401c460 con 0x7f372c0aee10 2024-03-04T07:28:38.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.538+0000 7f3722ffd640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f370c06b4d0 0x7f370c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.538+0000 7f3722ffd640 1 -- 172.21.15.64:0/149169309 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3714089f40 con 0x7f372c0aee10 2024-03-04T07:28:38.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.538+0000 7f372a59c640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f370c06b4d0 0x7f370c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.539+0000 7f372a59c640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f370c06b4d0 0x7f370c06d990 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f37240079a0 tx=0x7f3724008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.543+0000 7f3722ffd640 1 -- 172.21.15.64:0/149169309 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3714058770 con 0x7f372c0aee10 2024-03-04T07:28:38.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.720+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f372c0b31f0 con 0x7f370c06b4d0 2024-03-04T07:28:38.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.726+0000 7f3722ffd640 1 -- 172.21.15.64:0/149169309 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251888 (secure 0 0 0) 0x7f372c0b31f0 con 0x7f370c06b4d0 2024-03-04T07:28:38.728 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:38.729 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f370c06b4d0 msgr2=0x7f370c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f370c06b4d0 0x7f370c06d990 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f37240079a0 tx=0x7f3724008040 comp rx=0 tx=0).stop 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 msgr2=0x7f372c1463b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c1463b0 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f371400c430 tx=0x7f371400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 shutdown_connections 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f370c06b4d0 0x7f370c06d990 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f372c158520 0x7f372c0ad960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f372c0aee10 0x7f372c1463b0 unknown :-1 s=CLOSED pgs=538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 --2- 172.21.15.64:0/149169309 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f372c06dd60 0x7f372c14cdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 >> 172.21.15.64:0/149169309 conn(0x7f372c066d00 msgr2=0x7f372c0b4630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 shutdown_connections 2024-03-04T07:28:38.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.729+0000 7f3730d20640 1 -- 172.21.15.64:0/149169309 wait complete. 2024-03-04T07:28:38.739 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":95,"stamp":"2024-03-04T07:28:37.160910+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":67,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":254624,"kb_used_data":5128,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754720096,"statfs":{"total":773094113280,"available":772833378304,"internally_reserved":0,"allocated":5251072,"data_stored":2799902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000617"},"pg_stats":[{"pgid":"7.3f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390954+0000","last_change":"2024-03-04T07:28:35.386301+0000","last_active":"2024-03-04T07:28:35.390954+0000","last_peered":"2024-03-04T07:28:35.390954+0000","last_clean":"2024-03-04T07:28:35.390954+0000","last_became_active":"2024-03-04T07:28:35.385731+0000","last_became_peered":"2024-03-04T07:28:35.385731+0000","last_unstale":"2024-03-04T07:28:35.390954+0000","last_undegraded":"2024-03-04T07:28:35.390954+0000","last_fullsized":"2024-03-04T07:28:35.390954+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:20.808035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391273+0000","last_change":"2024-03-04T07:28:35.387855+0000","last_active":"2024-03-04T07:28:35.391273+0000","last_peered":"2024-03-04T07:28:35.391273+0000","last_clean":"2024-03-04T07:28:35.391273+0000","last_became_active":"2024-03-04T07:28:35.387557+0000","last_became_peered":"2024-03-04T07:28:35.387557+0000","last_unstale":"2024-03-04T07:28:35.391273+0000","last_undegraded":"2024-03-04T07:28:35.391273+0000","last_fullsized":"2024-03-04T07:28:35.391273+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:20.808035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.388400+0000","last_change":"2024-03-04T07:28:35.383936+0000","last_active":"2024-03-04T07:28:35.388400+0000","last_peered":"2024-03-04T07:28:35.388400+0000","last_clean":"2024-03-04T07:28:35.388400+0000","last_became_active":"2024-03-04T07:28:35.383665+0000","last_became_peered":"2024-03-04T07:28:35.383665+0000","last_unstale":"2024-03-04T07:28:35.388400+0000","last_undegraded":"2024-03-04T07:28:35.388400+0000","last_fullsized":"2024-03-04T07:28:35.388400+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:27:54.268773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398366+0000","last_change":"2024-03-04T07:28:35.393479+0000","last_active":"2024-03-04T07:28:35.398366+0000","last_peered":"2024-03-04T07:28:35.398366+0000","last_clean":"2024-03-04T07:28:35.398366+0000","last_became_active":"2024-03-04T07:28:35.393191+0000","last_became_peered":"2024-03-04T07:28:35.393191+0000","last_unstale":"2024-03-04T07:28:35.398366+0000","last_undegraded":"2024-03-04T07:28:35.398366+0000","last_fullsized":"2024-03-04T07:28:35.398366+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:27:54.268773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391019+0000","last_change":"2024-03-04T07:28:35.387862+0000","last_active":"2024-03-04T07:28:35.391019+0000","last_peered":"2024-03-04T07:28:35.391019+0000","last_clean":"2024-03-04T07:28:35.391019+0000","last_became_active":"2024-03-04T07:28:35.387563+0000","last_became_peered":"2024-03-04T07:28:35.387563+0000","last_unstale":"2024-03-04T07:28:35.391019+0000","last_undegraded":"2024-03-04T07:28:35.391019+0000","last_fullsized":"2024-03-04T07:28:35.391019+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:56:17.616203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396855+0000","last_change":"2024-03-04T07:28:35.390018+0000","last_active":"2024-03-04T07:28:35.396855+0000","last_peered":"2024-03-04T07:28:35.396855+0000","last_clean":"2024-03-04T07:28:35.396855+0000","last_became_active":"2024-03-04T07:28:35.389744+0000","last_became_peered":"2024-03-04T07:28:35.389744+0000","last_unstale":"2024-03-04T07:28:35.396855+0000","last_undegraded":"2024-03-04T07:28:35.396855+0000","last_fullsized":"2024-03-04T07:28:35.396855+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:21.080275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.39","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391039+0000","last_change":"2024-03-04T07:28:35.386005+0000","last_active":"2024-03-04T07:28:35.391039+0000","last_peered":"2024-03-04T07:28:35.391039+0000","last_clean":"2024-03-04T07:28:35.391039+0000","last_became_active":"2024-03-04T07:28:35.385676+0000","last_became_peered":"2024-03-04T07:28:35.385676+0000","last_unstale":"2024-03-04T07:28:35.391039+0000","last_undegraded":"2024-03-04T07:28:35.391039+0000","last_fullsized":"2024-03-04T07:28:35.391039+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:08:07.352168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.38","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398413+0000","last_change":"2024-03-04T07:28:35.393414+0000","last_active":"2024-03-04T07:28:35.398413+0000","last_peered":"2024-03-04T07:28:35.398413+0000","last_clean":"2024-03-04T07:28:35.398413+0000","last_became_active":"2024-03-04T07:28:35.392748+0000","last_became_peered":"2024-03-04T07:28:35.392748+0000","last_unstale":"2024-03-04T07:28:35.398413+0000","last_undegraded":"2024-03-04T07:28:35.398413+0000","last_fullsized":"2024-03-04T07:28:35.398413+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:48.003109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.37","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.395009+0000","last_change":"2024-03-04T07:28:35.388485+0000","last_active":"2024-03-04T07:28:35.395009+0000","last_peered":"2024-03-04T07:28:35.395009+0000","last_clean":"2024-03-04T07:28:35.395009+0000","last_became_active":"2024-03-04T07:28:35.387264+0000","last_became_peered":"2024-03-04T07:28:35.387264+0000","last_unstale":"2024-03-04T07:28:35.395009+0000","last_undegraded":"2024-03-04T07:28:35.395009+0000","last_fullsized":"2024-03-04T07:28:35.395009+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:15.340221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.36","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398013+0000","last_change":"2024-03-04T07:28:35.392528+0000","last_active":"2024-03-04T07:28:35.398013+0000","last_peered":"2024-03-04T07:28:35.398013+0000","last_clean":"2024-03-04T07:28:35.398013+0000","last_became_active":"2024-03-04T07:28:35.392051+0000","last_became_peered":"2024-03-04T07:28:35.392051+0000","last_unstale":"2024-03-04T07:28:35.398013+0000","last_undegraded":"2024-03-04T07:28:35.398013+0000","last_fullsized":"2024-03-04T07:28:35.398013+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:20:25.059386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.35","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396037+0000","last_change":"2024-03-04T07:28:35.390987+0000","last_active":"2024-03-04T07:28:35.396037+0000","last_peered":"2024-03-04T07:28:35.396037+0000","last_clean":"2024-03-04T07:28:35.396037+0000","last_became_active":"2024-03-04T07:28:35.390557+0000","last_became_peered":"2024-03-04T07:28:35.390557+0000","last_unstale":"2024-03-04T07:28:35.396037+0000","last_undegraded":"2024-03-04T07:28:35.396037+0000","last_fullsized":"2024-03-04T07:28:35.396037+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:39:56.900531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.34","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390932+0000","last_change":"2024-03-04T07:28:35.384783+0000","last_active":"2024-03-04T07:28:35.390932+0000","last_peered":"2024-03-04T07:28:35.390932+0000","last_clean":"2024-03-04T07:28:35.390932+0000","last_became_active":"2024-03-04T07:28:35.384490+0000","last_became_peered":"2024-03-04T07:28:35.384490+0000","last_unstale":"2024-03-04T07:28:35.390932+0000","last_undegraded":"2024-03-04T07:28:35.390932+0000","last_fullsized":"2024-03-04T07:28:35.390932+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:08:07.352168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.33","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390843+0000","last_change":"2024-03-04T07:28:35.389993+0000","last_active":"2024-03-04T07:28:35.390843+0000","last_peered":"2024-03-04T07:28:35.390843+0000","last_clean":"2024-03-04T07:28:35.390843+0000","last_became_active":"2024-03-04T07:28:35.388419+0000","last_became_peered":"2024-03-04T07:28:35.388419+0000","last_unstale":"2024-03-04T07:28:35.390843+0000","last_undegraded":"2024-03-04T07:28:35.390843+0000","last_fullsized":"2024-03-04T07:28:35.390843+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:07:25.627401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.32","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394895+0000","last_change":"2024-03-04T07:28:35.386955+0000","last_active":"2024-03-04T07:28:35.394895+0000","last_peered":"2024-03-04T07:28:35.394895+0000","last_clean":"2024-03-04T07:28:35.394895+0000","last_became_active":"2024-03-04T07:28:35.386539+0000","last_became_peered":"2024-03-04T07:28:35.386539+0000","last_unstale":"2024-03-04T07:28:35.394895+0000","last_undegraded":"2024-03-04T07:28:35.394895+0000","last_fullsized":"2024-03-04T07:28:35.394895+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:27.205310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.31","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393070+0000","last_change":"2024-03-04T07:28:35.387067+0000","last_active":"2024-03-04T07:28:35.393070+0000","last_peered":"2024-03-04T07:28:35.393070+0000","last_clean":"2024-03-04T07:28:35.393070+0000","last_became_active":"2024-03-04T07:28:35.386715+0000","last_became_peered":"2024-03-04T07:28:35.386715+0000","last_unstale":"2024-03-04T07:28:35.393070+0000","last_undegraded":"2024-03-04T07:28:35.393070+0000","last_fullsized":"2024-03-04T07:28:35.393070+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:35:56.294515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.30","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393024+0000","last_change":"2024-03-04T07:28:35.389480+0000","last_active":"2024-03-04T07:28:35.393024+0000","last_peered":"2024-03-04T07:28:35.393024+0000","last_clean":"2024-03-04T07:28:35.393024+0000","last_became_active":"2024-03-04T07:28:35.388674+0000","last_became_peered":"2024-03-04T07:28:35.388674+0000","last_unstale":"2024-03-04T07:28:35.393024+0000","last_undegraded":"2024-03-04T07:28:35.393024+0000","last_fullsized":"2024-03-04T07:28:35.393024+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:46:33.702514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.385746+0000","last_change":"2024-03-04T07:28:35.380670+0000","last_active":"2024-03-04T07:28:35.385746+0000","last_peered":"2024-03-04T07:28:35.385746+0000","last_clean":"2024-03-04T07:28:35.385746+0000","last_became_active":"2024-03-04T07:28:35.380384+0000","last_became_peered":"2024-03-04T07:28:35.380384+0000","last_unstale":"2024-03-04T07:28:35.385746+0000","last_undegraded":"2024-03-04T07:28:35.385746+0000","last_fullsized":"2024-03-04T07:28:35.385746+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:39:38.317488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394565+0000","last_change":"2024-03-04T07:28:35.391021+0000","last_active":"2024-03-04T07:28:35.394565+0000","last_peered":"2024-03-04T07:28:35.394565+0000","last_clean":"2024-03-04T07:28:35.394565+0000","last_became_active":"2024-03-04T07:28:35.390704+0000","last_became_peered":"2024-03-04T07:28:35.390704+0000","last_unstale":"2024-03-04T07:28:35.394565+0000","last_undegraded":"2024-03-04T07:28:35.394565+0000","last_fullsized":"2024-03-04T07:28:35.394565+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:42.397556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.2d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391884+0000","last_change":"2024-03-04T07:28:35.388426+0000","last_active":"2024-03-04T07:28:35.391884+0000","last_peered":"2024-03-04T07:28:35.391884+0000","last_clean":"2024-03-04T07:28:35.391884+0000","last_became_active":"2024-03-04T07:28:35.388044+0000","last_became_peered":"2024-03-04T07:28:35.388044+0000","last_unstale":"2024-03-04T07:28:35.391884+0000","last_undegraded":"2024-03-04T07:28:35.391884+0000","last_fullsized":"2024-03-04T07:28:35.391884+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:44:29.060773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.2c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390884+0000","last_change":"2024-03-04T07:28:35.387875+0000","last_active":"2024-03-04T07:28:35.390884+0000","last_peered":"2024-03-04T07:28:35.390884+0000","last_clean":"2024-03-04T07:28:35.390884+0000","last_became_active":"2024-03-04T07:28:35.387578+0000","last_became_peered":"2024-03-04T07:28:35.387578+0000","last_unstale":"2024-03-04T07:28:35.390884+0000","last_undegraded":"2024-03-04T07:28:35.390884+0000","last_fullsized":"2024-03-04T07:28:35.390884+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:47:27.934672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.2b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398513+0000","last_change":"2024-03-04T07:28:35.394789+0000","last_active":"2024-03-04T07:28:35.398513+0000","last_peered":"2024-03-04T07:28:35.398513+0000","last_clean":"2024-03-04T07:28:35.398513+0000","last_became_active":"2024-03-04T07:28:35.394517+0000","last_became_peered":"2024-03-04T07:28:35.394517+0000","last_unstale":"2024-03-04T07:28:35.398513+0000","last_undegraded":"2024-03-04T07:28:35.398513+0000","last_fullsized":"2024-03-04T07:28:35.398513+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:41:51.471223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398051+0000","last_change":"2024-03-04T07:28:35.392919+0000","last_active":"2024-03-04T07:28:35.398051+0000","last_peered":"2024-03-04T07:28:35.398051+0000","last_clean":"2024-03-04T07:28:35.398051+0000","last_became_active":"2024-03-04T07:28:35.392648+0000","last_became_peered":"2024-03-04T07:28:35.392648+0000","last_unstale":"2024-03-04T07:28:35.398051+0000","last_undegraded":"2024-03-04T07:28:35.398051+0000","last_fullsized":"2024-03-04T07:28:35.398051+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:39:38.317488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.29","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391912+0000","last_change":"2024-03-04T07:28:35.389395+0000","last_active":"2024-03-04T07:28:35.391912+0000","last_peered":"2024-03-04T07:28:35.391912+0000","last_clean":"2024-03-04T07:28:35.391912+0000","last_became_active":"2024-03-04T07:28:35.389040+0000","last_became_peered":"2024-03-04T07:28:35.389040+0000","last_unstale":"2024-03-04T07:28:35.391912+0000","last_undegraded":"2024-03-04T07:28:35.391912+0000","last_fullsized":"2024-03-04T07:28:35.391912+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:46:18.438852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.28","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393041+0000","last_change":"2024-03-04T07:28:35.391034+0000","last_active":"2024-03-04T07:28:35.393041+0000","last_peered":"2024-03-04T07:28:35.393041+0000","last_clean":"2024-03-04T07:28:35.393041+0000","last_became_active":"2024-03-04T07:28:35.390440+0000","last_became_peered":"2024-03-04T07:28:35.390440+0000","last_unstale":"2024-03-04T07:28:35.393041+0000","last_undegraded":"2024-03-04T07:28:35.393041+0000","last_fullsized":"2024-03-04T07:28:35.393041+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:27.205310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.27","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391590+0000","last_change":"2024-03-04T07:28:35.387632+0000","last_active":"2024-03-04T07:28:35.391590+0000","last_peered":"2024-03-04T07:28:35.391590+0000","last_clean":"2024-03-04T07:28:35.391590+0000","last_became_active":"2024-03-04T07:28:35.387329+0000","last_became_peered":"2024-03-04T07:28:35.387329+0000","last_unstale":"2024-03-04T07:28:35.391590+0000","last_undegraded":"2024-03-04T07:28:35.391590+0000","last_fullsized":"2024-03-04T07:28:35.391590+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:26.648603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.26","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390965+0000","last_change":"2024-03-04T07:28:35.388767+0000","last_active":"2024-03-04T07:28:35.390965+0000","last_peered":"2024-03-04T07:28:35.390965+0000","last_clean":"2024-03-04T07:28:35.390965+0000","last_became_active":"2024-03-04T07:28:35.388245+0000","last_became_peered":"2024-03-04T07:28:35.388245+0000","last_unstale":"2024-03-04T07:28:35.390965+0000","last_undegraded":"2024-03-04T07:28:35.390965+0000","last_fullsized":"2024-03-04T07:28:35.390965+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:30:31.008078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.25","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.388376+0000","last_change":"2024-03-04T07:28:35.382911+0000","last_active":"2024-03-04T07:28:35.388376+0000","last_peered":"2024-03-04T07:28:35.388376+0000","last_clean":"2024-03-04T07:28:35.388376+0000","last_became_active":"2024-03-04T07:28:35.382508+0000","last_became_peered":"2024-03-04T07:28:35.382508+0000","last_unstale":"2024-03-04T07:28:35.388376+0000","last_undegraded":"2024-03-04T07:28:35.388376+0000","last_fullsized":"2024-03-04T07:28:35.388376+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:36:13.026173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.24","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.388594+0000","last_change":"2024-03-04T07:28:35.382446+0000","last_active":"2024-03-04T07:28:35.388594+0000","last_peered":"2024-03-04T07:28:35.388594+0000","last_clean":"2024-03-04T07:28:35.388594+0000","last_became_active":"2024-03-04T07:28:35.382130+0000","last_became_peered":"2024-03-04T07:28:35.382130+0000","last_unstale":"2024-03-04T07:28:35.388594+0000","last_undegraded":"2024-03-04T07:28:35.388594+0000","last_fullsized":"2024-03-04T07:28:35.388594+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:24.035297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.23","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391963+0000","last_change":"2024-03-04T07:28:35.389333+0000","last_active":"2024-03-04T07:28:35.391963+0000","last_peered":"2024-03-04T07:28:35.391963+0000","last_clean":"2024-03-04T07:28:35.391963+0000","last_became_active":"2024-03-04T07:28:35.388996+0000","last_became_peered":"2024-03-04T07:28:35.388996+0000","last_unstale":"2024-03-04T07:28:35.391963+0000","last_undegraded":"2024-03-04T07:28:35.391963+0000","last_fullsized":"2024-03-04T07:28:35.391963+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:35:26.271770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.22","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391241+0000","last_change":"2024-03-04T07:28:35.386665+0000","last_active":"2024-03-04T07:28:35.391241+0000","last_peered":"2024-03-04T07:28:35.391241+0000","last_clean":"2024-03-04T07:28:35.391241+0000","last_became_active":"2024-03-04T07:28:35.386189+0000","last_became_peered":"2024-03-04T07:28:35.386189+0000","last_unstale":"2024-03-04T07:28:35.391241+0000","last_undegraded":"2024-03-04T07:28:35.391241+0000","last_fullsized":"2024-03-04T07:28:35.391241+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:48.003109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.21","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396818+0000","last_change":"2024-03-04T07:28:35.391088+0000","last_active":"2024-03-04T07:28:35.396818+0000","last_peered":"2024-03-04T07:28:35.396818+0000","last_clean":"2024-03-04T07:28:35.396818+0000","last_became_active":"2024-03-04T07:28:35.390835+0000","last_became_peered":"2024-03-04T07:28:35.390835+0000","last_unstale":"2024-03-04T07:28:35.396818+0000","last_undegraded":"2024-03-04T07:28:35.396818+0000","last_fullsized":"2024-03-04T07:28:35.396818+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:40:01.718326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.20","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.384670+0000","last_change":"2024-03-04T07:28:35.378799+0000","last_active":"2024-03-04T07:28:35.384670+0000","last_peered":"2024-03-04T07:28:35.384670+0000","last_clean":"2024-03-04T07:28:35.384670+0000","last_became_active":"2024-03-04T07:28:35.378549+0000","last_became_peered":"2024-03-04T07:28:35.378549+0000","last_unstale":"2024-03-04T07:28:35.384670+0000","last_undegraded":"2024-03-04T07:28:35.384670+0000","last_fullsized":"2024-03-04T07:28:35.384670+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:38:03.967307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394331+0000","last_change":"2024-03-04T07:28:35.389778+0000","last_active":"2024-03-04T07:28:35.394331+0000","last_peered":"2024-03-04T07:28:35.394331+0000","last_clean":"2024-03-04T07:28:35.394331+0000","last_became_active":"2024-03-04T07:28:35.389483+0000","last_became_peered":"2024-03-04T07:28:35.389483+0000","last_unstale":"2024-03-04T07:28:35.394331+0000","last_undegraded":"2024-03-04T07:28:35.394331+0000","last_fullsized":"2024-03-04T07:28:35.394331+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:27:54.268773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1f","version":"43'4","reported_seq":16,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.406497+0000","last_change":"2024-03-04T07:28:34.375428+0000","last_active":"2024-03-04T07:28:35.406497+0000","last_peered":"2024-03-04T07:28:35.406497+0000","last_clean":"2024-03-04T07:28:35.406497+0000","last_became_active":"2024-03-04T07:28:34.375154+0000","last_became_peered":"2024-03-04T07:28:34.375154+0000","last_unstale":"2024-03-04T07:28:35.406497+0000","last_undegraded":"2024-03-04T07:28:35.406497+0000","last_fullsized":"2024-03-04T07:28:35.406497+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:02.999903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391576+0000","last_change":"2024-03-04T07:28:35.387444+0000","last_active":"2024-03-04T07:28:35.391576+0000","last_peered":"2024-03-04T07:28:35.391576+0000","last_clean":"2024-03-04T07:28:35.391576+0000","last_became_active":"2024-03-04T07:28:35.387122+0000","last_became_peered":"2024-03-04T07:28:35.387122+0000","last_unstale":"2024-03-04T07:28:35.391576+0000","last_undegraded":"2024-03-04T07:28:35.391576+0000","last_fullsized":"2024-03-04T07:28:35.391576+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:22:05.234107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.1e","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.405762+0000","last_change":"2024-03-04T07:28:34.372736+0000","last_active":"2024-03-04T07:28:35.405762+0000","last_peered":"2024-03-04T07:28:35.405762+0000","last_clean":"2024-03-04T07:28:35.405762+0000","last_became_active":"2024-03-04T07:28:34.372341+0000","last_became_peered":"2024-03-04T07:28:34.372341+0000","last_unstale":"2024-03-04T07:28:35.405762+0000","last_undegraded":"2024-03-04T07:28:35.405762+0000","last_fullsized":"2024-03-04T07:28:35.405762+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:44:26.407962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.1c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394797+0000","last_change":"2024-03-04T07:28:35.392431+0000","last_active":"2024-03-04T07:28:35.394797+0000","last_peered":"2024-03-04T07:28:35.394797+0000","last_clean":"2024-03-04T07:28:35.394797+0000","last_became_active":"2024-03-04T07:28:35.391875+0000","last_became_peered":"2024-03-04T07:28:35.391875+0000","last_unstale":"2024-03-04T07:28:35.394797+0000","last_undegraded":"2024-03-04T07:28:35.394797+0000","last_fullsized":"2024-03-04T07:28:35.394797+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:04.006481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1d","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403441+0000","last_change":"2024-03-04T07:28:34.374183+0000","last_active":"2024-03-04T07:28:35.403441+0000","last_peered":"2024-03-04T07:28:35.403441+0000","last_clean":"2024-03-04T07:28:35.403441+0000","last_became_active":"2024-03-04T07:28:34.373834+0000","last_became_peered":"2024-03-04T07:28:34.373834+0000","last_unstale":"2024-03-04T07:28:35.403441+0000","last_undegraded":"2024-03-04T07:28:35.403441+0000","last_fullsized":"2024-03-04T07:28:35.403441+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:39:37.310910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.400829+0000","last_change":"2024-03-04T07:28:35.394680+0000","last_active":"2024-03-04T07:28:35.400829+0000","last_peered":"2024-03-04T07:28:35.400829+0000","last_clean":"2024-03-04T07:28:35.400829+0000","last_became_active":"2024-03-04T07:28:35.393884+0000","last_became_peered":"2024-03-04T07:28:35.393884+0000","last_unstale":"2024-03-04T07:28:35.400829+0000","last_undegraded":"2024-03-04T07:28:35.400829+0000","last_fullsized":"2024-03-04T07:28:35.400829+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:47:48.003109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1c","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403230+0000","last_change":"2024-03-04T07:28:34.373187+0000","last_active":"2024-03-04T07:28:35.403230+0000","last_peered":"2024-03-04T07:28:35.403230+0000","last_clean":"2024-03-04T07:28:35.403230+0000","last_became_active":"2024-03-04T07:28:34.372783+0000","last_became_peered":"2024-03-04T07:28:34.372783+0000","last_unstale":"2024-03-04T07:28:35.403230+0000","last_undegraded":"2024-03-04T07:28:35.403230+0000","last_fullsized":"2024-03-04T07:28:35.403230+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:29.155064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.1a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394548+0000","last_change":"2024-03-04T07:28:35.392054+0000","last_active":"2024-03-04T07:28:35.394548+0000","last_peered":"2024-03-04T07:28:35.394548+0000","last_clean":"2024-03-04T07:28:35.394548+0000","last_became_active":"2024-03-04T07:28:35.391748+0000","last_became_peered":"2024-03-04T07:28:35.391748+0000","last_unstale":"2024-03-04T07:28:35.394548+0000","last_undegraded":"2024-03-04T07:28:35.394548+0000","last_fullsized":"2024-03-04T07:28:35.394548+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:37.738646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403106+0000","last_change":"2024-03-04T07:28:34.373176+0000","last_active":"2024-03-04T07:28:35.403106+0000","last_peered":"2024-03-04T07:28:35.403106+0000","last_clean":"2024-03-04T07:28:35.403106+0000","last_became_active":"2024-03-04T07:28:34.372905+0000","last_became_peered":"2024-03-04T07:28:34.372905+0000","last_unstale":"2024-03-04T07:28:35.403106+0000","last_undegraded":"2024-03-04T07:28:35.403106+0000","last_fullsized":"2024-03-04T07:28:35.403106+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:28:11.075094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.392053+0000","last_change":"2024-03-04T07:28:35.386741+0000","last_active":"2024-03-04T07:28:35.392053+0000","last_peered":"2024-03-04T07:28:35.392053+0000","last_clean":"2024-03-04T07:28:35.392053+0000","last_became_active":"2024-03-04T07:28:35.386444+0000","last_became_peered":"2024-03-04T07:28:35.386444+0000","last_unstale":"2024-03-04T07:28:35.392053+0000","last_undegraded":"2024-03-04T07:28:35.392053+0000","last_fullsized":"2024-03-04T07:28:35.392053+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:57:26.300871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1a","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.397711+0000","last_change":"2024-03-04T07:28:34.374405+0000","last_active":"2024-03-04T07:28:35.397711+0000","last_peered":"2024-03-04T07:28:35.397711+0000","last_clean":"2024-03-04T07:28:35.397711+0000","last_became_active":"2024-03-04T07:28:34.373892+0000","last_became_peered":"2024-03-04T07:28:34.373892+0000","last_unstale":"2024-03-04T07:28:35.397711+0000","last_undegraded":"2024-03-04T07:28:35.397711+0000","last_fullsized":"2024-03-04T07:28:35.397711+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:14.333643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398435+0000","last_change":"2024-03-04T07:28:35.394047+0000","last_active":"2024-03-04T07:28:35.398435+0000","last_peered":"2024-03-04T07:28:35.398435+0000","last_clean":"2024-03-04T07:28:35.398435+0000","last_became_active":"2024-03-04T07:28:35.393677+0000","last_became_peered":"2024-03-04T07:28:35.393677+0000","last_unstale":"2024-03-04T07:28:35.398435+0000","last_undegraded":"2024-03-04T07:28:35.398435+0000","last_fullsized":"2024-03-04T07:28:35.398435+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:30.161642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.373689+0000","last_change":"2024-03-04T07:28:34.372986+0000","last_active":"2024-03-04T07:28:35.373689+0000","last_peered":"2024-03-04T07:28:35.373689+0000","last_clean":"2024-03-04T07:28:35.373689+0000","last_became_active":"2024-03-04T07:28:34.372642+0000","last_became_peered":"2024-03-04T07:28:34.372642+0000","last_unstale":"2024-03-04T07:28:35.373689+0000","last_undegraded":"2024-03-04T07:28:35.373689+0000","last_fullsized":"2024-03-04T07:28:35.373689+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:29.155064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394999+0000","last_change":"2024-03-04T07:28:35.389959+0000","last_active":"2024-03-04T07:28:35.394999+0000","last_peered":"2024-03-04T07:28:35.394999+0000","last_clean":"2024-03-04T07:28:35.394999+0000","last_became_active":"2024-03-04T07:28:35.389454+0000","last_became_peered":"2024-03-04T07:28:35.389454+0000","last_unstale":"2024-03-04T07:28:35.394999+0000","last_undegraded":"2024-03-04T07:28:35.394999+0000","last_fullsized":"2024-03-04T07:28:35.394999+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:57:26.300871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.18","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377550+0000","last_change":"2024-03-04T07:28:34.378843+0000","last_active":"2024-03-04T07:28:35.377550+0000","last_peered":"2024-03-04T07:28:35.377550+0000","last_clean":"2024-03-04T07:28:35.377550+0000","last_became_active":"2024-03-04T07:28:34.378564+0000","last_became_peered":"2024-03-04T07:28:34.378564+0000","last_unstale":"2024-03-04T07:28:35.377550+0000","last_undegraded":"2024-03-04T07:28:35.377550+0000","last_fullsized":"2024-03-04T07:28:35.377550+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:07.978415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.16","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393836+0000","last_change":"2024-03-04T07:28:35.389893+0000","last_active":"2024-03-04T07:28:35.393836+0000","last_peered":"2024-03-04T07:28:35.393836+0000","last_clean":"2024-03-04T07:28:35.393836+0000","last_became_active":"2024-03-04T07:28:35.389585+0000","last_became_peered":"2024-03-04T07:28:35.389585+0000","last_unstale":"2024-03-04T07:28:35.393836+0000","last_undegraded":"2024-03-04T07:28:35.393836+0000","last_fullsized":"2024-03-04T07:28:35.393836+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:30.161642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.17","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.374293+0000","last_change":"2024-03-04T07:28:34.376739+0000","last_active":"2024-03-04T07:28:35.374293+0000","last_peered":"2024-03-04T07:28:35.374293+0000","last_clean":"2024-03-04T07:28:35.374293+0000","last_became_active":"2024-03-04T07:28:34.376306+0000","last_became_peered":"2024-03-04T07:28:34.376306+0000","last_unstale":"2024-03-04T07:28:35.374293+0000","last_undegraded":"2024-03-04T07:28:35.374293+0000","last_fullsized":"2024-03-04T07:28:35.374293+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:26.379900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390976+0000","last_change":"2024-03-04T07:28:35.386548+0000","last_active":"2024-03-04T07:28:35.390976+0000","last_peered":"2024-03-04T07:28:35.390976+0000","last_clean":"2024-03-04T07:28:35.390976+0000","last_became_active":"2024-03-04T07:28:35.386008+0000","last_became_peered":"2024-03-04T07:28:35.386008+0000","last_unstale":"2024-03-04T07:28:35.390976+0000","last_undegraded":"2024-03-04T07:28:35.390976+0000","last_fullsized":"2024-03-04T07:28:35.390976+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:56:17.616203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.16","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.376961+0000","last_change":"2024-03-04T07:28:34.378266+0000","last_active":"2024-03-04T07:28:35.376961+0000","last_peered":"2024-03-04T07:28:35.376961+0000","last_clean":"2024-03-04T07:28:35.376961+0000","last_became_active":"2024-03-04T07:28:34.377944+0000","last_became_peered":"2024-03-04T07:28:34.377944+0000","last_unstale":"2024-03-04T07:28:35.376961+0000","last_undegraded":"2024-03-04T07:28:35.376961+0000","last_fullsized":"2024-03-04T07:28:35.376961+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:44:28.054195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396692+0000","last_change":"2024-03-04T07:28:35.388083+0000","last_active":"2024-03-04T07:28:35.396692+0000","last_peered":"2024-03-04T07:28:35.396692+0000","last_clean":"2024-03-04T07:28:35.396692+0000","last_became_active":"2024-03-04T07:28:35.387825+0000","last_became_peered":"2024-03-04T07:28:35.387825+0000","last_unstale":"2024-03-04T07:28:35.396692+0000","last_undegraded":"2024-03-04T07:28:35.396692+0000","last_fullsized":"2024-03-04T07:28:35.396692+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:53:29.133169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"43'3","reported_seq":15,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.405951+0000","last_change":"2024-03-04T07:28:34.378354+0000","last_active":"2024-03-04T07:28:35.405951+0000","last_peered":"2024-03-04T07:28:35.405951+0000","last_clean":"2024-03-04T07:28:35.405951+0000","last_became_active":"2024-03-04T07:28:34.378086+0000","last_became_peered":"2024-03-04T07:28:34.378086+0000","last_unstale":"2024-03-04T07:28:35.405951+0000","last_undegraded":"2024-03-04T07:28:35.405951+0000","last_fullsized":"2024-03-04T07:28:35.405951+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:36.732068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.395121+0000","last_change":"2024-03-04T07:28:35.392827+0000","last_active":"2024-03-04T07:28:35.395121+0000","last_peered":"2024-03-04T07:28:35.395121+0000","last_clean":"2024-03-04T07:28:35.395121+0000","last_became_active":"2024-03-04T07:28:35.392517+0000","last_became_peered":"2024-03-04T07:28:35.392517+0000","last_unstale":"2024-03-04T07:28:35.395121+0000","last_undegraded":"2024-03-04T07:28:35.395121+0000","last_fullsized":"2024-03-04T07:28:35.395121+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:28:12.081672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"43'6","reported_seq":18,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.406163+0000","last_change":"2024-03-04T07:28:34.372976+0000","last_active":"2024-03-04T07:28:35.406163+0000","last_peered":"2024-03-04T07:28:35.406163+0000","last_clean":"2024-03-04T07:28:35.406163+0000","last_became_active":"2024-03-04T07:28:34.372677+0000","last_became_peered":"2024-03-04T07:28:34.372677+0000","last_unstale":"2024-03-04T07:28:35.406163+0000","last_undegraded":"2024-03-04T07:28:35.406163+0000","last_fullsized":"2024-03-04T07:28:35.406163+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:00:40.924212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.12","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.392105+0000","last_change":"2024-03-04T07:28:35.389045+0000","last_active":"2024-03-04T07:28:35.392105+0000","last_peered":"2024-03-04T07:28:35.392105+0000","last_clean":"2024-03-04T07:28:35.392105+0000","last_became_active":"2024-03-04T07:28:35.388749+0000","last_became_peered":"2024-03-04T07:28:35.388749+0000","last_unstale":"2024-03-04T07:28:35.392105+0000","last_undegraded":"2024-03-04T07:28:35.392105+0000","last_fullsized":"2024-03-04T07:28:35.392105+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:08.984993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.402682+0000","last_change":"2024-03-04T07:28:34.378924+0000","last_active":"2024-03-04T07:28:35.402682+0000","last_peered":"2024-03-04T07:28:35.402682+0000","last_clean":"2024-03-04T07:28:35.402682+0000","last_became_active":"2024-03-04T07:28:34.378611+0000","last_became_peered":"2024-03-04T07:28:34.378611+0000","last_unstale":"2024-03-04T07:28:35.402682+0000","last_undegraded":"2024-03-04T07:28:35.402682+0000","last_fullsized":"2024-03-04T07:28:35.402682+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:19:49.334496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.397903+0000","last_change":"2024-03-04T07:28:35.391894+0000","last_active":"2024-03-04T07:28:35.397903+0000","last_peered":"2024-03-04T07:28:35.397903+0000","last_clean":"2024-03-04T07:28:35.397903+0000","last_became_active":"2024-03-04T07:28:35.391627+0000","last_became_peered":"2024-03-04T07:28:35.391627+0000","last_unstale":"2024-03-04T07:28:35.397903+0000","last_undegraded":"2024-03-04T07:28:35.397903+0000","last_fullsized":"2024-03-04T07:28:35.397903+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:44:27.414540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.12","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377915+0000","last_change":"2024-03-04T07:28:34.370402+0000","last_active":"2024-03-04T07:28:35.377915+0000","last_peered":"2024-03-04T07:28:35.377915+0000","last_clean":"2024-03-04T07:28:35.377915+0000","last_became_active":"2024-03-04T07:28:34.370062+0000","last_became_peered":"2024-03-04T07:28:34.370062+0000","last_unstale":"2024-03-04T07:28:35.377915+0000","last_undegraded":"2024-03-04T07:28:35.377915+0000","last_fullsized":"2024-03-04T07:28:35.377915+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:41.390978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.10","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393042+0000","last_change":"2024-03-04T07:28:35.386850+0000","last_active":"2024-03-04T07:28:35.393042+0000","last_peered":"2024-03-04T07:28:35.393042+0000","last_clean":"2024-03-04T07:28:35.393042+0000","last_became_active":"2024-03-04T07:28:35.386512+0000","last_became_peered":"2024-03-04T07:28:35.386512+0000","last_unstale":"2024-03-04T07:28:35.393042+0000","last_undegraded":"2024-03-04T07:28:35.393042+0000","last_fullsized":"2024-03-04T07:28:35.393042+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:09.518032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.11","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.380159+0000","last_change":"2024-03-04T07:28:34.379003+0000","last_active":"2024-03-04T07:28:35.380159+0000","last_peered":"2024-03-04T07:28:35.380159+0000","last_clean":"2024-03-04T07:28:35.380159+0000","last_became_active":"2024-03-04T07:28:34.378746+0000","last_became_peered":"2024-03-04T07:28:34.378746+0000","last_unstale":"2024-03-04T07:28:35.380159+0000","last_undegraded":"2024-03-04T07:28:35.380159+0000","last_fullsized":"2024-03-04T07:28:35.380159+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:19.801457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.399810+0000","last_change":"2024-03-04T07:28:35.393367+0000","last_active":"2024-03-04T07:28:35.399810+0000","last_peered":"2024-03-04T07:28:35.399810+0000","last_clean":"2024-03-04T07:28:35.399810+0000","last_became_active":"2024-03-04T07:28:35.393071+0000","last_became_peered":"2024-03-04T07:28:35.393071+0000","last_unstale":"2024-03-04T07:28:35.399810+0000","last_undegraded":"2024-03-04T07:28:35.399810+0000","last_fullsized":"2024-03-04T07:28:35.399810+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:19:50.341073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"43'8","reported_seq":20,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.409398+0000","last_change":"2024-03-04T07:28:34.380700+0000","last_active":"2024-03-04T07:28:35.409398+0000","last_peered":"2024-03-04T07:28:35.409398+0000","last_clean":"2024-03-04T07:28:35.409398+0000","last_became_active":"2024-03-04T07:28:34.380395+0000","last_became_peered":"2024-03-04T07:28:34.380395+0000","last_unstale":"2024-03-04T07:28:35.409398+0000","last_undegraded":"2024-03-04T07:28:35.409398+0000","last_fullsized":"2024-03-04T07:28:35.409398+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:35:25.265192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.383585+0000","last_change":"2024-03-04T07:28:35.377087+0000","last_active":"2024-03-04T07:28:35.383585+0000","last_peered":"2024-03-04T07:28:35.383585+0000","last_clean":"2024-03-04T07:28:35.383585+0000","last_became_active":"2024-03-04T07:28:35.376813+0000","last_became_peered":"2024-03-04T07:28:35.376813+0000","last_unstale":"2024-03-04T07:28:35.383585+0000","last_undegraded":"2024-03-04T07:28:35.383585+0000","last_fullsized":"2024-03-04T07:28:35.383585+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:23:19.913686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.402398+0000","last_change":"2024-03-04T07:28:34.373601+0000","last_active":"2024-03-04T07:28:35.402398+0000","last_peered":"2024-03-04T07:28:35.402398+0000","last_clean":"2024-03-04T07:28:35.402398+0000","last_became_active":"2024-03-04T07:28:34.373332+0000","last_became_peered":"2024-03-04T07:28:34.373332+0000","last_unstale":"2024-03-04T07:28:35.402398+0000","last_undegraded":"2024-03-04T07:28:35.402398+0000","last_fullsized":"2024-03-04T07:28:35.402398+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:06:34.443138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391821+0000","last_change":"2024-03-04T07:28:35.386018+0000","last_active":"2024-03-04T07:28:35.391821+0000","last_peered":"2024-03-04T07:28:35.391821+0000","last_clean":"2024-03-04T07:28:35.391821+0000","last_became_active":"2024-03-04T07:28:35.385596+0000","last_became_peered":"2024-03-04T07:28:35.385596+0000","last_unstale":"2024-03-04T07:28:35.391821+0000","last_undegraded":"2024-03-04T07:28:35.391821+0000","last_fullsized":"2024-03-04T07:28:35.391821+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:21:48.544037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.400759+0000","last_change":"2024-03-04T07:28:34.369493+0000","last_active":"2024-03-04T07:28:35.400759+0000","last_peered":"2024-03-04T07:28:35.400759+0000","last_clean":"2024-03-04T07:28:35.400759+0000","last_became_active":"2024-03-04T07:28:34.369199+0000","last_became_peered":"2024-03-04T07:28:34.369199+0000","last_unstale":"2024-03-04T07:28:35.400759+0000","last_undegraded":"2024-03-04T07:28:35.400759+0000","last_fullsized":"2024-03-04T07:28:35.400759+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:20:24.052808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.392988+0000","last_change":"2024-03-04T07:28:35.389946+0000","last_active":"2024-03-04T07:28:35.392988+0000","last_peered":"2024-03-04T07:28:35.392988+0000","last_clean":"2024-03-04T07:28:35.392988+0000","last_became_active":"2024-03-04T07:28:35.389263+0000","last_became_peered":"2024-03-04T07:28:35.389263+0000","last_unstale":"2024-03-04T07:28:35.392988+0000","last_undegraded":"2024-03-04T07:28:35.392988+0000","last_fullsized":"2024-03-04T07:28:35.392988+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:21:48.544037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"43'1","reported_seq":13,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403052+0000","last_change":"2024-03-04T07:28:34.369186+0000","last_active":"2024-03-04T07:28:35.403052+0000","last_peered":"2024-03-04T07:28:35.403052+0000","last_clean":"2024-03-04T07:28:35.403052+0000","last_became_active":"2024-03-04T07:28:34.368869+0000","last_became_peered":"2024-03-04T07:28:34.368869+0000","last_unstale":"2024-03-04T07:28:35.403052+0000","last_undegraded":"2024-03-04T07:28:35.403052+0000","last_fullsized":"2024-03-04T07:28:35.403052+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:42:37.468797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.397646+0000","last_change":"2024-03-04T07:28:35.393868+0000","last_active":"2024-03-04T07:28:35.397646+0000","last_peered":"2024-03-04T07:28:35.397646+0000","last_clean":"2024-03-04T07:28:35.397646+0000","last_became_active":"2024-03-04T07:28:35.393247+0000","last_became_peered":"2024-03-04T07:28:35.393247+0000","last_unstale":"2024-03-04T07:28:35.397646+0000","last_undegraded":"2024-03-04T07:28:35.397646+0000","last_fullsized":"2024-03-04T07:28:35.397646+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:27.386478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.381095+0000","last_change":"2024-03-04T07:28:34.376484+0000","last_active":"2024-03-04T07:28:35.381095+0000","last_peered":"2024-03-04T07:28:35.381095+0000","last_clean":"2024-03-04T07:28:35.381095+0000","last_became_active":"2024-03-04T07:28:34.376178+0000","last_became_peered":"2024-03-04T07:28:34.376178+0000","last_unstale":"2024-03-04T07:28:35.381095+0000","last_undegraded":"2024-03-04T07:28:35.381095+0000","last_fullsized":"2024-03-04T07:28:35.381095+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:56:16.609626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396798+0000","last_change":"2024-03-04T07:28:35.388379+0000","last_active":"2024-03-04T07:28:35.396798+0000","last_peered":"2024-03-04T07:28:35.396798+0000","last_clean":"2024-03-04T07:28:35.396798+0000","last_became_active":"2024-03-04T07:28:35.387541+0000","last_became_peered":"2024-03-04T07:28:35.387541+0000","last_unstale":"2024-03-04T07:28:35.396798+0000","last_undegraded":"2024-03-04T07:28:35.396798+0000","last_fullsized":"2024-03-04T07:28:35.396798+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:08.866801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.405845+0000","last_change":"2024-03-04T07:28:34.373024+0000","last_active":"2024-03-04T07:28:35.405845+0000","last_peered":"2024-03-04T07:28:35.405845+0000","last_clean":"2024-03-04T07:28:35.405845+0000","last_became_active":"2024-03-04T07:28:34.372705+0000","last_became_peered":"2024-03-04T07:28:34.372705+0000","last_unstale":"2024-03-04T07:28:35.405845+0000","last_undegraded":"2024-03-04T07:28:35.405845+0000","last_fullsized":"2024-03-04T07:28:35.405845+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:29:03.369932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391229+0000","last_change":"2024-03-04T07:28:35.383641+0000","last_active":"2024-03-04T07:28:35.391229+0000","last_peered":"2024-03-04T07:28:35.391229+0000","last_clean":"2024-03-04T07:28:35.391229+0000","last_became_active":"2024-03-04T07:28:35.383379+0000","last_became_peered":"2024-03-04T07:28:35.383379+0000","last_unstale":"2024-03-04T07:28:35.391229+0000","last_undegraded":"2024-03-04T07:28:35.391229+0000","last_fullsized":"2024-03-04T07:28:35.391229+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:36:13.026173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.375891+0000","last_change":"2024-03-04T07:28:34.376230+0000","last_active":"2024-03-04T07:28:35.375891+0000","last_peered":"2024-03-04T07:28:35.375891+0000","last_clean":"2024-03-04T07:28:35.375891+0000","last_became_active":"2024-03-04T07:28:34.375928+0000","last_became_peered":"2024-03-04T07:28:34.375928+0000","last_unstale":"2024-03-04T07:28:35.375891+0000","last_undegraded":"2024-03-04T07:28:35.375891+0000","last_fullsized":"2024-03-04T07:28:35.375891+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:41.390978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.390911+0000","last_change":"2024-03-04T07:28:35.387702+0000","last_active":"2024-03-04T07:28:35.390911+0000","last_peered":"2024-03-04T07:28:35.390911+0000","last_clean":"2024-03-04T07:28:35.390911+0000","last_became_active":"2024-03-04T07:28:35.387355+0000","last_became_peered":"2024-03-04T07:28:35.387355+0000","last_unstale":"2024-03-04T07:28:35.390911+0000","last_undegraded":"2024-03-04T07:28:35.390911+0000","last_fullsized":"2024-03-04T07:28:35.390911+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:11.668312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.379870+0000","last_change":"2024-03-04T07:28:34.380203+0000","last_active":"2024-03-04T07:28:35.379870+0000","last_peered":"2024-03-04T07:28:35.379870+0000","last_clean":"2024-03-04T07:28:35.379870+0000","last_became_active":"2024-03-04T07:28:34.379880+0000","last_became_peered":"2024-03-04T07:28:34.379880+0000","last_unstale":"2024-03-04T07:28:35.379870+0000","last_undegraded":"2024-03-04T07:28:35.379870+0000","last_fullsized":"2024-03-04T07:28:35.379870+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:46:32.695936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.400211+0000","last_change":"2024-03-04T07:28:35.394843+0000","last_active":"2024-03-04T07:28:35.400211+0000","last_peered":"2024-03-04T07:28:35.400211+0000","last_clean":"2024-03-04T07:28:35.400211+0000","last_became_active":"2024-03-04T07:28:35.394538+0000","last_became_peered":"2024-03-04T07:28:35.394538+0000","last_unstale":"2024-03-04T07:28:35.400211+0000","last_undegraded":"2024-03-04T07:28:35.400211+0000","last_fullsized":"2024-03-04T07:28:35.400211+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:07:42.397556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377996+0000","last_change":"2024-03-04T07:28:34.374763+0000","last_active":"2024-03-04T07:28:35.377996+0000","last_peered":"2024-03-04T07:28:35.377996+0000","last_clean":"2024-03-04T07:28:35.377996+0000","last_became_active":"2024-03-04T07:28:34.374443+0000","last_became_peered":"2024-03-04T07:28:34.374443+0000","last_unstale":"2024-03-04T07:28:35.377996+0000","last_undegraded":"2024-03-04T07:28:35.377996+0000","last_fullsized":"2024-03-04T07:28:35.377996+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:42:37.468797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.395065+0000","last_change":"2024-03-04T07:28:35.390316+0000","last_active":"2024-03-04T07:28:35.395065+0000","last_peered":"2024-03-04T07:28:35.395065+0000","last_clean":"2024-03-04T07:28:35.395065+0000","last_became_active":"2024-03-04T07:28:35.387957+0000","last_became_peered":"2024-03-04T07:28:35.387957+0000","last_unstale":"2024-03-04T07:28:35.395065+0000","last_undegraded":"2024-03-04T07:28:35.395065+0000","last_fullsized":"2024-03-04T07:28:35.395065+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:15.340221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.381791+0000","last_change":"2024-03-04T07:28:34.377932+0000","last_active":"2024-03-04T07:28:35.381791+0000","last_peered":"2024-03-04T07:28:35.381791+0000","last_clean":"2024-03-04T07:28:35.381791+0000","last_became_active":"2024-03-04T07:28:34.377652+0000","last_became_peered":"2024-03-04T07:28:34.377652+0000","last_unstale":"2024-03-04T07:28:35.381791+0000","last_undegraded":"2024-03-04T07:28:35.381791+0000","last_fullsized":"2024-03-04T07:28:35.381791+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:03:33.880246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.398340+0000","last_change":"2024-03-04T07:28:35.393147+0000","last_active":"2024-03-04T07:28:35.398340+0000","last_peered":"2024-03-04T07:28:35.398340+0000","last_clean":"2024-03-04T07:28:35.398340+0000","last_became_active":"2024-03-04T07:28:35.392224+0000","last_became_peered":"2024-03-04T07:28:35.392224+0000","last_unstale":"2024-03-04T07:28:35.398340+0000","last_undegraded":"2024-03-04T07:28:35.398340+0000","last_fullsized":"2024-03-04T07:28:35.398340+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:36:13.026173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.378280+0000","last_change":"2024-03-04T07:28:34.378349+0000","last_active":"2024-03-04T07:28:35.378280+0000","last_peered":"2024-03-04T07:28:35.378280+0000","last_clean":"2024-03-04T07:28:35.378280+0000","last_became_active":"2024-03-04T07:28:34.378057+0000","last_became_peered":"2024-03-04T07:28:34.378057+0000","last_unstale":"2024-03-04T07:28:35.378280+0000","last_undegraded":"2024-03-04T07:28:35.378280+0000","last_fullsized":"2024-03-04T07:28:35.378280+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:45:42.486910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.391013+0000","last_change":"2024-03-04T07:28:35.387189+0000","last_active":"2024-03-04T07:28:35.391013+0000","last_peered":"2024-03-04T07:28:35.391013+0000","last_clean":"2024-03-04T07:28:35.391013+0000","last_became_active":"2024-03-04T07:28:35.386897+0000","last_became_peered":"2024-03-04T07:28:35.386897+0000","last_unstale":"2024-03-04T07:28:35.391013+0000","last_undegraded":"2024-03-04T07:28:35.391013+0000","last_fullsized":"2024-03-04T07:28:35.391013+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:04.006481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.375353+0000","last_change":"2024-03-04T07:28:34.371108+0000","last_active":"2024-03-04T07:28:35.375353+0000","last_peered":"2024-03-04T07:28:35.375353+0000","last_clean":"2024-03-04T07:28:35.375353+0000","last_became_active":"2024-03-04T07:28:34.370806+0000","last_became_peered":"2024-03-04T07:28:34.370806+0000","last_unstale":"2024-03-04T07:28:35.375353+0000","last_undegraded":"2024-03-04T07:28:35.375353+0000","last_fullsized":"2024-03-04T07:28:35.375353+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:44:26.407962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.393016+0000","last_change":"2024-03-04T07:28:35.386310+0000","last_active":"2024-03-04T07:28:35.393016+0000","last_peered":"2024-03-04T07:28:35.393016+0000","last_clean":"2024-03-04T07:28:35.393016+0000","last_became_active":"2024-03-04T07:28:35.386020+0000","last_became_peered":"2024-03-04T07:28:35.386020+0000","last_unstale":"2024-03-04T07:28:35.393016+0000","last_undegraded":"2024-03-04T07:28:35.393016+0000","last_fullsized":"2024-03-04T07:28:35.393016+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:35:45.048101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.378888+0000","last_change":"2024-03-04T07:28:34.378354+0000","last_active":"2024-03-04T07:28:35.378888+0000","last_peered":"2024-03-04T07:28:35.378888+0000","last_clean":"2024-03-04T07:28:35.378888+0000","last_became_active":"2024-03-04T07:28:34.378034+0000","last_became_peered":"2024-03-04T07:28:34.378034+0000","last_unstale":"2024-03-04T07:28:35.378888+0000","last_undegraded":"2024-03-04T07:28:35.378888+0000","last_fullsized":"2024-03-04T07:28:35.378888+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:37:23.028719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.396746+0000","last_change":"2024-03-04T07:28:35.388926+0000","last_active":"2024-03-04T07:28:35.396746+0000","last_peered":"2024-03-04T07:28:35.396746+0000","last_clean":"2024-03-04T07:28:35.396746+0000","last_became_active":"2024-03-04T07:28:35.388659+0000","last_became_peered":"2024-03-04T07:28:35.388659+0000","last_unstale":"2024-03-04T07:28:35.396746+0000","last_undegraded":"2024-03-04T07:28:35.396746+0000","last_fullsized":"2024-03-04T07:28:35.396746+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:10.484767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.374385+0000","last_change":"2024-03-04T07:28:34.379297+0000","last_active":"2024-03-04T07:28:35.374385+0000","last_peered":"2024-03-04T07:28:35.374385+0000","last_clean":"2024-03-04T07:28:35.374385+0000","last_became_active":"2024-03-04T07:28:34.379041+0000","last_became_peered":"2024-03-04T07:28:34.379041+0000","last_unstale":"2024-03-04T07:28:35.374385+0000","last_undegraded":"2024-03-04T07:28:35.374385+0000","last_fullsized":"2024-03-04T07:28:35.374385+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:39:55.893953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.383050+0000","last_change":"2024-03-04T07:28:35.378306+0000","last_active":"2024-03-04T07:28:35.383050+0000","last_peered":"2024-03-04T07:28:35.383050+0000","last_clean":"2024-03-04T07:28:35.383050+0000","last_became_active":"2024-03-04T07:28:35.378008+0000","last_became_peered":"2024-03-04T07:28:35.378008+0000","last_unstale":"2024-03-04T07:28:35.383050+0000","last_undegraded":"2024-03-04T07:28:35.383050+0000","last_fullsized":"2024-03-04T07:28:35.383050+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:45:43.493488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.375606+0000","last_change":"2024-03-04T07:28:34.367749+0000","last_active":"2024-03-04T07:28:35.375606+0000","last_peered":"2024-03-04T07:28:35.375606+0000","last_clean":"2024-03-04T07:28:35.375606+0000","last_became_active":"2024-03-04T07:28:34.367417+0000","last_became_peered":"2024-03-04T07:28:34.367417+0000","last_unstale":"2024-03-04T07:28:35.375606+0000","last_undegraded":"2024-03-04T07:28:35.375606+0000","last_fullsized":"2024-03-04T07:28:35.375606+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:20:24.052808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394568+0000","last_change":"2024-03-04T07:28:35.391088+0000","last_active":"2024-03-04T07:28:35.394568+0000","last_peered":"2024-03-04T07:28:35.394568+0000","last_clean":"2024-03-04T07:28:35.394568+0000","last_became_active":"2024-03-04T07:28:35.390812+0000","last_became_peered":"2024-03-04T07:28:35.390812+0000","last_unstale":"2024-03-04T07:28:35.394568+0000","last_undegraded":"2024-03-04T07:28:35.394568+0000","last_fullsized":"2024-03-04T07:28:35.394568+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:42:38.475375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":12,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.377883+0000","last_change":"2024-03-04T07:28:34.378476+0000","last_active":"2024-03-04T07:28:35.377883+0000","last_peered":"2024-03-04T07:28:35.377883+0000","last_clean":"2024-03-04T07:28:35.377883+0000","last_became_active":"2024-03-04T07:28:34.378170+0000","last_became_peered":"2024-03-04T07:28:34.378170+0000","last_unstale":"2024-03-04T07:28:35.377883+0000","last_undegraded":"2024-03-04T07:28:35.377883+0000","last_fullsized":"2024-03-04T07:28:35.377883+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:46:17.432274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":9,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.394994+0000","last_change":"2024-03-04T07:28:35.390240+0000","last_active":"2024-03-04T07:28:35.394994+0000","last_peered":"2024-03-04T07:28:35.394994+0000","last_clean":"2024-03-04T07:28:35.394994+0000","last_became_active":"2024-03-04T07:28:35.389978+0000","last_became_peered":"2024-03-04T07:28:35.389978+0000","last_unstale":"2024-03-04T07:28:35.394994+0000","last_undegraded":"2024-03-04T07:28:35.394994+0000","last_fullsized":"2024-03-04T07:28:35.394994+0000","mapping_epoch":42,"log_start":"0'0","ondisk_log_start":"0'0","created":42,"last_epoch_clean":43,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:34.353765+0000","last_clean_scrub_stamp":"2024-03-04T07:28:34.353765+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:27.386478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"43'2","reported_seq":14,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.403169+0000","last_change":"2024-03-04T07:28:34.377128+0000","last_active":"2024-03-04T07:28:35.403169+0000","last_peered":"2024-03-04T07:28:35.403169+0000","last_clean":"2024-03-04T07:28:35.403169+0000","last_became_active":"2024-03-04T07:28:34.376617+0000","last_became_peered":"2024-03-04T07:28:34.376617+0000","last_unstale":"2024-03-04T07:28:35.403169+0000","last_undegraded":"2024-03-04T07:28:35.403169+0000","last_fullsized":"2024-03-04T07:28:35.403169+0000","mapping_epoch":41,"log_start":"0'0","ondisk_log_start":"0'0","created":41,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:28:33.347187+0000","last_clean_scrub_stamp":"2024-03-04T07:28:33.347187+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:58:36.732068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":211,"reported_epoch":43,"state":"active+clean","last_fresh":"2024-03-04T07:28:35.378107+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:28:35.378107+0000","last_peered":"2024-03-04T07:28:35.378107+0000","last_clean":"2024-03-04T07:28:35.378107+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:28:35.378107+0000","last_undegraded":"2024-03-04T07:28:35.378107+0000","last_fullsized":"2024-03-04T07:28:35.378107+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":6,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574876,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32264,"kb_used_data":1076,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339576,"statfs":{"total":96636764160,"available":96603725824,"internally_reserved":0,"allocated":1101824,"data_stored":793958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574875,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31688,"kb_used_data":500,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340152,"statfs":{"total":96636764160,"available":96604315648,"internally_reserved":0,"allocated":512000,"data_stored":204172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574875,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31684,"kb_used_data":496,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340156,"statfs":{"total":96636764160,"available":96604319744,"internally_reserved":0,"allocated":507904,"data_stored":203570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574876,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":516096,"data_stored":205112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574875,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31692,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340148,"statfs":{"total":96636764160,"available":96604311552,"internally_reserved":0,"allocated":516096,"data_stored":204138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574875,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31696,"kb_used_data":508,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340144,"statfs":{"total":96636764160,"available":96604307456,"internally_reserved":0,"allocated":520192,"data_stored":205134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574875,"num_pgs":10,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31664,"kb_used_data":480,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340176,"statfs":{"total":96636764160,"available":96604340224,"internally_reserved":0,"allocated":491520,"data_stored":196725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574875,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32244,"kb_used_data":1060,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339596,"statfs":{"total":96636764160,"available":96603746304,"internally_reserved":0,"allocated":1085440,"data_stored":787093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:28:38.743 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:28:38.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:28:38.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.889+0000 7f60173bc640 1 Processor -- start 2024-03-04T07:28:38.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.889+0000 7f60173bc640 1 -- start start 2024-03-04T07:28:38.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f60173bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f601015f500 0x7f60100ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f60173bc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f60100c5e30 0x7f60100c6280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f60173bc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f60100cd310 con 0x7f60100cf740 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f60173bc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f601015f8e0 con 0x7f60100c5e30 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f60173bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f60100c56c0 con 0x7f601015f500 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f6015158640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f601015f500 0x7f60100ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f6014957640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f60100cf740 0x7f60100ce840 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:33700/0 (socket says 172.21.15.64:33700) 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.890+0000 7f6015158640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f601015f500 0x7f60100ce300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:40378/0 (socket says 172.21.15.64:40378) 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f6014957640 1 -- 172.21.15.64:0/540056715 learned_addr learned my addr 172.21.15.64:0/540056715 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f6015959640 1 --2- 172.21.15.64:0/540056715 >> v2:172.21.15.66:3300/0 conn(0x7f60100c5e30 0x7f60100c6280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f600ffff640 1 -- 172.21.15.64:0/540056715 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2806715527 0 0) 0x7f60100cd310 con 0x7f60100cf740 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f600ffff640 1 -- 172.21.15.64:0/540056715 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5ff8003520 con 0x7f60100cf740 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f6015158640 1 -- 172.21.15.64:0/540056715 >> v2:172.21.15.66:3300/0 conn(0x7f60100c5e30 msgr2=0x7f60100c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f6015158640 1 --2- 172.21.15.64:0/540056715 >> v2:172.21.15.66:3300/0 conn(0x7f60100c5e30 0x7f60100c6280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f6015158640 1 -- 172.21.15.64:0/540056715 >> v1:172.21.15.66:6789/0 conn(0x7f60100cf740 legacy=0x7f60100ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.891+0000 7f6015158640 1 -- 172.21.15.64:0/540056715 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60100cd5b0 con 0x7f601015f500 2024-03-04T07:28:38.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f6015158640 1 --2- 172.21.15.64:0/540056715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f601015f500 0x7f60100ce300 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f60080097a0 tx=0x7f600802b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=59800691e25ffe9b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f600ffff640 1 -- 172.21.15.64:0/540056715 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60080315a0 con 0x7f601015f500 2024-03-04T07:28:38.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f600ffff640 1 -- 172.21.15.64:0/540056715 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6008037070 con 0x7f601015f500 2024-03-04T07:28:38.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f600ffff640 1 -- 172.21.15.64:0/540056715 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6008031960 con 0x7f601015f500 2024-03-04T07:28:38.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f60173bc640 1 -- 172.21.15.64:0/540056715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f601015f500 msgr2=0x7f60100ce300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f60173bc640 1 --2- 172.21.15.64:0/540056715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f601015f500 0x7f60100ce300 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f60080097a0 tx=0x7f600802b840 comp rx=0 tx=0).stop 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f60173bc640 1 -- 172.21.15.64:0/540056715 shutdown_connections 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f60173bc640 1 --2- 172.21.15.64:0/540056715 >> v2:172.21.15.66:3300/0 conn(0x7f60100c5e30 0x7f60100c6280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f60173bc640 1 --2- 172.21.15.64:0/540056715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f601015f500 0x7f60100ce300 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.892+0000 7f60173bc640 1 -- 172.21.15.64:0/540056715 >> 172.21.15.64:0/540056715 conn(0x7f60100b3700 msgr2=0x7f60100b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.893+0000 7f60173bc640 1 -- 172.21.15.64:0/540056715 shutdown_connections 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.893+0000 7f60173bc640 1 -- 172.21.15.64:0/540056715 wait complete. 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 Processor -- start 2024-03-04T07:28:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 -- start start 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60100c5e30 0x7f60101522a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60100cf740 0x7f60100b68b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f601015f500 0x7f601015a6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60100b43d0 con 0x7f60100cf740 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60100c59a0 con 0x7f601015f500 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f60173bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f60100cef50 con 0x7f60100c5e30 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.894+0000 7f6014957640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60100cf740 0x7f60100b68b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60100cf740 0x7f60100b68b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35024/0 (socket says 172.21.15.64:35024) 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 -- 172.21.15.64:0/2831021362 learned_addr learned my addr 172.21.15.64:0/2831021362 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6015959640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f601015f500 0x7f601015a6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6015158640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60100c5e30 0x7f60101522a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 -- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60100c5e30 msgr2=0x7f60101522a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60100c5e30 0x7f60101522a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 -- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f601015f500 msgr2=0x7f601015a6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f601015f500 0x7f601015a6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 -- 172.21.15.64:0/2831021362 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f601006b300 con 0x7f60100cf740 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f6014957640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60100cf740 0x7f60100b68b0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f600400c3e0 tx=0x7f600400c8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f600dffb640 1 -- 172.21.15.64:0/2831021362 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6004017070 con 0x7f60100cf740 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f600dffb640 1 -- 172.21.15.64:0/2831021362 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6004005ce0 con 0x7f60100cf740 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f601006b5e0 con 0x7f60100cf740 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f600dffb640 1 -- 172.21.15.64:0/2831021362 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60040072f0 con 0x7f60100cf740 2024-03-04T07:28:38.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.895+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f601006bba0 con 0x7f60100cf740 2024-03-04T07:28:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.897+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f601009fcf0 con 0x7f60100cf740 2024-03-04T07:28:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.897+0000 7f600dffb640 1 -- 172.21.15.64:0/2831021362 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f600401a070 con 0x7f60100cf740 2024-03-04T07:28:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.897+0000 7f600dffb640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ffc06b500 0x7f5ffc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.897+0000 7f600dffb640 1 -- 172.21.15.64:0/2831021362 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6004089480 con 0x7f60100cf740 2024-03-04T07:28:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.897+0000 7f6015158640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ffc06b500 0x7f5ffc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.898+0000 7f6015158640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ffc06b500 0x7f5ffc06d9c0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f6008036000 tx=0x7f600803d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:38.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:38.902+0000 7f600dffb640 1 -- 172.21.15.64:0/2831021362 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6004057d40 con 0x7f60100cf740 2024-03-04T07:28:39.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.149+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f60100c7c20 con 0x7f60100cf740 2024-03-04T07:28:39.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.151+0000 7f600dffb640 1 -- 172.21.15.64:0/2831021362 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v16) v1 ==== 190+0+147 (secure 0 0 0) 0x7f600405bde0 con 0x7f60100cf740 2024-03-04T07:28:39.151 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:28:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:28:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:28:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:28:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:28:39.151 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:28:39.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ffc06b500 msgr2=0x7f5ffc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ffc06b500 0x7f5ffc06d9c0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f6008036000 tx=0x7f600803d040 comp rx=0 tx=0).stop 2024-03-04T07:28:39.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60100cf740 msgr2=0x7f60100b68b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60100cf740 0x7f60100b68b0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f600400c3e0 tx=0x7f600400c8a0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 shutdown_connections 2024-03-04T07:28:39.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ffc06b500 0x7f5ffc06d9c0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f601015f500 0x7f601015a6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60100cf740 0x7f60100b68b0 unknown :-1 s=CLOSED pgs=539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 --2- 172.21.15.64:0/2831021362 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60100c5e30 0x7f60101522a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 >> 172.21.15.64:0/2831021362 conn(0x7f60100b3700 msgr2=0x7f60100c62f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:39.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 shutdown_connections 2024-03-04T07:28:39.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.153+0000 7f60173bc640 1 -- 172.21.15.64:0/2831021362 wait complete. 2024-03-04T07:28:39.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:39.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.265+0000 7f4c5dea4640 1 Processor -- start 2024-03-04T07:28:39.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.266+0000 7f4c5dea4640 1 -- start start 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.266+0000 7f4c5dea4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c580d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.266+0000 7f4c5dea4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4c580cee90 con 0x7f4c580b0e90 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.266+0000 7f4c5dea4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4c580cfc50 con 0x7f4c5806b5c0 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.266+0000 7f4c5dea4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4c580cfa00 con 0x7f4c580ae9b0 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.266+0000 7f4c56ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4c580b0e90 0x7f4c5806b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40922/0 (socket says 172.21.15.64:40922) 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.266+0000 7f4c56ffd640 1 -- 172.21.15.64:0/1856777559 learned_addr learned my addr 172.21.15.64:0/1856777559 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c57fff640 1 --2- 172.21.15.64:0/1856777559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c580d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c567fc640 1 -- 172.21.15.64:0/1856777559 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4236167808 0 0) 0x7f4c580cee90 con 0x7f4c580b0e90 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c567fc640 1 -- 172.21.15.64:0/1856777559 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4c40003520 con 0x7f4c580b0e90 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c567fc640 1 -- 172.21.15.64:0/1856777559 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1269853698 0 0) 0x7f4c580cfc50 con 0x7f4c5806b5c0 2024-03-04T07:28:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c567fc640 1 -- 172.21.15.64:0/1856777559 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4c580cee90 con 0x7f4c5806b5c0 2024-03-04T07:28:39.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c57fff640 1 -- 172.21.15.64:0/1856777559 >> v1:172.21.15.66:6789/0 conn(0x7f4c5806b5c0 legacy=0x7f4c580b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c57fff640 1 -- 172.21.15.64:0/1856777559 >> v1:172.21.15.64:6789/0 conn(0x7f4c580b0e90 legacy=0x7f4c5806b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.267+0000 7f4c57fff640 1 -- 172.21.15.64:0/1856777559 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c580cff30 con 0x7f4c580ae9b0 2024-03-04T07:28:39.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.268+0000 7f4c57fff640 1 --2- 172.21.15.64:0/1856777559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c580d03a0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f4c50009dc0 tx=0x7f4c5002be90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a4742fddfe17260d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:39.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.268+0000 7f4c567fc640 1 -- 172.21.15.64:0/1856777559 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c50039070 con 0x7f4c580ae9b0 2024-03-04T07:28:39.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.268+0000 7f4c567fc640 1 -- 172.21.15.64:0/1856777559 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4c5002cc40 con 0x7f4c580ae9b0 2024-03-04T07:28:39.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.268+0000 7f4c567fc640 1 -- 172.21.15.64:0/1856777559 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c50034440 con 0x7f4c580ae9b0 2024-03-04T07:28:39.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.269+0000 7f4c5dea4640 1 -- 172.21.15.64:0/1856777559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 msgr2=0x7f4c580d03a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.269+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/1856777559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c580d03a0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f4c50009dc0 tx=0x7f4c5002be90 comp rx=0 tx=0).stop 2024-03-04T07:28:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.269+0000 7f4c5dea4640 1 -- 172.21.15.64:0/1856777559 shutdown_connections 2024-03-04T07:28:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.269+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/1856777559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c580d03a0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.269+0000 7f4c5dea4640 1 -- 172.21.15.64:0/1856777559 >> 172.21.15.64:0/1856777559 conn(0x7f4c580b36d0 msgr2=0x7f4c580b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.269+0000 7f4c5dea4640 1 -- 172.21.15.64:0/1856777559 shutdown_connections 2024-03-04T07:28:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.269+0000 7f4c5dea4640 1 -- 172.21.15.64:0/1856777559 wait complete. 2024-03-04T07:28:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.271+0000 7f4c5dea4640 1 Processor -- start 2024-03-04T07:28:39.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.271+0000 7f4c5dea4640 1 -- start start 2024-03-04T07:28:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c5dea4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c5806b5c0 0x7f4c580c5810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c5dea4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c581f7cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c5dea4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c580b0e90 0x7f4c581dfff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c5dea4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c580aef80 con 0x7f4c580b0e90 2024-03-04T07:28:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c5dea4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c580aee00 con 0x7f4c5806b5c0 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c5dea4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4c580d08e0 con 0x7f4c580ae9b0 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c580b0e90 0x7f4c581dfff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c580b0e90 0x7f4c581dfff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35034/0 (socket says 172.21.15.64:35034) 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 -- 172.21.15.64:0/2960569251 learned_addr learned my addr 172.21.15.64:0/2960569251 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c577fe640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c5806b5c0 0x7f4c580c5810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c56ffd640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c581f7cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 -- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 msgr2=0x7f4c581f7cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c581f7cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 -- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c5806b5c0 msgr2=0x7f4c580c5810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c5806b5c0 0x7f4c580c5810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c57fff640 1 -- 172.21.15.64:0/2960569251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c581e39b0 con 0x7f4c580b0e90 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.272+0000 7f4c577fe640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c5806b5c0 0x7f4c580c5810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.273+0000 7f4c57fff640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c580b0e90 0x7f4c581dfff0 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f4c50037040 tx=0x7f4c50004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:39.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.273+0000 7f4c3ffff640 1 -- 172.21.15.64:0/2960569251 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c50039050 con 0x7f4c580b0e90 2024-03-04T07:28:39.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.273+0000 7f4c3ffff640 1 -- 172.21.15.64:0/2960569251 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4c50045070 con 0x7f4c580b0e90 2024-03-04T07:28:39.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.273+0000 7f4c3ffff640 1 -- 172.21.15.64:0/2960569251 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c50040690 con 0x7f4c580b0e90 2024-03-04T07:28:39.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.273+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4c581e3c90 con 0x7f4c580b0e90 2024-03-04T07:28:39.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.273+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4c581e4250 con 0x7f4c580b0e90 2024-03-04T07:28:39.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.274+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4c58068700 con 0x7f4c580b0e90 2024-03-04T07:28:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.274+0000 7f4c3ffff640 1 -- 172.21.15.64:0/2960569251 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4c50040830 con 0x7f4c580b0e90 2024-03-04T07:28:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.275+0000 7f4c3ffff640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c4406b4d0 0x7f4c4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.275+0000 7f4c3ffff640 1 -- 172.21.15.64:0/2960569251 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4c500acbf0 con 0x7f4c580b0e90 2024-03-04T07:28:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.275+0000 7f4c577fe640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c4406b4d0 0x7f4c4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.275+0000 7f4c577fe640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c4406b4d0 0x7f4c4406d990 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f4c480016b0 tx=0x7f4c48009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:39.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.279+0000 7f4c3ffff640 1 -- 172.21.15.64:0/2960569251 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4c5007b9f0 con 0x7f4c580b0e90 2024-03-04T07:28:39.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.498+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4c580c8460 con 0x7f4c580b0e90 2024-03-04T07:28:39.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.498+0000 7f4c3ffff640 1 -- 172.21.15.64:0/2960569251 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 22 v22) v1 ==== 94+0+4971 (secure 0 0 0) 0x7f4c5007b2f0 con 0x7f4c580b0e90 2024-03-04T07:28:39.498 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:39.499 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5138,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/627869633","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":627869633},{"type":"v1","addr":"172.21.15.66:6834","nonce":627869633}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5141,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2387393562","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2387393562},{"type":"v1","addr":"172.21.15.66:6835","nonce":2387393562}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5685,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/817733642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":817733642},{"type":"v1","addr":"172.21.15.64:6837","nonce":817733642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":18}],"filesystems":[{"mdsmap":{"epoch":22,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:28:34.726288+0000","modified":"2024-03-04T07:28:36.736175+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5688},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5688":{"gid":5688,"name":"a","rank":0,"incarnation":20,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/3981471748","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3981471748},{"type":"v1","addr":"172.21.15.64:6836","nonce":3981471748}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5688,"qdb_cluster":[5688]},"id":3}]} 2024-03-04T07:28:39.499 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 22 2024-03-04T07:28:39.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c4406b4d0 msgr2=0x7f4c4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c4406b4d0 0x7f4c4406d990 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f4c480016b0 tx=0x7f4c48009040 comp rx=0 tx=0).stop 2024-03-04T07:28:39.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c580b0e90 msgr2=0x7f4c581dfff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c580b0e90 0x7f4c581dfff0 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f4c50037040 tx=0x7f4c50004a40 comp rx=0 tx=0).stop 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 shutdown_connections 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c4406b4d0 0x7f4c4406d990 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c580b0e90 0x7f4c581dfff0 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c580ae9b0 0x7f4c581f7cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 --2- 172.21.15.64:0/2960569251 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c5806b5c0 0x7f4c580c5810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 >> 172.21.15.64:0/2960569251 conn(0x7f4c580b36d0 msgr2=0x7f4c580b1ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.500+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 shutdown_connections 2024-03-04T07:28:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:39.501+0000 7f4c5dea4640 1 -- 172.21.15.64:0/2960569251 wait complete. 2024-03-04T07:28:39.914 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 22, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:28:34.726288+0000', 'modified': '2024-03-04T07:28:36.736175+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 5688}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_5688': {'gid': 5688, 'name': 'a', 'rank': 0, 'incarnation': 20, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.64:6836/3981471748', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 3981471748}, {'type': 'v1', 'addr': '172.21.15.64:6836', 'nonce': 3981471748}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [7], 'metadata_pool': 6, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 5688, 'qdb_cluster': [5688]} 2024-03-04T07:28:39.914 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:28:39.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.a status 2024-03-04T07:28:40.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.018+0000 7f3b1c8ec640 1 Processor -- start 2024-03-04T07:28:40.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.018+0000 7f3b1c8ec640 1 -- start start 2024-03-04T07:28:40.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1c8ec640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3b1815df00 0x7f3b180d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1c8ec640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b9a30 0x7f3b180d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1c8ec640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3b180cea30 con 0x7f3b180b95a0 2024-03-04T07:28:40.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1c8ec640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3b180d2af0 con 0x7f3b1815df00 2024-03-04T07:28:40.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1c8ec640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b180cf3c0 con 0x7f3b180b9a30 2024-03-04T07:28:40.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1659c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3b1815df00 0x7f3b180d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b15d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3b180b95a0 0x7f3b180b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40926/0 (socket says 172.21.15.64:40926) 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b15d9b640 1 -- 172.21.15.64:0/1909740045 learned_addr learned my addr 172.21.15.64:0/1909740045 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1659c640 1 --2- 172.21.15.64:0/1909740045 >> v2:172.21.15.64:3300/0 conn(0x7f3b1815df00 0x7f3b180d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35040/0 (socket says 172.21.15.64:35040) 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/1909740045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b9a30 0x7f3b180d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1659c640 1 -- 172.21.15.64:0/1909740045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b9a30 msgr2=0x7f3b180d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1659c640 1 --2- 172.21.15.64:0/1909740045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b9a30 0x7f3b180d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1659c640 1 -- 172.21.15.64:0/1909740045 >> v1:172.21.15.64:6789/0 conn(0x7f3b180b95a0 legacy=0x7f3b180b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1659c640 1 -- 172.21.15.64:0/1909740045 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b180cf0e0 con 0x7f3b1815df00 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b1559a640 1 -- 172.21.15.64:0/1909740045 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4113960550 0 0) 0x7f3b180cea30 con 0x7f3b180b95a0 2024-03-04T07:28:40.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.019+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/1909740045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b9a30 0x7f3b180d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.020+0000 7f3b1659c640 1 --2- 172.21.15.64:0/1909740045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180d25b0 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f3b080097a0 tx=0x7f3b0802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4777ebab864431fb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.020+0000 7f3b1559a640 1 -- 172.21.15.64:0/1909740045 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b08035020 con 0x7f3b1815df00 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.020+0000 7f3b1559a640 1 -- 172.21.15.64:0/1909740045 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b08031830 con 0x7f3b1815df00 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.020+0000 7f3b1559a640 1 -- 172.21.15.64:0/1909740045 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b08031c00 con 0x7f3b1815df00 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/1909740045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 msgr2=0x7f3b180d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/1909740045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180d25b0 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f3b080097a0 tx=0x7f3b0802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/1909740045 shutdown_connections 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/1909740045 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b9a30 0x7f3b180d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/1909740045 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180d25b0 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/1909740045 >> 172.21.15.64:0/1909740045 conn(0x7f3b18071960 msgr2=0x7f3b18071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/1909740045 shutdown_connections 2024-03-04T07:28:40.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.021+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/1909740045 wait complete. 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.023+0000 7f3b1c8ec640 1 Processor -- start 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.023+0000 7f3b1c8ec640 1 -- start start 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b1c8ec640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b95a0 0x7f3b180cb740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b1c8ec640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180b5760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b1c8ec640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b180c7ea0 0x7f3b180c8300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b1c8ec640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b180cec70 con 0x7f3b1815df00 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b1c8ec640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b180ce750 con 0x7f3b180c7ea0 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b1c8ec640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b180cedf0 con 0x7f3b180b95a0 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180b5760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180b5760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35056/0 (socket says 172.21.15.64:35056) 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 -- 172.21.15.64:0/928803308 learned_addr learned my addr 172.21.15.64:0/928803308 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:40.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b1659c640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b95a0 0x7f3b180cb740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b180c7ea0 0x7f3b180c8300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 -- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b95a0 msgr2=0x7f3b180cb740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b95a0 0x7f3b180cb740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 -- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b180c7ea0 msgr2=0x7f3b180c8300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b180c7ea0 0x7f3b180c8300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.024+0000 7f3b15d9b640 1 -- 172.21.15.64:0/928803308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b180c8940 con 0x7f3b1815df00 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.025+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180b5760 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f3b0c00c500 tx=0x7f3b0c00c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.025+0000 7f3b06ffd640 1 -- 172.21.15.64:0/928803308 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b0c018070 con 0x7f3b1815df00 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.025+0000 7f3b06ffd640 1 -- 172.21.15.64:0/928803308 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b0c007580 con 0x7f3b1815df00 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.025+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b180c8c20 con 0x7f3b1815df00 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.025+0000 7f3b06ffd640 1 -- 172.21.15.64:0/928803308 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b0c004a20 con 0x7f3b1815df00 2024-03-04T07:28:40.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.025+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b180cb240 con 0x7f3b1815df00 2024-03-04T07:28:40.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.027+0000 7f3b06ffd640 1 -- 172.21.15.64:0/928803308 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3b0c01b070 con 0x7f3b1815df00 2024-03-04T07:28:40.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.027+0000 7f3b06ffd640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3af806b4d0 0x7f3af806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.027+0000 7f3b06ffd640 1 -- 172.21.15.64:0/928803308 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3b0c0894a0 con 0x7f3b1815df00 2024-03-04T07:28:40.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.027+0000 7f3b1659c640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3af806b4d0 0x7f3af806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.027+0000 7f3b1f5a8740 1 Processor -- start 2024-03-04T07:28:40.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1659c640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3af806b4d0 0x7f3af806d990 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f3b08036000 tx=0x7f3b08008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1f5a8740 1 -- start start 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9748870 0x558af9748c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af974c8c0 0x558af974e360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9778a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x558af96b26b0 con 0x558af9748870 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x558af965ba60 con 0x558af974c8c0 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x558af96ae750 con 0x558af9778600 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.028+0000 7f3b1659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9748870 0x558af9748c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9748870 0x558af9748c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35072/0 (socket says 172.21.15.64:35072) 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 -- 172.21.15.64:0/481490499 learned_addr learned my addr 172.21.15.64:0/481490499 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af974c8c0 0x558af974e360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 -- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 msgr2=0x558af9778a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9778a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9778a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 -- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af974c8c0 msgr2=0x558af974e360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af974c8c0 0x558af974e360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 -- 172.21.15.64:0/481490499 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558af9679f30 con 0x558af9748870 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af974c8c0 0x558af974e360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3b1659c640 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9748870 0x558af9748c50 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f3b08034040 tx=0x7f3b08030000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8418eddc70eeea48 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3af7fff640 1 -- 172.21.15.64:0/481490499 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b0804a070 con 0x558af9748870 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3af7fff640 1 -- 172.21.15.64:0/481490499 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b0803ab70 con 0x558af9748870 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.029+0000 7f3af7fff640 1 -- 172.21.15.64:0/481490499 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b08035040 con 0x558af9748870 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/481490499 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9748870 msgr2=0x558af9748c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9748870 0x558af9748c50 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f3b08034040 tx=0x7f3b08030000 comp rx=0 tx=0).stop 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/481490499 shutdown_connections 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9778a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af974c8c0 0x558af974e360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/481490499 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9748870 0x558af9748c50 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/481490499 >> 172.21.15.64:0/481490499 conn(0x558af974a8f0 msgr2=0x558af974c1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/481490499 shutdown_connections 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.030+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/481490499 wait complete. 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.031+0000 7f3b1f5a8740 1 -- start start 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.031+0000 7f3b1f5a8740 1 Processor -- start 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.031+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af9764950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9793760 0x558af9793b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9794080 0x558af97a75a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x558af974cd50 con 0x558af9778600 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x558af966eea0 con 0x558af9794080 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x558af974d030 con 0x558af9793760 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1f5a8740 10 client.0 _ll_register_callbacks cb 0x558af966e870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af9764950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af9764950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35074/0 (socket says 172.21.15.64:35074) 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 -- 172.21.15.64:0/1879342407 learned_addr learned my addr 172.21.15.64:0/1879342407 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9794080 0x558af97a75a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9793760 0x558af9793b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 -- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9793760 msgr2=0x558af9793b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9793760 0x558af9793b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 -- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9794080 msgr2=0x558af97a75a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9794080 0x558af97a75a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 -- 172.21.15.64:0/1879342407 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558af966f270 con 0x558af9778600 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1659c640 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af9764950 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f3b0802bee0 tx=0x7f3b0802e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e2f7de787283c08 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3af57fa640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3af57fa640 1 -- 172.21.15.64:0/1879342407 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b0804a070 con 0x558af9778600 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3af57fa640 1 -- 172.21.15.64:0/1879342407 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3b0804fd60 con 0x558af9778600 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.032+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3b0803ab70 con 0x558af9778600 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3af57fa640 1 -- 172.21.15.64:0/1879342407 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3b08060050 con 0x558af9778600 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 10 client.5829 fetch_fsmap learned FSMap version 22 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=22,monmap=2+}) v3 -- 0x558af93c72d0 con 0x558af9778600 2024-03-04T07:28:40.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3af57fa640 1 -- 172.21.15.64:0/1879342407 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b0803ab70 con 0x558af9778600 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3af57fa640 1 -- 172.21.15.64:0/1879342407 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 22) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f3b08058410 con 0x558af9778600 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 10 client.5829 fetch_fsmap finished waiting for FSMap version 22 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 10 client.5829 resolve_mds: resolved name 'a' to daemon mds.a 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 20 client.5829 populate_metadata read hostname 'smithi064' 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af97af9e0 0x558af97b1dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 4 client.5829 mds_command: new command op to 5688 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 --> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x558af96bfbf0 con 0x558af97af9e0 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af97af9e0 0x558af97b1dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3af57fa640 10 client.5829 ms_handle_connect on v2:172.21.15.64:6834/3981471748 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.033+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af97af9e0 0x558af97b1dd0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b7a9e6bf88337b70 server_cookie=ab1f39a2f187ced6 in_seq=0 out_seq=0 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3af57fa640 1 -- 172.21.15.64:0/1879342407 <== mds.0 v2:172.21.15.64:6834/3981471748 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x558af96bfbf0 con 0x558af97af9e0 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3af57fa640 10 client.5829 handle_command_reply: tid=0 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 client.5829 shutdown 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 msgr2=0x558af9764950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af9764950 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f3b0802bee0 tx=0x7f3b0802e000 comp rx=0 tx=0).stop 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 shutdown_connections 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af97af9e0 0x558af97b1dd0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9794080 0x558af97a75a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9793760 0x558af9793b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3af57fa640 0 client.5829 ms_handle_reset on v2:172.21.15.64:6834/3981471748 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/1879342407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af9764950 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 >> 172.21.15.64:0/1879342407 conn(0x558af9748870 msgr2=0x558af97765c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 shutdown_connections 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/1879342407 wait complete. 2024-03-04T07:28:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.034+0000 7f3b1f5a8740 20 client.5829 trim_cache size 0 max 16384 2024-03-04T07:28:40.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.064+0000 7f3b1f5a8740 1 Processor -- start 2024-03-04T07:28:40.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.064+0000 7f3b1f5a8740 1 -- start start 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af977c390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 0x558af9791990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9794080 0x558af9747740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b181ff1a0 con 0x558af9778600 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b181f7180 con 0x558af9793760 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b181f7780 con 0x558af9794080 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af977c390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af977c390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35080/0 (socket says 172.21.15.64:35080) 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 -- 172.21.15.64:0/3579397865 learned_addr learned my addr 172.21.15.64:0/3579397865 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 -- 172.21.15.64:0/3579397865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9794080 msgr2=0x558af9747740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 --2- 172.21.15.64:0/3579397865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9794080 0x558af9747740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 -- 172.21.15.64:0/3579397865 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 msgr2=0x558af9791990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 --2- 172.21.15.64:0/3579397865 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 0x558af9791990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 -- 172.21.15.64:0/3579397865 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558af9793eb0 con 0x558af9778600 2024-03-04T07:28:40.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.065+0000 7f3b1659c640 1 --2- 172.21.15.64:0/3579397865 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af977c390 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f3b08009560 tx=0x7f3b08047000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=605dee99ec412ff6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3af77fe640 1 -- 172.21.15.64:0/3579397865 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b0805d070 con 0x558af9778600 2024-03-04T07:28:40.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3af77fe640 1 -- 172.21.15.64:0/3579397865 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b08054070 con 0x558af9778600 2024-03-04T07:28:40.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3af77fe640 1 -- 172.21.15.64:0/3579397865 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b0804a040 con 0x558af9778600 2024-03-04T07:28:40.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3579397865 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 msgr2=0x558af977c390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3579397865 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af977c390 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f3b08009560 tx=0x7f3b08047000 comp rx=0 tx=0).stop 2024-03-04T07:28:40.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3579397865 shutdown_connections 2024-03-04T07:28:40.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3579397865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9794080 0x558af9747740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3579397865 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 0x558af9791990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3579397865 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9778600 0x558af977c390 unknown :-1 s=CLOSED pgs=547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3579397865 >> 172.21.15.64:0/3579397865 conn(0x558af9748870 msgr2=0x558af97789e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3579397865 shutdown_connections 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.066+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3579397865 wait complete. 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.067+0000 7f3b1f5a8740 1 -- start start 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.067+0000 7f3b1f5a8740 1 Processor -- start 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9774dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 0x558af97772b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1f5a8740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9794080 0x558af97789e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x558af974cd50 con 0x558af9794080 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x558af966eea0 con 0x558af9793760 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1f5a8740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x558af97a7fb0 con 0x558af9778600 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1f5a8740 10 client.0 _ll_register_callbacks cb 0x7f3b0803aa40 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9794080 0x558af97789e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9794080 0x558af97789e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35086/0 (socket says 172.21.15.64:35086) 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 -- 172.21.15.64:0/3727774997 learned_addr learned my addr 172.21.15.64:0/3727774997 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b15d9b640 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 0x558af97772b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b1659c640 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9774dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 -- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 msgr2=0x558af9774dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9774dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 -- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 msgr2=0x558af97772b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 0x558af97772b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 -- 172.21.15.64:0/3727774997 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558af94916b0 con 0x558af9794080 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3b16d9d640 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9794080 0x558af97789e0 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f3b10007220 tx=0x7f3b10004690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5423dbb8abce49bc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.068+0000 7f3af57fa640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3af57fa640 1 -- 172.21.15.64:0/3727774997 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b1001a020 con 0x558af9794080 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3af57fa640 1 -- 172.21.15.64:0/3727774997 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3b10007ca0 con 0x558af9794080 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3b08054070 con 0x558af9794080 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3af57fa640 1 -- 172.21.15.64:0/3727774997 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=22) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3b10024050 con 0x558af9794080 2024-03-04T07:28:40.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 10 client.5835 fetch_fsmap learned FSMap version 22 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=22,monmap=2+}) v3 -- 0x558af9773490 con 0x558af9794080 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3af57fa640 1 -- 172.21.15.64:0/3727774997 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b08054070 con 0x558af9794080 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3af57fa640 1 -- 172.21.15.64:0/3727774997 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 22) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f3b10027020 con 0x558af9794080 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 10 client.5835 fetch_fsmap finished waiting for FSMap version 22 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 10 client.5835 resolve_mds: resolved name 'a' to daemon mds.a 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 20 client.5835 populate_metadata read hostname 'smithi064' 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af9778f20 0x558af97627c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 4 client.5835 mds_command: new command op to 5688 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 --> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] -- command(tid 0: {"prefix": "status"}) v1 -- 0x558af96b9f40 con 0x558af9778f20 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3b1659c640 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af9778f20 0x558af97627c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.069+0000 7f3af57fa640 10 client.5835 ms_handle_connect on v2:172.21.15.64:6834/3981471748 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1659c640 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af9778f20 0x558af97627c0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8354f72bf10d0754 server_cookie=9fe20b80e33cc5d5 in_seq=0 out_seq=0 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3af57fa640 1 -- 172.21.15.64:0/3727774997 <== mds.0 v2:172.21.15.64:6834/3981471748 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x558af96b9f40 con 0x558af9778f20 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3af57fa640 10 client.5835 handle_command_reply: tid=0 2024-03-04T07:28:40.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 client.5835 shutdown 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9794080 msgr2=0x558af97789e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9794080 0x558af97789e0 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f3b10007220 tx=0x7f3b10004690 comp rx=0 tx=0).stop 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 shutdown_connections 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:6834/3981471748,v1:172.21.15.64:6836/3981471748] conn(0x558af9778f20 0x558af97627c0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x558af9794080 0x558af97789e0 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x558af9793760 0x558af97772b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3af57fa640 0 client.5835 ms_handle_reset on v2:172.21.15.64:6834/3981471748 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 --2- 172.21.15.64:0/3727774997 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x558af9778600 0x558af9774dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 >> 172.21.15.64:0/3727774997 conn(0x558af9748870 msgr2=0x558af9776790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 shutdown_connections 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 1 -- 172.21.15.64:0/3727774997 wait complete. 2024-03-04T07:28:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.070+0000 7f3b1f5a8740 20 client.5835 trim_cache size 0 max 16384 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 5688, 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 4.6895402170000002, 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 22, 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 44, 2024-03-04T07:28:40.077 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 44, 2024-03-04T07:28:40.078 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 9.7231528380000007 2024-03-04T07:28:40.078 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:28:40.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3af806b4d0 msgr2=0x7f3af806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3af806b4d0 0x7f3af806d990 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f3b08036000 tx=0x7f3b08008030 comp rx=0 tx=0).stop 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 msgr2=0x7f3b180b5760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180b5760 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f3b0c00c500 tx=0x7f3b0c00c9c0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 shutdown_connections 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3af806b4d0 0x7f3af806d990 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b180c7ea0 0x7f3b180c8300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b1815df00 0x7f3b180b5760 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 --2- 172.21.15.64:0/928803308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b180b95a0 0x7f3b180cb740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 >> 172.21.15.64:0/928803308 conn(0x7f3b18071960 msgr2=0x7f3b180b76b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 shutdown_connections 2024-03-04T07:28:40.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.071+0000 7f3b1c8ec640 1 -- 172.21.15.64:0/928803308 wait complete. 2024-03-04T07:28:40.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:28:40.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.184+0000 7fe601925640 1 Processor -- start 2024-03-04T07:28:40.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.184+0000 7fe601925640 1 -- start start 2024-03-04T07:28:40.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe601925640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe601925640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe5fc0b4260 0x7fe5fc0b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe601925640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe5fc0d06c0 0x7fe5fc0b1a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe601925640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe5fc0b0b80 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe601925640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe5fc0aeec0 con 0x7fe5fc0b4260 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe601925640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5fc0b0dd0 con 0x7fe5fc0d02e0 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe5fc0d06c0 0x7fe5fc0b1a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe5fc0d06c0 0x7fe5fc0b1a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35094/0 (socket says 172.21.15.64:35094) 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/1779054990 learned_addr learned my addr 172.21.15.64:0/1779054990 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/1779054990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 msgr2=0x7fe5fc0cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fa7fc640 1 --2- 172.21.15.64:0/1779054990 >> v2:172.21.15.66:3300/0 conn(0x7fe5fc0b4260 0x7fe5fc0b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5faffd640 1 --2- 172.21.15.64:0/1779054990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 --2- 172.21.15.64:0/1779054990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc0cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/1779054990 >> v2:172.21.15.66:3300/0 conn(0x7fe5fc0b4260 msgr2=0x7fe5fc0b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 --2- 172.21.15.64:0/1779054990 >> v2:172.21.15.66:3300/0 conn(0x7fe5fc0b4260 0x7fe5fc0b1500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.185+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/1779054990 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5fc0b10b0 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.186+0000 7fe5fa7fc640 1 --2- 172.21.15.64:0/1779054990 >> v2:172.21.15.66:3300/0 conn(0x7fe5fc0b4260 0x7fe5fc0b1500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.186+0000 7fe5faffd640 1 --2- 172.21.15.64:0/1779054990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.186+0000 7fe5fb7fe640 1 --2- 172.21.15.64:0/1779054990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc0b1a40 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fe5fc06eac0 tx=0x7fe5ec02bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=45d234b6bfbdee78 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.186+0000 7fe5f9ffb640 1 -- 172.21.15.64:0/1779054990 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5ec02cb00 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.186+0000 7fe5f9ffb640 1 -- 172.21.15.64:0/1779054990 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5ec02cca0 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.186+0000 7fe5f9ffb640 1 -- 172.21.15.64:0/1779054990 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5ec034610 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 -- 172.21.15.64:0/1779054990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 msgr2=0x7fe5fc0b1a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 --2- 172.21.15.64:0/1779054990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc0b1a40 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fe5fc06eac0 tx=0x7fe5ec02bf60 comp rx=0 tx=0).stop 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 -- 172.21.15.64:0/1779054990 shutdown_connections 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 --2- 172.21.15.64:0/1779054990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc0b1a40 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 --2- 172.21.15.64:0/1779054990 >> v2:172.21.15.66:3300/0 conn(0x7fe5fc0b4260 0x7fe5fc0b1500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 --2- 172.21.15.64:0/1779054990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 -- 172.21.15.64:0/1779054990 >> 172.21.15.64:0/1779054990 conn(0x7fe5fc0b36d0 msgr2=0x7fe5fc0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 -- 172.21.15.64:0/1779054990 shutdown_connections 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.187+0000 7fe601925640 1 -- 172.21.15.64:0/1779054990 wait complete. 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.188+0000 7fe601925640 1 Processor -- start 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe601925640 1 -- start start 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe601925640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5fc0b4260 0x7fe5fc142a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe601925640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc143940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe601925640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc143e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe601925640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5fc0b2100 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe601925640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5fc0b1f80 con 0x7fe5fc0b4260 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe601925640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5fc0b2280 con 0x7fe5fc0d02e0 2024-03-04T07:28:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.189+0000 7fe5fb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc143e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc143e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35096/0 (socket says 172.21.15.64:35096) 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/345959324 learned_addr learned my addr 172.21.15.64:0/345959324 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fa7fc640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc143940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5faffd640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5fc0b4260 0x7fe5fc142a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 msgr2=0x7fe5fc143940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc143940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5fc0b4260 msgr2=0x7fe5fc142a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5fc0b4260 0x7fe5fc142a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 -- 172.21.15.64:0/345959324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5fc1f1ed0 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5faffd640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5fc0b4260 0x7fe5fc142a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5fb7fe640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc143e80 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7fe5ec009f40 tx=0x7fe5ec004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5e3fff640 1 -- 172.21.15.64:0/345959324 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5ec044020 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5e3fff640 1 -- 172.21.15.64:0/345959324 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5ec02ce50 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe5fc1f2170 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe5e3fff640 1 -- 172.21.15.64:0/345959324 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5ec009510 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.190+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5fc1f2730 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.191+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe5fc04f3d0 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.192+0000 7fe5e3fff640 1 -- 172.21.15.64:0/345959324 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe5ec047020 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.192+0000 7fe5e3fff640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e806b4d0 0x7fe5e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.192+0000 7fe5e3fff640 1 -- 172.21.15.64:0/345959324 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe5ec0acb70 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.192+0000 7fe5faffd640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e806b4d0 0x7fe5e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:40.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.192+0000 7fe5faffd640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e806b4d0 0x7fe5e806d990 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fe5f4006fd0 tx=0x7fe5f4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:40.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.200+0000 7fe5e3fff640 1 -- 172.21.15.64:0/345959324 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe5fc04f3d0 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.423+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fe5fc068700 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.423+0000 7fe5e3fff640 1 -- 172.21.15.64:0/345959324 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v22) v1 ==== 53+0+73 (secure 0 0 0) 0x7fe5ec04e030 con 0x7fe5fc0d06c0 2024-03-04T07:28:40.423 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:28:40.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e806b4d0 msgr2=0x7fe5e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e806b4d0 0x7fe5e806d990 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fe5f4006fd0 tx=0x7fe5f4008040 comp rx=0 tx=0).stop 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 msgr2=0x7fe5fc143e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc143e80 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7fe5ec009f40 tx=0x7fe5ec004a40 comp rx=0 tx=0).stop 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 shutdown_connections 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e806b4d0 0x7fe5e806d990 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5fc0d06c0 0x7fe5fc143e80 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5fc0d02e0 0x7fe5fc143940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 --2- 172.21.15.64:0/345959324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5fc0b4260 0x7fe5fc142a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 >> 172.21.15.64:0/345959324 conn(0x7fe5fc0b36d0 msgr2=0x7fe5fc0cd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 shutdown_connections 2024-03-04T07:28:40.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:40.426+0000 7fe601925640 1 -- 172.21.15.64:0/345959324 wait complete. 2024-03-04T07:28:40.436 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:28:40.437 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:28:40.437 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:40.437 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:28:40.452 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:28:40.453 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42592sec preferred_lft 42592sec 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:28:40.454 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:28:40.455 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:28:40.455 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:28:40.455 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:28:40.455 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:28:40.455 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:40.455 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:28:40.518 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:28:40.520 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:40.520 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:28:40.575 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:28:40.575 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:28:40.575 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:28:40.575 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:28:40.576 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:28:40.576 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:40.576 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:28:40.576 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:28:40.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:28:40.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:28:40.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:28:40.576 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:28:40.672 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:40.672 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:28:40.686 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:28:40.686 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:28:40.686 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:28:40.686 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:40.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:28:40.752 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:28:40.752 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:28:40.753 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:40.807 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:28:40.807 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:28:40.807 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:40.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:28:40.928 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:28:40.983 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:28:40.983 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:28:40.984 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:28:40.985 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:28:40.986 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:40.986 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:28:41.037 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:28:41.038 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:28:41.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:28:41.108 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:28:41.164 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:28:41.164 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:28:41.164 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:28:41.165 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:28:41.166 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:28:41.167 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:28:41.167 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:28:41.167 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:28:41.167 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:28:41.167 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:41.167 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:28:41.168 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:28:41.168+0000 7fca45367480 -1 init, newargv = 0x55d8ada015b0 newargc=15 2024-03-04T07:28:41.168 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[56034]: starting ceph client 2024-03-04T07:28:41.172 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[56034]: starting fuse 2024-03-04T07:28:41.183 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:28:41.183 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:28:41.183 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:28:41.184 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:28:41.185 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:41.185 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:28:41.185 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:28:41.304 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.56034.asok 2024-03-04T07:28:41.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:28:41.313 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.56034.asok 2024-03-04T07:28:41.314 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:41.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.56034.asok status 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:28:41.451 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "56034", 2024-03-04T07:28:41.452 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:28:41.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:41.452 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:28:41.452 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:28:41.452 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 5850, 2024-03-04T07:28:41.452 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:28:41.453 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:28:41.453 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:28:41.453 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 5850 2024-03-04T07:28:41.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:41.453 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:28:41.453 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:28:41.454 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:28:41.454 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 326846069 2024-03-04T07:28:41.454 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:28:41.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:41.454 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:28:41.454 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:28:41.455 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:28:41.455 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 326846069 2024-03-04T07:28:41.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:28:41.455 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.5850 192.168.0.1:0/326846069", 2024-03-04T07:28:41.455 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/326846069", 2024-03-04T07:28:41.455 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:28:41.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 22, 2024-03-04T07:28:41.456 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 44, 2024-03-04T07:28:41.456 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:28:41.456 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:28:41.456 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:28:41.457 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:28:41.460 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:41.460 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:28:41.460 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:41.476 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:28:41.476 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:41.476 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:41.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:41.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_fragment_size_max 50 2024-03-04T07:28:41.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.688+0000 7f3ae75f6640 1 Processor -- start 2024-03-04T07:28:41.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.688+0000 7f3ae75f6640 1 -- start start 2024-03-04T07:28:41.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae75f6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3ae015b230 0x7f3ae0071160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae75f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 0x7f3ae0150a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae75f6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3ae00b2e30 con 0x7f3ae015ae50 2024-03-04T07:28:41.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae75f6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3ae0071ae0 con 0x7f3ae015b230 2024-03-04T07:28:41.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae75f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae006dfe0 con 0x7f3ae00716a0 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae5392640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3ae015ae50 0x7f3ae00708e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40936/0 (socket says 172.21.15.64:40936) 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae5392640 1 -- 172.21.15.64:0/2990952905 learned_addr learned my addr 172.21.15.64:0/2990952905 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae4b91640 1 --2- 172.21.15.64:0/2990952905 >> v2:172.21.15.66:3300/0 conn(0x7f3ae015b230 0x7f3ae0071160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 0x7f3ae0150a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.689+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 834740725 0 0) 0x7f3ad4001110 con 0x7f3ae015ae50 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3acc003520 con 0x7f3ae015ae50 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1012754552 0 0) 0x7f3acc003520 con 0x7f3ae015ae50 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 msgr2=0x7f3ae0150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 0x7f3ae0150a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 >> v2:172.21.15.66:3300/0 conn(0x7f3ae015b230 msgr2=0x7f3ae0071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 --2- 172.21.15.64:0/2990952905 >> v2:172.21.15.66:3300/0 conn(0x7f3ae015b230 0x7f3ae0071160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ae00b30b0 con 0x7f3ae015ae50 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f3ad4001840 con 0x7f3ae015ae50 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 >> v1:172.21.15.64:6789/0 conn(0x7f3ae015ae50 legacy=0x7f3ae00708e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 0x7f3acc0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3acc003f30 0x7f3acc004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3acc004d40 0x7f3acc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae0071ae0 con 0x7f3acc004d40 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae006dfe0 con 0x7f3acc003f30 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3acc00ba10 con 0x7f3ae00716a0 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3acc004d40 0x7f3acc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae5392640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 0x7f3acc0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae4b91640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3acc003f30 0x7f3acc004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 msgr2=0x7f3acc0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00716a0 0x7f3acc0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3acc003f30 msgr2=0x7f3acc004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3acc003f30 0x7f3acc004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.690+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2990952905 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3acc003520 con 0x7f3acc004d40 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3acc004d40 0x7f3acc0051c0 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f3ad800e170 tx=0x7f3acc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f753bff5597dabf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad8039070 con 0x7f3acc004d40 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ad802cbf0 con 0x7f3acc004d40 2024-03-04T07:28:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ad3fff640 1 -- 172.21.15.64:0/2990952905 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad8009c10 con 0x7f3acc004d40 2024-03-04T07:28:41.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2990952905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3acc004d40 msgr2=0x7f3acc0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae75f6640 1 --2- 172.21.15.64:0/2990952905 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3acc004d40 0x7f3acc0051c0 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f3ad800e170 tx=0x7f3acc005830 comp rx=0 tx=0).stop 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2990952905 reap_dead start 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2990952905 shutdown_connections 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2990952905 >> 172.21.15.64:0/2990952905 conn(0x7f3ae0066d00 msgr2=0x7f3ae0067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2990952905 shutdown_connections 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.692+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2990952905 wait complete. 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.693+0000 7f3ae75f6640 1 Processor -- start 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae75f6640 1 -- start start 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae75f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3acc004d40 0x7f3ae01f6f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae75f6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae00cb520 0x7f3ae00cb9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae75f6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0153980 0x7f3ae0153e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae75f6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae00b3950 con 0x7f3ae0153980 2024-03-04T07:28:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae75f6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae00b3670 con 0x7f3ae00cb520 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae75f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae00b4150 con 0x7f3acc004d40 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae5b93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0153980 0x7f3ae0153e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae5b93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0153980 0x7f3ae0153e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35106/0 (socket says 172.21.15.64:35106) 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.694+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2450358449 learned_addr learned my addr 172.21.15.64:0/2450358449 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae4b91640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae00cb520 0x7f3ae00cb9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae5392640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3acc004d40 0x7f3ae01f6f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3acc004d40 msgr2=0x7f3ae01f6f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3acc004d40 0x7f3ae01f6f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae00cb520 msgr2=0x7f3ae00cb9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae00cb520 0x7f3ae00cb9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae5b93640 1 -- 172.21.15.64:0/2450358449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ae00cc280 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae5b93640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0153980 0x7f3ae0153e00 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f3ad800e170 tx=0x7f3ad8009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ad1ffb640 1 -- 172.21.15.64:0/2450358449 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad8045070 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ad1ffb640 1 -- 172.21.15.64:0/2450358449 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ad803a040 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3ae00cbfa0 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ad1ffb640 1 -- 172.21.15.64:0/2450358449 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad8039070 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.695+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3ae0153180 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.696+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3ae00733c0 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.697+0000 7f3ad1ffb640 1 -- 172.21.15.64:0/2450358449 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3ad8048070 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.697+0000 7f3ad1ffb640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ac406b470 0x7f3ac406d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.698+0000 7f3ae5392640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ac406b470 0x7f3ac406d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.698+0000 7f3ad1ffb640 1 -- 172.21.15.64:0/2450358449 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3ad80ab850 con 0x7f3ae0153980 2024-03-04T07:28:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.698+0000 7f3ae5392640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ac406b470 0x7f3ac406d930 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f3ad40012b0 tx=0x7f3ad4007ac0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:41.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.705+0000 7f3ad1ffb640 1 -- 172.21.15.64:0/2450358449 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3ad807a1d0 con 0x7f3ae0153980 2024-03-04T07:28:41.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.886+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mds_bal_fragment_size_max}] v 0) v1 -- 0x7f3ae006cdb0 con 0x7f3ae0153980 2024-03-04T07:28:41.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.890+0000 7f3ad1ffb640 1 -- 172.21.15.64:0/2450358449 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_fragment_size_max}]=0 v13)=0 v13) v1 ==== 124+0+0 (secure 0 0 0) 0x7f3ad807e270 con 0x7f3ae0153980 2024-03-04T07:28:41.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ac406b470 msgr2=0x7f3ac406d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:41.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ac406b470 0x7f3ac406d930 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f3ad40012b0 tx=0x7f3ad4007ac0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0153980 msgr2=0x7f3ae0153e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0153980 0x7f3ae0153e00 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f3ad800e170 tx=0x7f3ad8009750 comp rx=0 tx=0).stop 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 shutdown_connections 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ac406b470 0x7f3ac406d930 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0153980 0x7f3ae0153e00 unknown :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae00cb520 0x7f3ae00cb9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 --2- 172.21.15.64:0/2450358449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3acc004d40 0x7f3ae01f6f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 >> 172.21.15.64:0/2450358449 conn(0x7f3ae0066d00 msgr2=0x7f3ae015c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 shutdown_connections 2024-03-04T07:28:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:41.892+0000 7f3ae75f6640 1 -- 172.21.15.64:0/2450358449 wait complete. 2024-03-04T07:28:47.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:28:47.465 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:28:51.904 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:28:51.904 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:51.919 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> import uuid 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> n = 51 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = False 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> if 0 > 0: 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = True 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> dpath = os.path.dirname(path) 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> fnameprefix = os.path.basename(path) 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> os.makedirs(dpath, exist_ok=True) 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> try: 2024-03-04T07:28:51.920 DEBUG:teuthology.orchestra.run.smithi064:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> for i in range(n): 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> with open(fpath, '"'"'w'"'"') as f: 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> f.write(f"{i}") 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> f.flush() 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(f.fileno()) 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> os.unlink(fpath) 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> if create_hard_links: 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> for j in range(0): 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> if True: 2024-03-04T07:28:51.921 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:28:51.922 DEBUG:teuthology.orchestra.run.smithi064:> finally: 2024-03-04T07:28:51.922 DEBUG:teuthology.orchestra.run.smithi064:> os.close(dirfd) 2024-03-04T07:28:51.922 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:28:52.117 INFO:teuthology.orchestra.run.smithi064.stderr:Traceback (most recent call last): 2024-03-04T07:28:52.117 INFO:teuthology.orchestra.run.smithi064.stderr: File "", line 20, in 2024-03-04T07:28:52.117 INFO:teuthology.orchestra.run.smithi064.stderr:OSError: [Errno 28] No space left on device: '/home/ubuntu/cephtest/mnt.0/subdir/file_50' 2024-03-04T07:28:52.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52 stdin-killer INFO: command exited with status 1: exiting normally with same code! 2024-03-04T07:28:52.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:52.127 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:28:52.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:28:52.220 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:28:52.266 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:28:52.266 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:28:52.266 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:52.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:52.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.423+0000 7fafb02b5640 1 Processor -- start 2024-03-04T07:28:52.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.423+0000 7fafb02b5640 1 -- start start 2024-03-04T07:28:52.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafb02b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fafa80b4260 0x7fafa80cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafb02b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fafa80d06c0 0x7fafa80b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafb02b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fafa80b0dd0 con 0x7fafa80d02e0 2024-03-04T07:28:52.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafb02b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fafa80b0b80 con 0x7fafa80b4260 2024-03-04T07:28:52.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafb02b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fafa80aeec0 con 0x7fafa80d06c0 2024-03-04T07:28:52.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafae051640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fafa80d02e0 0x7fafa80b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59018/0 (socket says 172.21.15.64:59018) 2024-03-04T07:28:52.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafae051640 1 -- 172.21.15.64:0/1499239003 learned_addr learned my addr 172.21.15.64:0/1499239003 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafad850640 1 --2- 172.21.15.64:0/1499239003 >> v2:172.21.15.64:3300/0 conn(0x7fafa80b4260 0x7fafa80cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafae852640 1 --2- 172.21.15.64:0/1499239003 >> v2:172.21.15.66:3300/0 conn(0x7fafa80d06c0 0x7fafa80b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafad04f640 1 -- 172.21.15.64:0/1499239003 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2109480948 0 0) 0x7fafa80b0dd0 con 0x7fafa80d02e0 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafad04f640 1 -- 172.21.15.64:0/1499239003 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7faf94003520 con 0x7fafa80d02e0 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafad850640 1 -- 172.21.15.64:0/1499239003 >> v2:172.21.15.66:3300/0 conn(0x7fafa80d06c0 msgr2=0x7fafa80b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafad850640 1 --2- 172.21.15.64:0/1499239003 >> v2:172.21.15.66:3300/0 conn(0x7fafa80d06c0 0x7fafa80b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafad850640 1 -- 172.21.15.64:0/1499239003 >> v1:172.21.15.64:6789/0 conn(0x7fafa80d02e0 legacy=0x7fafa80b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafad850640 1 -- 172.21.15.64:0/1499239003 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fafa80b10b0 con 0x7fafa80b4260 2024-03-04T07:28:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.425+0000 7fafae852640 1 --2- 172.21.15.64:0/1499239003 >> v2:172.21.15.66:3300/0 conn(0x7fafa80d06c0 0x7fafa80b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.426+0000 7fafad850640 1 --2- 172.21.15.64:0/1499239003 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80b4260 0x7fafa80cfa00 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7fafa80cc0e0 tx=0x7faf9802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c24442a1160527 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.426+0000 7fafad04f640 1 -- 172.21.15.64:0/1499239003 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf98034070 con 0x7fafa80b4260 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.426+0000 7fafad04f640 1 -- 172.21.15.64:0/1499239003 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faf98030610 con 0x7fafa80b4260 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.426+0000 7fafad04f640 1 -- 172.21.15.64:0/1499239003 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf980309e0 con 0x7fafa80b4260 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 -- 172.21.15.64:0/1499239003 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80b4260 msgr2=0x7fafa80cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 --2- 172.21.15.64:0/1499239003 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80b4260 0x7fafa80cfa00 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7fafa80cc0e0 tx=0x7faf9802b960 comp rx=0 tx=0).stop 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 -- 172.21.15.64:0/1499239003 shutdown_connections 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 --2- 172.21.15.64:0/1499239003 >> v2:172.21.15.66:3300/0 conn(0x7fafa80d06c0 0x7fafa80b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 --2- 172.21.15.64:0/1499239003 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80b4260 0x7fafa80cfa00 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 -- 172.21.15.64:0/1499239003 >> 172.21.15.64:0/1499239003 conn(0x7fafa80b36d0 msgr2=0x7fafa80b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 -- 172.21.15.64:0/1499239003 shutdown_connections 2024-03-04T07:28:52.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.427+0000 7fafb02b5640 1 -- 172.21.15.64:0/1499239003 wait complete. 2024-03-04T07:28:52.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.428+0000 7fafb02b5640 1 Processor -- start 2024-03-04T07:28:52.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.428+0000 7fafb02b5640 1 -- start start 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafb02b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fafa80b4260 0x7fafa8153810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafb02b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80d02e0 0x7fafa81572e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafb02b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fafa80d06c0 0x7fafa81f1290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafb02b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafa80b22d0 con 0x7fafa80d02e0 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafb02b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafa80b2150 con 0x7fafa80b4260 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafb02b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fafa80af350 con 0x7fafa80d06c0 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80d02e0 0x7fafa81572e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80d02e0 0x7fafa81572e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53644/0 (socket says 172.21.15.64:53644) 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 -- 172.21.15.64:0/3382491956 learned_addr learned my addr 172.21.15.64:0/3382491956 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafae051640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fafa80b4260 0x7fafa8153810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafae852640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fafa80d06c0 0x7fafa81f1290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 -- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fafa80d06c0 msgr2=0x7fafa81f1290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fafa80d06c0 0x7fafa81f1290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 -- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fafa80b4260 msgr2=0x7fafa8153810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fafa80b4260 0x7fafa8153810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 -- 172.21.15.64:0/3382491956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fafa8157820 con 0x7fafa80d02e0 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafae051640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fafa80b4260 0x7fafa8153810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7fafad850640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80d02e0 0x7fafa81572e0 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7faf98000c00 tx=0x7faf980046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.429+0000 7faf9effd640 1 -- 172.21.15.64:0/3382491956 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf98034040 con 0x7fafa80d02e0 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.430+0000 7faf9effd640 1 -- 172.21.15.64:0/3382491956 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faf98030410 con 0x7fafa80d02e0 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.430+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fafa8153f90 con 0x7fafa80d02e0 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.430+0000 7faf9effd640 1 -- 172.21.15.64:0/3382491956 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf98039c50 con 0x7fafa80d02e0 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.430+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fafa815d470 con 0x7fafa80d02e0 2024-03-04T07:28:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.430+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fafa8068700 con 0x7fafa80d02e0 2024-03-04T07:28:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.431+0000 7faf9effd640 1 -- 172.21.15.64:0/3382491956 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faf98037070 con 0x7fafa80d02e0 2024-03-04T07:28:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.432+0000 7faf9effd640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faf8c06b4d0 0x7faf8c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.432+0000 7faf9effd640 1 -- 172.21.15.64:0/3382491956 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7faf980ac160 con 0x7fafa80d02e0 2024-03-04T07:28:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.432+0000 7fafae051640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faf8c06b4d0 0x7faf8c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.432+0000 7fafae051640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faf8c06b4d0 0x7faf8c06d990 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fafa4005880 tx=0x7fafa4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:52.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.436+0000 7faf9effd640 1 -- 172.21.15.64:0/3382491956 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faf9807b070 con 0x7fafa80d02e0 2024-03-04T07:28:52.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.683+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fafa815bec0 con 0x7fafa80d02e0 2024-03-04T07:28:52.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.684+0000 7faf9effd640 1 -- 172.21.15.64:0/3382491956 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v44) v1 ==== 74+0+16565 (secure 0 0 0) 0x7faf9807a970 con 0x7fafa80d02e0 2024-03-04T07:28:52.685 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:52.685 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":44,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:36.369097+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:28:32.989901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-04T07:28:33.692080+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"44","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:52.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faf8c06b4d0 msgr2=0x7faf8c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faf8c06b4d0 0x7faf8c06d990 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fafa4005880 tx=0x7fafa4009040 comp rx=0 tx=0).stop 2024-03-04T07:28:52.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80d02e0 msgr2=0x7fafa81572e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80d02e0 0x7fafa81572e0 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7faf98000c00 tx=0x7faf980046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 shutdown_connections 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faf8c06b4d0 0x7faf8c06d990 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fafa80d06c0 0x7fafa81f1290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fafa80d02e0 0x7fafa81572e0 unknown :-1 s=CLOSED pgs=559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 --2- 172.21.15.64:0/3382491956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fafa80b4260 0x7fafa8153810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 >> 172.21.15.64:0/3382491956 conn(0x7fafa80b36d0 msgr2=0x7fafa8151c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.686+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 shutdown_connections 2024-03-04T07:28:52.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.687+0000 7fafb02b5640 1 -- 172.21.15.64:0/3382491956 wait complete. 2024-03-04T07:28:52.697 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 6, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:28:32.989901+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '43', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 7, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:28:33.692080+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '44', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:28:52.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:28:52.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.802+0000 7fd1abdf4640 1 Processor -- start 2024-03-04T07:28:52.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.803+0000 7fd1abdf4640 1 -- start start 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.803+0000 7fd1abdf4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1a40cea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.803+0000 7fd1abdf4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1a40cd310 con 0x7fd1a40c5e30 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.803+0000 7fd1abdf4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1a40cd5b0 con 0x7fd1a415f500 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.803+0000 7fd1abdf4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1a415f8e0 con 0x7fd1a40cf740 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1aa391640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd1a40c5e30 0x7fd1a40c6280 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59034/0 (socket says 172.21.15.64:59034) 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1aa391640 1 -- 172.21.15.64:0/1209185084 learned_addr learned my addr 172.21.15.64:0/1209185084 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a938f640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1a40cea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2185243407 0 0) 0x7fd1a40cd310 con 0x7fd1a40c5e30 2024-03-04T07:28:52.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd190003520 con 0x7fd1a40c5e30 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3421904568 0 0) 0x7fd1a40cd5b0 con 0x7fd1a415f500 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd1a40cd310 con 0x7fd1a415f500 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2706157005 0 0) 0x7fd190003520 con 0x7fd1a40c5e30 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 msgr2=0x7fd1a40cea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1a40cea10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 >> v1:172.21.15.66:6789/0 conn(0x7fd1a415f500 legacy=0x7fd1a40ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1a40cd890 con 0x7fd1a40c5e30 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd19c001a10 con 0x7fd1a40c5e30 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a938f640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1a40cea10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 >> v1:172.21.15.64:6789/0 conn(0x7fd1a40c5e30 legacy=0x7fd1a40c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190003b70 0x7fd190003f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd190004800 0x7fd190004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1900051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1a415f8e0 con 0x7fd190004800 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd19000bad0 con 0x7fd190003b70 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd19000bcd0 con 0x7fd1a40cf740 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a938f640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd190004800 0x7fd190004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.804+0000 7fd1a9b90640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190003b70 0x7fd190003f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1aa391640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1900051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1a938f640 1 -- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 msgr2=0x7fd1900051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1a938f640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1900051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1a938f640 1 -- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190003b70 msgr2=0x7fd190003f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1a938f640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190003b70 0x7fd190003f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1a938f640 1 -- 172.21.15.64:0/1209185084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd190003520 con 0x7fd190004800 2024-03-04T07:28:52.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1aa391640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a40cf740 0x7fd1900051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.805+0000 7fd1a938f640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd190004800 0x7fd190004c80 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7fd1a40cfba0 tx=0x7fd190005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1211acafc47a4f10 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd194039070 con 0x7fd190004800 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd19402c890 con 0x7fd190004800 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1a8b8e640 1 -- 172.21.15.64:0/1209185084 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd194009860 con 0x7fd190004800 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1abdf4640 1 -- 172.21.15.64:0/1209185084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd190004800 msgr2=0x7fd190004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1abdf4640 1 --2- 172.21.15.64:0/1209185084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd190004800 0x7fd190004c80 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7fd1a40cfba0 tx=0x7fd190005830 comp rx=0 tx=0).stop 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1aa391640 1 -- 172.21.15.64:0/1209185084 reap_dead start 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1abdf4640 1 -- 172.21.15.64:0/1209185084 shutdown_connections 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1abdf4640 1 -- 172.21.15.64:0/1209185084 >> 172.21.15.64:0/1209185084 conn(0x7fd1a40b3700 msgr2=0x7fd1a40b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1abdf4640 1 -- 172.21.15.64:0/1209185084 shutdown_connections 2024-03-04T07:28:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.806+0000 7fd1abdf4640 1 -- 172.21.15.64:0/1209185084 wait complete. 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.807+0000 7fd1abdf4640 1 Processor -- start 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.807+0000 7fd1abdf4640 1 -- start start 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1abdf4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190004800 0x7fd1a4151990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1abdf4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1a40b5ff0 0x7fd1a40b6470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1abdf4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a4156400 0x7fd1a4156880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1abdf4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1a40c59a0 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1abdf4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1a40c56c0 con 0x7fd190004800 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1abdf4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1a40cea10 con 0x7fd1a4156400 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1a938f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1a40b5ff0 0x7fd1a40b6470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1a938f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1a40b5ff0 0x7fd1a40b6470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53662/0 (socket says 172.21.15.64:53662) 2024-03-04T07:28:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1a938f640 1 -- 172.21.15.64:0/978428739 learned_addr learned my addr 172.21.15.64:0/978428739 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1a9b90640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190004800 0x7fd1a4151990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1aa391640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a4156400 0x7fd1a4156880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.808+0000 7fd1a938f640 1 -- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a4156400 msgr2=0x7fd1a4156880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1a938f640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a4156400 0x7fd1a4156880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1a938f640 1 -- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190004800 msgr2=0x7fd1a4151990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1a938f640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190004800 0x7fd1a4151990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1a938f640 1 -- 172.21.15.64:0/978428739 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1a4153060 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1a9b90640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190004800 0x7fd1a4151990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1a938f640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1a40b5ff0 0x7fd1a40b6470 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7fd194009d20 tx=0x7fd194004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd19a7fc640 1 -- 172.21.15.64:0/978428739 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd194042020 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd19a7fc640 1 -- 172.21.15.64:0/978428739 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd19403a040 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd1a4153340 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd19a7fc640 1 -- 172.21.15.64:0/978428739 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd194039070 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.809+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd1a4153900 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.810+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd1a409fcf0 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.810+0000 7fd19a7fc640 1 -- 172.21.15.64:0/978428739 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd194045020 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.811+0000 7fd19a7fc640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd18806b500 0x7fd18806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.811+0000 7fd1a9b90640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd18806b500 0x7fd18806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.811+0000 7fd19a7fc640 1 -- 172.21.15.64:0/978428739 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd1940abd40 con 0x7fd1a40b5ff0 2024-03-04T07:28:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.811+0000 7fd1a9b90640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd18806b500 0x7fd18806d9c0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fd1a00035f0 tx=0x7fd1a0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:52.815+0000 7fd19a7fc640 1 -- 172.21.15.64:0/978428739 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd19404c020 con 0x7fd1a40b5ff0 2024-03-04T07:28:53.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.010+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd1a404f410 con 0x7fd1a40b5ff0 2024-03-04T07:28:53.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.010+0000 7fd19a7fc640 1 -- 172.21.15.64:0/978428739 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v44) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd19407ab30 con 0x7fd1a40b5ff0 2024-03-04T07:28:53.010 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd18806b500 msgr2=0x7fd18806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd18806b500 0x7fd18806d9c0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fd1a00035f0 tx=0x7fd1a0009040 comp rx=0 tx=0).stop 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1a40b5ff0 msgr2=0x7fd1a40b6470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1a40b5ff0 0x7fd1a40b6470 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7fd194009d20 tx=0x7fd194004730 comp rx=0 tx=0).stop 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 shutdown_connections 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd18806b500 0x7fd18806d9c0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1a4156400 0x7fd1a4156880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1a40b5ff0 0x7fd1a40b6470 unknown :-1 s=CLOSED pgs=562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 --2- 172.21.15.64:0/978428739 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd190004800 0x7fd1a4151990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 >> 172.21.15.64:0/978428739 conn(0x7fd1a40b3700 msgr2=0x7fd1a406ff90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 shutdown_connections 2024-03-04T07:28:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.012+0000 7fd1abdf4640 1 -- 172.21.15.64:0/978428739 wait complete. 2024-03-04T07:28:53.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:28:53.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.128+0000 7f3bfb457640 1 Processor -- start 2024-03-04T07:28:53.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.128+0000 7f3bfb457640 1 -- start start 2024-03-04T07:28:53.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bfb457640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf4071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bfb457640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3bf40b2d30 0x7f3bf40ce770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bfb457640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3bf4071e60 con 0x7f3bf40cecb0 2024-03-04T07:28:53.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bfb457640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3bf40d3230 con 0x7f3bf40b2d30 2024-03-04T07:28:53.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bfb457640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3bf40d4960 con 0x7f3bf4071540 2024-03-04T07:28:53.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bf99f4640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3bf40cecb0 0x7f3bf40b72e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59044/0 (socket says 172.21.15.64:59044) 2024-03-04T07:28:53.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bf99f4640 1 -- 172.21.15.64:0/558709116 learned_addr learned my addr 172.21.15.64:0/558709116 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bf89f2640 1 --2- 172.21.15.64:0/558709116 >> v2:172.21.15.66:3300/0 conn(0x7f3bf40b2d30 0x7f3bf40ce770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bf91f3640 1 --2- 172.21.15.64:0/558709116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf4071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bebfff640 1 -- 172.21.15.64:0/558709116 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 742655855 0 0) 0x7f3bf4071e60 con 0x7f3bf40cecb0 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.129+0000 7f3bebfff640 1 -- 172.21.15.64:0/558709116 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3be0003520 con 0x7f3bf40cecb0 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.130+0000 7f3bf91f3640 1 -- 172.21.15.64:0/558709116 >> v2:172.21.15.66:3300/0 conn(0x7f3bf40b2d30 msgr2=0x7f3bf40ce770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.130+0000 7f3bf91f3640 1 --2- 172.21.15.64:0/558709116 >> v2:172.21.15.66:3300/0 conn(0x7f3bf40b2d30 0x7f3bf40ce770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.130+0000 7f3bebfff640 1 -- 172.21.15.64:0/558709116 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 166860849 0 0) 0x7f3be0003520 con 0x7f3bf40cecb0 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.130+0000 7f3bf91f3640 1 -- 172.21.15.64:0/558709116 >> v1:172.21.15.64:6789/0 conn(0x7f3bf40cecb0 legacy=0x7f3bf40b72e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.130+0000 7f3bf91f3640 1 -- 172.21.15.64:0/558709116 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3bf40d36c0 con 0x7f3bf4071540 2024-03-04T07:28:53.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.130+0000 7f3bf89f2640 1 --2- 172.21.15.64:0/558709116 >> v2:172.21.15.66:3300/0 conn(0x7f3bf40b2d30 0x7f3bf40ce770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:28:53.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.130+0000 7f3bf91f3640 1 --2- 172.21.15.64:0/558709116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf4071920 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f3bf00097a0 tx=0x7f3bf002b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8ba1a982167ae059 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bebfff640 1 -- 172.21.15.64:0/558709116 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bf0037070 con 0x7f3bf4071540 2024-03-04T07:28:53.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bebfff640 1 -- 172.21.15.64:0/558709116 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3bf0030510 con 0x7f3bf4071540 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bebfff640 1 -- 172.21.15.64:0/558709116 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bec0019f0 con 0x7f3bf4071540 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bfb457640 1 -- 172.21.15.64:0/558709116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 msgr2=0x7f3bf4071920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bfb457640 1 --2- 172.21.15.64:0/558709116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf4071920 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f3bf00097a0 tx=0x7f3bf002b790 comp rx=0 tx=0).stop 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bfb457640 1 -- 172.21.15.64:0/558709116 shutdown_connections 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bfb457640 1 --2- 172.21.15.64:0/558709116 >> v2:172.21.15.66:3300/0 conn(0x7f3bf40b2d30 0x7f3bf40ce770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bfb457640 1 --2- 172.21.15.64:0/558709116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf4071920 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.131+0000 7f3bfb457640 1 -- 172.21.15.64:0/558709116 >> 172.21.15.64:0/558709116 conn(0x7f3bf40d3d60 msgr2=0x7f3bf40d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.132+0000 7f3bfb457640 1 -- 172.21.15.64:0/558709116 shutdown_connections 2024-03-04T07:28:53.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.132+0000 7f3bfb457640 1 -- 172.21.15.64:0/558709116 wait complete. 2024-03-04T07:28:53.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.133+0000 7f3bfb457640 1 Processor -- start 2024-03-04T07:28:53.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.133+0000 7f3bfb457640 1 -- start start 2024-03-04T07:28:53.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bfb457640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf40cd870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bfb457640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3bf40b2d30 0x7f3bf41f3630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bfb457640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3bf40cecb0 0x7f3bf40c8710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bfb457640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bf40d2be0 con 0x7f3bf40b2d30 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bfb457640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bf40d2900 con 0x7f3bf40cecb0 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bfb457640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3bf40d2ec0 con 0x7f3bf4071540 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bf89f2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3bf40b2d30 0x7f3bf41f3630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bf89f2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3bf40b2d30 0x7f3bf41f3630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53676/0 (socket says 172.21.15.64:53676) 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.134+0000 7f3bf89f2640 1 -- 172.21.15.64:0/3930873443 learned_addr learned my addr 172.21.15.64:0/3930873443 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf99f4640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3bf40cecb0 0x7f3bf40c8710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf91f3640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf40cd870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf89f2640 1 -- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 msgr2=0x7f3bf40cd870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf89f2640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf40cd870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf89f2640 1 -- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3bf40cecb0 msgr2=0x7f3bf40c8710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf89f2640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3bf40cecb0 0x7f3bf40c8710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf89f2640 1 -- 172.21.15.64:0/3930873443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3bf4156eb0 con 0x7f3bf40b2d30 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf99f4640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3bf40cecb0 0x7f3bf40c8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:53.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bf89f2640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3bf40b2d30 0x7f3bf41f3630 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f3be400ebe0 tx=0x7f3be400c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3be9ffb640 1 -- 172.21.15.64:0/3930873443 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3be4017070 con 0x7f3bf40b2d30 2024-03-04T07:28:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3be9ffb640 1 -- 172.21.15.64:0/3930873443 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3be400cc30 con 0x7f3bf40b2d30 2024-03-04T07:28:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3bf40cddb0 con 0x7f3bf40b2d30 2024-03-04T07:28:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3be9ffb640 1 -- 172.21.15.64:0/3930873443 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3be4013470 con 0x7f3bf40b2d30 2024-03-04T07:28:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.135+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3bf41568f0 con 0x7f3bf40b2d30 2024-03-04T07:28:53.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.136+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3bf40cb1e0 con 0x7f3bf40b2d30 2024-03-04T07:28:53.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.137+0000 7f3be9ffb640 1 -- 172.21.15.64:0/3930873443 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3be401a070 con 0x7f3bf40b2d30 2024-03-04T07:28:53.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.137+0000 7f3be9ffb640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bd806b4d0 0x7f3bd806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.137+0000 7f3be9ffb640 1 -- 172.21.15.64:0/3930873443 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3be408a080 con 0x7f3bf40b2d30 2024-03-04T07:28:53.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.137+0000 7f3bf91f3640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bd806b4d0 0x7f3bd806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.138+0000 7f3bf91f3640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bd806b4d0 0x7f3bd806d990 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f3bf0034000 tx=0x7f3bf002f000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.141+0000 7f3be9ffb640 1 -- 172.21.15.64:0/3930873443 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3be4058f90 con 0x7f3bf40b2d30 2024-03-04T07:28:53.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.322+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f3bf404f410 con 0x7f3bf40b2d30 2024-03-04T07:28:53.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.322+0000 7f3be9ffb640 1 -- 172.21.15.64:0/3930873443 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v44) v1 ==== 104+0+11 (secure 0 0 0) 0x7f3be4058890 con 0x7f3bf40b2d30 2024-03-04T07:28:53.322 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:28:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bd806b4d0 msgr2=0x7f3bd806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bd806b4d0 0x7f3bd806d990 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f3bf0034000 tx=0x7f3bf002f000 comp rx=0 tx=0).stop 2024-03-04T07:28:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3bf40b2d30 msgr2=0x7f3bf41f3630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3bf40b2d30 0x7f3bf41f3630 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f3be400ebe0 tx=0x7f3be400c520 comp rx=0 tx=0).stop 2024-03-04T07:28:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 shutdown_connections 2024-03-04T07:28:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bd806b4d0 0x7f3bd806d990 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3bf40cecb0 0x7f3bf40c8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3bf40b2d30 0x7f3bf41f3630 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 --2- 172.21.15.64:0/3930873443 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3bf4071540 0x7f3bf40cd870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 >> 172.21.15.64:0/3930873443 conn(0x7f3bf40d3d60 msgr2=0x7f3bf40d1910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:53.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 shutdown_connections 2024-03-04T07:28:53.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.324+0000 7f3bfb457640 1 -- 172.21.15.64:0/3930873443 wait complete. 2024-03-04T07:28:53.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:28:53.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.442+0000 7f09eab16640 1 Processor -- start 2024-03-04T07:28:53.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.442+0000 7f09eab16640 1 -- start start 2024-03-04T07:28:53.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09eab16640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 0x7f09e40b4220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09eab16640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f09e415e4b0 con 0x7f09e40cdc20 2024-03-04T07:28:53.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09eab16640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f09e40cefc0 con 0x7f09e415e0d0 2024-03-04T07:28:53.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09eab16640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09e40cc1c0 con 0x7f09e40ceb00 2024-03-04T07:28:53.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e90b3640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f09e40cdc20 0x7f09e40cfd00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43556/0 (socket says 172.21.15.64:43556) 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e90b3640 1 -- 172.21.15.64:0/717826351 learned_addr learned my addr 172.21.15.64:0/717826351 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2468775750 0 0) 0x7f09e415e4b0 con 0x7f09e40cdc20 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f09d4003520 con 0x7f09e40cdc20 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e3fff640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 0x7f09e40b4220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2583078190 0 0) 0x7f09d4003520 con 0x7f09e40cdc20 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 msgr2=0x7f09e40b4220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 0x7f09e40b4220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 >> v1:172.21.15.66:6789/0 conn(0x7f09e415e0d0 legacy=0x7f09e40ce3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09e40cf2a0 con 0x7f09e40cdc20 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f09dc0019f0 con 0x7f09e40cdc20 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 >> v1:172.21.15.64:6789/0 conn(0x7f09e40cdc20 legacy=0x7f09e40cfd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09d40039e0 0x7f09d4003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09d4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 0x7f09d4005200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09e40cc1c0 con 0x7f09d4004800 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09d400bb20 con 0x7f09d40039e0 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.443+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09d400bd20 con 0x7f09e40ceb00 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e3fff640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09d4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e90b3640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 0x7f09d4005200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e88b2640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09d40039e0 0x7f09d4003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e3fff640 1 -- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 msgr2=0x7f09d4005200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e3fff640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40ceb00 0x7f09d4005200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e3fff640 1 -- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09d40039e0 msgr2=0x7f09d4003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e3fff640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09d40039e0 0x7f09d4003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.444+0000 7f09e3fff640 1 -- 172.21.15.64:0/717826351 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09d4003520 con 0x7f09d4004800 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09e3fff640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09d4004c80 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7f09d0009ac0 tx=0x7f09d4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=81fe4670318dfd42 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d0039070 con 0x7f09d4004800 2024-03-04T07:28:53.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f09d002c6b0 con 0x7f09d4004800 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09e37fe640 1 -- 172.21.15.64:0/717826351 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d0009690 con 0x7f09d4004800 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09eab16640 1 -- 172.21.15.64:0/717826351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 msgr2=0x7f09d4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09eab16640 1 --2- 172.21.15.64:0/717826351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09d4004c80 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7f09d0009ac0 tx=0x7f09d4005830 comp rx=0 tx=0).stop 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09e90b3640 1 -- 172.21.15.64:0/717826351 reap_dead start 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09eab16640 1 -- 172.21.15.64:0/717826351 shutdown_connections 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09eab16640 1 -- 172.21.15.64:0/717826351 >> 172.21.15.64:0/717826351 conn(0x7f09e40b3700 msgr2=0x7f09e40b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09eab16640 1 -- 172.21.15.64:0/717826351 shutdown_connections 2024-03-04T07:28:53.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.445+0000 7f09eab16640 1 -- 172.21.15.64:0/717826351 wait complete. 2024-03-04T07:28:53.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.446+0000 7f09eab16640 1 Processor -- start 2024-03-04T07:28:53.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09eab16640 1 -- start start 2024-03-04T07:28:53.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09eab16640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09e40c8c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09eab16640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40c6610 0x7f09e4149f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09eab16640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40c7ec0 0x7f09e40d1510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09eab16640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09e406b3d0 con 0x7f09d4004800 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09eab16640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09e40cc6f0 con 0x7f09e40c6610 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09eab16640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09e40b4220 con 0x7f09e40c7ec0 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09e88b2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09e40c8c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09e88b2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09e40c8c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36906/0 (socket says 172.21.15.64:36906) 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.447+0000 7f09e88b2640 1 -- 172.21.15.64:0/1441792161 learned_addr learned my addr 172.21.15.64:0/1441792161 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e3fff640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40c6610 0x7f09e4149f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e90b3640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40c7ec0 0x7f09e40d1510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e88b2640 1 -- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40c7ec0 msgr2=0x7f09e40d1510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e88b2640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40c7ec0 0x7f09e40d1510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e88b2640 1 -- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40c6610 msgr2=0x7f09e4149f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e88b2640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40c6610 0x7f09e4149f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e88b2640 1 -- 172.21.15.64:0/1441792161 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09e414a4d0 con 0x7f09d4004800 2024-03-04T07:28:53.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e88b2640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09e40c8c40 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f09d800e500 tx=0x7f09d800e900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e17fa640 1 -- 172.21.15.64:0/1441792161 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d80052e0 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e17fa640 1 -- 172.21.15.64:0/1441792161 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f09d8018070 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09e17fa640 1 -- 172.21.15.64:0/1441792161 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d8003810 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f09e40c7100 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.448+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f09e40c76c0 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.449+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f09e404f410 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.450+0000 7f09e17fa640 1 -- 172.21.15.64:0/1441792161 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f09d80039b0 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.450+0000 7f09e17fa640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09c806b4d0 0x7f09c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.450+0000 7f09e17fa640 1 -- 172.21.15.64:0/1441792161 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f09d8089d10 con 0x7f09d4004800 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.450+0000 7f09e3fff640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09c806b4d0 0x7f09c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.451+0000 7f09e3fff640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09c806b4d0 0x7f09c806d990 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f09d0036000 tx=0x7f09d0030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.454+0000 7f09e17fa640 1 -- 172.21.15.64:0/1441792161 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f09e414a4d0 con 0x7f09d4004800 2024-03-04T07:28:53.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.636+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f09e40b3fe0 con 0x7f09d4004800 2024-03-04T07:28:53.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.636+0000 7f09e17fa640 1 -- 172.21.15.64:0/1441792161 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v44) v1 ==== 100+0+11 (secure 0 0 0) 0x7f09d8058b90 con 0x7f09d4004800 2024-03-04T07:28:53.636 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09c806b4d0 msgr2=0x7f09c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09c806b4d0 0x7f09c806d990 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f09d0036000 tx=0x7f09d0030000 comp rx=0 tx=0).stop 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 msgr2=0x7f09e40c8c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09e40c8c40 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f09d800e500 tx=0x7f09d800e900 comp rx=0 tx=0).stop 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 shutdown_connections 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09c806b4d0 0x7f09c806d990 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e40c7ec0 0x7f09e40d1510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40c6610 0x7f09e4149f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 --2- 172.21.15.64:0/1441792161 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09d4004800 0x7f09e40c8c40 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.638+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 >> 172.21.15.64:0/1441792161 conn(0x7f09e40b3700 msgr2=0x7f09e40d0550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:53.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.639+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 shutdown_connections 2024-03-04T07:28:53.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.639+0000 7f09eab16640 1 -- 172.21.15.64:0/1441792161 wait complete. 2024-03-04T07:28:53.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:28:53.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.752+0000 7f161378a640 1 Processor -- start 2024-03-04T07:28:53.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.752+0000 7f161378a640 1 -- start start 2024-03-04T07:28:53.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f161378a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f160c0d02e0 0x7f160c0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f161378a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0b4260 0x7f160c0b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f161378a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f160c0b0dd0 con 0x7f160c0d06c0 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f161378a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f160c0b0b80 con 0x7f160c0d02e0 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f161378a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f160c0aeec0 con 0x7f160c0b4260 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f1611526640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f160c0d02e0 0x7f160c0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f1611d27640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f160c0d06c0 0x7f160c0b1a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43568/0 (socket says 172.21.15.64:43568) 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f1611d27640 1 -- 172.21.15.64:0/233596406 learned_addr learned my addr 172.21.15.64:0/233596406 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.753+0000 7f1610d25640 1 --2- 172.21.15.64:0/233596406 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0b4260 0x7f160c0b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.754+0000 7f1611526640 1 -- 172.21.15.64:0/233596406 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0b4260 msgr2=0x7f160c0b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.754+0000 7f1611526640 1 --2- 172.21.15.64:0/233596406 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0b4260 0x7f160c0b1500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.754+0000 7f1611526640 1 -- 172.21.15.64:0/233596406 >> v1:172.21.15.64:6789/0 conn(0x7f160c0d06c0 legacy=0x7f160c0b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.754+0000 7f160bfff640 1 -- 172.21.15.64:0/233596406 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3322228553 0 0) 0x7f160c0b0dd0 con 0x7f160c0d06c0 2024-03-04T07:28:53.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.754+0000 7f1611526640 1 -- 172.21.15.64:0/233596406 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f160c0b10b0 con 0x7f160c0d02e0 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.754+0000 7f1611526640 1 --2- 172.21.15.64:0/233596406 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d02e0 0x7f160c0cfa00 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f16040097a0 tx=0x7f160402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ca1029f2ae26975a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.754+0000 7f160bfff640 1 -- 172.21.15.64:0/233596406 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1604034070 con 0x7f160c0d02e0 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f160bfff640 1 -- 172.21.15.64:0/233596406 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1604030610 con 0x7f160c0d02e0 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f160bfff640 1 -- 172.21.15.64:0/233596406 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16040309e0 con 0x7f160c0d02e0 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 -- 172.21.15.64:0/233596406 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d02e0 msgr2=0x7f160c0cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 --2- 172.21.15.64:0/233596406 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d02e0 0x7f160c0cfa00 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f16040097a0 tx=0x7f160402b960 comp rx=0 tx=0).stop 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 -- 172.21.15.64:0/233596406 shutdown_connections 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 --2- 172.21.15.64:0/233596406 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0b4260 0x7f160c0b1500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 --2- 172.21.15.64:0/233596406 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d02e0 0x7f160c0cfa00 unknown :-1 s=CLOSED pgs=569 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 -- 172.21.15.64:0/233596406 >> 172.21.15.64:0/233596406 conn(0x7f160c0b36d0 msgr2=0x7f160c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:53.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 -- 172.21.15.64:0/233596406 shutdown_connections 2024-03-04T07:28:53.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.755+0000 7f161378a640 1 -- 172.21.15.64:0/233596406 wait complete. 2024-03-04T07:28:53.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.756+0000 7f161378a640 1 Processor -- start 2024-03-04T07:28:53.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.756+0000 7f161378a640 1 -- start start 2024-03-04T07:28:53.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f161378a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f160c0b4260 0x7f160c0ca5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f161378a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0d02e0 0x7f160c14f430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f161378a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d06c0 0x7f160c0c8bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f161378a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f160c0b22d0 con 0x7f160c0d06c0 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f161378a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f160c0b2150 con 0x7f160c0b4260 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f161378a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f160c0af350 con 0x7f160c0d02e0 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f1611d27640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d06c0 0x7f160c0c8bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f1611d27640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d06c0 0x7f160c0c8bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36928/0 (socket says 172.21.15.64:36928) 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f1611d27640 1 -- 172.21.15.64:0/2331307949 learned_addr learned my addr 172.21.15.64:0/2331307949 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.757+0000 7f1610d25640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0d02e0 0x7f160c14f430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611526640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f160c0b4260 0x7f160c0ca5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611d27640 1 -- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0d02e0 msgr2=0x7f160c14f430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611d27640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0d02e0 0x7f160c14f430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611d27640 1 -- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f160c0b4260 msgr2=0x7f160c0ca5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611d27640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f160c0b4260 0x7f160c0ca5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611d27640 1 -- 172.21.15.64:0/2331307949 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f160c0c9540 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611526640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f160c0b4260 0x7f160c0ca5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1611d27640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d06c0 0x7f160c0c8bd0 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f15fc00c700 tx=0x7f15fc00cbc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1609ffb640 1 -- 172.21.15.64:0/2331307949 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15fc018070 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1609ffb640 1 -- 172.21.15.64:0/2331307949 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f15fc00ce10 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f160c0c9820 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f1609ffb640 1 -- 172.21.15.64:0/2331307949 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15fc00a880 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.758+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f160c0c9de0 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.759+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f160c0b1c40 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.759+0000 7f1609ffb640 1 -- 172.21.15.64:0/2331307949 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f15fc01b070 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.760+0000 7f1609ffb640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f15f806b500 0x7f15f806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.760+0000 7f1609ffb640 1 -- 172.21.15.64:0/2331307949 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f15fc089720 con 0x7f160c0d06c0 2024-03-04T07:28:53.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.760+0000 7f1611526640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f15f806b500 0x7f15f806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:53.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.760+0000 7f1611526640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f15f806b500 0x7f15f806d9c0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f1604036000 tx=0x7f1604008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.764+0000 7f1609ffb640 1 -- 172.21.15.64:0/2331307949 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f160c0b1c40 con 0x7f160c0d06c0 2024-03-04T07:28:53.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.957+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f160c068700 con 0x7f160c0d06c0 2024-03-04T07:28:53.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.958+0000 7f1609ffb640 1 -- 172.21.15.64:0/2331307949 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v44) v1 ==== 80+0+0 (secure 0 0 0) 0x7f15fc0585a0 con 0x7f160c0d06c0 2024-03-04T07:28:53.957 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f15f806b500 msgr2=0x7f15f806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f15f806b500 0x7f15f806d9c0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f1604036000 tx=0x7f1604008030 comp rx=0 tx=0).stop 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d06c0 msgr2=0x7f160c0c8bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d06c0 0x7f160c0c8bd0 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f15fc00c700 tx=0x7f15fc00cbc0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 shutdown_connections 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f15f806b500 0x7f15f806d9c0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f160c0d06c0 0x7f160c0c8bd0 unknown :-1 s=CLOSED pgs=570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f160c0d02e0 0x7f160c14f430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 --2- 172.21.15.64:0/2331307949 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f160c0b4260 0x7f160c0ca5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 >> 172.21.15.64:0/2331307949 conn(0x7f160c0b36d0 msgr2=0x7f160c0d2350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 shutdown_connections 2024-03-04T07:28:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:53.960+0000 7f161378a640 1 -- 172.21.15.64:0/2331307949 wait complete. 2024-03-04T07:28:53.970 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:28:53.971 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:53.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:28:53.995 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[56034]: fuse finished with error 0 and tester_r 0 2024-03-04T07:28:53.997 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:28:53.997 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:53.998 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:28:54.168 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:28:54.169 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:54.187 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:28:54.187 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:28:54.187 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:54.187 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:28:54.245 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:28:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:54.246 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:28:54.246 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:54.261 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:28:54.261 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:28:54.261 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:54.261 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:28:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:54.316 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:28:54.316 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:28:54.316 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:28:54.334 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:28:54.334 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:28:54.334 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:28:54.334 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:28:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:28:54.392 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:28:54.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:54.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.494+0000 7ffbecde9640 1 Processor -- start 2024-03-04T07:28:54.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.494+0000 7ffbecde9640 1 -- start start 2024-03-04T07:28:54.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbecde9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ffbe80b0e90 0x7ffbe806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbecde9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffbe80cee90 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbecde9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffbe80cfc50 con 0x7ffbe806b5c0 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbecde9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffbe80cfa00 con 0x7ffbe80b0e90 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbdffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ffbe80b0e90 0x7ffbe806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbe6d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ffbe80ae9b0 0x7ffbe80d03a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43582/0 (socket says 172.21.15.64:43582) 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbe6d9d640 1 -- 172.21.15.64:0/856803216 learned_addr learned my addr 172.21.15.64:0/856803216 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbdffff640 1 -- 172.21.15.64:0/856803216 >> v1:172.21.15.66:6789/0 conn(0x7ffbe806b5c0 legacy=0x7ffbe80b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbdffff640 1 -- 172.21.15.64:0/856803216 >> v1:172.21.15.64:6789/0 conn(0x7ffbe80ae9b0 legacy=0x7ffbe80d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbdffff640 1 -- 172.21.15.64:0/856803216 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffbe80cff30 con 0x7ffbe80b0e90 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.495+0000 7ffbe5d9b640 1 -- 172.21.15.64:0/856803216 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 976005413 0 0) 0x7ffbe80cee90 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbdffff640 1 --2- 172.21.15.64:0/856803216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80b0e90 0x7ffbe806b9a0 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7ffbd00097a0 tx=0x7ffbd002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6707467f012f0fa3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbe5d9b640 1 -- 172.21.15.64:0/856803216 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbd0035020 con 0x7ffbe80b0e90 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbe5d9b640 1 -- 172.21.15.64:0/856803216 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffbd0031830 con 0x7ffbe80b0e90 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbe5d9b640 1 -- 172.21.15.64:0/856803216 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbd0031c00 con 0x7ffbe80b0e90 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbecde9640 1 -- 172.21.15.64:0/856803216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80b0e90 msgr2=0x7ffbe806b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbecde9640 1 --2- 172.21.15.64:0/856803216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80b0e90 0x7ffbe806b9a0 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7ffbd00097a0 tx=0x7ffbd002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbecde9640 1 -- 172.21.15.64:0/856803216 shutdown_connections 2024-03-04T07:28:54.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbecde9640 1 --2- 172.21.15.64:0/856803216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80b0e90 0x7ffbe806b9a0 unknown :-1 s=CLOSED pgs=572 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.496+0000 7ffbecde9640 1 -- 172.21.15.64:0/856803216 >> 172.21.15.64:0/856803216 conn(0x7ffbe80b36d0 msgr2=0x7ffbe80b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:54.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.497+0000 7ffbecde9640 1 -- 172.21.15.64:0/856803216 shutdown_connections 2024-03-04T07:28:54.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.497+0000 7ffbecde9640 1 -- 172.21.15.64:0/856803216 wait complete. 2024-03-04T07:28:54.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.498+0000 7ffbecde9640 1 Processor -- start 2024-03-04T07:28:54.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.498+0000 7ffbecde9640 1 -- start start 2024-03-04T07:28:54.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.498+0000 7ffbecde9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffbe806b5c0 0x7ffbe81580e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.498+0000 7ffbecde9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80ae9b0 0x7ffbe80b6640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbecde9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffbe80b0e90 0x7ffbe80c6d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbecde9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffbe806c910 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbecde9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffbe806c790 con 0x7ffbe80b0e90 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbecde9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffbe80d0ab0 con 0x7ffbe806b5c0 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80ae9b0 0x7ffbe80b6640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80ae9b0 0x7ffbe80b6640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36946/0 (socket says 172.21.15.64:36946) 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 -- 172.21.15.64:0/142828575 learned_addr learned my addr 172.21.15.64:0/142828575 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbe659c640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffbe806b5c0 0x7ffbe81580e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbe6d9d640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffbe80b0e90 0x7ffbe80c6d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 -- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffbe806b5c0 msgr2=0x7ffbe81580e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffbe806b5c0 0x7ffbe81580e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 -- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffbe80b0e90 msgr2=0x7ffbe80c6d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffbe80b0e90 0x7ffbe80c6d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 -- 172.21.15.64:0/142828575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffbe8158620 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbe659c640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffbe806b5c0 0x7ffbe81580e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbe6d9d640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffbe80b0e90 0x7ffbe80c6d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdffff640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80ae9b0 0x7ffbe80b6640 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7ffbd0000c00 tx=0x7ffbd00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdeffd640 1 -- 172.21.15.64:0/142828575 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbd0035050 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdeffd640 1 -- 172.21.15.64:0/142828575 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffbd0004230 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbdeffd640 1 -- 172.21.15.64:0/142828575 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffbd003b740 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffbe80c5630 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.499+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffbe80c5bf0 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.500+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffbe8068700 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.501+0000 7ffbdeffd640 1 -- 172.21.15.64:0/142828575 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffbd0038020 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.501+0000 7ffbdeffd640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffbc806b4d0 0x7ffbc806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.501+0000 7ffbdeffd640 1 -- 172.21.15.64:0/142828575 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ffbd00ac170 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.501+0000 7ffbe659c640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffbc806b4d0 0x7ffbc806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.502+0000 7ffbe659c640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffbc806b4d0 0x7ffbc806d990 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7ffbd8000f60 tx=0x7ffbd8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:54.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.510+0000 7ffbdeffd640 1 -- 172.21.15.64:0/142828575 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffbe8158620 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.727+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ffbe806daf0 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.727+0000 7ffbdeffd640 1 -- 172.21.15.64:0/142828575 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 22 v22) v1 ==== 94+0+4971 (secure 0 0 0) 0x7ffbd0040070 con 0x7ffbe80ae9b0 2024-03-04T07:28:54.727 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:54.727 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5138,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/627869633","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":627869633},{"type":"v1","addr":"172.21.15.66:6834","nonce":627869633}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5141,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2387393562","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2387393562},{"type":"v1","addr":"172.21.15.66:6835","nonce":2387393562}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5685,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/817733642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":817733642},{"type":"v1","addr":"172.21.15.64:6837","nonce":817733642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":18}],"filesystems":[{"mdsmap":{"epoch":22,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:28:34.726288+0000","modified":"2024-03-04T07:28:36.736175+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5688},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5688":{"gid":5688,"name":"a","rank":0,"incarnation":20,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/3981471748","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3981471748},{"type":"v1","addr":"172.21.15.64:6836","nonce":3981471748}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5688,"qdb_cluster":[5688]},"id":3}]} 2024-03-04T07:28:54.728 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 22 2024-03-04T07:28:54.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffbc806b4d0 msgr2=0x7ffbc806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffbc806b4d0 0x7ffbc806d990 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7ffbd8000f60 tx=0x7ffbd8009040 comp rx=0 tx=0).stop 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80ae9b0 msgr2=0x7ffbe80b6640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80ae9b0 0x7ffbe80b6640 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7ffbd0000c00 tx=0x7ffbd00046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 shutdown_connections 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffbc806b4d0 0x7ffbc806d990 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffbe80b0e90 0x7ffbe80c6d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffbe80ae9b0 0x7ffbe80b6640 unknown :-1 s=CLOSED pgs=573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 --2- 172.21.15.64:0/142828575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffbe806b5c0 0x7ffbe81580e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.729+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 >> 172.21.15.64:0/142828575 conn(0x7ffbe80b36d0 msgr2=0x7ffbe80cdf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.730+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 shutdown_connections 2024-03-04T07:28:54.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.730+0000 7ffbecde9640 1 -- 172.21.15.64:0/142828575 wait complete. 2024-03-04T07:28:54.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:54.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.847+0000 7f6bbefc7640 1 Processor -- start 2024-03-04T07:28:54.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.847+0000 7f6bbefc7640 1 -- start start 2024-03-04T07:28:54.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.847+0000 7f6bbefc7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6bb815df70 0x7f6bb80d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.847+0000 7f6bbefc7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb80d18c0 0x7f6bb8071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.847+0000 7f6bbefc7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6bb80b6f20 con 0x7f6bb8071a80 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bbefc7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6bb80d3030 con 0x7f6bb815df70 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bbefc7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6bb80d3310 con 0x7f6bb80d18c0 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bbd564640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6bb8071a80 0x7f6bb80b2700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43596/0 (socket says 172.21.15.64:43596) 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bbd564640 1 -- 172.21.15.64:0/3018817813 learned_addr learned my addr 172.21.15.64:0/3018817813 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bb7fff640 1 --2- 172.21.15.64:0/3018817813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb80d18c0 0x7f6bb8071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bbcd63640 1 --2- 172.21.15.64:0/3018817813 >> v2:172.21.15.66:3300/0 conn(0x7f6bb815df70 0x7f6bb80d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bb77fe640 1 -- 172.21.15.64:0/3018817813 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4825642 0 0) 0x7f6bb80b6f20 con 0x7f6bb8071a80 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bb77fe640 1 -- 172.21.15.64:0/3018817813 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6ba0003520 con 0x7f6bb8071a80 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bb7fff640 1 -- 172.21.15.64:0/3018817813 >> v2:172.21.15.66:3300/0 conn(0x7f6bb815df70 msgr2=0x7f6bb80d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:54.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bb7fff640 1 --2- 172.21.15.64:0/3018817813 >> v2:172.21.15.66:3300/0 conn(0x7f6bb815df70 0x7f6bb80d27b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bb7fff640 1 -- 172.21.15.64:0/3018817813 >> v1:172.21.15.64:6789/0 conn(0x7f6bb8071a80 legacy=0x7f6bb80b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.848+0000 7f6bb7fff640 1 -- 172.21.15.64:0/3018817813 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6bb80d35f0 con 0x7f6bb80d18c0 2024-03-04T07:28:54.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.849+0000 7f6bb7fff640 1 --2- 172.21.15.64:0/3018817813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb80d18c0 0x7f6bb8071540 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6bac0097a0 tx=0x7f6bac02b960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a5f3f5e2cf5ab263 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:54.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.849+0000 7f6bb77fe640 1 -- 172.21.15.64:0/3018817813 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bac034070 con 0x7f6bb80d18c0 2024-03-04T07:28:54.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.849+0000 7f6bb77fe640 1 -- 172.21.15.64:0/3018817813 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bac030610 con 0x7f6bb80d18c0 2024-03-04T07:28:54.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.849+0000 7f6bb77fe640 1 -- 172.21.15.64:0/3018817813 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bac0309c0 con 0x7f6bb80d18c0 2024-03-04T07:28:54.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 -- 172.21.15.64:0/3018817813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb80d18c0 msgr2=0x7f6bb8071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/3018817813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb80d18c0 0x7f6bb8071540 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6bac0097a0 tx=0x7f6bac02b960 comp rx=0 tx=0).stop 2024-03-04T07:28:54.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 -- 172.21.15.64:0/3018817813 shutdown_connections 2024-03-04T07:28:54.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/3018817813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb80d18c0 0x7f6bb8071540 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/3018817813 >> v2:172.21.15.66:3300/0 conn(0x7f6bb815df70 0x7f6bb80d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 -- 172.21.15.64:0/3018817813 >> 172.21.15.64:0/3018817813 conn(0x7f6bb80d3cf0 msgr2=0x7f6bb80d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 -- 172.21.15.64:0/3018817813 shutdown_connections 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.850+0000 7f6bbefc7640 1 -- 172.21.15.64:0/3018817813 wait complete. 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.851+0000 7f6bbefc7640 1 Processor -- start 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.851+0000 7f6bbefc7640 1 -- start start 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbefc7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb8071a80 0x7f6bb8157b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbefc7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bb80d18c0 0x7f6bb8159ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbefc7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bb815df70 0x7f6bb815a220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbefc7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bb815e350 con 0x7f6bb80d18c0 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbefc7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bb80d2cf0 con 0x7f6bb815df70 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbefc7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6bb815e4d0 con 0x7f6bb8071a80 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bb80d18c0 0x7f6bb8159ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bb80d18c0 0x7f6bb8159ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36958/0 (socket says 172.21.15.64:36958) 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 -- 172.21.15.64:0/4092869198 learned_addr learned my addr 172.21.15.64:0/4092869198 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbd564640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bb815df70 0x7f6bb815a220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbcd63640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb8071a80 0x7f6bb8157b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 -- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb8071a80 msgr2=0x7f6bb8157b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb8071a80 0x7f6bb8157b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 -- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bb815df70 msgr2=0x7f6bb815a220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bb815df70 0x7f6bb815a220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 -- 172.21.15.64:0/4092869198 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6bb8158150 con 0x7f6bb80d18c0 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bbcd63640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb8071a80 0x7f6bb8157b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.852+0000 7f6bb7fff640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bb80d18c0 0x7f6bb8159ce0 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f6bac02b9a0 tx=0x7f6bac0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.853+0000 7f6bb57fa640 1 -- 172.21.15.64:0/4092869198 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bac048070 con 0x7f6bb80d18c0 2024-03-04T07:28:54.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.853+0000 7f6bb57fa640 1 -- 172.21.15.64:0/4092869198 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bac0044c0 con 0x7f6bb80d18c0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.853+0000 7f6bb57fa640 1 -- 172.21.15.64:0/4092869198 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bac034050 con 0x7f6bb80d18c0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.853+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6bb8158430 con 0x7f6bb80d18c0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.853+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6bb81589f0 con 0x7f6bb80d18c0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.854+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6bb8067ff0 con 0x7f6bb80d18c0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.854+0000 7f6bb57fa640 1 -- 172.21.15.64:0/4092869198 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6bac04b070 con 0x7f6bb80d18c0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.855+0000 7f6bb57fa640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ba406b4d0 0x7f6ba406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.855+0000 7f6bb57fa640 1 -- 172.21.15.64:0/4092869198 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6bac0abdb0 con 0x7f6bb80d18c0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.855+0000 7f6bbcd63640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ba406b4d0 0x7f6ba406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:54.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.855+0000 7f6bbcd63640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ba406b4d0 0x7f6ba406d990 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f6bb80b3a90 tx=0x7f6ba8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:54.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:54.863+0000 7f6bb57fa640 1 -- 172.21.15.64:0/4092869198 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6bac0b0070 con 0x7f6bb80d18c0 2024-03-04T07:28:55.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.045+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6bb815f780 con 0x7f6bb80d18c0 2024-03-04T07:28:55.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.047+0000 7f6bb57fa640 1 -- 172.21.15.64:0/4092869198 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v44) v1 ==== 74+0+16565 (secure 0 0 0) 0x7f6bac07ac20 con 0x7f6bb80d18c0 2024-03-04T07:28:55.047 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:55.047 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":44,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:36.369097+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:28:32.989901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-04T07:28:33.692080+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"44","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ba406b4d0 msgr2=0x7f6ba406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ba406b4d0 0x7f6ba406d990 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f6bb80b3a90 tx=0x7f6ba8008040 comp rx=0 tx=0).stop 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bb80d18c0 msgr2=0x7f6bb8159ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bb80d18c0 0x7f6bb8159ce0 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f6bac02b9a0 tx=0x7f6bac0046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 shutdown_connections 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ba406b4d0 0x7f6ba406d990 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bb815df70 0x7f6bb815a220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bb80d18c0 0x7f6bb8159ce0 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 --2- 172.21.15.64:0/4092869198 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bb8071a80 0x7f6bb8157b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 >> 172.21.15.64:0/4092869198 conn(0x7f6bb80d3cf0 msgr2=0x7f6bb8160df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:55.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 shutdown_connections 2024-03-04T07:28:55.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.050+0000 7f6bbefc7640 1 -- 172.21.15.64:0/4092869198 wait complete. 2024-03-04T07:28:55.064 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 6, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:28:32.989901+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '43', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 7, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:28:33.692080+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '44', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:28:55.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:28:55.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.174+0000 7fd45506f640 1 Processor -- start 2024-03-04T07:28:55.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.174+0000 7fd45506f640 1 -- start start 2024-03-04T07:28:55.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd45506f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd45015df40 0x7fd450066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd45506f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd450067240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd45506f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd45006dc90 0x7fd45006e120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd45506f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd45015f000 con 0x7fd45015df40 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd45506f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd45015e6d0 con 0x7fd45006dc90 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd45506f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd45015e9b0 con 0x7fd4500b3ce0 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd44ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd45015df40 0x7fd450066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd44ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd45015df40 0x7fd450066d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36966/0 (socket says 172.21.15.64:36966) 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.175+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3849477146 learned_addr learned my addr 172.21.15.64:0/3849477146 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44f59e640 1 --2- 172.21.15.64:0/3849477146 >> v2:172.21.15.66:3300/0 conn(0x7fd45006dc90 0x7fd45006e120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44659c640 1 --2- 172.21.15.64:0/3849477146 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd450067240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3849477146 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 msgr2=0x7fd450067240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44ed9d640 1 --2- 172.21.15.64:0/3849477146 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd450067240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3849477146 >> v2:172.21.15.66:3300/0 conn(0x7fd45006dc90 msgr2=0x7fd45006e120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:55.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44ed9d640 1 --2- 172.21.15.64:0/3849477146 >> v2:172.21.15.66:3300/0 conn(0x7fd45006dc90 0x7fd45006e120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3849477146 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd45015ec90 con 0x7fd45015df40 2024-03-04T07:28:55.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44f59e640 1 --2- 172.21.15.64:0/3849477146 >> v2:172.21.15.66:3300/0 conn(0x7fd45006dc90 0x7fd45006e120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:55.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.176+0000 7fd44659c640 1 --2- 172.21.15.64:0/3849477146 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd450067240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.177+0000 7fd44ed9d640 1 --2- 172.21.15.64:0/3849477146 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45015df40 0x7fd450066d00 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7fd4500b3a40 tx=0x7fd44002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5e78967803dc4e30 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.177+0000 7fd44e59c640 1 -- 172.21.15.64:0/3849477146 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd440035020 con 0x7fd45015df40 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.177+0000 7fd44e59c640 1 -- 172.21.15.64:0/3849477146 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd440031870 con 0x7fd45015df40 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.177+0000 7fd44e59c640 1 -- 172.21.15.64:0/3849477146 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd440031c40 con 0x7fd45015df40 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 -- 172.21.15.64:0/3849477146 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45015df40 msgr2=0x7fd450066d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 --2- 172.21.15.64:0/3849477146 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45015df40 0x7fd450066d00 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7fd4500b3a40 tx=0x7fd44002b910 comp rx=0 tx=0).stop 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 -- 172.21.15.64:0/3849477146 shutdown_connections 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 --2- 172.21.15.64:0/3849477146 >> v2:172.21.15.66:3300/0 conn(0x7fd45006dc90 0x7fd45006e120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 --2- 172.21.15.64:0/3849477146 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd450067240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 --2- 172.21.15.64:0/3849477146 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45015df40 0x7fd450066d00 unknown :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 -- 172.21.15.64:0/3849477146 >> 172.21.15.64:0/3849477146 conn(0x7fd4500d09c0 msgr2=0x7fd4500d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:55.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 -- 172.21.15.64:0/3849477146 shutdown_connections 2024-03-04T07:28:55.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.178+0000 7fd45506f640 1 -- 172.21.15.64:0/3849477146 wait complete. 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.179+0000 7fd45506f640 1 Processor -- start 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.179+0000 7fd45506f640 1 -- start start 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd45506f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45006dc90 0x7fd4500afe70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd45506f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd4500b1bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd45506f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd45015df40 0x7fd450145020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd45506f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd45015e320 con 0x7fd45006dc90 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd45506f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd450071a20 con 0x7fd45015df40 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd45506f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd450145df0 con 0x7fd4500b3ce0 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45006dc90 0x7fd4500afe70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45006dc90 0x7fd4500afe70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36968/0 (socket says 172.21.15.64:36968) 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3596039363 learned_addr learned my addr 172.21.15.64:0/3596039363 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44f59e640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd45015df40 0x7fd450145020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44659c640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd4500b1bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 msgr2=0x7fd4500b1bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd4500b1bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd45015df40 msgr2=0x7fd450145020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd45015df40 0x7fd450145020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 -- 172.21.15.64:0/3596039363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4501460d0 con 0x7fd45006dc90 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.180+0000 7fd44ed9d640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45006dc90 0x7fd4500afe70 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fd44002b950 tx=0x7fd4400046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.181+0000 7fd447fff640 1 -- 172.21.15.64:0/3596039363 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd44003ad60 con 0x7fd45006dc90 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.181+0000 7fd447fff640 1 -- 172.21.15.64:0/3596039363 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd4400443f0 con 0x7fd45006dc90 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.181+0000 7fd447fff640 1 -- 172.21.15.64:0/3596039363 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4400447c0 con 0x7fd45006dc90 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.181+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4501463b0 con 0x7fd45006dc90 2024-03-04T07:28:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.181+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd450146970 con 0x7fd45006dc90 2024-03-04T07:28:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.182+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd4500733d0 con 0x7fd45006dc90 2024-03-04T07:28:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.182+0000 7fd447fff640 1 -- 172.21.15.64:0/3596039363 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd440038020 con 0x7fd45006dc90 2024-03-04T07:28:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.183+0000 7fd447fff640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd43006b500 0x7fd43006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.183+0000 7fd447fff640 1 -- 172.21.15.64:0/3596039363 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd4400ac2e0 con 0x7fd45006dc90 2024-03-04T07:28:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.183+0000 7fd44659c640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd43006b500 0x7fd43006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.183+0000 7fd44659c640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd43006b500 0x7fd43006d9c0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fd4501f7bb0 tx=0x7fd438008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.187+0000 7fd447fff640 1 -- 172.21.15.64:0/3596039363 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd44003f070 con 0x7fd45006dc90 2024-03-04T07:28:55.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.366+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd45006e330 con 0x7fd45006dc90 2024-03-04T07:28:55.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.367+0000 7fd447fff640 1 -- 172.21.15.64:0/3596039363 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v44) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd44007aa80 con 0x7fd45006dc90 2024-03-04T07:28:55.366 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd43006b500 msgr2=0x7fd43006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd43006b500 0x7fd43006d9c0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fd4501f7bb0 tx=0x7fd438008040 comp rx=0 tx=0).stop 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45006dc90 msgr2=0x7fd4500afe70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45006dc90 0x7fd4500afe70 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fd44002b950 tx=0x7fd4400046c0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 shutdown_connections 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd43006b500 0x7fd43006d9c0 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd45015df40 0x7fd450145020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4500b3ce0 0x7fd4500b1bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 --2- 172.21.15.64:0/3596039363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd45006dc90 0x7fd4500afe70 unknown :-1 s=CLOSED pgs=577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 >> 172.21.15.64:0/3596039363 conn(0x7fd4500d09c0 msgr2=0x7fd4500b4c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:55.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 shutdown_connections 2024-03-04T07:28:55.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.369+0000 7fd45506f640 1 -- 172.21.15.64:0/3596039363 wait complete. 2024-03-04T07:28:55.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:28:55.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.488+0000 7f19f4a15640 1 Processor -- start 2024-03-04T07:28:55.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.488+0000 7f19f4a15640 1 -- start start 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19f4a15640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f19f015dd90 0x7f19f00ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19f4a15640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f19f00b6d10 0x7f19f00b7150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19f4a15640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f19f00d3740 con 0x7f19f00d29d0 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19f4a15640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f19f00d3400 con 0x7f19f015dd90 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19f4a15640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f19f00b7690 con 0x7f19f00b6d10 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19ee59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f19f015dd90 0x7f19f00ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19edd9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f19f00d29d0 0x7f19f00d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43610/0 (socket says 172.21.15.64:43610) 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19edd9b640 1 -- 172.21.15.64:0/680739540 learned_addr learned my addr 172.21.15.64:0/680739540 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19eed9d640 1 --2- 172.21.15.64:0/680739540 >> v2:172.21.15.66:3300/0 conn(0x7f19f00b6d10 0x7f19f00b7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19ee59c640 1 -- 172.21.15.64:0/680739540 >> v2:172.21.15.66:3300/0 conn(0x7f19f00b6d10 msgr2=0x7f19f00b7150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19ee59c640 1 --2- 172.21.15.64:0/680739540 >> v2:172.21.15.66:3300/0 conn(0x7f19f00b6d10 0x7f19f00b7150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19ee59c640 1 -- 172.21.15.64:0/680739540 >> v1:172.21.15.64:6789/0 conn(0x7f19f00d29d0 legacy=0x7f19f00d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.490+0000 7f19ee59c640 1 -- 172.21.15.64:0/680739540 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19f00d3120 con 0x7f19f015dd90 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.491+0000 7f19ee59c640 1 --2- 172.21.15.64:0/680739540 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f00ceea0 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f19f00b5820 tx=0x7f19e002b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f660d45f44ae10f7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.491+0000 7f19ed59a640 1 -- 172.21.15.64:0/680739540 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19e0034070 con 0x7f19f015dd90 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.491+0000 7f19ed59a640 1 -- 172.21.15.64:0/680739540 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f19e0030610 con 0x7f19f015dd90 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.491+0000 7f19ed59a640 1 -- 172.21.15.64:0/680739540 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19e00309e0 con 0x7f19f015dd90 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.491+0000 7f19f4a15640 1 -- 172.21.15.64:0/680739540 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 msgr2=0x7f19f00ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.491+0000 7f19f4a15640 1 --2- 172.21.15.64:0/680739540 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f00ceea0 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f19f00b5820 tx=0x7f19e002b960 comp rx=0 tx=0).stop 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.492+0000 7f19f4a15640 1 -- 172.21.15.64:0/680739540 shutdown_connections 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.492+0000 7f19f4a15640 1 --2- 172.21.15.64:0/680739540 >> v2:172.21.15.66:3300/0 conn(0x7f19f00b6d10 0x7f19f00b7150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.492+0000 7f19f4a15640 1 --2- 172.21.15.64:0/680739540 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f00ceea0 unknown :-1 s=CLOSED pgs=579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.492+0000 7f19f4a15640 1 -- 172.21.15.64:0/680739540 >> 172.21.15.64:0/680739540 conn(0x7f19f0071960 msgr2=0x7f19f0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:55.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.492+0000 7f19f4a15640 1 -- 172.21.15.64:0/680739540 shutdown_connections 2024-03-04T07:28:55.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.492+0000 7f19f4a15640 1 -- 172.21.15.64:0/680739540 wait complete. 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.493+0000 7f19f4a15640 1 Processor -- start 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.493+0000 7f19f4a15640 1 -- start start 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19f4a15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19f00b6d10 0x7f19f00cad70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19f4a15640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19f00d29d0 0x7f19f01493c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19f4a15640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f0149900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19f4a15640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19f015e170 con 0x7f19f015dd90 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19f4a15640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19f00d3cc0 con 0x7f19f00d29d0 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19f4a15640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f19f00d2df0 con 0x7f19f00b6d10 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f0149900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f0149900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36982/0 (socket says 172.21.15.64:36982) 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 -- 172.21.15.64:0/693033797 learned_addr learned my addr 172.21.15.64:0/693033797 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19edd9b640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19f00d29d0 0x7f19f01493c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19ee59c640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19f00b6d10 0x7f19f00cad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 -- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19f00b6d10 msgr2=0x7f19f00cad70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19f00b6d10 0x7f19f00cad70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 -- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19f00d29d0 msgr2=0x7f19f01493c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19f00d29d0 0x7f19f01493c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 -- 172.21.15.64:0/693033797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19f0157660 con 0x7f19f015dd90 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19eed9d640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f0149900 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f19e800cda0 tx=0x7f19e800a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19e6ffd640 1 -- 172.21.15.64:0/693033797 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19e8003b90 con 0x7f19f015dd90 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19e6ffd640 1 -- 172.21.15.64:0/693033797 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f19e8003d30 con 0x7f19f015dd90 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.494+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f19f00c93b0 con 0x7f19f015dd90 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.495+0000 7f19e6ffd640 1 -- 172.21.15.64:0/693033797 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19e8004240 con 0x7f19f015dd90 2024-03-04T07:28:55.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.495+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f19f00c9970 con 0x7f19f015dd90 2024-03-04T07:28:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.496+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f19f0072a30 con 0x7f19f015dd90 2024-03-04T07:28:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.496+0000 7f19e6ffd640 1 -- 172.21.15.64:0/693033797 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f19e80043e0 con 0x7f19f015dd90 2024-03-04T07:28:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.496+0000 7f19e6ffd640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19d006b4d0 0x7f19d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.496+0000 7f19e6ffd640 1 -- 172.21.15.64:0/693033797 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f19e808abf0 con 0x7f19f015dd90 2024-03-04T07:28:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.496+0000 7f19ee59c640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19d006b4d0 0x7f19d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.497+0000 7f19ee59c640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19d006b4d0 0x7f19d006d990 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f19e0036000 tx=0x7f19e003b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.500+0000 7f19e6ffd640 1 -- 172.21.15.64:0/693033797 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f19e8059420 con 0x7f19f015dd90 2024-03-04T07:28:55.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.797+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f19f004f410 con 0x7f19f015dd90 2024-03-04T07:28:55.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.798+0000 7f19e6ffd640 1 -- 172.21.15.64:0/693033797 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v44) v1 ==== 104+0+11 (secure 0 0 0) 0x7f19e805d4c0 con 0x7f19f015dd90 2024-03-04T07:28:55.797 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:28:55.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19d006b4d0 msgr2=0x7f19d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19d006b4d0 0x7f19d006d990 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f19e0036000 tx=0x7f19e003b040 comp rx=0 tx=0).stop 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 msgr2=0x7f19f0149900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f0149900 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f19e800cda0 tx=0x7f19e800a660 comp rx=0 tx=0).stop 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 shutdown_connections 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19d006b4d0 0x7f19d006d990 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19f015dd90 0x7f19f0149900 unknown :-1 s=CLOSED pgs=580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19f00d29d0 0x7f19f01493c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 --2- 172.21.15.64:0/693033797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19f00b6d10 0x7f19f00cad70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 >> 172.21.15.64:0/693033797 conn(0x7f19f0071960 msgr2=0x7f19f00d0c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 shutdown_connections 2024-03-04T07:28:55.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.800+0000 7f19f4a15640 1 -- 172.21.15.64:0/693033797 wait complete. 2024-03-04T07:28:55.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:28:55.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.916+0000 7f764b45c640 1 Processor -- start 2024-03-04T07:28:55.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.916+0000 7f764b45c640 1 -- start start 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f764b45c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f76440ceea0 0x7f76440d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f764b45c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f76440b6d10 0x7f76440b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f764b45c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f76440d3740 con 0x7f764415dd90 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f764b45c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f76440d3400 con 0x7f76440b6d10 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f764b45c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f76440d3120 con 0x7f76440ceea0 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76499f9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f76440b6d10 0x7f76440b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76499f9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f76440b6d10 0x7f76440b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36990/0 (socket says 172.21.15.64:36990) 2024-03-04T07:28:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76499f9640 1 -- 172.21.15.64:0/2177853951 learned_addr learned my addr 172.21.15.64:0/2177853951 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76491f8640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f764415dd90 0x7f76440d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43614/0 (socket says 172.21.15.64:43614) 2024-03-04T07:28:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76489f7640 1 --2- 172.21.15.64:0/2177853951 >> v2:172.21.15.66:3300/0 conn(0x7f76440ceea0 0x7f76440d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76499f9640 1 -- 172.21.15.64:0/2177853951 >> v2:172.21.15.66:3300/0 conn(0x7f76440ceea0 msgr2=0x7f76440d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76499f9640 1 --2- 172.21.15.64:0/2177853951 >> v2:172.21.15.66:3300/0 conn(0x7f76440ceea0 0x7f76440d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76499f9640 1 -- 172.21.15.64:0/2177853951 >> v1:172.21.15.64:6789/0 conn(0x7f764415dd90 legacy=0x7f76440d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f763bfff640 1 -- 172.21.15.64:0/2177853951 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4086541029 0 0) 0x7f76440d3740 con 0x7f764415dd90 2024-03-04T07:28:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.917+0000 7f76499f9640 1 -- 172.21.15.64:0/2177853951 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76440d39e0 con 0x7f76440b6d10 2024-03-04T07:28:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.918+0000 7f76499f9640 1 --2- 172.21.15.64:0/2177853951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440b71a0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f76440724d0 tx=0x7f763c02ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=416e6376e6cfad77 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.918+0000 7f763bfff640 1 -- 172.21.15.64:0/2177853951 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f763c02f850 con 0x7f76440b6d10 2024-03-04T07:28:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.918+0000 7f763bfff640 1 -- 172.21.15.64:0/2177853951 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f763c02f9f0 con 0x7f76440b6d10 2024-03-04T07:28:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.918+0000 7f763bfff640 1 -- 172.21.15.64:0/2177853951 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f763c02fdc0 con 0x7f76440b6d10 2024-03-04T07:28:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 -- 172.21.15.64:0/2177853951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 msgr2=0x7f76440b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 --2- 172.21.15.64:0/2177853951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440b71a0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f76440724d0 tx=0x7f763c02ba30 comp rx=0 tx=0).stop 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 -- 172.21.15.64:0/2177853951 shutdown_connections 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 --2- 172.21.15.64:0/2177853951 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440b71a0 unknown :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 --2- 172.21.15.64:0/2177853951 >> v2:172.21.15.66:3300/0 conn(0x7f76440ceea0 0x7f76440d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 -- 172.21.15.64:0/2177853951 >> 172.21.15.64:0/2177853951 conn(0x7f7644071960 msgr2=0x7f7644071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 -- 172.21.15.64:0/2177853951 shutdown_connections 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.919+0000 7f764b45c640 1 -- 172.21.15.64:0/2177853951 wait complete. 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.920+0000 7f764b45c640 1 Processor -- start 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.920+0000 7f764b45c640 1 -- start start 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f764b45c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440c8800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f764b45c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76440ceea0 0x7f76441f3dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f764b45c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f764415dd90 0x7f7644146b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f764b45c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76440d48f0 con 0x7f76440b6d10 2024-03-04T07:28:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f764b45c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f764415e170 con 0x7f764415dd90 2024-03-04T07:28:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f764b45c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f76440d4a70 con 0x7f76440ceea0 2024-03-04T07:28:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440c8800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440c8800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36996/0 (socket says 172.21.15.64:36996) 2024-03-04T07:28:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 -- 172.21.15.64:0/1923959120 learned_addr learned my addr 172.21.15.64:0/1923959120 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76499f9640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f764415dd90 0x7f7644146b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76489f7640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76440ceea0 0x7f76441f3dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 -- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76440ceea0 msgr2=0x7f76441f3dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76440ceea0 0x7f76441f3dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 -- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f764415dd90 msgr2=0x7f7644146b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f764415dd90 0x7f7644146b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 -- 172.21.15.64:0/1923959120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7644147090 con 0x7f76440b6d10 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f76491f8640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440c8800 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f764000c7f0 tx=0x7f764000ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f7639ffb640 1 -- 172.21.15.64:0/1923959120 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7640018070 con 0x7f76440b6d10 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f7639ffb640 1 -- 172.21.15.64:0/1923959120 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f764000a5e0 con 0x7f76440b6d10 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f76440c8de0 con 0x7f76440b6d10 2024-03-04T07:28:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.921+0000 7f7639ffb640 1 -- 172.21.15.64:0/1923959120 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f764000a9d0 con 0x7f76440b6d10 2024-03-04T07:28:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.922+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f76440c93a0 con 0x7f76440b6d10 2024-03-04T07:28:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.923+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f76440d26d0 con 0x7f76440b6d10 2024-03-04T07:28:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.923+0000 7f7639ffb640 1 -- 172.21.15.64:0/1923959120 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f764001b070 con 0x7f76440b6d10 2024-03-04T07:28:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.923+0000 7f7639ffb640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f762806b4d0 0x7f762806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.924+0000 7f76489f7640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f762806b4d0 0x7f762806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.924+0000 7f7639ffb640 1 -- 172.21.15.64:0/1923959120 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7640089730 con 0x7f76440b6d10 2024-03-04T07:28:55.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.924+0000 7f76489f7640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f762806b4d0 0x7f762806d990 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f76440ca7f0 tx=0x7f7634008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:55.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:55.928+0000 7f7639ffb640 1 -- 172.21.15.64:0/1923959120 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7640057f60 con 0x7f76440b6d10 2024-03-04T07:28:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.109+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f764404f410 con 0x7f76440b6d10 2024-03-04T07:28:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.110+0000 7f7639ffb640 1 -- 172.21.15.64:0/1923959120 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v44) v1 ==== 100+0+11 (secure 0 0 0) 0x7f764005c000 con 0x7f76440b6d10 2024-03-04T07:28:56.110 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f762806b4d0 msgr2=0x7f762806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f762806b4d0 0x7f762806d990 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f76440ca7f0 tx=0x7f7634008040 comp rx=0 tx=0).stop 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 msgr2=0x7f76440c8800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440c8800 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f764000c7f0 tx=0x7f764000ccb0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 shutdown_connections 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f762806b4d0 0x7f762806d990 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f764415dd90 0x7f7644146b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f76440ceea0 0x7f76441f3dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 --2- 172.21.15.64:0/1923959120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76440b6d10 0x7f76440c8800 unknown :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 >> 172.21.15.64:0/1923959120 conn(0x7f7644071960 msgr2=0x7f76440d0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:56.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 shutdown_connections 2024-03-04T07:28:56.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.112+0000 7f764b45c640 1 -- 172.21.15.64:0/1923959120 wait complete. 2024-03-04T07:28:56.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:56.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.225+0000 7fef0b756640 1 Processor -- start 2024-03-04T07:28:56.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.225+0000 7fef0b756640 1 -- start start 2024-03-04T07:28:56.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.226+0000 7fef0b756640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef040b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.226+0000 7fef0b756640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef040b9a10 0x7fef040b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef0b756640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fef040cea30 con 0x7fef0415df00 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef0b756640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fef040cf3c0 con 0x7fef040b9a10 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef0b756640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fef040cf0e0 con 0x7fef040b95a0 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef09cf3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef040b9a10 0x7fef040b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef09cf3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef040b9a10 0x7fef040b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37002/0 (socket says 172.21.15.64:37002) 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef09cf3640 1 -- 172.21.15.64:0/2600619229 learned_addr learned my addr 172.21.15.64:0/2600619229 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef094f2640 1 --1- 172.21.15.64:0/2600619229 >> v1:172.21.15.64:6789/0 conn(0x7fef0415df00 0x7fef040d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43628/0 (socket says 172.21.15.64:43628) 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef08cf1640 1 --2- 172.21.15.64:0/2600619229 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef040b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef09cf3640 1 -- 172.21.15.64:0/2600619229 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 msgr2=0x7fef040b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef09cf3640 1 --2- 172.21.15.64:0/2600619229 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef040b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef09cf3640 1 -- 172.21.15.64:0/2600619229 >> v1:172.21.15.64:6789/0 conn(0x7fef0415df00 legacy=0x7fef040d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7fef09cf3640 1 -- 172.21.15.64:0/2600619229 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef040ce470 con 0x7fef040b9a10 2024-03-04T07:28:56.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.227+0000 7feefbfff640 1 -- 172.21.15.64:0/2600619229 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3252405772 0 0) 0x7fef040cea30 con 0x7fef0415df00 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7fef09cf3640 1 --2- 172.21.15.64:0/2600619229 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef040b9a10 0x7fef040b5c40 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7feefc009e50 tx=0x7feefc02bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a45cafaaf0b2819 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7feefbfff640 1 -- 172.21.15.64:0/2600619229 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feefc02cb00 con 0x7fef040b9a10 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7feefbfff640 1 -- 172.21.15.64:0/2600619229 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feefc02cca0 con 0x7fef040b9a10 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7feefbfff640 1 -- 172.21.15.64:0/2600619229 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feefc034610 con 0x7fef040b9a10 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7fef0b756640 1 -- 172.21.15.64:0/2600619229 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef040b9a10 msgr2=0x7fef040b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7fef0b756640 1 --2- 172.21.15.64:0/2600619229 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef040b9a10 0x7fef040b5c40 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7feefc009e50 tx=0x7feefc02bf60 comp rx=0 tx=0).stop 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7fef0b756640 1 -- 172.21.15.64:0/2600619229 shutdown_connections 2024-03-04T07:28:56.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7fef0b756640 1 --2- 172.21.15.64:0/2600619229 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef040b9a10 0x7fef040b5c40 unknown :-1 s=CLOSED pgs=585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7fef0b756640 1 --2- 172.21.15.64:0/2600619229 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef040b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.228+0000 7fef0b756640 1 -- 172.21.15.64:0/2600619229 >> 172.21.15.64:0/2600619229 conn(0x7fef04071960 msgr2=0x7fef04071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:56.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.229+0000 7fef0b756640 1 -- 172.21.15.64:0/2600619229 shutdown_connections 2024-03-04T07:28:56.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.229+0000 7fef0b756640 1 -- 172.21.15.64:0/2600619229 wait complete. 2024-03-04T07:28:56.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 Processor -- start 2024-03-04T07:28:56.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 -- start start 2024-03-04T07:28:56.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef0414e030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef040b9a10 0x7fef04159610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef0415df00 0x7fef04157b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef040cedf0 con 0x7fef0415df00 2024-03-04T07:28:56.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef040cec70 con 0x7fef040b9a10 2024-03-04T07:28:56.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef0b756640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fef040d3220 con 0x7fef040b95a0 2024-03-04T07:28:56.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef09cf3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef0415df00 0x7fef04157b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef09cf3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef0415df00 0x7fef04157b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37014/0 (socket says 172.21.15.64:37014) 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.230+0000 7fef09cf3640 1 -- 172.21.15.64:0/1839683517 learned_addr learned my addr 172.21.15.64:0/1839683517 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef094f2640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef0414e030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef08cf1640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef040b9a10 0x7fef04159610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef09cf3640 1 -- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 msgr2=0x7fef0414e030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef09cf3640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef0414e030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef09cf3640 1 -- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef040b9a10 msgr2=0x7fef04159610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef09cf3640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef040b9a10 0x7fef04159610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef09cf3640 1 -- 172.21.15.64:0/1839683517 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef04159b50 con 0x7fef0415df00 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef08cf1640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef040b9a10 0x7fef04159610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef09cf3640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef0415df00 0x7fef04157b90 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7feefc009e50 tx=0x7feefc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7feef9ffb640 1 -- 172.21.15.64:0/1839683517 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feefc043030 con 0x7fef0415df00 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7feef9ffb640 1 -- 172.21.15.64:0/1839683517 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feefc02ce50 con 0x7fef0415df00 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fef04158570 con 0x7fef0415df00 2024-03-04T07:28:56.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7feef9ffb640 1 -- 172.21.15.64:0/1839683517 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feefc009510 con 0x7fef0415df00 2024-03-04T07:28:56.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.231+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef04158b30 con 0x7fef0415df00 2024-03-04T07:28:56.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.233+0000 7feef9ffb640 1 -- 172.21.15.64:0/1839683517 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feefc005d10 con 0x7fef0415df00 2024-03-04T07:28:56.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.233+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fef04067ff0 con 0x7fef0415df00 2024-03-04T07:28:56.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.233+0000 7feef9ffb640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feee806b500 0x7feee806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.233+0000 7fef094f2640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feee806b500 0x7feee806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.233+0000 7feef9ffb640 1 -- 172.21.15.64:0/1839683517 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7feefc0ac140 con 0x7fef0415df00 2024-03-04T07:28:56.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.234+0000 7fef094f2640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feee806b500 0x7feee806d9c0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fef00007fd0 tx=0x7fef00007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.237+0000 7feef9ffb640 1 -- 172.21.15.64:0/1839683517 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feefc04d020 con 0x7fef0415df00 2024-03-04T07:28:56.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.459+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fef0415d570 con 0x7fef0415df00 2024-03-04T07:28:56.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.459+0000 7feef9ffb640 1 -- 172.21.15.64:0/1839683517 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 22 v22) v1 ==== 94+0+4971 (secure 0 0 0) 0x7feefc07b050 con 0x7fef0415df00 2024-03-04T07:28:56.459 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:56.459 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5138,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/627869633","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":627869633},{"type":"v1","addr":"172.21.15.66:6834","nonce":627869633}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5141,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2387393562","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2387393562},{"type":"v1","addr":"172.21.15.66:6835","nonce":2387393562}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5685,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/817733642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":817733642},{"type":"v1","addr":"172.21.15.64:6837","nonce":817733642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":18}],"filesystems":[{"mdsmap":{"epoch":22,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:28:34.726288+0000","modified":"2024-03-04T07:28:36.736175+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5688},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5688":{"gid":5688,"name":"a","rank":0,"incarnation":20,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/3981471748","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3981471748},{"type":"v1","addr":"172.21.15.64:6836","nonce":3981471748}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5688,"qdb_cluster":[5688]},"id":3}]} 2024-03-04T07:28:56.460 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 22 2024-03-04T07:28:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feee806b500 msgr2=0x7feee806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feee806b500 0x7feee806d9c0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fef00007fd0 tx=0x7fef00007de0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef0415df00 msgr2=0x7fef04157b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef0415df00 0x7fef04157b90 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7feefc009e50 tx=0x7feefc004a40 comp rx=0 tx=0).stop 2024-03-04T07:28:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 shutdown_connections 2024-03-04T07:28:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feee806b500 0x7feee806d9c0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef0415df00 0x7fef04157b90 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef040b9a10 0x7fef04159610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 --2- 172.21.15.64:0/1839683517 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef040b95a0 0x7fef0414e030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.461+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 >> 172.21.15.64:0/1839683517 conn(0x7fef04071960 msgr2=0x7fef04071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:56.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.462+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 shutdown_connections 2024-03-04T07:28:56.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.462+0000 7fef0b756640 1 -- 172.21.15.64:0/1839683517 wait complete. 2024-03-04T07:28:56.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:56.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.578+0000 7f60c2de2640 1 Processor -- start 2024-03-04T07:28:56.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.578+0000 7f60c2de2640 1 -- start start 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.579+0000 7f60c2de2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f60bc0d02e0 0x7f60bc0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.579+0000 7f60c2de2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f60bc0d06c0 0x7f60bc0b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.579+0000 7f60c2de2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f60bc0b0dd0 con 0x7f60bc0b4260 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.579+0000 7f60c2de2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f60bc0b0b80 con 0x7f60bc0d02e0 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.579+0000 7f60c2de2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f60bc0aeec0 con 0x7f60bc0d06c0 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c0b7e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f60bc0d02e0 0x7f60bc0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c0b7e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f60bc0d02e0 0x7f60bc0cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37020/0 (socket says 172.21.15.64:37020) 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c0b7e640 1 -- 172.21.15.64:0/2339342668 learned_addr learned my addr 172.21.15.64:0/2339342668 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c0b7e640 1 -- 172.21.15.64:0/2339342668 >> v2:172.21.15.66:3300/0 conn(0x7f60bc0d06c0 msgr2=0x7f60bc0b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c137f640 1 --2- 172.21.15.64:0/2339342668 >> v2:172.21.15.66:3300/0 conn(0x7f60bc0d06c0 0x7f60bc0b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c0b7e640 1 --2- 172.21.15.64:0/2339342668 >> v2:172.21.15.66:3300/0 conn(0x7f60bc0d06c0 0x7f60bc0b1c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c0b7e640 1 -- 172.21.15.64:0/2339342668 >> v1:172.21.15.66:6789/0 conn(0x7f60bc0b4260 legacy=0x7f60bc0b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:56.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.580+0000 7f60c0b7e640 1 -- 172.21.15.64:0/2339342668 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60bc0b10b0 con 0x7f60bc0d02e0 2024-03-04T07:28:56.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.581+0000 7f60c0b7e640 1 --2- 172.21.15.64:0/2339342668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0d02e0 0x7f60bc0cfa00 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f60bc0cc0e0 tx=0x7f60b002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cfbe92ff9252a1f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.581+0000 7f60bb7fe640 1 -- 172.21.15.64:0/2339342668 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60b0034070 con 0x7f60bc0d02e0 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.581+0000 7f60bb7fe640 1 -- 172.21.15.64:0/2339342668 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f60b0030860 con 0x7f60bc0d02e0 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.581+0000 7f60bb7fe640 1 -- 172.21.15.64:0/2339342668 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60b0030c30 con 0x7f60bc0d02e0 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.581+0000 7f60c2de2640 1 -- 172.21.15.64:0/2339342668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0d02e0 msgr2=0x7f60bc0cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.581+0000 7f60c2de2640 1 --2- 172.21.15.64:0/2339342668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0d02e0 0x7f60bc0cfa00 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f60bc0cc0e0 tx=0x7f60b002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.582+0000 7f60c2de2640 1 -- 172.21.15.64:0/2339342668 shutdown_connections 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.582+0000 7f60c2de2640 1 --2- 172.21.15.64:0/2339342668 >> v2:172.21.15.66:3300/0 conn(0x7f60bc0d06c0 0x7f60bc0b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.582+0000 7f60c2de2640 1 --2- 172.21.15.64:0/2339342668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0d02e0 0x7f60bc0cfa00 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.582+0000 7f60c2de2640 1 -- 172.21.15.64:0/2339342668 >> 172.21.15.64:0/2339342668 conn(0x7f60bc0b36d0 msgr2=0x7f60bc0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.582+0000 7f60c2de2640 1 -- 172.21.15.64:0/2339342668 shutdown_connections 2024-03-04T07:28:56.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.582+0000 7f60c2de2640 1 -- 172.21.15.64:0/2339342668 wait complete. 2024-03-04T07:28:56.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.583+0000 7f60c2de2640 1 Processor -- start 2024-03-04T07:28:56.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.583+0000 7f60c2de2640 1 -- start start 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c2de2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0b4260 0x7f60bc0c5b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c2de2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60bc0d02e0 0x7f60bc0c9c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c2de2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60bc0d06c0 0x7f60bc15bc40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c2de2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60bc0b22d0 con 0x7f60bc0b4260 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c2de2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60bc0b2150 con 0x7f60bc0d06c0 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c2de2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f60bc0af350 con 0x7f60bc0d02e0 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0b4260 0x7f60bc0c5b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0b4260 0x7f60bc0c5b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37022/0 (socket says 172.21.15.64:37022) 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 -- 172.21.15.64:0/192181307 learned_addr learned my addr 172.21.15.64:0/192181307 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c137f640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60bc0d06c0 0x7f60bc15bc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60bbfff640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60bc0d02e0 0x7f60bc0c9c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 -- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60bc0d02e0 msgr2=0x7f60bc0c9c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60bc0d02e0 0x7f60bc0c9c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 -- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60bc0d06c0 msgr2=0x7f60bc15bc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60bc0d06c0 0x7f60bc15bc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 -- 172.21.15.64:0/192181307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60bc142b20 con 0x7f60bc0b4260 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c137f640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60bc0d06c0 0x7f60bc15bc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60bbfff640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60bc0d02e0 0x7f60bc0c9c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.585+0000 7f60c0b7e640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0b4260 0x7f60bc0c5b40 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7f60b002b9e0 tx=0x7f60b0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.586+0000 7f60b97fa640 1 -- 172.21.15.64:0/192181307 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60b0048070 con 0x7f60bc0b4260 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.586+0000 7f60b97fa640 1 -- 172.21.15.64:0/192181307 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f60b0039d90 con 0x7f60bc0b4260 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.586+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f60bc0c6080 con 0x7f60bc0b4260 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.586+0000 7f60b97fa640 1 -- 172.21.15.64:0/192181307 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60b0034040 con 0x7f60bc0b4260 2024-03-04T07:28:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.586+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f60bc0ca950 con 0x7f60bc0b4260 2024-03-04T07:28:56.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.587+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f60bc068700 con 0x7f60bc0b4260 2024-03-04T07:28:56.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.587+0000 7f60b97fa640 1 -- 172.21.15.64:0/192181307 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f60b004b070 con 0x7f60bc0b4260 2024-03-04T07:28:56.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.588+0000 7f60b97fa640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60a006b4d0 0x7f60a006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.588+0000 7f60bbfff640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60a006b4d0 0x7f60a006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.588+0000 7f60b97fa640 1 -- 172.21.15.64:0/192181307 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f60b00ac430 con 0x7f60bc0b4260 2024-03-04T07:28:56.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.588+0000 7f60bbfff640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60a006b4d0 0x7f60a006d990 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f60bc0cbcd0 tx=0x7f60a8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.592+0000 7f60b97fa640 1 -- 172.21.15.64:0/192181307 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f60b007b2b0 con 0x7f60bc0b4260 2024-03-04T07:28:56.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.776+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f60bc0ce650 con 0x7f60bc0b4260 2024-03-04T07:28:56.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.777+0000 7f60b97fa640 1 -- 172.21.15.64:0/192181307 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v44) v1 ==== 74+0+16565 (secure 0 0 0) 0x7f60b007abb0 con 0x7f60bc0b4260 2024-03-04T07:28:56.777 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:56.778 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":44,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:36.369097+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:28:32.989901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-04T07:28:33.692080+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"44","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:56.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60a006b4d0 msgr2=0x7f60a006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60a006b4d0 0x7f60a006d990 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f60bc0cbcd0 tx=0x7f60a8009040 comp rx=0 tx=0).stop 2024-03-04T07:28:56.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0b4260 msgr2=0x7f60bc0c5b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0b4260 0x7f60bc0c5b40 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7f60b002b9e0 tx=0x7f60b0004730 comp rx=0 tx=0).stop 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 shutdown_connections 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60a006b4d0 0x7f60a006d990 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60bc0d06c0 0x7f60bc15bc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60bc0d02e0 0x7f60bc0c9c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 --2- 172.21.15.64:0/192181307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f60bc0b4260 0x7f60bc0c5b40 unknown :-1 s=CLOSED pgs=588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 >> 172.21.15.64:0/192181307 conn(0x7f60bc0b36d0 msgr2=0x7f60bc140840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 shutdown_connections 2024-03-04T07:28:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.780+0000 7f60c2de2640 1 -- 172.21.15.64:0/192181307 wait complete. 2024-03-04T07:28:56.795 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:28:56.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:56.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.896+0000 7f3988634640 1 Processor -- start 2024-03-04T07:28:56.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.896+0000 7f3988634640 1 -- start start 2024-03-04T07:28:56.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3988634640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3988634640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f39800aec40 0x7f39800b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3988634640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3980071240 con 0x7f3980158520 2024-03-04T07:28:56.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3988634640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3980071b70 con 0x7f39800aec40 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3988634640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39800af0b0 con 0x7f398006df30 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3985bcf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3986bd1640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f39800aec40 0x7f39800b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f39863d0640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f3980158520 0x7f398006d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:51068/0 (socket says 172.21.15.64:51068) 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3986bd1640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f39800aec40 0x7f39800b3640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:41856/0 (socket says 172.21.15.64:41856) 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f3985bcf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:35904/0 (socket says 172.21.15.64:35904) 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.898+0000 7f39863d0640 1 -- 172.21.15.64:0/4189544431 learned_addr learned my addr 172.21.15.64:0/4189544431 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.899+0000 7f39853ce640 1 -- 172.21.15.64:0/4189544431 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4048743174 0 0) 0x7f3980071240 con 0x7f3980158520 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.899+0000 7f3985bcf640 1 -- 172.21.15.64:0/4189544431 >> v2:172.21.15.66:3300/0 conn(0x7f39800aec40 msgr2=0x7f39800b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.899+0000 7f3985bcf640 1 --2- 172.21.15.64:0/4189544431 >> v2:172.21.15.66:3300/0 conn(0x7f39800aec40 0x7f39800b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.899+0000 7f3985bcf640 1 -- 172.21.15.64:0/4189544431 >> v1:172.21.15.66:6789/0 conn(0x7f3980158520 legacy=0x7f398006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.899+0000 7f3985bcf640 1 -- 172.21.15.64:0/4189544431 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3980071520 con 0x7f398006df30 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.899+0000 7f3985bcf640 1 --2- 172.21.15.64:0/4189544431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800ae700 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f397c0097a0 tx=0x7f397c02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f8b5a016d56fdce6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f39853ce640 1 -- 172.21.15.64:0/4189544431 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f397c035020 con 0x7f398006df30 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f39853ce640 1 -- 172.21.15.64:0/4189544431 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f397c031830 con 0x7f398006df30 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f39853ce640 1 -- 172.21.15.64:0/4189544431 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f397c031c00 con 0x7f398006df30 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 -- 172.21.15.64:0/4189544431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 msgr2=0x7f39800ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 --2- 172.21.15.64:0/4189544431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800ae700 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f397c0097a0 tx=0x7f397c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 -- 172.21.15.64:0/4189544431 shutdown_connections 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 --2- 172.21.15.64:0/4189544431 >> v2:172.21.15.66:3300/0 conn(0x7f39800aec40 0x7f39800b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 --2- 172.21.15.64:0/4189544431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800ae700 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 -- 172.21.15.64:0/4189544431 >> 172.21.15.64:0/4189544431 conn(0x7f3980066d00 msgr2=0x7f39800670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 -- 172.21.15.64:0/4189544431 shutdown_connections 2024-03-04T07:28:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.900+0000 7f3988634640 1 -- 172.21.15.64:0/4189544431 wait complete. 2024-03-04T07:28:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.901+0000 7f3988634640 1 Processor -- start 2024-03-04T07:28:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.901+0000 7f3988634640 1 -- start start 2024-03-04T07:28:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3988634640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800b20e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3988634640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39800aec40 0x7f39800ccbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3988634640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3980158520 0x7f39800cd110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3988634640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3980158900 con 0x7f3980158520 2024-03-04T07:28:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3988634640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3980072000 con 0x7f39800aec40 2024-03-04T07:28:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3988634640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3980158a80 con 0x7f398006df30 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3986bd1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3980158520 0x7f39800cd110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3986bd1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3980158520 0x7f39800cd110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37030/0 (socket says 172.21.15.64:37030) 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3986bd1640 1 -- 172.21.15.64:0/1978061584 learned_addr learned my addr 172.21.15.64:0/1978061584 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f39863d0640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800b20e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.902+0000 7f3985bcf640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39800aec40 0x7f39800ccbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3986bd1640 1 -- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 msgr2=0x7f39800b20e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3986bd1640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800b20e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3986bd1640 1 -- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39800aec40 msgr2=0x7f39800ccbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3986bd1640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39800aec40 0x7f39800ccbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3986bd1640 1 -- 172.21.15.64:0/1978061584 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3980148160 con 0x7f3980158520 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3986bd1640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3980158520 0x7f39800cd110 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f39780092e0 tx=0x7f397800e490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3976ffd640 1 -- 172.21.15.64:0/1978061584 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f397801a070 con 0x7f3980158520 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3976ffd640 1 -- 172.21.15.64:0/1978061584 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f397800ebe0 con 0x7f3980158520 2024-03-04T07:28:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3980148440 con 0x7f3980158520 2024-03-04T07:28:56.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3976ffd640 1 -- 172.21.15.64:0/1978061584 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3978003990 con 0x7f3980158520 2024-03-04T07:28:56.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.903+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3980148a00 con 0x7f3980158520 2024-03-04T07:28:56.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.904+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3980067ff0 con 0x7f3980158520 2024-03-04T07:28:56.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.905+0000 7f3976ffd640 1 -- 172.21.15.64:0/1978061584 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f397801d070 con 0x7f3980158520 2024-03-04T07:28:56.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.905+0000 7f3976ffd640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b4d0 0x7f396406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:56.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.905+0000 7f3976ffd640 1 -- 172.21.15.64:0/1978061584 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f397808a350 con 0x7f3980158520 2024-03-04T07:28:56.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.905+0000 7f39863d0640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b4d0 0x7f396406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.906+0000 7f39863d0640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b4d0 0x7f396406d990 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f3970000f60 tx=0x7f3970007c00 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:56.910+0000 7f3976ffd640 1 -- 172.21.15.64:0/1978061584 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f39780591c0 con 0x7f3980158520 2024-03-04T07:28:57.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.131+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f39800b31f0 con 0x7f3980158520 2024-03-04T07:28:57.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.131+0000 7f3976ffd640 1 -- 172.21.15.64:0/1978061584 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 22 v22) v1 ==== 94+0+4971 (secure 0 0 0) 0x7f3978058ac0 con 0x7f3980158520 2024-03-04T07:28:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:57.131 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5138,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/627869633","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":627869633},{"type":"v1","addr":"172.21.15.66:6834","nonce":627869633}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5141,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2387393562","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2387393562},{"type":"v1","addr":"172.21.15.66:6835","nonce":2387393562}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5685,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/817733642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":817733642},{"type":"v1","addr":"172.21.15.64:6837","nonce":817733642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":18}],"filesystems":[{"mdsmap":{"epoch":22,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:28:34.726288+0000","modified":"2024-03-04T07:28:36.736175+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5688},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5688":{"gid":5688,"name":"a","rank":0,"incarnation":20,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/3981471748","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3981471748},{"type":"v1","addr":"172.21.15.64:6836","nonce":3981471748}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5688,"qdb_cluster":[5688]},"id":3}]} 2024-03-04T07:28:57.132 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 22 2024-03-04T07:28:57.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b4d0 msgr2=0x7f396406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b4d0 0x7f396406d990 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f3970000f60 tx=0x7f3970007c00 comp rx=0 tx=0).stop 2024-03-04T07:28:57.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3980158520 msgr2=0x7f39800cd110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3980158520 0x7f39800cd110 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f39780092e0 tx=0x7f397800e490 comp rx=0 tx=0).stop 2024-03-04T07:28:57.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 shutdown_connections 2024-03-04T07:28:57.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f396406b4d0 0x7f396406d990 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3980158520 0x7f39800cd110 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39800aec40 0x7f39800ccbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 --2- 172.21.15.64:0/1978061584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f398006df30 0x7f39800b20e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 >> 172.21.15.64:0/1978061584 conn(0x7f3980066d00 msgr2=0x7f398006d200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:57.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 shutdown_connections 2024-03-04T07:28:57.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.133+0000 7f3988634640 1 -- 172.21.15.64:0/1978061584 wait complete. 2024-03-04T07:28:57.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:28:57.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.246+0000 7f802fec2640 1 Processor -- start 2024-03-04T07:28:57.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.246+0000 7f802fec2640 1 -- start start 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.246+0000 7f802fec2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 0x7f80280cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802fec2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f80280d06c0 0x7f80280b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802fec2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f80280b0dd0 con 0x7f80280d02e0 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802fec2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f80280b0b80 con 0x7f80280d06c0 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802fec2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80280aeec0 con 0x7f80280b4260 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802dc5e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f80280d02e0 0x7f80280b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43632/0 (socket says 172.21.15.64:43632) 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802dc5e640 1 -- 172.21.15.64:0/2005144491 learned_addr learned my addr 172.21.15.64:0/2005144491 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802e45f640 1 --2- 172.21.15.64:0/2005144491 >> v2:172.21.15.66:3300/0 conn(0x7f80280d06c0 0x7f80280b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3681886219 0 0) 0x7f80280b0dd0 con 0x7f80280d02e0 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802d45d640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 0x7f80280cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8014003520 con 0x7f80280d02e0 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2412586881 0 0) 0x7f8014003520 con 0x7f80280d02e0 2024-03-04T07:28:57.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 msgr2=0x7f80280cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 0x7f80280cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 >> v2:172.21.15.66:3300/0 conn(0x7f80280d06c0 msgr2=0x7f80280b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 --2- 172.21.15.64:0/2005144491 >> v2:172.21.15.66:3300/0 conn(0x7f80280d06c0 0x7f80280b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80280b10b0 con 0x7f80280d02e0 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f8024001670 con 0x7f80280d02e0 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 >> v1:172.21.15.64:6789/0 conn(0x7f80280d02e0 legacy=0x7f80280b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80140039b0 0x7f8014003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 0x7f8014004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8014004d40 0x7f80140051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80280b0b80 con 0x7f80140039b0 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80280aeec0 con 0x7f8014004d40 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f801400b9c0 con 0x7f80280b4260 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.247+0000 7f802dc5e640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80140039b0 0x7f8014003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802e45f640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8014004d40 0x7f80140051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802d45d640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 0x7f8014004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802dc5e640 1 -- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 msgr2=0x7f8014004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802dc5e640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280b4260 0x7f8014004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802dc5e640 1 -- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8014004d40 msgr2=0x7f80140051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802dc5e640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8014004d40 0x7f80140051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802dc5e640 1 -- 172.21.15.64:0/2005144491 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8014003520 con 0x7f80140039b0 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802dc5e640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80140039b0 0x7f8014003dd0 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f8024001050 tx=0x7f8014005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=78b3de996ee8a8ed server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.248+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8024007d70 con 0x7f80140039b0 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8024030070 con 0x7f80140039b0 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802cc5c640 1 -- 172.21.15.64:0/2005144491 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f802402cd80 con 0x7f80140039b0 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802fec2640 1 -- 172.21.15.64:0/2005144491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80140039b0 msgr2=0x7f8014003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802fec2640 1 --2- 172.21.15.64:0/2005144491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80140039b0 0x7f8014003dd0 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f8024001050 tx=0x7f8014005830 comp rx=0 tx=0).stop 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802e45f640 1 -- 172.21.15.64:0/2005144491 reap_dead start 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802fec2640 1 -- 172.21.15.64:0/2005144491 shutdown_connections 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802fec2640 1 -- 172.21.15.64:0/2005144491 >> 172.21.15.64:0/2005144491 conn(0x7f80280b36d0 msgr2=0x7f80280b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802fec2640 1 -- 172.21.15.64:0/2005144491 shutdown_connections 2024-03-04T07:28:57.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.249+0000 7f802fec2640 1 -- 172.21.15.64:0/2005144491 wait complete. 2024-03-04T07:28:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.250+0000 7f802fec2640 1 Processor -- start 2024-03-04T07:28:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.250+0000 7f802fec2640 1 -- start start 2024-03-04T07:28:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802fec2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80140039b0 0x7f80280caaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802fec2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80281f4b70 0x7f802814f8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802fec2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280ccc30 0x7f80280cd0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802fec2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80280b22d0 con 0x7f80280ccc30 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802fec2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80280b2150 con 0x7f80140039b0 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802fec2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80280cfa00 con 0x7f80281f4b70 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280ccc30 0x7f80280cd0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280ccc30 0x7f80280cd0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37052/0 (socket says 172.21.15.64:37052) 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 -- 172.21.15.64:0/3761142447 learned_addr learned my addr 172.21.15.64:0/3761142447 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802dc5e640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80140039b0 0x7f80280caaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802d45d640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80281f4b70 0x7f802814f8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 -- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80281f4b70 msgr2=0x7f802814f8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80281f4b70 0x7f802814f8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 -- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80140039b0 msgr2=0x7f80280caaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80140039b0 0x7f80280caaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.251+0000 7f802e45f640 1 -- 172.21.15.64:0/3761142447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80281561c0 con 0x7f80280ccc30 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.252+0000 7f802e45f640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280ccc30 0x7f80280cd0b0 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f80280d0b70 tx=0x7f80200115a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.252+0000 7f801e7fc640 1 -- 172.21.15.64:0/3761142447 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8020011e40 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.252+0000 7f801e7fc640 1 -- 172.21.15.64:0/3761142447 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f802001b070 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.252+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8028152630 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.252+0000 7f801e7fc640 1 -- 172.21.15.64:0/3761142447 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8020003990 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.252+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80281f53a0 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.253+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8028152800 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.253+0000 7f801e7fc640 1 -- 172.21.15.64:0/3761142447 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8020005d10 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.254+0000 7f801e7fc640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800c06b500 0x7f800c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.254+0000 7f802dc5e640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800c06b500 0x7f800c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.254+0000 7f801e7fc640 1 -- 172.21.15.64:0/3761142447 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f802008a3d0 con 0x7f80280ccc30 2024-03-04T07:28:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.254+0000 7f802dc5e640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800c06b500 0x7f800c06d9c0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8024034000 tx=0x7f8024007000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.258+0000 7f801e7fc640 1 -- 172.21.15.64:0/3761142447 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8020058c90 con 0x7f80280ccc30 2024-03-04T07:28:57.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.479+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8028068700 con 0x7f80280ccc30 2024-03-04T07:28:57.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.479+0000 7f801e7fc640 1 -- 172.21.15.64:0/3761142447 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 22 v22) v1 ==== 94+0+4971 (secure 0 0 0) 0x7f802005cd30 con 0x7f80280ccc30 2024-03-04T07:28:57.479 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:57.479 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":22,"default_fscid":3,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5138,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/627869633","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":627869633},{"type":"v1","addr":"172.21.15.66:6834","nonce":627869633}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5141,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2387393562","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2387393562},{"type":"v1","addr":"172.21.15.66:6835","nonce":2387393562}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5685,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/817733642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":817733642},{"type":"v1","addr":"172.21.15.64:6837","nonce":817733642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":18}],"filesystems":[{"mdsmap":{"epoch":22,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:28:34.726288+0000","modified":"2024-03-04T07:28:36.736175+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":5688},"failed":[],"damaged":[],"stopped":[],"info":{"gid_5688":{"gid":5688,"name":"a","rank":0,"incarnation":20,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/3981471748","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3981471748},{"type":"v1","addr":"172.21.15.64:6836","nonce":3981471748}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[7],"metadata_pool":6,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":5688,"qdb_cluster":[5688]},"id":3}]} 2024-03-04T07:28:57.479 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 22 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.481+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800c06b500 msgr2=0x7f800c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.481+0000 7f802fec2640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800c06b500 0x7f800c06d9c0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8024034000 tx=0x7f8024007000 comp rx=0 tx=0).stop 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.481+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280ccc30 msgr2=0x7f80280cd0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280ccc30 0x7f80280cd0b0 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f80280d0b70 tx=0x7f80200115a0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 shutdown_connections 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800c06b500 0x7f800c06d9c0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280ccc30 0x7f80280cd0b0 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80281f4b70 0x7f802814f8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 --2- 172.21.15.64:0/3761142447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80140039b0 0x7f80280caaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 >> 172.21.15.64:0/3761142447 conn(0x7f80280b36d0 msgr2=0x7f8028151e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 shutdown_connections 2024-03-04T07:28:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.482+0000 7f802fec2640 1 -- 172.21.15.64:0/3761142447 wait complete. 2024-03-04T07:28:57.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:28:57.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.594+0000 7f5cdd525640 1 Processor -- start 2024-03-04T07:28:57.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.594+0000 7f5cdd525640 1 -- start start 2024-03-04T07:28:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cdd525640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5cd8066d00 0x7f5cd815be60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cdd525640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5cd815f740 0x7f5cd80670e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cdd525640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd80719b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cdd525640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5cd8067620 con 0x7f5cd8066d00 2024-03-04T07:28:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cdd525640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5cd815fcc0 con 0x7f5cd815f740 2024-03-04T07:28:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cdd525640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5cd815ffa0 con 0x7f5cd806fdc0 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cd6ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5cd8066d00 0x7f5cd815be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cd6ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5cd8066d00 0x7f5cd815be60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37066/0 (socket says 172.21.15.64:37066) 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.595+0000 7f5cd6ffd640 1 -- 172.21.15.64:0/1573529719 learned_addr learned my addr 172.21.15.64:0/1573529719 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd67fc640 1 --2- 172.21.15.64:0/1573529719 >> v2:172.21.15.66:3300/0 conn(0x7f5cd815f740 0x7f5cd80670e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd77fe640 1 --2- 172.21.15.64:0/1573529719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd80719b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd6ffd640 1 -- 172.21.15.64:0/1573529719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 msgr2=0x7f5cd80719b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd6ffd640 1 --2- 172.21.15.64:0/1573529719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd80719b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd6ffd640 1 -- 172.21.15.64:0/1573529719 >> v2:172.21.15.66:3300/0 conn(0x7f5cd815f740 msgr2=0x7f5cd80670e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd6ffd640 1 --2- 172.21.15.64:0/1573529719 >> v2:172.21.15.66:3300/0 conn(0x7f5cd815f740 0x7f5cd80670e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd6ffd640 1 -- 172.21.15.64:0/1573529719 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5cd8160280 con 0x7f5cd8066d00 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd67fc640 1 --2- 172.21.15.64:0/1573529719 >> v2:172.21.15.66:3300/0 conn(0x7f5cd815f740 0x7f5cd80670e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd77fe640 1 --2- 172.21.15.64:0/1573529719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd80719b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.596+0000 7f5cd6ffd640 1 --2- 172.21.15.64:0/1573529719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd8066d00 0x7f5cd815be60 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f5cd8072c00 tx=0x7f5ccc02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=86c3fec44008466f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cd5ffb640 1 -- 172.21.15.64:0/1573529719 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ccc035020 con 0x7f5cd8066d00 2024-03-04T07:28:57.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cd5ffb640 1 -- 172.21.15.64:0/1573529719 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5ccc031870 con 0x7f5cd8066d00 2024-03-04T07:28:57.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cd5ffb640 1 -- 172.21.15.64:0/1573529719 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ccc031c40 con 0x7f5cd8066d00 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 -- 172.21.15.64:0/1573529719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd8066d00 msgr2=0x7f5cd815be60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 --2- 172.21.15.64:0/1573529719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd8066d00 0x7f5cd815be60 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f5cd8072c00 tx=0x7f5ccc02b910 comp rx=0 tx=0).stop 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 -- 172.21.15.64:0/1573529719 shutdown_connections 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 --2- 172.21.15.64:0/1573529719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd80719b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 --2- 172.21.15.64:0/1573529719 >> v2:172.21.15.66:3300/0 conn(0x7f5cd815f740 0x7f5cd80670e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 --2- 172.21.15.64:0/1573529719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd8066d00 0x7f5cd815be60 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 -- 172.21.15.64:0/1573529719 >> 172.21.15.64:0/1573529719 conn(0x7f5cd80d0990 msgr2=0x7f5cd80d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 -- 172.21.15.64:0/1573529719 shutdown_connections 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.597+0000 7f5cdd525640 1 -- 172.21.15.64:0/1573529719 wait complete. 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.599+0000 7f5cdd525640 1 Processor -- start 2024-03-04T07:28:57.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.599+0000 7f5cdd525640 1 -- start start 2024-03-04T07:28:57.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.600+0000 7f5cdd525640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cd8066d00 0x7f5cd8141330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.600+0000 7f5cdd525640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd81544d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.600+0000 7f5cdd525640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd815f740 0x7f5cd8154a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.600+0000 7f5cdd525640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5cd80703d0 con 0x7f5cd815f740 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.600+0000 7f5cdd525640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5cd8070250 con 0x7f5cd8066d00 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.600+0000 7f5cdd525640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5cd8071ef0 con 0x7f5cd806fdc0 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd815f740 0x7f5cd8154a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd815f740 0x7f5cd8154a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37072/0 (socket says 172.21.15.64:37072) 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 -- 172.21.15.64:0/89419361 learned_addr learned my addr 172.21.15.64:0/89419361 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd67fc640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd81544d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd6ffd640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cd8066d00 0x7f5cd8141330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 -- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 msgr2=0x7f5cd81544d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd81544d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 -- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cd8066d00 msgr2=0x7f5cd8141330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cd8066d00 0x7f5cd8141330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 -- 172.21.15.64:0/89419361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5cd81e6dc0 con 0x7f5cd815f740 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cd77fe640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd815f740 0x7f5cd8154a10 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f5cd000cef0 tx=0x7f5cd000a710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cbf7fe640 1 -- 172.21.15.64:0/89419361 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cd001a070 con 0x7f5cd815f740 2024-03-04T07:28:57.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cbf7fe640 1 -- 172.21.15.64:0/89419361 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5cd000d860 con 0x7f5cd815f740 2024-03-04T07:28:57.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5cd8152310 con 0x7f5cd815f740 2024-03-04T07:28:57.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cbf7fe640 1 -- 172.21.15.64:0/89419361 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cd000dc50 con 0x7f5cd815f740 2024-03-04T07:28:57.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.601+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5cd81528d0 con 0x7f5cd815f740 2024-03-04T07:28:57.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.602+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5cd8152ad0 con 0x7f5cd815f740 2024-03-04T07:28:57.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.603+0000 7f5cbf7fe640 1 -- 172.21.15.64:0/89419361 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5cd001d070 con 0x7f5cd815f740 2024-03-04T07:28:57.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.603+0000 7f5cbf7fe640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc406b4d0 0x7f5cc406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.604+0000 7f5cd6ffd640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc406b4d0 0x7f5cc406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.604+0000 7f5cbf7fe640 1 -- 172.21.15.64:0/89419361 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5cd008a990 con 0x7f5cd815f740 2024-03-04T07:28:57.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.604+0000 7f5cd6ffd640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc406b4d0 0x7f5cc406d990 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f5ccc036000 tx=0x7f5ccc03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.608+0000 7f5cbf7fe640 1 -- 172.21.15.64:0/89419361 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5cd8152ad0 con 0x7f5cd815f740 2024-03-04T07:28:57.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.789+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f5cd80d1790 con 0x7f5cd815f740 2024-03-04T07:28:57.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.790+0000 7f5cbf7fe640 1 -- 172.21.15.64:0/89419361 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v44) v1 ==== 74+0+16565 (secure 0 0 0) 0x7f5cd0059310 con 0x7f5cd815f740 2024-03-04T07:28:57.790 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:28:57.790 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":44,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:28:36.369097+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":6,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:28:32.989901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"43","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":7,"pool_name":"cephfs_data","create_time":"2024-03-04T07:28:33.692080+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"44","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc406b4d0 msgr2=0x7f5cc406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc406b4d0 0x7f5cc406d990 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f5ccc036000 tx=0x7f5ccc03c040 comp rx=0 tx=0).stop 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd815f740 msgr2=0x7f5cd8154a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd815f740 0x7f5cd8154a10 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f5cd000cef0 tx=0x7f5cd000a710 comp rx=0 tx=0).stop 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 shutdown_connections 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc406b4d0 0x7f5cc406d990 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cd815f740 0x7f5cd8154a10 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cd806fdc0 0x7f5cd81544d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 --2- 172.21.15.64:0/89419361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cd8066d00 0x7f5cd8141330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 >> 172.21.15.64:0/89419361 conn(0x7f5cd80d0990 msgr2=0x7f5cd806af20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:57.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 shutdown_connections 2024-03-04T07:28:57.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.792+0000 7f5cdd525640 1 -- 172.21.15.64:0/89419361 wait complete. 2024-03-04T07:28:57.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:28:57.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.909+0000 7fa6b2e4e640 1 Processor -- start 2024-03-04T07:28:57.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.909+0000 7fa6b2e4e640 1 -- start start 2024-03-04T07:28:57.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b2e4e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6ac071540 0x7fa6ac0b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b2e4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0b2b50 0x7fa6ac0b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b2e4e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa6ac0ce610 con 0x7fa6ac0d3400 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b2e4e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6ac0d2f70 con 0x7fa6ac071540 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b2e4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6ac0d4be0 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b0bea640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6ac071540 0x7fa6ac0b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b0bea640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6ac071540 0x7fa6ac0b30a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37074/0 (socket says 172.21.15.64:37074) 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b0bea640 1 -- 172.21.15.64:0/490952754 learned_addr learned my addr 172.21.15.64:0/490952754 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b13eb640 1 --2- 172.21.15.64:0/490952754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0b2b50 0x7fa6ac0b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b0bea640 1 -- 172.21.15.64:0/490952754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0b2b50 msgr2=0x7fa6ac0b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b0bea640 1 --2- 172.21.15.64:0/490952754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0b2b50 0x7fa6ac0b6e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b0bea640 1 -- 172.21.15.64:0/490952754 >> v1:172.21.15.66:6789/0 conn(0x7fa6ac0d3400 legacy=0x7fa6ac071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b0bea640 1 -- 172.21.15.64:0/490952754 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6ac0d2920 con 0x7fa6ac071540 2024-03-04T07:28:57.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.910+0000 7fa6b13eb640 1 --2- 172.21.15.64:0/490952754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0b2b50 0x7fa6ac0b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.911+0000 7fa6b0bea640 1 --2- 172.21.15.64:0/490952754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac071540 0x7fa6ac0b30a0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7fa6a4009870 tx=0x7fa6a402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8511b15684deda8b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.911+0000 7fa6ab7fe640 1 -- 172.21.15.64:0/490952754 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6a4034070 con 0x7fa6ac071540 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.911+0000 7fa6ab7fe640 1 -- 172.21.15.64:0/490952754 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa6a4030840 con 0x7fa6ac071540 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.911+0000 7fa6ab7fe640 1 -- 172.21.15.64:0/490952754 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6a4030c10 con 0x7fa6ac071540 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/490952754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac071540 msgr2=0x7fa6ac0b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/490952754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac071540 0x7fa6ac0b30a0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7fa6a4009870 tx=0x7fa6a402b980 comp rx=0 tx=0).stop 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/490952754 shutdown_connections 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/490952754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0b2b50 0x7fa6ac0b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/490952754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac071540 0x7fa6ac0b30a0 unknown :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/490952754 >> 172.21.15.64:0/490952754 conn(0x7fa6ac0d3ca0 msgr2=0x7fa6ac0d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/490952754 shutdown_connections 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.912+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/490952754 wait complete. 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.913+0000 7fa6b2e4e640 1 Processor -- start 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.913+0000 7fa6b2e4e640 1 -- start start 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b2e4e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6ac071540 0x7fa6ac14c610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b2e4e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac0b2b50 0x7fa6ac1e70d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b2e4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0d3400 0x7fa6ac14f830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b2e4e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6ac15e370 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b2e4e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6ac0ce8f0 con 0x7fa6ac071540 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b2e4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6ac1e7610 con 0x7fa6ac0d3400 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac0b2b50 0x7fa6ac1e70d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac0b2b50 0x7fa6ac1e70d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37088/0 (socket says 172.21.15.64:37088) 2024-03-04T07:28:57.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 -- 172.21.15.64:0/2342222357 learned_addr learned my addr 172.21.15.64:0/2342222357 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b13eb640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0d3400 0x7fa6ac14f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6b0bea640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6ac071540 0x7fa6ac14c610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 -- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0d3400 msgr2=0x7fa6ac14f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0d3400 0x7fa6ac14f830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 -- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6ac071540 msgr2=0x7fa6ac14c610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6ac071540 0x7fa6ac14c610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.914+0000 7fa6abfff640 1 -- 172.21.15.64:0/2342222357 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6ac1e78b0 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.915+0000 7fa6abfff640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac0b2b50 0x7fa6ac1e70d0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7fa6a000c470 tx=0x7fa6a000c930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.915+0000 7fa6a97fa640 1 -- 172.21.15.64:0/2342222357 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6a00052d0 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.915+0000 7fa6a97fa640 1 -- 172.21.15.64:0/2342222357 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa6a0005490 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.915+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa6ac1e7b90 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.915+0000 7fa6a97fa640 1 -- 172.21.15.64:0/2342222357 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6a0007260 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.915+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6ac1e8150 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.916+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa6ac068700 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.916+0000 7fa6a97fa640 1 -- 172.21.15.64:0/2342222357 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa6a0007400 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.917+0000 7fa6a97fa640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa69806b4d0 0x7fa69806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:57.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.917+0000 7fa6a97fa640 1 -- 172.21.15.64:0/2342222357 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa6a00896f0 con 0x7fa6ac0b2b50 2024-03-04T07:28:57.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.917+0000 7fa6b0bea640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa69806b4d0 0x7fa69806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:57.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.917+0000 7fa6b0bea640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa69806b4d0 0x7fa69806d990 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fa6a4036000 tx=0x7fa6a403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:57.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:57.921+0000 7fa6a97fa640 1 -- 172.21.15.64:0/2342222357 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa6a00584e0 con 0x7fa6ac0b2b50 2024-03-04T07:28:58.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.140+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7fa6ac0b3100 con 0x7fa6ac0b2b50 2024-03-04T07:28:58.140 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:28:58.141+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:28:58.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.142+0000 7fa6a97fa640 1 -- 172.21.15.64:0/2342222357 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v23) v1 ==== 161+0+0 (secure 0 0 0) 0x7fa6a0057de0 con 0x7fa6ac0b2b50 2024-03-04T07:28:58.142 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa69806b4d0 msgr2=0x7fa69806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa69806b4d0 0x7fa69806d990 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fa6a4036000 tx=0x7fa6a403b040 comp rx=0 tx=0).stop 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac0b2b50 msgr2=0x7fa6ac1e70d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac0b2b50 0x7fa6ac1e70d0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7fa6a000c470 tx=0x7fa6a000c930 comp rx=0 tx=0).stop 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 shutdown_connections 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa69806b4d0 0x7fa69806d990 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6ac0d3400 0x7fa6ac14f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6ac0b2b50 0x7fa6ac1e70d0 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 --2- 172.21.15.64:0/2342222357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6ac071540 0x7fa6ac14c610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 >> 172.21.15.64:0/2342222357 conn(0x7fa6ac0d3ca0 msgr2=0x7fa6ac160570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 shutdown_connections 2024-03-04T07:28:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.145+0000 7fa6b2e4e640 1 -- 172.21.15.64:0/2342222357 wait complete. 2024-03-04T07:28:58.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:28:58.183 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:28:58.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.263+0000 7f11d0955640 1 Processor -- start 2024-03-04T07:28:58.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.263+0000 7f11d0955640 1 -- start start 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11d0955640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f11cc1457b0 0x7f11cc0722e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11d0955640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 0x7f11cc15c190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11d0955640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f11cc15cbc0 con 0x7f11cc15c6d0 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11d0955640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f11cc15cea0 con 0x7f11cc1457b0 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11d0955640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11cc071ae0 con 0x7f11cc067930 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11ca59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f11cc1457b0 0x7f11cc0722e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11ca59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f11cc1457b0 0x7f11cc0722e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37110/0 (socket says 172.21.15.64:37110) 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11ca59c640 1 -- 172.21.15.64:0/3099144005 learned_addr learned my addr 172.21.15.64:0/3099144005 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11ca59c640 1 -- 172.21.15.64:0/3099144005 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 msgr2=0x7f11cc15c190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11ca59c640 1 --2- 172.21.15.64:0/3099144005 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 0x7f11cc15c190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11ca59c640 1 -- 172.21.15.64:0/3099144005 >> v1:172.21.15.66:6789/0 conn(0x7f11cc15c6d0 legacy=0x7f11cc066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.264+0000 7f11ca59c640 1 -- 172.21.15.64:0/3099144005 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11cc071dc0 con 0x7f11cc1457b0 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11ca59c640 1 --2- 172.21.15.64:0/3099144005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc0722e0 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f11bc009870 tx=0x7f11bc02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5de5c64238e0bda3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11c9d9b640 1 -- 172.21.15.64:0/3099144005 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11bc035020 con 0x7f11cc1457b0 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11c9d9b640 1 -- 172.21.15.64:0/3099144005 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11bc031640 con 0x7f11cc1457b0 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11c9d9b640 1 -- 172.21.15.64:0/3099144005 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11bc02d040 con 0x7f11cc1457b0 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 -- 172.21.15.64:0/3099144005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 msgr2=0x7f11cc0722e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 --2- 172.21.15.64:0/3099144005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc0722e0 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f11bc009870 tx=0x7f11bc02b8f0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 -- 172.21.15.64:0/3099144005 shutdown_connections 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 --2- 172.21.15.64:0/3099144005 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 0x7f11cc15c190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 --2- 172.21.15.64:0/3099144005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc0722e0 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 -- 172.21.15.64:0/3099144005 >> 172.21.15.64:0/3099144005 conn(0x7f11cc0b26b0 msgr2=0x7f11cc0b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 -- 172.21.15.64:0/3099144005 shutdown_connections 2024-03-04T07:28:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.265+0000 7f11d0955640 1 -- 172.21.15.64:0/3099144005 wait complete. 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 Processor -- start 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 -- start start 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 0x7f11cc1e62a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc1e67e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11cc15c6d0 0x7f11cc1f79f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11cc067590 con 0x7f11cc1457b0 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11cc067410 con 0x7f11cc15c6d0 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.267+0000 7f11d0955640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11cc06d9a0 con 0x7f11cc067930 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc1e67e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc1e67e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37120/0 (socket says 172.21.15.64:37120) 2024-03-04T07:28:58.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 -- 172.21.15.64:0/2346664549 learned_addr learned my addr 172.21.15.64:0/2346664549 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11ca59c640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 0x7f11cc1e62a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11cad9d640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11cc15c6d0 0x7f11cc1f79f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 -- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 msgr2=0x7f11cc1e62a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 0x7f11cc1e62a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 -- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11cc15c6d0 msgr2=0x7f11cc1f79f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11cc15c6d0 0x7f11cc1f79f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 -- 172.21.15.64:0/2346664549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11cc1588e0 con 0x7f11cc1457b0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11cad9d640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11cc15c6d0 0x7f11cc1f79f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c3fff640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc1e67e0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7f11b400b4a0 tx=0x7f11b400b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c2ffd640 1 -- 172.21.15.64:0/2346664549 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11b4014070 con 0x7f11cc1457b0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c2ffd640 1 -- 172.21.15.64:0/2346664549 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11b4010610 con 0x7f11cc1457b0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f11cc139d60 con 0x7f11cc1457b0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11c2ffd640 1 -- 172.21.15.64:0/2346664549 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11b4010a00 con 0x7f11cc1457b0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.268+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f11cc158b80 con 0x7f11cc1457b0 2024-03-04T07:28:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.269+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f11cc15c190 con 0x7f11cc1457b0 2024-03-04T07:28:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.270+0000 7f11c2ffd640 1 -- 172.21.15.64:0/2346664549 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f11b4017070 con 0x7f11cc1457b0 2024-03-04T07:28:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.270+0000 7f11c2ffd640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11ac06b4d0 0x7f11ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.270+0000 7f11c2ffd640 1 -- 172.21.15.64:0/2346664549 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(45..45 src has 1..45) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f11b4089c50 con 0x7f11cc1457b0 2024-03-04T07:28:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.270+0000 7f11ca59c640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11ac06b4d0 0x7f11ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.271+0000 7f11ca59c640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11ac06b4d0 0x7f11ac06d990 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f11bc036000 tx=0x7f11bc03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:58.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.274+0000 7f11c2ffd640 1 -- 172.21.15.64:0/2346664549 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f11cc15c190 con 0x7f11cc1457b0 2024-03-04T07:28:58.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:58.499+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f11cc1e7120 con 0x7f11cc1457b0 2024-03-04T07:28:59.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.144+0000 7f11c2ffd640 1 -- 172.21.15.64:0/2346664549 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v24) v1 ==== 104+0+0 (secure 0 0 0) 0x7f11b408f020 con 0x7f11cc1457b0 2024-03-04T07:28:59.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11ac06b4d0 msgr2=0x7f11ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:59.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11ac06b4d0 0x7f11ac06d990 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f11bc036000 tx=0x7f11bc03c040 comp rx=0 tx=0).stop 2024-03-04T07:28:59.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 msgr2=0x7f11cc1e67e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:59.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc1e67e0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7f11b400b4a0 tx=0x7f11b400b960 comp rx=0 tx=0).stop 2024-03-04T07:28:59.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 shutdown_connections 2024-03-04T07:28:59.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11ac06b4d0 0x7f11ac06d990 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11cc15c6d0 0x7f11cc1f79f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11cc1457b0 0x7f11cc1e67e0 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 --2- 172.21.15.64:0/2346664549 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11cc067930 0x7f11cc1e62a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 >> 172.21.15.64:0/2346664549 conn(0x7f11cc0b26b0 msgr2=0x7f11cc0d0a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:59.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 shutdown_connections 2024-03-04T07:28:59.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.146+0000 7f11d0955640 1 -- 172.21.15.64:0/2346664549 wait complete. 2024-03-04T07:28:59.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:28:59.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.310+0000 7f9993a1c640 1 Processor -- start 2024-03-04T07:28:59.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.310+0000 7f9993a1c640 1 -- start start 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9993a1c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f998c0b3060 0x7f998c071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9993a1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 0x7f998c0d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9993a1c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f998c15e4b0 con 0x7f998c0717b0 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9993a1c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f998c0d2e10 con 0x7f998c0b3060 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9993a1c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f998c0b97e0 con 0x7f998c0b2b10 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9990fb7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f998c0b3060 0x7f998c071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9990fb7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f998c0b3060 0x7f998c071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37122/0 (socket says 172.21.15.64:37122) 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9990fb7640 1 -- 172.21.15.64:0/3476124834 learned_addr learned my addr 172.21.15.64:0/3476124834 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9990fb7640 1 -- 172.21.15.64:0/3476124834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 msgr2=0x7f998c0d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9990fb7640 1 --2- 172.21.15.64:0/3476124834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 0x7f998c0d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9990fb7640 1 -- 172.21.15.64:0/3476124834 >> v1:172.21.15.66:6789/0 conn(0x7f998c0717b0 legacy=0x7f998c0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:28:59.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.311+0000 7f9990fb7640 1 -- 172.21.15.64:0/3476124834 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f998c0b9a80 con 0x7f998c0b3060 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9990fb7640 1 --2- 172.21.15.64:0/3476124834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0b3060 0x7f998c071b90 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f997c009870 tx=0x7f997c02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=19bf3eb9ce61abd4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9983fff640 1 -- 172.21.15.64:0/3476124834 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f997c035020 con 0x7f998c0b3060 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9983fff640 1 -- 172.21.15.64:0/3476124834 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f997c031640 con 0x7f998c0b3060 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9983fff640 1 -- 172.21.15.64:0/3476124834 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f997c02d040 con 0x7f998c0b3060 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9993a1c640 1 -- 172.21.15.64:0/3476124834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0b3060 msgr2=0x7f998c071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9993a1c640 1 --2- 172.21.15.64:0/3476124834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0b3060 0x7f998c071b90 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f997c009870 tx=0x7f997c02b8f0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9993a1c640 1 -- 172.21.15.64:0/3476124834 shutdown_connections 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9993a1c640 1 --2- 172.21.15.64:0/3476124834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 0x7f998c0d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9993a1c640 1 --2- 172.21.15.64:0/3476124834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0b3060 0x7f998c071b90 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9993a1c640 1 -- 172.21.15.64:0/3476124834 >> 172.21.15.64:0/3476124834 conn(0x7f998c0d3d20 msgr2=0x7f998c0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.312+0000 7f9993a1c640 1 -- 172.21.15.64:0/3476124834 shutdown_connections 2024-03-04T07:28:59.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.313+0000 7f9993a1c640 1 -- 172.21.15.64:0/3476124834 wait complete. 2024-03-04T07:28:59.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 Processor -- start 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 -- start start 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0717b0 0x7f998c15ab70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 0x7f998c1e9270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f998c0b3060 0x7f998c15c570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f998c0b7560 con 0x7f998c0717b0 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f998c0d4bf0 con 0x7f998c0b3060 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f9993a1c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f998c0b76e0 con 0x7f998c0b2b10 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.314+0000 7f99917b8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0717b0 0x7f998c15ab70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0717b0 0x7f998c15ab70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37136/0 (socket says 172.21.15.64:37136) 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 -- 172.21.15.64:0/2387386811 learned_addr learned my addr 172.21.15.64:0/2387386811 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f9990fb7640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 0x7f998c1e9270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f9991fb9640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f998c0b3060 0x7f998c15c570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:59.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 -- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 msgr2=0x7f998c1e9270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 0x7f998c1e9270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 -- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f998c0b3060 msgr2=0x7f998c15c570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f998c0b3060 0x7f998c15c570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 -- 172.21.15.64:0/2387386811 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f998c1e81d0 con 0x7f998c0717b0 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f99917b8640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0717b0 0x7f998c15ab70 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f998800c410 tx=0x7f998800c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f9981ffb640 1 -- 172.21.15.64:0/2387386811 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9988017070 con 0x7f998c0717b0 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f9981ffb640 1 -- 172.21.15.64:0/2387386811 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99880035f0 con 0x7f998c0717b0 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f9981ffb640 1 -- 172.21.15.64:0/2387386811 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9988003a20 con 0x7f998c0717b0 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f998c1e84b0 con 0x7f998c0717b0 2024-03-04T07:28:59.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.315+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f998c1e8a70 con 0x7f998c0717b0 2024-03-04T07:28:59.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.317+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f998c06ce40 con 0x7f998c0717b0 2024-03-04T07:28:59.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.317+0000 7f9981ffb640 1 -- 172.21.15.64:0/2387386811 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f998801a070 con 0x7f998c0717b0 2024-03-04T07:28:59.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.317+0000 7f9981ffb640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f997006b4d0 0x7f997006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:28:59.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.317+0000 7f9981ffb640 1 -- 172.21.15.64:0/2387386811 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(45..45 src has 1..45) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f99880896c0 con 0x7f998c0717b0 2024-03-04T07:28:59.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.317+0000 7f9990fb7640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f997006b4d0 0x7f997006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:28:59.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.318+0000 7f9990fb7640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f997006b4d0 0x7f997006d990 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f997c036000 tx=0x7f997c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:28:59.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.322+0000 7f9981ffb640 1 -- 172.21.15.64:0/2387386811 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9988058570 con 0x7f998c0717b0 2024-03-04T07:28:59.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:28:59.506+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f998c04f410 con 0x7f998c0717b0 2024-03-04T07:29:00.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.439+0000 7f9981ffb640 1 -- 172.21.15.64:0/2387386811 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v46) v1 ==== 181+0+0 (secure 0 0 0) 0x7f9988057e70 con 0x7f998c0717b0 2024-03-04T07:29:00.439 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:29:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.442+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f997006b4d0 msgr2=0x7f997006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:00.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.442+0000 7f9993a1c640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f997006b4d0 0x7f997006d990 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f997c036000 tx=0x7f997c03c040 comp rx=0 tx=0).stop 2024-03-04T07:29:00.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.442+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0717b0 msgr2=0x7f998c15ab70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:00.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.442+0000 7f9993a1c640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0717b0 0x7f998c15ab70 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f998800c410 tx=0x7f998800c8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 shutdown_connections 2024-03-04T07:29:00.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f997006b4d0 0x7f997006d990 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f998c0b3060 0x7f998c15c570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998c0b2b10 0x7f998c1e9270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 --2- 172.21.15.64:0/2387386811 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f998c0717b0 0x7f998c15ab70 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 >> 172.21.15.64:0/2387386811 conn(0x7f998c0d3d20 msgr2=0x7f998c0ce030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:00.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 shutdown_connections 2024-03-04T07:29:00.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.443+0000 7f9993a1c640 1 -- 172.21.15.64:0/2387386811 wait complete. 2024-03-04T07:29:00.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:29:00.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.569+0000 7fe82164e640 1 Processor -- start 2024-03-04T07:29:00.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.569+0000 7fe82164e640 1 -- start start 2024-03-04T07:29:00.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe82164e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe81c071540 0x7fe81c071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe82164e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe81c0b2d30 0x7fe81c0ce770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe82164e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe81c071e60 con 0x7fe81c0cecb0 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe82164e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe81c0d3230 con 0x7fe81c071540 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe82164e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe81c0d4960 con 0x7fe81c0b2d30 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe81affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe81c071540 0x7fe81c071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe81b7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe81c0cecb0 0x7fe81c0b72e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43658/0 (socket says 172.21.15.64:43658) 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.570+0000 7fe81b7fe640 1 -- 172.21.15.64:0/3947836150 learned_addr learned my addr 172.21.15.64:0/3947836150 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.571+0000 7fe81a7fc640 1 --2- 172.21.15.64:0/3947836150 >> v2:172.21.15.66:3300/0 conn(0x7fe81c0b2d30 0x7fe81c0ce770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.571+0000 7fe81affd640 1 -- 172.21.15.64:0/3947836150 >> v2:172.21.15.66:3300/0 conn(0x7fe81c0b2d30 msgr2=0x7fe81c0ce770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.571+0000 7fe81affd640 1 --2- 172.21.15.64:0/3947836150 >> v2:172.21.15.66:3300/0 conn(0x7fe81c0b2d30 0x7fe81c0ce770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.571+0000 7fe81affd640 1 -- 172.21.15.64:0/3947836150 >> v1:172.21.15.64:6789/0 conn(0x7fe81c0cecb0 legacy=0x7fe81c0b72e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.571+0000 7fe819ffb640 1 -- 172.21.15.64:0/3947836150 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 225328076 0 0) 0x7fe81c071e60 con 0x7fe81c0cecb0 2024-03-04T07:29:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.571+0000 7fe81affd640 1 -- 172.21.15.64:0/3947836150 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe81c0d36c0 con 0x7fe81c071540 2024-03-04T07:29:00.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.571+0000 7fe81a7fc640 1 --2- 172.21.15.64:0/3947836150 >> v2:172.21.15.66:3300/0 conn(0x7fe81c0b2d30 0x7fe81c0ce770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe81affd640 1 --2- 172.21.15.64:0/3947836150 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c071540 0x7fe81c071920 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7fe81c0c7690 tx=0x7fe80c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=97327abf8969e07d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe819ffb640 1 -- 172.21.15.64:0/3947836150 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe80c039070 con 0x7fe81c071540 2024-03-04T07:29:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe819ffb640 1 -- 172.21.15.64:0/3947836150 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe80c02cc80 con 0x7fe81c071540 2024-03-04T07:29:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe819ffb640 1 -- 172.21.15.64:0/3947836150 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe80c034660 con 0x7fe81c071540 2024-03-04T07:29:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 -- 172.21.15.64:0/3947836150 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c071540 msgr2=0x7fe81c071920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 --2- 172.21.15.64:0/3947836150 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c071540 0x7fe81c071920 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7fe81c0c7690 tx=0x7fe80c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 -- 172.21.15.64:0/3947836150 shutdown_connections 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 --2- 172.21.15.64:0/3947836150 >> v2:172.21.15.66:3300/0 conn(0x7fe81c0b2d30 0x7fe81c0ce770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 --2- 172.21.15.64:0/3947836150 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c071540 0x7fe81c071920 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 -- 172.21.15.64:0/3947836150 >> 172.21.15.64:0/3947836150 conn(0x7fe81c0d3d60 msgr2=0x7fe81c0d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 -- 172.21.15.64:0/3947836150 shutdown_connections 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.572+0000 7fe82164e640 1 -- 172.21.15.64:0/3947836150 wait complete. 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 Processor -- start 2024-03-04T07:29:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 -- start start 2024-03-04T07:29:00.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe81c071540 0x7fe81c15b470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:00.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c0b2d30 0x7fe81c0c95b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe81c0cecb0 0x7fe81c0c9af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe81c0d2be0 con 0x7fe81c0b2d30 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe81c0d2900 con 0x7fe81c071540 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe82164e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe81c0d2ec0 con 0x7fe81c0cecb0 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.574+0000 7fe81a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c0b2d30 0x7fe81c0c95b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c0b2d30 0x7fe81c0c95b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37150/0 (socket says 172.21.15.64:37150) 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 -- 172.21.15.64:0/1728106449 learned_addr learned my addr 172.21.15.64:0/1728106449 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81affd640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe81c071540 0x7fe81c15b470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81b7fe640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe81c0cecb0 0x7fe81c0c9af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 -- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe81c0cecb0 msgr2=0x7fe81c0c9af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe81c0cecb0 0x7fe81c0c9af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 -- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe81c071540 msgr2=0x7fe81c15b470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:00.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe81c071540 0x7fe81c15b470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 -- 172.21.15.64:0/1728106449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe81c1e3640 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81b7fe640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe81c0cecb0 0x7fe81c0c9af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe81a7fc640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c0b2d30 0x7fe81c0c95b0 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7fe80400c820 tx=0x7fe80400cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe8137fe640 1 -- 172.21.15.64:0/1728106449 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe804004230 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe8137fe640 1 -- 172.21.15.64:0/1728106449 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe8040043d0 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe81c1e3920 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe8137fe640 1 -- 172.21.15.64:0/1728106449 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe804007a70 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.575+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe81c1e3ee0 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.576+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe81c068a30 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.577+0000 7fe8137fe640 1 -- 172.21.15.64:0/1728106449 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe804007c10 con 0x7fe81c0b2d30 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.577+0000 7fe8137fe640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe7fc06b4d0 0x7fe7fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.577+0000 7fe81affd640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe7fc06b4d0 0x7fe7fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.577+0000 7fe8137fe640 1 -- 172.21.15.64:0/1728106449 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5357+0+0 (secure 0 0 0) 0x7fe80408a070 con 0x7fe81c0b2d30 2024-03-04T07:29:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.578+0000 7fe81affd640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe7fc06b4d0 0x7fe7fc06d990 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fe80c036000 tx=0x7fe80c030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:00.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.581+0000 7fe8137fe640 1 -- 172.21.15.64:0/1728106449 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe804015020 con 0x7fe81c0b2d30 2024-03-04T07:29:00.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:00.761+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fe81c072070 con 0x7fe81c0b2d30 2024-03-04T07:29:01.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.444+0000 7fe8137fe640 1 -- 172.21.15.64:0/1728106449 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v47) v1 ==== 169+0+0 (secure 0 0 0) 0x7fe804058a90 con 0x7fe81c0b2d30 2024-03-04T07:29:01.444 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:29:01.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe7fc06b4d0 msgr2=0x7fe7fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe7fc06b4d0 0x7fe7fc06d990 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fe80c036000 tx=0x7fe80c030000 comp rx=0 tx=0).stop 2024-03-04T07:29:01.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c0b2d30 msgr2=0x7fe81c0c95b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c0b2d30 0x7fe81c0c95b0 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7fe80400c820 tx=0x7fe80400cce0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 shutdown_connections 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe7fc06b4d0 0x7fe7fc06d990 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe81c0cecb0 0x7fe81c0c9af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe81c0b2d30 0x7fe81c0c95b0 secure :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7fe80400c820 tx=0x7fe80400cce0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.447+0000 7fe82164e640 1 --2- 172.21.15.64:0/1728106449 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe81c071540 0x7fe81c15b470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.448+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 >> 172.21.15.64:0/1728106449 conn(0x7fe81c0d3d60 msgr2=0x7fe81c0cae30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:01.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.448+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 shutdown_connections 2024-03-04T07:29:01.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.448+0000 7fe82164e640 1 -- 172.21.15.64:0/1728106449 wait complete. 2024-03-04T07:29:01.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:29:01.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.570+0000 7f6b4c579640 1 Processor -- start 2024-03-04T07:29:01.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.570+0000 7f6b4c579640 1 -- start start 2024-03-04T07:29:01.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.571+0000 7f6b4c579640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b4415dd90 0x7f6b440ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.571+0000 7f6b4c579640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6b440d29d0 0x7f6b440d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.571+0000 7f6b4c579640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b440d4720 0x7f6b440b6d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.571+0000 7f6b4c579640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b440d3400 con 0x7f6b4415dd90 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.571+0000 7f6b4c579640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b440d3740 con 0x7f6b440d29d0 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.571+0000 7f6b4c579640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b440d3120 con 0x7f6b440d4720 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b4415dd90 0x7f6b440ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b4415dd90 0x7f6b440ceea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37154/0 (socket says 172.21.15.64:37154) 2024-03-04T07:29:01.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 -- 172.21.15.64:0/3029246973 learned_addr learned my addr 172.21.15.64:0/3029246973 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b49b14640 1 --2- 172.21.15.64:0/3029246973 >> v2:172.21.15.66:3300/0 conn(0x7f6b440d29d0 0x7f6b440d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4ab16640 1 --2- 172.21.15.64:0/3029246973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b440d4720 0x7f6b440b6d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 -- 172.21.15.64:0/3029246973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b440d4720 msgr2=0x7f6b440b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 --2- 172.21.15.64:0/3029246973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b440d4720 0x7f6b440b6d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 -- 172.21.15.64:0/3029246973 >> v2:172.21.15.66:3300/0 conn(0x7f6b440d29d0 msgr2=0x7f6b440d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 --2- 172.21.15.64:0/3029246973 >> v2:172.21.15.66:3300/0 conn(0x7f6b440d29d0 0x7f6b440d41e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b4a315640 1 -- 172.21.15.64:0/3029246973 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b440d39e0 con 0x7f6b4415dd90 2024-03-04T07:29:01.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.572+0000 7f6b49b14640 1 --2- 172.21.15.64:0/3029246973 >> v2:172.21.15.66:3300/0 conn(0x7f6b440d29d0 0x7f6b440d41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4a315640 1 --2- 172.21.15.64:0/3029246973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b4415dd90 0x7f6b440ceea0 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f6b440724d0 tx=0x7f6b4002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ffb3b72660af7d0d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b49313640 1 -- 172.21.15.64:0/3029246973 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b40034070 con 0x7f6b4415dd90 2024-03-04T07:29:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b49313640 1 -- 172.21.15.64:0/3029246973 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b40030860 con 0x7f6b4415dd90 2024-03-04T07:29:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b49313640 1 -- 172.21.15.64:0/3029246973 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b40030c30 con 0x7f6b4415dd90 2024-03-04T07:29:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 -- 172.21.15.64:0/3029246973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b4415dd90 msgr2=0x7f6b440ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 --2- 172.21.15.64:0/3029246973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b4415dd90 0x7f6b440ceea0 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f6b440724d0 tx=0x7f6b4002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 -- 172.21.15.64:0/3029246973 shutdown_connections 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 --2- 172.21.15.64:0/3029246973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b440d4720 0x7f6b440b6d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 --2- 172.21.15.64:0/3029246973 >> v2:172.21.15.66:3300/0 conn(0x7f6b440d29d0 0x7f6b440d41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 --2- 172.21.15.64:0/3029246973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b4415dd90 0x7f6b440ceea0 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 -- 172.21.15.64:0/3029246973 >> 172.21.15.64:0/3029246973 conn(0x7f6b44071960 msgr2=0x7f6b44071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.573+0000 7f6b4c579640 1 -- 172.21.15.64:0/3029246973 shutdown_connections 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.574+0000 7f6b4c579640 1 -- 172.21.15.64:0/3029246973 wait complete. 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 Processor -- start 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 -- start start 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b440d29d0 0x7f6b440cc3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b440d4720 0x7f6b440c8800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b4415dd90 0x7f6b440c8d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b440d2df0 con 0x7f6b440d4720 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b4415e170 con 0x7f6b440d29d0 2024-03-04T07:29:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b4c579640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b440b7250 con 0x7f6b4415dd90 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b49b14640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b440d4720 0x7f6b440c8800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b49b14640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b440d4720 0x7f6b440c8800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37156/0 (socket says 172.21.15.64:37156) 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.575+0000 7f6b49b14640 1 -- 172.21.15.64:0/4288850283 learned_addr learned my addr 172.21.15.64:0/4288850283 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b4a315640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b440d29d0 0x7f6b440cc3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b4ab16640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b4415dd90 0x7f6b440c8d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b49b14640 1 -- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b4415dd90 msgr2=0x7f6b440c8d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b49b14640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b4415dd90 0x7f6b440c8d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b49b14640 1 -- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b440d29d0 msgr2=0x7f6b440cc3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b49b14640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b440d29d0 0x7f6b440cc3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b49b14640 1 -- 172.21.15.64:0/4288850283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b440ccdf0 con 0x7f6b440d4720 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b4a315640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b440d29d0 0x7f6b440cc3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b49b14640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b440d4720 0x7f6b440c8800 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f6b3800c7c0 tx=0x7f6b3800cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b36ffd640 1 -- 172.21.15.64:0/4288850283 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b38015070 con 0x7f6b440d4720 2024-03-04T07:29:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b36ffd640 1 -- 172.21.15.64:0/4288850283 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b380079d0 con 0x7f6b440d4720 2024-03-04T07:29:01.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b440cd0d0 con 0x7f6b440d4720 2024-03-04T07:29:01.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b36ffd640 1 -- 172.21.15.64:0/4288850283 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b38007da0 con 0x7f6b440d4720 2024-03-04T07:29:01.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.576+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b440cd690 con 0x7f6b440d4720 2024-03-04T07:29:01.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.577+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b44072100 con 0x7f6b440d4720 2024-03-04T07:29:01.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.578+0000 7f6b36ffd640 1 -- 172.21.15.64:0/4288850283 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b38018070 con 0x7f6b440d4720 2024-03-04T07:29:01.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.578+0000 7f6b36ffd640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b2806b4d0 0x7f6b2806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.578+0000 7f6b36ffd640 1 -- 172.21.15.64:0/4288850283 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f6b38089d10 con 0x7f6b440d4720 2024-03-04T07:29:01.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.578+0000 7f6b4a315640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b2806b4d0 0x7f6b2806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.578+0000 7f6b4a315640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b2806b4d0 0x7f6b2806d990 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f6b40036000 tx=0x7f6b4003b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:01.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.582+0000 7f6b36ffd640 1 -- 172.21.15.64:0/4288850283 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6b44072100 con 0x7f6b440d4720 2024-03-04T07:29:01.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.762+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f6b44072a30 con 0x7f6b440d4720 2024-03-04T07:29:01.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.765+0000 7f6b36ffd640 1 -- 172.21.15.64:0/4288850283 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v14) v1 ==== 104+0+0 (secure 0 0 0) 0x7f6b38058f20 con 0x7f6b440d4720 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.767+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b2806b4d0 msgr2=0x7f6b2806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.767+0000 7f6b4c579640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b2806b4d0 0x7f6b2806d990 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f6b40036000 tx=0x7f6b4003b040 comp rx=0 tx=0).stop 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.767+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b440d4720 msgr2=0x7f6b440c8800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b440d4720 0x7f6b440c8800 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f6b3800c7c0 tx=0x7f6b3800cc80 comp rx=0 tx=0).stop 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 shutdown_connections 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b2806b4d0 0x7f6b2806d990 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b4415dd90 0x7f6b440c8d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b440d4720 0x7f6b440c8800 unknown :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 --2- 172.21.15.64:0/4288850283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b440d29d0 0x7f6b440cc3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 >> 172.21.15.64:0/4288850283 conn(0x7f6b44071960 msgr2=0x7f6b440d0c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:01.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 shutdown_connections 2024-03-04T07:29:01.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.768+0000 7f6b4c579640 1 -- 172.21.15.64:0/4288850283 wait complete. 2024-03-04T07:29:01.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_fragment_size_max 2024-03-04T07:29:01.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.887+0000 7fd788d5e640 1 Processor -- start 2024-03-04T07:29:01.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.887+0000 7fd788d5e640 1 -- start start 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd788d5e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd78415e000 0x7fd7840d1e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd788d5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 0x7fd7840ceae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd788d5e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd7840d2700 con 0x7fd7840b9710 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd788d5e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7840b7ec0 con 0x7fd78415e000 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd788d5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7840d29e0 con 0x7fd7840d37a0 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd782d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd7840b9710 0x7fd7840b5790 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43668/0 (socket says 172.21.15.64:43668) 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd782d9d640 1 -- 172.21.15.64:0/2421488367 learned_addr learned my addr 172.21.15.64:0/2421488367 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd78259c640 1 --2- 172.21.15.64:0/2421488367 >> v2:172.21.15.66:3300/0 conn(0x7fd78415e000 0x7fd7840d1e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd779d9b640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 0x7fd7840ceae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4091792071 0 0) 0x7fd7840d2700 con 0x7fd7840b9710 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.888+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd770003520 con 0x7fd7840b9710 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1995590174 0 0) 0x7fd770003520 con 0x7fd7840b9710 2024-03-04T07:29:01.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 msgr2=0x7fd7840ceae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 0x7fd7840ceae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 >> v2:172.21.15.66:3300/0 conn(0x7fd78415e000 msgr2=0x7fd7840d1e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 --2- 172.21.15.64:0/2421488367 >> v2:172.21.15.66:3300/0 conn(0x7fd78415e000 0x7fd7840d1e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7840d2cc0 con 0x7fd7840b9710 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd77c0019f0 con 0x7fd7840b9710 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 >> v1:172.21.15.64:6789/0 conn(0x7fd7840b9710 legacy=0x7fd7840b5790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7700039b0 0x7fd770003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 0x7fd770004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd770004d40 0x7fd7700051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7840b7ec0 con 0x7fd7700039b0 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7840d29e0 con 0x7fd770004d40 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd77000b9f0 con 0x7fd7840d37a0 2024-03-04T07:29:01.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd78259c640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7700039b0 0x7fd770003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd779d9b640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 0x7fd770004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd782d9d640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd770004d40 0x7fd7700051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd78259c640 1 -- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 msgr2=0x7fd770004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd78259c640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 0x7fd770004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd78259c640 1 -- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd770004d40 msgr2=0x7fd7700051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd78259c640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd770004d40 0x7fd7700051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd78259c640 1 -- 172.21.15.64:0/2421488367 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd770003520 con 0x7fd7700039b0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.889+0000 7fd779d9b640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7840d37a0 0x7fd770004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd78259c640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7700039b0 0x7fd770003dd0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7fd774009ac0 tx=0x7fd770005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=861e2b71b4f5f86c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd774039070 con 0x7fd7700039b0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd77402c710 con 0x7fd7700039b0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd781d9b640 1 -- 172.21.15.64:0/2421488367 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd774009690 con 0x7fd7700039b0 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd788d5e640 1 -- 172.21.15.64:0/2421488367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7700039b0 msgr2=0x7fd770003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd788d5e640 1 --2- 172.21.15.64:0/2421488367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7700039b0 0x7fd770003dd0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7fd774009ac0 tx=0x7fd770005830 comp rx=0 tx=0).stop 2024-03-04T07:29:01.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd782d9d640 1 -- 172.21.15.64:0/2421488367 reap_dead start 2024-03-04T07:29:01.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd788d5e640 1 -- 172.21.15.64:0/2421488367 shutdown_connections 2024-03-04T07:29:01.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd788d5e640 1 -- 172.21.15.64:0/2421488367 >> 172.21.15.64:0/2421488367 conn(0x7fd784066d00 msgr2=0x7fd784067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:01.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd788d5e640 1 -- 172.21.15.64:0/2421488367 shutdown_connections 2024-03-04T07:29:01.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.890+0000 7fd788d5e640 1 -- 172.21.15.64:0/2421488367 wait complete. 2024-03-04T07:29:01.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.892+0000 7fd788d5e640 1 Processor -- start 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.892+0000 7fd788d5e640 1 -- start start 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd788d5e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7700039b0 0x7fd7841596a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd788d5e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7840c9110 0x7fd7840c9590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd788d5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7841e64c0 0x7fd7841e6940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd788d5e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd78415e3e0 con 0x7fd7840c9110 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd788d5e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7840d3280 con 0x7fd7700039b0 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd788d5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7840d2340 con 0x7fd7841e64c0 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7840c9110 0x7fd7840c9590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7840c9110 0x7fd7840c9590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37162/0 (socket says 172.21.15.64:37162) 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 -- 172.21.15.64:0/3637230189 learned_addr learned my addr 172.21.15.64:0/3637230189 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd78259c640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7700039b0 0x7fd7841596a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd782d9d640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7841e64c0 0x7fd7841e6940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 -- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7841e64c0 msgr2=0x7fd7841e6940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7841e64c0 0x7fd7841e6940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 -- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7700039b0 msgr2=0x7fd7841596a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7700039b0 0x7fd7841596a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 -- 172.21.15.64:0/3637230189 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd784159be0 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd782d9d640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7841e64c0 0x7fd7841e6940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.893+0000 7fd779d9b640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7840c9110 0x7fd7840c9590 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7fd76c0116d0 tx=0x7fd76c011b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.894+0000 7fd77b7fe640 1 -- 172.21.15.64:0/3637230189 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd76c00f9c0 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.894+0000 7fd77b7fe640 1 -- 172.21.15.64:0/3637230189 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd76c00fb60 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.894+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd7840c9c80 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.894+0000 7fd77b7fe640 1 -- 172.21.15.64:0/3637230189 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd76c015460 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.894+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd7841e8a00 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.895+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd784072a30 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.896+0000 7fd77b7fe640 1 -- 172.21.15.64:0/3637230189 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd76c015600 con 0x7fd7840c9110 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.897+0000 7fd77b7fe640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76406b500 0x7fd76406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:01.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.897+0000 7fd77b7fe640 1 -- 172.21.15.64:0/3637230189 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fd76c08a8b0 con 0x7fd7840c9110 2024-03-04T07:29:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.897+0000 7fd78259c640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76406b500 0x7fd76406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.898+0000 7fd78259c640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76406b500 0x7fd76406d9c0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fd774009ac0 tx=0x7fd774030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:01.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:01.903+0000 7fd77b7fe640 1 -- 172.21.15.64:0/3637230189 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd76c019030 con 0x7fd7840c9110 2024-03-04T07:29:02.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.085+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"} v 0) v1 -- 0x7fd7840d01d0 con 0x7fd7840c9110 2024-03-04T07:29:02.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.088+0000 7fd77b7fe640 1 -- 172.21.15.64:0/3637230189 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"}]=0 v15) v1 ==== 108+0+0 (secure 0 0 0) 0x7fd76c059990 con 0x7fd7840c9110 2024-03-04T07:29:02.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76406b500 msgr2=0x7fd76406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:02.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76406b500 0x7fd76406d9c0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fd774009ac0 tx=0x7fd774030000 comp rx=0 tx=0).stop 2024-03-04T07:29:02.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7840c9110 msgr2=0x7fd7840c9590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7840c9110 0x7fd7840c9590 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7fd76c0116d0 tx=0x7fd76c011b90 comp rx=0 tx=0).stop 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 shutdown_connections 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76406b500 0x7fd76406d9c0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7841e64c0 0x7fd7841e6940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7840c9110 0x7fd7840c9590 unknown :-1 s=CLOSED pgs=611 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 --2- 172.21.15.64:0/3637230189 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7700039b0 0x7fd7841596a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 >> 172.21.15.64:0/3637230189 conn(0x7fd784066d00 msgr2=0x7fd78415c7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 shutdown_connections 2024-03-04T07:29:02.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.090+0000 7fd788d5e640 1 -- 172.21.15.64:0/3637230189 wait complete. 2024-03-04T07:29:02.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit' 2024-03-04T07:29:02.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.204+0000 7fc39d78c640 1 Processor -- start 2024-03-04T07:29:02.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.204+0000 7fc39d78c640 1 -- start start 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc39d78c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc398071540 0x7fc3980b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc39d78c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc3980d3400 0x7fc398071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc39d78c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc398071e60 con 0x7fc3980b2b50 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc39d78c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc3980d2f70 con 0x7fc3980d3400 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc39d78c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc3980d4be0 con 0x7fc398071540 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc3967fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc3980d3400 0x7fc398071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc3977fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc3980b2b50 0x7fc3980d1870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:43680/0 (socket says 172.21.15.64:43680) 2024-03-04T07:29:02.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc3977fe640 1 -- 172.21.15.64:0/1842076960 learned_addr learned my addr 172.21.15.64:0/1842076960 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc3967fc640 1 -- 172.21.15.64:0/1842076960 >> v2:172.21.15.66:3300/0 conn(0x7fc398071540 msgr2=0x7fc3980b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc3967fc640 1 --2- 172.21.15.64:0/1842076960 >> v2:172.21.15.66:3300/0 conn(0x7fc398071540 0x7fc3980b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc3967fc640 1 -- 172.21.15.64:0/1842076960 >> v1:172.21.15.64:6789/0 conn(0x7fc3980b2b50 legacy=0x7fc3980d1870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc395ffb640 1 -- 172.21.15.64:0/1842076960 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1256938080 0 0) 0x7fc398071e60 con 0x7fc3980b2b50 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.205+0000 7fc3967fc640 1 -- 172.21.15.64:0/1842076960 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3980ce610 con 0x7fc3980d3400 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.206+0000 7fc3967fc640 1 --2- 172.21.15.64:0/1842076960 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3980d3400 0x7fc398071920 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7fc3980d4da0 tx=0x7fc38002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=16ad5f6845f0669f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.206+0000 7fc395ffb640 1 -- 172.21.15.64:0/1842076960 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc380035020 con 0x7fc3980d3400 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.206+0000 7fc395ffb640 1 -- 172.21.15.64:0/1842076960 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc380031830 con 0x7fc3980d3400 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.206+0000 7fc395ffb640 1 -- 172.21.15.64:0/1842076960 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc380031c00 con 0x7fc3980d3400 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 -- 172.21.15.64:0/1842076960 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3980d3400 msgr2=0x7fc398071920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 --2- 172.21.15.64:0/1842076960 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3980d3400 0x7fc398071920 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7fc3980d4da0 tx=0x7fc38002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 -- 172.21.15.64:0/1842076960 shutdown_connections 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 --2- 172.21.15.64:0/1842076960 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3980d3400 0x7fc398071920 unknown :-1 s=CLOSED pgs=613 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 --2- 172.21.15.64:0/1842076960 >> v2:172.21.15.66:3300/0 conn(0x7fc398071540 0x7fc3980b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 -- 172.21.15.64:0/1842076960 >> 172.21.15.64:0/1842076960 conn(0x7fc3980d3ca0 msgr2=0x7fc3980d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 -- 172.21.15.64:0/1842076960 shutdown_connections 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.207+0000 7fc39d78c640 1 -- 172.21.15.64:0/1842076960 wait complete. 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.208+0000 7fc39d78c640 1 Processor -- start 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.208+0000 7fc39d78c640 1 -- start start 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc398071540 0x7fc398156fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3980b2b50 0x7fc3980c85e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3980d3400 0x7fc398152f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3980ce8f0 con 0x7fc398071540 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3980d2c00 con 0x7fc3980b2b50 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc39815e370 con 0x7fc3980d3400 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc398071540 0x7fc398156fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc398071540 0x7fc398156fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37192/0 (socket says 172.21.15.64:37192) 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 -- 172.21.15.64:0/2915502597 learned_addr learned my addr 172.21.15.64:0/2915502597 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 -- 172.21.15.64:0/2915502597 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3980d3400 msgr2=0x7fc398152f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3980d3400 0x7fc398152f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 -- 172.21.15.64:0/2915502597 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3980b2b50 msgr2=0x7fc3980c85e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3980b2b50 0x7fc3980c85e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 -- 172.21.15.64:0/2915502597 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3980b3be0 con 0x7fc398071540 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc396ffd640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc398071540 0x7fc398156fa0 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7fc38800ba80 tx=0x7fc38800bf40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc38f7fe640 1 -- 172.21.15.64:0/2915502597 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc388019070 con 0x7fc398071540 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc38f7fe640 1 -- 172.21.15.64:0/2915502597 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc38800cc20 con 0x7fc398071540 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3980b3ec0 con 0x7fc398071540 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc38f7fe640 1 -- 172.21.15.64:0/2915502597 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc388014440 con 0x7fc398071540 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.209+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3980b33d0 con 0x7fc398071540 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.210+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc398068700 con 0x7fc398071540 2024-03-04T07:29:02.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.211+0000 7fc38f7fe640 1 -- 172.21.15.64:0/2915502597 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc38801c070 con 0x7fc398071540 2024-03-04T07:29:02.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.211+0000 7fc38f7fe640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc37806b4d0 0x7fc37806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:02.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.211+0000 7fc38f7fe640 1 -- 172.21.15.64:0/2915502597 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fc388089bc0 con 0x7fc398071540 2024-03-04T07:29:02.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.211+0000 7fc3967fc640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc37806b4d0 0x7fc37806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:02.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.212+0000 7fc3967fc640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc37806b4d0 0x7fc37806d990 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7fc380036000 tx=0x7fc380030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:02.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.215+0000 7fc38f7fe640 1 -- 172.21.15.64:0/2915502597 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc3880586f0 con 0x7fc398071540 2024-03-04T07:29:02.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:02.470+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]} v 0) v1 -- 0x7fc39815fd50 con 0x7fc398071540 2024-03-04T07:29:03.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.454+0000 7fc38f7fe640 1 -- 172.21.15.64:0/2915502597 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit"]}]=0 v133) v1 ==== 133+0+0 (secure 0 0 0) 0x7fc38805c790 con 0x7fc398071540 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc37806b4d0 msgr2=0x7fc37806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc37806b4d0 0x7fc37806d990 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7fc380036000 tx=0x7fc380030000 comp rx=0 tx=0).stop 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc398071540 msgr2=0x7fc398156fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc398071540 0x7fc398156fa0 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7fc38800ba80 tx=0x7fc38800bf40 comp rx=0 tx=0).stop 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 shutdown_connections 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc37806b4d0 0x7fc37806d990 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3980d3400 0x7fc398152f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3980b2b50 0x7fc3980c85e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 --2- 172.21.15.64:0/2915502597 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc398071540 0x7fc398156fa0 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 >> 172.21.15.64:0/2915502597 conn(0x7fc3980d3ca0 msgr2=0x7fc398160d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 shutdown_connections 2024-03-04T07:29:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.457+0000 7fc39d78c640 1 -- 172.21.15.64:0/2915502597 wait complete. 2024-03-04T07:29:03.470 INFO:tasks.cephfs_test_runner:That the directory fragment size cannot exceed mds_bal_fragment_size_max (using a limit of 50 in all configurations). ... ok 2024-03-04T07:29:03.470 INFO:tasks.cephfs_test_runner:Starting test: test_dirfrag_limit_fragmented (tasks.cephfs.test_strays.TestStrays) That fragmentation (forced) will allow more entries to be created. 2024-03-04T07:29:03.470 INFO:tasks.cephfs_test_runner:test_dirfrag_limit_fragmented (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:29:03.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:03.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.579+0000 7faedd639640 1 Processor -- start 2024-03-04T07:29:03.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.579+0000 7faedd639640 1 -- start start 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faedd639640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7faed815df00 0x7faed80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faedd639640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faed80b95a0 0x7faed80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faedd639640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7faed80cea30 con 0x7faed80b9a30 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faedd639640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faed80d2af0 con 0x7faed80b95a0 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faedd639640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faed80cf3c0 con 0x7faed815df00 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed67fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faed80b95a0 0x7faed80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed67fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faed80b95a0 0x7faed80b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34344/0 (socket says 172.21.15.64:34344) 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7faed80b9a30 0x7faed80d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51246/0 (socket says 172.21.15.64:51246) 2024-03-04T07:29:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed67fc640 1 -- 172.21.15.64:0/679962341 learned_addr learned my addr 172.21.15.64:0/679962341 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed6ffd640 1 --2- 172.21.15.64:0/679962341 >> v2:172.21.15.66:3300/0 conn(0x7faed815df00 0x7faed80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed67fc640 1 -- 172.21.15.64:0/679962341 >> v2:172.21.15.66:3300/0 conn(0x7faed815df00 msgr2=0x7faed80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed67fc640 1 --2- 172.21.15.64:0/679962341 >> v2:172.21.15.66:3300/0 conn(0x7faed815df00 0x7faed80d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed67fc640 1 -- 172.21.15.64:0/679962341 >> v1:172.21.15.64:6789/0 conn(0x7faed80b9a30 legacy=0x7faed80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed67fc640 1 -- 172.21.15.64:0/679962341 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faed80cf0e0 con 0x7faed80b95a0 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.580+0000 7faed5ffb640 1 -- 172.21.15.64:0/679962341 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 176247312 0 0) 0x7faed80cea30 con 0x7faed80b9a30 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.581+0000 7faed67fc640 1 --2- 172.21.15.64:0/679962341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed80b95a0 0x7faed80b5700 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7faed80d11b0 tx=0x7faecc02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b95046b625669f88 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.581+0000 7faed5ffb640 1 -- 172.21.15.64:0/679962341 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faecc034070 con 0x7faed80b95a0 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.581+0000 7faed5ffb640 1 -- 172.21.15.64:0/679962341 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faecc030860 con 0x7faed80b95a0 2024-03-04T07:29:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.581+0000 7faed5ffb640 1 -- 172.21.15.64:0/679962341 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faecc030c30 con 0x7faed80b95a0 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 -- 172.21.15.64:0/679962341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed80b95a0 msgr2=0x7faed80b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 --2- 172.21.15.64:0/679962341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed80b95a0 0x7faed80b5700 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7faed80d11b0 tx=0x7faecc02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 -- 172.21.15.64:0/679962341 shutdown_connections 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 --2- 172.21.15.64:0/679962341 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed80b95a0 0x7faed80b5700 unknown :-1 s=CLOSED pgs=616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 --2- 172.21.15.64:0/679962341 >> v2:172.21.15.66:3300/0 conn(0x7faed815df00 0x7faed80d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 -- 172.21.15.64:0/679962341 >> 172.21.15.64:0/679962341 conn(0x7faed8071960 msgr2=0x7faed8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 -- 172.21.15.64:0/679962341 shutdown_connections 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.582+0000 7faedd639640 1 -- 172.21.15.64:0/679962341 wait complete. 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.583+0000 7faedd639640 1 Processor -- start 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.583+0000 7faedd639640 1 -- start start 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faedd639640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faed80b95a0 0x7faed815a860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faedd639640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed815df00 0x7faed81f78d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faedd639640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faed815ae40 0x7faed8161060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faedd639640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faed80cec70 con 0x7faed815df00 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faedd639640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faed80ce750 con 0x7faed815ae40 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faedd639640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faed80cedf0 con 0x7faed80b95a0 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed67fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed815df00 0x7faed81f78d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed67fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed815df00 0x7faed81f78d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34354/0 (socket says 172.21.15.64:34354) 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed67fc640 1 -- 172.21.15.64:0/2664132583 learned_addr learned my addr 172.21.15.64:0/2664132583 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed77fe640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faed815ae40 0x7faed8161060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed6ffd640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faed80b95a0 0x7faed815a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed67fc640 1 -- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faed80b95a0 msgr2=0x7faed815a860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed67fc640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faed80b95a0 0x7faed815a860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.584+0000 7faed67fc640 1 -- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faed815ae40 msgr2=0x7faed8161060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faed67fc640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faed815ae40 0x7faed8161060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faed67fc640 1 -- 172.21.15.64:0/2664132583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faed81f6e30 con 0x7faed815df00 2024-03-04T07:29:03.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faed77fe640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faed815ae40 0x7faed8161060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faed6ffd640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faed80b95a0 0x7faed815a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faed67fc640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed815df00 0x7faed81f78d0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7faecc02b9e0 tx=0x7faecc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faebf7fe640 1 -- 172.21.15.64:0/2664132583 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faecc048070 con 0x7faed815df00 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faebf7fe640 1 -- 172.21.15.64:0/2664132583 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faecc030660 con 0x7faed815df00 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faebf7fe640 1 -- 172.21.15.64:0/2664132583 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faecc034040 con 0x7faed815df00 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faed81e8db0 con 0x7faed815df00 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.585+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faed81f73b0 con 0x7faed815df00 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.586+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faed8067ff0 con 0x7faed815df00 2024-03-04T07:29:03.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.586+0000 7faebf7fe640 1 -- 172.21.15.64:0/2664132583 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faecc037070 con 0x7faed815df00 2024-03-04T07:29:03.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.587+0000 7faebf7fe640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.587+0000 7faebf7fe640 1 -- 172.21.15.64:0/2664132583 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7faecc0abdd0 con 0x7faed815df00 2024-03-04T07:29:03.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.587+0000 7faed6ffd640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.587+0000 7faed6ffd640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7faec8009f50 tx=0x7faec800b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:03.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.591+0000 7faebf7fe640 1 -- 172.21.15.64:0/2664132583 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faecc07b070 con 0x7faed815df00 2024-03-04T07:29:03.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.773+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7faed80ce0a0 con 0x7faed815df00 2024-03-04T07:29:03.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.774+0000 7faebf7fe640 1 -- 172.21.15.64:0/2664132583 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v47) v1 ==== 74+0+12972 (secure 0 0 0) 0x7faecc0ad090 con 0x7faed815df00 2024-03-04T07:29:03.773 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:03.774 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":47,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:01.440258+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6836/3981471748":"2024-03-05T07:28:58.141833+0000","172.21.15.64:6834/3981471748":"2024-03-05T07:28:58.141833+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:03.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 msgr2=0x7faec406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7faec8009f50 tx=0x7faec800b040 comp rx=0 tx=0).stop 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed815df00 msgr2=0x7faed81f78d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed815df00 0x7faed81f78d0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7faecc02b9e0 tx=0x7faecc004730 comp rx=0 tx=0).stop 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 shutdown_connections 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faed815ae40 0x7faed8161060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faed815df00 0x7faed81f78d0 unknown :-1 s=CLOSED pgs=617 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 --2- 172.21.15.64:0/2664132583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faed80b95a0 0x7faed815a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 >> 172.21.15.64:0/2664132583 conn(0x7faed8071960 msgr2=0x7faed80b8730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 shutdown_connections 2024-03-04T07:29:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.776+0000 7faedd639640 1 -- 172.21.15.64:0/2664132583 wait complete. 2024-03-04T07:29:03.787 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:29:03.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:29:03.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.892+0000 7f931b53e640 1 Processor -- start 2024-03-04T07:29:03.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.892+0000 7f931b53e640 1 -- start start 2024-03-04T07:29:03.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f931b53e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f931415f5c0 0x7f93140cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f931b53e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f93140cff40 0x7f93140d03d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f931b53e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f93140b33b0 con 0x7f93140b1470 2024-03-04T07:29:03.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f931b53e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f93140b0af0 con 0x7f93140cff40 2024-03-04T07:29:03.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f931b53e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f93140b0d40 con 0x7f931415f5c0 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9318ad9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f93140cff40 0x7f93140d03d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9319adb640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f93140b1470 0x7f93140b18a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51262/0 (socket says 172.21.15.64:51262) 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9318ad9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f93140cff40 0x7f93140d03d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34358/0 (socket says 172.21.15.64:34358) 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9319adb640 1 -- 172.21.15.64:0/1605699792 learned_addr learned my addr 172.21.15.64:0/1605699792 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f93192da640 1 --2- 172.21.15.64:0/1605699792 >> v2:172.21.15.66:3300/0 conn(0x7f931415f5c0 0x7f93140cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9318ad9640 1 -- 172.21.15.64:0/1605699792 >> v2:172.21.15.66:3300/0 conn(0x7f931415f5c0 msgr2=0x7f93140cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9318ad9640 1 --2- 172.21.15.64:0/1605699792 >> v2:172.21.15.66:3300/0 conn(0x7f931415f5c0 0x7f93140cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9318ad9640 1 -- 172.21.15.64:0/1605699792 >> v1:172.21.15.64:6789/0 conn(0x7f93140b1470 legacy=0x7f93140b18a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f9318ad9640 1 -- 172.21.15.64:0/1605699792 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f93140b1020 con 0x7f93140cff40 2024-03-04T07:29:03.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.893+0000 7f930bfff640 1 -- 172.21.15.64:0/1605699792 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 440583489 0 0) 0x7f93140b33b0 con 0x7f93140b1470 2024-03-04T07:29:03.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.894+0000 7f9318ad9640 1 --2- 172.21.15.64:0/1605699792 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140cff40 0x7f93140d03d0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f9314071e70 tx=0x7f930402b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=79d0a4bf7a9f630d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.894+0000 7f930bfff640 1 -- 172.21.15.64:0/1605699792 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f93040315a0 con 0x7f93140cff40 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.894+0000 7f930bfff640 1 -- 172.21.15.64:0/1605699792 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9304037070 con 0x7f93140cff40 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.894+0000 7f930bfff640 1 -- 172.21.15.64:0/1605699792 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9304031960 con 0x7f93140cff40 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 -- 172.21.15.64:0/1605699792 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140cff40 msgr2=0x7f93140d03d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 --2- 172.21.15.64:0/1605699792 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140cff40 0x7f93140d03d0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f9314071e70 tx=0x7f930402b840 comp rx=0 tx=0).stop 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 -- 172.21.15.64:0/1605699792 shutdown_connections 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 --2- 172.21.15.64:0/1605699792 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140cff40 0x7f93140d03d0 unknown :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 --2- 172.21.15.64:0/1605699792 >> v2:172.21.15.66:3300/0 conn(0x7f931415f5c0 0x7f93140cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 -- 172.21.15.64:0/1605699792 >> 172.21.15.64:0/1605699792 conn(0x7f93140b36d0 msgr2=0x7f93140b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 -- 172.21.15.64:0/1605699792 shutdown_connections 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.895+0000 7f931b53e640 1 -- 172.21.15.64:0/1605699792 wait complete. 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.896+0000 7f931b53e640 1 Processor -- start 2024-03-04T07:29:03.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.896+0000 7f931b53e640 1 -- start start 2024-03-04T07:29:03.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f931b53e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140b1470 0x7f9314151f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f931b53e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93140cff40 0x7f93140cacc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f931b53e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f931415f5c0 0x7f93140cb200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f931b53e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f931415f9a0 con 0x7f93140b1470 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f931b53e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93140ced80 con 0x7f93140cff40 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f931b53e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f93140cf2a0 con 0x7f931415f5c0 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140b1470 0x7f9314151f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140b1470 0x7f9314151f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34360/0 (socket says 172.21.15.64:34360) 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 -- 172.21.15.64:0/748565571 learned_addr learned my addr 172.21.15.64:0/748565571 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 -- 172.21.15.64:0/748565571 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f931415f5c0 msgr2=0x7f93140cb200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f931415f5c0 0x7f93140cb200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 -- 172.21.15.64:0/748565571 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93140cff40 msgr2=0x7f93140cacc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93140cff40 0x7f93140cacc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 -- 172.21.15.64:0/748565571 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f93140b5420 con 0x7f93140b1470 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f93192da640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140b1470 0x7f9314151f10 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f930c00d830 tx=0x7f930c00dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f9309ffb640 1 -- 172.21.15.64:0/748565571 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f930c00b6c0 con 0x7f93140b1470 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f9309ffb640 1 -- 172.21.15.64:0/748565571 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f930c019070 con 0x7f93140b1470 2024-03-04T07:29:03.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.897+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f93140b5700 con 0x7f93140b1470 2024-03-04T07:29:03.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.898+0000 7f9309ffb640 1 -- 172.21.15.64:0/748565571 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f930c00ba60 con 0x7f93140b1470 2024-03-04T07:29:03.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.898+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f93140b5cc0 con 0x7f93140b1470 2024-03-04T07:29:03.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.899+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f93140d16a0 con 0x7f93140b1470 2024-03-04T07:29:03.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.899+0000 7f9309ffb640 1 -- 172.21.15.64:0/748565571 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f930c01e460 con 0x7f93140b1470 2024-03-04T07:29:03.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.899+0000 7f9309ffb640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f92f806b4d0 0x7f92f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:03.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.900+0000 7f9309ffb640 1 -- 172.21.15.64:0/748565571 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f930c08a580 con 0x7f93140b1470 2024-03-04T07:29:03.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.900+0000 7f9318ad9640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f92f806b4d0 0x7f92f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:03.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.900+0000 7f9318ad9640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f92f806b4d0 0x7f92f806d990 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f9304036000 tx=0x7f9304030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:03.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:03.904+0000 7f9309ffb640 1 -- 172.21.15.64:0/748565571 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f93140d16a0 con 0x7f93140b1470 2024-03-04T07:29:04.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.084+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f931404f410 con 0x7f93140b1470 2024-03-04T07:29:04.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.084+0000 7f9309ffb640 1 -- 172.21.15.64:0/748565571 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v47) v1 ==== 93+0+10 (secure 0 0 0) 0x7f930c059790 con 0x7f93140b1470 2024-03-04T07:29:04.084 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:29:04.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f92f806b4d0 msgr2=0x7f92f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:04.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f92f806b4d0 0x7f92f806d990 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f9304036000 tx=0x7f9304030000 comp rx=0 tx=0).stop 2024-03-04T07:29:04.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140b1470 msgr2=0x7f9314151f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140b1470 0x7f9314151f10 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f930c00d830 tx=0x7f930c00dcf0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 shutdown_connections 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f92f806b4d0 0x7f92f806d990 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f931415f5c0 0x7f93140cb200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f93140cff40 0x7f93140cacc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 --2- 172.21.15.64:0/748565571 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93140b1470 0x7f9314151f10 unknown :-1 s=CLOSED pgs=620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 >> 172.21.15.64:0/748565571 conn(0x7f93140b36d0 msgr2=0x7f931415e7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.086+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 shutdown_connections 2024-03-04T07:29:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.087+0000 7f931b53e640 1 -- 172.21.15.64:0/748565571 wait complete. 2024-03-04T07:29:04.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented' 2024-03-04T07:29:04.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.199+0000 7f7dc64e1640 1 Processor -- start 2024-03-04T07:29:04.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc64e1640 1 -- start start 2024-03-04T07:29:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc64e1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7dc00b1020 0x7f7dc006af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc64e1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc006ed10 0x7f7dc00cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc64e1640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7dc00cc0e0 con 0x7f7dc00cd6c0 2024-03-04T07:29:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc64e1640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7dc00d28b0 con 0x7f7dc00b1020 2024-03-04T07:29:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc64e1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7dc00cf320 con 0x7f7dc006ed10 2024-03-04T07:29:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dbffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7dc00b1020 0x7f7dc006af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc4a7e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7dc00cd6c0 0x7f7dc015d700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51276/0 (socket says 172.21.15.64:51276) 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.200+0000 7f7dc4a7e640 1 -- 172.21.15.64:0/3645527473 learned_addr learned my addr 172.21.15.64:0/3645527473 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.201+0000 7f7dbf7fe640 1 --2- 172.21.15.64:0/3645527473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc006ed10 0x7f7dc00cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.201+0000 7f7dbffff640 1 -- 172.21.15.64:0/3645527473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc006ed10 msgr2=0x7f7dc00cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.201+0000 7f7dbffff640 1 --2- 172.21.15.64:0/3645527473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc006ed10 0x7f7dc00cd180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.201+0000 7f7dbffff640 1 -- 172.21.15.64:0/3645527473 >> v1:172.21.15.64:6789/0 conn(0x7f7dc00cd6c0 legacy=0x7f7dc015d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.201+0000 7f7dbffff640 1 -- 172.21.15.64:0/3645527473 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7dc00cc3c0 con 0x7f7dc00b1020 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.201+0000 7f7dbeffd640 1 -- 172.21.15.64:0/3645527473 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2783856606 0 0) 0x7f7dc00cc0e0 con 0x7f7dc00cd6c0 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dbffff640 1 --2- 172.21.15.64:0/3645527473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc00b1020 0x7f7dc006af20 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f7db00097a0 tx=0x7f7db002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6ded6f5c469ffd98 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dbeffd640 1 -- 172.21.15.64:0/3645527473 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7db0035020 con 0x7f7dc00b1020 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dbeffd640 1 -- 172.21.15.64:0/3645527473 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7db0031830 con 0x7f7dc00b1020 2024-03-04T07:29:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dbeffd640 1 -- 172.21.15.64:0/3645527473 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7db0031c00 con 0x7f7dc00b1020 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3645527473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc00b1020 msgr2=0x7f7dc006af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3645527473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc00b1020 0x7f7dc006af20 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f7db00097a0 tx=0x7f7db002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3645527473 shutdown_connections 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3645527473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc006ed10 0x7f7dc00cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3645527473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc00b1020 0x7f7dc006af20 unknown :-1 s=CLOSED pgs=622 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3645527473 >> 172.21.15.64:0/3645527473 conn(0x7f7dc00d1a90 msgr2=0x7f7dc00d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3645527473 shutdown_connections 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.202+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3645527473 wait complete. 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.203+0000 7f7dc64e1640 1 Processor -- start 2024-03-04T07:29:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dc64e1640 1 -- start start 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dc64e1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc006ed10 0x7f7dc0156e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dc64e1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7dc00b1020 0x7f7dc00c5c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dc64e1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc00cd6c0 0x7f7dc00c6160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dc64e1640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7dc00ccc60 con 0x7f7dc006ed10 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dc64e1640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7dc00cc980 con 0x7f7dc00b1020 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dc64e1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7dc00cdb10 con 0x7f7dc00cd6c0 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dbffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc006ed10 0x7f7dc0156e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dbffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc006ed10 0x7f7dc0156e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34382/0 (socket says 172.21.15.64:34382) 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.204+0000 7f7dbffff640 1 -- 172.21.15.64:0/3307542133 learned_addr learned my addr 172.21.15.64:0/3307542133 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dc4a7e640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc00cd6c0 0x7f7dc00c6160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbffff640 1 -- 172.21.15.64:0/3307542133 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc00cd6c0 msgr2=0x7f7dc00c6160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbffff640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc00cd6c0 0x7f7dc00c6160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbffff640 1 -- 172.21.15.64:0/3307542133 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7dc00b1020 msgr2=0x7f7dc00c5c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbffff640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7dc00b1020 0x7f7dc00c5c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbffff640 1 -- 172.21.15.64:0/3307542133 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7dc00b2020 con 0x7f7dc006ed10 2024-03-04T07:29:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbffff640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc006ed10 0x7f7dc0156e50 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7f7db0009890 tx=0x7f7db00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbcff9640 1 -- 172.21.15.64:0/3307542133 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7db0041070 con 0x7f7dc006ed10 2024-03-04T07:29:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbcff9640 1 -- 172.21.15.64:0/3307542133 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7db003b920 con 0x7f7dc006ed10 2024-03-04T07:29:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7dc00b2300 con 0x7f7dc006ed10 2024-03-04T07:29:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dbcff9640 1 -- 172.21.15.64:0/3307542133 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7db0035040 con 0x7f7dc006ed10 2024-03-04T07:29:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.205+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7dc00b28c0 con 0x7f7dc006ed10 2024-03-04T07:29:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.206+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7dc0067ff0 con 0x7f7dc006ed10 2024-03-04T07:29:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.206+0000 7f7dbcff9640 1 -- 172.21.15.64:0/3307542133 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7db004c060 con 0x7f7dc006ed10 2024-03-04T07:29:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.207+0000 7f7dbcff9640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7dac06b4d0 0x7f7dac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.207+0000 7f7dbcff9640 1 -- 172.21.15.64:0/3307542133 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f7db00ab920 con 0x7f7dc006ed10 2024-03-04T07:29:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.207+0000 7f7dbf7fe640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7dac06b4d0 0x7f7dac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.207+0000 7f7dbf7fe640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7dac06b4d0 0x7f7dac06d990 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f7dc00d1120 tx=0x7f7db4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:04.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.211+0000 7f7dbcff9640 1 -- 172.21.15.64:0/3307542133 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7db007aab0 con 0x7f7dc006ed10 2024-03-04T07:29:04.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:04.461+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]} v 0) v1 -- 0x7f7dc00d1fe0 con 0x7f7dc006ed10 2024-03-04T07:29:05.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.462+0000 7f7dbcff9640 1 -- 172.21.15.64:0/3307542133 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]}]=0 v135) v1 ==== 147+0+0 (secure 0 0 0) 0x7f7db007a3b0 con 0x7f7dc006ed10 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7dac06b4d0 msgr2=0x7f7dac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7dac06b4d0 0x7f7dac06d990 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f7dc00d1120 tx=0x7f7db4008040 comp rx=0 tx=0).stop 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc006ed10 msgr2=0x7f7dc0156e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc006ed10 0x7f7dc0156e50 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7f7db0009890 tx=0x7f7db00046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 shutdown_connections 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7dac06b4d0 0x7f7dac06d990 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7dc00cd6c0 0x7f7dc00c6160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7dc00b1020 0x7f7dc00c5c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 --2- 172.21.15.64:0/3307542133 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7dc006ed10 0x7f7dc0156e50 unknown :-1 s=CLOSED pgs=623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 >> 172.21.15.64:0/3307542133 conn(0x7f7dc00d1a90 msgr2=0x7f7dc00d1310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 shutdown_connections 2024-03-04T07:29:05.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.465+0000 7f7dc64e1640 1 -- 172.21.15.64:0/3307542133 wait complete. 2024-03-04T07:29:05.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:29:05.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.589+0000 7f5e5a5e2640 1 Processor -- start 2024-03-04T07:29:05.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.589+0000 7f5e5a5e2640 1 -- start start 2024-03-04T07:29:05.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e5a5e2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5e5415dbd0 0x7f5e5415ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e5a5e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 0x7f5e54066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e5a5e2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5e5415f3b0 con 0x7f5e540b5840 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e5a5e2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5e5406f070 con 0x7f5e5415dbd0 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e5a5e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5e54071940 con 0x7f5e540b40c0 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e537fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5e540b5840 0x7f5e540b39b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51286/0 (socket says 172.21.15.64:51286) 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e537fe640 1 -- 172.21.15.64:0/3403321753 learned_addr learned my addr 172.21.15.64:0/3403321753 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 0x7f5e54066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e53fff640 1 --2- 172.21.15.64:0/3403321753 >> v2:172.21.15.66:3300/0 conn(0x7f5e5415dbd0 0x7f5e5415ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3017203810 0 0) 0x7f5e5415f3b0 con 0x7f5e540b5840 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5e3c003520 con 0x7f5e540b5840 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4149335193 0 0) 0x7f5e3c003520 con 0x7f5e540b5840 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 msgr2=0x7f5e54066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:05.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 0x7f5e54066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 >> v2:172.21.15.66:3300/0 conn(0x7f5e5415dbd0 msgr2=0x7f5e5415ee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 --2- 172.21.15.64:0/3403321753 >> v2:172.21.15.66:3300/0 conn(0x7f5e5415dbd0 0x7f5e5415ee70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5e540b5db0 con 0x7f5e540b5840 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f5e48001670 con 0x7f5e540b5840 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.590+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 >> v1:172.21.15.64:6789/0 conn(0x7f5e540b5840 legacy=0x7f5e540b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e52ffd640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 0x7f5e3c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e52ffd640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e3c003f30 0x7f5e3c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e52ffd640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e3c004d40 0x7f5e3c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e5406f070 con 0x7f5e3c004d40 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e54071940 con 0x7f5e3c003f30 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5e3c00b9c0 con 0x7f5e540b40c0 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e3c004d40 0x7f5e3c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e53fff640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 0x7f5e3c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e537fe640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e3c003f30 0x7f5e3c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e58b7f640 1 -- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 msgr2=0x7f5e3c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e540b40c0 0x7f5e3c0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e58b7f640 1 -- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e3c003f30 msgr2=0x7f5e3c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e3c003f30 0x7f5e3c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.591+0000 7f5e58b7f640 1 -- 172.21.15.64:0/3403321753 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5e3c003520 con 0x7f5e3c004d40 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e3c004d40 0x7f5e3c0051c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f5e4c009fc0 tx=0x7f5e3c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=361aa89047f76611 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e4c039070 con 0x7f5e3c004d40 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5e4c02cc00 con 0x7f5e3c004d40 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e52ffd640 1 -- 172.21.15.64:0/3403321753 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e4c009c10 con 0x7f5e3c004d40 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/3403321753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e3c004d40 msgr2=0x7f5e3c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e5a5e2640 1 --2- 172.21.15.64:0/3403321753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e3c004d40 0x7f5e3c0051c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f5e4c009fc0 tx=0x7f5e3c005830 comp rx=0 tx=0).stop 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e58b7f640 1 -- 172.21.15.64:0/3403321753 reap_dead start 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/3403321753 shutdown_connections 2024-03-04T07:29:05.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/3403321753 >> 172.21.15.64:0/3403321753 conn(0x7f5e540b26b0 msgr2=0x7f5e540b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:05.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/3403321753 shutdown_connections 2024-03-04T07:29:05.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.592+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/3403321753 wait complete. 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 Processor -- start 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 -- start start 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e5414b4d0 0x7f5e540ae9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e5415e990 0x7f5e540afb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e540b0050 0x7f5e540ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e540721c0 con 0x7f5e540b0050 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e54071ee0 con 0x7f5e5414b4d0 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.594+0000 7f5e5a5e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5e540726e0 con 0x7f5e5415e990 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e540b0050 0x7f5e540ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e540b0050 0x7f5e540ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34414/0 (socket says 172.21.15.64:34414) 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 -- 172.21.15.64:0/328301015 learned_addr learned my addr 172.21.15.64:0/328301015 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e537fe640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e5415e990 0x7f5e540afb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e53fff640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e5414b4d0 0x7f5e540ae9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 -- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e5415e990 msgr2=0x7f5e540afb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e5415e990 0x7f5e540afb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 -- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e5414b4d0 msgr2=0x7f5e540ae9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e5414b4d0 0x7f5e540ae9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 -- 172.21.15.64:0/328301015 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5e5414b8b0 con 0x7f5e540b0050 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e58b7f640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e540b0050 0x7f5e540ad8b0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f5e4c037040 tx=0x7f5e4c009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e50ff9640 1 -- 172.21.15.64:0/328301015 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e4c039030 con 0x7f5e540b0050 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e50ff9640 1 -- 172.21.15.64:0/328301015 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5e4c03a040 con 0x7f5e540b0050 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5e540aeee0 con 0x7f5e540b0050 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e50ff9640 1 -- 172.21.15.64:0/328301015 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e4c009c10 con 0x7f5e540b0050 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.595+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5e5415f240 con 0x7f5e540b0050 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.596+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5e54073380 con 0x7f5e540b0050 2024-03-04T07:29:05.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.597+0000 7f5e50ff9640 1 -- 172.21.15.64:0/328301015 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5e4c045030 con 0x7f5e540b0050 2024-03-04T07:29:05.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.597+0000 7f5e50ff9640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e4006b4d0 0x7f5e4006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:05.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.597+0000 7f5e50ff9640 1 -- 172.21.15.64:0/328301015 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f5e4c0abb80 con 0x7f5e540b0050 2024-03-04T07:29:05.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.597+0000 7f5e53fff640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e4006b4d0 0x7f5e4006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:05.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.597+0000 7f5e53fff640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e4006b4d0 0x7f5e4006d990 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f5e44009b80 tx=0x7f5e44008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:05.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.601+0000 7f5e50ff9640 1 -- 172.21.15.64:0/328301015 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5e4c04c060 con 0x7f5e540b0050 2024-03-04T07:29:05.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.782+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f5e54161ec0 con 0x7f5e540b0050 2024-03-04T07:29:05.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.787+0000 7f5e50ff9640 1 -- 172.21.15.64:0/328301015 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v16)=0 v16) v1 ==== 122+0+0 (secure 0 0 0) 0x7f5e4c07ad90 con 0x7f5e540b0050 2024-03-04T07:29:05.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e4006b4d0 msgr2=0x7f5e4006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e4006b4d0 0x7f5e4006d990 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f5e44009b80 tx=0x7f5e44008040 comp rx=0 tx=0).stop 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e540b0050 msgr2=0x7f5e540ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e540b0050 0x7f5e540ad8b0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f5e4c037040 tx=0x7f5e4c009750 comp rx=0 tx=0).stop 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 shutdown_connections 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e4006b4d0 0x7f5e4006d990 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e540b0050 0x7f5e540ad8b0 unknown :-1 s=CLOSED pgs=626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e5415e990 0x7f5e540afb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 --2- 172.21.15.64:0/328301015 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e5414b4d0 0x7f5e540ae9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 >> 172.21.15.64:0/328301015 conn(0x7f5e540b26b0 msgr2=0x7f5e540b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 shutdown_connections 2024-03-04T07:29:05.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:05.789+0000 7f5e5a5e2640 1 -- 172.21.15.64:0/328301015 wait complete. 2024-03-04T07:29:05.799 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:29:05.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:29:05.844 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:29:05.896 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:29:05.896 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:05.896 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:05.911 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:29:05.911 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:29:05.912 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:05.930 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:29:05.930 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:05.930 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:05.965 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:29:05.966 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:29:05.966 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:05.987 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:29:05.988 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:29:05.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:06.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.107+0000 7f8183bbd640 1 Processor -- start 2024-03-04T07:29:06.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.107+0000 7f8183bbd640 1 -- start start 2024-03-04T07:29:06.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f8183bbd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f817c06ed10 0x7f817c06af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f8183bbd640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f817c0cc0e0 con 0x7f817c0cd890 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f8183bbd640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f817c0cc3c0 con 0x7f817c0b1020 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f8183bbd640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f817c0d28b0 con 0x7f817c06ed10 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f818215a640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f817c0cd890 0x7f817c15d700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51298/0 (socket says 172.21.15.64:51298) 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f818215a640 1 -- 172.21.15.64:0/3537816718 learned_addr learned my addr 172.21.15.64:0/3537816718 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 289706448 0 0) 0x7f817c0cc0e0 con 0x7f817c0cd890 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8168003520 con 0x7f817c0cd890 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.108+0000 7f8181158640 1 --2- 172.21.15.64:0/3537816718 >> v2:172.21.15.66:3300/0 conn(0x7f817c06ed10 0x7f817c06af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2513788244 0 0) 0x7f8168003520 con 0x7f817c0cd890 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 >> v2:172.21.15.66:3300/0 conn(0x7f817c06ed10 msgr2=0x7f817c06af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 --2- 172.21.15.64:0/3537816718 >> v2:172.21.15.66:3300/0 conn(0x7f817c06ed10 0x7f817c06af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 >> v1:172.21.15.66:6789/0 conn(0x7f817c0b1020 legacy=0x7f817c0cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f817c0cc6a0 con 0x7f817c0cd890 2024-03-04T07:29:06.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f81780019f0 con 0x7f817c0cd890 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 >> v1:172.21.15.64:6789/0 conn(0x7f817c0cd890 legacy=0x7f817c15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680039b0 0x7f8168003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8168004800 0x7f8168004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680051c0 0x7f816800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f817c0d28b0 con 0x7f81680039b0 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f816800c110 con 0x7f81680051c0 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f816800c310 con 0x7f8168004800 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181959640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680039b0 0x7f8168003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f818215a640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680051c0 0x7f816800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181158640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8168004800 0x7f8168004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181959640 1 -- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8168004800 msgr2=0x7f8168004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181959640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8168004800 0x7f8168004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181959640 1 -- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680051c0 msgr2=0x7f816800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:06.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181959640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680051c0 0x7f816800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f818215a640 1 -- 172.21.15.64:0/3537816718 reap_dead start 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181959640 1 -- 172.21.15.64:0/3537816718 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8168003520 con 0x7f81680039b0 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.109+0000 7f8181158640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8168004800 0x7f8168004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.110+0000 7f8181959640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680039b0 0x7f8168003dd0 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f8174001750 tx=0x7f8168005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ca96edbdcb6fb6c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.110+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8174001370 con 0x7f81680039b0 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.110+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81740052d0 con 0x7f81680039b0 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.110+0000 7f8180957640 1 -- 172.21.15.64:0/3537816718 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8174003830 con 0x7f81680039b0 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.111+0000 7f8183bbd640 1 -- 172.21.15.64:0/3537816718 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680039b0 msgr2=0x7f8168003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.111+0000 7f8183bbd640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680039b0 0x7f8168003dd0 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f8174001750 tx=0x7f8168005830 comp rx=0 tx=0).stop 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.111+0000 7f8183bbd640 1 -- 172.21.15.64:0/3537816718 shutdown_connections 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.111+0000 7f8183bbd640 1 --2- 172.21.15.64:0/3537816718 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680039b0 0x7f8168003dd0 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.111+0000 7f8183bbd640 1 -- 172.21.15.64:0/3537816718 >> 172.21.15.64:0/3537816718 conn(0x7f817c0d1a90 msgr2=0x7f817c0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.111+0000 7f8183bbd640 1 -- 172.21.15.64:0/3537816718 shutdown_connections 2024-03-04T07:29:06.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.111+0000 7f8183bbd640 1 -- 172.21.15.64:0/3537816718 wait complete. 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.112+0000 7f8183bbd640 1 Processor -- start 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.112+0000 7f8183bbd640 1 -- start start 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680039b0 0x7f817c0b3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f817c15dba0 0x7f817c15dfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f817c0b6720 0x7f817c0b6ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f817c0cdca0 con 0x7f817c0b6720 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f817c0ccc60 con 0x7f81680039b0 2024-03-04T07:29:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f817c0cde20 con 0x7f817c15dba0 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f817c0b6720 0x7f817c0b6ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f817c0b6720 0x7f817c0b6ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34442/0 (socket says 172.21.15.64:34442) 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 -- 172.21.15.64:0/2692757344 learned_addr learned my addr 172.21.15.64:0/2692757344 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8181158640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f817c15dba0 0x7f817c15dfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8181959640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680039b0 0x7f817c0b3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 -- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f817c15dba0 msgr2=0x7f817c15dfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f817c15dba0 0x7f817c15dfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 -- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680039b0 msgr2=0x7f817c0b3750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680039b0 0x7f817c0b3750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 -- 172.21.15.64:0/2692757344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f817c15e520 con 0x7f817c0b6720 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f818215a640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f817c0b6720 0x7f817c0b6ba0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f817c072150 tx=0x7f8178007b20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f81727fc640 1 -- 172.21.15.64:0/2692757344 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f817800ab60 con 0x7f817c0b6720 2024-03-04T07:29:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f81727fc640 1 -- 172.21.15.64:0/2692757344 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8178019070 con 0x7f817c0b6720 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f81727fc640 1 -- 172.21.15.64:0/2692757344 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81780133f0 con 0x7f817c0b6720 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f817c0b73e0 con 0x7f817c0b6720 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.113+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f817c0c5810 con 0x7f817c0b6720 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.114+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f817c067ff0 con 0x7f817c0b6720 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.115+0000 7f81727fc640 1 -- 172.21.15.64:0/2692757344 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8178013590 con 0x7f817c0b6720 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.115+0000 7f81727fc640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816006b500 0x7f816006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.115+0000 7f81727fc640 1 -- 172.21.15.64:0/2692757344 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f817808a5f0 con 0x7f817c0b6720 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.115+0000 7f8181959640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816006b500 0x7f816006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.116+0000 7f8181959640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816006b500 0x7f816006d9c0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f8174033000 tx=0x7f8174009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:06.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.119+0000 7f81727fc640 1 -- 172.21.15.64:0/2692757344 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f81780596e0 con 0x7f817c0b6720 2024-03-04T07:29:06.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.339+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f817c06f2f0 con 0x7f817c0b6720 2024-03-04T07:29:06.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.339+0000 7f81727fc640 1 -- 172.21.15.64:0/2692757344 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+3693 (secure 0 0 0) 0x7f8178058fe0 con 0x7f817c0b6720 2024-03-04T07:29:06.339 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:06.339 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":24,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5138,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/627869633","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":627869633},{"type":"v1","addr":"172.21.15.66:6834","nonce":627869633}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5141,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2387393562","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2387393562},{"type":"v1","addr":"172.21.15.66:6835","nonce":2387393562}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":19},{"gid":5685,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/817733642","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":817733642},{"type":"v1","addr":"172.21.15.64:6837","nonce":817733642}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":18},{"gid":5973,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/2951924014","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2951924014},{"type":"v1","addr":"172.21.15.64:6836","nonce":2951924014}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":24}],"filesystems":[]} 2024-03-04T07:29:06.339 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 24 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816006b500 msgr2=0x7f816006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816006b500 0x7f816006d9c0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f8174033000 tx=0x7f8174009030 comp rx=0 tx=0).stop 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f817c0b6720 msgr2=0x7f817c0b6ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f817c0b6720 0x7f817c0b6ba0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f817c072150 tx=0x7f8178007b20 comp rx=0 tx=0).stop 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 shutdown_connections 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816006b500 0x7f816006d9c0 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f817c0b6720 0x7f817c0b6ba0 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f817c15dba0 0x7f817c15dfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 --2- 172.21.15.64:0/2692757344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81680039b0 0x7f817c0b3750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 >> 172.21.15.64:0/2692757344 conn(0x7f817c0d1a90 msgr2=0x7f817c0cfc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:06.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 shutdown_connections 2024-03-04T07:29:06.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.341+0000 7f8183bbd640 1 -- 172.21.15.64:0/2692757344 wait complete. 2024-03-04T07:29:06.352 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:29:06.352 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:29:06.352 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:29:06.353 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:06.353 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:29:06.353 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:29:06.353 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:29:06.353 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:06.353 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:29:06.353 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:29:06.353 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:29:06.354 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:06.354 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:29:06.354 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:29:06.354 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:29:06.354 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:06.362 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:29:06.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:29:06.363 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:29:06.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:29:06.365 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:29:06.366 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:29:06.367 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:29:06.367 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:29:06.405 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:29:06.405 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:06.406 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:29:06.406 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:06.409 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:29:06.409 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:06.411 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:29:06.411 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:06.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:29:06.481 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:29:06.483 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:29:06.500 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:29:06.501 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:29:06.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.518+0000 7f01c1ac3640 1 Processor -- start 2024-03-04T07:29:06.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.518+0000 7f01c1ac3640 1 -- start start 2024-03-04T07:29:06.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01c1ac3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f01bc0d02e0 0x7f01bc0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01c1ac3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0d06c0 0x7f01bc0b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01c1ac3640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f01bc0b0dd0 con 0x7f01bc0b4260 2024-03-04T07:29:06.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01c1ac3640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f01bc0b0b80 con 0x7f01bc0d02e0 2024-03-04T07:29:06.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01c1ac3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f01bc0aeec0 con 0x7f01bc0d06c0 2024-03-04T07:29:06.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01bb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f01bc0d02e0 0x7f01bc0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01bb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f01bc0d02e0 0x7f01bc0cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34476/0 (socket says 172.21.15.64:34476) 2024-03-04T07:29:06.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01bb7fe640 1 -- 172.21.15.64:0/3308298071 learned_addr learned my addr 172.21.15.64:0/3308298071 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:06.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.519+0000 7f01bbfff640 1 --2- 172.21.15.64:0/3308298071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0d06c0 0x7f01bc0b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.520+0000 7f01bb7fe640 1 -- 172.21.15.64:0/3308298071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0d06c0 msgr2=0x7f01bc0b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:06.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.520+0000 7f01bb7fe640 1 --2- 172.21.15.64:0/3308298071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0d06c0 0x7f01bc0b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.520+0000 7f01bb7fe640 1 -- 172.21.15.64:0/3308298071 >> v1:172.21.15.66:6789/0 conn(0x7f01bc0b4260 legacy=0x7f01bc0b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.520+0000 7f01bb7fe640 1 -- 172.21.15.64:0/3308298071 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01bc0b10b0 con 0x7f01bc0d02e0 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.520+0000 7f01bb7fe640 1 --2- 172.21.15.64:0/3308298071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc0d02e0 0x7f01bc0cfa00 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f01b0009870 tx=0x7f01b002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8de6494f5c173c94 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01ba7fc640 1 -- 172.21.15.64:0/3308298071 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01b002f840 con 0x7f01bc0d02e0 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01ba7fc640 1 -- 172.21.15.64:0/3308298071 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f01b002f9e0 con 0x7f01bc0d02e0 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01ba7fc640 1 -- 172.21.15.64:0/3308298071 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01b002fdb0 con 0x7f01bc0d02e0 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 -- 172.21.15.64:0/3308298071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc0d02e0 msgr2=0x7f01bc0cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/3308298071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc0d02e0 0x7f01bc0cfa00 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f01b0009870 tx=0x7f01b002ba20 comp rx=0 tx=0).stop 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 -- 172.21.15.64:0/3308298071 shutdown_connections 2024-03-04T07:29:06.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/3308298071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0d06c0 0x7f01bc0b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/3308298071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc0d02e0 0x7f01bc0cfa00 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 -- 172.21.15.64:0/3308298071 >> 172.21.15.64:0/3308298071 conn(0x7f01bc0b36d0 msgr2=0x7f01bc0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:06.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 -- 172.21.15.64:0/3308298071 shutdown_connections 2024-03-04T07:29:06.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.521+0000 7f01c1ac3640 1 -- 172.21.15.64:0/3308298071 wait complete. 2024-03-04T07:29:06.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.522+0000 7f01c1ac3640 1 Processor -- start 2024-03-04T07:29:06.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.522+0000 7f01c1ac3640 1 -- start start 2024-03-04T07:29:06.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01c1ac3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0b4260 0x7f01bc144680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01c1ac3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01bc0d06c0 0x7f01bc0c7f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01c1ac3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc143d60 0x7f01bc1554a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01c1ac3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01bc0b22d0 con 0x7f01bc143d60 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01c1ac3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01bc0b2150 con 0x7f01bc0d06c0 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01c1ac3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f01bc0af350 con 0x7f01bc0b4260 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc143d60 0x7f01bc1554a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc143d60 0x7f01bc1554a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34490/0 (socket says 172.21.15.64:34490) 2024-03-04T07:29:06.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 -- 172.21.15.64:0/200376987 learned_addr learned my addr 172.21.15.64:0/200376987 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bb7fe640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0b4260 0x7f01bc144680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 -- 172.21.15.64:0/200376987 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0b4260 msgr2=0x7f01bc144680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0b4260 0x7f01bc144680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 -- 172.21.15.64:0/200376987 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01bc0d06c0 msgr2=0x7f01bc0c7f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01bc0d06c0 0x7f01bc0c7f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.523+0000 7f01bbfff640 1 -- 172.21.15.64:0/200376987 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01bc142f40 con 0x7f01bc143d60 2024-03-04T07:29:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.524+0000 7f01bbfff640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc143d60 0x7f01bc1554a0 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f01b400efc0 tx=0x7f01b400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.524+0000 7f01a3fff640 1 -- 172.21.15.64:0/200376987 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01b4018020 con 0x7f01bc143d60 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.524+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01bc142980 con 0x7f01bc143d60 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.524+0000 7f01a3fff640 1 -- 172.21.15.64:0/200376987 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f01b4014480 con 0x7f01bc143d60 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.524+0000 7f01a3fff640 1 -- 172.21.15.64:0/200376987 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01b4014870 con 0x7f01bc143d60 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.524+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f01bc154ca0 con 0x7f01bc143d60 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.525+0000 7f01a3fff640 1 -- 172.21.15.64:0/200376987 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f01b4005d10 con 0x7f01bc143d60 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.525+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f01bc068700 con 0x7f01bc143d60 2024-03-04T07:29:06.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.526+0000 7f01a3fff640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01a806b4d0 0x7f01a806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:06.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.526+0000 7f01a3fff640 1 -- 172.21.15.64:0/200376987 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f01b408ad20 con 0x7f01bc143d60 2024-03-04T07:29:06.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.526+0000 7f01bb7fe640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01a806b4d0 0x7f01a806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:06.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.526+0000 7f01bb7fe640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01a806b4d0 0x7f01a806d990 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f01b0036000 tx=0x7f01b0039040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:06.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.530+0000 7f01a3fff640 1 -- 172.21.15.64:0/200376987 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f01b4022030 con 0x7f01bc143d60 2024-03-04T07:29:06.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:06.722+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f01bc0cf0b0 con 0x7f01bc143d60 2024-03-04T07:29:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.481+0000 7f01a3fff640 1 -- 172.21.15.64:0/200376987 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v49) v1 ==== 97+0+0 (secure 0 0 0) 0x7f01b4010040 con 0x7f01bc143d60 2024-03-04T07:29:07.481 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:29:07.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01a806b4d0 msgr2=0x7f01a806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01a806b4d0 0x7f01a806d990 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f01b0036000 tx=0x7f01b0039040 comp rx=0 tx=0).stop 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc143d60 msgr2=0x7f01bc1554a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc143d60 0x7f01bc1554a0 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f01b400efc0 tx=0x7f01b400c490 comp rx=0 tx=0).stop 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 shutdown_connections 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01a806b4d0 0x7f01a806d990 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01bc143d60 0x7f01bc1554a0 unknown :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01bc0d06c0 0x7f01bc0c7f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 --2- 172.21.15.64:0/200376987 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01bc0b4260 0x7f01bc144680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.483+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 >> 172.21.15.64:0/200376987 conn(0x7f01bc0b36d0 msgr2=0x7f01bc0b12f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.484+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 shutdown_connections 2024-03-04T07:29:07.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.484+0000 7f01c1ac3640 1 -- 172.21.15.64:0/200376987 wait complete. 2024-03-04T07:29:07.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:29:07.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.602+0000 7fd833e84640 1 Processor -- start 2024-03-04T07:29:07.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.603+0000 7fd833e84640 1 -- start start 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.603+0000 7fd833e84640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd82c15df00 0x7fd82c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.603+0000 7fd833e84640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd82c0cea30 con 0x7fd82c0b9a30 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.603+0000 7fd833e84640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd82c0d2af0 con 0x7fd82c0b95a0 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.603+0000 7fd833e84640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd82c0cf3c0 con 0x7fd82c15df00 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.603+0000 7fd832421640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd82c0b9a30 0x7fd82c0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51326/0 (socket says 172.21.15.64:51326) 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.603+0000 7fd832421640 1 -- 172.21.15.64:0/3628554869 learned_addr learned my addr 172.21.15.64:0/3628554869 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2910035988 0 0) 0x7fd82c0cea30 con 0x7fd82c0b9a30 2024-03-04T07:29:07.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd818003520 con 0x7fd82c0b9a30 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd831c20640 1 --2- 172.21.15.64:0/3628554869 >> v2:172.21.15.66:3300/0 conn(0x7fd82c15df00 0x7fd82c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 481321345 0 0) 0x7fd818003520 con 0x7fd82c0b9a30 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 >> v2:172.21.15.66:3300/0 conn(0x7fd82c15df00 msgr2=0x7fd82c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 --2- 172.21.15.64:0/3628554869 >> v2:172.21.15.66:3300/0 conn(0x7fd82c15df00 0x7fd82c0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 >> v1:172.21.15.66:6789/0 conn(0x7fd82c0b95a0 legacy=0x7fd82c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd82c0cf0e0 con 0x7fd82c0b9a30 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd8240019f0 con 0x7fd82c0b9a30 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 >> v1:172.21.15.64:6789/0 conn(0x7fd82c0b9a30 legacy=0x7fd82c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180039b0 0x7fd818003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd818004800 0x7fd818004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8180051c0 0x7fd81800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd82c0cf3c0 con 0x7fd8180051c0 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd81800c110 con 0x7fd818004800 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd81800c310 con 0x7fd8180039b0 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd832421640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8180051c0 0x7fd81800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd83141f640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd818004800 0x7fd818004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd831c20640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180039b0 0x7fd818003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd832421640 1 -- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180039b0 msgr2=0x7fd818003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd832421640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180039b0 0x7fd818003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd832421640 1 -- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd818004800 msgr2=0x7fd818004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd832421640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd818004800 0x7fd818004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.604+0000 7fd832421640 1 -- 172.21.15.64:0/3628554869 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd818003520 con 0x7fd8180051c0 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.605+0000 7fd832421640 1 -- 172.21.15.64:0/3628554869 reap_dead start 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.605+0000 7fd832421640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8180051c0 0x7fd81800b9c0 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7fd82400a840 tx=0x7fd818005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=58479a53a8acf96f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.605+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd82400a460 con 0x7fd8180051c0 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.605+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd824034650 con 0x7fd8180051c0 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.605+0000 7fd830c1e640 1 -- 172.21.15.64:0/3628554869 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd824034a40 con 0x7fd8180051c0 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.606+0000 7fd833e84640 1 -- 172.21.15.64:0/3628554869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8180051c0 msgr2=0x7fd81800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.606+0000 7fd833e84640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8180051c0 0x7fd81800b9c0 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7fd82400a840 tx=0x7fd818005830 comp rx=0 tx=0).stop 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.606+0000 7fd833e84640 1 -- 172.21.15.64:0/3628554869 shutdown_connections 2024-03-04T07:29:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.606+0000 7fd833e84640 1 --2- 172.21.15.64:0/3628554869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd8180051c0 0x7fd81800b9c0 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.606+0000 7fd833e84640 1 -- 172.21.15.64:0/3628554869 >> 172.21.15.64:0/3628554869 conn(0x7fd82c071960 msgr2=0x7fd82c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:07.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.606+0000 7fd833e84640 1 -- 172.21.15.64:0/3628554869 shutdown_connections 2024-03-04T07:29:07.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.606+0000 7fd833e84640 1 -- 172.21.15.64:0/3628554869 wait complete. 2024-03-04T07:29:07.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.607+0000 7fd833e84640 1 Processor -- start 2024-03-04T07:29:07.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.607+0000 7fd833e84640 1 -- start start 2024-03-04T07:29:07.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd833e84640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180051c0 0x7fd82c15e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd833e84640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd82c15b040 0x7fd82c1f2800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd833e84640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd82c1f2d40 0x7fd82c15ffe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd833e84640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd82c0cec70 con 0x7fd82c1f2d40 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd833e84640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd82c0ce750 con 0x7fd82c15b040 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd833e84640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd82c0cedf0 con 0x7fd8180051c0 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd82c1f2d40 0x7fd82c15ffe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd82c1f2d40 0x7fd82c15ffe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34508/0 (socket says 172.21.15.64:34508) 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 -- 172.21.15.64:0/3969106008 learned_addr learned my addr 172.21.15.64:0/3969106008 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd83141f640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd82c15b040 0x7fd82c1f2800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd831c20640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180051c0 0x7fd82c15e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 -- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180051c0 msgr2=0x7fd82c15e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180051c0 0x7fd82c15e2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 -- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd82c15b040 msgr2=0x7fd82c1f2800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd82c15b040 0x7fd82c1f2800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 -- 172.21.15.64:0/3969106008 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd82c14fc40 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.608+0000 7fd832421640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd82c1f2d40 0x7fd82c15ffe0 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7fd82400a930 tx=0x7fd82c0d0620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.609+0000 7fd81e7fc640 1 -- 172.21.15.64:0/3969106008 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd824034800 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.609+0000 7fd81e7fc640 1 -- 172.21.15.64:0/3969106008 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd824045070 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.609+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd82c15ea30 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.609+0000 7fd81e7fc640 1 -- 172.21.15.64:0/3969106008 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd824034ba0 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.609+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd82c0c9590 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.610+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd82c067ff0 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.610+0000 7fd81e7fc640 1 -- 172.21.15.64:0/3969106008 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd82404a460 con 0x7fd82c1f2d40 2024-03-04T07:29:07.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.610+0000 7fd81e7fc640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd81006b4d0 0x7fd81006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.610+0000 7fd81e7fc640 1 -- 172.21.15.64:0/3969106008 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(51..51 src has 1..51) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fd8240ac5a0 con 0x7fd82c1f2d40 2024-03-04T07:29:07.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.610+0000 7fd831c20640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd81006b4d0 0x7fd81006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.611+0000 7fd831c20640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd81006b4d0 0x7fd81006d990 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fd820006fd0 tx=0x7fd820008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:07.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.615+0000 7fd81e7fc640 1 -- 172.21.15.64:0/3969106008 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd82407b890 con 0x7fd82c1f2d40 2024-03-04T07:29:07.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.864+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7fd82c0720f0 con 0x7fd82c1f2d40 2024-03-04T07:29:07.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.864+0000 7fd81e7fc640 1 -- 172.21.15.64:0/3969106008 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v16) v1 ==== 80+0+5304 (secure 0 0 0) 0x7fd82407b190 con 0x7fd82c1f2d40 2024-03-04T07:29:07.864 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:07.864 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:29:07.865 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.866 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.868 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.869 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.870 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:29:07.872 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.873 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow r", 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:29:07.875 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.876 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:29:07.877 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:29:07.878 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:29:07.880 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:29:07.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd81006b4d0 msgr2=0x7fd81006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd81006b4d0 0x7fd81006d990 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fd820006fd0 tx=0x7fd820008040 comp rx=0 tx=0).stop 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd82c1f2d40 msgr2=0x7fd82c15ffe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd82c1f2d40 0x7fd82c15ffe0 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7fd82400a930 tx=0x7fd82c0d0620 comp rx=0 tx=0).stop 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 shutdown_connections 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd81006b4d0 0x7fd81006d990 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd82c1f2d40 0x7fd82c15ffe0 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd82c15b040 0x7fd82c1f2800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 --2- 172.21.15.64:0/3969106008 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd8180051c0 0x7fd82c15e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 >> 172.21.15.64:0/3969106008 conn(0x7fd82c071960 msgr2=0x7fd82c15d1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 shutdown_connections 2024-03-04T07:29:07.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.866+0000 7fd833e84640 1 -- 172.21.15.64:0/3969106008 wait complete. 2024-03-04T07:29:07.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:07.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.986+0000 7f5bd6581640 1 Processor -- start 2024-03-04T07:29:07.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.986+0000 7f5bd6581640 1 -- start start 2024-03-04T07:29:07.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bd6581640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 0x7f5bd00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bd6581640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5bd00b95a0 0x7f5bd00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bd6581640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5bd00cea30 con 0x7f5bd00b9a30 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bd6581640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5bd00d2af0 con 0x7f5bd00b95a0 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bd6581640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5bd00cf3c0 con 0x7f5bd015df00 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bd4b1e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5bd00b9a30 0x7f5bd00d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51340/0 (socket says 172.21.15.64:51340) 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bd4b1e640 1 -- 172.21.15.64:0/2334510130 learned_addr learned my addr 172.21.15.64:0/2334510130 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2334510130 >> v2:172.21.15.66:3300/0 conn(0x7f5bd00b95a0 0x7f5bd00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4007664515 0 0) 0x7f5bd00cea30 con 0x7f5bd00b9a30 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bcffff640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 0x7f5bd00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5bb8003520 con 0x7f5bd00b9a30 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 854195588 0 0) 0x7f5bb8003520 con 0x7f5bd00b9a30 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 msgr2=0x7f5bd00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 0x7f5bd00d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 >> v2:172.21.15.66:3300/0 conn(0x7f5bd00b95a0 msgr2=0x7f5bd00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:07.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 --2- 172.21.15.64:0/2334510130 >> v2:172.21.15.66:3300/0 conn(0x7f5bd00b95a0 0x7f5bd00b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5bd00cf0e0 con 0x7f5bd00b9a30 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f5bc80019f0 con 0x7f5bd00b9a30 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 >> v1:172.21.15.64:6789/0 conn(0x7f5bd00b9a30 legacy=0x7f5bd00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 0x7f5bb80039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.987+0000 7f5bceffd640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bb8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bceffd640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bb8004d40 0x7f5bb80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5bd00d2af0 con 0x7f5bb8003f30 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5bd00cf3c0 con 0x7f5bb8004d40 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5bb800b9c0 con 0x7f5bd015df00 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bb8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bcffff640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 0x7f5bb80039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bd4b1e640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bb8004d40 0x7f5bb80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bcf7fe640 1 -- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 msgr2=0x7f5bb80039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd015df00 0x7f5bb80039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bcf7fe640 1 -- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bb8004d40 msgr2=0x7f5bb80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bb8004d40 0x7f5bb80051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.988+0000 7f5bcf7fe640 1 -- 172.21.15.64:0/2334510130 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5bb8003520 con 0x7f5bb8003f30 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bb8004800 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7f5bc0009ac0 tx=0x7f5bb8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc7bb0d8e7438bf7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bc0039070 con 0x7f5bb8003f30 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5bc002c6d0 con 0x7f5bb8003f30 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bceffd640 1 -- 172.21.15.64:0/2334510130 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bc0009690 con 0x7f5bb8003f30 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bd6581640 1 -- 172.21.15.64:0/2334510130 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 msgr2=0x7f5bb8004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bd6581640 1 --2- 172.21.15.64:0/2334510130 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bb8004800 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7f5bc0009ac0 tx=0x7f5bb8005830 comp rx=0 tx=0).stop 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bd4b1e640 1 -- 172.21.15.64:0/2334510130 reap_dead start 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bd6581640 1 -- 172.21.15.64:0/2334510130 shutdown_connections 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bd6581640 1 -- 172.21.15.64:0/2334510130 >> 172.21.15.64:0/2334510130 conn(0x7f5bd0071960 msgr2=0x7f5bd0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:07.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bd6581640 1 -- 172.21.15.64:0/2334510130 shutdown_connections 2024-03-04T07:29:07.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.989+0000 7f5bd6581640 1 -- 172.21.15.64:0/2334510130 wait complete. 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.990+0000 7f5bd6581640 1 Processor -- start 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bd6581640 1 -- start start 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bd6581640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bd00b4710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bd6581640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd014b620 0x7f5bd0149b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bd6581640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bd014a070 0x7f5bd01f7010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bd6581640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5bd00cec70 con 0x7f5bb8003f30 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bd6581640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5bd00ce750 con 0x7f5bd014a070 2024-03-04T07:29:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bd6581640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5bd00cedf0 con 0x7f5bd014b620 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bcffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bd00b4710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bcffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bd00b4710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34526/0 (socket says 172.21.15.64:34526) 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.991+0000 7f5bcffff640 1 -- 172.21.15.64:0/2985358086 learned_addr learned my addr 172.21.15.64:0/2985358086 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bd4b1e640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bd014a070 0x7f5bd01f7010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd014b620 0x7f5bd0149b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcffff640 1 -- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd014b620 msgr2=0x7f5bd0149b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcffff640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd014b620 0x7f5bd0149b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcffff640 1 -- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bd014a070 msgr2=0x7f5bd01f7010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcffff640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bd014a070 0x7f5bd01f7010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcffff640 1 -- 172.21.15.64:0/2985358086 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5bd014a580 con 0x7f5bb8003f30 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd014b620 0x7f5bd0149b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bcffff640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bd00b4710 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7f5bc4011a00 tx=0x7f5bc4011ec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bccff9640 1 -- 172.21.15.64:0/2985358086 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bc400fb50 con 0x7f5bb8003f30 2024-03-04T07:29:07.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bccff9640 1 -- 172.21.15.64:0/2985358086 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5bc400fcf0 con 0x7f5bb8003f30 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5bd014a860 con 0x7f5bb8003f30 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bccff9640 1 -- 172.21.15.64:0/2985358086 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bc400fea0 con 0x7f5bb8003f30 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.992+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5bd014ae20 con 0x7f5bb8003f30 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.993+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5bd0067ff0 con 0x7f5bb8003f30 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.994+0000 7f5bccff9640 1 -- 172.21.15.64:0/2985358086 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5bc4012040 con 0x7f5bb8003f30 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.994+0000 7f5bccff9640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5bbc06b500 0x7f5bbc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.994+0000 7f5bccff9640 1 -- 172.21.15.64:0/2985358086 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(51..51 src has 1..51) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f5bc408a3b0 con 0x7f5bb8003f30 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.994+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5bbc06b500 0x7f5bbc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:07.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.994+0000 7f5bcf7fe640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5bbc06b500 0x7f5bbc06d9c0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f5bc0036000 tx=0x7f5bc0030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:07.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:07.998+0000 7f5bccff9640 1 -- 172.21.15.64:0/2985358086 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5bc4019030 con 0x7f5bb8003f30 2024-03-04T07:29:08.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.181+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f5bd00d2ee0 con 0x7f5bb8003f30 2024-03-04T07:29:08.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.182+0000 7f5bccff9640 1 -- 172.21.15.64:0/2985358086 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v51) v1 ==== 74+0+12843 (secure 0 0 0) 0x7f5bc4059610 con 0x7f5bb8003f30 2024-03-04T07:29:08.182 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:08.182 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":51,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:07.495641+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":7,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:08.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5bbc06b500 msgr2=0x7f5bbc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5bbc06b500 0x7f5bbc06d9c0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f5bc0036000 tx=0x7f5bc0030000 comp rx=0 tx=0).stop 2024-03-04T07:29:08.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 msgr2=0x7f5bd00b4710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bd00b4710 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7f5bc4011a00 tx=0x7f5bc4011ec0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 shutdown_connections 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5bbc06b500 0x7f5bbc06d9c0 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bd014a070 0x7f5bd01f7010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bd014b620 0x7f5bd0149b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 --2- 172.21.15.64:0/2985358086 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bb8003f30 0x7f5bd00b4710 unknown :-1 s=CLOSED pgs=642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 >> 172.21.15.64:0/2985358086 conn(0x7f5bd0071960 msgr2=0x7f5bd015bfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 shutdown_connections 2024-03-04T07:29:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.184+0000 7f5bd6581640 1 -- 172.21.15.64:0/2985358086 wait complete. 2024-03-04T07:29:08.196 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:29:08.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:29:08.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.301+0000 7fc5faff6640 1 Processor -- start 2024-03-04T07:29:08.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.301+0000 7fc5faff6640 1 -- start start 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5faff6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc5f40717b0 0x7fc5f4071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5faff6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40b2b10 0x7fc5f40d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5faff6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5f415e4b0 con 0x7fc5f40b3060 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5faff6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5f40d2e10 con 0x7fc5f40717b0 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5faff6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5f40b97e0 con 0x7fc5f40b2b10 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5f8d92640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc5f40717b0 0x7fc5f4071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5f8d92640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc5f40717b0 0x7fc5f4071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34542/0 (socket says 172.21.15.64:34542) 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5f8d92640 1 -- 172.21.15.64:0/1546286285 learned_addr learned my addr 172.21.15.64:0/1546286285 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5f8d92640 1 -- 172.21.15.64:0/1546286285 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40b2b10 msgr2=0x7fc5f40d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5f8d92640 1 --2- 172.21.15.64:0/1546286285 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40b2b10 0x7fc5f40d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5f8d92640 1 -- 172.21.15.64:0/1546286285 >> v1:172.21.15.66:6789/0 conn(0x7fc5f40b3060 legacy=0x7fc5f40b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:08.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.302+0000 7fc5f8d92640 1 -- 172.21.15.64:0/1546286285 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5f40b9a80 con 0x7fc5f40717b0 2024-03-04T07:29:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5f8d92640 1 --2- 172.21.15.64:0/1546286285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40717b0 0x7fc5f4071b90 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fc5f0009870 tx=0x7fc5f002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4af216d58bab2db0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5eb7fe640 1 -- 172.21.15.64:0/1546286285 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5f0035020 con 0x7fc5f40717b0 2024-03-04T07:29:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5eb7fe640 1 -- 172.21.15.64:0/1546286285 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5f0031640 con 0x7fc5f40717b0 2024-03-04T07:29:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5eb7fe640 1 -- 172.21.15.64:0/1546286285 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5f002d040 con 0x7fc5f40717b0 2024-03-04T07:29:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5faff6640 1 -- 172.21.15.64:0/1546286285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40717b0 msgr2=0x7fc5f4071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5faff6640 1 --2- 172.21.15.64:0/1546286285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40717b0 0x7fc5f4071b90 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fc5f0009870 tx=0x7fc5f002b8f0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5faff6640 1 -- 172.21.15.64:0/1546286285 shutdown_connections 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5faff6640 1 --2- 172.21.15.64:0/1546286285 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40b2b10 0x7fc5f40d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5faff6640 1 --2- 172.21.15.64:0/1546286285 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40717b0 0x7fc5f4071b90 unknown :-1 s=CLOSED pgs=643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.303+0000 7fc5faff6640 1 -- 172.21.15.64:0/1546286285 >> 172.21.15.64:0/1546286285 conn(0x7fc5f40d3d20 msgr2=0x7fc5f40d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.304+0000 7fc5faff6640 1 -- 172.21.15.64:0/1546286285 shutdown_connections 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.304+0000 7fc5faff6640 1 -- 172.21.15.64:0/1546286285 wait complete. 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 Processor -- start 2024-03-04T07:29:08.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 -- start start 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40717b0 0x7fc5f41e58c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5f40b2b10 0x7fc5f40cbe40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40b3060 0x7fc5f41f6c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5f40b7560 con 0x7fc5f40b3060 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5f40d4bf0 con 0x7fc5f40b2b10 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.305+0000 7fc5faff6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5f40b76e0 con 0x7fc5f40717b0 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40b3060 0x7fc5f41f6c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40b3060 0x7fc5f41f6c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34544/0 (socket says 172.21.15.64:34544) 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 -- 172.21.15.64:0/2302049506 learned_addr learned my addr 172.21.15.64:0/2302049506 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5ebfff640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5f40b2b10 0x7fc5f40cbe40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f8d92640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40717b0 0x7fc5f41e58c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 -- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40717b0 msgr2=0x7fc5f41e58c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40717b0 0x7fc5f41e58c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 -- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5f40b2b10 msgr2=0x7fc5f40cbe40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5f40b2b10 0x7fc5f40cbe40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 -- 172.21.15.64:0/2302049506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5f40cc940 con 0x7fc5f40b3060 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f8d92640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40717b0 0x7fc5f41e58c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5f9593640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40b3060 0x7fc5f41f6c50 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7fc5ec00b970 tx=0x7fc5ec00be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5e97fa640 1 -- 172.21.15.64:0/2302049506 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5ec019070 con 0x7fc5f40b3060 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5e97fa640 1 -- 172.21.15.64:0/2302049506 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5ec00cb60 con 0x7fc5f40b3060 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5f40ccc20 con 0x7fc5f40b3060 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5e97fa640 1 -- 172.21.15.64:0/2302049506 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5ec014440 con 0x7fc5f40b3060 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.306+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5f40cd1e0 con 0x7fc5f40b3060 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.307+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5f406ce40 con 0x7fc5f40b3060 2024-03-04T07:29:08.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.308+0000 7fc5e97fa640 1 -- 172.21.15.64:0/2302049506 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc5ec01c070 con 0x7fc5f40b3060 2024-03-04T07:29:08.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.308+0000 7fc5e97fa640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc5e006b4d0 0x7fc5e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.308+0000 7fc5e97fa640 1 -- 172.21.15.64:0/2302049506 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(51..51 src has 1..51) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fc5ec089b60 con 0x7fc5f40b3060 2024-03-04T07:29:08.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.308+0000 7fc5f8d92640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc5e006b4d0 0x7fc5e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.309+0000 7fc5f8d92640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc5e006b4d0 0x7fc5e006d990 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fc5f0036000 tx=0x7fc5f003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.312+0000 7fc5e97fa640 1 -- 172.21.15.64:0/2302049506 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc5ec058dc0 con 0x7fc5f40b3060 2024-03-04T07:29:08.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.492+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fc5f40b3da0 con 0x7fc5f40b3060 2024-03-04T07:29:08.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.493+0000 7fc5e97fa640 1 -- 172.21.15.64:0/2302049506 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v51) v1 ==== 93+0+10 (secure 0 0 0) 0x7fc5ec0586c0 con 0x7fc5f40b3060 2024-03-04T07:29:08.493 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:29:08.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.494+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc5e006b4d0 msgr2=0x7fc5e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.494+0000 7fc5faff6640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc5e006b4d0 0x7fc5e006d990 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fc5f0036000 tx=0x7fc5f003c040 comp rx=0 tx=0).stop 2024-03-04T07:29:08.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.494+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40b3060 msgr2=0x7fc5f41f6c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.494+0000 7fc5faff6640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40b3060 0x7fc5f41f6c50 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7fc5ec00b970 tx=0x7fc5ec00be30 comp rx=0 tx=0).stop 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 shutdown_connections 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc5e006b4d0 0x7fc5e006d990 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5f40b3060 0x7fc5f41f6c50 unknown :-1 s=CLOSED pgs=644 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5f40b2b10 0x7fc5f40cbe40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 --2- 172.21.15.64:0/2302049506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5f40717b0 0x7fc5f41e58c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 >> 172.21.15.64:0/2302049506 conn(0x7fc5f40d3d20 msgr2=0x7fc5f415c1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 shutdown_connections 2024-03-04T07:29:08.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.495+0000 7fc5faff6640 1 -- 172.21.15.64:0/2302049506 wait complete. 2024-03-04T07:29:08.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:29:08.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.607+0000 7fd62aa8f640 1 Processor -- start 2024-03-04T07:29:08.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.607+0000 7fd62aa8f640 1 -- start start 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd62aa8f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd6240b1020 0x7fd62406af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd62aa8f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd62406ed10 0x7fd6240cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd62aa8f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd6240cc0e0 con 0x7fd6240cd6c0 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd62aa8f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6240d28b0 con 0x7fd62406ed10 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd62aa8f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6240cf320 con 0x7fd6240b1020 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd623fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd62406ed10 0x7fd6240cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd623fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd62406ed10 0x7fd6240cd180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34560/0 (socket says 172.21.15.64:34560) 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd623fff640 1 -- 172.21.15.64:0/856858196 learned_addr learned my addr 172.21.15.64:0/856858196 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:08.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd62882b640 1 --2- 172.21.15.64:0/856858196 >> v2:172.21.15.66:3300/0 conn(0x7fd6240b1020 0x7fd62406af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd623fff640 1 -- 172.21.15.64:0/856858196 >> v2:172.21.15.66:3300/0 conn(0x7fd6240b1020 msgr2=0x7fd62406af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:08.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd623fff640 1 --2- 172.21.15.64:0/856858196 >> v2:172.21.15.66:3300/0 conn(0x7fd6240b1020 0x7fd62406af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd623fff640 1 -- 172.21.15.64:0/856858196 >> v1:172.21.15.66:6789/0 conn(0x7fd6240cd6c0 legacy=0x7fd62415d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:08.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.608+0000 7fd623fff640 1 -- 172.21.15.64:0/856858196 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6240cc3c0 con 0x7fd62406ed10 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.609+0000 7fd623fff640 1 --2- 172.21.15.64:0/856858196 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd62406ed10 0x7fd6240cd180 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fd624070d50 tx=0x7fd61002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1dd3bcc878cf8bb4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.609+0000 7fd6237fe640 1 -- 172.21.15.64:0/856858196 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd610035020 con 0x7fd62406ed10 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.609+0000 7fd6237fe640 1 -- 172.21.15.64:0/856858196 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd610031870 con 0x7fd62406ed10 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.609+0000 7fd6237fe640 1 -- 172.21.15.64:0/856858196 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd610031c40 con 0x7fd62406ed10 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 -- 172.21.15.64:0/856858196 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd62406ed10 msgr2=0x7fd6240cd180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/856858196 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd62406ed10 0x7fd6240cd180 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fd624070d50 tx=0x7fd61002b910 comp rx=0 tx=0).stop 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 -- 172.21.15.64:0/856858196 shutdown_connections 2024-03-04T07:29:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/856858196 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd62406ed10 0x7fd6240cd180 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/856858196 >> v2:172.21.15.66:3300/0 conn(0x7fd6240b1020 0x7fd62406af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 -- 172.21.15.64:0/856858196 >> 172.21.15.64:0/856858196 conn(0x7fd6240d1a90 msgr2=0x7fd6240d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:08.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 -- 172.21.15.64:0/856858196 shutdown_connections 2024-03-04T07:29:08.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.610+0000 7fd62aa8f640 1 -- 172.21.15.64:0/856858196 wait complete. 2024-03-04T07:29:08.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.611+0000 7fd62aa8f640 1 Processor -- start 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.611+0000 7fd62aa8f640 1 -- start start 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62aa8f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd62406ed10 0x7fd6240b3340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62aa8f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6240b1020 0x7fd6240b3c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62aa8f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240cd6c0 0x7fd6240cfc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62aa8f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6240ccc60 con 0x7fd6240cd6c0 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62aa8f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6240cc980 con 0x7fd62406ed10 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62aa8f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd6240cdad0 con 0x7fd6240b1020 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240cd6c0 0x7fd6240cfc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240cd6c0 0x7fd6240cfc90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34564/0 (socket says 172.21.15.64:34564) 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 -- 172.21.15.64:0/586389399 learned_addr learned my addr 172.21.15.64:0/586389399 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd623fff640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6240b1020 0x7fd6240b3c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62882b640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd62406ed10 0x7fd6240b3340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 -- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6240b1020 msgr2=0x7fd6240b3c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6240b1020 0x7fd6240b3c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 -- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd62406ed10 msgr2=0x7fd6240b3340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd62406ed10 0x7fd6240b3340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62902c640 1 -- 172.21.15.64:0/586389399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6240c7c00 con 0x7fd6240cd6c0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd623fff640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6240b1020 0x7fd6240b3c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.612+0000 7fd62882b640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd62406ed10 0x7fd6240b3340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.613+0000 7fd62902c640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240cd6c0 0x7fd6240cfc90 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7fd61c00b970 tx=0x7fd61c00be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.613+0000 7fd6217fa640 1 -- 172.21.15.64:0/586389399 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd61c019070 con 0x7fd6240cd6c0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.613+0000 7fd6217fa640 1 -- 172.21.15.64:0/586389399 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd61c00cb40 con 0x7fd6240cd6c0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.613+0000 7fd6217fa640 1 -- 172.21.15.64:0/586389399 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd61c014440 con 0x7fd6240cd6c0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.613+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd6240c7ee0 con 0x7fd6240cd6c0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.613+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6240b1400 con 0x7fd6240cd6c0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.614+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd6240b4680 con 0x7fd6240cd6c0 2024-03-04T07:29:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.614+0000 7fd6217fa640 1 -- 172.21.15.64:0/586389399 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd61c01c070 con 0x7fd6240cd6c0 2024-03-04T07:29:08.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.614+0000 7fd6217fa640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd60806b4d0 0x7fd60806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.615+0000 7fd6217fa640 1 -- 172.21.15.64:0/586389399 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(51..51 src has 1..51) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fd61c089b70 con 0x7fd6240cd6c0 2024-03-04T07:29:08.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.615+0000 7fd62882b640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd60806b4d0 0x7fd60806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.615+0000 7fd62882b640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd60806b4d0 0x7fd60806d990 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7fd618006fd0 tx=0x7fd618008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.619+0000 7fd6217fa640 1 -- 172.21.15.64:0/586389399 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd6240b4680 con 0x7fd6240cd6c0 2024-03-04T07:29:08.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.802+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7fd62406f2f0 con 0x7fd6240cd6c0 2024-03-04T07:29:08.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.802+0000 7fd6217fa640 1 -- 172.21.15.64:0/586389399 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v51) v1 ==== 84+0+65 (secure 0 0 0) 0x7fd61c058ca0 con 0x7fd6240cd6c0 2024-03-04T07:29:08.802 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:08.802 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:29:08.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:29:08.802 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:29:08.802 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:29:08.802 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:08.802 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd60806b4d0 msgr2=0x7fd60806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd60806b4d0 0x7fd60806d990 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7fd618006fd0 tx=0x7fd618008040 comp rx=0 tx=0).stop 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240cd6c0 msgr2=0x7fd6240cfc90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240cd6c0 0x7fd6240cfc90 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7fd61c00b970 tx=0x7fd61c00be30 comp rx=0 tx=0).stop 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 shutdown_connections 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd60806b4d0 0x7fd60806d990 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6240cd6c0 0x7fd6240cfc90 unknown :-1 s=CLOSED pgs=646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6240b1020 0x7fd6240b3c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 --2- 172.21.15.64:0/586389399 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd62406ed10 0x7fd6240b3340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 >> 172.21.15.64:0/586389399 conn(0x7fd6240d1a90 msgr2=0x7fd6240d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:08.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 shutdown_connections 2024-03-04T07:29:08.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.804+0000 7fd62aa8f640 1 -- 172.21.15.64:0/586389399 wait complete. 2024-03-04T07:29:08.815 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:29:08.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:29:08.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.920+0000 7f8b17781640 1 Processor -- start 2024-03-04T07:29:08.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.921+0000 7f8b17781640 1 -- start start 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.921+0000 7f8b17781640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8b100d07c0 0x7f8b101624c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.921+0000 7f8b17781640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8b100b4950 con 0x7f8b100cf9f0 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.921+0000 7f8b17781640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8b1006d4d0 con 0x7f8b10072a30 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.921+0000 7f8b17781640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8b1006dba0 con 0x7f8b100d07c0 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b15d1e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8b100cf9f0 0x7f8b1015d660 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51352/0 (socket says 172.21.15.64:51352) 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b15d1e640 1 -- 172.21.15.64:0/760135661 learned_addr learned my addr 172.21.15.64:0/760135661 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b14d1c640 1 --2- 172.21.15.64:0/760135661 >> v2:172.21.15.66:3300/0 conn(0x7f8b100d07c0 0x7f8b101624c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b0ffff640 1 -- 172.21.15.64:0/760135661 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 921230625 0 0) 0x7f8b100b4950 con 0x7f8b100cf9f0 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b0ffff640 1 -- 172.21.15.64:0/760135661 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8af8003520 con 0x7f8b100cf9f0 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b14d1c640 1 -- 172.21.15.64:0/760135661 >> v1:172.21.15.66:6789/0 conn(0x7f8b10072a30 legacy=0x7f8b100d00b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b14d1c640 1 -- 172.21.15.64:0/760135661 >> v1:172.21.15.64:6789/0 conn(0x7f8b100cf9f0 legacy=0x7f8b1015d660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.922+0000 7f8b14d1c640 1 -- 172.21.15.64:0/760135661 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b1006d7b0 con 0x7f8b100d07c0 2024-03-04T07:29:08.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.923+0000 7f8b14d1c640 1 --2- 172.21.15.64:0/760135661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100d07c0 0x7f8b101624c0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f8b040097a0 tx=0x7f8b0402b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a782d3aade2427cc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.923+0000 7f8b0ffff640 1 -- 172.21.15.64:0/760135661 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b040315a0 con 0x7f8b100d07c0 2024-03-04T07:29:08.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.923+0000 7f8b0ffff640 1 -- 172.21.15.64:0/760135661 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b04037070 con 0x7f8b100d07c0 2024-03-04T07:29:08.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.923+0000 7f8b0ffff640 1 -- 172.21.15.64:0/760135661 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b04031940 con 0x7f8b100d07c0 2024-03-04T07:29:08.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.924+0000 7f8b17781640 1 -- 172.21.15.64:0/760135661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100d07c0 msgr2=0x7f8b101624c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.924+0000 7f8b17781640 1 --2- 172.21.15.64:0/760135661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100d07c0 0x7f8b101624c0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f8b040097a0 tx=0x7f8b0402b840 comp rx=0 tx=0).stop 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.924+0000 7f8b17781640 1 -- 172.21.15.64:0/760135661 shutdown_connections 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.924+0000 7f8b17781640 1 --2- 172.21.15.64:0/760135661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100d07c0 0x7f8b101624c0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.924+0000 7f8b17781640 1 -- 172.21.15.64:0/760135661 >> 172.21.15.64:0/760135661 conn(0x7f8b1006af20 msgr2=0x7f8b1006b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.924+0000 7f8b17781640 1 -- 172.21.15.64:0/760135661 shutdown_connections 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.924+0000 7f8b17781640 1 -- 172.21.15.64:0/760135661 wait complete. 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 Processor -- start 2024-03-04T07:29:08.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 -- start start 2024-03-04T07:29:08.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8b10072a30 0x7f8b1015f120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100cf9f0 0x7f8b1015a8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8b100d07c0 0x7f8b1009ceb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b10072e10 con 0x7f8b100d07c0 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b10071130 con 0x7f8b100cf9f0 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.925+0000 7f8b17781640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8b10072f90 con 0x7f8b10072a30 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8b100d07c0 0x7f8b1009ceb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8b100d07c0 0x7f8b1009ceb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34570/0 (socket says 172.21.15.64:34570) 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 -- 172.21.15.64:0/2714017640 learned_addr learned my addr 172.21.15.64:0/2714017640 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b1551d640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8b10072a30 0x7f8b1015f120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b14d1c640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100cf9f0 0x7f8b1015a8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 -- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8b10072a30 msgr2=0x7f8b1015f120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8b10072a30 0x7f8b1015f120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 -- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100cf9f0 msgr2=0x7f8b1015a8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100cf9f0 0x7f8b1015a8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 -- 172.21.15.64:0/2714017640 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b1015b330 con 0x7f8b100d07c0 2024-03-04T07:29:08.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b14d1c640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100cf9f0 0x7f8b1015a8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b15d1e640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8b100d07c0 0x7f8b1009ceb0 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f8b0000c420 tx=0x7f8b0000c8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b0dffb640 1 -- 172.21.15.64:0/2714017640 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b00017070 con 0x7f8b100d07c0 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b0dffb640 1 -- 172.21.15.64:0/2714017640 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8b00005ce0 con 0x7f8b100d07c0 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8b1015b610 con 0x7f8b100d07c0 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.926+0000 7f8b0dffb640 1 -- 172.21.15.64:0/2714017640 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b000072d0 con 0x7f8b100d07c0 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.927+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8b10159360 con 0x7f8b100d07c0 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.927+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8b1009e320 con 0x7f8b100d07c0 2024-03-04T07:29:08.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.928+0000 7f8b0dffb640 1 -- 172.21.15.64:0/2714017640 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8b0001a070 con 0x7f8b100d07c0 2024-03-04T07:29:08.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.928+0000 7f8b0dffb640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8afc06b4d0 0x7f8afc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:08.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.928+0000 7f8b0dffb640 1 -- 172.21.15.64:0/2714017640 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(51..51 src has 1..51) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f8b00089160 con 0x7f8b100d07c0 2024-03-04T07:29:08.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.928+0000 7f8b1551d640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8afc06b4d0 0x7f8afc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:08.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.929+0000 7f8b1551d640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8afc06b4d0 0x7f8afc06d990 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f8b08001f10 tx=0x7f8b08009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:08.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:08.933+0000 7f8b0dffb640 1 -- 172.21.15.64:0/2714017640 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8b00057ce0 con 0x7f8b100d07c0 2024-03-04T07:29:09.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.112+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f8b1004f410 con 0x7f8b100d07c0 2024-03-04T07:29:09.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.485+0000 7f8b0dffb640 1 -- 172.21.15.64:0/2714017640 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v52) v1 ==== 138+0+0 (secure 0 0 0) 0x7f8b0005bd80 con 0x7f8b100d07c0 2024-03-04T07:29:09.485 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:29:09.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8afc06b4d0 msgr2=0x7f8afc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:09.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8afc06b4d0 0x7f8afc06d990 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f8b08001f10 tx=0x7f8b08009040 comp rx=0 tx=0).stop 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8b100d07c0 msgr2=0x7f8b1009ceb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8b100d07c0 0x7f8b1009ceb0 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f8b0000c420 tx=0x7f8b0000c8e0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 shutdown_connections 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8afc06b4d0 0x7f8afc06d990 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8b100d07c0 0x7f8b1009ceb0 unknown :-1 s=CLOSED pgs=648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8b100cf9f0 0x7f8b1015a8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 --2- 172.21.15.64:0/2714017640 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8b10072a30 0x7f8b1015f120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.487+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 >> 172.21.15.64:0/2714017640 conn(0x7f8b1006af20 msgr2=0x7f8b100b4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.488+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 shutdown_connections 2024-03-04T07:29:09.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.488+0000 7f8b17781640 1 -- 172.21.15.64:0/2714017640 wait complete. 2024-03-04T07:29:09.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:29:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.611+0000 7ff901af4640 1 Processor -- start 2024-03-04T07:29:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.611+0000 7ff901af4640 1 -- start start 2024-03-04T07:29:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff901af4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15f1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff901af4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff8fc06d250 0x7ff8fc06d6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff901af4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8fc0cf5a0 con 0x7ff8fc161bb0 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff901af4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff8fc15ed20 con 0x7ff8fc06d250 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff901af4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8fc072890 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff8fbfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff8fc161bb0 0x7ff8fc0ad8b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51364/0 (socket says 172.21.15.64:51364) 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff8fbfff640 1 -- 172.21.15.64:0/3674940267 learned_addr learned my addr 172.21.15.64:0/3674940267 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff8faffd640 1 --2- 172.21.15.64:0/3674940267 >> v2:172.21.15.66:3300/0 conn(0x7ff8fc06d250 0x7ff8fc06d6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff8fb7fe640 1 --2- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15f1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff8fa7fc640 1 -- 172.21.15.64:0/3674940267 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 951737299 0 0) 0x7ff8f4001490 con 0x7ff8fc161bb0 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.612+0000 7ff8fa7fc640 1 -- 172.21.15.64:0/3674940267 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff8e4003520 con 0x7ff8fc161bb0 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.613+0000 7ff8faffd640 1 -- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc0cf1c0 msgr2=0x7ff8fc15f1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.613+0000 7ff8faffd640 1 --2- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15f1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.613+0000 7ff8faffd640 1 -- 172.21.15.64:0/3674940267 >> v1:172.21.15.64:6789/0 conn(0x7ff8fc161bb0 legacy=0x7ff8fc0ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.613+0000 7ff8fa7fc640 1 -- 172.21.15.64:0/3674940267 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2934551929 0 0) 0x7ff8e4003520 con 0x7ff8fc161bb0 2024-03-04T07:29:09.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.613+0000 7ff8faffd640 1 -- 172.21.15.64:0/3674940267 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8fc162790 con 0x7ff8fc06d250 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.613+0000 7ff8faffd640 1 --2- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc06d250 0x7ff8fc06d6d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7ff8ec0097a0 tx=0x7ff8ec02b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cec127fe9b502fa2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff8fa7fc640 1 -- 172.21.15.64:0/3674940267 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8ec037070 con 0x7ff8fc06d250 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff8fa7fc640 1 -- 172.21.15.64:0/3674940267 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff8ec030510 con 0x7ff8fc06d250 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff8fa7fc640 1 -- 172.21.15.64:0/3674940267 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8f4001bc0 con 0x7ff8fc06d250 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 -- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc06d250 msgr2=0x7ff8fc06d6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 --2- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc06d250 0x7ff8fc06d6d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7ff8ec0097a0 tx=0x7ff8ec02b790 comp rx=0 tx=0).stop 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 -- 172.21.15.64:0/3674940267 shutdown_connections 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 --2- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc06d250 0x7ff8fc06d6d0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 --2- 172.21.15.64:0/3674940267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15f1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 -- 172.21.15.64:0/3674940267 >> 172.21.15.64:0/3674940267 conn(0x7ff8fc066d00 msgr2=0x7ff8fc067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:09.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 -- 172.21.15.64:0/3674940267 shutdown_connections 2024-03-04T07:29:09.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.614+0000 7ff901af4640 1 -- 172.21.15.64:0/3674940267 wait complete. 2024-03-04T07:29:09.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.615+0000 7ff901af4640 1 Processor -- start 2024-03-04T07:29:09.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.615+0000 7ff901af4640 1 -- start start 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff901af4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc06d250 0x7ff8fc15a7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff901af4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15b670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff901af4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc161bb0 0x7ff8fc158d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff901af4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8fc15e9b0 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff901af4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8fc15e6d0 con 0x7ff8fc161bb0 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff901af4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8fc162000 con 0x7ff8fc06d250 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff8faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15b670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff8faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15b670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34586/0 (socket says 172.21.15.64:34586) 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff8faffd640 1 -- 172.21.15.64:0/2587286544 learned_addr learned my addr 172.21.15.64:0/2587286544 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.616+0000 7ff8fbfff640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc161bb0 0x7ff8fc158d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8fb7fe640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc06d250 0x7ff8fc15a7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8faffd640 1 -- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc06d250 msgr2=0x7ff8fc15a7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8faffd640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc06d250 0x7ff8fc15a7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8faffd640 1 -- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc161bb0 msgr2=0x7ff8fc158d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8faffd640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc161bb0 0x7ff8fc158d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8faffd640 1 -- 172.21.15.64:0/2587286544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8fc159750 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8fb7fe640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc06d250 0x7ff8fc15a7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8faffd640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15b670 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7ff8ec02b7d0 tx=0x7ff8ec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8e3fff640 1 -- 172.21.15.64:0/2587286544 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8ec041020 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8e3fff640 1 -- 172.21.15.64:0/2587286544 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff8ec03c890 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff8e3fff640 1 -- 172.21.15.64:0/2587286544 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8ec037040 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8fc159a30 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.617+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8fc159ff0 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.618+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff8fc0733c0 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.619+0000 7ff8e3fff640 1 -- 172.21.15.64:0/2587286544 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff8ec044020 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.619+0000 7ff8e3fff640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8e806b4d0 0x7ff8e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:09.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.619+0000 7ff8e3fff640 1 -- 172.21.15.64:0/2587286544 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(52..52 src has 1..52) v4 ==== 5227+0+0 (secure 0 0 0) 0x7ff8ec0ab930 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.619+0000 7ff8fb7fe640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8e806b4d0 0x7ff8e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:09.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.619+0000 7ff8fb7fe640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8e806b4d0 0x7ff8e806d990 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7ff8fc158ac0 tx=0x7ff8f0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:09.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.624+0000 7ff8e3fff640 1 -- 172.21.15.64:0/2587286544 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff8ec04b030 con 0x7ff8fc0cf1c0 2024-03-04T07:29:09.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:09.806+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7ff8fc0cd8c0 con 0x7ff8fc0cf1c0 2024-03-04T07:29:10.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.494+0000 7ff8e3fff640 1 -- 172.21.15.64:0/2587286544 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v53) v1 ==== 170+0+0 (secure 0 0 0) 0x7ff8ec07a990 con 0x7ff8fc0cf1c0 2024-03-04T07:29:10.495 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8e806b4d0 msgr2=0x7ff8e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8e806b4d0 0x7ff8e806d990 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7ff8fc158ac0 tx=0x7ff8f0008040 comp rx=0 tx=0).stop 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8fc0cf1c0 msgr2=0x7ff8fc15b670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15b670 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7ff8ec02b7d0 tx=0x7ff8ec0046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 shutdown_connections 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8e806b4d0 0x7ff8e806d990 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8fc161bb0 0x7ff8fc158d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8fc0cf1c0 0x7ff8fc15b670 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 --2- 172.21.15.64:0/2587286544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8fc06d250 0x7ff8fc15a7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 >> 172.21.15.64:0/2587286544 conn(0x7ff8fc066d00 msgr2=0x7ff8fc0716a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 shutdown_connections 2024-03-04T07:29:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.497+0000 7ff901af4640 1 -- 172.21.15.64:0/2587286544 wait complete. 2024-03-04T07:29:10.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:29:10.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.629+0000 7f04077b6640 1 Processor -- start 2024-03-04T07:29:10.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.629+0000 7f04077b6640 1 -- start start 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f04077b6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0400162300 0x7f04000b3130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f04077b6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0400162790 0x7f04000d2160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f04077b6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04000ce590 con 0x7f040015e150 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f04077b6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f040015e530 con 0x7f0400162790 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f04077b6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04000ce870 con 0x7f0400162300 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f0405d53640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0400162790 0x7f04000d2160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f0405d53640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0400162790 0x7f04000d2160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34598/0 (socket says 172.21.15.64:34598) 2024-03-04T07:29:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f0405d53640 1 -- 172.21.15.64:0/566392209 learned_addr learned my addr 172.21.15.64:0/566392209 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.630+0000 7f0404d51640 1 --2- 172.21.15.64:0/566392209 >> v2:172.21.15.66:3300/0 conn(0x7f0400162300 0x7f04000b3130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f0405d53640 1 -- 172.21.15.64:0/566392209 >> v2:172.21.15.66:3300/0 conn(0x7f0400162300 msgr2=0x7f04000b3130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f0405d53640 1 --2- 172.21.15.64:0/566392209 >> v2:172.21.15.66:3300/0 conn(0x7f0400162300 0x7f04000b3130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f0405d53640 1 -- 172.21.15.64:0/566392209 >> v1:172.21.15.66:6789/0 conn(0x7f040015e150 legacy=0x7f0400161bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f0405d53640 1 -- 172.21.15.64:0/566392209 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04000cee10 con 0x7f0400162790 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f0404d51640 1 --2- 172.21.15.64:0/566392209 >> v2:172.21.15.66:3300/0 conn(0x7f0400162300 0x7f04000b3130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f0405d53640 1 --2- 172.21.15.64:0/566392209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000d2160 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7f040006cc80 tx=0x7f03f002be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d84577cf1066fd2a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f03fffff640 1 -- 172.21.15.64:0/566392209 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03f002c9f0 con 0x7f0400162790 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f03fffff640 1 -- 172.21.15.64:0/566392209 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03f002cb90 con 0x7f0400162790 2024-03-04T07:29:10.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.631+0000 7f03fffff640 1 -- 172.21.15.64:0/566392209 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03f00343f0 con 0x7f0400162790 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 -- 172.21.15.64:0/566392209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 msgr2=0x7f04000d2160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 --2- 172.21.15.64:0/566392209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000d2160 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7f040006cc80 tx=0x7f03f002be50 comp rx=0 tx=0).stop 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 -- 172.21.15.64:0/566392209 shutdown_connections 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 --2- 172.21.15.64:0/566392209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000d2160 unknown :-1 s=CLOSED pgs=651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 --2- 172.21.15.64:0/566392209 >> v2:172.21.15.66:3300/0 conn(0x7f0400162300 0x7f04000b3130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 -- 172.21.15.64:0/566392209 >> 172.21.15.64:0/566392209 conn(0x7f04000d40c0 msgr2=0x7f04000d44d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 -- 172.21.15.64:0/566392209 shutdown_connections 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.632+0000 7f04077b6640 1 -- 172.21.15.64:0/566392209 wait complete. 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.633+0000 7f04077b6640 1 Processor -- start 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.633+0000 7f04077b6640 1 -- start start 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f04077b6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f040015e150 0x7f04001607e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:10.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f04077b6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0400162300 0x7f04000c63c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f04077b6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000c6900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f04077b6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04000cf550 con 0x7f0400162790 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f04077b6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04000cf3d0 con 0x7f0400162300 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f04077b6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04000d26a0 con 0x7f040015e150 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000c6900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000c6900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34612/0 (socket says 172.21.15.64:34612) 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 -- 172.21.15.64:0/4277201365 learned_addr learned my addr 172.21.15.64:0/4277201365 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0404d51640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0400162300 0x7f04000c63c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405552640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f040015e150 0x7f04001607e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 -- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f040015e150 msgr2=0x7f04001607e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f040015e150 0x7f04001607e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 -- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0400162300 msgr2=0x7f04000c63c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0400162300 0x7f04000c63c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0405d53640 1 -- 172.21.15.64:0/4277201365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04000c8390 con 0x7f0400162790 2024-03-04T07:29:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.634+0000 7f0404d51640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0400162300 0x7f04000c63c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.635+0000 7f0405d53640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000c6900 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f03f002c460 tx=0x7f03f0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.635+0000 7f03fdffb640 1 -- 172.21.15.64:0/4277201365 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03f0043020 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.635+0000 7f03fdffb640 1 -- 172.21.15.64:0/4277201365 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03f003e450 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.635+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04000c8670 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.635+0000 7f03fdffb640 1 -- 172.21.15.64:0/4277201365 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03f003e820 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.635+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04000c8c30 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.636+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f040006cdb0 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.636+0000 7f03fdffb640 1 -- 172.21.15.64:0/4277201365 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f03f0046020 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.637+0000 7f03fdffb640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03ec06b4d0 0x7f03ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.637+0000 7f03fdffb640 1 -- 172.21.15.64:0/4277201365 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(53..53 src has 1..53) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f03f00acb90 con 0x7f0400162790 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.637+0000 7f0405552640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03ec06b4d0 0x7f03ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:10.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.637+0000 7f0405552640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03ec06b4d0 0x7f03ec06d990 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f03f8004ea0 tx=0x7f03f8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:10.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.641+0000 7f03fdffb640 1 -- 172.21.15.64:0/4277201365 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f03f004d030 con 0x7f0400162790 2024-03-04T07:29:10.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:10.860+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f04000c8e00 con 0x7f0400162790 2024-03-04T07:29:11.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.509+0000 7f03fdffb640 1 -- 172.21.15.64:0/4277201365 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 8 and data pool 9 v27) v1 ==== 172+0+0 (secure 0 0 0) 0x7f03f007bb00 con 0x7f0400162790 2024-03-04T07:29:11.509 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 8 and data pool 9 2024-03-04T07:29:11.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.518+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03ec06b4d0 msgr2=0x7f03ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.518+0000 7f04077b6640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03ec06b4d0 0x7f03ec06d990 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f03f8004ea0 tx=0x7f03f8009040 comp rx=0 tx=0).stop 2024-03-04T07:29:11.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.518+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 msgr2=0x7f04000c6900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.518+0000 7f04077b6640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000c6900 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f03f002c460 tx=0x7f03f0004a40 comp rx=0 tx=0).stop 2024-03-04T07:29:11.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.519+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 shutdown_connections 2024-03-04T07:29:11.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.519+0000 7f04077b6640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03ec06b4d0 0x7f03ec06d990 secure :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f03f8004ea0 tx=0x7f03f8009040 comp rx=0 tx=0).stop 2024-03-04T07:29:11.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.519+0000 7f04077b6640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0400162790 0x7f04000c6900 unknown :-1 s=CLOSED pgs=652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.519+0000 7f04077b6640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0400162300 0x7f04000c63c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.519+0000 7f04077b6640 1 --2- 172.21.15.64:0/4277201365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f040015e150 0x7f04001607e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.519+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 >> 172.21.15.64:0/4277201365 conn(0x7f04000d40c0 msgr2=0x7f0400072090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:11.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.523+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 shutdown_connections 2024-03-04T07:29:11.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.523+0000 7f04077b6640 1 -- 172.21.15.64:0/4277201365 wait complete. 2024-03-04T07:29:11.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:11.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.639+0000 7f1f31cdf640 1 Processor -- start 2024-03-04T07:29:11.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.639+0000 7f1f31cdf640 1 -- start start 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f31cdf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 0x7f1f2c06af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f31cdf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1f2c0cd890 0x7f1f2c15d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f31cdf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1f2c0cc0e0 con 0x7f1f2c06ed10 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f31cdf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1f2c0d28b0 con 0x7f1f2c0cd890 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f31cdf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0cf320 con 0x7f1f2c0b1020 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2affd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1f2c06ed10 0x7f1f2c0cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51370/0 (socket says 172.21.15.64:51370) 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2affd640 1 -- 172.21.15.64:0/2737201575 learned_addr learned my addr 172.21.15.64:0/2737201575 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2bfff640 1 --2- 172.21.15.64:0/2737201575 >> v2:172.21.15.66:3300/0 conn(0x7f1f2c0cd890 0x7f1f2c15d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2b7fe640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 0x7f1f2c06af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2610256879 0 0) 0x7f1f2c0cc0e0 con 0x7f1f2c06ed10 2024-03-04T07:29:11.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1f14003520 con 0x7f1f2c06ed10 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3773147319 0 0) 0x7f1f14003520 con 0x7f1f2c06ed10 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 msgr2=0x7f1f2c06af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 0x7f1f2c06af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 >> v2:172.21.15.66:3300/0 conn(0x7f1f2c0cd890 msgr2=0x7f1f2c15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 --2- 172.21.15.64:0/2737201575 >> v2:172.21.15.66:3300/0 conn(0x7f1f2c0cd890 0x7f1f2c15d700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f2c0cc3c0 con 0x7f1f2c06ed10 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2bfff640 1 --2- 172.21.15.64:0/2737201575 >> v2:172.21.15.66:3300/0 conn(0x7f1f2c0cd890 0x7f1f2c15d700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f1f20001670 con 0x7f1f2c06ed10 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 >> v1:172.21.15.64:6789/0 conn(0x7f1f2c06ed10 legacy=0x7f1f2c0cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 0x7f1f140039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f14004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f14004d40 0x7f1f140051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0d28b0 con 0x7f1f14003f30 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0cf320 con 0x7f1f14004d40 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.640+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f1400b9c0 con 0x7f1f2c0b1020 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2affd640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f14004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2b7fe640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 0x7f1f140039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2bfff640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f14004d40 0x7f1f140051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2affd640 1 -- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 msgr2=0x7f1f140039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2affd640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c0b1020 0x7f1f140039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2affd640 1 -- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f14004d40 msgr2=0x7f1f140051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2affd640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f14004d40 0x7f1f140051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.641+0000 7f1f2affd640 1 -- 172.21.15.64:0/2737201575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f14003520 con 0x7f1f14003f30 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f2affd640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f14004800 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f1f20001050 tx=0x7f1f14005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=62b7c579132dad7b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f20007d70 con 0x7f1f14003f30 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f20030070 con 0x7f1f14003f30 2024-03-04T07:29:11.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f2a7fc640 1 -- 172.21.15.64:0/2737201575 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f2002cd70 con 0x7f1f14003f30 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f31cdf640 1 -- 172.21.15.64:0/2737201575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 msgr2=0x7f1f14004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f31cdf640 1 --2- 172.21.15.64:0/2737201575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f14004800 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f1f20001050 tx=0x7f1f14005830 comp rx=0 tx=0).stop 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f2bfff640 1 -- 172.21.15.64:0/2737201575 reap_dead start 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f31cdf640 1 -- 172.21.15.64:0/2737201575 shutdown_connections 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f31cdf640 1 -- 172.21.15.64:0/2737201575 >> 172.21.15.64:0/2737201575 conn(0x7f1f2c0d1a90 msgr2=0x7f1f2c0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f31cdf640 1 -- 172.21.15.64:0/2737201575 shutdown_connections 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.642+0000 7f1f31cdf640 1 -- 172.21.15.64:0/2737201575 wait complete. 2024-03-04T07:29:11.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.643+0000 7f1f31cdf640 1 Processor -- start 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f31cdf640 1 -- start start 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f31cdf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f2c0b1400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f31cdf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c1520b0 0x7f1f2c0c7af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f31cdf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f2c151600 0x7f1f2c0c8030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f31cdf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0ccc60 con 0x7f1f14003f30 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f31cdf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0cc980 con 0x7f1f2c151600 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f31cdf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f2c0cdcc0 con 0x7f1f2c1520b0 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f2c0b1400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f2c0b1400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34634/0 (socket says 172.21.15.64:34634) 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.644+0000 7f1f2b7fe640 1 -- 172.21.15.64:0/3701856889 learned_addr learned my addr 172.21.15.64:0/3701856889 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2affd640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c1520b0 0x7f1f2c0c7af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2bfff640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f2c151600 0x7f1f2c0c8030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2b7fe640 1 -- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c1520b0 msgr2=0x7f1f2c0c7af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2b7fe640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c1520b0 0x7f1f2c0c7af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2b7fe640 1 -- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f2c151600 msgr2=0x7f1f2c0c8030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2b7fe640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f2c151600 0x7f1f2c0c8030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2b7fe640 1 -- 172.21.15.64:0/3701856889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f2c0b2860 con 0x7f1f14003f30 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2bfff640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f2c151600 0x7f1f2c0c8030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f2b7fe640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f2c0b1400 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f1f1c011910 tx=0x7f1f1c011dd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f13fff640 1 -- 172.21.15.64:0/3701856889 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f1c00fa90 con 0x7f1f14003f30 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f13fff640 1 -- 172.21.15.64:0/3701856889 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f1c00fc30 con 0x7f1f14003f30 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f13fff640 1 -- 172.21.15.64:0/3701856889 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f1c014440 con 0x7f1f14003f30 2024-03-04T07:29:11.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1f2c0b1ce0 con 0x7f1f14003f30 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.645+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1f2c0b22a0 con 0x7f1f14003f30 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.646+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1f2c067ff0 con 0x7f1f14003f30 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.646+0000 7f1f13fff640 1 -- 172.21.15.64:0/3701856889 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1f1c012040 con 0x7f1f14003f30 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.647+0000 7f1f13fff640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f1806b500 0x7f1f1806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.647+0000 7f1f13fff640 1 -- 172.21.15.64:0/3701856889 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1f1c08a620 con 0x7f1f14003f30 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.647+0000 7f1f2affd640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f1806b500 0x7f1f1806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.647+0000 7f1f2affd640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f1806b500 0x7f1f1806d9c0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f1f20034000 tx=0x7f1f20007000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:11.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.651+0000 7f1f13fff640 1 -- 172.21.15.64:0/3701856889 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1f1c019030 con 0x7f1f14003f30 2024-03-04T07:29:11.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.832+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1f2c06f2f0 con 0x7f1f14003f30 2024-03-04T07:29:11.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.833+0000 7f1f13fff640 1 -- 172.21.15.64:0/3701856889 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v54) v1 ==== 74+0+16630 (secure 0 0 0) 0x7f1f1c0594a0 con 0x7f1f14003f30 2024-03-04T07:29:11.833 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:11.833 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":54,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:10.861108+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:11.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f1806b500 msgr2=0x7f1f1806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f1806b500 0x7f1f1806d9c0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f1f20034000 tx=0x7f1f20007000 comp rx=0 tx=0).stop 2024-03-04T07:29:11.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 msgr2=0x7f1f2c0b1400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f2c0b1400 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f1f1c011910 tx=0x7f1f1c011dd0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 shutdown_connections 2024-03-04T07:29:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f1806b500 0x7f1f1806d9c0 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f2c151600 0x7f1f2c0c8030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f2c1520b0 0x7f1f2c0c7af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 --2- 172.21.15.64:0/3701856889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f14003f30 0x7f1f2c0b1400 unknown :-1 s=CLOSED pgs=655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 >> 172.21.15.64:0/3701856889 conn(0x7f1f2c0d1a90 msgr2=0x7f1f2c0afa10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 shutdown_connections 2024-03-04T07:29:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.835+0000 7f1f31cdf640 1 -- 172.21.15.64:0/3701856889 wait complete. 2024-03-04T07:29:11.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:11.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.948+0000 7f755615e640 1 Processor -- start 2024-03-04T07:29:11.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.948+0000 7f755615e640 1 -- start start 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f755615e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f755006e700 0x7f755015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f755615e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75500b4160 con 0x7f75500b55d0 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f755615e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75500b5b40 con 0x7f7550072660 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f755615e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75500b3ba0 con 0x7f755006e700 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f755006e700 0x7f755015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f75500b55d0 0x7f7550071980 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51378/0 (socket says 172.21.15.64:51378) 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f755006e700 0x7f755015ef80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34644/0 (socket says 172.21.15.64:34644) 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754ffff640 1 -- 172.21.15.64:0/1965386784 learned_addr learned my addr 172.21.15.64:0/1965386784 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754f7fe640 1 -- 172.21.15.64:0/1965386784 >> v1:172.21.15.66:6789/0 conn(0x7f7550072660 legacy=0x7f755006d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754f7fe640 1 -- 172.21.15.64:0/1965386784 >> v1:172.21.15.64:6789/0 conn(0x7f75500b55d0 legacy=0x7f7550071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754e7fc640 1 -- 172.21.15.64:0/1965386784 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1772117306 0 0) 0x7f75500b4160 con 0x7f75500b55d0 2024-03-04T07:29:11.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.949+0000 7f754f7fe640 1 -- 172.21.15.64:0/1965386784 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75500b3e80 con 0x7f755006e700 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.950+0000 7f754f7fe640 1 --2- 172.21.15.64:0/1965386784 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f755015ef80 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7f75440097a0 tx=0x7f754402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c1a8bbb952fb1f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.950+0000 7f754e7fc640 1 -- 172.21.15.64:0/1965386784 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7544035020 con 0x7f755006e700 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.950+0000 7f754e7fc640 1 -- 172.21.15.64:0/1965386784 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7544031830 con 0x7f755006e700 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.950+0000 7f754e7fc640 1 -- 172.21.15.64:0/1965386784 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7544031c00 con 0x7f755006e700 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.950+0000 7f755615e640 1 -- 172.21.15.64:0/1965386784 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 msgr2=0x7f755015ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.950+0000 7f755615e640 1 --2- 172.21.15.64:0/1965386784 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f755015ef80 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7f75440097a0 tx=0x7f754402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.950+0000 7f755615e640 1 -- 172.21.15.64:0/1965386784 shutdown_connections 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.951+0000 7f755615e640 1 --2- 172.21.15.64:0/1965386784 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f755015ef80 unknown :-1 s=CLOSED pgs=657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.951+0000 7f755615e640 1 -- 172.21.15.64:0/1965386784 >> 172.21.15.64:0/1965386784 conn(0x7f75500b2720 msgr2=0x7f75500b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.951+0000 7f755615e640 1 -- 172.21.15.64:0/1965386784 shutdown_connections 2024-03-04T07:29:11.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.951+0000 7f755615e640 1 -- 172.21.15.64:0/1965386784 wait complete. 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.952+0000 7f755615e640 1 Processor -- start 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.952+0000 7f755615e640 1 -- start start 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f755615e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f75500cf130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f755615e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7550072660 0x7f75501f7ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f755615e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75500b55d0 0x7f7550156e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f755615e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7550072210 con 0x7f755006e700 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f755615e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7550072090 con 0x7f7550072660 2024-03-04T07:29:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f755615e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75500670e0 con 0x7f75500b55d0 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f75500cf130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f75500cf130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34648/0 (socket says 172.21.15.64:34648) 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 -- 172.21.15.64:0/4022114300 learned_addr learned my addr 172.21.15.64:0/4022114300 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754effd640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7550072660 0x7f75501f7ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754ffff640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75500b55d0 0x7f7550156e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 -- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75500b55d0 msgr2=0x7f7550156e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75500b55d0 0x7f7550156e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 -- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7550072660 msgr2=0x7f75501f7ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7550072660 0x7f75501f7ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 -- 172.21.15.64:0/4022114300 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75500cf7f0 con 0x7f755006e700 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754effd640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7550072660 0x7f75501f7ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f754f7fe640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f75500cf130 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f75440097a0 tx=0x7f75440046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f7537fff640 1 -- 172.21.15.64:0/4022114300 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7544004230 con 0x7f755006e700 2024-03-04T07:29:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f7537fff640 1 -- 172.21.15.64:0/4022114300 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75440043d0 con 0x7f755006e700 2024-03-04T07:29:11.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f7537fff640 1 -- 172.21.15.64:0/4022114300 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f754403cad0 con 0x7f755006e700 2024-03-04T07:29:11.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.953+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f75500cfad0 con 0x7f755006e700 2024-03-04T07:29:11.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.954+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f75500d0090 con 0x7f755006e700 2024-03-04T07:29:11.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.955+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7550067ff0 con 0x7f755006e700 2024-03-04T07:29:11.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.955+0000 7f7537fff640 1 -- 172.21.15.64:0/4022114300 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7544038020 con 0x7f755006e700 2024-03-04T07:29:11.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.955+0000 7f7537fff640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f753c06b4d0 0x7f753c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:11.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.956+0000 7f754effd640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f753c06b4d0 0x7f753c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:11.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.956+0000 7f7537fff640 1 -- 172.21.15.64:0/4022114300 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f75440ac430 con 0x7f755006e700 2024-03-04T07:29:11.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.956+0000 7f754effd640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f753c06b4d0 0x7f753c06d990 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f7550071da0 tx=0x7f7540009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:11.960+0000 7f7537fff640 1 -- 172.21.15.64:0/4022114300 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7544041070 con 0x7f755006e700 2024-03-04T07:29:12.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.143+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7550072c30 con 0x7f755006e700 2024-03-04T07:29:12.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.144+0000 7f7537fff640 1 -- 172.21.15.64:0/4022114300 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v54) v1 ==== 74+0+16630 (secure 0 0 0) 0x7f754407b2b0 con 0x7f755006e700 2024-03-04T07:29:12.143 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:12.144 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":54,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:10.861108+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.145+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f753c06b4d0 msgr2=0x7f753c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.145+0000 7f755615e640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f753c06b4d0 0x7f753c06d990 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f7550071da0 tx=0x7f7540009040 comp rx=0 tx=0).stop 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 msgr2=0x7f75500cf130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f75500cf130 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f75440097a0 tx=0x7f75440046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 shutdown_connections 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f753c06b4d0 0x7f753c06d990 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75500b55d0 0x7f7550156e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7550072660 0x7f75501f7ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 --2- 172.21.15.64:0/4022114300 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f755006e700 0x7f75500cf130 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 >> 172.21.15.64:0/4022114300 conn(0x7f75500b2720 msgr2=0x7f755015c090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 shutdown_connections 2024-03-04T07:29:12.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.146+0000 7f755615e640 1 -- 172.21.15.64:0/4022114300 wait complete. 2024-03-04T07:29:12.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:29:12.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.261+0000 7fa8dc81c640 1 Processor -- start 2024-03-04T07:29:12.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.261+0000 7fa8dc81c640 1 -- start start 2024-03-04T07:29:12.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8dc81c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa8d80ceea0 0x7fa8d80d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8dc81c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa8d80b6d10 0x7fa8d80b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8dc81c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa8d80d3740 con 0x7fa8d815dd90 2024-03-04T07:29:12.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8dc81c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa8d80d3400 con 0x7fa8d80b6d10 2024-03-04T07:29:12.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8dc81c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa8d80d3120 con 0x7fa8d80ceea0 2024-03-04T07:29:12.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8d6d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa8d80b6d10 0x7fa8d80b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8d6d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa8d80b6d10 0x7fa8d80b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34662/0 (socket says 172.21.15.64:34662) 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.262+0000 7fa8d6d9d640 1 -- 172.21.15.64:0/1228389757 learned_addr learned my addr 172.21.15.64:0/1228389757 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.263+0000 7fa8cffff640 1 --2- 172.21.15.64:0/1228389757 >> v2:172.21.15.66:3300/0 conn(0x7fa8d80ceea0 0x7fa8d80d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.263+0000 7fa8d6d9d640 1 -- 172.21.15.64:0/1228389757 >> v2:172.21.15.66:3300/0 conn(0x7fa8d80ceea0 msgr2=0x7fa8d80d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.263+0000 7fa8d6d9d640 1 --2- 172.21.15.64:0/1228389757 >> v2:172.21.15.66:3300/0 conn(0x7fa8d80ceea0 0x7fa8d80d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.263+0000 7fa8d6d9d640 1 -- 172.21.15.64:0/1228389757 >> v1:172.21.15.66:6789/0 conn(0x7fa8d815dd90 legacy=0x7fa8d80d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.263+0000 7fa8d6d9d640 1 -- 172.21.15.64:0/1228389757 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8d80d39e0 con 0x7fa8d80b6d10 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.263+0000 7fa8d6d9d640 1 --2- 172.21.15.64:0/1228389757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80b71a0 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7fa8d80724d0 tx=0x7fa8d002c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4f5770eeae12086c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8d5d9b640 1 -- 172.21.15.64:0/1228389757 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8d002cbb0 con 0x7fa8d80b6d10 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8d5d9b640 1 -- 172.21.15.64:0/1228389757 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8d002cd50 con 0x7fa8d80b6d10 2024-03-04T07:29:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8d5d9b640 1 -- 172.21.15.64:0/1228389757 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8d0034610 con 0x7fa8d80b6d10 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1228389757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 msgr2=0x7fa8d80b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1228389757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80b71a0 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7fa8d80724d0 tx=0x7fa8d002c010 comp rx=0 tx=0).stop 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1228389757 shutdown_connections 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1228389757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80b71a0 unknown :-1 s=CLOSED pgs=659 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1228389757 >> v2:172.21.15.66:3300/0 conn(0x7fa8d80ceea0 0x7fa8d80d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1228389757 >> 172.21.15.64:0/1228389757 conn(0x7fa8d8071960 msgr2=0x7fa8d8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1228389757 shutdown_connections 2024-03-04T07:29:12.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.264+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1228389757 wait complete. 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.265+0000 7fa8dc81c640 1 Processor -- start 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.265+0000 7fa8dc81c640 1 -- start start 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8dc81c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80c6340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8dc81c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8d80ceea0 0x7fa8d8158170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8dc81c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8d815dd90 0x7fa8d815e380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8dc81c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8d80d48f0 con 0x7fa8d80b6d10 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8dc81c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8d815e170 con 0x7fa8d80ceea0 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8dc81c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa8d80d4a70 con 0x7fa8d815dd90 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8d659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80c6340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8d659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80c6340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34676/0 (socket says 172.21.15.64:34676) 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8d659c640 1 -- 172.21.15.64:0/1685739747 learned_addr learned my addr 172.21.15.64:0/1685739747 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8cffff640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8d80ceea0 0x7fa8d8158170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.266+0000 7fa8d6d9d640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8d815dd90 0x7fa8d815e380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8d659c640 1 -- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8d815dd90 msgr2=0x7fa8d815e380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8d659c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8d815dd90 0x7fa8d815e380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8d659c640 1 -- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8d80ceea0 msgr2=0x7fa8d8158170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8d659c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8d80ceea0 0x7fa8d8158170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8d659c640 1 -- 172.21.15.64:0/1685739747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8d80c80a0 con 0x7fa8d80b6d10 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8d659c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80c6340 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7fa8c800c3c0 tx=0x7fa8c800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8ceffd640 1 -- 172.21.15.64:0/1685739747 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8c80052d0 con 0x7fa8d80b6d10 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8ceffd640 1 -- 172.21.15.64:0/1685739747 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa8c8005490 con 0x7fa8d80b6d10 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa8d80c8380 con 0x7fa8d80b6d10 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8ceffd640 1 -- 172.21.15.64:0/1685739747 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8c8007260 con 0x7fa8d80b6d10 2024-03-04T07:29:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.267+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8d80c8940 con 0x7fa8d80b6d10 2024-03-04T07:29:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.268+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa8d8072a30 con 0x7fa8d80b6d10 2024-03-04T07:29:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.268+0000 7fa8ceffd640 1 -- 172.21.15.64:0/1685739747 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa8c8007400 con 0x7fa8d80b6d10 2024-03-04T07:29:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.269+0000 7fa8ceffd640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa8b806b4d0 0x7fa8b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.269+0000 7fa8cffff640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa8b806b4d0 0x7fa8b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.269+0000 7fa8ceffd640 1 -- 172.21.15.64:0/1685739747 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(54..54 src has 1..54) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa8c80896e0 con 0x7fa8d80b6d10 2024-03-04T07:29:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.269+0000 7fa8cffff640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa8b806b4d0 0x7fa8b806d990 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7fa8d8159240 tx=0x7fa8c0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:12.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.273+0000 7fa8ceffd640 1 -- 172.21.15.64:0/1685739747 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa8c8058560 con 0x7fa8d80b6d10 2024-03-04T07:29:12.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.491+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7fa8d8068480 con 0x7fa8d80b6d10 2024-03-04T07:29:12.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.504+0000 7fa8ceffd640 1 -- 172.21.15.64:0/1685739747 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v28) v1 ==== 118+0+0 (secure 0 0 0) 0x7fa8c8057e60 con 0x7fa8d80b6d10 2024-03-04T07:29:12.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa8b806b4d0 msgr2=0x7fa8b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa8b806b4d0 0x7fa8b806d990 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7fa8d8159240 tx=0x7fa8c0008040 comp rx=0 tx=0).stop 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 msgr2=0x7fa8d80c6340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80c6340 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7fa8c800c3c0 tx=0x7fa8c800c880 comp rx=0 tx=0).stop 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 shutdown_connections 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa8b806b4d0 0x7fa8b806d990 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa8d815dd90 0x7fa8d815e380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa8d80ceea0 0x7fa8d8158170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 --2- 172.21.15.64:0/1685739747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa8d80b6d10 0x7fa8d80c6340 unknown :-1 s=CLOSED pgs=660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 >> 172.21.15.64:0/1685739747 conn(0x7fa8d8071960 msgr2=0x7fa8d80d0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 shutdown_connections 2024-03-04T07:29:12.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.506+0000 7fa8dc81c640 1 -- 172.21.15.64:0/1685739747 wait complete. 2024-03-04T07:29:12.519 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:29:12.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:29:12.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.625+0000 7f47ae271640 1 Processor -- start 2024-03-04T07:29:12.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47ae271640 1 -- start start 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47ae271640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 0x7f47a80ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47ae271640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f47a80c5e30 0x7f47a80c6280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47ae271640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f47a80cd310 con 0x7f47a80cf740 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47ae271640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f47a815f8e0 con 0x7f47a80c5e30 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47ae271640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47a80c56c0 con 0x7f47a815f500 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47a77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f47a80cf740 0x7f47a80ce840 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51386/0 (socket says 172.21.15.64:51386) 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.626+0000 7f47a77fe640 1 -- 172.21.15.64:0/3225991357 learned_addr learned my addr 172.21.15.64:0/3225991357 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47ac80e640 1 --2- 172.21.15.64:0/3225991357 >> v2:172.21.15.66:3300/0 conn(0x7f47a80c5e30 0x7f47a80c6280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a7fff640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 0x7f47a80ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1614049117 0 0) 0x7f47a80cd310 con 0x7f47a80cf740 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4790003520 con 0x7f47a80cf740 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1528471706 0 0) 0x7f4790003520 con 0x7f47a80cf740 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 msgr2=0x7f47a80ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 0x7f47a80ce300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 >> v2:172.21.15.66:3300/0 conn(0x7f47a80c5e30 msgr2=0x7f47a80c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 --2- 172.21.15.64:0/3225991357 >> v2:172.21.15.66:3300/0 conn(0x7f47a80c5e30 0x7f47a80c6280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47a80cd5b0 con 0x7f47a80cf740 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f479c001690 con 0x7f47a80cf740 2024-03-04T07:29:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a7fff640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 0x7f47a80ce300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47ac80e640 1 --2- 172.21.15.64:0/3225991357 >> v2:172.21.15.66:3300/0 conn(0x7f47a80c5e30 0x7f47a80c6280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 >> v1:172.21.15.64:6789/0 conn(0x7f47a80cf740 legacy=0x7f47a80ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47900039b0 0x7f4790003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 0x7f4790004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4790004d40 0x7f47900051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47a815f8e0 con 0x7f4790004d40 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47a80c56c0 con 0x7f47900039b0 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f479000b9c0 con 0x7f47a815f500 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47ac80e640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4790004d40 0x7f47900051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a7fff640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47900039b0 0x7f4790003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.627+0000 7f47a77fe640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 0x7f4790004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47ac80e640 1 -- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 msgr2=0x7f4790004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47ac80e640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a815f500 0x7f4790004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47ac80e640 1 -- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47900039b0 msgr2=0x7f4790003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47ac80e640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47900039b0 0x7f4790003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47ac80e640 1 -- 172.21.15.64:0/3225991357 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4790003520 con 0x7f4790004d40 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47a7fff640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47900039b0 0x7f4790003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47ac80e640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4790004d40 0x7f47900051c0 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f47a000e350 tx=0x7f4790005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b702766da9534eb6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47a0039070 con 0x7f4790004d40 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f47a002cdf0 con 0x7f4790004d40 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.628+0000 7f47a6ffd640 1 -- 172.21.15.64:0/3225991357 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47a0009de0 con 0x7f4790004d40 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.629+0000 7f47ae271640 1 -- 172.21.15.64:0/3225991357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4790004d40 msgr2=0x7f47900051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.629+0000 7f47ae271640 1 --2- 172.21.15.64:0/3225991357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4790004d40 0x7f47900051c0 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f47a000e350 tx=0x7f4790005830 comp rx=0 tx=0).stop 2024-03-04T07:29:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.629+0000 7f47ac80e640 1 -- 172.21.15.64:0/3225991357 reap_dead start 2024-03-04T07:29:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.629+0000 7f47ae271640 1 -- 172.21.15.64:0/3225991357 shutdown_connections 2024-03-04T07:29:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.629+0000 7f47ae271640 1 -- 172.21.15.64:0/3225991357 >> 172.21.15.64:0/3225991357 conn(0x7f47a80b3700 msgr2=0x7f47a80b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.629+0000 7f47ae271640 1 -- 172.21.15.64:0/3225991357 shutdown_connections 2024-03-04T07:29:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.629+0000 7f47ae271640 1 -- 172.21.15.64:0/3225991357 wait complete. 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.630+0000 7f47ae271640 1 Processor -- start 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.630+0000 7f47ae271640 1 -- start start 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47ae271640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47a80c9df0 0x7f47a806b850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47ae271640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47a806cce0 0x7f47a8150780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47ae271640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a80ca1d0 0x7f47a815c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47ae271640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47a80b43d0 con 0x7f47a806cce0 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47ae271640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47a80c59a0 con 0x7f47a80c9df0 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47ae271640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47a80cef50 con 0x7f47a80ca1d0 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47a806cce0 0x7f47a8150780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47a806cce0 0x7f47a8150780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34696/0 (socket says 172.21.15.64:34696) 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 -- 172.21.15.64:0/843357450 learned_addr learned my addr 172.21.15.64:0/843357450 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:12.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a7fff640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47a80c9df0 0x7f47a806b850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47ac80e640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a80ca1d0 0x7f47a815c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 -- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a80ca1d0 msgr2=0x7f47a815c8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a80ca1d0 0x7f47a815c8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 -- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47a80c9df0 msgr2=0x7f47a806b850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47a80c9df0 0x7f47a806b850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.631+0000 7f47a77fe640 1 -- 172.21.15.64:0/843357450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47a81516b0 con 0x7f47a806cce0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47a7fff640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47a80c9df0 0x7f47a806b850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47ac80e640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a80ca1d0 0x7f47a815c8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47a77fe640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47a806cce0 0x7f47a8150780 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f479c004840 tx=0x7f479c007720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47a4ff9640 1 -- 172.21.15.64:0/843357450 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f479c00f990 con 0x7f47a806cce0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47a4ff9640 1 -- 172.21.15.64:0/843357450 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f479c018070 con 0x7f47a806cce0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47a81510f0 con 0x7f47a806cce0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47a4ff9640 1 -- 172.21.15.64:0/843357450 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f479c00fd90 con 0x7f47a806cce0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.632+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47a806b050 con 0x7f47a806cce0 2024-03-04T07:29:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.633+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f47a809fcf0 con 0x7f47a806cce0 2024-03-04T07:29:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.633+0000 7f47a4ff9640 1 -- 172.21.15.64:0/843357450 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f479c01d460 con 0x7f47a806cce0 2024-03-04T07:29:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.634+0000 7f47a4ff9640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f479406b4d0 0x7f479406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.634+0000 7f47a4ff9640 1 -- 172.21.15.64:0/843357450 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f479c08a390 con 0x7f47a806cce0 2024-03-04T07:29:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.634+0000 7f47a7fff640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f479406b4d0 0x7f479406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.634+0000 7f47a7fff640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f479406b4d0 0x7f479406d990 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f4798009560 tx=0x7f4798008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.638+0000 7f47a4ff9640 1 -- 172.21.15.64:0/843357450 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f479c0592a0 con 0x7f47a806cce0 2024-03-04T07:29:12.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:12.858+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f47a80cbf00 con 0x7f47a806cce0 2024-03-04T07:29:13.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.512+0000 7f47a4ff9640 1 -- 172.21.15.64:0/843357450 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v29) v1 ==== 122+0+0 (secure 0 0 0) 0x7f479c058ba0 con 0x7f47a806cce0 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f479406b4d0 msgr2=0x7f479406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f479406b4d0 0x7f479406d990 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f4798009560 tx=0x7f4798008040 comp rx=0 tx=0).stop 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47a806cce0 msgr2=0x7f47a8150780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47a806cce0 0x7f47a8150780 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f479c004840 tx=0x7f479c007720 comp rx=0 tx=0).stop 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 shutdown_connections 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f479406b4d0 0x7f479406d990 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47a80ca1d0 0x7f47a815c8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47a806cce0 0x7f47a8150780 unknown :-1 s=CLOSED pgs=663 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 --2- 172.21.15.64:0/843357450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47a80c9df0 0x7f47a806b850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 >> 172.21.15.64:0/843357450 conn(0x7f47a80b3700 msgr2=0x7f47a80c7730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 shutdown_connections 2024-03-04T07:29:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.515+0000 7f47ae271640 1 -- 172.21.15.64:0/843357450 wait complete. 2024-03-04T07:29:13.526 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:29:13.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:13.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.628+0000 7f4911c81640 1 Processor -- start 2024-03-04T07:29:13.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.628+0000 7f4911c81640 1 -- start start 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f4911c81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f4911c81640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f490c0b95a0 0x7f490c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f4911c81640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f490c0cea30 con 0x7f490c0b9a30 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f4911c81640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f490c0d2af0 con 0x7f490c0b95a0 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f4911c81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f490c0cf3c0 con 0x7f490c15df00 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f490bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f490c0b9a30 0x7f490c0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35664/0 (socket says 172.21.15.64:35664) 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f490bfff640 1 -- 172.21.15.64:0/3341485207 learned_addr learned my addr 172.21.15.64:0/3341485207 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.629+0000 7f490affd640 1 --2- 172.21.15.64:0/3341485207 >> v2:172.21.15.66:3300/0 conn(0x7f490c0b95a0 0x7f490c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.630+0000 7f490b7fe640 1 --2- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.630+0000 7f490a7fc640 1 -- 172.21.15.64:0/3341485207 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3622950185 0 0) 0x7f490c0cea30 con 0x7f490c0b9a30 2024-03-04T07:29:13.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.630+0000 7f490a7fc640 1 -- 172.21.15.64:0/3341485207 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f48f4003520 con 0x7f490c0b9a30 2024-03-04T07:29:13.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.630+0000 7f490affd640 1 -- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 msgr2=0x7f490c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:13.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.630+0000 7f490affd640 1 --2- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.630+0000 7f490affd640 1 -- 172.21.15.64:0/3341485207 >> v1:172.21.15.64:6789/0 conn(0x7f490c0b9a30 legacy=0x7f490c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.630+0000 7f490affd640 1 -- 172.21.15.64:0/3341485207 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f490c0cf0e0 con 0x7f490c0b95a0 2024-03-04T07:29:13.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f490affd640 1 --2- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 0x7f490c0b5700 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f49000097a0 tx=0x7f490002b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6f91c241d68ba7c4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:13.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f490a7fc640 1 -- 172.21.15.64:0/3341485207 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4900035020 con 0x7f490c0b95a0 2024-03-04T07:29:13.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f490a7fc640 1 -- 172.21.15.64:0/3341485207 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4900031830 con 0x7f490c0b95a0 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f490a7fc640 1 -- 172.21.15.64:0/3341485207 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4900031c00 con 0x7f490c0b95a0 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 -- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 msgr2=0x7f490c0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 --2- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 0x7f490c0b5700 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f49000097a0 tx=0x7f490002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 -- 172.21.15.64:0/3341485207 shutdown_connections 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 --2- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 0x7f490c0b5700 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 --2- 172.21.15.64:0/3341485207 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 -- 172.21.15.64:0/3341485207 >> 172.21.15.64:0/3341485207 conn(0x7f490c071960 msgr2=0x7f490c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 -- 172.21.15.64:0/3341485207 shutdown_connections 2024-03-04T07:29:13.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.631+0000 7f4911c81640 1 -- 172.21.15.64:0/3341485207 wait complete. 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.633+0000 7f4911c81640 1 Processor -- start 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.633+0000 7f4911c81640 1 -- start start 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f4911c81640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 0x7f490c0b3c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f4911c81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0b42d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f4911c81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f490c0b4810 0x7f490c0cc320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f4911c81640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f490c0cec70 con 0x7f490c0b4810 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f4911c81640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f490c0ce750 con 0x7f490c0b95a0 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f4911c81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f490c0cedf0 con 0x7f490c15df00 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f490c0b4810 0x7f490c0cc320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f490c0b4810 0x7f490c0cc320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39230/0 (socket says 172.21.15.64:39230) 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 -- 172.21.15.64:0/1560320390 learned_addr learned my addr 172.21.15.64:0/1560320390 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490b7fe640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 0x7f490c0b3c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490affd640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0b42d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 -- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 msgr2=0x7f490c0b42d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0b42d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 -- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 msgr2=0x7f490c0b3c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 0x7f490c0b3c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 -- 172.21.15.64:0/1560320390 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f490c0b4d00 con 0x7f490c0b4810 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.635+0000 7f490bfff640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f490c0b4810 0x7f490c0cc320 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f490400cbe0 tx=0x7f490400a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:13.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.636+0000 7f48f3fff640 1 -- 172.21.15.64:0/1560320390 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4904017070 con 0x7f490c0b4810 2024-03-04T07:29:13.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.636+0000 7f48f3fff640 1 -- 172.21.15.64:0/1560320390 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f490400ad40 con 0x7f490c0b4810 2024-03-04T07:29:13.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.636+0000 7f48f3fff640 1 -- 172.21.15.64:0/1560320390 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4904012600 con 0x7f490c0b4810 2024-03-04T07:29:13.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.636+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f490c0b4fe0 con 0x7f490c0b4810 2024-03-04T07:29:13.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.636+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f490c0b55a0 con 0x7f490c0b4810 2024-03-04T07:29:13.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.637+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f490c067ff0 con 0x7f490c0b4810 2024-03-04T07:29:13.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.637+0000 7f48f3fff640 1 -- 172.21.15.64:0/1560320390 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f490401a070 con 0x7f490c0b4810 2024-03-04T07:29:13.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.638+0000 7f48f3fff640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f48f806b4d0 0x7f48f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.638+0000 7f490b7fe640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f48f806b4d0 0x7f48f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.638+0000 7f48f3fff640 1 -- 172.21.15.64:0/1560320390 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4904089f80 con 0x7f490c0b4810 2024-03-04T07:29:13.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.638+0000 7f490b7fe640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f48f806b4d0 0x7f48f806d990 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f48fc009aa0 tx=0x7f48fc009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:13.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.642+0000 7f48f3fff640 1 -- 172.21.15.64:0/1560320390 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4904058730 con 0x7f490c0b4810 2024-03-04T07:29:13.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.864+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f490c06b780 con 0x7f490c0b4810 2024-03-04T07:29:13.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.865+0000 7f48f3fff640 1 -- 172.21.15.64:0/1560320390 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f490405c7d0 con 0x7f490c0b4810 2024-03-04T07:29:13.864 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:13.865 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:13.865 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:13.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f48f806b4d0 msgr2=0x7f48f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f48f806b4d0 0x7f48f806d990 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f48fc009aa0 tx=0x7f48fc009190 comp rx=0 tx=0).stop 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f490c0b4810 msgr2=0x7f490c0cc320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f490c0b4810 0x7f490c0cc320 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f490400cbe0 tx=0x7f490400a430 comp rx=0 tx=0).stop 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 shutdown_connections 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f48f806b4d0 0x7f48f806d990 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f490c0b4810 0x7f490c0cc320 unknown :-1 s=CLOSED pgs=665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f490c15df00 0x7f490c0b42d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 --2- 172.21.15.64:0/1560320390 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f490c0b95a0 0x7f490c0b3c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.866+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 >> 172.21.15.64:0/1560320390 conn(0x7f490c071960 msgr2=0x7f490c0b8400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.867+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 shutdown_connections 2024-03-04T07:29:13.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.867+0000 7f4911c81640 1 -- 172.21.15.64:0/1560320390 wait complete. 2024-03-04T07:29:13.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:13.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.980+0000 7f95586c2640 1 Processor -- start 2024-03-04T07:29:13.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.980+0000 7f95586c2640 1 -- start start 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f95586c2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f95500d06b0 0x7f95500ce3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f95586c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500ce900 0x7f95500b13a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f95586c2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f95500cf610 con 0x7f955015bdc0 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f95586c2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f95500cf8f0 con 0x7f95500d06b0 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f95586c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95500cfbd0 con 0x7f95500ce900 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9555c5d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f95500d06b0 0x7f95500ce3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9555c5d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f95500d06b0 0x7f95500ce3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39238/0 (socket says 172.21.15.64:39238) 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9555c5d640 1 -- 172.21.15.64:0/1119997757 learned_addr learned my addr 172.21.15.64:0/1119997757 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9556c5f640 1 --2- 172.21.15.64:0/1119997757 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500ce900 0x7f95500b13a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9555c5d640 1 -- 172.21.15.64:0/1119997757 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500ce900 msgr2=0x7f95500b13a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9555c5d640 1 --2- 172.21.15.64:0/1119997757 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500ce900 0x7f95500b13a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9555c5d640 1 -- 172.21.15.64:0/1119997757 >> v1:172.21.15.66:6789/0 conn(0x7f955015bdc0 legacy=0x7f95500cd210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.981+0000 7f9555c5d640 1 -- 172.21.15.64:0/1119997757 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95500cfeb0 con 0x7f95500d06b0 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.982+0000 7f9555c5d640 1 --2- 172.21.15.64:0/1119997757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95500d06b0 0x7f95500ce3c0 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f9540009870 tx=0x7f954002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b25e74f9e52d8a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.982+0000 7f955545c640 1 -- 172.21.15.64:0/1119997757 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f954002f840 con 0x7f95500d06b0 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.982+0000 7f955545c640 1 -- 172.21.15.64:0/1119997757 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f954002f9e0 con 0x7f95500d06b0 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.982+0000 7f955545c640 1 -- 172.21.15.64:0/1119997757 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f954002fdb0 con 0x7f95500d06b0 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 -- 172.21.15.64:0/1119997757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95500d06b0 msgr2=0x7f95500ce3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 --2- 172.21.15.64:0/1119997757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95500d06b0 0x7f95500ce3c0 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f9540009870 tx=0x7f954002ba20 comp rx=0 tx=0).stop 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 -- 172.21.15.64:0/1119997757 shutdown_connections 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 --2- 172.21.15.64:0/1119997757 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500ce900 0x7f95500b13a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 --2- 172.21.15.64:0/1119997757 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95500d06b0 0x7f95500ce3c0 unknown :-1 s=CLOSED pgs=666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 -- 172.21.15.64:0/1119997757 >> 172.21.15.64:0/1119997757 conn(0x7f95500d1a40 msgr2=0x7f95500d1e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:13.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 -- 172.21.15.64:0/1119997757 shutdown_connections 2024-03-04T07:29:13.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.983+0000 7f95586c2640 1 -- 172.21.15.64:0/1119997757 wait complete. 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.984+0000 7f95586c2640 1 Processor -- start 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.984+0000 7f95586c2640 1 -- start start 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95500ce900 0x7f95500ca150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500d06b0 0x7f95500c46b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f955015bdc0 0x7f95500cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95500d2760 con 0x7f955015bdc0 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95500d0b10 con 0x7f95500ce900 2024-03-04T07:29:13.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f955015c1a0 con 0x7f95500d06b0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f955015bdc0 0x7f95500cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f955015bdc0 0x7f95500cad50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39242/0 (socket says 172.21.15.64:39242) 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 -- 172.21.15.64:0/2604533329 learned_addr learned my addr 172.21.15.64:0/2604533329 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f955645e640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95500ce900 0x7f95500ca150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9555c5d640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500d06b0 0x7f95500c46b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 -- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500d06b0 msgr2=0x7f95500c46b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500d06b0 0x7f95500c46b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 -- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95500ce900 msgr2=0x7f95500ca150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95500ce900 0x7f95500ca150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 -- 172.21.15.64:0/2604533329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95500c4bf0 con 0x7f955015bdc0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9555c5d640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500d06b0 0x7f95500c46b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9556c5f640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f955015bdc0 0x7f95500cad50 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f9548007e40 tx=0x7f954800e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9546ffd640 1 -- 172.21.15.64:0/2604533329 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9548014620 con 0x7f955015bdc0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9546ffd640 1 -- 172.21.15.64:0/2604533329 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9548019070 con 0x7f955015bdc0 2024-03-04T07:29:13.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95500b6780 con 0x7f955015bdc0 2024-03-04T07:29:13.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f9546ffd640 1 -- 172.21.15.64:0/2604533329 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9548014a20 con 0x7f955015bdc0 2024-03-04T07:29:13.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.985+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95500b6d40 con 0x7f955015bdc0 2024-03-04T07:29:13.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.987+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9550068700 con 0x7f955015bdc0 2024-03-04T07:29:13.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.987+0000 7f9546ffd640 1 -- 172.21.15.64:0/2604533329 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9548014bc0 con 0x7f955015bdc0 2024-03-04T07:29:13.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.987+0000 7f9546ffd640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953406b4d0 0x7f953406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:13.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.987+0000 7f9546ffd640 1 -- 172.21.15.64:0/2604533329 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f954808aab0 con 0x7f955015bdc0 2024-03-04T07:29:13.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.987+0000 7f955645e640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953406b4d0 0x7f953406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:13.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.988+0000 7f955645e640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953406b4d0 0x7f953406d990 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f954c0016e0 tx=0x7f954c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:13.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:13.991+0000 7f9546ffd640 1 -- 172.21.15.64:0/2604533329 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9548059910 con 0x7f955015bdc0 2024-03-04T07:29:14.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.181+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f95500c84f0 con 0x7f955015bdc0 2024-03-04T07:29:14.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.183+0000 7f9546ffd640 1 -- 172.21.15.64:0/2604533329 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v55) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f9548059210 con 0x7f955015bdc0 2024-03-04T07:29:14.182 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:14.183 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:12.504021+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:14.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953406b4d0 msgr2=0x7f953406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953406b4d0 0x7f953406d990 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f954c0016e0 tx=0x7f954c009040 comp rx=0 tx=0).stop 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f955015bdc0 msgr2=0x7f95500cad50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f955015bdc0 0x7f95500cad50 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f9548007e40 tx=0x7f954800e4b0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 shutdown_connections 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953406b4d0 0x7f953406d990 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f955015bdc0 0x7f95500cad50 unknown :-1 s=CLOSED pgs=667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500d06b0 0x7f95500c46b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 --2- 172.21.15.64:0/2604533329 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95500ce900 0x7f95500ca150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.184+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 >> 172.21.15.64:0/2604533329 conn(0x7f95500d1a40 msgr2=0x7f95500cd000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.185+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 shutdown_connections 2024-03-04T07:29:14.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.185+0000 7f95586c2640 1 -- 172.21.15.64:0/2604533329 wait complete. 2024-03-04T07:29:14.196 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:29:14.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:29:14.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.297+0000 7f1728442640 1 Processor -- start 2024-03-04T07:29:14.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.297+0000 7f1728442640 1 -- start start 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f1728442640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f17200b95a0 0x7f17200b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f1728442640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f17200b9a10 0x7f17200b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f1728442640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17200cea30 con 0x7f172015df00 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f1728442640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17200cf3c0 con 0x7f17200b95a0 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f1728442640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17200cf0e0 con 0x7f17200b9a10 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17259dd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f17200b95a0 0x7f17200b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17259dd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f17200b95a0 0x7f17200b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39248/0 (socket says 172.21.15.64:39248) 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17259dd640 1 -- 172.21.15.64:0/4196907417 learned_addr learned my addr 172.21.15.64:0/4196907417 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:14.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17269df640 1 --2- 172.21.15.64:0/4196907417 >> v2:172.21.15.66:3300/0 conn(0x7f17200b9a10 0x7f17200b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17259dd640 1 -- 172.21.15.64:0/4196907417 >> v2:172.21.15.66:3300/0 conn(0x7f17200b9a10 msgr2=0x7f17200b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:14.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17259dd640 1 --2- 172.21.15.64:0/4196907417 >> v2:172.21.15.66:3300/0 conn(0x7f17200b9a10 0x7f17200b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17259dd640 1 -- 172.21.15.64:0/4196907417 >> v1:172.21.15.66:6789/0 conn(0x7f172015df00 legacy=0x7f17200d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.298+0000 7f17259dd640 1 -- 172.21.15.64:0/4196907417 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17200ce470 con 0x7f17200b95a0 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.299+0000 7f17259dd640 1 --2- 172.21.15.64:0/4196907417 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f17200b5700 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7f17200d4d20 tx=0x7f171002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=267cac7dec4f0133 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.299+0000 7f17251dc640 1 -- 172.21.15.64:0/4196907417 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1710034070 con 0x7f17200b95a0 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.299+0000 7f17251dc640 1 -- 172.21.15.64:0/4196907417 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1710030860 con 0x7f17200b95a0 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.299+0000 7f17251dc640 1 -- 172.21.15.64:0/4196907417 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1710030c30 con 0x7f17200b95a0 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 -- 172.21.15.64:0/4196907417 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 msgr2=0x7f17200b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 --2- 172.21.15.64:0/4196907417 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f17200b5700 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7f17200d4d20 tx=0x7f171002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 -- 172.21.15.64:0/4196907417 shutdown_connections 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 --2- 172.21.15.64:0/4196907417 >> v2:172.21.15.66:3300/0 conn(0x7f17200b9a10 0x7f17200b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 --2- 172.21.15.64:0/4196907417 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f17200b5700 unknown :-1 s=CLOSED pgs=668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 -- 172.21.15.64:0/4196907417 >> 172.21.15.64:0/4196907417 conn(0x7f1720071960 msgr2=0x7f1720071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:14.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 -- 172.21.15.64:0/4196907417 shutdown_connections 2024-03-04T07:29:14.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.300+0000 7f1728442640 1 -- 172.21.15.64:0/4196907417 wait complete. 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.301+0000 7f1728442640 1 Processor -- start 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.301+0000 7f1728442640 1 -- start start 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f1728442640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f172015b300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f1728442640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17200b9a10 0x7f172015e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f1728442640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f172015df00 0x7f17201606a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f1728442640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17200cedf0 con 0x7f17200b95a0 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f1728442640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17200cec70 con 0x7f17200b9a10 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f1728442640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17200d3220 con 0x7f172015df00 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f172015b300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f172015b300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39260/0 (socket says 172.21.15.64:39260) 2024-03-04T07:29:14.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 -- 172.21.15.64:0/4247176790 learned_addr learned my addr 172.21.15.64:0/4247176790 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 -- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f172015df00 msgr2=0x7f17201606a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17259dd640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17200b9a10 0x7f172015e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17269df640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f172015df00 0x7f17201606a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f172015df00 0x7f17201606a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 -- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17200b9a10 msgr2=0x7f172015e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17200b9a10 0x7f172015e2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 -- 172.21.15.64:0/4247176790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17200cc950 con 0x7f17200b95a0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.302+0000 7f17261de640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f172015b300 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f171c00c3c0 tx=0x7f171c00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.303+0000 7f1716ffd640 1 -- 172.21.15.64:0/4247176790 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f171c0052d0 con 0x7f17200b95a0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.303+0000 7f1716ffd640 1 -- 172.21.15.64:0/4247176790 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f171c005490 con 0x7f17200b95a0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.303+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17200ccc30 con 0x7f17200b95a0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.303+0000 7f1716ffd640 1 -- 172.21.15.64:0/4247176790 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f171c007260 con 0x7f17200b95a0 2024-03-04T07:29:14.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.303+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17200cd1f0 con 0x7f17200b95a0 2024-03-04T07:29:14.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.304+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1720067ff0 con 0x7f17200b95a0 2024-03-04T07:29:14.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.304+0000 7f1716ffd640 1 -- 172.21.15.64:0/4247176790 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f171c007400 con 0x7f17200b95a0 2024-03-04T07:29:14.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.305+0000 7f1716ffd640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f170406b500 0x7f170406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.305+0000 7f17259dd640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f170406b500 0x7f170406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.305+0000 7f1716ffd640 1 -- 172.21.15.64:0/4247176790 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f171c089650 con 0x7f17200b95a0 2024-03-04T07:29:14.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.305+0000 7f17259dd640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f170406b500 0x7f170406d9c0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f1710036000 tx=0x7f171003b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:14.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.309+0000 7f1716ffd640 1 -- 172.21.15.64:0/4247176790 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f171c0584d0 con 0x7f17200b95a0 2024-03-04T07:29:14.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.577+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f1720068950 con 0x7f170406b500 2024-03-04T07:29:14.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.583+0000 7f1716ffd640 1 -- 172.21.15.64:0/4247176790 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7f1720068950 con 0x7f170406b500 2024-03-04T07:29:14.587 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:14.588 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:29:14.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f170406b500 msgr2=0x7f170406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f170406b500 0x7f170406d9c0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f1710036000 tx=0x7f171003b040 comp rx=0 tx=0).stop 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 msgr2=0x7f172015b300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f172015b300 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f171c00c3c0 tx=0x7f171c00c880 comp rx=0 tx=0).stop 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 shutdown_connections 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f170406b500 0x7f170406d9c0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f172015df00 0x7f17201606a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17200b9a10 0x7f172015e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 --2- 172.21.15.64:0/4247176790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17200b95a0 0x7f172015b300 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 >> 172.21.15.64:0/4247176790 conn(0x7f1720071960 msgr2=0x7f1720071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 shutdown_connections 2024-03-04T07:29:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.587+0000 7f1728442640 1 -- 172.21.15.64:0/4247176790 wait complete. 2024-03-04T07:29:14.598 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2024-03-04T07:29:13.167846+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":255580,"kb_used_data":6084,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754719140,"statfs":{"total":773094113280,"available":772832399360,"internally_reserved":0,"allocated":6230016,"data_stored":3241271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000828"},"pg_stats":[{"pgid":"9.3f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503695+0000","last_change":"2024-03-04T07:29:10.503695+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503695+0000","last_undegraded":"2024-03-04T07:29:10.503695+0000","last_fullsized":"2024-03-04T07:29:10.503695+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.509497+0000","last_change":"2024-03-04T07:29:10.509497+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.509497+0000","last_undegraded":"2024-03-04T07:29:10.509497+0000","last_fullsized":"2024-03-04T07:29:10.509497+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.3d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506870+0000","last_change":"2024-03-04T07:29:10.506870+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506870+0000","last_undegraded":"2024-03-04T07:29:10.506870+0000","last_fullsized":"2024-03-04T07:29:10.506870+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.3c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502699+0000","last_change":"2024-03-04T07:29:10.502699+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502699+0000","last_undegraded":"2024-03-04T07:29:10.502699+0000","last_fullsized":"2024-03-04T07:29:10.502699+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.3b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503688+0000","last_change":"2024-03-04T07:29:10.503688+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503688+0000","last_undegraded":"2024-03-04T07:29:10.503688+0000","last_fullsized":"2024-03-04T07:29:10.503688+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510758+0000","last_change":"2024-03-04T07:29:10.510758+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510758+0000","last_undegraded":"2024-03-04T07:29:10.510758+0000","last_fullsized":"2024-03-04T07:29:10.510758+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.39","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.38","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506502+0000","last_change":"2024-03-04T07:29:10.506502+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506502+0000","last_undegraded":"2024-03-04T07:29:10.506502+0000","last_fullsized":"2024-03-04T07:29:10.506502+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.37","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500610+0000","last_change":"2024-03-04T07:29:10.500610+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500610+0000","last_undegraded":"2024-03-04T07:29:10.500610+0000","last_fullsized":"2024-03-04T07:29:10.500610+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.36","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502657+0000","last_change":"2024-03-04T07:29:10.502657+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502657+0000","last_undegraded":"2024-03-04T07:29:10.502657+0000","last_fullsized":"2024-03-04T07:29:10.502657+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.35","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504279+0000","last_change":"2024-03-04T07:29:10.504279+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504279+0000","last_undegraded":"2024-03-04T07:29:10.504279+0000","last_fullsized":"2024-03-04T07:29:10.504279+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.34","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500622+0000","last_change":"2024-03-04T07:29:10.500622+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500622+0000","last_undegraded":"2024-03-04T07:29:10.500622+0000","last_fullsized":"2024-03-04T07:29:10.500622+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.33","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505723+0000","last_change":"2024-03-04T07:29:10.505723+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505723+0000","last_undegraded":"2024-03-04T07:29:10.505723+0000","last_fullsized":"2024-03-04T07:29:10.505723+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.32","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500946+0000","last_change":"2024-03-04T07:29:10.500946+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500946+0000","last_undegraded":"2024-03-04T07:29:10.500946+0000","last_fullsized":"2024-03-04T07:29:10.500946+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.31","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500479+0000","last_change":"2024-03-04T07:29:10.500479+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500479+0000","last_undegraded":"2024-03-04T07:29:10.500479+0000","last_fullsized":"2024-03-04T07:29:10.500479+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.30","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506809+0000","last_change":"2024-03-04T07:29:10.506809+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506809+0000","last_undegraded":"2024-03-04T07:29:10.506809+0000","last_fullsized":"2024-03-04T07:29:10.506809+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.2f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507972+0000","last_change":"2024-03-04T07:29:10.507972+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507972+0000","last_undegraded":"2024-03-04T07:29:10.507972+0000","last_fullsized":"2024-03-04T07:29:10.507972+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505474+0000","last_change":"2024-03-04T07:29:10.505474+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505474+0000","last_undegraded":"2024-03-04T07:29:10.505474+0000","last_fullsized":"2024-03-04T07:29:10.505474+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503206+0000","last_change":"2024-03-04T07:29:10.503206+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503206+0000","last_undegraded":"2024-03-04T07:29:10.503206+0000","last_fullsized":"2024-03-04T07:29:10.503206+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.2c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507000+0000","last_change":"2024-03-04T07:29:10.507000+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507000+0000","last_undegraded":"2024-03-04T07:29:10.507000+0000","last_fullsized":"2024-03-04T07:29:10.507000+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504903+0000","last_change":"2024-03-04T07:29:10.504903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504903+0000","last_undegraded":"2024-03-04T07:29:10.504903+0000","last_fullsized":"2024-03-04T07:29:10.504903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505803+0000","last_change":"2024-03-04T07:29:10.505803+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505803+0000","last_undegraded":"2024-03-04T07:29:10.505803+0000","last_fullsized":"2024-03-04T07:29:10.505803+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.29","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506903+0000","last_change":"2024-03-04T07:29:10.506903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506903+0000","last_undegraded":"2024-03-04T07:29:10.506903+0000","last_fullsized":"2024-03-04T07:29:10.506903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.28","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505190+0000","last_change":"2024-03-04T07:29:10.505190+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505190+0000","last_undegraded":"2024-03-04T07:29:10.505190+0000","last_fullsized":"2024-03-04T07:29:10.505190+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.27","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502380+0000","last_change":"2024-03-04T07:29:10.502380+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502380+0000","last_undegraded":"2024-03-04T07:29:10.502380+0000","last_fullsized":"2024-03-04T07:29:10.502380+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.26","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499795+0000","last_change":"2024-03-04T07:29:10.499795+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499795+0000","last_undegraded":"2024-03-04T07:29:10.499795+0000","last_fullsized":"2024-03-04T07:29:10.499795+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.25","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507608+0000","last_change":"2024-03-04T07:29:10.507608+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507608+0000","last_undegraded":"2024-03-04T07:29:10.507608+0000","last_fullsized":"2024-03-04T07:29:10.507608+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.24","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502297+0000","last_change":"2024-03-04T07:29:10.502297+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502297+0000","last_undegraded":"2024-03-04T07:29:10.502297+0000","last_fullsized":"2024-03-04T07:29:10.502297+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.23","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507197+0000","last_change":"2024-03-04T07:29:10.507197+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507197+0000","last_undegraded":"2024-03-04T07:29:10.507197+0000","last_fullsized":"2024-03-04T07:29:10.507197+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.22","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505391+0000","last_change":"2024-03-04T07:29:10.505391+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505391+0000","last_undegraded":"2024-03-04T07:29:10.505391+0000","last_fullsized":"2024-03-04T07:29:10.505391+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.21","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502303+0000","last_change":"2024-03-04T07:29:10.502303+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502303+0000","last_undegraded":"2024-03-04T07:29:10.502303+0000","last_fullsized":"2024-03-04T07:29:10.502303+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.20","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506384+0000","last_change":"2024-03-04T07:29:10.506384+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506384+0000","last_undegraded":"2024-03-04T07:29:10.506384+0000","last_fullsized":"2024-03-04T07:29:10.506384+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505456+0000","last_change":"2024-03-04T07:29:10.505456+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505456+0000","last_undegraded":"2024-03-04T07:29:10.505456+0000","last_fullsized":"2024-03-04T07:29:10.505456+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.513909+0000","last_change":"2024-03-04T07:29:10.512338+0000","last_active":"2024-03-04T07:29:10.513909+0000","last_peered":"2024-03-04T07:29:10.513909+0000","last_clean":"2024-03-04T07:29:10.513909+0000","last_became_active":"2024-03-04T07:29:10.512059+0000","last_became_peered":"2024-03-04T07:29:10.512059+0000","last_unstale":"2024-03-04T07:29:10.513909+0000","last_undegraded":"2024-03-04T07:29:10.513909+0000","last_fullsized":"2024-03-04T07:29:10.513909+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503363+0000","last_change":"2024-03-04T07:29:10.503363+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503363+0000","last_undegraded":"2024-03-04T07:29:10.503363+0000","last_fullsized":"2024-03-04T07:29:10.503363+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518723+0000","last_change":"2024-03-04T07:29:10.515404+0000","last_active":"2024-03-04T07:29:10.518723+0000","last_peered":"2024-03-04T07:29:10.518723+0000","last_clean":"2024-03-04T07:29:10.518723+0000","last_became_active":"2024-03-04T07:29:10.515135+0000","last_became_peered":"2024-03-04T07:29:10.515135+0000","last_unstale":"2024-03-04T07:29:10.518723+0000","last_undegraded":"2024-03-04T07:29:10.518723+0000","last_fullsized":"2024-03-04T07:29:10.518723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:12:54.069572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505684+0000","last_change":"2024-03-04T07:29:10.505684+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505684+0000","last_undegraded":"2024-03-04T07:29:10.505684+0000","last_fullsized":"2024-03-04T07:29:10.505684+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518470+0000","last_change":"2024-03-04T07:29:10.515994+0000","last_active":"2024-03-04T07:29:10.518470+0000","last_peered":"2024-03-04T07:29:10.518470+0000","last_clean":"2024-03-04T07:29:10.518470+0000","last_became_active":"2024-03-04T07:29:10.515751+0000","last_became_peered":"2024-03-04T07:29:10.515751+0000","last_unstale":"2024-03-04T07:29:10.518470+0000","last_undegraded":"2024-03-04T07:29:10.518470+0000","last_fullsized":"2024-03-04T07:29:10.518470+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507219+0000","last_change":"2024-03-04T07:29:10.507219+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507219+0000","last_undegraded":"2024-03-04T07:29:10.507219+0000","last_fullsized":"2024-03-04T07:29:10.507219+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516461+0000","last_change":"2024-03-04T07:29:10.509745+0000","last_active":"2024-03-04T07:29:10.516461+0000","last_peered":"2024-03-04T07:29:10.516461+0000","last_clean":"2024-03-04T07:29:10.516461+0000","last_became_active":"2024-03-04T07:29:10.509471+0000","last_became_peered":"2024-03-04T07:29:10.509471+0000","last_unstale":"2024-03-04T07:29:10.516461+0000","last_undegraded":"2024-03-04T07:29:10.516461+0000","last_fullsized":"2024-03-04T07:29:10.516461+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505406+0000","last_change":"2024-03-04T07:29:10.505406+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505406+0000","last_undegraded":"2024-03-04T07:29:10.505406+0000","last_fullsized":"2024-03-04T07:29:10.505406+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516437+0000","last_change":"2024-03-04T07:29:10.510878+0000","last_active":"2024-03-04T07:29:10.516437+0000","last_peered":"2024-03-04T07:29:10.516437+0000","last_clean":"2024-03-04T07:29:10.516437+0000","last_became_active":"2024-03-04T07:29:10.510509+0000","last_became_peered":"2024-03-04T07:29:10.510509+0000","last_unstale":"2024-03-04T07:29:10.516437+0000","last_undegraded":"2024-03-04T07:29:10.516437+0000","last_fullsized":"2024-03-04T07:29:10.516437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:51:59.390945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508983+0000","last_change":"2024-03-04T07:29:10.508983+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508983+0000","last_undegraded":"2024-03-04T07:29:10.508983+0000","last_fullsized":"2024-03-04T07:29:10.508983+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519100+0000","last_change":"2024-03-04T07:29:10.517386+0000","last_active":"2024-03-04T07:29:10.519100+0000","last_peered":"2024-03-04T07:29:10.519100+0000","last_clean":"2024-03-04T07:29:10.519100+0000","last_became_active":"2024-03-04T07:29:10.517120+0000","last_became_peered":"2024-03-04T07:29:10.517120+0000","last_unstale":"2024-03-04T07:29:10.519100+0000","last_undegraded":"2024-03-04T07:29:10.519100+0000","last_fullsized":"2024-03-04T07:29:10.519100+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.18","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504666+0000","last_change":"2024-03-04T07:29:10.504666+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504666+0000","last_undegraded":"2024-03-04T07:29:10.504666+0000","last_fullsized":"2024-03-04T07:29:10.504666+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516345+0000","last_change":"2024-03-04T07:29:10.509713+0000","last_active":"2024-03-04T07:29:10.516345+0000","last_peered":"2024-03-04T07:29:10.516345+0000","last_clean":"2024-03-04T07:29:10.516345+0000","last_became_active":"2024-03-04T07:29:10.509409+0000","last_became_peered":"2024-03-04T07:29:10.509409+0000","last_unstale":"2024-03-04T07:29:10.516345+0000","last_undegraded":"2024-03-04T07:29:10.516345+0000","last_fullsized":"2024-03-04T07:29:10.516345+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508131+0000","last_change":"2024-03-04T07:29:10.508131+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508131+0000","last_undegraded":"2024-03-04T07:29:10.508131+0000","last_fullsized":"2024-03-04T07:29:10.508131+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516481+0000","last_change":"2024-03-04T07:29:10.512481+0000","last_active":"2024-03-04T07:29:10.516481+0000","last_peered":"2024-03-04T07:29:10.516481+0000","last_clean":"2024-03-04T07:29:10.516481+0000","last_became_active":"2024-03-04T07:29:10.512206+0000","last_became_peered":"2024-03-04T07:29:10.512206+0000","last_unstale":"2024-03-04T07:29:10.516481+0000","last_undegraded":"2024-03-04T07:29:10.516481+0000","last_fullsized":"2024-03-04T07:29:10.516481+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:19.657851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.16","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501436+0000","last_change":"2024-03-04T07:29:10.501436+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501436+0000","last_undegraded":"2024-03-04T07:29:10.501436+0000","last_fullsized":"2024-03-04T07:29:10.501436+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.17","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518954+0000","last_change":"2024-03-04T07:29:10.514467+0000","last_active":"2024-03-04T07:29:10.518954+0000","last_peered":"2024-03-04T07:29:10.518954+0000","last_clean":"2024-03-04T07:29:10.518954+0000","last_became_active":"2024-03-04T07:29:10.514160+0000","last_became_peered":"2024-03-04T07:29:10.514160+0000","last_unstale":"2024-03-04T07:29:10.518954+0000","last_undegraded":"2024-03-04T07:29:10.518954+0000","last_fullsized":"2024-03-04T07:29:10.518954+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.17","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504882+0000","last_change":"2024-03-04T07:29:10.504882+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504882+0000","last_undegraded":"2024-03-04T07:29:10.504882+0000","last_fullsized":"2024-03-04T07:29:10.504882+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.16","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518939+0000","last_change":"2024-03-04T07:29:10.517195+0000","last_active":"2024-03-04T07:29:10.518939+0000","last_peered":"2024-03-04T07:29:10.518939+0000","last_clean":"2024-03-04T07:29:10.518939+0000","last_became_active":"2024-03-04T07:29:10.516936+0000","last_became_peered":"2024-03-04T07:29:10.516936+0000","last_unstale":"2024-03-04T07:29:10.518939+0000","last_undegraded":"2024-03-04T07:29:10.518939+0000","last_fullsized":"2024-03-04T07:29:10.518939+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:00.174138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502894+0000","last_change":"2024-03-04T07:29:10.502894+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502894+0000","last_undegraded":"2024-03-04T07:29:10.502894+0000","last_fullsized":"2024-03-04T07:29:10.502894+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.15","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518958+0000","last_change":"2024-03-04T07:29:10.515515+0000","last_active":"2024-03-04T07:29:10.518958+0000","last_peered":"2024-03-04T07:29:10.518958+0000","last_clean":"2024-03-04T07:29:10.518958+0000","last_became_active":"2024-03-04T07:29:10.515229+0000","last_became_peered":"2024-03-04T07:29:10.515229+0000","last_unstale":"2024-03-04T07:29:10.518958+0000","last_undegraded":"2024-03-04T07:29:10.518958+0000","last_fullsized":"2024-03-04T07:29:10.518958+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.15","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505698+0000","last_change":"2024-03-04T07:29:10.505698+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505698+0000","last_undegraded":"2024-03-04T07:29:10.505698+0000","last_fullsized":"2024-03-04T07:29:10.505698+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.14","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516384+0000","last_change":"2024-03-04T07:29:10.509271+0000","last_active":"2024-03-04T07:29:10.516384+0000","last_peered":"2024-03-04T07:29:10.516384+0000","last_clean":"2024-03-04T07:29:10.516384+0000","last_became_active":"2024-03-04T07:29:10.508965+0000","last_became_peered":"2024-03-04T07:29:10.508965+0000","last_unstale":"2024-03-04T07:29:10.516384+0000","last_undegraded":"2024-03-04T07:29:10.516384+0000","last_fullsized":"2024-03-04T07:29:10.516384+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:18:49.238841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.12","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500095+0000","last_change":"2024-03-04T07:29:10.500095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500095+0000","last_undegraded":"2024-03-04T07:29:10.500095+0000","last_fullsized":"2024-03-04T07:29:10.500095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.13","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519067+0000","last_change":"2024-03-04T07:29:10.517336+0000","last_active":"2024-03-04T07:29:10.519067+0000","last_peered":"2024-03-04T07:29:10.519067+0000","last_clean":"2024-03-04T07:29:10.519067+0000","last_became_active":"2024-03-04T07:29:10.517088+0000","last_became_peered":"2024-03-04T07:29:10.517088+0000","last_unstale":"2024-03-04T07:29:10.519067+0000","last_undegraded":"2024-03-04T07:29:10.519067+0000","last_fullsized":"2024-03-04T07:29:10.519067+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:26:41.225141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504221+0000","last_change":"2024-03-04T07:29:10.504221+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504221+0000","last_undegraded":"2024-03-04T07:29:10.504221+0000","last_fullsized":"2024-03-04T07:29:10.504221+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.12","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518895+0000","last_change":"2024-03-04T07:29:10.515222+0000","last_active":"2024-03-04T07:29:10.518895+0000","last_peered":"2024-03-04T07:29:10.518895+0000","last_clean":"2024-03-04T07:29:10.518895+0000","last_became_active":"2024-03-04T07:29:10.514882+0000","last_became_peered":"2024-03-04T07:29:10.514882+0000","last_unstale":"2024-03-04T07:29:10.518895+0000","last_undegraded":"2024-03-04T07:29:10.518895+0000","last_fullsized":"2024-03-04T07:29:10.518895+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:36:31.420388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.10","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500037+0000","last_change":"2024-03-04T07:29:10.500037+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500037+0000","last_undegraded":"2024-03-04T07:29:10.500037+0000","last_fullsized":"2024-03-04T07:29:10.500037+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.11","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518799+0000","last_change":"2024-03-04T07:29:10.515029+0000","last_active":"2024-03-04T07:29:10.518799+0000","last_peered":"2024-03-04T07:29:10.518799+0000","last_clean":"2024-03-04T07:29:10.518799+0000","last_became_active":"2024-03-04T07:29:10.514722+0000","last_became_peered":"2024-03-04T07:29:10.514722+0000","last_unstale":"2024-03-04T07:29:10.518799+0000","last_undegraded":"2024-03-04T07:29:10.518799+0000","last_fullsized":"2024-03-04T07:29:10.518799+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506446+0000","last_change":"2024-03-04T07:29:10.506446+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506446+0000","last_undegraded":"2024-03-04T07:29:10.506446+0000","last_fullsized":"2024-03-04T07:29:10.506446+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.10","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519102+0000","last_change":"2024-03-04T07:29:10.517145+0000","last_active":"2024-03-04T07:29:10.519102+0000","last_peered":"2024-03-04T07:29:10.519102+0000","last_clean":"2024-03-04T07:29:10.519102+0000","last_became_active":"2024-03-04T07:29:10.516791+0000","last_became_peered":"2024-03-04T07:29:10.516791+0000","last_unstale":"2024-03-04T07:29:10.519102+0000","last_undegraded":"2024-03-04T07:29:10.519102+0000","last_fullsized":"2024-03-04T07:29:10.519102+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504566+0000","last_change":"2024-03-04T07:29:10.504566+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504566+0000","last_undegraded":"2024-03-04T07:29:10.504566+0000","last_fullsized":"2024-03-04T07:29:10.504566+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519012+0000","last_change":"2024-03-04T07:29:10.517764+0000","last_active":"2024-03-04T07:29:10.519012+0000","last_peered":"2024-03-04T07:29:10.519012+0000","last_clean":"2024-03-04T07:29:10.519012+0000","last_became_active":"2024-03-04T07:29:10.516895+0000","last_became_peered":"2024-03-04T07:29:10.516895+0000","last_unstale":"2024-03-04T07:29:10.519012+0000","last_undegraded":"2024-03-04T07:29:10.519012+0000","last_fullsized":"2024-03-04T07:29:10.519012+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:04:10.056881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501820+0000","last_change":"2024-03-04T07:29:10.501820+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501820+0000","last_undegraded":"2024-03-04T07:29:10.501820+0000","last_fullsized":"2024-03-04T07:29:10.501820+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518517+0000","last_change":"2024-03-04T07:29:10.515628+0000","last_active":"2024-03-04T07:29:10.518517+0000","last_peered":"2024-03-04T07:29:10.518517+0000","last_clean":"2024-03-04T07:29:10.518517+0000","last_became_active":"2024-03-04T07:29:10.515205+0000","last_became_peered":"2024-03-04T07:29:10.515205+0000","last_unstale":"2024-03-04T07:29:10.518517+0000","last_undegraded":"2024-03-04T07:29:10.518517+0000","last_fullsized":"2024-03-04T07:29:10.518517+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506522+0000","last_change":"2024-03-04T07:29:10.506522+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506522+0000","last_undegraded":"2024-03-04T07:29:10.506522+0000","last_fullsized":"2024-03-04T07:29:10.506522+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518651+0000","last_change":"2024-03-04T07:29:10.512208+0000","last_active":"2024-03-04T07:29:10.518651+0000","last_peered":"2024-03-04T07:29:10.518651+0000","last_clean":"2024-03-04T07:29:10.518651+0000","last_became_active":"2024-03-04T07:29:10.511806+0000","last_became_peered":"2024-03-04T07:29:10.511806+0000","last_unstale":"2024-03-04T07:29:10.518651+0000","last_undegraded":"2024-03-04T07:29:10.518651+0000","last_fullsized":"2024-03-04T07:29:10.518651+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504634+0000","last_change":"2024-03-04T07:29:10.504634+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504634+0000","last_undegraded":"2024-03-04T07:29:10.504634+0000","last_fullsized":"2024-03-04T07:29:10.504634+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519131+0000","last_change":"2024-03-04T07:29:10.516595+0000","last_active":"2024-03-04T07:29:10.519131+0000","last_peered":"2024-03-04T07:29:10.519131+0000","last_clean":"2024-03-04T07:29:10.519131+0000","last_became_active":"2024-03-04T07:29:10.516327+0000","last_became_peered":"2024-03-04T07:29:10.516327+0000","last_unstale":"2024-03-04T07:29:10.519131+0000","last_undegraded":"2024-03-04T07:29:10.519131+0000","last_fullsized":"2024-03-04T07:29:10.519131+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499557+0000","last_change":"2024-03-04T07:29:10.499557+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499557+0000","last_undegraded":"2024-03-04T07:29:10.499557+0000","last_fullsized":"2024-03-04T07:29:10.499557+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519087+0000","last_change":"2024-03-04T07:29:10.517775+0000","last_active":"2024-03-04T07:29:10.519087+0000","last_peered":"2024-03-04T07:29:10.519087+0000","last_clean":"2024-03-04T07:29:10.519087+0000","last_became_active":"2024-03-04T07:29:10.517496+0000","last_became_peered":"2024-03-04T07:29:10.517496+0000","last_unstale":"2024-03-04T07:29:10.519087+0000","last_undegraded":"2024-03-04T07:29:10.519087+0000","last_fullsized":"2024-03-04T07:29:10.519087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:00.707197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510312+0000","last_change":"2024-03-04T07:29:10.510312+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510312+0000","last_undegraded":"2024-03-04T07:29:10.510312+0000","last_fullsized":"2024-03-04T07:29:10.510312+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519200+0000","last_change":"2024-03-04T07:29:10.514774+0000","last_active":"2024-03-04T07:29:10.519200+0000","last_peered":"2024-03-04T07:29:10.519200+0000","last_clean":"2024-03-04T07:29:10.519200+0000","last_became_active":"2024-03-04T07:29:10.514489+0000","last_became_peered":"2024-03-04T07:29:10.514489+0000","last_unstale":"2024-03-04T07:29:10.519200+0000","last_undegraded":"2024-03-04T07:29:10.519200+0000","last_fullsized":"2024-03-04T07:29:10.519200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:20.173974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506687+0000","last_change":"2024-03-04T07:29:10.506687+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506687+0000","last_undegraded":"2024-03-04T07:29:10.506687+0000","last_fullsized":"2024-03-04T07:29:10.506687+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518751+0000","last_change":"2024-03-04T07:29:10.513482+0000","last_active":"2024-03-04T07:29:10.518751+0000","last_peered":"2024-03-04T07:29:10.518751+0000","last_clean":"2024-03-04T07:29:10.518751+0000","last_became_active":"2024-03-04T07:29:10.513209+0000","last_became_peered":"2024-03-04T07:29:10.513209+0000","last_unstale":"2024-03-04T07:29:10.518751+0000","last_undegraded":"2024-03-04T07:29:10.518751+0000","last_fullsized":"2024-03-04T07:29:10.518751+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499078+0000","last_change":"2024-03-04T07:29:10.499078+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499078+0000","last_undegraded":"2024-03-04T07:29:10.499078+0000","last_fullsized":"2024-03-04T07:29:10.499078+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519230+0000","last_change":"2024-03-04T07:29:10.516454+0000","last_active":"2024-03-04T07:29:10.519230+0000","last_peered":"2024-03-04T07:29:10.519230+0000","last_clean":"2024-03-04T07:29:10.519230+0000","last_became_active":"2024-03-04T07:29:10.514845+0000","last_became_peered":"2024-03-04T07:29:10.514845+0000","last_unstale":"2024-03-04T07:29:10.519230+0000","last_undegraded":"2024-03-04T07:29:10.519230+0000","last_fullsized":"2024-03-04T07:29:10.519230+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516782+0000","last_change":"2024-03-04T07:29:10.514474+0000","last_active":"2024-03-04T07:29:10.516782+0000","last_peered":"2024-03-04T07:29:10.516782+0000","last_clean":"2024-03-04T07:29:10.516782+0000","last_became_active":"2024-03-04T07:29:10.514215+0000","last_became_peered":"2024-03-04T07:29:10.514215+0000","last_unstale":"2024-03-04T07:29:10.516782+0000","last_undegraded":"2024-03-04T07:29:10.516782+0000","last_fullsized":"2024-03-04T07:29:10.516782+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502443+0000","last_change":"2024-03-04T07:29:10.502443+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502443+0000","last_undegraded":"2024-03-04T07:29:10.502443+0000","last_fullsized":"2024-03-04T07:29:10.502443+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519002+0000","last_change":"2024-03-04T07:29:10.515670+0000","last_active":"2024-03-04T07:29:10.519002+0000","last_peered":"2024-03-04T07:29:10.519002+0000","last_clean":"2024-03-04T07:29:10.519002+0000","last_became_active":"2024-03-04T07:29:10.515327+0000","last_became_peered":"2024-03-04T07:29:10.515327+0000","last_unstale":"2024-03-04T07:29:10.519002+0000","last_undegraded":"2024-03-04T07:29:10.519002+0000","last_fullsized":"2024-03-04T07:29:10.519002+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:54:04.259042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506095+0000","last_change":"2024-03-04T07:29:10.506095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506095+0000","last_undegraded":"2024-03-04T07:29:10.506095+0000","last_fullsized":"2024-03-04T07:29:10.506095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.509863+0000","last_change":"2024-03-04T07:29:10.506371+0000","last_active":"2024-03-04T07:29:10.509863+0000","last_peered":"2024-03-04T07:29:10.509863+0000","last_clean":"2024-03-04T07:29:10.509863+0000","last_became_active":"2024-03-04T07:29:10.506084+0000","last_became_peered":"2024-03-04T07:29:10.506084+0000","last_unstale":"2024-03-04T07:29:10.509863+0000","last_undegraded":"2024-03-04T07:29:10.509863+0000","last_fullsized":"2024-03-04T07:29:10.509863+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504947+0000","last_change":"2024-03-04T07:29:10.504947+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504947+0000","last_undegraded":"2024-03-04T07:29:10.504947+0000","last_fullsized":"2024-03-04T07:29:10.504947+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.514479+0000","last_change":"2024-03-04T07:29:10.510685+0000","last_active":"2024-03-04T07:29:10.514479+0000","last_peered":"2024-03-04T07:29:10.514479+0000","last_clean":"2024-03-04T07:29:10.514479+0000","last_became_active":"2024-03-04T07:29:10.510433+0000","last_became_peered":"2024-03-04T07:29:10.510433+0000","last_unstale":"2024-03-04T07:29:10.514479+0000","last_undegraded":"2024-03-04T07:29:10.514479+0000","last_fullsized":"2024-03-04T07:29:10.514479+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:04:10.012697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503678+0000","last_change":"2024-03-04T07:29:10.503678+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503678+0000","last_undegraded":"2024-03-04T07:29:10.503678+0000","last_fullsized":"2024-03-04T07:29:10.503678+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518944+0000","last_change":"2024-03-04T07:29:10.513884+0000","last_active":"2024-03-04T07:29:10.518944+0000","last_peered":"2024-03-04T07:29:10.518944+0000","last_clean":"2024-03-04T07:29:10.518944+0000","last_became_active":"2024-03-04T07:29:10.513574+0000","last_became_peered":"2024-03-04T07:29:10.513574+0000","last_unstale":"2024-03-04T07:29:10.518944+0000","last_undegraded":"2024-03-04T07:29:10.518944+0000","last_fullsized":"2024-03-04T07:29:10.518944+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:39.944442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502740+0000","last_change":"2024-03-04T07:29:10.502740+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502740+0000","last_undegraded":"2024-03-04T07:29:10.502740+0000","last_fullsized":"2024-03-04T07:29:10.502740+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516550+0000","last_change":"2024-03-04T07:29:10.514430+0000","last_active":"2024-03-04T07:29:10.516550+0000","last_peered":"2024-03-04T07:29:10.516550+0000","last_clean":"2024-03-04T07:29:10.516550+0000","last_became_active":"2024-03-04T07:29:10.514146+0000","last_became_peered":"2024-03-04T07:29:10.514146+0000","last_unstale":"2024-03-04T07:29:10.516550+0000","last_undegraded":"2024-03-04T07:29:10.516550+0000","last_fullsized":"2024-03-04T07:29:10.516550+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:29:39.502383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507659+0000","last_change":"2024-03-04T07:29:10.507659+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507659+0000","last_undegraded":"2024-03-04T07:29:10.507659+0000","last_fullsized":"2024-03-04T07:29:10.507659+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.521576+0000","last_change":"2024-03-04T07:29:10.519578+0000","last_active":"2024-03-04T07:29:10.521576+0000","last_peered":"2024-03-04T07:29:10.521576+0000","last_clean":"2024-03-04T07:29:10.521576+0000","last_became_active":"2024-03-04T07:29:10.519248+0000","last_became_peered":"2024-03-04T07:29:10.519248+0000","last_unstale":"2024-03-04T07:29:10.521576+0000","last_undegraded":"2024-03-04T07:29:10.521576+0000","last_fullsized":"2024-03-04T07:29:10.521576+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:11.247866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506035+0000","last_change":"2024-03-04T07:29:10.506035+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506035+0000","last_undegraded":"2024-03-04T07:29:10.506035+0000","last_fullsized":"2024-03-04T07:29:10.506035+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516800+0000","last_change":"2024-03-04T07:29:10.513166+0000","last_active":"2024-03-04T07:29:10.516800+0000","last_peered":"2024-03-04T07:29:10.516800+0000","last_clean":"2024-03-04T07:29:10.516800+0000","last_became_active":"2024-03-04T07:29:10.512608+0000","last_became_peered":"2024-03-04T07:29:10.512608+0000","last_unstale":"2024-03-04T07:29:10.516800+0000","last_undegraded":"2024-03-04T07:29:10.516800+0000","last_fullsized":"2024-03-04T07:29:10.516800+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":230,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.507585+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:29:10.507585+0000","last_peered":"2024-03-04T07:29:10.507585+0000","last_clean":"2024-03-04T07:29:10.507585+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:29:10.507585+0000","last_undegraded":"2024-03-04T07:29:10.507585+0000","last_fullsized":"2024-03-04T07:29:10.507585+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":8,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574883,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32376,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339464,"statfs":{"total":96636764160,"available":96603611136,"internally_reserved":0,"allocated":1216512,"data_stored":844018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31812,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340028,"statfs":{"total":96636764160,"available":96604188672,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574883,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31808,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340032,"statfs":{"total":96636764160,"available":96604192768,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31788,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340052,"statfs":{"total":96636764160,"available":96604213248,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574882,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31796,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340044,"statfs":{"total":96636764160,"available":96604205056,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574882,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32392,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339448,"statfs":{"total":96636764160,"available":96603594752,"internally_reserved":0,"allocated":1228800,"data_stored":850285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:29:14.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:29:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.739+0000 7f5c800c9640 1 Processor -- start 2024-03-04T07:29:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.739+0000 7f5c800c9640 1 -- start start 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.740+0000 7f5c800c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5c7815e080 0x7f5c78066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.740+0000 7f5c800c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c78161c20 con 0x7f5c7815dca0 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.740+0000 7f5c800c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c78161f00 con 0x7f5c780b00c0 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.740+0000 7f5c800c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5c781619d0 con 0x7f5c7815e080 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.740+0000 7f5c7de65640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5c7815dca0 0x7f5c7806fd10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35678/0 (socket says 172.21.15.64:35678) 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.740+0000 7f5c7de65640 1 -- 172.21.15.64:0/767733603 learned_addr learned my addr 172.21.15.64:0/767733603 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3156467065 0 0) 0x7f5c78161c20 con 0x7f5c7815dca0 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5c64003520 con 0x7f5c7815dca0 2024-03-04T07:29:14.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7d664640 1 --2- 172.21.15.64:0/767733603 >> v2:172.21.15.66:3300/0 conn(0x7f5c7815e080 0x7f5c78066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2311609421 0 0) 0x7f5c64003520 con 0x7f5c7815dca0 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 >> v2:172.21.15.66:3300/0 conn(0x7f5c7815e080 msgr2=0x7f5c78066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 --2- 172.21.15.64:0/767733603 >> v2:172.21.15.66:3300/0 conn(0x7f5c7815e080 0x7f5c78066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 >> v1:172.21.15.66:6789/0 conn(0x7f5c780b00c0 legacy=0x7f5c780b3ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c781621e0 con 0x7f5c7815dca0 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f5c74001670 con 0x7f5c7815dca0 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 >> v1:172.21.15.64:6789/0 conn(0x7f5c7815dca0 legacy=0x7f5c7806fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640039b0 0x7f5c64003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c64004800 0x7f5c64004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c640051c0 0x7f5c6400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c781619d0 con 0x7f5c640051c0 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c6400c160 con 0x7f5c64004800 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c6400c360 con 0x7f5c640039b0 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7e666640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c640051c0 0x7f5c6400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7de65640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640039b0 0x7f5c64003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7d664640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c64004800 0x7f5c64004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7e666640 1 -- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640039b0 msgr2=0x7f5c64003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7e666640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640039b0 0x7f5c64003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7e666640 1 -- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c64004800 msgr2=0x7f5c64004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7e666640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c64004800 0x7f5c64004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7e666640 1 -- 172.21.15.64:0/767733603 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c64003520 con 0x7f5c640051c0 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7de65640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640039b0 0x7f5c64003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.741+0000 7f5c7d664640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c64004800 0x7f5c64004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.742+0000 7f5c7e666640 1 -- 172.21.15.64:0/767733603 reap_dead start 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.742+0000 7f5c7e666640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c640051c0 0x7f5c6400b9c0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f5c70001e50 tx=0x7f5c64005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bef1f325015f08bf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.742+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c70001a70 con 0x7f5c640051c0 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.742+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c70003bb0 con 0x7f5c640051c0 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.742+0000 7f5c7ce63640 1 -- 172.21.15.64:0/767733603 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c700050d0 con 0x7f5c640051c0 2024-03-04T07:29:14.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.743+0000 7f5c800c9640 1 -- 172.21.15.64:0/767733603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c640051c0 msgr2=0x7f5c6400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.743+0000 7f5c800c9640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c640051c0 0x7f5c6400b9c0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f5c70001e50 tx=0x7f5c64005830 comp rx=0 tx=0).stop 2024-03-04T07:29:14.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.743+0000 7f5c800c9640 1 -- 172.21.15.64:0/767733603 shutdown_connections 2024-03-04T07:29:14.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.743+0000 7f5c800c9640 1 --2- 172.21.15.64:0/767733603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c640051c0 0x7f5c6400b9c0 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.743+0000 7f5c800c9640 1 -- 172.21.15.64:0/767733603 >> 172.21.15.64:0/767733603 conn(0x7f5c780d0990 msgr2=0x7f5c780d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:14.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.743+0000 7f5c800c9640 1 -- 172.21.15.64:0/767733603 shutdown_connections 2024-03-04T07:29:14.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.743+0000 7f5c800c9640 1 -- 172.21.15.64:0/767733603 wait complete. 2024-03-04T07:29:14.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.744+0000 7f5c800c9640 1 Processor -- start 2024-03-04T07:29:14.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.744+0000 7f5c800c9640 1 -- start start 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c800c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640051c0 0x7f5c780cd430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c800c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c78142440 0x7f5c78159ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c800c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c7815f6a0 0x7f5c7815fb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c800c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c7806fe90 con 0x7f5c7815f6a0 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c800c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c7806fd10 con 0x7f5c78142440 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c800c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c78070010 con 0x7f5c640051c0 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c7e666640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c7815f6a0 0x7f5c7815fb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c7e666640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c7815f6a0 0x7f5c7815fb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39292/0 (socket says 172.21.15.64:39292) 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c7e666640 1 -- 172.21.15.64:0/4077460515 learned_addr learned my addr 172.21.15.64:0/4077460515 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c7d664640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c78142440 0x7f5c78159ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c7de65640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640051c0 0x7f5c780cd430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.745+0000 7f5c7e666640 1 -- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640051c0 msgr2=0x7f5c780cd430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c7e666640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640051c0 0x7f5c780cd430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c7e666640 1 -- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c78142440 msgr2=0x7f5c78159ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c7e666640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c78142440 0x7f5c78159ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c7e666640 1 -- 172.21.15.64:0/4077460515 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c78160060 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c7de65640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640051c0 0x7f5c780cd430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c7d664640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c78142440 0x7f5c78159ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c7e666640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c7815f6a0 0x7f5c7815fb20 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f5c70001f70 tx=0x7f5c700014a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c6e7fc640 1 -- 172.21.15.64:0/4077460515 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c70006d60 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c6e7fc640 1 -- 172.21.15.64:0/4077460515 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c70007030 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c780d0080 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c6e7fc640 1 -- 172.21.15.64:0/4077460515 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c700073e0 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.746+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c7815f180 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.747+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c7815c380 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.747+0000 7f5c6e7fc640 1 -- 172.21.15.64:0/4077460515 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5c70007580 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.748+0000 7f5c6e7fc640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c5c06b4d0 0x7f5c5c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.748+0000 7f5c6e7fc640 1 -- 172.21.15.64:0/4077460515 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5c700abe90 con 0x7f5c7815f6a0 2024-03-04T07:29:14.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.748+0000 7f5c7de65640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c5c06b4d0 0x7f5c5c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:14.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.748+0000 7f5c7de65640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c5c06b4d0 0x7f5c5c06d990 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f5c74000fd0 tx=0x7f5c74009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:14.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.753+0000 7f5c6e7fc640 1 -- 172.21.15.64:0/4077460515 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5c70033030 con 0x7f5c7815f6a0 2024-03-04T07:29:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.930+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f5c7806f330 con 0x7f5c5c06b4d0 2024-03-04T07:29:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.936+0000 7f5c6e7fc640 1 -- 172.21.15.64:0/4077460515 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7f5c7806f330 con 0x7f5c5c06b4d0 2024-03-04T07:29:14.939 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:14.940 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:29:14.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c5c06b4d0 msgr2=0x7f5c5c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c5c06b4d0 0x7f5c5c06d990 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f5c74000fd0 tx=0x7f5c74009040 comp rx=0 tx=0).stop 2024-03-04T07:29:14.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c7815f6a0 msgr2=0x7f5c7815fb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:14.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c7815f6a0 0x7f5c7815fb20 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f5c70001f70 tx=0x7f5c700014a0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 shutdown_connections 2024-03-04T07:29:14.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c5c06b4d0 0x7f5c5c06d990 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c7815f6a0 0x7f5c7815fb20 unknown :-1 s=CLOSED pgs=672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c78142440 0x7f5c78159ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 --2- 172.21.15.64:0/4077460515 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c640051c0 0x7f5c780cd430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:14.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 >> 172.21.15.64:0/4077460515 conn(0x7f5c780d0990 msgr2=0x7f5c780b3ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:14.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 shutdown_connections 2024-03-04T07:29:14.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:14.939+0000 7f5c800c9640 1 -- 172.21.15.64:0/4077460515 wait complete. 2024-03-04T07:29:14.951 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2024-03-04T07:29:13.167846+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":255580,"kb_used_data":6084,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754719140,"statfs":{"total":773094113280,"available":772832399360,"internally_reserved":0,"allocated":6230016,"data_stored":3241271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000828"},"pg_stats":[{"pgid":"9.3f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503695+0000","last_change":"2024-03-04T07:29:10.503695+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503695+0000","last_undegraded":"2024-03-04T07:29:10.503695+0000","last_fullsized":"2024-03-04T07:29:10.503695+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.509497+0000","last_change":"2024-03-04T07:29:10.509497+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.509497+0000","last_undegraded":"2024-03-04T07:29:10.509497+0000","last_fullsized":"2024-03-04T07:29:10.509497+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.3d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506870+0000","last_change":"2024-03-04T07:29:10.506870+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506870+0000","last_undegraded":"2024-03-04T07:29:10.506870+0000","last_fullsized":"2024-03-04T07:29:10.506870+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.3c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502699+0000","last_change":"2024-03-04T07:29:10.502699+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502699+0000","last_undegraded":"2024-03-04T07:29:10.502699+0000","last_fullsized":"2024-03-04T07:29:10.502699+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.3b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503688+0000","last_change":"2024-03-04T07:29:10.503688+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503688+0000","last_undegraded":"2024-03-04T07:29:10.503688+0000","last_fullsized":"2024-03-04T07:29:10.503688+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510758+0000","last_change":"2024-03-04T07:29:10.510758+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510758+0000","last_undegraded":"2024-03-04T07:29:10.510758+0000","last_fullsized":"2024-03-04T07:29:10.510758+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.39","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.38","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506502+0000","last_change":"2024-03-04T07:29:10.506502+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506502+0000","last_undegraded":"2024-03-04T07:29:10.506502+0000","last_fullsized":"2024-03-04T07:29:10.506502+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.37","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500610+0000","last_change":"2024-03-04T07:29:10.500610+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500610+0000","last_undegraded":"2024-03-04T07:29:10.500610+0000","last_fullsized":"2024-03-04T07:29:10.500610+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.36","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502657+0000","last_change":"2024-03-04T07:29:10.502657+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502657+0000","last_undegraded":"2024-03-04T07:29:10.502657+0000","last_fullsized":"2024-03-04T07:29:10.502657+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.35","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504279+0000","last_change":"2024-03-04T07:29:10.504279+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504279+0000","last_undegraded":"2024-03-04T07:29:10.504279+0000","last_fullsized":"2024-03-04T07:29:10.504279+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.34","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500622+0000","last_change":"2024-03-04T07:29:10.500622+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500622+0000","last_undegraded":"2024-03-04T07:29:10.500622+0000","last_fullsized":"2024-03-04T07:29:10.500622+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.33","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505723+0000","last_change":"2024-03-04T07:29:10.505723+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505723+0000","last_undegraded":"2024-03-04T07:29:10.505723+0000","last_fullsized":"2024-03-04T07:29:10.505723+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.32","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500946+0000","last_change":"2024-03-04T07:29:10.500946+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500946+0000","last_undegraded":"2024-03-04T07:29:10.500946+0000","last_fullsized":"2024-03-04T07:29:10.500946+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.31","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500479+0000","last_change":"2024-03-04T07:29:10.500479+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500479+0000","last_undegraded":"2024-03-04T07:29:10.500479+0000","last_fullsized":"2024-03-04T07:29:10.500479+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.30","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506809+0000","last_change":"2024-03-04T07:29:10.506809+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506809+0000","last_undegraded":"2024-03-04T07:29:10.506809+0000","last_fullsized":"2024-03-04T07:29:10.506809+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.2f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507972+0000","last_change":"2024-03-04T07:29:10.507972+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507972+0000","last_undegraded":"2024-03-04T07:29:10.507972+0000","last_fullsized":"2024-03-04T07:29:10.507972+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505474+0000","last_change":"2024-03-04T07:29:10.505474+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505474+0000","last_undegraded":"2024-03-04T07:29:10.505474+0000","last_fullsized":"2024-03-04T07:29:10.505474+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503206+0000","last_change":"2024-03-04T07:29:10.503206+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503206+0000","last_undegraded":"2024-03-04T07:29:10.503206+0000","last_fullsized":"2024-03-04T07:29:10.503206+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.2c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507000+0000","last_change":"2024-03-04T07:29:10.507000+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507000+0000","last_undegraded":"2024-03-04T07:29:10.507000+0000","last_fullsized":"2024-03-04T07:29:10.507000+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504903+0000","last_change":"2024-03-04T07:29:10.504903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504903+0000","last_undegraded":"2024-03-04T07:29:10.504903+0000","last_fullsized":"2024-03-04T07:29:10.504903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505803+0000","last_change":"2024-03-04T07:29:10.505803+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505803+0000","last_undegraded":"2024-03-04T07:29:10.505803+0000","last_fullsized":"2024-03-04T07:29:10.505803+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.29","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506903+0000","last_change":"2024-03-04T07:29:10.506903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506903+0000","last_undegraded":"2024-03-04T07:29:10.506903+0000","last_fullsized":"2024-03-04T07:29:10.506903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.28","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505190+0000","last_change":"2024-03-04T07:29:10.505190+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505190+0000","last_undegraded":"2024-03-04T07:29:10.505190+0000","last_fullsized":"2024-03-04T07:29:10.505190+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.27","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502380+0000","last_change":"2024-03-04T07:29:10.502380+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502380+0000","last_undegraded":"2024-03-04T07:29:10.502380+0000","last_fullsized":"2024-03-04T07:29:10.502380+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.26","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499795+0000","last_change":"2024-03-04T07:29:10.499795+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499795+0000","last_undegraded":"2024-03-04T07:29:10.499795+0000","last_fullsized":"2024-03-04T07:29:10.499795+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.25","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507608+0000","last_change":"2024-03-04T07:29:10.507608+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507608+0000","last_undegraded":"2024-03-04T07:29:10.507608+0000","last_fullsized":"2024-03-04T07:29:10.507608+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.24","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502297+0000","last_change":"2024-03-04T07:29:10.502297+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502297+0000","last_undegraded":"2024-03-04T07:29:10.502297+0000","last_fullsized":"2024-03-04T07:29:10.502297+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.23","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507197+0000","last_change":"2024-03-04T07:29:10.507197+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507197+0000","last_undegraded":"2024-03-04T07:29:10.507197+0000","last_fullsized":"2024-03-04T07:29:10.507197+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.22","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505391+0000","last_change":"2024-03-04T07:29:10.505391+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505391+0000","last_undegraded":"2024-03-04T07:29:10.505391+0000","last_fullsized":"2024-03-04T07:29:10.505391+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.21","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502303+0000","last_change":"2024-03-04T07:29:10.502303+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502303+0000","last_undegraded":"2024-03-04T07:29:10.502303+0000","last_fullsized":"2024-03-04T07:29:10.502303+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.20","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506384+0000","last_change":"2024-03-04T07:29:10.506384+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506384+0000","last_undegraded":"2024-03-04T07:29:10.506384+0000","last_fullsized":"2024-03-04T07:29:10.506384+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505456+0000","last_change":"2024-03-04T07:29:10.505456+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505456+0000","last_undegraded":"2024-03-04T07:29:10.505456+0000","last_fullsized":"2024-03-04T07:29:10.505456+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.513909+0000","last_change":"2024-03-04T07:29:10.512338+0000","last_active":"2024-03-04T07:29:10.513909+0000","last_peered":"2024-03-04T07:29:10.513909+0000","last_clean":"2024-03-04T07:29:10.513909+0000","last_became_active":"2024-03-04T07:29:10.512059+0000","last_became_peered":"2024-03-04T07:29:10.512059+0000","last_unstale":"2024-03-04T07:29:10.513909+0000","last_undegraded":"2024-03-04T07:29:10.513909+0000","last_fullsized":"2024-03-04T07:29:10.513909+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503363+0000","last_change":"2024-03-04T07:29:10.503363+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503363+0000","last_undegraded":"2024-03-04T07:29:10.503363+0000","last_fullsized":"2024-03-04T07:29:10.503363+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518723+0000","last_change":"2024-03-04T07:29:10.515404+0000","last_active":"2024-03-04T07:29:10.518723+0000","last_peered":"2024-03-04T07:29:10.518723+0000","last_clean":"2024-03-04T07:29:10.518723+0000","last_became_active":"2024-03-04T07:29:10.515135+0000","last_became_peered":"2024-03-04T07:29:10.515135+0000","last_unstale":"2024-03-04T07:29:10.518723+0000","last_undegraded":"2024-03-04T07:29:10.518723+0000","last_fullsized":"2024-03-04T07:29:10.518723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:12:54.069572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505684+0000","last_change":"2024-03-04T07:29:10.505684+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505684+0000","last_undegraded":"2024-03-04T07:29:10.505684+0000","last_fullsized":"2024-03-04T07:29:10.505684+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518470+0000","last_change":"2024-03-04T07:29:10.515994+0000","last_active":"2024-03-04T07:29:10.518470+0000","last_peered":"2024-03-04T07:29:10.518470+0000","last_clean":"2024-03-04T07:29:10.518470+0000","last_became_active":"2024-03-04T07:29:10.515751+0000","last_became_peered":"2024-03-04T07:29:10.515751+0000","last_unstale":"2024-03-04T07:29:10.518470+0000","last_undegraded":"2024-03-04T07:29:10.518470+0000","last_fullsized":"2024-03-04T07:29:10.518470+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507219+0000","last_change":"2024-03-04T07:29:10.507219+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507219+0000","last_undegraded":"2024-03-04T07:29:10.507219+0000","last_fullsized":"2024-03-04T07:29:10.507219+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516461+0000","last_change":"2024-03-04T07:29:10.509745+0000","last_active":"2024-03-04T07:29:10.516461+0000","last_peered":"2024-03-04T07:29:10.516461+0000","last_clean":"2024-03-04T07:29:10.516461+0000","last_became_active":"2024-03-04T07:29:10.509471+0000","last_became_peered":"2024-03-04T07:29:10.509471+0000","last_unstale":"2024-03-04T07:29:10.516461+0000","last_undegraded":"2024-03-04T07:29:10.516461+0000","last_fullsized":"2024-03-04T07:29:10.516461+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505406+0000","last_change":"2024-03-04T07:29:10.505406+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505406+0000","last_undegraded":"2024-03-04T07:29:10.505406+0000","last_fullsized":"2024-03-04T07:29:10.505406+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516437+0000","last_change":"2024-03-04T07:29:10.510878+0000","last_active":"2024-03-04T07:29:10.516437+0000","last_peered":"2024-03-04T07:29:10.516437+0000","last_clean":"2024-03-04T07:29:10.516437+0000","last_became_active":"2024-03-04T07:29:10.510509+0000","last_became_peered":"2024-03-04T07:29:10.510509+0000","last_unstale":"2024-03-04T07:29:10.516437+0000","last_undegraded":"2024-03-04T07:29:10.516437+0000","last_fullsized":"2024-03-04T07:29:10.516437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:51:59.390945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508983+0000","last_change":"2024-03-04T07:29:10.508983+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508983+0000","last_undegraded":"2024-03-04T07:29:10.508983+0000","last_fullsized":"2024-03-04T07:29:10.508983+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519100+0000","last_change":"2024-03-04T07:29:10.517386+0000","last_active":"2024-03-04T07:29:10.519100+0000","last_peered":"2024-03-04T07:29:10.519100+0000","last_clean":"2024-03-04T07:29:10.519100+0000","last_became_active":"2024-03-04T07:29:10.517120+0000","last_became_peered":"2024-03-04T07:29:10.517120+0000","last_unstale":"2024-03-04T07:29:10.519100+0000","last_undegraded":"2024-03-04T07:29:10.519100+0000","last_fullsized":"2024-03-04T07:29:10.519100+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.18","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504666+0000","last_change":"2024-03-04T07:29:10.504666+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504666+0000","last_undegraded":"2024-03-04T07:29:10.504666+0000","last_fullsized":"2024-03-04T07:29:10.504666+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516345+0000","last_change":"2024-03-04T07:29:10.509713+0000","last_active":"2024-03-04T07:29:10.516345+0000","last_peered":"2024-03-04T07:29:10.516345+0000","last_clean":"2024-03-04T07:29:10.516345+0000","last_became_active":"2024-03-04T07:29:10.509409+0000","last_became_peered":"2024-03-04T07:29:10.509409+0000","last_unstale":"2024-03-04T07:29:10.516345+0000","last_undegraded":"2024-03-04T07:29:10.516345+0000","last_fullsized":"2024-03-04T07:29:10.516345+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508131+0000","last_change":"2024-03-04T07:29:10.508131+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508131+0000","last_undegraded":"2024-03-04T07:29:10.508131+0000","last_fullsized":"2024-03-04T07:29:10.508131+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516481+0000","last_change":"2024-03-04T07:29:10.512481+0000","last_active":"2024-03-04T07:29:10.516481+0000","last_peered":"2024-03-04T07:29:10.516481+0000","last_clean":"2024-03-04T07:29:10.516481+0000","last_became_active":"2024-03-04T07:29:10.512206+0000","last_became_peered":"2024-03-04T07:29:10.512206+0000","last_unstale":"2024-03-04T07:29:10.516481+0000","last_undegraded":"2024-03-04T07:29:10.516481+0000","last_fullsized":"2024-03-04T07:29:10.516481+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:19.657851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.16","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501436+0000","last_change":"2024-03-04T07:29:10.501436+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501436+0000","last_undegraded":"2024-03-04T07:29:10.501436+0000","last_fullsized":"2024-03-04T07:29:10.501436+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.17","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518954+0000","last_change":"2024-03-04T07:29:10.514467+0000","last_active":"2024-03-04T07:29:10.518954+0000","last_peered":"2024-03-04T07:29:10.518954+0000","last_clean":"2024-03-04T07:29:10.518954+0000","last_became_active":"2024-03-04T07:29:10.514160+0000","last_became_peered":"2024-03-04T07:29:10.514160+0000","last_unstale":"2024-03-04T07:29:10.518954+0000","last_undegraded":"2024-03-04T07:29:10.518954+0000","last_fullsized":"2024-03-04T07:29:10.518954+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.17","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504882+0000","last_change":"2024-03-04T07:29:10.504882+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504882+0000","last_undegraded":"2024-03-04T07:29:10.504882+0000","last_fullsized":"2024-03-04T07:29:10.504882+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.16","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518939+0000","last_change":"2024-03-04T07:29:10.517195+0000","last_active":"2024-03-04T07:29:10.518939+0000","last_peered":"2024-03-04T07:29:10.518939+0000","last_clean":"2024-03-04T07:29:10.518939+0000","last_became_active":"2024-03-04T07:29:10.516936+0000","last_became_peered":"2024-03-04T07:29:10.516936+0000","last_unstale":"2024-03-04T07:29:10.518939+0000","last_undegraded":"2024-03-04T07:29:10.518939+0000","last_fullsized":"2024-03-04T07:29:10.518939+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:00.174138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502894+0000","last_change":"2024-03-04T07:29:10.502894+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502894+0000","last_undegraded":"2024-03-04T07:29:10.502894+0000","last_fullsized":"2024-03-04T07:29:10.502894+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.15","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518958+0000","last_change":"2024-03-04T07:29:10.515515+0000","last_active":"2024-03-04T07:29:10.518958+0000","last_peered":"2024-03-04T07:29:10.518958+0000","last_clean":"2024-03-04T07:29:10.518958+0000","last_became_active":"2024-03-04T07:29:10.515229+0000","last_became_peered":"2024-03-04T07:29:10.515229+0000","last_unstale":"2024-03-04T07:29:10.518958+0000","last_undegraded":"2024-03-04T07:29:10.518958+0000","last_fullsized":"2024-03-04T07:29:10.518958+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.15","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505698+0000","last_change":"2024-03-04T07:29:10.505698+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505698+0000","last_undegraded":"2024-03-04T07:29:10.505698+0000","last_fullsized":"2024-03-04T07:29:10.505698+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.14","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516384+0000","last_change":"2024-03-04T07:29:10.509271+0000","last_active":"2024-03-04T07:29:10.516384+0000","last_peered":"2024-03-04T07:29:10.516384+0000","last_clean":"2024-03-04T07:29:10.516384+0000","last_became_active":"2024-03-04T07:29:10.508965+0000","last_became_peered":"2024-03-04T07:29:10.508965+0000","last_unstale":"2024-03-04T07:29:10.516384+0000","last_undegraded":"2024-03-04T07:29:10.516384+0000","last_fullsized":"2024-03-04T07:29:10.516384+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:18:49.238841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.12","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500095+0000","last_change":"2024-03-04T07:29:10.500095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500095+0000","last_undegraded":"2024-03-04T07:29:10.500095+0000","last_fullsized":"2024-03-04T07:29:10.500095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.13","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519067+0000","last_change":"2024-03-04T07:29:10.517336+0000","last_active":"2024-03-04T07:29:10.519067+0000","last_peered":"2024-03-04T07:29:10.519067+0000","last_clean":"2024-03-04T07:29:10.519067+0000","last_became_active":"2024-03-04T07:29:10.517088+0000","last_became_peered":"2024-03-04T07:29:10.517088+0000","last_unstale":"2024-03-04T07:29:10.519067+0000","last_undegraded":"2024-03-04T07:29:10.519067+0000","last_fullsized":"2024-03-04T07:29:10.519067+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:26:41.225141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504221+0000","last_change":"2024-03-04T07:29:10.504221+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504221+0000","last_undegraded":"2024-03-04T07:29:10.504221+0000","last_fullsized":"2024-03-04T07:29:10.504221+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.12","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518895+0000","last_change":"2024-03-04T07:29:10.515222+0000","last_active":"2024-03-04T07:29:10.518895+0000","last_peered":"2024-03-04T07:29:10.518895+0000","last_clean":"2024-03-04T07:29:10.518895+0000","last_became_active":"2024-03-04T07:29:10.514882+0000","last_became_peered":"2024-03-04T07:29:10.514882+0000","last_unstale":"2024-03-04T07:29:10.518895+0000","last_undegraded":"2024-03-04T07:29:10.518895+0000","last_fullsized":"2024-03-04T07:29:10.518895+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:36:31.420388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.10","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500037+0000","last_change":"2024-03-04T07:29:10.500037+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500037+0000","last_undegraded":"2024-03-04T07:29:10.500037+0000","last_fullsized":"2024-03-04T07:29:10.500037+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.11","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518799+0000","last_change":"2024-03-04T07:29:10.515029+0000","last_active":"2024-03-04T07:29:10.518799+0000","last_peered":"2024-03-04T07:29:10.518799+0000","last_clean":"2024-03-04T07:29:10.518799+0000","last_became_active":"2024-03-04T07:29:10.514722+0000","last_became_peered":"2024-03-04T07:29:10.514722+0000","last_unstale":"2024-03-04T07:29:10.518799+0000","last_undegraded":"2024-03-04T07:29:10.518799+0000","last_fullsized":"2024-03-04T07:29:10.518799+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506446+0000","last_change":"2024-03-04T07:29:10.506446+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506446+0000","last_undegraded":"2024-03-04T07:29:10.506446+0000","last_fullsized":"2024-03-04T07:29:10.506446+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.10","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519102+0000","last_change":"2024-03-04T07:29:10.517145+0000","last_active":"2024-03-04T07:29:10.519102+0000","last_peered":"2024-03-04T07:29:10.519102+0000","last_clean":"2024-03-04T07:29:10.519102+0000","last_became_active":"2024-03-04T07:29:10.516791+0000","last_became_peered":"2024-03-04T07:29:10.516791+0000","last_unstale":"2024-03-04T07:29:10.519102+0000","last_undegraded":"2024-03-04T07:29:10.519102+0000","last_fullsized":"2024-03-04T07:29:10.519102+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504566+0000","last_change":"2024-03-04T07:29:10.504566+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504566+0000","last_undegraded":"2024-03-04T07:29:10.504566+0000","last_fullsized":"2024-03-04T07:29:10.504566+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519012+0000","last_change":"2024-03-04T07:29:10.517764+0000","last_active":"2024-03-04T07:29:10.519012+0000","last_peered":"2024-03-04T07:29:10.519012+0000","last_clean":"2024-03-04T07:29:10.519012+0000","last_became_active":"2024-03-04T07:29:10.516895+0000","last_became_peered":"2024-03-04T07:29:10.516895+0000","last_unstale":"2024-03-04T07:29:10.519012+0000","last_undegraded":"2024-03-04T07:29:10.519012+0000","last_fullsized":"2024-03-04T07:29:10.519012+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:04:10.056881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501820+0000","last_change":"2024-03-04T07:29:10.501820+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501820+0000","last_undegraded":"2024-03-04T07:29:10.501820+0000","last_fullsized":"2024-03-04T07:29:10.501820+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518517+0000","last_change":"2024-03-04T07:29:10.515628+0000","last_active":"2024-03-04T07:29:10.518517+0000","last_peered":"2024-03-04T07:29:10.518517+0000","last_clean":"2024-03-04T07:29:10.518517+0000","last_became_active":"2024-03-04T07:29:10.515205+0000","last_became_peered":"2024-03-04T07:29:10.515205+0000","last_unstale":"2024-03-04T07:29:10.518517+0000","last_undegraded":"2024-03-04T07:29:10.518517+0000","last_fullsized":"2024-03-04T07:29:10.518517+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506522+0000","last_change":"2024-03-04T07:29:10.506522+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506522+0000","last_undegraded":"2024-03-04T07:29:10.506522+0000","last_fullsized":"2024-03-04T07:29:10.506522+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518651+0000","last_change":"2024-03-04T07:29:10.512208+0000","last_active":"2024-03-04T07:29:10.518651+0000","last_peered":"2024-03-04T07:29:10.518651+0000","last_clean":"2024-03-04T07:29:10.518651+0000","last_became_active":"2024-03-04T07:29:10.511806+0000","last_became_peered":"2024-03-04T07:29:10.511806+0000","last_unstale":"2024-03-04T07:29:10.518651+0000","last_undegraded":"2024-03-04T07:29:10.518651+0000","last_fullsized":"2024-03-04T07:29:10.518651+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504634+0000","last_change":"2024-03-04T07:29:10.504634+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504634+0000","last_undegraded":"2024-03-04T07:29:10.504634+0000","last_fullsized":"2024-03-04T07:29:10.504634+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519131+0000","last_change":"2024-03-04T07:29:10.516595+0000","last_active":"2024-03-04T07:29:10.519131+0000","last_peered":"2024-03-04T07:29:10.519131+0000","last_clean":"2024-03-04T07:29:10.519131+0000","last_became_active":"2024-03-04T07:29:10.516327+0000","last_became_peered":"2024-03-04T07:29:10.516327+0000","last_unstale":"2024-03-04T07:29:10.519131+0000","last_undegraded":"2024-03-04T07:29:10.519131+0000","last_fullsized":"2024-03-04T07:29:10.519131+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499557+0000","last_change":"2024-03-04T07:29:10.499557+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499557+0000","last_undegraded":"2024-03-04T07:29:10.499557+0000","last_fullsized":"2024-03-04T07:29:10.499557+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519087+0000","last_change":"2024-03-04T07:29:10.517775+0000","last_active":"2024-03-04T07:29:10.519087+0000","last_peered":"2024-03-04T07:29:10.519087+0000","last_clean":"2024-03-04T07:29:10.519087+0000","last_became_active":"2024-03-04T07:29:10.517496+0000","last_became_peered":"2024-03-04T07:29:10.517496+0000","last_unstale":"2024-03-04T07:29:10.519087+0000","last_undegraded":"2024-03-04T07:29:10.519087+0000","last_fullsized":"2024-03-04T07:29:10.519087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:00.707197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510312+0000","last_change":"2024-03-04T07:29:10.510312+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510312+0000","last_undegraded":"2024-03-04T07:29:10.510312+0000","last_fullsized":"2024-03-04T07:29:10.510312+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519200+0000","last_change":"2024-03-04T07:29:10.514774+0000","last_active":"2024-03-04T07:29:10.519200+0000","last_peered":"2024-03-04T07:29:10.519200+0000","last_clean":"2024-03-04T07:29:10.519200+0000","last_became_active":"2024-03-04T07:29:10.514489+0000","last_became_peered":"2024-03-04T07:29:10.514489+0000","last_unstale":"2024-03-04T07:29:10.519200+0000","last_undegraded":"2024-03-04T07:29:10.519200+0000","last_fullsized":"2024-03-04T07:29:10.519200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:20.173974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506687+0000","last_change":"2024-03-04T07:29:10.506687+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506687+0000","last_undegraded":"2024-03-04T07:29:10.506687+0000","last_fullsized":"2024-03-04T07:29:10.506687+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518751+0000","last_change":"2024-03-04T07:29:10.513482+0000","last_active":"2024-03-04T07:29:10.518751+0000","last_peered":"2024-03-04T07:29:10.518751+0000","last_clean":"2024-03-04T07:29:10.518751+0000","last_became_active":"2024-03-04T07:29:10.513209+0000","last_became_peered":"2024-03-04T07:29:10.513209+0000","last_unstale":"2024-03-04T07:29:10.518751+0000","last_undegraded":"2024-03-04T07:29:10.518751+0000","last_fullsized":"2024-03-04T07:29:10.518751+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499078+0000","last_change":"2024-03-04T07:29:10.499078+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499078+0000","last_undegraded":"2024-03-04T07:29:10.499078+0000","last_fullsized":"2024-03-04T07:29:10.499078+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519230+0000","last_change":"2024-03-04T07:29:10.516454+0000","last_active":"2024-03-04T07:29:10.519230+0000","last_peered":"2024-03-04T07:29:10.519230+0000","last_clean":"2024-03-04T07:29:10.519230+0000","last_became_active":"2024-03-04T07:29:10.514845+0000","last_became_peered":"2024-03-04T07:29:10.514845+0000","last_unstale":"2024-03-04T07:29:10.519230+0000","last_undegraded":"2024-03-04T07:29:10.519230+0000","last_fullsized":"2024-03-04T07:29:10.519230+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516782+0000","last_change":"2024-03-04T07:29:10.514474+0000","last_active":"2024-03-04T07:29:10.516782+0000","last_peered":"2024-03-04T07:29:10.516782+0000","last_clean":"2024-03-04T07:29:10.516782+0000","last_became_active":"2024-03-04T07:29:10.514215+0000","last_became_peered":"2024-03-04T07:29:10.514215+0000","last_unstale":"2024-03-04T07:29:10.516782+0000","last_undegraded":"2024-03-04T07:29:10.516782+0000","last_fullsized":"2024-03-04T07:29:10.516782+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502443+0000","last_change":"2024-03-04T07:29:10.502443+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502443+0000","last_undegraded":"2024-03-04T07:29:10.502443+0000","last_fullsized":"2024-03-04T07:29:10.502443+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519002+0000","last_change":"2024-03-04T07:29:10.515670+0000","last_active":"2024-03-04T07:29:10.519002+0000","last_peered":"2024-03-04T07:29:10.519002+0000","last_clean":"2024-03-04T07:29:10.519002+0000","last_became_active":"2024-03-04T07:29:10.515327+0000","last_became_peered":"2024-03-04T07:29:10.515327+0000","last_unstale":"2024-03-04T07:29:10.519002+0000","last_undegraded":"2024-03-04T07:29:10.519002+0000","last_fullsized":"2024-03-04T07:29:10.519002+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:54:04.259042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506095+0000","last_change":"2024-03-04T07:29:10.506095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506095+0000","last_undegraded":"2024-03-04T07:29:10.506095+0000","last_fullsized":"2024-03-04T07:29:10.506095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.509863+0000","last_change":"2024-03-04T07:29:10.506371+0000","last_active":"2024-03-04T07:29:10.509863+0000","last_peered":"2024-03-04T07:29:10.509863+0000","last_clean":"2024-03-04T07:29:10.509863+0000","last_became_active":"2024-03-04T07:29:10.506084+0000","last_became_peered":"2024-03-04T07:29:10.506084+0000","last_unstale":"2024-03-04T07:29:10.509863+0000","last_undegraded":"2024-03-04T07:29:10.509863+0000","last_fullsized":"2024-03-04T07:29:10.509863+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504947+0000","last_change":"2024-03-04T07:29:10.504947+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504947+0000","last_undegraded":"2024-03-04T07:29:10.504947+0000","last_fullsized":"2024-03-04T07:29:10.504947+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.514479+0000","last_change":"2024-03-04T07:29:10.510685+0000","last_active":"2024-03-04T07:29:10.514479+0000","last_peered":"2024-03-04T07:29:10.514479+0000","last_clean":"2024-03-04T07:29:10.514479+0000","last_became_active":"2024-03-04T07:29:10.510433+0000","last_became_peered":"2024-03-04T07:29:10.510433+0000","last_unstale":"2024-03-04T07:29:10.514479+0000","last_undegraded":"2024-03-04T07:29:10.514479+0000","last_fullsized":"2024-03-04T07:29:10.514479+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:04:10.012697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503678+0000","last_change":"2024-03-04T07:29:10.503678+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503678+0000","last_undegraded":"2024-03-04T07:29:10.503678+0000","last_fullsized":"2024-03-04T07:29:10.503678+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518944+0000","last_change":"2024-03-04T07:29:10.513884+0000","last_active":"2024-03-04T07:29:10.518944+0000","last_peered":"2024-03-04T07:29:10.518944+0000","last_clean":"2024-03-04T07:29:10.518944+0000","last_became_active":"2024-03-04T07:29:10.513574+0000","last_became_peered":"2024-03-04T07:29:10.513574+0000","last_unstale":"2024-03-04T07:29:10.518944+0000","last_undegraded":"2024-03-04T07:29:10.518944+0000","last_fullsized":"2024-03-04T07:29:10.518944+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:39.944442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502740+0000","last_change":"2024-03-04T07:29:10.502740+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502740+0000","last_undegraded":"2024-03-04T07:29:10.502740+0000","last_fullsized":"2024-03-04T07:29:10.502740+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516550+0000","last_change":"2024-03-04T07:29:10.514430+0000","last_active":"2024-03-04T07:29:10.516550+0000","last_peered":"2024-03-04T07:29:10.516550+0000","last_clean":"2024-03-04T07:29:10.516550+0000","last_became_active":"2024-03-04T07:29:10.514146+0000","last_became_peered":"2024-03-04T07:29:10.514146+0000","last_unstale":"2024-03-04T07:29:10.516550+0000","last_undegraded":"2024-03-04T07:29:10.516550+0000","last_fullsized":"2024-03-04T07:29:10.516550+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:29:39.502383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507659+0000","last_change":"2024-03-04T07:29:10.507659+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507659+0000","last_undegraded":"2024-03-04T07:29:10.507659+0000","last_fullsized":"2024-03-04T07:29:10.507659+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.521576+0000","last_change":"2024-03-04T07:29:10.519578+0000","last_active":"2024-03-04T07:29:10.521576+0000","last_peered":"2024-03-04T07:29:10.521576+0000","last_clean":"2024-03-04T07:29:10.521576+0000","last_became_active":"2024-03-04T07:29:10.519248+0000","last_became_peered":"2024-03-04T07:29:10.519248+0000","last_unstale":"2024-03-04T07:29:10.521576+0000","last_undegraded":"2024-03-04T07:29:10.521576+0000","last_fullsized":"2024-03-04T07:29:10.521576+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:11.247866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506035+0000","last_change":"2024-03-04T07:29:10.506035+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506035+0000","last_undegraded":"2024-03-04T07:29:10.506035+0000","last_fullsized":"2024-03-04T07:29:10.506035+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516800+0000","last_change":"2024-03-04T07:29:10.513166+0000","last_active":"2024-03-04T07:29:10.516800+0000","last_peered":"2024-03-04T07:29:10.516800+0000","last_clean":"2024-03-04T07:29:10.516800+0000","last_became_active":"2024-03-04T07:29:10.512608+0000","last_became_peered":"2024-03-04T07:29:10.512608+0000","last_unstale":"2024-03-04T07:29:10.516800+0000","last_undegraded":"2024-03-04T07:29:10.516800+0000","last_fullsized":"2024-03-04T07:29:10.516800+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":230,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.507585+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:29:10.507585+0000","last_peered":"2024-03-04T07:29:10.507585+0000","last_clean":"2024-03-04T07:29:10.507585+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:29:10.507585+0000","last_undegraded":"2024-03-04T07:29:10.507585+0000","last_fullsized":"2024-03-04T07:29:10.507585+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":8,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574883,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32376,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339464,"statfs":{"total":96636764160,"available":96603611136,"internally_reserved":0,"allocated":1216512,"data_stored":844018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31812,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340028,"statfs":{"total":96636764160,"available":96604188672,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574883,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31808,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340032,"statfs":{"total":96636764160,"available":96604192768,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31788,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340052,"statfs":{"total":96636764160,"available":96604213248,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574882,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31796,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340044,"statfs":{"total":96636764160,"available":96604205056,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574882,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32392,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339448,"statfs":{"total":96636764160,"available":96603594752,"internally_reserved":0,"allocated":1228800,"data_stored":850285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:29:14.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:29:15.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.093+0000 7f20f18ed640 1 Processor -- start 2024-03-04T07:29:15.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.093+0000 7f20f18ed640 1 -- start start 2024-03-04T07:29:15.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.094+0000 7f20f18ed640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f20ec06b5c0 0x7f20ec0b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.094+0000 7f20f18ed640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f20ec0b3340 con 0x7f20ec0b0e90 2024-03-04T07:29:15.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.094+0000 7f20f18ed640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f20ec0cee90 con 0x7f20ec0ae9b0 2024-03-04T07:29:15.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.094+0000 7f20f18ed640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f20ec0cfa00 con 0x7f20ec06b5c0 2024-03-04T07:29:15.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20eaffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f20ec06b5c0 0x7f20ec0b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20ea7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f20ec0b0e90 0x7f20ec06b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35682/0 (socket says 172.21.15.64:35682) 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20eaffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f20ec06b5c0 0x7f20ec0b2e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39308/0 (socket says 172.21.15.64:39308) 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20ea7fc640 1 -- 172.21.15.64:0/488495844 learned_addr learned my addr 172.21.15.64:0/488495844 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20eaffd640 1 -- 172.21.15.64:0/488495844 >> v1:172.21.15.66:6789/0 conn(0x7f20ec0ae9b0 legacy=0x7f20ec0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20eaffd640 1 -- 172.21.15.64:0/488495844 >> v1:172.21.15.64:6789/0 conn(0x7f20ec0b0e90 legacy=0x7f20ec06b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20eaffd640 1 -- 172.21.15.64:0/488495844 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20ec0cfc50 con 0x7f20ec06b5c0 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.095+0000 7f20e9ffb640 1 -- 172.21.15.64:0/488495844 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3248982242 0 0) 0x7f20ec0b3340 con 0x7f20ec0b0e90 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20eaffd640 1 --2- 172.21.15.64:0/488495844 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec06b5c0 0x7f20ec0b2e00 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7f20dc0097a0 tx=0x7f20dc02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e35c78820a6958a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20e9ffb640 1 -- 172.21.15.64:0/488495844 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20dc035020 con 0x7f20ec06b5c0 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20e9ffb640 1 -- 172.21.15.64:0/488495844 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20dc031830 con 0x7f20ec06b5c0 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20e9ffb640 1 -- 172.21.15.64:0/488495844 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20dc031c00 con 0x7f20ec06b5c0 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20f18ed640 1 -- 172.21.15.64:0/488495844 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec06b5c0 msgr2=0x7f20ec0b2e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20f18ed640 1 --2- 172.21.15.64:0/488495844 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec06b5c0 0x7f20ec0b2e00 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7f20dc0097a0 tx=0x7f20dc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20f18ed640 1 -- 172.21.15.64:0/488495844 shutdown_connections 2024-03-04T07:29:15.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20f18ed640 1 --2- 172.21.15.64:0/488495844 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec06b5c0 0x7f20ec0b2e00 unknown :-1 s=CLOSED pgs=674 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20f18ed640 1 -- 172.21.15.64:0/488495844 >> 172.21.15.64:0/488495844 conn(0x7f20ec0b36d0 msgr2=0x7f20ec0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:15.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20f18ed640 1 -- 172.21.15.64:0/488495844 shutdown_connections 2024-03-04T07:29:15.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.096+0000 7f20f18ed640 1 -- 172.21.15.64:0/488495844 wait complete. 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.098+0000 7f20f18ed640 1 Processor -- start 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.098+0000 7f20f18ed640 1 -- start start 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.098+0000 7f20f18ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20ec06b5c0 0x7f20ec1550b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.098+0000 7f20f18ed640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20ec0ae9b0 0x7f20ec0c8e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20f18ed640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec0b0e90 0x7f20ec0b59e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20f18ed640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20ec06c910 con 0x7f20ec0b0e90 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20f18ed640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20ec06c790 con 0x7f20ec0ae9b0 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20f18ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20ec0d0ab0 con 0x7f20ec06b5c0 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec0b0e90 0x7f20ec0b59e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec0b0e90 0x7f20ec0b59e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39312/0 (socket says 172.21.15.64:39312) 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 -- 172.21.15.64:0/1757775672 learned_addr learned my addr 172.21.15.64:0/1757775672 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eaffd640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20ec06b5c0 0x7f20ec1550b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20ea7fc640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20ec0ae9b0 0x7f20ec0c8e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 -- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20ec06b5c0 msgr2=0x7f20ec1550b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20ec06b5c0 0x7f20ec1550b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 -- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20ec0ae9b0 msgr2=0x7f20ec0c8e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20ec0ae9b0 0x7f20ec0c8e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 -- 172.21.15.64:0/1757775672 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20ec0c5a30 con 0x7f20ec0b0e90 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20eb7fe640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec0b0e90 0x7f20ec0b59e0 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7f20e400c7d0 tx=0x7f20e400cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20d3fff640 1 -- 172.21.15.64:0/1757775672 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20e4003b70 con 0x7f20ec0b0e90 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20d3fff640 1 -- 172.21.15.64:0/1757775672 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f20e4003d30 con 0x7f20ec0b0e90 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20ec0c5d10 con 0x7f20ec0b0e90 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20d3fff640 1 -- 172.21.15.64:0/1757775672 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20e400aa90 con 0x7f20ec0b0e90 2024-03-04T07:29:15.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.099+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20ec0c62d0 con 0x7f20ec0b0e90 2024-03-04T07:29:15.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.101+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f20ec068700 con 0x7f20ec0b0e90 2024-03-04T07:29:15.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.101+0000 7f20d3fff640 1 -- 172.21.15.64:0/1757775672 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f20e400ac30 con 0x7f20ec0b0e90 2024-03-04T07:29:15.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.101+0000 7f20d3fff640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d806b4d0 0x7f20d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.101+0000 7f20eaffd640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d806b4d0 0x7f20d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.101+0000 7f20d3fff640 1 -- 172.21.15.64:0/1757775672 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f20e4089f40 con 0x7f20ec0b0e90 2024-03-04T07:29:15.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.102+0000 7f20eaffd640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d806b4d0 0x7f20d806d990 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f20dc036000 tx=0x7f20dc008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.106+0000 7f20d3fff640 1 -- 172.21.15.64:0/1757775672 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f20e4058800 con 0x7f20ec0b0e90 2024-03-04T07:29:15.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.282+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f20ec0008d0 con 0x7f20d806b4d0 2024-03-04T07:29:15.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.288+0000 7f20d3fff640 1 -- 172.21.15.64:0/1757775672 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7f20ec0008d0 con 0x7f20d806b4d0 2024-03-04T07:29:15.291 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:15.293 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:29:15.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d806b4d0 msgr2=0x7f20d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d806b4d0 0x7f20d806d990 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f20dc036000 tx=0x7f20dc008030 comp rx=0 tx=0).stop 2024-03-04T07:29:15.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec0b0e90 msgr2=0x7f20ec0b59e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec0b0e90 0x7f20ec0b59e0 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7f20e400c7d0 tx=0x7f20e400cc90 comp rx=0 tx=0).stop 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 shutdown_connections 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f20d806b4d0 0x7f20d806d990 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f20ec0b0e90 0x7f20ec0b59e0 unknown :-1 s=CLOSED pgs=675 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f20ec0ae9b0 0x7f20ec0c8e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 --2- 172.21.15.64:0/1757775672 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f20ec06b5c0 0x7f20ec1550b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 >> 172.21.15.64:0/1757775672 conn(0x7f20ec0b36d0 msgr2=0x7f20ec0701d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 shutdown_connections 2024-03-04T07:29:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.291+0000 7f20f18ed640 1 -- 172.21.15.64:0/1757775672 wait complete. 2024-03-04T07:29:15.303 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":125,"stamp":"2024-03-04T07:29:15.168119+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":255580,"kb_used_data":6084,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754719140,"statfs":{"total":773094113280,"available":772832399360,"internally_reserved":0,"allocated":6230016,"data_stored":3241271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.679194"},"pg_stats":[{"pgid":"9.3f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503695+0000","last_change":"2024-03-04T07:29:10.503695+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503695+0000","last_undegraded":"2024-03-04T07:29:10.503695+0000","last_fullsized":"2024-03-04T07:29:10.503695+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.509497+0000","last_change":"2024-03-04T07:29:10.509497+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.509497+0000","last_undegraded":"2024-03-04T07:29:10.509497+0000","last_fullsized":"2024-03-04T07:29:10.509497+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.3d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506870+0000","last_change":"2024-03-04T07:29:10.506870+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506870+0000","last_undegraded":"2024-03-04T07:29:10.506870+0000","last_fullsized":"2024-03-04T07:29:10.506870+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.3c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502699+0000","last_change":"2024-03-04T07:29:10.502699+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502699+0000","last_undegraded":"2024-03-04T07:29:10.502699+0000","last_fullsized":"2024-03-04T07:29:10.502699+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.3b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503688+0000","last_change":"2024-03-04T07:29:10.503688+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503688+0000","last_undegraded":"2024-03-04T07:29:10.503688+0000","last_fullsized":"2024-03-04T07:29:10.503688+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510758+0000","last_change":"2024-03-04T07:29:10.510758+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510758+0000","last_undegraded":"2024-03-04T07:29:10.510758+0000","last_fullsized":"2024-03-04T07:29:10.510758+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.39","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.38","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506502+0000","last_change":"2024-03-04T07:29:10.506502+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506502+0000","last_undegraded":"2024-03-04T07:29:10.506502+0000","last_fullsized":"2024-03-04T07:29:10.506502+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.37","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500610+0000","last_change":"2024-03-04T07:29:10.500610+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500610+0000","last_undegraded":"2024-03-04T07:29:10.500610+0000","last_fullsized":"2024-03-04T07:29:10.500610+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.36","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502657+0000","last_change":"2024-03-04T07:29:10.502657+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502657+0000","last_undegraded":"2024-03-04T07:29:10.502657+0000","last_fullsized":"2024-03-04T07:29:10.502657+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.35","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504279+0000","last_change":"2024-03-04T07:29:10.504279+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504279+0000","last_undegraded":"2024-03-04T07:29:10.504279+0000","last_fullsized":"2024-03-04T07:29:10.504279+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.34","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500622+0000","last_change":"2024-03-04T07:29:10.500622+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500622+0000","last_undegraded":"2024-03-04T07:29:10.500622+0000","last_fullsized":"2024-03-04T07:29:10.500622+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.33","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505723+0000","last_change":"2024-03-04T07:29:10.505723+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505723+0000","last_undegraded":"2024-03-04T07:29:10.505723+0000","last_fullsized":"2024-03-04T07:29:10.505723+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.32","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500946+0000","last_change":"2024-03-04T07:29:10.500946+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500946+0000","last_undegraded":"2024-03-04T07:29:10.500946+0000","last_fullsized":"2024-03-04T07:29:10.500946+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.31","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500479+0000","last_change":"2024-03-04T07:29:10.500479+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500479+0000","last_undegraded":"2024-03-04T07:29:10.500479+0000","last_fullsized":"2024-03-04T07:29:10.500479+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.30","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506809+0000","last_change":"2024-03-04T07:29:10.506809+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506809+0000","last_undegraded":"2024-03-04T07:29:10.506809+0000","last_fullsized":"2024-03-04T07:29:10.506809+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.2f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507972+0000","last_change":"2024-03-04T07:29:10.507972+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507972+0000","last_undegraded":"2024-03-04T07:29:10.507972+0000","last_fullsized":"2024-03-04T07:29:10.507972+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505474+0000","last_change":"2024-03-04T07:29:10.505474+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505474+0000","last_undegraded":"2024-03-04T07:29:10.505474+0000","last_fullsized":"2024-03-04T07:29:10.505474+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503206+0000","last_change":"2024-03-04T07:29:10.503206+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503206+0000","last_undegraded":"2024-03-04T07:29:10.503206+0000","last_fullsized":"2024-03-04T07:29:10.503206+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.2c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507000+0000","last_change":"2024-03-04T07:29:10.507000+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507000+0000","last_undegraded":"2024-03-04T07:29:10.507000+0000","last_fullsized":"2024-03-04T07:29:10.507000+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504903+0000","last_change":"2024-03-04T07:29:10.504903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504903+0000","last_undegraded":"2024-03-04T07:29:10.504903+0000","last_fullsized":"2024-03-04T07:29:10.504903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505803+0000","last_change":"2024-03-04T07:29:10.505803+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505803+0000","last_undegraded":"2024-03-04T07:29:10.505803+0000","last_fullsized":"2024-03-04T07:29:10.505803+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.29","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506903+0000","last_change":"2024-03-04T07:29:10.506903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506903+0000","last_undegraded":"2024-03-04T07:29:10.506903+0000","last_fullsized":"2024-03-04T07:29:10.506903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.28","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505190+0000","last_change":"2024-03-04T07:29:10.505190+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505190+0000","last_undegraded":"2024-03-04T07:29:10.505190+0000","last_fullsized":"2024-03-04T07:29:10.505190+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.27","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502380+0000","last_change":"2024-03-04T07:29:10.502380+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502380+0000","last_undegraded":"2024-03-04T07:29:10.502380+0000","last_fullsized":"2024-03-04T07:29:10.502380+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.26","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499795+0000","last_change":"2024-03-04T07:29:10.499795+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499795+0000","last_undegraded":"2024-03-04T07:29:10.499795+0000","last_fullsized":"2024-03-04T07:29:10.499795+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.25","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507608+0000","last_change":"2024-03-04T07:29:10.507608+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507608+0000","last_undegraded":"2024-03-04T07:29:10.507608+0000","last_fullsized":"2024-03-04T07:29:10.507608+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.24","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502297+0000","last_change":"2024-03-04T07:29:10.502297+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502297+0000","last_undegraded":"2024-03-04T07:29:10.502297+0000","last_fullsized":"2024-03-04T07:29:10.502297+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.23","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507197+0000","last_change":"2024-03-04T07:29:10.507197+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507197+0000","last_undegraded":"2024-03-04T07:29:10.507197+0000","last_fullsized":"2024-03-04T07:29:10.507197+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.22","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505391+0000","last_change":"2024-03-04T07:29:10.505391+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505391+0000","last_undegraded":"2024-03-04T07:29:10.505391+0000","last_fullsized":"2024-03-04T07:29:10.505391+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.21","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502303+0000","last_change":"2024-03-04T07:29:10.502303+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502303+0000","last_undegraded":"2024-03-04T07:29:10.502303+0000","last_fullsized":"2024-03-04T07:29:10.502303+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.20","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506384+0000","last_change":"2024-03-04T07:29:10.506384+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506384+0000","last_undegraded":"2024-03-04T07:29:10.506384+0000","last_fullsized":"2024-03-04T07:29:10.506384+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505456+0000","last_change":"2024-03-04T07:29:10.505456+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505456+0000","last_undegraded":"2024-03-04T07:29:10.505456+0000","last_fullsized":"2024-03-04T07:29:10.505456+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.513909+0000","last_change":"2024-03-04T07:29:10.512338+0000","last_active":"2024-03-04T07:29:10.513909+0000","last_peered":"2024-03-04T07:29:10.513909+0000","last_clean":"2024-03-04T07:29:10.513909+0000","last_became_active":"2024-03-04T07:29:10.512059+0000","last_became_peered":"2024-03-04T07:29:10.512059+0000","last_unstale":"2024-03-04T07:29:10.513909+0000","last_undegraded":"2024-03-04T07:29:10.513909+0000","last_fullsized":"2024-03-04T07:29:10.513909+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503363+0000","last_change":"2024-03-04T07:29:10.503363+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503363+0000","last_undegraded":"2024-03-04T07:29:10.503363+0000","last_fullsized":"2024-03-04T07:29:10.503363+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518723+0000","last_change":"2024-03-04T07:29:10.515404+0000","last_active":"2024-03-04T07:29:10.518723+0000","last_peered":"2024-03-04T07:29:10.518723+0000","last_clean":"2024-03-04T07:29:10.518723+0000","last_became_active":"2024-03-04T07:29:10.515135+0000","last_became_peered":"2024-03-04T07:29:10.515135+0000","last_unstale":"2024-03-04T07:29:10.518723+0000","last_undegraded":"2024-03-04T07:29:10.518723+0000","last_fullsized":"2024-03-04T07:29:10.518723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:12:54.069572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505684+0000","last_change":"2024-03-04T07:29:10.505684+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505684+0000","last_undegraded":"2024-03-04T07:29:10.505684+0000","last_fullsized":"2024-03-04T07:29:10.505684+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518470+0000","last_change":"2024-03-04T07:29:10.515994+0000","last_active":"2024-03-04T07:29:10.518470+0000","last_peered":"2024-03-04T07:29:10.518470+0000","last_clean":"2024-03-04T07:29:10.518470+0000","last_became_active":"2024-03-04T07:29:10.515751+0000","last_became_peered":"2024-03-04T07:29:10.515751+0000","last_unstale":"2024-03-04T07:29:10.518470+0000","last_undegraded":"2024-03-04T07:29:10.518470+0000","last_fullsized":"2024-03-04T07:29:10.518470+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507219+0000","last_change":"2024-03-04T07:29:10.507219+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507219+0000","last_undegraded":"2024-03-04T07:29:10.507219+0000","last_fullsized":"2024-03-04T07:29:10.507219+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516461+0000","last_change":"2024-03-04T07:29:10.509745+0000","last_active":"2024-03-04T07:29:10.516461+0000","last_peered":"2024-03-04T07:29:10.516461+0000","last_clean":"2024-03-04T07:29:10.516461+0000","last_became_active":"2024-03-04T07:29:10.509471+0000","last_became_peered":"2024-03-04T07:29:10.509471+0000","last_unstale":"2024-03-04T07:29:10.516461+0000","last_undegraded":"2024-03-04T07:29:10.516461+0000","last_fullsized":"2024-03-04T07:29:10.516461+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505406+0000","last_change":"2024-03-04T07:29:10.505406+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505406+0000","last_undegraded":"2024-03-04T07:29:10.505406+0000","last_fullsized":"2024-03-04T07:29:10.505406+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516437+0000","last_change":"2024-03-04T07:29:10.510878+0000","last_active":"2024-03-04T07:29:10.516437+0000","last_peered":"2024-03-04T07:29:10.516437+0000","last_clean":"2024-03-04T07:29:10.516437+0000","last_became_active":"2024-03-04T07:29:10.510509+0000","last_became_peered":"2024-03-04T07:29:10.510509+0000","last_unstale":"2024-03-04T07:29:10.516437+0000","last_undegraded":"2024-03-04T07:29:10.516437+0000","last_fullsized":"2024-03-04T07:29:10.516437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:51:59.390945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508983+0000","last_change":"2024-03-04T07:29:10.508983+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508983+0000","last_undegraded":"2024-03-04T07:29:10.508983+0000","last_fullsized":"2024-03-04T07:29:10.508983+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519100+0000","last_change":"2024-03-04T07:29:10.517386+0000","last_active":"2024-03-04T07:29:10.519100+0000","last_peered":"2024-03-04T07:29:10.519100+0000","last_clean":"2024-03-04T07:29:10.519100+0000","last_became_active":"2024-03-04T07:29:10.517120+0000","last_became_peered":"2024-03-04T07:29:10.517120+0000","last_unstale":"2024-03-04T07:29:10.519100+0000","last_undegraded":"2024-03-04T07:29:10.519100+0000","last_fullsized":"2024-03-04T07:29:10.519100+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.18","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504666+0000","last_change":"2024-03-04T07:29:10.504666+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504666+0000","last_undegraded":"2024-03-04T07:29:10.504666+0000","last_fullsized":"2024-03-04T07:29:10.504666+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516345+0000","last_change":"2024-03-04T07:29:10.509713+0000","last_active":"2024-03-04T07:29:10.516345+0000","last_peered":"2024-03-04T07:29:10.516345+0000","last_clean":"2024-03-04T07:29:10.516345+0000","last_became_active":"2024-03-04T07:29:10.509409+0000","last_became_peered":"2024-03-04T07:29:10.509409+0000","last_unstale":"2024-03-04T07:29:10.516345+0000","last_undegraded":"2024-03-04T07:29:10.516345+0000","last_fullsized":"2024-03-04T07:29:10.516345+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508131+0000","last_change":"2024-03-04T07:29:10.508131+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508131+0000","last_undegraded":"2024-03-04T07:29:10.508131+0000","last_fullsized":"2024-03-04T07:29:10.508131+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516481+0000","last_change":"2024-03-04T07:29:10.512481+0000","last_active":"2024-03-04T07:29:10.516481+0000","last_peered":"2024-03-04T07:29:10.516481+0000","last_clean":"2024-03-04T07:29:10.516481+0000","last_became_active":"2024-03-04T07:29:10.512206+0000","last_became_peered":"2024-03-04T07:29:10.512206+0000","last_unstale":"2024-03-04T07:29:10.516481+0000","last_undegraded":"2024-03-04T07:29:10.516481+0000","last_fullsized":"2024-03-04T07:29:10.516481+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:19.657851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.16","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501436+0000","last_change":"2024-03-04T07:29:10.501436+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501436+0000","last_undegraded":"2024-03-04T07:29:10.501436+0000","last_fullsized":"2024-03-04T07:29:10.501436+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.17","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518954+0000","last_change":"2024-03-04T07:29:10.514467+0000","last_active":"2024-03-04T07:29:10.518954+0000","last_peered":"2024-03-04T07:29:10.518954+0000","last_clean":"2024-03-04T07:29:10.518954+0000","last_became_active":"2024-03-04T07:29:10.514160+0000","last_became_peered":"2024-03-04T07:29:10.514160+0000","last_unstale":"2024-03-04T07:29:10.518954+0000","last_undegraded":"2024-03-04T07:29:10.518954+0000","last_fullsized":"2024-03-04T07:29:10.518954+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.17","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504882+0000","last_change":"2024-03-04T07:29:10.504882+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504882+0000","last_undegraded":"2024-03-04T07:29:10.504882+0000","last_fullsized":"2024-03-04T07:29:10.504882+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.16","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518939+0000","last_change":"2024-03-04T07:29:10.517195+0000","last_active":"2024-03-04T07:29:10.518939+0000","last_peered":"2024-03-04T07:29:10.518939+0000","last_clean":"2024-03-04T07:29:10.518939+0000","last_became_active":"2024-03-04T07:29:10.516936+0000","last_became_peered":"2024-03-04T07:29:10.516936+0000","last_unstale":"2024-03-04T07:29:10.518939+0000","last_undegraded":"2024-03-04T07:29:10.518939+0000","last_fullsized":"2024-03-04T07:29:10.518939+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:00.174138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502894+0000","last_change":"2024-03-04T07:29:10.502894+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502894+0000","last_undegraded":"2024-03-04T07:29:10.502894+0000","last_fullsized":"2024-03-04T07:29:10.502894+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.15","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518958+0000","last_change":"2024-03-04T07:29:10.515515+0000","last_active":"2024-03-04T07:29:10.518958+0000","last_peered":"2024-03-04T07:29:10.518958+0000","last_clean":"2024-03-04T07:29:10.518958+0000","last_became_active":"2024-03-04T07:29:10.515229+0000","last_became_peered":"2024-03-04T07:29:10.515229+0000","last_unstale":"2024-03-04T07:29:10.518958+0000","last_undegraded":"2024-03-04T07:29:10.518958+0000","last_fullsized":"2024-03-04T07:29:10.518958+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.15","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505698+0000","last_change":"2024-03-04T07:29:10.505698+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505698+0000","last_undegraded":"2024-03-04T07:29:10.505698+0000","last_fullsized":"2024-03-04T07:29:10.505698+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.14","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516384+0000","last_change":"2024-03-04T07:29:10.509271+0000","last_active":"2024-03-04T07:29:10.516384+0000","last_peered":"2024-03-04T07:29:10.516384+0000","last_clean":"2024-03-04T07:29:10.516384+0000","last_became_active":"2024-03-04T07:29:10.508965+0000","last_became_peered":"2024-03-04T07:29:10.508965+0000","last_unstale":"2024-03-04T07:29:10.516384+0000","last_undegraded":"2024-03-04T07:29:10.516384+0000","last_fullsized":"2024-03-04T07:29:10.516384+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:18:49.238841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.12","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500095+0000","last_change":"2024-03-04T07:29:10.500095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500095+0000","last_undegraded":"2024-03-04T07:29:10.500095+0000","last_fullsized":"2024-03-04T07:29:10.500095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.13","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519067+0000","last_change":"2024-03-04T07:29:10.517336+0000","last_active":"2024-03-04T07:29:10.519067+0000","last_peered":"2024-03-04T07:29:10.519067+0000","last_clean":"2024-03-04T07:29:10.519067+0000","last_became_active":"2024-03-04T07:29:10.517088+0000","last_became_peered":"2024-03-04T07:29:10.517088+0000","last_unstale":"2024-03-04T07:29:10.519067+0000","last_undegraded":"2024-03-04T07:29:10.519067+0000","last_fullsized":"2024-03-04T07:29:10.519067+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:26:41.225141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504221+0000","last_change":"2024-03-04T07:29:10.504221+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504221+0000","last_undegraded":"2024-03-04T07:29:10.504221+0000","last_fullsized":"2024-03-04T07:29:10.504221+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.12","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518895+0000","last_change":"2024-03-04T07:29:10.515222+0000","last_active":"2024-03-04T07:29:10.518895+0000","last_peered":"2024-03-04T07:29:10.518895+0000","last_clean":"2024-03-04T07:29:10.518895+0000","last_became_active":"2024-03-04T07:29:10.514882+0000","last_became_peered":"2024-03-04T07:29:10.514882+0000","last_unstale":"2024-03-04T07:29:10.518895+0000","last_undegraded":"2024-03-04T07:29:10.518895+0000","last_fullsized":"2024-03-04T07:29:10.518895+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:36:31.420388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.10","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500037+0000","last_change":"2024-03-04T07:29:10.500037+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500037+0000","last_undegraded":"2024-03-04T07:29:10.500037+0000","last_fullsized":"2024-03-04T07:29:10.500037+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.11","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518799+0000","last_change":"2024-03-04T07:29:10.515029+0000","last_active":"2024-03-04T07:29:10.518799+0000","last_peered":"2024-03-04T07:29:10.518799+0000","last_clean":"2024-03-04T07:29:10.518799+0000","last_became_active":"2024-03-04T07:29:10.514722+0000","last_became_peered":"2024-03-04T07:29:10.514722+0000","last_unstale":"2024-03-04T07:29:10.518799+0000","last_undegraded":"2024-03-04T07:29:10.518799+0000","last_fullsized":"2024-03-04T07:29:10.518799+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506446+0000","last_change":"2024-03-04T07:29:10.506446+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506446+0000","last_undegraded":"2024-03-04T07:29:10.506446+0000","last_fullsized":"2024-03-04T07:29:10.506446+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.10","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519102+0000","last_change":"2024-03-04T07:29:10.517145+0000","last_active":"2024-03-04T07:29:10.519102+0000","last_peered":"2024-03-04T07:29:10.519102+0000","last_clean":"2024-03-04T07:29:10.519102+0000","last_became_active":"2024-03-04T07:29:10.516791+0000","last_became_peered":"2024-03-04T07:29:10.516791+0000","last_unstale":"2024-03-04T07:29:10.519102+0000","last_undegraded":"2024-03-04T07:29:10.519102+0000","last_fullsized":"2024-03-04T07:29:10.519102+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504566+0000","last_change":"2024-03-04T07:29:10.504566+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504566+0000","last_undegraded":"2024-03-04T07:29:10.504566+0000","last_fullsized":"2024-03-04T07:29:10.504566+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519012+0000","last_change":"2024-03-04T07:29:10.517764+0000","last_active":"2024-03-04T07:29:10.519012+0000","last_peered":"2024-03-04T07:29:10.519012+0000","last_clean":"2024-03-04T07:29:10.519012+0000","last_became_active":"2024-03-04T07:29:10.516895+0000","last_became_peered":"2024-03-04T07:29:10.516895+0000","last_unstale":"2024-03-04T07:29:10.519012+0000","last_undegraded":"2024-03-04T07:29:10.519012+0000","last_fullsized":"2024-03-04T07:29:10.519012+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:04:10.056881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501820+0000","last_change":"2024-03-04T07:29:10.501820+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501820+0000","last_undegraded":"2024-03-04T07:29:10.501820+0000","last_fullsized":"2024-03-04T07:29:10.501820+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518517+0000","last_change":"2024-03-04T07:29:10.515628+0000","last_active":"2024-03-04T07:29:10.518517+0000","last_peered":"2024-03-04T07:29:10.518517+0000","last_clean":"2024-03-04T07:29:10.518517+0000","last_became_active":"2024-03-04T07:29:10.515205+0000","last_became_peered":"2024-03-04T07:29:10.515205+0000","last_unstale":"2024-03-04T07:29:10.518517+0000","last_undegraded":"2024-03-04T07:29:10.518517+0000","last_fullsized":"2024-03-04T07:29:10.518517+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506522+0000","last_change":"2024-03-04T07:29:10.506522+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506522+0000","last_undegraded":"2024-03-04T07:29:10.506522+0000","last_fullsized":"2024-03-04T07:29:10.506522+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518651+0000","last_change":"2024-03-04T07:29:10.512208+0000","last_active":"2024-03-04T07:29:10.518651+0000","last_peered":"2024-03-04T07:29:10.518651+0000","last_clean":"2024-03-04T07:29:10.518651+0000","last_became_active":"2024-03-04T07:29:10.511806+0000","last_became_peered":"2024-03-04T07:29:10.511806+0000","last_unstale":"2024-03-04T07:29:10.518651+0000","last_undegraded":"2024-03-04T07:29:10.518651+0000","last_fullsized":"2024-03-04T07:29:10.518651+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504634+0000","last_change":"2024-03-04T07:29:10.504634+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504634+0000","last_undegraded":"2024-03-04T07:29:10.504634+0000","last_fullsized":"2024-03-04T07:29:10.504634+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519131+0000","last_change":"2024-03-04T07:29:10.516595+0000","last_active":"2024-03-04T07:29:10.519131+0000","last_peered":"2024-03-04T07:29:10.519131+0000","last_clean":"2024-03-04T07:29:10.519131+0000","last_became_active":"2024-03-04T07:29:10.516327+0000","last_became_peered":"2024-03-04T07:29:10.516327+0000","last_unstale":"2024-03-04T07:29:10.519131+0000","last_undegraded":"2024-03-04T07:29:10.519131+0000","last_fullsized":"2024-03-04T07:29:10.519131+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499557+0000","last_change":"2024-03-04T07:29:10.499557+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499557+0000","last_undegraded":"2024-03-04T07:29:10.499557+0000","last_fullsized":"2024-03-04T07:29:10.499557+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519087+0000","last_change":"2024-03-04T07:29:10.517775+0000","last_active":"2024-03-04T07:29:10.519087+0000","last_peered":"2024-03-04T07:29:10.519087+0000","last_clean":"2024-03-04T07:29:10.519087+0000","last_became_active":"2024-03-04T07:29:10.517496+0000","last_became_peered":"2024-03-04T07:29:10.517496+0000","last_unstale":"2024-03-04T07:29:10.519087+0000","last_undegraded":"2024-03-04T07:29:10.519087+0000","last_fullsized":"2024-03-04T07:29:10.519087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:00.707197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510312+0000","last_change":"2024-03-04T07:29:10.510312+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510312+0000","last_undegraded":"2024-03-04T07:29:10.510312+0000","last_fullsized":"2024-03-04T07:29:10.510312+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519200+0000","last_change":"2024-03-04T07:29:10.514774+0000","last_active":"2024-03-04T07:29:10.519200+0000","last_peered":"2024-03-04T07:29:10.519200+0000","last_clean":"2024-03-04T07:29:10.519200+0000","last_became_active":"2024-03-04T07:29:10.514489+0000","last_became_peered":"2024-03-04T07:29:10.514489+0000","last_unstale":"2024-03-04T07:29:10.519200+0000","last_undegraded":"2024-03-04T07:29:10.519200+0000","last_fullsized":"2024-03-04T07:29:10.519200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:20.173974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506687+0000","last_change":"2024-03-04T07:29:10.506687+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506687+0000","last_undegraded":"2024-03-04T07:29:10.506687+0000","last_fullsized":"2024-03-04T07:29:10.506687+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518751+0000","last_change":"2024-03-04T07:29:10.513482+0000","last_active":"2024-03-04T07:29:10.518751+0000","last_peered":"2024-03-04T07:29:10.518751+0000","last_clean":"2024-03-04T07:29:10.518751+0000","last_became_active":"2024-03-04T07:29:10.513209+0000","last_became_peered":"2024-03-04T07:29:10.513209+0000","last_unstale":"2024-03-04T07:29:10.518751+0000","last_undegraded":"2024-03-04T07:29:10.518751+0000","last_fullsized":"2024-03-04T07:29:10.518751+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499078+0000","last_change":"2024-03-04T07:29:10.499078+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499078+0000","last_undegraded":"2024-03-04T07:29:10.499078+0000","last_fullsized":"2024-03-04T07:29:10.499078+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519230+0000","last_change":"2024-03-04T07:29:10.516454+0000","last_active":"2024-03-04T07:29:10.519230+0000","last_peered":"2024-03-04T07:29:10.519230+0000","last_clean":"2024-03-04T07:29:10.519230+0000","last_became_active":"2024-03-04T07:29:10.514845+0000","last_became_peered":"2024-03-04T07:29:10.514845+0000","last_unstale":"2024-03-04T07:29:10.519230+0000","last_undegraded":"2024-03-04T07:29:10.519230+0000","last_fullsized":"2024-03-04T07:29:10.519230+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516782+0000","last_change":"2024-03-04T07:29:10.514474+0000","last_active":"2024-03-04T07:29:10.516782+0000","last_peered":"2024-03-04T07:29:10.516782+0000","last_clean":"2024-03-04T07:29:10.516782+0000","last_became_active":"2024-03-04T07:29:10.514215+0000","last_became_peered":"2024-03-04T07:29:10.514215+0000","last_unstale":"2024-03-04T07:29:10.516782+0000","last_undegraded":"2024-03-04T07:29:10.516782+0000","last_fullsized":"2024-03-04T07:29:10.516782+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502443+0000","last_change":"2024-03-04T07:29:10.502443+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502443+0000","last_undegraded":"2024-03-04T07:29:10.502443+0000","last_fullsized":"2024-03-04T07:29:10.502443+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519002+0000","last_change":"2024-03-04T07:29:10.515670+0000","last_active":"2024-03-04T07:29:10.519002+0000","last_peered":"2024-03-04T07:29:10.519002+0000","last_clean":"2024-03-04T07:29:10.519002+0000","last_became_active":"2024-03-04T07:29:10.515327+0000","last_became_peered":"2024-03-04T07:29:10.515327+0000","last_unstale":"2024-03-04T07:29:10.519002+0000","last_undegraded":"2024-03-04T07:29:10.519002+0000","last_fullsized":"2024-03-04T07:29:10.519002+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:54:04.259042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506095+0000","last_change":"2024-03-04T07:29:10.506095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506095+0000","last_undegraded":"2024-03-04T07:29:10.506095+0000","last_fullsized":"2024-03-04T07:29:10.506095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.509863+0000","last_change":"2024-03-04T07:29:10.506371+0000","last_active":"2024-03-04T07:29:10.509863+0000","last_peered":"2024-03-04T07:29:10.509863+0000","last_clean":"2024-03-04T07:29:10.509863+0000","last_became_active":"2024-03-04T07:29:10.506084+0000","last_became_peered":"2024-03-04T07:29:10.506084+0000","last_unstale":"2024-03-04T07:29:10.509863+0000","last_undegraded":"2024-03-04T07:29:10.509863+0000","last_fullsized":"2024-03-04T07:29:10.509863+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504947+0000","last_change":"2024-03-04T07:29:10.504947+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504947+0000","last_undegraded":"2024-03-04T07:29:10.504947+0000","last_fullsized":"2024-03-04T07:29:10.504947+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.514479+0000","last_change":"2024-03-04T07:29:10.510685+0000","last_active":"2024-03-04T07:29:10.514479+0000","last_peered":"2024-03-04T07:29:10.514479+0000","last_clean":"2024-03-04T07:29:10.514479+0000","last_became_active":"2024-03-04T07:29:10.510433+0000","last_became_peered":"2024-03-04T07:29:10.510433+0000","last_unstale":"2024-03-04T07:29:10.514479+0000","last_undegraded":"2024-03-04T07:29:10.514479+0000","last_fullsized":"2024-03-04T07:29:10.514479+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:04:10.012697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503678+0000","last_change":"2024-03-04T07:29:10.503678+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503678+0000","last_undegraded":"2024-03-04T07:29:10.503678+0000","last_fullsized":"2024-03-04T07:29:10.503678+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518944+0000","last_change":"2024-03-04T07:29:10.513884+0000","last_active":"2024-03-04T07:29:10.518944+0000","last_peered":"2024-03-04T07:29:10.518944+0000","last_clean":"2024-03-04T07:29:10.518944+0000","last_became_active":"2024-03-04T07:29:10.513574+0000","last_became_peered":"2024-03-04T07:29:10.513574+0000","last_unstale":"2024-03-04T07:29:10.518944+0000","last_undegraded":"2024-03-04T07:29:10.518944+0000","last_fullsized":"2024-03-04T07:29:10.518944+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:39.944442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502740+0000","last_change":"2024-03-04T07:29:10.502740+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502740+0000","last_undegraded":"2024-03-04T07:29:10.502740+0000","last_fullsized":"2024-03-04T07:29:10.502740+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516550+0000","last_change":"2024-03-04T07:29:10.514430+0000","last_active":"2024-03-04T07:29:10.516550+0000","last_peered":"2024-03-04T07:29:10.516550+0000","last_clean":"2024-03-04T07:29:10.516550+0000","last_became_active":"2024-03-04T07:29:10.514146+0000","last_became_peered":"2024-03-04T07:29:10.514146+0000","last_unstale":"2024-03-04T07:29:10.516550+0000","last_undegraded":"2024-03-04T07:29:10.516550+0000","last_fullsized":"2024-03-04T07:29:10.516550+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:29:39.502383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507659+0000","last_change":"2024-03-04T07:29:10.507659+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507659+0000","last_undegraded":"2024-03-04T07:29:10.507659+0000","last_fullsized":"2024-03-04T07:29:10.507659+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.521576+0000","last_change":"2024-03-04T07:29:10.519578+0000","last_active":"2024-03-04T07:29:10.521576+0000","last_peered":"2024-03-04T07:29:10.521576+0000","last_clean":"2024-03-04T07:29:10.521576+0000","last_became_active":"2024-03-04T07:29:10.519248+0000","last_became_peered":"2024-03-04T07:29:10.519248+0000","last_unstale":"2024-03-04T07:29:10.521576+0000","last_undegraded":"2024-03-04T07:29:10.521576+0000","last_fullsized":"2024-03-04T07:29:10.521576+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:11.247866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506035+0000","last_change":"2024-03-04T07:29:10.506035+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506035+0000","last_undegraded":"2024-03-04T07:29:10.506035+0000","last_fullsized":"2024-03-04T07:29:10.506035+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516800+0000","last_change":"2024-03-04T07:29:10.513166+0000","last_active":"2024-03-04T07:29:10.516800+0000","last_peered":"2024-03-04T07:29:10.516800+0000","last_clean":"2024-03-04T07:29:10.516800+0000","last_became_active":"2024-03-04T07:29:10.512608+0000","last_became_peered":"2024-03-04T07:29:10.512608+0000","last_unstale":"2024-03-04T07:29:10.516800+0000","last_undegraded":"2024-03-04T07:29:10.516800+0000","last_fullsized":"2024-03-04T07:29:10.516800+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":230,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.507585+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:29:10.507585+0000","last_peered":"2024-03-04T07:29:10.507585+0000","last_clean":"2024-03-04T07:29:10.507585+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:29:10.507585+0000","last_undegraded":"2024-03-04T07:29:10.507585+0000","last_fullsized":"2024-03-04T07:29:10.507585+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":8,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574883,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32376,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339464,"statfs":{"total":96636764160,"available":96603611136,"internally_reserved":0,"allocated":1216512,"data_stored":844018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31812,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340028,"statfs":{"total":96636764160,"available":96604188672,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574883,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31808,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340032,"statfs":{"total":96636764160,"available":96604192768,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31788,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340052,"statfs":{"total":96636764160,"available":96604213248,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574882,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31796,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340044,"statfs":{"total":96636764160,"available":96604205056,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574882,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32392,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339448,"statfs":{"total":96636764160,"available":96603594752,"internally_reserved":0,"allocated":1228800,"data_stored":850285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:29:15.306 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.3f is not active+clean 2024-03-04T07:29:15.306 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.3f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.503695+0000', 'last_change': '2024-03-04T07:29:10.503695+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.503695+0000', 'last_undegraded': '2024-03-04T07:29:10.503695+0000', 'last_fullsized': '2024-03-04T07:29:10.503695+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 3], 'acting': [2, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.3e is not active+clean 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.3e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.509497+0000', 'last_change': '2024-03-04T07:29:10.509497+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.509497+0000', 'last_undegraded': '2024-03-04T07:29:10.509497+0000', 'last_fullsized': '2024-03-04T07:29:10.509497+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 1], 'acting': [3, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.3d is not active+clean 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.3d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506870+0000', 'last_change': '2024-03-04T07:29:10.506870+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506870+0000', 'last_undegraded': '2024-03-04T07:29:10.506870+0000', 'last_fullsized': '2024-03-04T07:29:10.506870+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4], 'acting': [7, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.3c is not active+clean 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.3c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502699+0000', 'last_change': '2024-03-04T07:29:10.502699+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502699+0000', 'last_undegraded': '2024-03-04T07:29:10.502699+0000', 'last_fullsized': '2024-03-04T07:29:10.502699+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 2], 'acting': [5, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.3b is not active+clean 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.3b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.503688+0000', 'last_change': '2024-03-04T07:29:10.503688+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.503688+0000', 'last_undegraded': '2024-03-04T07:29:10.503688+0000', 'last_fullsized': '2024-03-04T07:29:10.503688+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 5], 'acting': [2, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.3a is not active+clean 2024-03-04T07:29:15.307 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.3a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.510758+0000', 'last_change': '2024-03-04T07:29:10.510758+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.510758+0000', 'last_undegraded': '2024-03-04T07:29:10.510758+0000', 'last_fullsized': '2024-03-04T07:29:10.510758+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 2], 'acting': [0, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.39 is not active+clean 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.39', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.501042+0000', 'last_change': '2024-03-04T07:29:10.501042+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.501042+0000', 'last_undegraded': '2024-03-04T07:29:10.501042+0000', 'last_fullsized': '2024-03-04T07:29:10.501042+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 5], 'acting': [4, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.38 is not active+clean 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.38', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506502+0000', 'last_change': '2024-03-04T07:29:10.506502+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506502+0000', 'last_undegraded': '2024-03-04T07:29:10.506502+0000', 'last_fullsized': '2024-03-04T07:29:10.506502+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 4], 'acting': [3, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.37 is not active+clean 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.37', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.500610+0000', 'last_change': '2024-03-04T07:29:10.500610+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.500610+0000', 'last_undegraded': '2024-03-04T07:29:10.500610+0000', 'last_fullsized': '2024-03-04T07:29:10.500610+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 0], 'acting': [4, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.36 is not active+clean 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.36', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502657+0000', 'last_change': '2024-03-04T07:29:10.502657+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502657+0000', 'last_undegraded': '2024-03-04T07:29:10.502657+0000', 'last_fullsized': '2024-03-04T07:29:10.502657+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 6], 'acting': [1, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.35 is not active+clean 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.35', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504279+0000', 'last_change': '2024-03-04T07:29:10.504279+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504279+0000', 'last_undegraded': '2024-03-04T07:29:10.504279+0000', 'last_fullsized': '2024-03-04T07:29:10.504279+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 3], 'acting': [1, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.34 is not active+clean 2024-03-04T07:29:15.308 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.34', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.500622+0000', 'last_change': '2024-03-04T07:29:10.500622+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.500622+0000', 'last_undegraded': '2024-03-04T07:29:10.500622+0000', 'last_fullsized': '2024-03-04T07:29:10.500622+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 7], 'acting': [5, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.33 is not active+clean 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.33', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505723+0000', 'last_change': '2024-03-04T07:29:10.505723+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505723+0000', 'last_undegraded': '2024-03-04T07:29:10.505723+0000', 'last_fullsized': '2024-03-04T07:29:10.505723+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0], 'acting': [6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.32 is not active+clean 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.32', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.500946+0000', 'last_change': '2024-03-04T07:29:10.500946+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.500946+0000', 'last_undegraded': '2024-03-04T07:29:10.500946+0000', 'last_fullsized': '2024-03-04T07:29:10.500946+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 2], 'acting': [4, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.31 is not active+clean 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.31', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.500479+0000', 'last_change': '2024-03-04T07:29:10.500479+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.500479+0000', 'last_undegraded': '2024-03-04T07:29:10.500479+0000', 'last_fullsized': '2024-03-04T07:29:10.500479+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 1], 'acting': [4, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.30 is not active+clean 2024-03-04T07:29:15.309 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.30', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506809+0000', 'last_change': '2024-03-04T07:29:10.506809+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506809+0000', 'last_undegraded': '2024-03-04T07:29:10.506809+0000', 'last_fullsized': '2024-03-04T07:29:10.506809+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.310 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.2f is not active+clean 2024-03-04T07:29:15.310 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.2f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.507972+0000', 'last_change': '2024-03-04T07:29:10.507972+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.507972+0000', 'last_undegraded': '2024-03-04T07:29:10.507972+0000', 'last_fullsized': '2024-03-04T07:29:10.507972+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 6], 'acting': [3, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.310 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.2e is not active+clean 2024-03-04T07:29:15.310 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.2e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505474+0000', 'last_change': '2024-03-04T07:29:10.505474+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505474+0000', 'last_undegraded': '2024-03-04T07:29:10.505474+0000', 'last_fullsized': '2024-03-04T07:29:10.505474+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 0], 'acting': [7, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.310 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.2d is not active+clean 2024-03-04T07:29:15.310 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.2d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.503206+0000', 'last_change': '2024-03-04T07:29:10.503206+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.503206+0000', 'last_undegraded': '2024-03-04T07:29:10.503206+0000', 'last_fullsized': '2024-03-04T07:29:10.503206+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 3], 'acting': [2, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-04T07:29:15.310 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.2c is not active+clean 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.2c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.507000+0000', 'last_change': '2024-03-04T07:29:10.507000+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.507000+0000', 'last_undegraded': '2024-03-04T07:29:10.507000+0000', 'last_fullsized': '2024-03-04T07:29:10.507000+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 0], 'acting': [3, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.2b is not active+clean 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.2b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504903+0000', 'last_change': '2024-03-04T07:29:10.504903+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504903+0000', 'last_undegraded': '2024-03-04T07:29:10.504903+0000', 'last_fullsized': '2024-03-04T07:29:10.504903+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 5], 'acting': [7, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.2a is not active+clean 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.2a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505803+0000', 'last_change': '2024-03-04T07:29:10.505803+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505803+0000', 'last_undegraded': '2024-03-04T07:29:10.505803+0000', 'last_fullsized': '2024-03-04T07:29:10.505803+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 2], 'acting': [3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.29 is not active+clean 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.29', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506903+0000', 'last_change': '2024-03-04T07:29:10.506903+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506903+0000', 'last_undegraded': '2024-03-04T07:29:10.506903+0000', 'last_fullsized': '2024-03-04T07:29:10.506903+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 3], 'acting': [7, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.28 is not active+clean 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.28', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505190+0000', 'last_change': '2024-03-04T07:29:10.505190+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505190+0000', 'last_undegraded': '2024-03-04T07:29:10.505190+0000', 'last_fullsized': '2024-03-04T07:29:10.505190+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 0], 'acting': [6, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.27 is not active+clean 2024-03-04T07:29:15.311 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.27', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502380+0000', 'last_change': '2024-03-04T07:29:10.502380+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502380+0000', 'last_undegraded': '2024-03-04T07:29:10.502380+0000', 'last_fullsized': '2024-03-04T07:29:10.502380+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.26 is not active+clean 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.26', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.499795+0000', 'last_change': '2024-03-04T07:29:10.499795+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.499795+0000', 'last_undegraded': '2024-03-04T07:29:10.499795+0000', 'last_fullsized': '2024-03-04T07:29:10.499795+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 4], 'acting': [5, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.25 is not active+clean 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.25', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.507608+0000', 'last_change': '2024-03-04T07:29:10.507608+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.507608+0000', 'last_undegraded': '2024-03-04T07:29:10.507608+0000', 'last_fullsized': '2024-03-04T07:29:10.507608+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.24 is not active+clean 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.24', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502297+0000', 'last_change': '2024-03-04T07:29:10.502297+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502297+0000', 'last_undegraded': '2024-03-04T07:29:10.502297+0000', 'last_fullsized': '2024-03-04T07:29:10.502297+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 5], 'acting': [2, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.23 is not active+clean 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.23', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.507197+0000', 'last_change': '2024-03-04T07:29:10.507197+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.507197+0000', 'last_undegraded': '2024-03-04T07:29:10.507197+0000', 'last_fullsized': '2024-03-04T07:29:10.507197+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 5], 'acting': [3, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.22 is not active+clean 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.22', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505391+0000', 'last_change': '2024-03-04T07:29:10.505391+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505391+0000', 'last_undegraded': '2024-03-04T07:29:10.505391+0000', 'last_fullsized': '2024-03-04T07:29:10.505391+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4], 'acting': [7, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.21 is not active+clean 2024-03-04T07:29:15.312 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.21', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502303+0000', 'last_change': '2024-03-04T07:29:10.502303+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502303+0000', 'last_undegraded': '2024-03-04T07:29:10.502303+0000', 'last_fullsized': '2024-03-04T07:29:10.502303+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.20 is not active+clean 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.20', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506384+0000', 'last_change': '2024-03-04T07:29:10.506384+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506384+0000', 'last_undegraded': '2024-03-04T07:29:10.506384+0000', 'last_fullsized': '2024-03-04T07:29:10.506384+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 3], 'acting': [0, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.1e is not active+clean 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.1e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505456+0000', 'last_change': '2024-03-04T07:29:10.505456+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505456+0000', 'last_undegraded': '2024-03-04T07:29:10.505456+0000', 'last_fullsized': '2024-03-04T07:29:10.505456+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 7], 'acting': [0, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.1f is not active+clean 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.1f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.503363+0000', 'last_change': '2024-03-04T07:29:10.503363+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.503363+0000', 'last_undegraded': '2024-03-04T07:29:10.503363+0000', 'last_fullsized': '2024-03-04T07:29:10.503363+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 7], 'acting': [1, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.1c is not active+clean 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.1c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505684+0000', 'last_change': '2024-03-04T07:29:10.505684+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505684+0000', 'last_undegraded': '2024-03-04T07:29:10.505684+0000', 'last_fullsized': '2024-03-04T07:29:10.505684+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 1], 'acting': [0, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.1d is not active+clean 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.1d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.507219+0000', 'last_change': '2024-03-04T07:29:10.507219+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.507219+0000', 'last_undegraded': '2024-03-04T07:29:10.507219+0000', 'last_fullsized': '2024-03-04T07:29:10.507219+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 5], 'acting': [0, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.1a is not active+clean 2024-03-04T07:29:15.313 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.1a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505406+0000', 'last_change': '2024-03-04T07:29:10.505406+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505406+0000', 'last_undegraded': '2024-03-04T07:29:10.505406+0000', 'last_fullsized': '2024-03-04T07:29:10.505406+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 2], 'acting': [3, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.1b is not active+clean 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.1b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.508983+0000', 'last_change': '2024-03-04T07:29:10.508983+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.508983+0000', 'last_undegraded': '2024-03-04T07:29:10.508983+0000', 'last_fullsized': '2024-03-04T07:29:10.508983+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 0], 'acting': [3, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.18 is not active+clean 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.18', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504666+0000', 'last_change': '2024-03-04T07:29:10.504666+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504666+0000', 'last_undegraded': '2024-03-04T07:29:10.504666+0000', 'last_fullsized': '2024-03-04T07:29:10.504666+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 7], 'acting': [6, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.19 is not active+clean 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.19', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.508131+0000', 'last_change': '2024-03-04T07:29:10.508131+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.508131+0000', 'last_undegraded': '2024-03-04T07:29:10.508131+0000', 'last_fullsized': '2024-03-04T07:29:10.508131+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 6], 'acting': [3, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.16 is not active+clean 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.16', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.501436+0000', 'last_change': '2024-03-04T07:29:10.501436+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.501436+0000', 'last_undegraded': '2024-03-04T07:29:10.501436+0000', 'last_fullsized': '2024-03-04T07:29:10.501436+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 1], 'acting': [4, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.17 is not active+clean 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.17', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504882+0000', 'last_change': '2024-03-04T07:29:10.504882+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504882+0000', 'last_undegraded': '2024-03-04T07:29:10.504882+0000', 'last_fullsized': '2024-03-04T07:29:10.504882+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 1], 'acting': [7, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.314 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.14 is not active+clean 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.14', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502894+0000', 'last_change': '2024-03-04T07:29:10.502894+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502894+0000', 'last_undegraded': '2024-03-04T07:29:10.502894+0000', 'last_fullsized': '2024-03-04T07:29:10.502894+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 4], 'acting': [1, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.15 is not active+clean 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.15', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.505698+0000', 'last_change': '2024-03-04T07:29:10.505698+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.505698+0000', 'last_undegraded': '2024-03-04T07:29:10.505698+0000', 'last_fullsized': '2024-03-04T07:29:10.505698+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 1], 'acting': [7, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.12 is not active+clean 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.12', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.500095+0000', 'last_change': '2024-03-04T07:29:10.500095+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.500095+0000', 'last_undegraded': '2024-03-04T07:29:10.500095+0000', 'last_fullsized': '2024-03-04T07:29:10.500095+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 1], 'acting': [5, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.13 is not active+clean 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.13', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504221+0000', 'last_change': '2024-03-04T07:29:10.504221+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504221+0000', 'last_undegraded': '2024-03-04T07:29:10.504221+0000', 'last_fullsized': '2024-03-04T07:29:10.504221+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 6], 'acting': [7, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.10 is not active+clean 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.10', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.500037+0000', 'last_change': '2024-03-04T07:29:10.500037+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.500037+0000', 'last_undegraded': '2024-03-04T07:29:10.500037+0000', 'last_fullsized': '2024-03-04T07:29:10.500037+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 7], 'acting': [4, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.11 is not active+clean 2024-03-04T07:29:15.315 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.11', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506446+0000', 'last_change': '2024-03-04T07:29:10.506446+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506446+0000', 'last_undegraded': '2024-03-04T07:29:10.506446+0000', 'last_fullsized': '2024-03-04T07:29:10.506446+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 3], 'acting': [7, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.e is not active+clean 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.e', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504566+0000', 'last_change': '2024-03-04T07:29:10.504566+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504566+0000', 'last_undegraded': '2024-03-04T07:29:10.504566+0000', 'last_fullsized': '2024-03-04T07:29:10.504566+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 4], 'acting': [7, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.f is not active+clean 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.f', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.501820+0000', 'last_change': '2024-03-04T07:29:10.501820+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.501820+0000', 'last_undegraded': '2024-03-04T07:29:10.501820+0000', 'last_fullsized': '2024-03-04T07:29:10.501820+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 1], 'acting': [5, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.c is not active+clean 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.c', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506522+0000', 'last_change': '2024-03-04T07:29:10.506522+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506522+0000', 'last_undegraded': '2024-03-04T07:29:10.506522+0000', 'last_fullsized': '2024-03-04T07:29:10.506522+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 4], 'acting': [3, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.d is not active+clean 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.d', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504634+0000', 'last_change': '2024-03-04T07:29:10.504634+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504634+0000', 'last_undegraded': '2024-03-04T07:29:10.504634+0000', 'last_fullsized': '2024-03-04T07:29:10.504634+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 1], 'acting': [7, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.a is not active+clean 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.a', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.499557+0000', 'last_change': '2024-03-04T07:29:10.499557+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.499557+0000', 'last_undegraded': '2024-03-04T07:29:10.499557+0000', 'last_fullsized': '2024-03-04T07:29:10.499557+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 1], 'acting': [5, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.316 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.b is not active+clean 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.b', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.510312+0000', 'last_change': '2024-03-04T07:29:10.510312+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.510312+0000', 'last_undegraded': '2024-03-04T07:29:10.510312+0000', 'last_fullsized': '2024-03-04T07:29:10.510312+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 6], 'acting': [0, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.9 is not active+clean 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.9', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506687+0000', 'last_change': '2024-03-04T07:29:10.506687+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506687+0000', 'last_undegraded': '2024-03-04T07:29:10.506687+0000', 'last_fullsized': '2024-03-04T07:29:10.506687+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 7], 'acting': [0, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.6 is not active+clean 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.6', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.499078+0000', 'last_change': '2024-03-04T07:29:10.499078+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.499078+0000', 'last_undegraded': '2024-03-04T07:29:10.499078+0000', 'last_fullsized': '2024-03-04T07:29:10.499078+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 2], 'acting': [5, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.7 is not active+clean 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.7', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.501042+0000', 'last_change': '2024-03-04T07:29:10.501042+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.501042+0000', 'last_undegraded': '2024-03-04T07:29:10.501042+0000', 'last_fullsized': '2024-03-04T07:29:10.501042+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [5, 7], 'acting': [5, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 5, 'acting_primary': 5, 'purged_snaps': []} 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.4 is not active+clean 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.4', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502443+0000', 'last_change': '2024-03-04T07:29:10.502443+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502443+0000', 'last_undegraded': '2024-03-04T07:29:10.502443+0000', 'last_fullsized': '2024-03-04T07:29:10.502443+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 4], 'acting': [1, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.5 is not active+clean 2024-03-04T07:29:15.317 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.5', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506095+0000', 'last_change': '2024-03-04T07:29:10.506095+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506095+0000', 'last_undegraded': '2024-03-04T07:29:10.506095+0000', 'last_fullsized': '2024-03-04T07:29:10.506095+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 5], 'acting': [0, 5], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.2 is not active+clean 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.2', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.504947+0000', 'last_change': '2024-03-04T07:29:10.504947+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.504947+0000', 'last_undegraded': '2024-03-04T07:29:10.504947+0000', 'last_fullsized': '2024-03-04T07:29:10.504947+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 1], 'acting': [3, 1], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.3 is not active+clean 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.3', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.503678+0000', 'last_change': '2024-03-04T07:29:10.503678+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.503678+0000', 'last_undegraded': '2024-03-04T07:29:10.503678+0000', 'last_fullsized': '2024-03-04T07:29:10.503678+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [7, 3], 'acting': [7, 3], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 7, 'acting_primary': 7, 'purged_snaps': []} 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.0 is not active+clean 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.0', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.502740+0000', 'last_change': '2024-03-04T07:29:10.502740+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.502740+0000', 'last_undegraded': '2024-03-04T07:29:10.502740+0000', 'last_fullsized': '2024-03-04T07:29:10.502740+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [1, 6], 'acting': [1, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 1, 'acting_primary': 1, 'purged_snaps': []} 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.1 is not active+clean 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.1', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.507659+0000', 'last_change': '2024-03-04T07:29:10.507659+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.507659+0000', 'last_undegraded': '2024-03-04T07:29:10.507659+0000', 'last_fullsized': '2024-03-04T07:29:10.507659+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 6], 'acting': [3, 6], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:PG 9.8 is not active+clean 2024-03-04T07:29:15.318 INFO:tasks.cephfs.filesystem.ceph_manager:{'pgid': '9.8', 'version': "0'0", 'reported_seq': 1, 'reported_epoch': 53, 'state': 'creating+peering', 'last_fresh': '2024-03-04T07:29:10.506035+0000', 'last_change': '2024-03-04T07:29:10.506035+0000', 'last_active': '2024-03-04T07:29:10.486506+0000', 'last_peered': '2024-03-04T07:29:10.486506+0000', 'last_clean': '2024-03-04T07:29:10.486506+0000', 'last_became_active': '0.000000', 'last_became_peered': '0.000000', 'last_unstale': '2024-03-04T07:29:10.506035+0000', 'last_undegraded': '2024-03-04T07:29:10.506035+0000', 'last_fullsized': '2024-03-04T07:29:10.506035+0000', 'mapping_epoch': 53, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 53, 'last_epoch_clean': 0, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'last_clean_scrub_stamp': '2024-03-04T07:29:10.486506+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'no scrub is scheduled', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [3, 0], 'acting': [3, 0], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [], 'up_primary': 3, 'acting_primary': 3, 'purged_snaps': []} 2024-03-04T07:29:15.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json 2024-03-04T07:29:15.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.444+0000 7fa12a8c9640 1 Processor -- start 2024-03-04T07:29:15.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.444+0000 7fa12a8c9640 1 -- start start 2024-03-04T07:29:15.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa12a8c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa12415df00 0x7fa1240d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa12a8c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b95a0 0x7fa1240b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa12a8c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa1240cea30 con 0x7fa1240b9a30 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa12a8c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa1240d2af0 con 0x7fa12415df00 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa12a8c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa1240cf3c0 con 0x7fa1240b95a0 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa123fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa12415df00 0x7fa1240d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa123fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa12415df00 0x7fa1240d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39326/0 (socket says 172.21.15.64:39326) 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa123fff640 1 -- 172.21.15.64:0/1972946621 learned_addr learned my addr 172.21.15.64:0/1972946621 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa1237fe640 1 --2- 172.21.15.64:0/1972946621 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b95a0 0x7fa1240b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa123fff640 1 -- 172.21.15.64:0/1972946621 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b95a0 msgr2=0x7fa1240b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa123fff640 1 --2- 172.21.15.64:0/1972946621 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b95a0 0x7fa1240b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.445+0000 7fa123fff640 1 -- 172.21.15.64:0/1972946621 >> v1:172.21.15.66:6789/0 conn(0x7fa1240b9a30 legacy=0x7fa1240d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.446+0000 7fa123fff640 1 -- 172.21.15.64:0/1972946621 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1240cf0e0 con 0x7fa12415df00 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.446+0000 7fa123fff640 1 --2- 172.21.15.64:0/1972946621 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa12415df00 0x7fa1240d25b0 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fa114009870 tx=0x7fa11402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f33b6d43b81b2d4a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.446+0000 7fa122ffd640 1 -- 172.21.15.64:0/1972946621 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa114034070 con 0x7fa12415df00 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.446+0000 7fa122ffd640 1 -- 172.21.15.64:0/1972946621 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa114030840 con 0x7fa12415df00 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.446+0000 7fa122ffd640 1 -- 172.21.15.64:0/1972946621 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa114030c10 con 0x7fa12415df00 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1972946621 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa12415df00 msgr2=0x7fa1240d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1972946621 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa12415df00 0x7fa1240d25b0 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fa114009870 tx=0x7fa11402b980 comp rx=0 tx=0).stop 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1972946621 shutdown_connections 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1972946621 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b95a0 0x7fa1240b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1972946621 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa12415df00 0x7fa1240d25b0 unknown :-1 s=CLOSED pgs=676 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1972946621 >> 172.21.15.64:0/1972946621 conn(0x7fa124071960 msgr2=0x7fa124071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1972946621 shutdown_connections 2024-03-04T07:29:15.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.447+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1972946621 wait complete. 2024-03-04T07:29:15.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.448+0000 7fa12a8c9640 1 Processor -- start 2024-03-04T07:29:15.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.448+0000 7fa12a8c9640 1 -- start start 2024-03-04T07:29:15.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa12a8c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1240b95a0 0x7fa12415f620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa12a8c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa12415df00 0x7fa12415c070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa12a8c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b1c60 0x7fa1240b2110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa12a8c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1240cec70 con 0x7fa1240b95a0 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa12a8c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1240ce750 con 0x7fa12415df00 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa12a8c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa1240cedf0 con 0x7fa1240b1c60 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1240b95a0 0x7fa12415f620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1240b95a0 0x7fa12415f620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39328/0 (socket says 172.21.15.64:39328) 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 -- 172.21.15.64:0/1408389785 learned_addr learned my addr 172.21.15.64:0/1408389785 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 -- 172.21.15.64:0/1408389785 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b1c60 msgr2=0x7fa1240b2110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b1c60 0x7fa1240b2110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 -- 172.21.15.64:0/1408389785 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa12415df00 msgr2=0x7fa12415c070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa12415df00 0x7fa12415c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.449+0000 7fa123fff640 1 -- 172.21.15.64:0/1408389785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa12415e2e0 con 0x7fa1240b95a0 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.450+0000 7fa123fff640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1240b95a0 0x7fa12415f620 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7fa114000c00 tx=0x7fa1140046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.450+0000 7fa120ff9640 1 -- 172.21.15.64:0/1408389785 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa114034040 con 0x7fa1240b95a0 2024-03-04T07:29:15.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.450+0000 7fa120ff9640 1 -- 172.21.15.64:0/1408389785 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa114039d90 con 0x7fa1240b95a0 2024-03-04T07:29:15.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.450+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa12415e580 con 0x7fa1240b95a0 2024-03-04T07:29:15.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.450+0000 7fa120ff9640 1 -- 172.21.15.64:0/1408389785 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa114043610 con 0x7fa1240b95a0 2024-03-04T07:29:15.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.450+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa12415eb40 con 0x7fa1240b95a0 2024-03-04T07:29:15.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.451+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa1240b2810 con 0x7fa1240b95a0 2024-03-04T07:29:15.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.451+0000 7fa120ff9640 1 -- 172.21.15.64:0/1408389785 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa11404a060 con 0x7fa1240b95a0 2024-03-04T07:29:15.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.452+0000 7fa120ff9640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa11006b4d0 0x7fa11006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.452+0000 7fa1237fe640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa11006b4d0 0x7fa11006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.452+0000 7fa120ff9640 1 -- 172.21.15.64:0/1408389785 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa1140ac2b0 con 0x7fa1240b95a0 2024-03-04T07:29:15.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.452+0000 7fa1237fe640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa11006b4d0 0x7fa11006d990 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fa1240c76e0 tx=0x7fa118008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.457+0000 7fa120ff9640 1 -- 172.21.15.64:0/1408389785 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa1240b2810 con 0x7fa1240b95a0 2024-03-04T07:29:15.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.706+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "status", "format": "json"} v 0) v1 -- 0x7fa124067ff0 con 0x7fa1240b95a0 2024-03-04T07:29:15.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.707+0000 7fa120ff9640 1 -- 172.21.15.64:0/1408389785 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json"}]=0 v0) v1 ==== 72+0+1453 (secure 0 0 0) 0x7fa11403e070 con 0x7fa1240b95a0 2024-03-04T07:29:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:15.708 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":8,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":166,"monmap":{"epoch":1,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1709537200,"num_in_osds":8,"osd_in_since":1709537191,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"creating+peering","count":64},{"state_name":"active+clean","count":33}],"num_pgs":97,"num_pools":3,"num_objects":2,"data_bytes":590368,"bytes_used":261713920,"bytes_avail":772832399360,"bytes_total":773094113280,"inactive_pgs_ratio":0.65979379415512085},"fsmap":{"epoch":29,"id":4,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":4,"rank":0,"name":"c","status":"up:active","gid":6069}],"up:standby":3},"mgrmap":{"available":true,"num_standbys":1,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":4,"modified":"2024-03-04T07:28:25.158803+0000","services":{"mds":{"daemons":{"summary":"","b":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"c":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"d":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-03-04T07:29:15.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa11006b4d0 msgr2=0x7fa11006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa11006b4d0 0x7fa11006d990 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fa1240c76e0 tx=0x7fa118008040 comp rx=0 tx=0).stop 2024-03-04T07:29:15.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1240b95a0 msgr2=0x7fa12415f620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1240b95a0 0x7fa12415f620 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7fa114000c00 tx=0x7fa1140046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 shutdown_connections 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa11006b4d0 0x7fa11006d990 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1240b1c60 0x7fa1240b2110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa12415df00 0x7fa12415c070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 --2- 172.21.15.64:0/1408389785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1240b95a0 0x7fa12415f620 unknown :-1 s=CLOSED pgs=677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.709+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 >> 172.21.15.64:0/1408389785 conn(0x7fa124071960 msgr2=0x7fa1240d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.710+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 shutdown_connections 2024-03-04T07:29:15.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.710+0000 7fa12a8c9640 1 -- 172.21.15.64:0/1408389785 wait complete. 2024-03-04T07:29:15.721 INFO:tasks.cephfs.filesystem.ceph_manager:no progress seen, keeping timeout for now 2024-03-04T07:29:15.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:29:15.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.822+0000 7fdd657ac640 1 Processor -- start 2024-03-04T07:29:15.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.822+0000 7fdd657ac640 1 -- start start 2024-03-04T07:29:15.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.823+0000 7fdd657ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 0x7fdd600cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.823+0000 7fdd657ac640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdd600b2320 con 0x7fdd600d02e0 2024-03-04T07:29:15.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.823+0000 7fdd657ac640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdd600b0dd0 con 0x7fdd600b4260 2024-03-04T07:29:15.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.823+0000 7fdd657ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd600b0b80 con 0x7fdd600b0360 2024-03-04T07:29:15.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.823+0000 7fdd5effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fdd600d02e0 0x7fdd600b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35696/0 (socket says 172.21.15.64:35696) 2024-03-04T07:29:15.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.823+0000 7fdd5effd640 1 -- 172.21.15.64:0/2331151458 learned_addr learned my addr 172.21.15.64:0/2331151458 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:15.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5f7fe640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 0x7fdd600cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3137310854 0 0) 0x7fdd600b2320 con 0x7fdd600d02e0 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fdd4c003520 con 0x7fdd600d02e0 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3951929510 0 0) 0x7fdd4c003520 con 0x7fdd600d02e0 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 msgr2=0x7fdd600cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 0x7fdd600cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 >> v1:172.21.15.66:6789/0 conn(0x7fdd600b4260 legacy=0x7fdd600b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd600b10b0 con 0x7fdd600d02e0 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fdd58001670 con 0x7fdd600d02e0 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 >> v1:172.21.15.64:6789/0 conn(0x7fdd600d02e0 legacy=0x7fdd600b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c0039e0 0x7fdd4c003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd4c004390 0x7fdd4c004860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 0x7fdd4c0050c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd600b0b80 con 0x7fdd4c004390 2024-03-04T07:29:15.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd4c00b9f0 con 0x7fdd4c0039e0 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd4c00bbf0 con 0x7fdd600b0360 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5e7fc640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd4c004390 0x7fdd4c004860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5effd640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c0039e0 0x7fdd4c003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5f7fe640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 0x7fdd4c0050c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5e7fc640 1 -- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 msgr2=0x7fdd4c0050c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5e7fc640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd600b0360 0x7fdd4c0050c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5e7fc640 1 -- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c0039e0 msgr2=0x7fdd4c003dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5e7fc640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c0039e0 0x7fdd4c003dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.824+0000 7fdd5e7fc640 1 -- 172.21.15.64:0/2331151458 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd4c003520 con 0x7fdd4c004390 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd5e7fc640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd4c004390 0x7fdd4c004860 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7fdd48001860 tx=0x7fdd4c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e4911f7dd5140f5d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd48038070 con 0x7fdd4c004390 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd48033410 con 0x7fdd4c004390 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd5dffb640 1 -- 172.21.15.64:0/2331151458 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd48001430 con 0x7fdd4c004390 2024-03-04T07:29:15.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd657ac640 1 -- 172.21.15.64:0/2331151458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd4c004390 msgr2=0x7fdd4c004860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd657ac640 1 --2- 172.21.15.64:0/2331151458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd4c004390 0x7fdd4c004860 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7fdd48001860 tx=0x7fdd4c005830 comp rx=0 tx=0).stop 2024-03-04T07:29:15.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd5f7fe640 1 -- 172.21.15.64:0/2331151458 reap_dead start 2024-03-04T07:29:15.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd657ac640 1 -- 172.21.15.64:0/2331151458 shutdown_connections 2024-03-04T07:29:15.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd657ac640 1 -- 172.21.15.64:0/2331151458 >> 172.21.15.64:0/2331151458 conn(0x7fdd600b36d0 msgr2=0x7fdd600b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:15.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd657ac640 1 -- 172.21.15.64:0/2331151458 shutdown_connections 2024-03-04T07:29:15.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.826+0000 7fdd657ac640 1 -- 172.21.15.64:0/2331151458 wait complete. 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.828+0000 7fdd657ac640 1 Processor -- start 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.828+0000 7fdd657ac640 1 -- start start 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c004390 0x7fdd601524a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd60154f50 0x7fdd601f3420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd60152eb0 0x7fdd60153330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd600d06c0 con 0x7fdd60154f50 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd600aeec0 con 0x7fdd4c004390 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdd600d0840 con 0x7fdd60152eb0 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd60154f50 0x7fdd601f3420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd60154f50 0x7fdd601f3420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39354/0 (socket says 172.21.15.64:39354) 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 -- 172.21.15.64:0/2091390378 learned_addr learned my addr 172.21.15.64:0/2091390378 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5effd640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c004390 0x7fdd601524a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5f7fe640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd60152eb0 0x7fdd60153330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 -- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd60152eb0 msgr2=0x7fdd60153330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd60152eb0 0x7fdd60153330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 -- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c004390 msgr2=0x7fdd601524a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c004390 0x7fdd601524a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 -- 172.21.15.64:0/2091390378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd600cabe0 con 0x7fdd60154f50 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd5e7fc640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd60154f50 0x7fdd601f3420 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7fdd48008fd0 tx=0x7fdd48001230 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd577fe640 1 -- 172.21.15.64:0/2091390378 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd48041020 con 0x7fdd60154f50 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd577fe640 1 -- 172.21.15.64:0/2091390378 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdd48006e00 con 0x7fdd60154f50 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdd600cae80 con 0x7fdd60154f50 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd577fe640 1 -- 172.21.15.64:0/2091390378 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdd48038040 con 0x7fdd60154f50 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.830+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdd60155780 con 0x7fdd60154f50 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.831+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdd600cb3a0 con 0x7fdd60154f50 2024-03-04T07:29:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.832+0000 7fdd577fe640 1 -- 172.21.15.64:0/2091390378 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdd48044020 con 0x7fdd60154f50 2024-03-04T07:29:15.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.832+0000 7fdd577fe640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdd4006b500 0x7fdd4006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:15.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.832+0000 7fdd577fe640 1 -- 172.21.15.64:0/2091390378 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fdd480ab2b0 con 0x7fdd60154f50 2024-03-04T07:29:15.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.832+0000 7fdd5effd640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdd4006b500 0x7fdd4006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:15.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.833+0000 7fdd5effd640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdd4006b500 0x7fdd4006d9c0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fdd58001050 tx=0x7fdd58001cb0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:15.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:15.836+0000 7fdd577fe640 1 -- 172.21.15.64:0/2091390378 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdd600cb3a0 con 0x7fdd60154f50 2024-03-04T07:29:16.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.013+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fdd600c8a20 con 0x7fdd4006b500 2024-03-04T07:29:16.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.019+0000 7fdd577fe640 1 -- 172.21.15.64:0/2091390378 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+246638 (secure 0 0 0) 0x7fdd600c8a20 con 0x7fdd4006b500 2024-03-04T07:29:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:16.024 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:29:16.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdd4006b500 msgr2=0x7fdd4006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:16.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdd4006b500 0x7fdd4006d9c0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fdd58001050 tx=0x7fdd58001cb0 comp rx=0 tx=0).stop 2024-03-04T07:29:16.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd60154f50 msgr2=0x7fdd601f3420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd60154f50 0x7fdd601f3420 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7fdd48008fd0 tx=0x7fdd48001230 comp rx=0 tx=0).stop 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 shutdown_connections 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdd4006b500 0x7fdd4006d9c0 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdd60152eb0 0x7fdd60153330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdd60154f50 0x7fdd601f3420 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 --2- 172.21.15.64:0/2091390378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdd4c004390 0x7fdd601524a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 >> 172.21.15.64:0/2091390378 conn(0x7fdd600b36d0 msgr2=0x7fdd60151a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 shutdown_connections 2024-03-04T07:29:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:16.022+0000 7fdd657ac640 1 -- 172.21.15.64:0/2091390378 wait complete. 2024-03-04T07:29:16.037 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":125,"stamp":"2024-03-04T07:29:15.168119+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":255580,"kb_used_data":6084,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754719140,"statfs":{"total":773094113280,"available":772832399360,"internally_reserved":0,"allocated":6230016,"data_stored":3241271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.679194"},"pg_stats":[{"pgid":"9.3f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503695+0000","last_change":"2024-03-04T07:29:10.503695+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503695+0000","last_undegraded":"2024-03-04T07:29:10.503695+0000","last_fullsized":"2024-03-04T07:29:10.503695+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.509497+0000","last_change":"2024-03-04T07:29:10.509497+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.509497+0000","last_undegraded":"2024-03-04T07:29:10.509497+0000","last_fullsized":"2024-03-04T07:29:10.509497+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.3d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506870+0000","last_change":"2024-03-04T07:29:10.506870+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506870+0000","last_undegraded":"2024-03-04T07:29:10.506870+0000","last_fullsized":"2024-03-04T07:29:10.506870+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.3c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502699+0000","last_change":"2024-03-04T07:29:10.502699+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502699+0000","last_undegraded":"2024-03-04T07:29:10.502699+0000","last_fullsized":"2024-03-04T07:29:10.502699+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.3b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503688+0000","last_change":"2024-03-04T07:29:10.503688+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503688+0000","last_undegraded":"2024-03-04T07:29:10.503688+0000","last_fullsized":"2024-03-04T07:29:10.503688+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510758+0000","last_change":"2024-03-04T07:29:10.510758+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510758+0000","last_undegraded":"2024-03-04T07:29:10.510758+0000","last_fullsized":"2024-03-04T07:29:10.510758+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.39","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.38","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506502+0000","last_change":"2024-03-04T07:29:10.506502+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506502+0000","last_undegraded":"2024-03-04T07:29:10.506502+0000","last_fullsized":"2024-03-04T07:29:10.506502+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.37","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500610+0000","last_change":"2024-03-04T07:29:10.500610+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500610+0000","last_undegraded":"2024-03-04T07:29:10.500610+0000","last_fullsized":"2024-03-04T07:29:10.500610+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.36","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502657+0000","last_change":"2024-03-04T07:29:10.502657+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502657+0000","last_undegraded":"2024-03-04T07:29:10.502657+0000","last_fullsized":"2024-03-04T07:29:10.502657+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.35","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504279+0000","last_change":"2024-03-04T07:29:10.504279+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504279+0000","last_undegraded":"2024-03-04T07:29:10.504279+0000","last_fullsized":"2024-03-04T07:29:10.504279+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.34","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500622+0000","last_change":"2024-03-04T07:29:10.500622+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500622+0000","last_undegraded":"2024-03-04T07:29:10.500622+0000","last_fullsized":"2024-03-04T07:29:10.500622+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.33","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505723+0000","last_change":"2024-03-04T07:29:10.505723+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505723+0000","last_undegraded":"2024-03-04T07:29:10.505723+0000","last_fullsized":"2024-03-04T07:29:10.505723+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.32","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500946+0000","last_change":"2024-03-04T07:29:10.500946+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500946+0000","last_undegraded":"2024-03-04T07:29:10.500946+0000","last_fullsized":"2024-03-04T07:29:10.500946+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.31","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500479+0000","last_change":"2024-03-04T07:29:10.500479+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500479+0000","last_undegraded":"2024-03-04T07:29:10.500479+0000","last_fullsized":"2024-03-04T07:29:10.500479+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.30","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506809+0000","last_change":"2024-03-04T07:29:10.506809+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506809+0000","last_undegraded":"2024-03-04T07:29:10.506809+0000","last_fullsized":"2024-03-04T07:29:10.506809+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.2f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507972+0000","last_change":"2024-03-04T07:29:10.507972+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507972+0000","last_undegraded":"2024-03-04T07:29:10.507972+0000","last_fullsized":"2024-03-04T07:29:10.507972+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505474+0000","last_change":"2024-03-04T07:29:10.505474+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505474+0000","last_undegraded":"2024-03-04T07:29:10.505474+0000","last_fullsized":"2024-03-04T07:29:10.505474+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503206+0000","last_change":"2024-03-04T07:29:10.503206+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503206+0000","last_undegraded":"2024-03-04T07:29:10.503206+0000","last_fullsized":"2024-03-04T07:29:10.503206+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.2c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507000+0000","last_change":"2024-03-04T07:29:10.507000+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507000+0000","last_undegraded":"2024-03-04T07:29:10.507000+0000","last_fullsized":"2024-03-04T07:29:10.507000+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504903+0000","last_change":"2024-03-04T07:29:10.504903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504903+0000","last_undegraded":"2024-03-04T07:29:10.504903+0000","last_fullsized":"2024-03-04T07:29:10.504903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505803+0000","last_change":"2024-03-04T07:29:10.505803+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505803+0000","last_undegraded":"2024-03-04T07:29:10.505803+0000","last_fullsized":"2024-03-04T07:29:10.505803+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.29","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506903+0000","last_change":"2024-03-04T07:29:10.506903+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506903+0000","last_undegraded":"2024-03-04T07:29:10.506903+0000","last_fullsized":"2024-03-04T07:29:10.506903+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.28","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505190+0000","last_change":"2024-03-04T07:29:10.505190+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505190+0000","last_undegraded":"2024-03-04T07:29:10.505190+0000","last_fullsized":"2024-03-04T07:29:10.505190+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.27","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502380+0000","last_change":"2024-03-04T07:29:10.502380+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502380+0000","last_undegraded":"2024-03-04T07:29:10.502380+0000","last_fullsized":"2024-03-04T07:29:10.502380+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.26","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499795+0000","last_change":"2024-03-04T07:29:10.499795+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499795+0000","last_undegraded":"2024-03-04T07:29:10.499795+0000","last_fullsized":"2024-03-04T07:29:10.499795+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.25","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507608+0000","last_change":"2024-03-04T07:29:10.507608+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507608+0000","last_undegraded":"2024-03-04T07:29:10.507608+0000","last_fullsized":"2024-03-04T07:29:10.507608+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.24","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502297+0000","last_change":"2024-03-04T07:29:10.502297+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502297+0000","last_undegraded":"2024-03-04T07:29:10.502297+0000","last_fullsized":"2024-03-04T07:29:10.502297+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.23","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507197+0000","last_change":"2024-03-04T07:29:10.507197+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507197+0000","last_undegraded":"2024-03-04T07:29:10.507197+0000","last_fullsized":"2024-03-04T07:29:10.507197+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.22","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505391+0000","last_change":"2024-03-04T07:29:10.505391+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505391+0000","last_undegraded":"2024-03-04T07:29:10.505391+0000","last_fullsized":"2024-03-04T07:29:10.505391+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.21","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502303+0000","last_change":"2024-03-04T07:29:10.502303+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502303+0000","last_undegraded":"2024-03-04T07:29:10.502303+0000","last_fullsized":"2024-03-04T07:29:10.502303+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.20","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506384+0000","last_change":"2024-03-04T07:29:10.506384+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506384+0000","last_undegraded":"2024-03-04T07:29:10.506384+0000","last_fullsized":"2024-03-04T07:29:10.506384+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505456+0000","last_change":"2024-03-04T07:29:10.505456+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505456+0000","last_undegraded":"2024-03-04T07:29:10.505456+0000","last_fullsized":"2024-03-04T07:29:10.505456+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.513909+0000","last_change":"2024-03-04T07:29:10.512338+0000","last_active":"2024-03-04T07:29:10.513909+0000","last_peered":"2024-03-04T07:29:10.513909+0000","last_clean":"2024-03-04T07:29:10.513909+0000","last_became_active":"2024-03-04T07:29:10.512059+0000","last_became_peered":"2024-03-04T07:29:10.512059+0000","last_unstale":"2024-03-04T07:29:10.513909+0000","last_undegraded":"2024-03-04T07:29:10.513909+0000","last_fullsized":"2024-03-04T07:29:10.513909+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503363+0000","last_change":"2024-03-04T07:29:10.503363+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503363+0000","last_undegraded":"2024-03-04T07:29:10.503363+0000","last_fullsized":"2024-03-04T07:29:10.503363+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518723+0000","last_change":"2024-03-04T07:29:10.515404+0000","last_active":"2024-03-04T07:29:10.518723+0000","last_peered":"2024-03-04T07:29:10.518723+0000","last_clean":"2024-03-04T07:29:10.518723+0000","last_became_active":"2024-03-04T07:29:10.515135+0000","last_became_peered":"2024-03-04T07:29:10.515135+0000","last_unstale":"2024-03-04T07:29:10.518723+0000","last_undegraded":"2024-03-04T07:29:10.518723+0000","last_fullsized":"2024-03-04T07:29:10.518723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:12:54.069572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505684+0000","last_change":"2024-03-04T07:29:10.505684+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505684+0000","last_undegraded":"2024-03-04T07:29:10.505684+0000","last_fullsized":"2024-03-04T07:29:10.505684+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518470+0000","last_change":"2024-03-04T07:29:10.515994+0000","last_active":"2024-03-04T07:29:10.518470+0000","last_peered":"2024-03-04T07:29:10.518470+0000","last_clean":"2024-03-04T07:29:10.518470+0000","last_became_active":"2024-03-04T07:29:10.515751+0000","last_became_peered":"2024-03-04T07:29:10.515751+0000","last_unstale":"2024-03-04T07:29:10.518470+0000","last_undegraded":"2024-03-04T07:29:10.518470+0000","last_fullsized":"2024-03-04T07:29:10.518470+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507219+0000","last_change":"2024-03-04T07:29:10.507219+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507219+0000","last_undegraded":"2024-03-04T07:29:10.507219+0000","last_fullsized":"2024-03-04T07:29:10.507219+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516461+0000","last_change":"2024-03-04T07:29:10.509745+0000","last_active":"2024-03-04T07:29:10.516461+0000","last_peered":"2024-03-04T07:29:10.516461+0000","last_clean":"2024-03-04T07:29:10.516461+0000","last_became_active":"2024-03-04T07:29:10.509471+0000","last_became_peered":"2024-03-04T07:29:10.509471+0000","last_unstale":"2024-03-04T07:29:10.516461+0000","last_undegraded":"2024-03-04T07:29:10.516461+0000","last_fullsized":"2024-03-04T07:29:10.516461+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505406+0000","last_change":"2024-03-04T07:29:10.505406+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505406+0000","last_undegraded":"2024-03-04T07:29:10.505406+0000","last_fullsized":"2024-03-04T07:29:10.505406+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516437+0000","last_change":"2024-03-04T07:29:10.510878+0000","last_active":"2024-03-04T07:29:10.516437+0000","last_peered":"2024-03-04T07:29:10.516437+0000","last_clean":"2024-03-04T07:29:10.516437+0000","last_became_active":"2024-03-04T07:29:10.510509+0000","last_became_peered":"2024-03-04T07:29:10.510509+0000","last_unstale":"2024-03-04T07:29:10.516437+0000","last_undegraded":"2024-03-04T07:29:10.516437+0000","last_fullsized":"2024-03-04T07:29:10.516437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:51:59.390945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508983+0000","last_change":"2024-03-04T07:29:10.508983+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508983+0000","last_undegraded":"2024-03-04T07:29:10.508983+0000","last_fullsized":"2024-03-04T07:29:10.508983+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519100+0000","last_change":"2024-03-04T07:29:10.517386+0000","last_active":"2024-03-04T07:29:10.519100+0000","last_peered":"2024-03-04T07:29:10.519100+0000","last_clean":"2024-03-04T07:29:10.519100+0000","last_became_active":"2024-03-04T07:29:10.517120+0000","last_became_peered":"2024-03-04T07:29:10.517120+0000","last_unstale":"2024-03-04T07:29:10.519100+0000","last_undegraded":"2024-03-04T07:29:10.519100+0000","last_fullsized":"2024-03-04T07:29:10.519100+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.18","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504666+0000","last_change":"2024-03-04T07:29:10.504666+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504666+0000","last_undegraded":"2024-03-04T07:29:10.504666+0000","last_fullsized":"2024-03-04T07:29:10.504666+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516345+0000","last_change":"2024-03-04T07:29:10.509713+0000","last_active":"2024-03-04T07:29:10.516345+0000","last_peered":"2024-03-04T07:29:10.516345+0000","last_clean":"2024-03-04T07:29:10.516345+0000","last_became_active":"2024-03-04T07:29:10.509409+0000","last_became_peered":"2024-03-04T07:29:10.509409+0000","last_unstale":"2024-03-04T07:29:10.516345+0000","last_undegraded":"2024-03-04T07:29:10.516345+0000","last_fullsized":"2024-03-04T07:29:10.516345+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.508131+0000","last_change":"2024-03-04T07:29:10.508131+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.508131+0000","last_undegraded":"2024-03-04T07:29:10.508131+0000","last_fullsized":"2024-03-04T07:29:10.508131+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.18","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516481+0000","last_change":"2024-03-04T07:29:10.512481+0000","last_active":"2024-03-04T07:29:10.516481+0000","last_peered":"2024-03-04T07:29:10.516481+0000","last_clean":"2024-03-04T07:29:10.516481+0000","last_became_active":"2024-03-04T07:29:10.512206+0000","last_became_peered":"2024-03-04T07:29:10.512206+0000","last_unstale":"2024-03-04T07:29:10.516481+0000","last_undegraded":"2024-03-04T07:29:10.516481+0000","last_fullsized":"2024-03-04T07:29:10.516481+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:19.657851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.16","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501436+0000","last_change":"2024-03-04T07:29:10.501436+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501436+0000","last_undegraded":"2024-03-04T07:29:10.501436+0000","last_fullsized":"2024-03-04T07:29:10.501436+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.17","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518954+0000","last_change":"2024-03-04T07:29:10.514467+0000","last_active":"2024-03-04T07:29:10.518954+0000","last_peered":"2024-03-04T07:29:10.518954+0000","last_clean":"2024-03-04T07:29:10.518954+0000","last_became_active":"2024-03-04T07:29:10.514160+0000","last_became_peered":"2024-03-04T07:29:10.514160+0000","last_unstale":"2024-03-04T07:29:10.518954+0000","last_undegraded":"2024-03-04T07:29:10.518954+0000","last_fullsized":"2024-03-04T07:29:10.518954+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.17","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504882+0000","last_change":"2024-03-04T07:29:10.504882+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504882+0000","last_undegraded":"2024-03-04T07:29:10.504882+0000","last_fullsized":"2024-03-04T07:29:10.504882+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.16","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518939+0000","last_change":"2024-03-04T07:29:10.517195+0000","last_active":"2024-03-04T07:29:10.518939+0000","last_peered":"2024-03-04T07:29:10.518939+0000","last_clean":"2024-03-04T07:29:10.518939+0000","last_became_active":"2024-03-04T07:29:10.516936+0000","last_became_peered":"2024-03-04T07:29:10.516936+0000","last_unstale":"2024-03-04T07:29:10.518939+0000","last_undegraded":"2024-03-04T07:29:10.518939+0000","last_fullsized":"2024-03-04T07:29:10.518939+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:00.174138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502894+0000","last_change":"2024-03-04T07:29:10.502894+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502894+0000","last_undegraded":"2024-03-04T07:29:10.502894+0000","last_fullsized":"2024-03-04T07:29:10.502894+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.15","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518958+0000","last_change":"2024-03-04T07:29:10.515515+0000","last_active":"2024-03-04T07:29:10.518958+0000","last_peered":"2024-03-04T07:29:10.518958+0000","last_clean":"2024-03-04T07:29:10.518958+0000","last_became_active":"2024-03-04T07:29:10.515229+0000","last_became_peered":"2024-03-04T07:29:10.515229+0000","last_unstale":"2024-03-04T07:29:10.518958+0000","last_undegraded":"2024-03-04T07:29:10.518958+0000","last_fullsized":"2024-03-04T07:29:10.518958+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.15","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.505698+0000","last_change":"2024-03-04T07:29:10.505698+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.505698+0000","last_undegraded":"2024-03-04T07:29:10.505698+0000","last_fullsized":"2024-03-04T07:29:10.505698+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.14","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516384+0000","last_change":"2024-03-04T07:29:10.509271+0000","last_active":"2024-03-04T07:29:10.516384+0000","last_peered":"2024-03-04T07:29:10.516384+0000","last_clean":"2024-03-04T07:29:10.516384+0000","last_became_active":"2024-03-04T07:29:10.508965+0000","last_became_peered":"2024-03-04T07:29:10.508965+0000","last_unstale":"2024-03-04T07:29:10.516384+0000","last_undegraded":"2024-03-04T07:29:10.516384+0000","last_fullsized":"2024-03-04T07:29:10.516384+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:18:49.238841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.12","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500095+0000","last_change":"2024-03-04T07:29:10.500095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500095+0000","last_undegraded":"2024-03-04T07:29:10.500095+0000","last_fullsized":"2024-03-04T07:29:10.500095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.13","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519067+0000","last_change":"2024-03-04T07:29:10.517336+0000","last_active":"2024-03-04T07:29:10.519067+0000","last_peered":"2024-03-04T07:29:10.519067+0000","last_clean":"2024-03-04T07:29:10.519067+0000","last_became_active":"2024-03-04T07:29:10.517088+0000","last_became_peered":"2024-03-04T07:29:10.517088+0000","last_unstale":"2024-03-04T07:29:10.519067+0000","last_undegraded":"2024-03-04T07:29:10.519067+0000","last_fullsized":"2024-03-04T07:29:10.519067+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:26:41.225141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504221+0000","last_change":"2024-03-04T07:29:10.504221+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504221+0000","last_undegraded":"2024-03-04T07:29:10.504221+0000","last_fullsized":"2024-03-04T07:29:10.504221+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.12","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518895+0000","last_change":"2024-03-04T07:29:10.515222+0000","last_active":"2024-03-04T07:29:10.518895+0000","last_peered":"2024-03-04T07:29:10.518895+0000","last_clean":"2024-03-04T07:29:10.518895+0000","last_became_active":"2024-03-04T07:29:10.514882+0000","last_became_peered":"2024-03-04T07:29:10.514882+0000","last_unstale":"2024-03-04T07:29:10.518895+0000","last_undegraded":"2024-03-04T07:29:10.518895+0000","last_fullsized":"2024-03-04T07:29:10.518895+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:36:31.420388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.10","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.500037+0000","last_change":"2024-03-04T07:29:10.500037+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.500037+0000","last_undegraded":"2024-03-04T07:29:10.500037+0000","last_fullsized":"2024-03-04T07:29:10.500037+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.11","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518799+0000","last_change":"2024-03-04T07:29:10.515029+0000","last_active":"2024-03-04T07:29:10.518799+0000","last_peered":"2024-03-04T07:29:10.518799+0000","last_clean":"2024-03-04T07:29:10.518799+0000","last_became_active":"2024-03-04T07:29:10.514722+0000","last_became_peered":"2024-03-04T07:29:10.514722+0000","last_unstale":"2024-03-04T07:29:10.518799+0000","last_undegraded":"2024-03-04T07:29:10.518799+0000","last_fullsized":"2024-03-04T07:29:10.518799+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506446+0000","last_change":"2024-03-04T07:29:10.506446+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506446+0000","last_undegraded":"2024-03-04T07:29:10.506446+0000","last_fullsized":"2024-03-04T07:29:10.506446+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.10","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519102+0000","last_change":"2024-03-04T07:29:10.517145+0000","last_active":"2024-03-04T07:29:10.519102+0000","last_peered":"2024-03-04T07:29:10.519102+0000","last_clean":"2024-03-04T07:29:10.519102+0000","last_became_active":"2024-03-04T07:29:10.516791+0000","last_became_peered":"2024-03-04T07:29:10.516791+0000","last_unstale":"2024-03-04T07:29:10.519102+0000","last_undegraded":"2024-03-04T07:29:10.519102+0000","last_fullsized":"2024-03-04T07:29:10.519102+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504566+0000","last_change":"2024-03-04T07:29:10.504566+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504566+0000","last_undegraded":"2024-03-04T07:29:10.504566+0000","last_fullsized":"2024-03-04T07:29:10.504566+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519012+0000","last_change":"2024-03-04T07:29:10.517764+0000","last_active":"2024-03-04T07:29:10.519012+0000","last_peered":"2024-03-04T07:29:10.519012+0000","last_clean":"2024-03-04T07:29:10.519012+0000","last_became_active":"2024-03-04T07:29:10.516895+0000","last_became_peered":"2024-03-04T07:29:10.516895+0000","last_unstale":"2024-03-04T07:29:10.519012+0000","last_undegraded":"2024-03-04T07:29:10.519012+0000","last_fullsized":"2024-03-04T07:29:10.519012+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:04:10.056881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501820+0000","last_change":"2024-03-04T07:29:10.501820+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501820+0000","last_undegraded":"2024-03-04T07:29:10.501820+0000","last_fullsized":"2024-03-04T07:29:10.501820+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518517+0000","last_change":"2024-03-04T07:29:10.515628+0000","last_active":"2024-03-04T07:29:10.518517+0000","last_peered":"2024-03-04T07:29:10.518517+0000","last_clean":"2024-03-04T07:29:10.518517+0000","last_became_active":"2024-03-04T07:29:10.515205+0000","last_became_peered":"2024-03-04T07:29:10.515205+0000","last_unstale":"2024-03-04T07:29:10.518517+0000","last_undegraded":"2024-03-04T07:29:10.518517+0000","last_fullsized":"2024-03-04T07:29:10.518517+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506522+0000","last_change":"2024-03-04T07:29:10.506522+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506522+0000","last_undegraded":"2024-03-04T07:29:10.506522+0000","last_fullsized":"2024-03-04T07:29:10.506522+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518651+0000","last_change":"2024-03-04T07:29:10.512208+0000","last_active":"2024-03-04T07:29:10.518651+0000","last_peered":"2024-03-04T07:29:10.518651+0000","last_clean":"2024-03-04T07:29:10.518651+0000","last_became_active":"2024-03-04T07:29:10.511806+0000","last_became_peered":"2024-03-04T07:29:10.511806+0000","last_unstale":"2024-03-04T07:29:10.518651+0000","last_undegraded":"2024-03-04T07:29:10.518651+0000","last_fullsized":"2024-03-04T07:29:10.518651+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:54.138136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504634+0000","last_change":"2024-03-04T07:29:10.504634+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504634+0000","last_undegraded":"2024-03-04T07:29:10.504634+0000","last_fullsized":"2024-03-04T07:29:10.504634+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519131+0000","last_change":"2024-03-04T07:29:10.516595+0000","last_active":"2024-03-04T07:29:10.519131+0000","last_peered":"2024-03-04T07:29:10.519131+0000","last_clean":"2024-03-04T07:29:10.519131+0000","last_became_active":"2024-03-04T07:29:10.516327+0000","last_became_peered":"2024-03-04T07:29:10.516327+0000","last_unstale":"2024-03-04T07:29:10.519131+0000","last_undegraded":"2024-03-04T07:29:10.519131+0000","last_fullsized":"2024-03-04T07:29:10.519131+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499557+0000","last_change":"2024-03-04T07:29:10.499557+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499557+0000","last_undegraded":"2024-03-04T07:29:10.499557+0000","last_fullsized":"2024-03-04T07:29:10.499557+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519087+0000","last_change":"2024-03-04T07:29:10.517775+0000","last_active":"2024-03-04T07:29:10.519087+0000","last_peered":"2024-03-04T07:29:10.519087+0000","last_clean":"2024-03-04T07:29:10.519087+0000","last_became_active":"2024-03-04T07:29:10.517496+0000","last_became_peered":"2024-03-04T07:29:10.517496+0000","last_unstale":"2024-03-04T07:29:10.519087+0000","last_undegraded":"2024-03-04T07:29:10.519087+0000","last_fullsized":"2024-03-04T07:29:10.519087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:00.707197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.510312+0000","last_change":"2024-03-04T07:29:10.510312+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.510312+0000","last_undegraded":"2024-03-04T07:29:10.510312+0000","last_fullsized":"2024-03-04T07:29:10.510312+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519200+0000","last_change":"2024-03-04T07:29:10.514774+0000","last_active":"2024-03-04T07:29:10.519200+0000","last_peered":"2024-03-04T07:29:10.519200+0000","last_clean":"2024-03-04T07:29:10.519200+0000","last_became_active":"2024-03-04T07:29:10.514489+0000","last_became_peered":"2024-03-04T07:29:10.514489+0000","last_unstale":"2024-03-04T07:29:10.519200+0000","last_undegraded":"2024-03-04T07:29:10.519200+0000","last_fullsized":"2024-03-04T07:29:10.519200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:20.173974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506687+0000","last_change":"2024-03-04T07:29:10.506687+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506687+0000","last_undegraded":"2024-03-04T07:29:10.506687+0000","last_fullsized":"2024-03-04T07:29:10.506687+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518751+0000","last_change":"2024-03-04T07:29:10.513482+0000","last_active":"2024-03-04T07:29:10.518751+0000","last_peered":"2024-03-04T07:29:10.518751+0000","last_clean":"2024-03-04T07:29:10.518751+0000","last_became_active":"2024-03-04T07:29:10.513209+0000","last_became_peered":"2024-03-04T07:29:10.513209+0000","last_unstale":"2024-03-04T07:29:10.518751+0000","last_undegraded":"2024-03-04T07:29:10.518751+0000","last_fullsized":"2024-03-04T07:29:10.518751+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:07.903221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.499078+0000","last_change":"2024-03-04T07:29:10.499078+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.499078+0000","last_undegraded":"2024-03-04T07:29:10.499078+0000","last_fullsized":"2024-03-04T07:29:10.499078+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519230+0000","last_change":"2024-03-04T07:29:10.516454+0000","last_active":"2024-03-04T07:29:10.519230+0000","last_peered":"2024-03-04T07:29:10.519230+0000","last_clean":"2024-03-04T07:29:10.519230+0000","last_became_active":"2024-03-04T07:29:10.514845+0000","last_became_peered":"2024-03-04T07:29:10.514845+0000","last_unstale":"2024-03-04T07:29:10.519230+0000","last_undegraded":"2024-03-04T07:29:10.519230+0000","last_fullsized":"2024-03-04T07:29:10.519230+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.501042+0000","last_change":"2024-03-04T07:29:10.501042+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.501042+0000","last_undegraded":"2024-03-04T07:29:10.501042+0000","last_fullsized":"2024-03-04T07:29:10.501042+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516782+0000","last_change":"2024-03-04T07:29:10.514474+0000","last_active":"2024-03-04T07:29:10.516782+0000","last_peered":"2024-03-04T07:29:10.516782+0000","last_clean":"2024-03-04T07:29:10.516782+0000","last_became_active":"2024-03-04T07:29:10.514215+0000","last_became_peered":"2024-03-04T07:29:10.514215+0000","last_unstale":"2024-03-04T07:29:10.516782+0000","last_undegraded":"2024-03-04T07:29:10.516782+0000","last_fullsized":"2024-03-04T07:29:10.516782+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502443+0000","last_change":"2024-03-04T07:29:10.502443+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502443+0000","last_undegraded":"2024-03-04T07:29:10.502443+0000","last_fullsized":"2024-03-04T07:29:10.502443+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.519002+0000","last_change":"2024-03-04T07:29:10.515670+0000","last_active":"2024-03-04T07:29:10.519002+0000","last_peered":"2024-03-04T07:29:10.519002+0000","last_clean":"2024-03-04T07:29:10.519002+0000","last_became_active":"2024-03-04T07:29:10.515327+0000","last_became_peered":"2024-03-04T07:29:10.515327+0000","last_unstale":"2024-03-04T07:29:10.519002+0000","last_undegraded":"2024-03-04T07:29:10.519002+0000","last_fullsized":"2024-03-04T07:29:10.519002+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:54:04.259042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506095+0000","last_change":"2024-03-04T07:29:10.506095+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506095+0000","last_undegraded":"2024-03-04T07:29:10.506095+0000","last_fullsized":"2024-03-04T07:29:10.506095+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.509863+0000","last_change":"2024-03-04T07:29:10.506371+0000","last_active":"2024-03-04T07:29:10.509863+0000","last_peered":"2024-03-04T07:29:10.509863+0000","last_clean":"2024-03-04T07:29:10.509863+0000","last_became_active":"2024-03-04T07:29:10.506084+0000","last_became_peered":"2024-03-04T07:29:10.506084+0000","last_unstale":"2024-03-04T07:29:10.509863+0000","last_undegraded":"2024-03-04T07:29:10.509863+0000","last_fullsized":"2024-03-04T07:29:10.509863+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.504947+0000","last_change":"2024-03-04T07:29:10.504947+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.504947+0000","last_undegraded":"2024-03-04T07:29:10.504947+0000","last_fullsized":"2024-03-04T07:29:10.504947+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.514479+0000","last_change":"2024-03-04T07:29:10.510685+0000","last_active":"2024-03-04T07:29:10.514479+0000","last_peered":"2024-03-04T07:29:10.514479+0000","last_clean":"2024-03-04T07:29:10.514479+0000","last_became_active":"2024-03-04T07:29:10.510433+0000","last_became_peered":"2024-03-04T07:29:10.510433+0000","last_unstale":"2024-03-04T07:29:10.514479+0000","last_undegraded":"2024-03-04T07:29:10.514479+0000","last_fullsized":"2024-03-04T07:29:10.514479+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:04:10.012697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.503678+0000","last_change":"2024-03-04T07:29:10.503678+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.503678+0000","last_undegraded":"2024-03-04T07:29:10.503678+0000","last_fullsized":"2024-03-04T07:29:10.503678+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.518944+0000","last_change":"2024-03-04T07:29:10.513884+0000","last_active":"2024-03-04T07:29:10.518944+0000","last_peered":"2024-03-04T07:29:10.518944+0000","last_clean":"2024-03-04T07:29:10.518944+0000","last_became_active":"2024-03-04T07:29:10.513574+0000","last_became_peered":"2024-03-04T07:29:10.513574+0000","last_unstale":"2024-03-04T07:29:10.518944+0000","last_undegraded":"2024-03-04T07:29:10.518944+0000","last_fullsized":"2024-03-04T07:29:10.518944+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:39.944442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.502740+0000","last_change":"2024-03-04T07:29:10.502740+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.502740+0000","last_undegraded":"2024-03-04T07:29:10.502740+0000","last_fullsized":"2024-03-04T07:29:10.502740+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516550+0000","last_change":"2024-03-04T07:29:10.514430+0000","last_active":"2024-03-04T07:29:10.516550+0000","last_peered":"2024-03-04T07:29:10.516550+0000","last_clean":"2024-03-04T07:29:10.516550+0000","last_became_active":"2024-03-04T07:29:10.514146+0000","last_became_peered":"2024-03-04T07:29:10.514146+0000","last_unstale":"2024-03-04T07:29:10.516550+0000","last_undegraded":"2024-03-04T07:29:10.516550+0000","last_fullsized":"2024-03-04T07:29:10.516550+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:29:39.502383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.507659+0000","last_change":"2024-03-04T07:29:10.507659+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.507659+0000","last_undegraded":"2024-03-04T07:29:10.507659+0000","last_fullsized":"2024-03-04T07:29:10.507659+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.521576+0000","last_change":"2024-03-04T07:29:10.519578+0000","last_active":"2024-03-04T07:29:10.521576+0000","last_peered":"2024-03-04T07:29:10.521576+0000","last_clean":"2024-03-04T07:29:10.521576+0000","last_became_active":"2024-03-04T07:29:10.519248+0000","last_became_peered":"2024-03-04T07:29:10.519248+0000","last_unstale":"2024-03-04T07:29:10.521576+0000","last_undegraded":"2024-03-04T07:29:10.521576+0000","last_fullsized":"2024-03-04T07:29:10.521576+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:11.247866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":1,"reported_epoch":53,"state":"creating+peering","last_fresh":"2024-03-04T07:29:10.506035+0000","last_change":"2024-03-04T07:29:10.506035+0000","last_active":"2024-03-04T07:29:10.486506+0000","last_peered":"2024-03-04T07:29:10.486506+0000","last_clean":"2024-03-04T07:29:10.486506+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-03-04T07:29:10.506035+0000","last_undegraded":"2024-03-04T07:29:10.506035+0000","last_fullsized":"2024-03-04T07:29:10.506035+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":9,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.516800+0000","last_change":"2024-03-04T07:29:10.513166+0000","last_active":"2024-03-04T07:29:10.516800+0000","last_peered":"2024-03-04T07:29:10.516800+0000","last_clean":"2024-03-04T07:29:10.516800+0000","last_became_active":"2024-03-04T07:29:10.512608+0000","last_became_peered":"2024-03-04T07:29:10.512608+0000","last_unstale":"2024-03-04T07:29:10.516800+0000","last_undegraded":"2024-03-04T07:29:10.516800+0000","last_fullsized":"2024-03-04T07:29:10.516800+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:44.643905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":230,"reported_epoch":53,"state":"active+clean","last_fresh":"2024-03-04T07:29:10.507585+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:29:10.507585+0000","last_peered":"2024-03-04T07:29:10.507585+0000","last_clean":"2024-03-04T07:29:10.507585+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:29:10.507585+0000","last_undegraded":"2024-03-04T07:29:10.507585+0000","last_fullsized":"2024-03-04T07:29:10.507585+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":8,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574883,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32376,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339464,"statfs":{"total":96636764160,"available":96603611136,"internally_reserved":0,"allocated":1216512,"data_stored":844018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31812,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340028,"statfs":{"total":96636764160,"available":96604188672,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574883,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31804,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340036,"statfs":{"total":96636764160,"available":96604196864,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31808,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340032,"statfs":{"total":96636764160,"available":96604192768,"internally_reserved":0,"allocated":634880,"data_stored":259917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574882,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31788,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340052,"statfs":{"total":96636764160,"available":96604213248,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574882,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31796,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340044,"statfs":{"total":96636764160,"available":96604205056,"internally_reserved":0,"allocated":622592,"data_stored":253650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574882,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32392,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339448,"statfs":{"total":96636764160,"available":96603594752,"internally_reserved":0,"allocated":1228800,"data_stored":850285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:29:17.482 INFO:tasks.ceph.mds.c.smithi064.stderr:2024-03-04T07:29:17.480+0000 7f7dea22e640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:29:17.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:29:17.530 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:29:19.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:29:19.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.144+0000 7eff9d316640 1 Processor -- start 2024-03-04T07:29:19.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.144+0000 7eff9d316640 1 -- start start 2024-03-04T07:29:19.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.145+0000 7eff9d316640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.145+0000 7eff9d316640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7eff980b2b50 0x7eff980b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.145+0000 7eff9d316640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7eff980ce610 con 0x7eff98071540 2024-03-04T07:29:19.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.145+0000 7eff9d316640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7eff980d2f70 con 0x7eff980b2b50 2024-03-04T07:29:19.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.145+0000 7eff9d316640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff980d4be0 con 0x7eff980d3400 2024-03-04T07:29:19.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.145+0000 7eff977fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7eff980b2b50 0x7eff980b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.146+0000 7eff977fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7eff980b2b50 0x7eff980b6e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39362/0 (socket says 172.21.15.64:39362) 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.146+0000 7eff977fe640 1 -- 172.21.15.64:0/898293266 learned_addr learned my addr 172.21.15.64:0/898293266 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.146+0000 7eff967fc640 1 --2- 172.21.15.64:0/898293266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.146+0000 7eff977fe640 1 -- 172.21.15.64:0/898293266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 msgr2=0x7eff980b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.146+0000 7eff977fe640 1 --2- 172.21.15.64:0/898293266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980b30a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.146+0000 7eff977fe640 1 -- 172.21.15.64:0/898293266 >> v1:172.21.15.66:6789/0 conn(0x7eff98071540 legacy=0x7eff98071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.146+0000 7eff977fe640 1 -- 172.21.15.64:0/898293266 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff980d2920 con 0x7eff980b2b50 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff977fe640 1 --2- 172.21.15.64:0/898293266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980b6e20 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7eff90009e50 tx=0x7eff9002bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=27218a63798396b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff95ffb640 1 -- 172.21.15.64:0/898293266 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff9002cb00 con 0x7eff980b2b50 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff95ffb640 1 -- 172.21.15.64:0/898293266 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff9002cca0 con 0x7eff980b2b50 2024-03-04T07:29:19.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff95ffb640 1 -- 172.21.15.64:0/898293266 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff90034610 con 0x7eff980b2b50 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff9d316640 1 -- 172.21.15.64:0/898293266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 msgr2=0x7eff980b6e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff9d316640 1 --2- 172.21.15.64:0/898293266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980b6e20 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7eff90009e50 tx=0x7eff9002bf60 comp rx=0 tx=0).stop 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff9d316640 1 -- 172.21.15.64:0/898293266 shutdown_connections 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff9d316640 1 --2- 172.21.15.64:0/898293266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980b6e20 unknown :-1 s=CLOSED pgs=681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff9d316640 1 --2- 172.21.15.64:0/898293266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff9d316640 1 -- 172.21.15.64:0/898293266 >> 172.21.15.64:0/898293266 conn(0x7eff980d3ca0 msgr2=0x7eff980d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.147+0000 7eff9d316640 1 -- 172.21.15.64:0/898293266 shutdown_connections 2024-03-04T07:29:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.148+0000 7eff9d316640 1 -- 172.21.15.64:0/898293266 wait complete. 2024-03-04T07:29:19.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.149+0000 7eff9d316640 1 Processor -- start 2024-03-04T07:29:19.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.149+0000 7eff9d316640 1 -- start start 2024-03-04T07:29:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff9d316640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff98071540 0x7eff9815a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff9d316640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980cc1e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff9d316640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980aa6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff9d316640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff9815e370 con 0x7eff980b2b50 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff9d316640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff980ce8f0 con 0x7eff98071540 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff9d316640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff9815ab30 con 0x7eff980d3400 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980cc1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980cc1e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39366/0 (socket says 172.21.15.64:39366) 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 -- 172.21.15.64:0/2368855232 learned_addr learned my addr 172.21.15.64:0/2368855232 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff977fe640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980aa6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff96ffd640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff98071540 0x7eff9815a5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 -- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 msgr2=0x7eff980aa6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980aa6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 -- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff98071540 msgr2=0x7eff9815a5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff98071540 0x7eff9815a5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 -- 172.21.15.64:0/2368855232 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff980ccf80 con 0x7eff980b2b50 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff977fe640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980aa6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff967fc640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980cc1e0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7eff8000d600 tx=0x7eff8000dac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff8f7fe640 1 -- 172.21.15.64:0/2368855232 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff80004240 con 0x7eff980b2b50 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff8f7fe640 1 -- 172.21.15.64:0/2368855232 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff800043e0 con 0x7eff980b2b50 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff980ccca0 con 0x7eff980b2b50 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.151+0000 7eff8f7fe640 1 -- 172.21.15.64:0/2368855232 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff80003830 con 0x7eff980b2b50 2024-03-04T07:29:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.152+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff980c8810 con 0x7eff980b2b50 2024-03-04T07:29:19.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.153+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7eff98068700 con 0x7eff980b2b50 2024-03-04T07:29:19.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.153+0000 7eff8f7fe640 1 -- 172.21.15.64:0/2368855232 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7eff800039d0 con 0x7eff980b2b50 2024-03-04T07:29:19.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.153+0000 7eff8f7fe640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7806b4d0 0x7eff7806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.154+0000 7eff96ffd640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7806b4d0 0x7eff7806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.154+0000 7eff8f7fe640 1 -- 172.21.15.64:0/2368855232 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7eff8008a430 con 0x7eff980b2b50 2024-03-04T07:29:19.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.154+0000 7eff96ffd640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7806b4d0 0x7eff7806d990 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7eff880015a0 tx=0x7eff88009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.158+0000 7eff8f7fe640 1 -- 172.21.15.64:0/2368855232 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7eff8001c360 con 0x7eff980b2b50 2024-03-04T07:29:19.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.335+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7eff980b26b0 con 0x7eff7806b4d0 2024-03-04T07:29:19.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.341+0000 7eff8f7fe640 1 -- 172.21.15.64:0/2368855232 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+251955 (secure 0 0 0) 0x7eff980b26b0 con 0x7eff7806b4d0 2024-03-04T07:29:19.345 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:19.347 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.344+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7806b4d0 msgr2=0x7eff7806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.344+0000 7eff9d316640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7806b4d0 0x7eff7806d990 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7eff880015a0 tx=0x7eff88009040 comp rx=0 tx=0).stop 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.344+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 msgr2=0x7eff980cc1e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.344+0000 7eff9d316640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980cc1e0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7eff8000d600 tx=0x7eff8000dac0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 shutdown_connections 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7806b4d0 0x7eff7806d990 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff980d3400 0x7eff980aa6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff980b2b50 0x7eff980cc1e0 unknown :-1 s=CLOSED pgs=682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 --2- 172.21.15.64:0/2368855232 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff98071540 0x7eff9815a5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 >> 172.21.15.64:0/2368855232 conn(0x7eff980d3ca0 msgr2=0x7eff98160870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:19.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 shutdown_connections 2024-03-04T07:29:19.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.345+0000 7eff9d316640 1 -- 172.21.15.64:0/2368855232 wait complete. 2024-03-04T07:29:19.356 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":127,"stamp":"2024-03-04T07:29:19.168836+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":255816,"kb_used_data":6376,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754718904,"statfs":{"total":773094113280,"available":772832157696,"internally_reserved":0,"allocated":6529024,"data_stored":3369492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":18,"apply_latency_ms":18,"commit_latency_ns":18000000,"apply_latency_ns":18000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001434"},"pg_stats":[{"pgid":"9.3f","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.557142+0000","last_change":"2024-03-04T07:29:11.518659+0000","last_active":"2024-03-04T07:29:12.557142+0000","last_peered":"2024-03-04T07:29:12.557142+0000","last_clean":"2024-03-04T07:29:12.557142+0000","last_became_active":"2024-03-04T07:29:11.518347+0000","last_became_peered":"2024-03-04T07:29:11.518347+0000","last_unstale":"2024-03-04T07:29:12.557142+0000","last_undegraded":"2024-03-04T07:29:12.557142+0000","last_fullsized":"2024-03-04T07:29:12.557142+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:56.940776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3e","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510718+0000","last_change":"2024-03-04T07:29:11.515749+0000","last_active":"2024-03-04T07:29:12.510718+0000","last_peered":"2024-03-04T07:29:12.510718+0000","last_clean":"2024-03-04T07:29:12.510718+0000","last_became_active":"2024-03-04T07:29:11.515492+0000","last_became_peered":"2024-03-04T07:29:11.515492+0000","last_unstale":"2024-03-04T07:29:12.510718+0000","last_undegraded":"2024-03-04T07:29:12.510718+0000","last_fullsized":"2024-03-04T07:29:12.510718+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:00:39.478817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.3d","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511849+0000","last_change":"2024-03-04T07:29:11.518842+0000","last_active":"2024-03-04T07:29:12.511849+0000","last_peered":"2024-03-04T07:29:12.511849+0000","last_clean":"2024-03-04T07:29:12.511849+0000","last_became_active":"2024-03-04T07:29:11.518504+0000","last_became_peered":"2024-03-04T07:29:11.518504+0000","last_unstale":"2024-03-04T07:29:12.511849+0000","last_undegraded":"2024-03-04T07:29:12.511849+0000","last_fullsized":"2024-03-04T07:29:12.511849+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:03.263959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.3c","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592314+0000","last_change":"2024-03-04T07:29:11.511260+0000","last_active":"2024-03-04T07:29:12.592314+0000","last_peered":"2024-03-04T07:29:12.592314+0000","last_clean":"2024-03-04T07:29:12.592314+0000","last_became_active":"2024-03-04T07:29:11.510148+0000","last_became_peered":"2024-03-04T07:29:11.510148+0000","last_unstale":"2024-03-04T07:29:12.592314+0000","last_undegraded":"2024-03-04T07:29:12.592314+0000","last_fullsized":"2024-03-04T07:29:12.592314+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:10.326465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.3b","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.557078+0000","last_change":"2024-03-04T07:29:11.510604+0000","last_active":"2024-03-04T07:29:12.557078+0000","last_peered":"2024-03-04T07:29:12.557078+0000","last_clean":"2024-03-04T07:29:12.557078+0000","last_became_active":"2024-03-04T07:29:11.510348+0000","last_became_peered":"2024-03-04T07:29:11.510348+0000","last_unstale":"2024-03-04T07:29:12.557078+0000","last_undegraded":"2024-03-04T07:29:12.557078+0000","last_fullsized":"2024-03-04T07:29:12.557078+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:08:43.484909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3a","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513208+0000","last_change":"2024-03-04T07:29:11.512450+0000","last_active":"2024-03-04T07:29:12.513208+0000","last_peered":"2024-03-04T07:29:12.513208+0000","last_clean":"2024-03-04T07:29:12.513208+0000","last_became_active":"2024-03-04T07:29:11.511961+0000","last_became_peered":"2024-03-04T07:29:11.511961+0000","last_unstale":"2024-03-04T07:29:12.513208+0000","last_undegraded":"2024-03-04T07:29:12.513208+0000","last_fullsized":"2024-03-04T07:29:12.513208+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:46:38.755909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.39","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511875+0000","last_change":"2024-03-04T07:29:11.515147+0000","last_active":"2024-03-04T07:29:12.511875+0000","last_peered":"2024-03-04T07:29:12.511875+0000","last_clean":"2024-03-04T07:29:12.511875+0000","last_became_active":"2024-03-04T07:29:11.514457+0000","last_became_peered":"2024-03-04T07:29:11.514457+0000","last_unstale":"2024-03-04T07:29:12.511875+0000","last_undegraded":"2024-03-04T07:29:12.511875+0000","last_fullsized":"2024-03-04T07:29:12.511875+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:04.504102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.38","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.509940+0000","last_change":"2024-03-04T07:29:11.514317+0000","last_active":"2024-03-04T07:29:12.509940+0000","last_peered":"2024-03-04T07:29:12.509940+0000","last_clean":"2024-03-04T07:29:12.509940+0000","last_became_active":"2024-03-04T07:29:11.514032+0000","last_became_peered":"2024-03-04T07:29:11.514032+0000","last_unstale":"2024-03-04T07:29:12.509940+0000","last_undegraded":"2024-03-04T07:29:12.509940+0000","last_fullsized":"2024-03-04T07:29:12.509940+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:43:41.712584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.37","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512703+0000","last_change":"2024-03-04T07:29:11.520779+0000","last_active":"2024-03-04T07:29:12.512703+0000","last_peered":"2024-03-04T07:29:12.512703+0000","last_clean":"2024-03-04T07:29:12.512703+0000","last_became_active":"2024-03-04T07:29:11.518203+0000","last_became_peered":"2024-03-04T07:29:11.518203+0000","last_unstale":"2024-03-04T07:29:12.512703+0000","last_undegraded":"2024-03-04T07:29:12.512703+0000","last_fullsized":"2024-03-04T07:29:12.512703+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:26:42.232009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.36","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591860+0000","last_change":"2024-03-04T07:29:11.503497+0000","last_active":"2024-03-04T07:29:12.591860+0000","last_peered":"2024-03-04T07:29:12.591860+0000","last_clean":"2024-03-04T07:29:12.591860+0000","last_became_active":"2024-03-04T07:29:11.503242+0000","last_became_peered":"2024-03-04T07:29:11.503242+0000","last_unstale":"2024-03-04T07:29:12.591860+0000","last_undegraded":"2024-03-04T07:29:12.591860+0000","last_fullsized":"2024-03-04T07:29:12.591860+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:18.010859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.35","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592081+0000","last_change":"2024-03-04T07:29:11.507308+0000","last_active":"2024-03-04T07:29:12.592081+0000","last_peered":"2024-03-04T07:29:12.592081+0000","last_clean":"2024-03-04T07:29:12.592081+0000","last_became_active":"2024-03-04T07:29:11.507040+0000","last_became_peered":"2024-03-04T07:29:11.507040+0000","last_unstale":"2024-03-04T07:29:12.592081+0000","last_undegraded":"2024-03-04T07:29:12.592081+0000","last_fullsized":"2024-03-04T07:29:12.592081+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:49:08.910089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.34","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592351+0000","last_change":"2024-03-04T07:29:11.514795+0000","last_active":"2024-03-04T07:29:12.592351+0000","last_peered":"2024-03-04T07:29:12.592351+0000","last_clean":"2024-03-04T07:29:12.592351+0000","last_became_active":"2024-03-04T07:29:11.513409+0000","last_became_peered":"2024-03-04T07:29:11.513409+0000","last_unstale":"2024-03-04T07:29:12.592351+0000","last_undegraded":"2024-03-04T07:29:12.592351+0000","last_fullsized":"2024-03-04T07:29:12.592351+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:03.263959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.33","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.516119+0000","last_change":"2024-03-04T07:29:11.516577+0000","last_active":"2024-03-04T07:29:12.516119+0000","last_peered":"2024-03-04T07:29:12.516119+0000","last_clean":"2024-03-04T07:29:12.516119+0000","last_became_active":"2024-03-04T07:29:11.515942+0000","last_became_peered":"2024-03-04T07:29:11.515942+0000","last_unstale":"2024-03-04T07:29:12.516119+0000","last_undegraded":"2024-03-04T07:29:12.516119+0000","last_fullsized":"2024-03-04T07:29:12.516119+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:10:40.951309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.32","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512557+0000","last_change":"2024-03-04T07:29:11.512384+0000","last_active":"2024-03-04T07:29:12.512557+0000","last_peered":"2024-03-04T07:29:12.512557+0000","last_clean":"2024-03-04T07:29:12.512557+0000","last_became_active":"2024-03-04T07:29:11.511509+0000","last_became_peered":"2024-03-04T07:29:11.511509+0000","last_unstale":"2024-03-04T07:29:12.512557+0000","last_undegraded":"2024-03-04T07:29:12.512557+0000","last_fullsized":"2024-03-04T07:29:12.512557+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:01.181005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.31","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512109+0000","last_change":"2024-03-04T07:29:11.512114+0000","last_active":"2024-03-04T07:29:12.512109+0000","last_peered":"2024-03-04T07:29:12.512109+0000","last_clean":"2024-03-04T07:29:12.512109+0000","last_became_active":"2024-03-04T07:29:11.511804+0000","last_became_peered":"2024-03-04T07:29:11.511804+0000","last_unstale":"2024-03-04T07:29:12.512109+0000","last_undegraded":"2024-03-04T07:29:12.512109+0000","last_fullsized":"2024-03-04T07:29:12.512109+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:12:55.076440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.30","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513129+0000","last_change":"2024-03-04T07:29:11.516436+0000","last_active":"2024-03-04T07:29:12.513129+0000","last_peered":"2024-03-04T07:29:12.513129+0000","last_clean":"2024-03-04T07:29:12.513129+0000","last_became_active":"2024-03-04T07:29:11.516031+0000","last_became_peered":"2024-03-04T07:29:11.516031+0000","last_unstale":"2024-03-04T07:29:12.513129+0000","last_undegraded":"2024-03-04T07:29:12.513129+0000","last_fullsized":"2024-03-04T07:29:12.513129+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:48:27.443923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.2f","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510289+0000","last_change":"2024-03-04T07:29:11.513918+0000","last_active":"2024-03-04T07:29:12.510289+0000","last_peered":"2024-03-04T07:29:12.510289+0000","last_clean":"2024-03-04T07:29:12.510289+0000","last_became_active":"2024-03-04T07:29:11.510697+0000","last_became_peered":"2024-03-04T07:29:11.510697+0000","last_unstale":"2024-03-04T07:29:12.510289+0000","last_undegraded":"2024-03-04T07:29:12.510289+0000","last_fullsized":"2024-03-04T07:29:12.510289+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:37:03.872242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2e","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512817+0000","last_change":"2024-03-04T07:29:11.520409+0000","last_active":"2024-03-04T07:29:12.512817+0000","last_peered":"2024-03-04T07:29:12.512817+0000","last_clean":"2024-03-04T07:29:12.512817+0000","last_became_active":"2024-03-04T07:29:11.520085+0000","last_became_peered":"2024-03-04T07:29:11.520085+0000","last_unstale":"2024-03-04T07:29:12.512817+0000","last_undegraded":"2024-03-04T07:29:12.512817+0000","last_fullsized":"2024-03-04T07:29:12.512817+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:01:13.487945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2d","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.557011+0000","last_change":"2024-03-04T07:29:11.517899+0000","last_active":"2024-03-04T07:29:12.557011+0000","last_peered":"2024-03-04T07:29:12.557011+0000","last_clean":"2024-03-04T07:29:12.557011+0000","last_became_active":"2024-03-04T07:29:11.517562+0000","last_became_peered":"2024-03-04T07:29:11.517562+0000","last_unstale":"2024-03-04T07:29:12.557011+0000","last_undegraded":"2024-03-04T07:29:12.557011+0000","last_fullsized":"2024-03-04T07:29:12.557011+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:19.626228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.2c","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510301+0000","last_change":"2024-03-04T07:29:11.516916+0000","last_active":"2024-03-04T07:29:12.510301+0000","last_peered":"2024-03-04T07:29:12.510301+0000","last_clean":"2024-03-04T07:29:12.510301+0000","last_became_active":"2024-03-04T07:29:11.516406+0000","last_became_peered":"2024-03-04T07:29:11.516406+0000","last_unstale":"2024-03-04T07:29:12.510301+0000","last_undegraded":"2024-03-04T07:29:12.510301+0000","last_fullsized":"2024-03-04T07:29:12.510301+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:43:27.630014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.2b","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511533+0000","last_change":"2024-03-04T07:29:11.518655+0000","last_active":"2024-03-04T07:29:12.511533+0000","last_peered":"2024-03-04T07:29:12.511533+0000","last_clean":"2024-03-04T07:29:12.511533+0000","last_became_active":"2024-03-04T07:29:11.518314+0000","last_became_peered":"2024-03-04T07:29:11.518314+0000","last_unstale":"2024-03-04T07:29:12.511533+0000","last_undegraded":"2024-03-04T07:29:12.511533+0000","last_fullsized":"2024-03-04T07:29:12.511533+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:16.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2a","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510245+0000","last_change":"2024-03-04T07:29:11.517987+0000","last_active":"2024-03-04T07:29:12.510245+0000","last_peered":"2024-03-04T07:29:12.510245+0000","last_clean":"2024-03-04T07:29:12.510245+0000","last_became_active":"2024-03-04T07:29:11.517161+0000","last_became_peered":"2024-03-04T07:29:11.517161+0000","last_unstale":"2024-03-04T07:29:12.510245+0000","last_undegraded":"2024-03-04T07:29:12.510245+0000","last_fullsized":"2024-03-04T07:29:12.510245+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:59:43.040080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.29","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511819+0000","last_change":"2024-03-04T07:29:11.520580+0000","last_active":"2024-03-04T07:29:12.511819+0000","last_peered":"2024-03-04T07:29:12.511819+0000","last_clean":"2024-03-04T07:29:12.511819+0000","last_became_active":"2024-03-04T07:29:11.520277+0000","last_became_peered":"2024-03-04T07:29:11.520277+0000","last_unstale":"2024-03-04T07:29:12.511819+0000","last_undegraded":"2024-03-04T07:29:12.511819+0000","last_fullsized":"2024-03-04T07:29:12.511819+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:10.326465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.28","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.515820+0000","last_change":"2024-03-04T07:29:11.517320+0000","last_active":"2024-03-04T07:29:12.515820+0000","last_peered":"2024-03-04T07:29:12.515820+0000","last_clean":"2024-03-04T07:29:12.515820+0000","last_became_active":"2024-03-04T07:29:11.517050+0000","last_became_peered":"2024-03-04T07:29:11.517050+0000","last_unstale":"2024-03-04T07:29:12.515820+0000","last_undegraded":"2024-03-04T07:29:12.515820+0000","last_fullsized":"2024-03-04T07:29:12.515820+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:52:00.397813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.27","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.557201+0000","last_change":"2024-03-04T07:29:11.509834+0000","last_active":"2024-03-04T07:29:12.557201+0000","last_peered":"2024-03-04T07:29:12.557201+0000","last_clean":"2024-03-04T07:29:12.557201+0000","last_became_active":"2024-03-04T07:29:11.509578+0000","last_became_peered":"2024-03-04T07:29:11.509578+0000","last_unstale":"2024-03-04T07:29:12.557201+0000","last_undegraded":"2024-03-04T07:29:12.557201+0000","last_fullsized":"2024-03-04T07:29:12.557201+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:56:53.748944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.26","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592532+0000","last_change":"2024-03-04T07:29:11.514937+0000","last_active":"2024-03-04T07:29:12.592532+0000","last_peered":"2024-03-04T07:29:12.592532+0000","last_clean":"2024-03-04T07:29:12.592532+0000","last_became_active":"2024-03-04T07:29:11.514562+0000","last_became_peered":"2024-03-04T07:29:11.514562+0000","last_unstale":"2024-03-04T07:29:12.592532+0000","last_undegraded":"2024-03-04T07:29:12.592532+0000","last_fullsized":"2024-03-04T07:29:12.592532+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:48:27.443923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.25","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513672+0000","last_change":"2024-03-04T07:29:11.515045+0000","last_active":"2024-03-04T07:29:12.513672+0000","last_peered":"2024-03-04T07:29:12.513672+0000","last_clean":"2024-03-04T07:29:12.513672+0000","last_became_active":"2024-03-04T07:29:11.514792+0000","last_became_peered":"2024-03-04T07:29:11.514792+0000","last_unstale":"2024-03-04T07:29:12.513672+0000","last_undegraded":"2024-03-04T07:29:12.513672+0000","last_fullsized":"2024-03-04T07:29:12.513672+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:01:13.487945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.24","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.557062+0000","last_change":"2024-03-04T07:29:11.511337+0000","last_active":"2024-03-04T07:29:12.557062+0000","last_peered":"2024-03-04T07:29:12.557062+0000","last_clean":"2024-03-04T07:29:12.557062+0000","last_became_active":"2024-03-04T07:29:11.510999+0000","last_became_peered":"2024-03-04T07:29:11.510999+0000","last_unstale":"2024-03-04T07:29:12.557062+0000","last_undegraded":"2024-03-04T07:29:12.557062+0000","last_fullsized":"2024-03-04T07:29:12.557062+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:38:00.168037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.23","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510694+0000","last_change":"2024-03-04T07:29:11.520966+0000","last_active":"2024-03-04T07:29:12.510694+0000","last_peered":"2024-03-04T07:29:12.510694+0000","last_clean":"2024-03-04T07:29:12.510694+0000","last_became_active":"2024-03-04T07:29:11.520705+0000","last_became_peered":"2024-03-04T07:29:11.520705+0000","last_unstale":"2024-03-04T07:29:12.510694+0000","last_undegraded":"2024-03-04T07:29:12.510694+0000","last_fullsized":"2024-03-04T07:29:12.510694+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:37.385953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.22","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511644+0000","last_change":"2024-03-04T07:29:11.518066+0000","last_active":"2024-03-04T07:29:12.511644+0000","last_peered":"2024-03-04T07:29:12.511644+0000","last_clean":"2024-03-04T07:29:12.511644+0000","last_became_active":"2024-03-04T07:29:11.517793+0000","last_became_peered":"2024-03-04T07:29:11.517793+0000","last_unstale":"2024-03-04T07:29:12.511644+0000","last_undegraded":"2024-03-04T07:29:12.511644+0000","last_fullsized":"2024-03-04T07:29:12.511644+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:35:11.391348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.21","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.556870+0000","last_change":"2024-03-04T07:29:11.512779+0000","last_active":"2024-03-04T07:29:12.556870+0000","last_peered":"2024-03-04T07:29:12.556870+0000","last_clean":"2024-03-04T07:29:12.556870+0000","last_became_active":"2024-03-04T07:29:11.512491+0000","last_became_peered":"2024-03-04T07:29:11.512491+0000","last_unstale":"2024-03-04T07:29:12.556870+0000","last_undegraded":"2024-03-04T07:29:12.556870+0000","last_fullsized":"2024-03-04T07:29:12.556870+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:59:13.871387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.20","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513191+0000","last_change":"2024-03-04T07:29:11.517086+0000","last_active":"2024-03-04T07:29:12.513191+0000","last_peered":"2024-03-04T07:29:12.513191+0000","last_clean":"2024-03-04T07:29:12.513191+0000","last_became_active":"2024-03-04T07:29:11.516787+0000","last_became_peered":"2024-03-04T07:29:11.516787+0000","last_unstale":"2024-03-04T07:29:12.513191+0000","last_undegraded":"2024-03-04T07:29:12.513191+0000","last_fullsized":"2024-03-04T07:29:12.513191+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:03.263959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1e","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513484+0000","last_change":"2024-03-04T07:29:11.517240+0000","last_active":"2024-03-04T07:29:12.513484+0000","last_peered":"2024-03-04T07:29:12.513484+0000","last_clean":"2024-03-04T07:29:12.513484+0000","last_became_active":"2024-03-04T07:29:11.516974+0000","last_became_peered":"2024-03-04T07:29:11.516974+0000","last_unstale":"2024-03-04T07:29:12.513484+0000","last_undegraded":"2024-03-04T07:29:12.513484+0000","last_fullsized":"2024-03-04T07:29:12.513484+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:37.622116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1f","version":"54'4","reported_seq":18,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513489+0000","last_change":"2024-03-04T07:29:10.512338+0000","last_active":"2024-03-04T07:29:12.513489+0000","last_peered":"2024-03-04T07:29:12.513489+0000","last_clean":"2024-03-04T07:29:12.513489+0000","last_became_active":"2024-03-04T07:29:10.512059+0000","last_became_peered":"2024-03-04T07:29:10.512059+0000","last_unstale":"2024-03-04T07:29:12.513489+0000","last_undegraded":"2024-03-04T07:29:12.513489+0000","last_fullsized":"2024-03-04T07:29:12.513489+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:06:17.042964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1f","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591874+0000","last_change":"2024-03-04T07:29:11.518723+0000","last_active":"2024-03-04T07:29:12.591874+0000","last_peered":"2024-03-04T07:29:12.591874+0000","last_clean":"2024-03-04T07:29:12.591874+0000","last_became_active":"2024-03-04T07:29:11.518388+0000","last_became_peered":"2024-03-04T07:29:11.518388+0000","last_unstale":"2024-03-04T07:29:12.591874+0000","last_undegraded":"2024-03-04T07:29:12.591874+0000","last_fullsized":"2024-03-04T07:29:12.591874+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:03:55.145004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1e","version":"54'1","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510674+0000","last_change":"2024-03-04T07:29:10.515404+0000","last_active":"2024-03-04T07:29:12.510674+0000","last_peered":"2024-03-04T07:29:12.510674+0000","last_clean":"2024-03-04T07:29:12.510674+0000","last_became_active":"2024-03-04T07:29:10.515135+0000","last_became_peered":"2024-03-04T07:29:10.515135+0000","last_unstale":"2024-03-04T07:29:12.510674+0000","last_undegraded":"2024-03-04T07:29:12.510674+0000","last_fullsized":"2024-03-04T07:29:12.510674+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:34:33.188616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1c","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513348+0000","last_change":"2024-03-04T07:29:11.510499+0000","last_active":"2024-03-04T07:29:12.513348+0000","last_peered":"2024-03-04T07:29:12.513348+0000","last_clean":"2024-03-04T07:29:12.513348+0000","last_became_active":"2024-03-04T07:29:11.510235+0000","last_became_peered":"2024-03-04T07:29:11.510235+0000","last_unstale":"2024-03-04T07:29:12.513348+0000","last_undegraded":"2024-03-04T07:29:12.513348+0000","last_fullsized":"2024-03-04T07:29:12.513348+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:19:43.023161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1d","version":"54'2","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513778+0000","last_change":"2024-03-04T07:29:10.515994+0000","last_active":"2024-03-04T07:29:12.513778+0000","last_peered":"2024-03-04T07:29:12.513778+0000","last_clean":"2024-03-04T07:29:12.513778+0000","last_became_active":"2024-03-04T07:29:10.515751+0000","last_became_peered":"2024-03-04T07:29:10.515751+0000","last_unstale":"2024-03-04T07:29:12.513778+0000","last_undegraded":"2024-03-04T07:29:12.513778+0000","last_fullsized":"2024-03-04T07:29:12.513778+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:03.497234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1d","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513682+0000","last_change":"2024-03-04T07:29:11.512425+0000","last_active":"2024-03-04T07:29:12.513682+0000","last_peered":"2024-03-04T07:29:12.513682+0000","last_clean":"2024-03-04T07:29:12.513682+0000","last_became_active":"2024-03-04T07:29:11.512165+0000","last_became_peered":"2024-03-04T07:29:11.512165+0000","last_unstale":"2024-03-04T07:29:12.513682+0000","last_undegraded":"2024-03-04T07:29:12.513682+0000","last_fullsized":"2024-03-04T07:29:12.513682+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:43:22.542266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1c","version":"54'2","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592462+0000","last_change":"2024-03-04T07:29:10.509745+0000","last_active":"2024-03-04T07:29:12.592462+0000","last_peered":"2024-03-04T07:29:12.592462+0000","last_clean":"2024-03-04T07:29:12.592462+0000","last_became_active":"2024-03-04T07:29:10.509471+0000","last_became_peered":"2024-03-04T07:29:10.509471+0000","last_unstale":"2024-03-04T07:29:12.592462+0000","last_undegraded":"2024-03-04T07:29:12.592462+0000","last_fullsized":"2024-03-04T07:29:12.592462+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:03.497234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.1a","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510390+0000","last_change":"2024-03-04T07:29:11.515464+0000","last_active":"2024-03-04T07:29:12.510390+0000","last_peered":"2024-03-04T07:29:12.510390+0000","last_clean":"2024-03-04T07:29:12.510390+0000","last_became_active":"2024-03-04T07:29:11.514631+0000","last_became_peered":"2024-03-04T07:29:11.514631+0000","last_unstale":"2024-03-04T07:29:12.510390+0000","last_undegraded":"2024-03-04T07:29:12.510390+0000","last_fullsized":"2024-03-04T07:29:12.510390+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:34:35.617725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1b","version":"54'1","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512756+0000","last_change":"2024-03-04T07:29:10.510878+0000","last_active":"2024-03-04T07:29:12.512756+0000","last_peered":"2024-03-04T07:29:12.512756+0000","last_clean":"2024-03-04T07:29:12.512756+0000","last_became_active":"2024-03-04T07:29:10.510509+0000","last_became_peered":"2024-03-04T07:29:10.510509+0000","last_unstale":"2024-03-04T07:29:12.512756+0000","last_undegraded":"2024-03-04T07:29:12.512756+0000","last_fullsized":"2024-03-04T07:29:12.512756+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:03:03.354668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1b","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510554+0000","last_change":"2024-03-04T07:29:11.518742+0000","last_active":"2024-03-04T07:29:12.510554+0000","last_peered":"2024-03-04T07:29:12.510554+0000","last_clean":"2024-03-04T07:29:12.510554+0000","last_became_active":"2024-03-04T07:29:11.518481+0000","last_became_peered":"2024-03-04T07:29:11.518481+0000","last_unstale":"2024-03-04T07:29:12.510554+0000","last_undegraded":"2024-03-04T07:29:12.510554+0000","last_fullsized":"2024-03-04T07:29:12.510554+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:45:02.138479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1a","version":"54'1","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513175+0000","last_change":"2024-03-04T07:29:10.517386+0000","last_active":"2024-03-04T07:29:12.513175+0000","last_peered":"2024-03-04T07:29:12.513175+0000","last_clean":"2024-03-04T07:29:12.513175+0000","last_became_active":"2024-03-04T07:29:10.517120+0000","last_became_peered":"2024-03-04T07:29:10.517120+0000","last_unstale":"2024-03-04T07:29:12.513175+0000","last_undegraded":"2024-03-04T07:29:12.513175+0000","last_fullsized":"2024-03-04T07:29:12.513175+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:33:57.523018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.18","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.515907+0000","last_change":"2024-03-04T07:29:11.514095+0000","last_active":"2024-03-04T07:29:12.515907+0000","last_peered":"2024-03-04T07:29:12.515907+0000","last_clean":"2024-03-04T07:29:12.515907+0000","last_became_active":"2024-03-04T07:29:11.513791+0000","last_became_peered":"2024-03-04T07:29:11.513791+0000","last_unstale":"2024-03-04T07:29:12.515907+0000","last_undegraded":"2024-03-04T07:29:12.515907+0000","last_fullsized":"2024-03-04T07:29:12.515907+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:20.664719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592030+0000","last_change":"2024-03-04T07:29:10.509713+0000","last_active":"2024-03-04T07:29:12.592030+0000","last_peered":"2024-03-04T07:29:12.592030+0000","last_clean":"2024-03-04T07:29:12.592030+0000","last_became_active":"2024-03-04T07:29:10.509409+0000","last_became_peered":"2024-03-04T07:29:10.509409+0000","last_unstale":"2024-03-04T07:29:12.592030+0000","last_undegraded":"2024-03-04T07:29:12.592030+0000","last_fullsized":"2024-03-04T07:29:12.592030+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:33:57.523018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.19","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510108+0000","last_change":"2024-03-04T07:29:11.509524+0000","last_active":"2024-03-04T07:29:12.510108+0000","last_peered":"2024-03-04T07:29:12.510108+0000","last_clean":"2024-03-04T07:29:12.510108+0000","last_became_active":"2024-03-04T07:29:11.509273+0000","last_became_peered":"2024-03-04T07:29:11.509273+0000","last_unstale":"2024-03-04T07:29:12.510108+0000","last_undegraded":"2024-03-04T07:29:12.510108+0000","last_fullsized":"2024-03-04T07:29:12.510108+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:39:09.854646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.18","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512222+0000","last_change":"2024-03-04T07:29:10.512481+0000","last_active":"2024-03-04T07:29:12.512222+0000","last_peered":"2024-03-04T07:29:12.512222+0000","last_clean":"2024-03-04T07:29:12.512222+0000","last_became_active":"2024-03-04T07:29:10.512206+0000","last_became_peered":"2024-03-04T07:29:10.512206+0000","last_unstale":"2024-03-04T07:29:12.512222+0000","last_undegraded":"2024-03-04T07:29:12.512222+0000","last_fullsized":"2024-03-04T07:29:12.512222+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:59.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.16","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513007+0000","last_change":"2024-03-04T07:29:11.511362+0000","last_active":"2024-03-04T07:29:12.513007+0000","last_peered":"2024-03-04T07:29:12.513007+0000","last_clean":"2024-03-04T07:29:12.513007+0000","last_became_active":"2024-03-04T07:29:11.510967+0000","last_became_peered":"2024-03-04T07:29:11.510967+0000","last_unstale":"2024-03-04T07:29:12.513007+0000","last_undegraded":"2024-03-04T07:29:12.513007+0000","last_fullsized":"2024-03-04T07:29:12.513007+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:04:11.063749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.17","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513102+0000","last_change":"2024-03-04T07:29:10.514467+0000","last_active":"2024-03-04T07:29:12.513102+0000","last_peered":"2024-03-04T07:29:12.513102+0000","last_clean":"2024-03-04T07:29:12.513102+0000","last_became_active":"2024-03-04T07:29:10.514160+0000","last_became_peered":"2024-03-04T07:29:10.514160+0000","last_unstale":"2024-03-04T07:29:12.513102+0000","last_undegraded":"2024-03-04T07:29:12.513102+0000","last_fullsized":"2024-03-04T07:29:12.513102+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:06:17.042964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.17","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513165+0000","last_change":"2024-03-04T07:29:11.524731+0000","last_active":"2024-03-04T07:29:12.513165+0000","last_peered":"2024-03-04T07:29:12.513165+0000","last_clean":"2024-03-04T07:29:12.513165+0000","last_became_active":"2024-03-04T07:29:11.524062+0000","last_became_peered":"2024-03-04T07:29:11.524062+0000","last_unstale":"2024-03-04T07:29:12.513165+0000","last_undegraded":"2024-03-04T07:29:12.513165+0000","last_fullsized":"2024-03-04T07:29:12.513165+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:54:26.472758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.16","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510784+0000","last_change":"2024-03-04T07:29:10.517195+0000","last_active":"2024-03-04T07:29:12.510784+0000","last_peered":"2024-03-04T07:29:12.510784+0000","last_clean":"2024-03-04T07:29:12.510784+0000","last_became_active":"2024-03-04T07:29:10.516936+0000","last_became_peered":"2024-03-04T07:29:10.516936+0000","last_unstale":"2024-03-04T07:29:12.510784+0000","last_undegraded":"2024-03-04T07:29:12.510784+0000","last_fullsized":"2024-03-04T07:29:12.510784+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:35:10.384480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.14","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591677+0000","last_change":"2024-03-04T07:29:11.507475+0000","last_active":"2024-03-04T07:29:12.591677+0000","last_peered":"2024-03-04T07:29:12.591677+0000","last_clean":"2024-03-04T07:29:12.591677+0000","last_became_active":"2024-03-04T07:29:11.506876+0000","last_became_peered":"2024-03-04T07:29:11.506876+0000","last_unstale":"2024-03-04T07:29:12.591677+0000","last_undegraded":"2024-03-04T07:29:12.591677+0000","last_fullsized":"2024-03-04T07:29:12.591677+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:59.337127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.15","version":"54'3","reported_seq":17,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512469+0000","last_change":"2024-03-04T07:29:10.515515+0000","last_active":"2024-03-04T07:29:12.512469+0000","last_peered":"2024-03-04T07:29:12.512469+0000","last_clean":"2024-03-04T07:29:12.512469+0000","last_became_active":"2024-03-04T07:29:10.515229+0000","last_became_peered":"2024-03-04T07:29:10.515229+0000","last_unstale":"2024-03-04T07:29:12.512469+0000","last_undegraded":"2024-03-04T07:29:12.512469+0000","last_fullsized":"2024-03-04T07:29:12.512469+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:33:57.523018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.15","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512024+0000","last_change":"2024-03-04T07:29:11.519745+0000","last_active":"2024-03-04T07:29:12.512024+0000","last_peered":"2024-03-04T07:29:12.512024+0000","last_clean":"2024-03-04T07:29:12.512024+0000","last_became_active":"2024-03-04T07:29:11.519426+0000","last_became_peered":"2024-03-04T07:29:11.519426+0000","last_unstale":"2024-03-04T07:29:12.512024+0000","last_undegraded":"2024-03-04T07:29:12.512024+0000","last_fullsized":"2024-03-04T07:29:12.512024+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:48:27.443923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.14","version":"54'6","reported_seq":20,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513107+0000","last_change":"2024-03-04T07:29:10.509271+0000","last_active":"2024-03-04T07:29:12.513107+0000","last_peered":"2024-03-04T07:29:12.513107+0000","last_clean":"2024-03-04T07:29:12.513107+0000","last_became_active":"2024-03-04T07:29:10.508965+0000","last_became_peered":"2024-03-04T07:29:10.508965+0000","last_unstale":"2024-03-04T07:29:12.513107+0000","last_undegraded":"2024-03-04T07:29:12.513107+0000","last_fullsized":"2024-03-04T07:29:12.513107+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:22:17.003992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.12","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592223+0000","last_change":"2024-03-04T07:29:11.510822+0000","last_active":"2024-03-04T07:29:12.592223+0000","last_peered":"2024-03-04T07:29:12.592223+0000","last_clean":"2024-03-04T07:29:12.592223+0000","last_became_active":"2024-03-04T07:29:11.510526+0000","last_became_peered":"2024-03-04T07:29:11.510526+0000","last_unstale":"2024-03-04T07:29:12.592223+0000","last_undegraded":"2024-03-04T07:29:12.592223+0000","last_fullsized":"2024-03-04T07:29:12.592223+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:35:11.391348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.13","version":"54'2","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510538+0000","last_change":"2024-03-04T07:29:10.517336+0000","last_active":"2024-03-04T07:29:12.510538+0000","last_peered":"2024-03-04T07:29:12.510538+0000","last_clean":"2024-03-04T07:29:12.510538+0000","last_became_active":"2024-03-04T07:29:10.517088+0000","last_became_peered":"2024-03-04T07:29:10.517088+0000","last_unstale":"2024-03-04T07:29:12.510538+0000","last_undegraded":"2024-03-04T07:29:12.510538+0000","last_fullsized":"2024-03-04T07:29:12.510538+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:15.530601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512141+0000","last_change":"2024-03-04T07:29:11.515327+0000","last_active":"2024-03-04T07:29:12.512141+0000","last_peered":"2024-03-04T07:29:12.512141+0000","last_clean":"2024-03-04T07:29:12.512141+0000","last_became_active":"2024-03-04T07:29:11.515050+0000","last_became_peered":"2024-03-04T07:29:11.515050+0000","last_unstale":"2024-03-04T07:29:12.512141+0000","last_undegraded":"2024-03-04T07:29:12.512141+0000","last_fullsized":"2024-03-04T07:29:12.512141+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:19:43.023161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.12","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.515962+0000","last_change":"2024-03-04T07:29:10.515222+0000","last_active":"2024-03-04T07:29:12.515962+0000","last_peered":"2024-03-04T07:29:12.515962+0000","last_clean":"2024-03-04T07:29:12.515962+0000","last_became_active":"2024-03-04T07:29:10.514882+0000","last_became_peered":"2024-03-04T07:29:10.514882+0000","last_unstale":"2024-03-04T07:29:12.515962+0000","last_undegraded":"2024-03-04T07:29:12.515962+0000","last_fullsized":"2024-03-04T07:29:12.515962+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:45.610640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.10","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511471+0000","last_change":"2024-03-04T07:29:11.517352+0000","last_active":"2024-03-04T07:29:12.511471+0000","last_peered":"2024-03-04T07:29:12.511471+0000","last_clean":"2024-03-04T07:29:12.511471+0000","last_became_active":"2024-03-04T07:29:11.517044+0000","last_became_peered":"2024-03-04T07:29:11.517044+0000","last_unstale":"2024-03-04T07:29:12.511471+0000","last_undegraded":"2024-03-04T07:29:12.511471+0000","last_fullsized":"2024-03-04T07:29:12.511471+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:13:31.528309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.11","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592043+0000","last_change":"2024-03-04T07:29:10.515029+0000","last_active":"2024-03-04T07:29:12.592043+0000","last_peered":"2024-03-04T07:29:12.592043+0000","last_clean":"2024-03-04T07:29:12.592043+0000","last_became_active":"2024-03-04T07:29:10.514722+0000","last_became_peered":"2024-03-04T07:29:10.514722+0000","last_unstale":"2024-03-04T07:29:12.592043+0000","last_undegraded":"2024-03-04T07:29:12.592043+0000","last_fullsized":"2024-03-04T07:29:12.592043+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:13:30.521442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512002+0000","last_change":"2024-03-04T07:29:11.521359+0000","last_active":"2024-03-04T07:29:12.512002+0000","last_peered":"2024-03-04T07:29:12.512002+0000","last_clean":"2024-03-04T07:29:12.512002+0000","last_became_active":"2024-03-04T07:29:11.521045+0000","last_became_peered":"2024-03-04T07:29:11.521045+0000","last_unstale":"2024-03-04T07:29:12.512002+0000","last_undegraded":"2024-03-04T07:29:12.512002+0000","last_fullsized":"2024-03-04T07:29:12.512002+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:46:38.755909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.10","version":"54'8","reported_seq":22,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511571+0000","last_change":"2024-03-04T07:29:10.517145+0000","last_active":"2024-03-04T07:29:12.511571+0000","last_peered":"2024-03-04T07:29:12.511571+0000","last_clean":"2024-03-04T07:29:12.511571+0000","last_became_active":"2024-03-04T07:29:10.516791+0000","last_became_peered":"2024-03-04T07:29:10.516791+0000","last_unstale":"2024-03-04T07:29:12.511571+0000","last_undegraded":"2024-03-04T07:29:12.511571+0000","last_fullsized":"2024-03-04T07:29:12.511571+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:21.125602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512930+0000","last_change":"2024-03-04T07:29:11.519002+0000","last_active":"2024-03-04T07:29:12.512930+0000","last_peered":"2024-03-04T07:29:12.512930+0000","last_clean":"2024-03-04T07:29:12.512930+0000","last_became_active":"2024-03-04T07:29:11.518693+0000","last_became_peered":"2024-03-04T07:29:11.518693+0000","last_unstale":"2024-03-04T07:29:12.512930+0000","last_undegraded":"2024-03-04T07:29:12.512930+0000","last_fullsized":"2024-03-04T07:29:12.512930+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:43:22.542266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"54'1","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510533+0000","last_change":"2024-03-04T07:29:10.517764+0000","last_active":"2024-03-04T07:29:12.510533+0000","last_peered":"2024-03-04T07:29:12.510533+0000","last_clean":"2024-03-04T07:29:12.510533+0000","last_became_active":"2024-03-04T07:29:10.516895+0000","last_became_peered":"2024-03-04T07:29:10.516895+0000","last_unstale":"2024-03-04T07:29:12.510533+0000","last_undegraded":"2024-03-04T07:29:12.510533+0000","last_fullsized":"2024-03-04T07:29:12.510533+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:53:35.871464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592584+0000","last_change":"2024-03-04T07:29:11.511640+0000","last_active":"2024-03-04T07:29:12.592584+0000","last_peered":"2024-03-04T07:29:12.592584+0000","last_clean":"2024-03-04T07:29:12.592584+0000","last_became_active":"2024-03-04T07:29:11.511332+0000","last_became_peered":"2024-03-04T07:29:11.511332+0000","last_unstale":"2024-03-04T07:29:12.592584+0000","last_undegraded":"2024-03-04T07:29:12.592584+0000","last_fullsized":"2024-03-04T07:29:12.592584+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:19:43.023161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"54'2","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513534+0000","last_change":"2024-03-04T07:29:10.515628+0000","last_active":"2024-03-04T07:29:12.513534+0000","last_peered":"2024-03-04T07:29:12.513534+0000","last_clean":"2024-03-04T07:29:12.513534+0000","last_became_active":"2024-03-04T07:29:10.515205+0000","last_became_peered":"2024-03-04T07:29:10.515205+0000","last_unstale":"2024-03-04T07:29:12.513534+0000","last_undegraded":"2024-03-04T07:29:12.513534+0000","last_fullsized":"2024-03-04T07:29:12.513534+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:13:30.521442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510325+0000","last_change":"2024-03-04T07:29:11.510194+0000","last_active":"2024-03-04T07:29:12.510325+0000","last_peered":"2024-03-04T07:29:12.510325+0000","last_clean":"2024-03-04T07:29:12.510325+0000","last_became_active":"2024-03-04T07:29:11.509929+0000","last_became_peered":"2024-03-04T07:29:11.509929+0000","last_unstale":"2024-03-04T07:29:12.510325+0000","last_undegraded":"2024-03-04T07:29:12.510325+0000","last_fullsized":"2024-03-04T07:29:12.510325+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:44:10.910920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"54'1","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592438+0000","last_change":"2024-03-04T07:29:10.512208+0000","last_active":"2024-03-04T07:29:12.592438+0000","last_peered":"2024-03-04T07:29:12.592438+0000","last_clean":"2024-03-04T07:29:12.592438+0000","last_became_active":"2024-03-04T07:29:10.511806+0000","last_became_peered":"2024-03-04T07:29:10.511806+0000","last_unstale":"2024-03-04T07:29:12.592438+0000","last_undegraded":"2024-03-04T07:29:12.592438+0000","last_fullsized":"2024-03-04T07:29:12.592438+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:45:10.807134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512202+0000","last_change":"2024-03-04T07:29:11.525439+0000","last_active":"2024-03-04T07:29:12.512202+0000","last_peered":"2024-03-04T07:29:12.512202+0000","last_clean":"2024-03-04T07:29:12.512202+0000","last_became_active":"2024-03-04T07:29:11.525161+0000","last_became_peered":"2024-03-04T07:29:11.525161+0000","last_unstale":"2024-03-04T07:29:12.512202+0000","last_undegraded":"2024-03-04T07:29:12.512202+0000","last_fullsized":"2024-03-04T07:29:12.512202+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:28.667236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513293+0000","last_change":"2024-03-04T07:29:10.516595+0000","last_active":"2024-03-04T07:29:12.513293+0000","last_peered":"2024-03-04T07:29:12.513293+0000","last_clean":"2024-03-04T07:29:12.513293+0000","last_became_active":"2024-03-04T07:29:10.516327+0000","last_became_peered":"2024-03-04T07:29:10.516327+0000","last_unstale":"2024-03-04T07:29:12.513293+0000","last_undegraded":"2024-03-04T07:29:12.513293+0000","last_fullsized":"2024-03-04T07:29:12.513293+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:45:10.807134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592111+0000","last_change":"2024-03-04T07:29:11.509924+0000","last_active":"2024-03-04T07:29:12.592111+0000","last_peered":"2024-03-04T07:29:12.592111+0000","last_clean":"2024-03-04T07:29:12.592111+0000","last_became_active":"2024-03-04T07:29:11.509615+0000","last_became_peered":"2024-03-04T07:29:11.509615+0000","last_unstale":"2024-03-04T07:29:12.592111+0000","last_undegraded":"2024-03-04T07:29:12.592111+0000","last_fullsized":"2024-03-04T07:29:12.592111+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:36:16.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"54'2","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510646+0000","last_change":"2024-03-04T07:29:10.517775+0000","last_active":"2024-03-04T07:29:12.510646+0000","last_peered":"2024-03-04T07:29:12.510646+0000","last_clean":"2024-03-04T07:29:12.510646+0000","last_became_active":"2024-03-04T07:29:10.517496+0000","last_became_peered":"2024-03-04T07:29:10.517496+0000","last_unstale":"2024-03-04T07:29:12.510646+0000","last_undegraded":"2024-03-04T07:29:12.510646+0000","last_fullsized":"2024-03-04T07:29:12.510646+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:09.319598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513421+0000","last_change":"2024-03-04T07:29:11.510481+0000","last_active":"2024-03-04T07:29:12.513421+0000","last_peered":"2024-03-04T07:29:12.513421+0000","last_clean":"2024-03-04T07:29:12.513421+0000","last_became_active":"2024-03-04T07:29:11.510221+0000","last_became_peered":"2024-03-04T07:29:11.510221+0000","last_unstale":"2024-03-04T07:29:12.513421+0000","last_undegraded":"2024-03-04T07:29:12.513421+0000","last_fullsized":"2024-03-04T07:29:12.513421+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:28.667236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591685+0000","last_change":"2024-03-04T07:29:10.514774+0000","last_active":"2024-03-04T07:29:12.591685+0000","last_peered":"2024-03-04T07:29:12.591685+0000","last_clean":"2024-03-04T07:29:12.591685+0000","last_became_active":"2024-03-04T07:29:10.514489+0000","last_became_peered":"2024-03-04T07:29:10.514489+0000","last_unstale":"2024-03-04T07:29:12.591685+0000","last_undegraded":"2024-03-04T07:29:12.591685+0000","last_fullsized":"2024-03-04T07:29:12.591685+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:54:04.259042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513165+0000","last_change":"2024-03-04T07:29:11.513809+0000","last_active":"2024-03-04T07:29:12.513165+0000","last_peered":"2024-03-04T07:29:12.513165+0000","last_clean":"2024-03-04T07:29:12.513165+0000","last_became_active":"2024-03-04T07:29:11.513545+0000","last_became_peered":"2024-03-04T07:29:11.513545+0000","last_unstale":"2024-03-04T07:29:12.513165+0000","last_undegraded":"2024-03-04T07:29:12.513165+0000","last_fullsized":"2024-03-04T07:29:12.513165+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:10.326465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591998+0000","last_change":"2024-03-04T07:29:10.513482+0000","last_active":"2024-03-04T07:29:12.591998+0000","last_peered":"2024-03-04T07:29:12.591998+0000","last_clean":"2024-03-04T07:29:12.591998+0000","last_became_active":"2024-03-04T07:29:10.513209+0000","last_became_peered":"2024-03-04T07:29:10.513209+0000","last_unstale":"2024-03-04T07:29:12.591998+0000","last_undegraded":"2024-03-04T07:29:12.591998+0000","last_fullsized":"2024-03-04T07:29:12.591998+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:06:17.042964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592514+0000","last_change":"2024-03-04T07:29:11.509678+0000","last_active":"2024-03-04T07:29:12.592514+0000","last_peered":"2024-03-04T07:29:12.592514+0000","last_clean":"2024-03-04T07:29:12.592514+0000","last_became_active":"2024-03-04T07:29:11.509370+0000","last_became_peered":"2024-03-04T07:29:11.509370+0000","last_unstale":"2024-03-04T07:29:12.592514+0000","last_undegraded":"2024-03-04T07:29:12.592514+0000","last_fullsized":"2024-03-04T07:29:12.592514+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:46:38.755909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.7","version":"54'2","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591833+0000","last_change":"2024-03-04T07:29:10.516454+0000","last_active":"2024-03-04T07:29:12.591833+0000","last_peered":"2024-03-04T07:29:12.591833+0000","last_clean":"2024-03-04T07:29:12.591833+0000","last_became_active":"2024-03-04T07:29:10.514845+0000","last_became_peered":"2024-03-04T07:29:10.514845+0000","last_unstale":"2024-03-04T07:29:12.591833+0000","last_undegraded":"2024-03-04T07:29:12.591833+0000","last_fullsized":"2024-03-04T07:29:12.591833+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:56.206148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592622+0000","last_change":"2024-03-04T07:29:11.514256+0000","last_active":"2024-03-04T07:29:12.592622+0000","last_peered":"2024-03-04T07:29:12.592622+0000","last_clean":"2024-03-04T07:29:12.592622+0000","last_became_active":"2024-03-04T07:29:11.513951+0000","last_became_peered":"2024-03-04T07:29:11.513951+0000","last_unstale":"2024-03-04T07:29:12.592622+0000","last_undegraded":"2024-03-04T07:29:12.592622+0000","last_fullsized":"2024-03-04T07:29:12.592622+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:09:28.667236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.6","version":"55'1","reported_seq":15,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513645+0000","last_change":"2024-03-04T07:29:10.514474+0000","last_active":"2024-03-04T07:29:12.513645+0000","last_peered":"2024-03-04T07:29:12.513645+0000","last_clean":"2024-03-04T07:29:12.513645+0000","last_became_active":"2024-03-04T07:29:10.514215+0000","last_became_peered":"2024-03-04T07:29:10.514215+0000","last_unstale":"2024-03-04T07:29:12.513645+0000","last_undegraded":"2024-03-04T07:29:12.513645+0000","last_fullsized":"2024-03-04T07:29:12.513645+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:12:54.069572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591715+0000","last_change":"2024-03-04T07:29:11.507147+0000","last_active":"2024-03-04T07:29:12.591715+0000","last_peered":"2024-03-04T07:29:12.591715+0000","last_clean":"2024-03-04T07:29:12.591715+0000","last_became_active":"2024-03-04T07:29:11.506524+0000","last_became_peered":"2024-03-04T07:29:11.506524+0000","last_unstale":"2024-03-04T07:29:12.591715+0000","last_undegraded":"2024-03-04T07:29:12.591715+0000","last_fullsized":"2024-03-04T07:29:12.591715+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:00.866317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.516044+0000","last_change":"2024-03-04T07:29:10.515670+0000","last_active":"2024-03-04T07:29:12.516044+0000","last_peered":"2024-03-04T07:29:12.516044+0000","last_clean":"2024-03-04T07:29:12.516044+0000","last_became_active":"2024-03-04T07:29:10.515327+0000","last_became_peered":"2024-03-04T07:29:10.515327+0000","last_unstale":"2024-03-04T07:29:12.516044+0000","last_undegraded":"2024-03-04T07:29:12.516044+0000","last_fullsized":"2024-03-04T07:29:12.516044+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:43.992674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.513819+0000","last_change":"2024-03-04T07:29:11.511793+0000","last_active":"2024-03-04T07:29:12.513819+0000","last_peered":"2024-03-04T07:29:12.513819+0000","last_clean":"2024-03-04T07:29:12.513819+0000","last_became_active":"2024-03-04T07:29:11.511562+0000","last_became_peered":"2024-03-04T07:29:11.511562+0000","last_unstale":"2024-03-04T07:29:12.513819+0000","last_undegraded":"2024-03-04T07:29:12.513819+0000","last_fullsized":"2024-03-04T07:29:12.513819+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:54:26.472758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.592619+0000","last_change":"2024-03-04T07:29:10.506371+0000","last_active":"2024-03-04T07:29:12.592619+0000","last_peered":"2024-03-04T07:29:12.592619+0000","last_clean":"2024-03-04T07:29:12.592619+0000","last_became_active":"2024-03-04T07:29:10.506084+0000","last_became_peered":"2024-03-04T07:29:10.506084+0000","last_unstale":"2024-03-04T07:29:12.592619+0000","last_undegraded":"2024-03-04T07:29:12.592619+0000","last_fullsized":"2024-03-04T07:29:12.592619+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:12:54.069572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510437+0000","last_change":"2024-03-04T07:29:11.517040+0000","last_active":"2024-03-04T07:29:12.510437+0000","last_peered":"2024-03-04T07:29:12.510437+0000","last_clean":"2024-03-04T07:29:12.510437+0000","last_became_active":"2024-03-04T07:29:11.516765+0000","last_became_peered":"2024-03-04T07:29:11.516765+0000","last_unstale":"2024-03-04T07:29:12.510437+0000","last_undegraded":"2024-03-04T07:29:12.510437+0000","last_fullsized":"2024-03-04T07:29:12.510437+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:18:28.207257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591561+0000","last_change":"2024-03-04T07:29:10.510685+0000","last_active":"2024-03-04T07:29:12.591561+0000","last_peered":"2024-03-04T07:29:12.591561+0000","last_clean":"2024-03-04T07:29:12.591561+0000","last_became_active":"2024-03-04T07:29:10.510433+0000","last_became_peered":"2024-03-04T07:29:10.510433+0000","last_unstale":"2024-03-04T07:29:12.591561+0000","last_undegraded":"2024-03-04T07:29:12.591561+0000","last_fullsized":"2024-03-04T07:29:12.591561+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:29:45.610640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512325+0000","last_change":"2024-03-04T07:29:11.522800+0000","last_active":"2024-03-04T07:29:12.512325+0000","last_peered":"2024-03-04T07:29:12.512325+0000","last_clean":"2024-03-04T07:29:12.512325+0000","last_became_active":"2024-03-04T07:29:11.522457+0000","last_became_peered":"2024-03-04T07:29:11.522457+0000","last_unstale":"2024-03-04T07:29:12.512325+0000","last_undegraded":"2024-03-04T07:29:12.512325+0000","last_fullsized":"2024-03-04T07:29:12.512325+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:37.622116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.512675+0000","last_change":"2024-03-04T07:29:10.513884+0000","last_active":"2024-03-04T07:29:12.512675+0000","last_peered":"2024-03-04T07:29:12.512675+0000","last_clean":"2024-03-04T07:29:12.512675+0000","last_became_active":"2024-03-04T07:29:10.513574+0000","last_became_peered":"2024-03-04T07:29:10.513574+0000","last_unstale":"2024-03-04T07:29:12.512675+0000","last_undegraded":"2024-03-04T07:29:12.512675+0000","last_fullsized":"2024-03-04T07:29:12.512675+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:05:58.330259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591520+0000","last_change":"2024-03-04T07:29:11.502806+0000","last_active":"2024-03-04T07:29:12.591520+0000","last_peered":"2024-03-04T07:29:12.591520+0000","last_clean":"2024-03-04T07:29:12.591520+0000","last_became_active":"2024-03-04T07:29:11.502534+0000","last_became_peered":"2024-03-04T07:29:11.502534+0000","last_unstale":"2024-03-04T07:29:12.591520+0000","last_undegraded":"2024-03-04T07:29:12.591520+0000","last_fullsized":"2024-03-04T07:29:12.591520+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:03:04.361535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.591563+0000","last_change":"2024-03-04T07:29:10.514430+0000","last_active":"2024-03-04T07:29:12.591563+0000","last_peered":"2024-03-04T07:29:12.591563+0000","last_clean":"2024-03-04T07:29:12.591563+0000","last_became_active":"2024-03-04T07:29:10.514146+0000","last_became_peered":"2024-03-04T07:29:10.514146+0000","last_unstale":"2024-03-04T07:29:12.591563+0000","last_undegraded":"2024-03-04T07:29:12.591563+0000","last_fullsized":"2024-03-04T07:29:12.591563+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:10:43.992674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510234+0000","last_change":"2024-03-04T07:29:11.507963+0000","last_active":"2024-03-04T07:29:12.510234+0000","last_peered":"2024-03-04T07:29:12.510234+0000","last_clean":"2024-03-04T07:29:12.510234+0000","last_became_active":"2024-03-04T07:29:11.507701+0000","last_became_peered":"2024-03-04T07:29:11.507701+0000","last_unstale":"2024-03-04T07:29:12.510234+0000","last_undegraded":"2024-03-04T07:29:12.510234+0000","last_fullsized":"2024-03-04T07:29:12.510234+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:18:20.216420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510036+0000","last_change":"2024-03-04T07:29:10.519578+0000","last_active":"2024-03-04T07:29:12.510036+0000","last_peered":"2024-03-04T07:29:12.510036+0000","last_clean":"2024-03-04T07:29:12.510036+0000","last_became_active":"2024-03-04T07:29:10.519248+0000","last_became_peered":"2024-03-04T07:29:10.519248+0000","last_unstale":"2024-03-04T07:29:12.510036+0000","last_undegraded":"2024-03-04T07:29:12.510036+0000","last_fullsized":"2024-03-04T07:29:12.510036+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:11:20.751197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.510001+0000","last_change":"2024-03-04T07:29:11.519071+0000","last_active":"2024-03-04T07:29:12.510001+0000","last_peered":"2024-03-04T07:29:12.510001+0000","last_clean":"2024-03-04T07:29:12.510001+0000","last_became_active":"2024-03-04T07:29:11.518809+0000","last_became_peered":"2024-03-04T07:29:11.518809+0000","last_unstale":"2024-03-04T07:29:12.510001+0000","last_undegraded":"2024-03-04T07:29:12.510001+0000","last_fullsized":"2024-03-04T07:29:12.510001+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:10.486506+0000","last_clean_scrub_stamp":"2024-03-04T07:29:10.486506+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:59.174617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":14,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.515855+0000","last_change":"2024-03-04T07:29:10.513166+0000","last_active":"2024-03-04T07:29:12.515855+0000","last_peered":"2024-03-04T07:29:12.515855+0000","last_clean":"2024-03-04T07:29:12.515855+0000","last_became_active":"2024-03-04T07:29:10.512608+0000","last_became_peered":"2024-03-04T07:29:10.512608+0000","last_unstale":"2024-03-04T07:29:12.515855+0000","last_undegraded":"2024-03-04T07:29:12.515855+0000","last_fullsized":"2024-03-04T07:29:12.515855+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:29:09.479638+0000","last_clean_scrub_stamp":"2024-03-04T07:29:09.479638+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:22:56.206148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":234,"reported_epoch":55,"state":"active+clean","last_fresh":"2024-03-04T07:29:12.511702+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:29:12.511702+0000","last_peered":"2024-03-04T07:29:12.511702+0000","last_clean":"2024-03-04T07:29:12.511702+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:29:12.511702+0000","last_undegraded":"2024-03-04T07:29:12.511702+0000","last_fullsized":"2024-03-04T07:29:12.511702+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":8,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574884,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32412,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339428,"statfs":{"total":96636764160,"available":96603574272,"internally_reserved":0,"allocated":1261568,"data_stored":863520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574883,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31824,"kb_used_data":644,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340016,"statfs":{"total":96636764160,"available":96604176384,"internally_reserved":0,"allocated":659456,"data_stored":272992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574883,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31832,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340008,"statfs":{"total":96636764160,"available":96604168192,"internally_reserved":0,"allocated":667648,"data_stored":273644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574884,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31836,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340004,"statfs":{"total":96636764160,"available":96604164096,"internally_reserved":0,"allocated":671744,"data_stored":274604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574883,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31832,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340008,"statfs":{"total":96636764160,"available":96604168192,"internally_reserved":0,"allocated":667648,"data_stored":273048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574883,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31832,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340008,"statfs":{"total":96636764160,"available":96604168192,"internally_reserved":0,"allocated":667648,"data_stored":273048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574883,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31828,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340012,"statfs":{"total":96636764160,"available":96604172288,"internally_reserved":0,"allocated":663552,"data_stored":273082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574883,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32420,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339420,"statfs":{"total":96636764160,"available":96603566080,"internally_reserved":0,"allocated":1269760,"data_stored":865554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:29:19.360 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:29:19.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:29:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.504+0000 7f6a84bd6640 1 Processor -- start 2024-03-04T07:29:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.504+0000 7f6a84bd6640 1 -- start start 2024-03-04T07:29:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a84bd6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800717b0 0x7f6a80071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a84bd6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a8015e4b0 con 0x7f6a800b3060 2024-03-04T07:29:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a84bd6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a800b97e0 con 0x7f6a800b2b10 2024-03-04T07:29:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a84bd6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a800b7c70 con 0x7f6a800717b0 2024-03-04T07:29:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a77fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6a800b3060 0x7f6a800b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35710/0 (socket says 172.21.15.64:35710) 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a77fff640 1 -- 172.21.15.64:0/2562906694 learned_addr learned my addr 172.21.15.64:0/2562906694 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7e59c640 1 --2- 172.21.15.64:0/2562906694 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800717b0 0x7f6a80071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7dd9b640 1 -- 172.21.15.64:0/2562906694 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 590228408 0 0) 0x7f6a800b97e0 con 0x7f6a800b2b10 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7dd9b640 1 -- 172.21.15.64:0/2562906694 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6a6c003520 con 0x7f6a800b2b10 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7dd9b640 1 -- 172.21.15.64:0/2562906694 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 381923309 0 0) 0x7f6a68001110 con 0x7f6a800b3060 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7dd9b640 1 -- 172.21.15.64:0/2562906694 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6a800b97e0 con 0x7f6a800b3060 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7e59c640 1 -- 172.21.15.64:0/2562906694 >> v1:172.21.15.66:6789/0 conn(0x7f6a800b2b10 legacy=0x7f6a800b7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7e59c640 1 -- 172.21.15.64:0/2562906694 >> v1:172.21.15.64:6789/0 conn(0x7f6a800b3060 legacy=0x7f6a800b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.505+0000 7f6a7e59c640 1 -- 172.21.15.64:0/2562906694 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a800b9a80 con 0x7f6a800717b0 2024-03-04T07:29:19.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.506+0000 7f6a7e59c640 1 --2- 172.21.15.64:0/2562906694 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800717b0 0x7f6a80071b90 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f6a700097d0 tx=0x7f6a7002b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a3236db949661701 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a7dd9b640 1 -- 172.21.15.64:0/2562906694 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a70035020 con 0x7f6a800717b0 2024-03-04T07:29:19.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a7dd9b640 1 -- 172.21.15.64:0/2562906694 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a70031890 con 0x7f6a800717b0 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a7dd9b640 1 -- 172.21.15.64:0/2562906694 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a70031c60 con 0x7f6a800717b0 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a84bd6640 1 -- 172.21.15.64:0/2562906694 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800717b0 msgr2=0x7f6a80071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/2562906694 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800717b0 0x7f6a80071b90 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f6a700097d0 tx=0x7f6a7002b930 comp rx=0 tx=0).stop 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a84bd6640 1 -- 172.21.15.64:0/2562906694 shutdown_connections 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/2562906694 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800717b0 0x7f6a80071b90 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a84bd6640 1 -- 172.21.15.64:0/2562906694 >> 172.21.15.64:0/2562906694 conn(0x7f6a800d3d20 msgr2=0x7f6a800d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a84bd6640 1 -- 172.21.15.64:0/2562906694 shutdown_connections 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.507+0000 7f6a84bd6640 1 -- 172.21.15.64:0/2562906694 wait complete. 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.509+0000 7f6a84bd6640 1 Processor -- start 2024-03-04T07:29:19.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.509+0000 7f6a84bd6640 1 -- start start 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a84bd6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a800717b0 0x7f6a801494e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a84bd6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a800b2b10 0x7f6a801e8770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a84bd6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800b3060 0x7f6a8014ab60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a84bd6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a800d4bf0 con 0x7f6a800b2b10 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a84bd6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a800d2e10 con 0x7f6a800717b0 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a84bd6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a800d32a0 con 0x7f6a800b3060 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a7e59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a800717b0 0x7f6a801494e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a77fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a800b2b10 0x7f6a801e8770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a7e59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a800717b0 0x7f6a801494e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:34908/0 (socket says 172.21.15.64:34908) 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a77fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a800b2b10 0x7f6a801e8770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39372/0 (socket says 172.21.15.64:39372) 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a77fff640 1 -- 172.21.15.64:0/1593530587 learned_addr learned my addr 172.21.15.64:0/1593530587 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.510+0000 7f6a7ed9d640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800b3060 0x7f6a8014ab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a77fff640 1 -- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800b3060 msgr2=0x7f6a8014ab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a77fff640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800b3060 0x7f6a8014ab60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a77fff640 1 -- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a800717b0 msgr2=0x7f6a801494e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a77fff640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a800717b0 0x7f6a801494e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a77fff640 1 -- 172.21.15.64:0/1593530587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a80149ac0 con 0x7f6a800b2b10 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a7e59c640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a800717b0 0x7f6a801494e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a77fff640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a800b2b10 0x7f6a801e8770 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f6a6800c3d0 tx=0x7f6a6800c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a76ffd640 1 -- 172.21.15.64:0/1593530587 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a68017070 con 0x7f6a800b2b10 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a76ffd640 1 -- 172.21.15.64:0/1593530587 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a68005ce0 con 0x7f6a800b2b10 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a76ffd640 1 -- 172.21.15.64:0/1593530587 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a68007310 con 0x7f6a800b2b10 2024-03-04T07:29:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a80149da0 con 0x7f6a800b2b10 2024-03-04T07:29:19.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.511+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a8014a360 con 0x7f6a800b2b10 2024-03-04T07:29:19.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.512+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6a8006ce40 con 0x7f6a800b2b10 2024-03-04T07:29:19.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.513+0000 7f6a76ffd640 1 -- 172.21.15.64:0/1593530587 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6a6801a070 con 0x7f6a800b2b10 2024-03-04T07:29:19.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.513+0000 7f6a76ffd640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a6006b4d0 0x7f6a6006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.513+0000 7f6a7e59c640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a6006b4d0 0x7f6a6006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.513+0000 7f6a76ffd640 1 -- 172.21.15.64:0/1593530587 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6a680895c0 con 0x7f6a800b2b10 2024-03-04T07:29:19.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.514+0000 7f6a7e59c640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a6006b4d0 0x7f6a6006d990 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f6a70036000 tx=0x7f6a70008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.519+0000 7f6a76ffd640 1 -- 172.21.15.64:0/1593530587 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6a8015e4b0 con 0x7f6a800b2b10 2024-03-04T07:29:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.820+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f6a800c9dc0 con 0x7f6a800b2b10 2024-03-04T07:29:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.825+0000 7f6a76ffd640 1 -- 172.21.15.64:0/1593530587 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v17) v1 ==== 190+0+147 (secure 0 0 0) 0x7f6a68057df0 con 0x7f6a800b2b10 2024-03-04T07:29:19.825 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:29:19.825 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:29:19.826 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:29:19.826 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:29:19.826 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:29:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:29:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.826+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a6006b4d0 msgr2=0x7f6a6006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.826+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a6006b4d0 0x7f6a6006d990 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f6a70036000 tx=0x7f6a70008030 comp rx=0 tx=0).stop 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.826+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a800b2b10 msgr2=0x7f6a801e8770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.826+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a800b2b10 0x7f6a801e8770 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f6a6800c3d0 tx=0x7f6a6800c890 comp rx=0 tx=0).stop 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.826+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 shutdown_connections 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.826+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a6006b4d0 0x7f6a6006d990 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.826+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a800b3060 0x7f6a8014ab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.827+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a800b2b10 0x7f6a801e8770 unknown :-1 s=CLOSED pgs=684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.827+0000 7f6a84bd6640 1 --2- 172.21.15.64:0/1593530587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a800717b0 0x7f6a801494e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.827+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 >> 172.21.15.64:0/1593530587 conn(0x7f6a800d3d20 msgr2=0x7f6a800cce00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.827+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 shutdown_connections 2024-03-04T07:29:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.827+0000 7f6a84bd6640 1 -- 172.21.15.64:0/1593530587 wait complete. 2024-03-04T07:29:19.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.943+0000 7f258feaf640 1 Processor -- start 2024-03-04T07:29:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.943+0000 7f258feaf640 1 -- start start 2024-03-04T07:29:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258feaf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2588157f60 0x7f258815b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258feaf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f2588070b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258feaf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f258806c7b0 con 0x7f2588157b80 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258feaf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f258806ca90 con 0x7f2588157f60 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258feaf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f258806cd70 con 0x7f25880cbab0 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258d44a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2588157f60 0x7f258815b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258e44c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f2588070b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258dc4b640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f2588157b80 0x7f258815b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:36796/0 (socket says 172.21.15.64:36796) 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258d44a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2588157f60 0x7f258815b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:34910/0 (socket says 172.21.15.64:34910) 2024-03-04T07:29:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258dc4b640 1 -- 172.21.15.64:0/2507245126 learned_addr learned my addr 172.21.15.64:0/2507245126 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258d44a640 1 -- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 msgr2=0x7f2588070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258d44a640 1 --2- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f2588070b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258d44a640 1 -- 172.21.15.64:0/2507245126 >> v1:172.21.15.66:6789/0 conn(0x7f2588157b80 legacy=0x7f258815b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258d44a640 1 -- 172.21.15.64:0/2507245126 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f258806d050 con 0x7f2588157f60 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.945+0000 7f258e44c640 1 --2- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f2588070b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.946+0000 7f258d44a640 1 --2- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 0x7f258815b260 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f257c0097a0 tx=0x7f257c02b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=949884aafffc3e65 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.946+0000 7f258cc49640 1 -- 172.21.15.64:0/2507245126 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f257c035020 con 0x7f2588157f60 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.946+0000 7f258cc49640 1 -- 172.21.15.64:0/2507245126 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f257c031830 con 0x7f2588157f60 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.946+0000 7f258cc49640 1 -- 172.21.15.64:0/2507245126 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f257c031c00 con 0x7f2588157f60 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 -- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 msgr2=0x7f258815b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 --2- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 0x7f258815b260 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f257c0097a0 tx=0x7f257c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 -- 172.21.15.64:0/2507245126 shutdown_connections 2024-03-04T07:29:19.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 --2- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f2588070b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 --2- 172.21.15.64:0/2507245126 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 0x7f258815b260 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 -- 172.21.15.64:0/2507245126 >> 172.21.15.64:0/2507245126 conn(0x7f2588066d00 msgr2=0x7f25880670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:19.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 -- 172.21.15.64:0/2507245126 shutdown_connections 2024-03-04T07:29:19.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.947+0000 7f258feaf640 1 -- 172.21.15.64:0/2507245126 wait complete. 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.948+0000 7f258feaf640 1 Processor -- start 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.948+0000 7f258feaf640 1 -- start start 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258feaf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f25881472a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258feaf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2588157b80 0x7f2588152560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258feaf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 0x7f25880b02c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258feaf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2588071080 con 0x7f2588157b80 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258feaf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f25880cc260 con 0x7f2588157f60 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258feaf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2588071200 con 0x7f25880cbab0 2024-03-04T07:29:19.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2588157b80 0x7f2588152560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2588157b80 0x7f2588152560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39382/0 (socket says 172.21.15.64:39382) 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 -- 172.21.15.64:0/3644501928 learned_addr learned my addr 172.21.15.64:0/3644501928 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258dc4b640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f25881472a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258e44c640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 0x7f25880b02c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 -- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 msgr2=0x7f25881472a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f25881472a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 -- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 msgr2=0x7f25880b02c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 0x7f25880b02c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.949+0000 7f258d44a640 1 -- 172.21.15.64:0/3644501928 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2588155c90 con 0x7f2588157b80 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.950+0000 7f258d44a640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2588157b80 0x7f2588152560 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f257c034040 tx=0x7f257c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.950+0000 7f257a7fc640 1 -- 172.21.15.64:0/3644501928 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f257c040070 con 0x7f2588157b80 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.950+0000 7f257a7fc640 1 -- 172.21.15.64:0/3644501928 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f257c03a850 con 0x7f2588157b80 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.950+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2588155f70 con 0x7f2588157b80 2024-03-04T07:29:19.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.950+0000 7f257a7fc640 1 -- 172.21.15.64:0/3644501928 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f257c035040 con 0x7f2588157b80 2024-03-04T07:29:19.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.950+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2588156530 con 0x7f2588157b80 2024-03-04T07:29:19.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.951+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2588156a10 con 0x7f2588157b80 2024-03-04T07:29:19.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.951+0000 7f257a7fc640 1 -- 172.21.15.64:0/3644501928 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f257c038020 con 0x7f2588157b80 2024-03-04T07:29:19.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.952+0000 7f257a7fc640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f256c06b4d0 0x7f256c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:19.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.952+0000 7f257a7fc640 1 -- 172.21.15.64:0/3644501928 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f257c0ab9b0 con 0x7f2588157b80 2024-03-04T07:29:19.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.952+0000 7f258dc4b640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f256c06b4d0 0x7f256c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:19.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.952+0000 7f258dc4b640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f256c06b4d0 0x7f256c06d990 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f2584007fd0 tx=0x7f2584007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:19.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:19.956+0000 7f257a7fc640 1 -- 172.21.15.64:0/3644501928 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2588156a10 con 0x7f2588157b80 2024-03-04T07:29:20.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.179+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f25880d07a0 con 0x7f2588157b80 2024-03-04T07:29:20.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.179+0000 7f257a7fc640 1 -- 172.21.15.64:0/3644501928 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f257c031800 con 0x7f2588157b80 2024-03-04T07:29:20.180 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:20.180 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:20.180 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:20.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f256c06b4d0 msgr2=0x7f256c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f256c06b4d0 0x7f256c06d990 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f2584007fd0 tx=0x7f2584007e80 comp rx=0 tx=0).stop 2024-03-04T07:29:20.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2588157b80 msgr2=0x7f2588152560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2588157b80 0x7f2588152560 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f257c034040 tx=0x7f257c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 shutdown_connections 2024-03-04T07:29:20.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f256c06b4d0 0x7f256c06d990 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2588157f60 0x7f25880b02c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2588157b80 0x7f2588152560 unknown :-1 s=CLOSED pgs=685 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 --2- 172.21.15.64:0/3644501928 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25880cbab0 0x7f25881472a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 >> 172.21.15.64:0/3644501928 conn(0x7f2588066d00 msgr2=0x7f258815b980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 shutdown_connections 2024-03-04T07:29:20.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.181+0000 7f258feaf640 1 -- 172.21.15.64:0/3644501928 wait complete. 2024-03-04T07:29:20.595 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 29, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:29:10.861414+0000', 'modified': '2024-03-04T07:29:12.860001+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 6069}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6069': {'gid': 6069, 'name': 'c', 'rank': 0, 'incarnation': 27, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.64:6837/1811282816', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6836', 'nonce': 1811282816}, {'type': 'v1', 'addr': '172.21.15.64:6837', 'nonce': 1811282816}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [9], 'metadata_pool': 8, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 6069, 'qdb_cluster': [6069]} 2024-03-04T07:29:20.595 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:29:20.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-04T07:29:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.697+0000 7f89f8046640 1 Processor -- start 2024-03-04T07:29:20.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.697+0000 7f89f8046640 1 -- start start 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f8046640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f89f00ae9b0 0x7f89f00d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f8046640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f89f00cee90 con 0x7f89f006b5c0 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f8046640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f89f00cfc50 con 0x7f89f00b0e90 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f8046640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f89f00cfa00 con 0x7f89f00ae9b0 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f5de2640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f89f006b5c0 0x7f89f00b2e00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35720/0 (socket says 172.21.15.64:35720) 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f5de2640 1 -- 172.21.15.64:0/1403886367 learned_addr learned my addr 172.21.15.64:0/1403886367 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f65e3640 1 --2- 172.21.15.64:0/1403886367 >> v2:172.21.15.66:3300/0 conn(0x7f89f00ae9b0 0x7f89f00d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f4de0640 1 -- 172.21.15.64:0/1403886367 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3383143159 0 0) 0x7f89f00cee90 con 0x7f89f006b5c0 2024-03-04T07:29:20.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f4de0640 1 -- 172.21.15.64:0/1403886367 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f89dc003520 con 0x7f89f006b5c0 2024-03-04T07:29:20.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f4de0640 1 -- 172.21.15.64:0/1403886367 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3837581479 0 0) 0x7f89dc003520 con 0x7f89f006b5c0 2024-03-04T07:29:20.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f65e3640 1 -- 172.21.15.64:0/1403886367 >> v1:172.21.15.66:6789/0 conn(0x7f89f00b0e90 legacy=0x7f89f006b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f65e3640 1 -- 172.21.15.64:0/1403886367 >> v1:172.21.15.64:6789/0 conn(0x7f89f006b5c0 legacy=0x7f89f00b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.699+0000 7f89f65e3640 1 -- 172.21.15.64:0/1403886367 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89f00cff30 con 0x7f89f00ae9b0 2024-03-04T07:29:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.700+0000 7f89f65e3640 1 --2- 172.21.15.64:0/1403886367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00ae9b0 0x7f89f00d03a0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f89e8009d90 tx=0x7f89e802bd80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=481f63d8e92c384e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.700+0000 7f89f4de0640 1 -- 172.21.15.64:0/1403886367 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89e8039070 con 0x7f89f00ae9b0 2024-03-04T07:29:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.700+0000 7f89f4de0640 1 -- 172.21.15.64:0/1403886367 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89e802c8b0 con 0x7f89f00ae9b0 2024-03-04T07:29:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.700+0000 7f89f4de0640 1 -- 172.21.15.64:0/1403886367 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89ec001670 con 0x7f89f00ae9b0 2024-03-04T07:29:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.701+0000 7f89f8046640 1 -- 172.21.15.64:0/1403886367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00ae9b0 msgr2=0x7f89f00d03a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.701+0000 7f89f8046640 1 --2- 172.21.15.64:0/1403886367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00ae9b0 0x7f89f00d03a0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f89e8009d90 tx=0x7f89e802bd80 comp rx=0 tx=0).stop 2024-03-04T07:29:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.701+0000 7f89f8046640 1 -- 172.21.15.64:0/1403886367 shutdown_connections 2024-03-04T07:29:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.701+0000 7f89f8046640 1 --2- 172.21.15.64:0/1403886367 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00ae9b0 0x7f89f00d03a0 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.701+0000 7f89f8046640 1 -- 172.21.15.64:0/1403886367 >> 172.21.15.64:0/1403886367 conn(0x7f89f00b36d0 msgr2=0x7f89f00b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.701+0000 7f89f8046640 1 -- 172.21.15.64:0/1403886367 shutdown_connections 2024-03-04T07:29:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.701+0000 7f89f8046640 1 -- 172.21.15.64:0/1403886367 wait complete. 2024-03-04T07:29:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.702+0000 7f89f8046640 1 Processor -- start 2024-03-04T07:29:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.702+0000 7f89f8046640 1 -- start start 2024-03-04T07:29:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f8046640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89f006b5c0 0x7f89f01ddd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f8046640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89f00ae9b0 0x7f89f00b6f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f8046640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00b0e90 0x7f89f00b1350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f8046640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89f00aef40 con 0x7f89f006b5c0 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f8046640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89f00aedc0 con 0x7f89f00b0e90 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f8046640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89f00d08e0 con 0x7f89f00ae9b0 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89f006b5c0 0x7f89f01ddd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89f006b5c0 0x7f89f01ddd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39388/0 (socket says 172.21.15.64:39388) 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 -- 172.21.15.64:0/2327389427 learned_addr learned my addr 172.21.15.64:0/2327389427 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f65e3640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00b0e90 0x7f89f00b1350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 -- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89f00ae9b0 msgr2=0x7f89f00b6f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89f00ae9b0 0x7f89f00b6f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 -- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00b0e90 msgr2=0x7f89f00b1350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00b0e90 0x7f89f00b1350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f5de2640 1 -- 172.21.15.64:0/2327389427 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89f01de2d0 con 0x7f89f006b5c0 2024-03-04T07:29:20.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.703+0000 7f89f65e3640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00b0e90 0x7f89f00b1350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:20.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.704+0000 7f89f5de2640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89f006b5c0 0x7f89f01ddd90 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f89ec00c760 tx=0x7f89ec00cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.704+0000 7f89e27fc640 1 -- 172.21.15.64:0/2327389427 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89ec017070 con 0x7f89f006b5c0 2024-03-04T07:29:20.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.704+0000 7f89e27fc640 1 -- 172.21.15.64:0/2327389427 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89ec005af0 con 0x7f89f006b5c0 2024-03-04T07:29:20.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.704+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f89f01f7720 con 0x7f89f006b5c0 2024-03-04T07:29:20.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.704+0000 7f89e27fc640 1 -- 172.21.15.64:0/2327389427 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89ec007070 con 0x7f89f006b5c0 2024-03-04T07:29:20.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.704+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89f01f7ce0 con 0x7f89f006b5c0 2024-03-04T07:29:20.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.705+0000 7f89e27fc640 1 -- 172.21.15.64:0/2327389427 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f89ec01a070 con 0x7f89f006b5c0 2024-03-04T07:29:20.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.706+0000 7f89e27fc640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89d406b4d0 0x7f89d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.706+0000 7f89e27fc640 1 -- 172.21.15.64:0/2327389427 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f89ec089720 con 0x7f89f006b5c0 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.706+0000 7f89f55e1640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89d406b4d0 0x7f89d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.706+0000 7f89f55e1640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89d406b4d0 0x7f89d406d990 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f89e4000f60 tx=0x7f89e40079b0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.707+0000 7f89fac2a740 1 Processor -- start 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.707+0000 7f89fac2a740 1 -- start start 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79541870 0x559e79541c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795458c0 0x559e79547360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e79571600 0x559e79571a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89fac2a740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e794ab6b0 con 0x559e79571600 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e79454a60 con 0x559e79541870 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x559e794a7750 con 0x559e795458c0 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e79571600 0x559e79571a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e79571600 0x559e79571a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39396/0 (socket says 172.21.15.64:39396) 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 -- 172.21.15.64:0/502732266 learned_addr learned my addr 172.21.15.64:0/502732266 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f5de2640 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79541870 0x559e79541c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f55e1640 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795458c0 0x559e79547360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 -- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795458c0 msgr2=0x559e79547360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795458c0 0x559e79547360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 -- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79541870 msgr2=0x559e79541c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79541870 0x559e79541c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 -- 172.21.15.64:0/502732266 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559e79472f30 con 0x559e79571600 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.708+0000 7f89f65e3640 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e79571600 0x559e79571a80 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f89e8009eb0 tx=0x7f89e802e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bf84eb25e0c33ca0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89d37fe640 1 -- 172.21.15.64:0/502732266 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89e8039040 con 0x559e79571600 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89d37fe640 1 -- 172.21.15.64:0/502732266 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89e802cb70 con 0x559e79571600 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89d37fe640 1 -- 172.21.15.64:0/502732266 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89e803f440 con 0x559e79571600 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 -- 172.21.15.64:0/502732266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e79571600 msgr2=0x559e79571a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e79571600 0x559e79571a80 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f89e8009eb0 tx=0x7f89e802e000 comp rx=0 tx=0).stop 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 -- 172.21.15.64:0/502732266 shutdown_connections 2024-03-04T07:29:20.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e79571600 0x559e79571a80 unknown :-1 s=CLOSED pgs=688 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795458c0 0x559e79547360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 --2- 172.21.15.64:0/502732266 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79541870 0x559e79541c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 -- 172.21.15.64:0/502732266 >> 172.21.15.64:0/502732266 conn(0x559e795438f0 msgr2=0x559e795451b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 -- 172.21.15.64:0/502732266 shutdown_connections 2024-03-04T07:29:20.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.709+0000 7f89fac2a740 1 -- 172.21.15.64:0/502732266 wait complete. 2024-03-04T07:29:20.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.710+0000 7f89fac2a740 1 -- start start 2024-03-04T07:29:20.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.710+0000 7f89fac2a740 1 Processor -- start 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79563690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e7956d190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79564a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89fac2a740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e79545d50 con 0x559e795645a0 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e79467ea0 con 0x559e79571600 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x559e79546030 con 0x559e795651c0 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89fac2a740 10 client.0 _ll_register_callbacks cb 0x559e79467870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79564a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79564a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39398/0 (socket says 172.21.15.64:39398) 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 -- 172.21.15.64:0/2733136653 learned_addr learned my addr 172.21.15.64:0/2733136653 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:20.717 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f5de2640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79563690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f55e1640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e7956d190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 -- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 msgr2=0x559e7956d190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e7956d190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 -- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 msgr2=0x559e79563690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79563690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f65e3640 1 -- 172.21.15.64:0/2733136653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x559e79468270 con 0x559e795645a0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.711+0000 7f89f5de2640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79563690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89f65e3640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79564a20 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7f89e802c390 tx=0x7f89e802e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ecbbce6608eba073 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89d0ff9640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89d0ff9640 1 -- 172.21.15.64:0/2733136653 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89e8039050 con 0x559e795645a0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f89e802cb70 con 0x559e795645a0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89d0ff9640 1 -- 172.21.15.64:0/2733136653 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f89e8004040 con 0x559e795645a0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89d0ff9640 1 -- 172.21.15.64:0/2733136653 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) v2 ==== 24+0+0 (secure 0 0 0) 0x7f89e805a050 con 0x559e795645a0 2024-03-04T07:29:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 10 client.6225 fetch_fsmap learned FSMap version 29 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=2+}) v3 -- 0x559e791c02d0 con 0x559e795645a0 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89d0ff9640 1 -- 172.21.15.64:0/2733136653 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89e802cb70 con 0x559e795645a0 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89d0ff9640 1 -- 172.21.15.64:0/2733136653 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 29) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f89e805d020 con 0x559e795645a0 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 10 client.6225 fetch_fsmap finished waiting for FSMap version 29 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 10 client.6225 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 20 client.6225 populate_metadata read hostname 'smithi064' 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e795a06e0 0x559e795ab380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 4 client.6225 mds_command: new command op to 6069 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 --> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559e794b8bf0 con 0x559e795a06e0 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.712+0000 7f89f5de2640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e795a06e0 0x559e795ab380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89d0ff9640 10 client.6225 ms_handle_connect on v2:172.21.15.64:6836/1811282816 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89f5de2640 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e795a06e0 0x559e795ab380 crc :-1 s=READY pgs=11 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=45586d4a4b7dc31e server_cookie=ab6c06d2d4baaf05 in_seq=0 out_seq=0 2024-03-04T07:29:20.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89d0ff9640 1 -- 172.21.15.64:0/2733136653 <== mds.0 v2:172.21.15.64:6836/1811282816 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x559e794b8bf0 con 0x559e795a06e0 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89d0ff9640 10 client.6225 handle_command_reply: tid=0 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 client.6225 shutdown 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 msgr2=0x559e79564a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79564a20 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7f89e802c390 tx=0x7f89e802e070 comp rx=0 tx=0).stop 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 shutdown_connections 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e795a06e0 0x559e795ab380 crc :-1 s=READY pgs=11 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79564a20 unknown :-1 s=CLOSED pgs=689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e7956d190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89d0ff9640 0 client.6225 ms_handle_reset on v2:172.21.15.64:6836/1811282816 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 --2- 172.21.15.64:0/2733136653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79563690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.713+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 >> 172.21.15.64:0/2733136653 conn(0x559e79541870 msgr2=0x559e7956f5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.714+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 shutdown_connections 2024-03-04T07:29:20.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.714+0000 7f89fac2a740 1 -- 172.21.15.64:0/2733136653 wait complete. 2024-03-04T07:29:20.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.714+0000 7f89fac2a740 20 client.6225 trim_cache size 0 max 16384 2024-03-04T07:29:20.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 Processor -- start 2024-03-04T07:29:20.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 -- start start 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79566ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e795719e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79571f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89f01fff20 con 0x559e795645a0 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89f01ff060 con 0x559e79571600 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.731+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89f01fe460 con 0x559e795651c0 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79566ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79566ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39402/0 (socket says 172.21.15.64:39402) 2024-03-04T07:29:20.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 -- 172.21.15.64:0/1534688924 learned_addr learned my addr 172.21.15.64:0/1534688924 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f65e3640 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79571f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f55e1640 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e795719e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 -- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 msgr2=0x559e795719e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e795719e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 -- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 msgr2=0x559e79571f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79571f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 -- 172.21.15.64:0/1534688924 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559e7956d500 con 0x559e795645a0 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f65e3640 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79571f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89f5de2640 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79566ba0 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7f89ec057ba0 tx=0x7f89ec0541c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=56cf211715cfb54a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89d2ffd640 1 -- 172.21.15.64:0/1534688924 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89ec06be60 con 0x559e795645a0 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89d2ffd640 1 -- 172.21.15.64:0/1534688924 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89ec086070 con 0x559e795645a0 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.732+0000 7f89d2ffd640 1 -- 172.21.15.64:0/1534688924 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89ec0561a0 con 0x559e795645a0 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 -- 172.21.15.64:0/1534688924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 msgr2=0x559e79566ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79566ba0 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7f89ec057ba0 tx=0x7f89ec0541c0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 -- 172.21.15.64:0/1534688924 shutdown_connections 2024-03-04T07:29:20.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e79571600 0x559e79571f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e795651c0 0x559e795719e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 --2- 172.21.15.64:0/1534688924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795645a0 0x559e79566ba0 unknown :-1 s=CLOSED pgs=690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 -- 172.21.15.64:0/1534688924 >> 172.21.15.64:0/1534688924 conn(0x559e79541870 msgr2=0x559e795634a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 -- 172.21.15.64:0/1534688924 shutdown_connections 2024-03-04T07:29:20.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 -- 172.21.15.64:0/1534688924 wait complete. 2024-03-04T07:29:20.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 -- start start 2024-03-04T07:29:20.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.733+0000 7f89fac2a740 1 Processor -- start 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e795645a0 0x559e7956bbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795651c0 0x559e795fc290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89fac2a740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e79571600 0x559e79547130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89fac2a740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e7956e1a0 con 0x559e795651c0 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89f01fea60 con 0x559e795645a0 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89fac2a740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x559e7956dec0 con 0x559e79571600 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89fac2a740 10 client.0 _ll_register_callbacks cb 0x7f89e802c970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89f55e1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795651c0 0x559e795fc290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89f55e1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795651c0 0x559e795fc290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39412/0 (socket says 172.21.15.64:39412) 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89f55e1640 1 -- 172.21.15.64:0/3433679491 learned_addr learned my addr 172.21.15.64:0/3433679491 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.734+0000 7f89f5de2640 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e795645a0 0x559e7956bbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89f65e3640 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e79571600 0x559e79547130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89f55e1640 1 -- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e79571600 msgr2=0x559e79547130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89f55e1640 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e79571600 0x559e79547130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89f55e1640 1 -- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e795645a0 msgr2=0x559e7956bbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89f55e1640 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e795645a0 0x559e7956bbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89f55e1640 1 -- 172.21.15.64:0/3433679491 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x559e7928a6b0 con 0x559e795651c0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89f55e1640 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795651c0 0x559e795fc290 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x559e79543670 tx=0x7f89e400f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc8da5529c8e63a8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89d0ff9640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89d0ff9640 1 -- 172.21.15.64:0/3433679491 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89e4028060 con 0x559e795651c0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89d0ff9640 1 -- 172.21.15.64:0/3433679491 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f89e4023ac0 con 0x559e795651c0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f89ec086070 con 0x559e795651c0 2024-03-04T07:29:20.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89d0ff9640 1 -- 172.21.15.64:0/3433679491 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) v2 ==== 24+0+0 (secure 0 0 0) 0x7f89e4023cc0 con 0x559e795651c0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89fac2a740 10 client.6231 fetch_fsmap learned FSMap version 29 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=2+}) v3 -- 0x559e79547670 con 0x559e795651c0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89d0ff9640 1 -- 172.21.15.64:0/3433679491 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89ec086070 con 0x559e795651c0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.735+0000 7f89d0ff9640 1 -- 172.21.15.64:0/3433679491 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 29) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f89e40035f0 con 0x559e795651c0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 10 client.6231 fetch_fsmap finished waiting for FSMap version 29 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 10 client.6231 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 20 client.6231 populate_metadata read hostname 'smithi064' 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e7959eba0 0x559e79540510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 4 client.6231 mds_command: new command op to 6069 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 --> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] -- command(tid 0: {"prefix": "status"}) v1 -- 0x559e79544050 con 0x559e7959eba0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89f5de2640 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e7959eba0 0x559e79540510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89d0ff9640 10 client.6231 ms_handle_connect on v2:172.21.15.64:6836/1811282816 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89f5de2640 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e7959eba0 0x559e79540510 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=49991ad91f001fd1 server_cookie=6862414b73d235a4 in_seq=0 out_seq=0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89d0ff9640 1 -- 172.21.15.64:0/3433679491 <== mds.0 v2:172.21.15.64:6836/1811282816 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x559e79544050 con 0x559e7959eba0 2024-03-04T07:29:20.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89d0ff9640 10 client.6231 handle_command_reply: tid=0 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 client.6231 shutdown 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795651c0 msgr2=0x559e795fc290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795651c0 0x559e795fc290 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x559e79543670 tx=0x7f89e400f000 comp rx=0 tx=0).stop 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 shutdown_connections 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559e79571600 0x559e79547130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:6836/1811282816,v1:172.21.15.64:6837/1811282816] conn(0x559e7959eba0 0x559e79540510 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559e795651c0 0x559e795fc290 unknown :-1 s=CLOSED pgs=691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 --2- 172.21.15.64:0/3433679491 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559e795645a0 0x559e7956bbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89d0ff9640 0 client.6231 ms_handle_reset on v2:172.21.15.64:6836/1811282816 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.736+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 >> 172.21.15.64:0/3433679491 conn(0x559e79541870 msgr2=0x559e79545080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 shutdown_connections 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89fac2a740 1 -- 172.21.15.64:0/3433679491 wait complete. 2024-03-04T07:29:20.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89fac2a740 20 client.6231 trim_cache size 0 max 16384 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 6069, 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 9.2333155219999998, 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 29, 2024-03-04T07:29:20.744 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 55, 2024-03-04T07:29:20.745 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 55, 2024-03-04T07:29:20.745 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 14.251474249999999 2024-03-04T07:29:20.745 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89d406b4d0 msgr2=0x7f89d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89d406b4d0 0x7f89d406d990 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f89e4000f60 tx=0x7f89e40079b0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89f006b5c0 msgr2=0x7f89f01ddd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89f006b5c0 0x7f89f01ddd90 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f89ec00c760 tx=0x7f89ec00cc20 comp rx=0 tx=0).stop 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 shutdown_connections 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89d406b4d0 0x7f89d406d990 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89f00b0e90 0x7f89f00b1350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89f00ae9b0 0x7f89f00b6f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 --2- 172.21.15.64:0/2327389427 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89f006b5c0 0x7f89f01ddd90 unknown :-1 s=CLOSED pgs=687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 >> 172.21.15.64:0/2327389427 conn(0x7f89f00b36d0 msgr2=0x7f89f00b2100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 shutdown_connections 2024-03-04T07:29:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.737+0000 7f89f8046640 1 -- 172.21.15.64:0/2327389427 wait complete. 2024-03-04T07:29:20.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:29:20.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.850+0000 7fb2c0229640 1 Processor -- start 2024-03-04T07:29:20.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.850+0000 7fb2c0229640 1 -- start start 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.850+0000 7fb2c0229640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb2b80b95a0 0x7fb2b80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2c0229640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb2b80cea30 con 0x7fb2b815df00 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2c0229640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb2b80cf3c0 con 0x7fb2b80b9a10 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2c0229640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb2b80cf0e0 con 0x7fb2b80b95a0 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2bd7c4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb2b80b95a0 0x7fb2b80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2bd7c4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb2b80b95a0 0x7fb2b80b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39428/0 (socket says 172.21.15.64:39428) 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2bd7c4640 1 -- 172.21.15.64:0/3586871877 learned_addr learned my addr 172.21.15.64:0/3586871877 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2bd7c4640 1 -- 172.21.15.64:0/3586871877 >> v1:172.21.15.66:6789/0 conn(0x7fb2b80b9a10 legacy=0x7fb2b80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2bd7c4640 1 -- 172.21.15.64:0/3586871877 >> v1:172.21.15.64:6789/0 conn(0x7fb2b815df00 legacy=0x7fb2b80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2bd7c4640 1 -- 172.21.15.64:0/3586871877 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2b80ce470 con 0x7fb2b80b95a0 2024-03-04T07:29:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.851+0000 7fb2bcfc3640 1 -- 172.21.15.64:0/3586871877 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 301792574 0 0) 0x7fb2b80cea30 con 0x7fb2b815df00 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2bd7c4640 1 --2- 172.21.15.64:0/3586871877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b80b95a0 0x7fb2b80b5700 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7fb2b80cf590 tx=0x7fb2a802ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=500f1696dcf0bc8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2bcfc3640 1 -- 172.21.15.64:0/3586871877 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2a802f840 con 0x7fb2b80b95a0 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2bcfc3640 1 -- 172.21.15.64:0/3586871877 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb2a802f9e0 con 0x7fb2b80b95a0 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2bcfc3640 1 -- 172.21.15.64:0/3586871877 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2a802fdb0 con 0x7fb2b80b95a0 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2c0229640 1 -- 172.21.15.64:0/3586871877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b80b95a0 msgr2=0x7fb2b80b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2c0229640 1 --2- 172.21.15.64:0/3586871877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b80b95a0 0x7fb2b80b5700 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7fb2b80cf590 tx=0x7fb2a802ba20 comp rx=0 tx=0).stop 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2c0229640 1 -- 172.21.15.64:0/3586871877 shutdown_connections 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2c0229640 1 --2- 172.21.15.64:0/3586871877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b80b95a0 0x7fb2b80b5700 unknown :-1 s=CLOSED pgs=693 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.852+0000 7fb2c0229640 1 -- 172.21.15.64:0/3586871877 >> 172.21.15.64:0/3586871877 conn(0x7fb2b8071960 msgr2=0x7fb2b8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.853+0000 7fb2c0229640 1 -- 172.21.15.64:0/3586871877 shutdown_connections 2024-03-04T07:29:20.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.853+0000 7fb2c0229640 1 -- 172.21.15.64:0/3586871877 wait complete. 2024-03-04T07:29:20.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 Processor -- start 2024-03-04T07:29:20.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 -- start start 2024-03-04T07:29:20.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2b80b95a0 0x7fb2b8159f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb2b80b9a10 0x7fb2b815a460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b815df00 0x7fb2b81f6c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2b80cedf0 con 0x7fb2b815df00 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2b80cec70 con 0x7fb2b80b9a10 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.854+0000 7fb2c0229640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb2b80b5c40 con 0x7fb2b80b95a0 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b815df00 0x7fb2b81f6c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b815df00 0x7fb2b81f6c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39444/0 (socket says 172.21.15.64:39444) 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 -- 172.21.15.64:0/1047202272 learned_addr learned my addr 172.21.15.64:0/1047202272 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2bdfc5640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2b80b95a0 0x7fb2b8159f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2bd7c4640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb2b80b9a10 0x7fb2b815a460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 -- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2b80b95a0 msgr2=0x7fb2b8159f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2b80b95a0 0x7fb2b8159f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 -- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb2b80b9a10 msgr2=0x7fb2b815a460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb2b80b9a10 0x7fb2b815a460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 -- 172.21.15.64:0/1047202272 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2b81f7910 con 0x7fb2b815df00 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2bdfc5640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2b80b95a0 0x7fb2b8159f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:20.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2bd7c4640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb2b80b9a10 0x7fb2b815a460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2be7c6640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b815df00 0x7fb2b81f6c70 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7fb2b0001eb0 tx=0x7fb2b000c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2ae7fc640 1 -- 172.21.15.64:0/1047202272 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2b0017070 con 0x7fb2b815df00 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2ae7fc640 1 -- 172.21.15.64:0/1047202272 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb2b00052d0 con 0x7fb2b815df00 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb2b80cc390 con 0x7fb2b815df00 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2ae7fc640 1 -- 172.21.15.64:0/1047202272 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2b0003810 con 0x7fb2b815df00 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.855+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb2b80cc950 con 0x7fb2b815df00 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.856+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb2b8067ff0 con 0x7fb2b815df00 2024-03-04T07:29:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.857+0000 7fb2ae7fc640 1 -- 172.21.15.64:0/1047202272 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb2b001a070 con 0x7fb2b815df00 2024-03-04T07:29:20.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.857+0000 7fb2ae7fc640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb29c06b500 0x7fb29c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:20.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.857+0000 7fb2bdfc5640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb29c06b500 0x7fb29c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:20.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.857+0000 7fb2ae7fc640 1 -- 172.21.15.64:0/1047202272 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb2b0089580 con 0x7fb2b815df00 2024-03-04T07:29:20.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.858+0000 7fb2bdfc5640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb29c06b500 0x7fb29c06d9c0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fb2b4007fd0 tx=0x7fb2b4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:20.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:20.861+0000 7fb2ae7fc640 1 -- 172.21.15.64:0/1047202272 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb2b0058400 con 0x7fb2b815df00 2024-03-04T07:29:21.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.079+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fb2b80c7530 con 0x7fb2b815df00 2024-03-04T07:29:21.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.079+0000 7fb2ae7fc640 1 -- 172.21.15.64:0/1047202272 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v29) v1 ==== 53+0+73 (secure 0 0 0) 0x7fb2b0057d00 con 0x7fb2b815df00 2024-03-04T07:29:21.080 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb29c06b500 msgr2=0x7fb29c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb29c06b500 0x7fb29c06d9c0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fb2b4007fd0 tx=0x7fb2b4009040 comp rx=0 tx=0).stop 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b815df00 msgr2=0x7fb2b81f6c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b815df00 0x7fb2b81f6c70 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7fb2b0001eb0 tx=0x7fb2b000c890 comp rx=0 tx=0).stop 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 shutdown_connections 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb29c06b500 0x7fb29c06d9c0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2b815df00 0x7fb2b81f6c70 unknown :-1 s=CLOSED pgs=694 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb2b80b9a10 0x7fb2b815a460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 --2- 172.21.15.64:0/1047202272 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2b80b95a0 0x7fb2b8159f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 >> 172.21.15.64:0/1047202272 conn(0x7fb2b8071960 msgr2=0x7fb2b80b6d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 shutdown_connections 2024-03-04T07:29:21.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21.081+0000 7fb2c0229640 1 -- 172.21.15.64:0/1047202272 wait complete. 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:29:21.093 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:29:21.093 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.093 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:29:21.108 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:29:21.108 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:29:21.108 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:29:21.109 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42551sec preferred_lft 42551sec 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:29:21.110 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:21.111 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:29:21.111 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:21.111 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:29:21.111 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:21.111 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.111 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:29:21.175 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:29:21.177 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.177 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:29:21.232 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:29:21.232 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:29:21.232 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:29:21.232 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:29:21.233 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:29:21.233 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.233 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:29:21.233 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:29:21.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:29:21.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:29:21.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:29:21.233 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:29:21.327 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.328 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:29:21.342 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:29:21.342 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:29:21.342 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:29:21.343 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:29:21.411 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:29:21.411 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:29:21.411 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:21.468 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:29:21.468 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:29:21.469 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:21.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:29:21.592 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:29:21.647 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:29:21.648 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:29:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:21.650 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.650 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:29:21.702 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:29:21.703 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:29:21.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:29:21.771 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:29:21.827 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:29:21.828 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:21.829 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.829 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:29:21.837 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:29:21.836+0000 7fae6560d480 -1 init, newargv = 0x564c87b4f4e0 newargc=15 2024-03-04T07:29:21.837 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[58959]: starting ceph client 2024-03-04T07:29:21.841 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[58959]: starting fuse 2024-03-04T07:29:21.851 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:29:21.852 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:29:21.852 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:29:21.853 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:29:21.854 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:21.854 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:29:21.854 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:29:21.971 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.58959.asok 2024-03-04T07:29:21.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:21 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:29:21.982 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.58959.asok 2024-03-04T07:29:21.983 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:21.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.58959.asok status 2024-03-04T07:29:22.123 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:29:22.123 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:29:22.123 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:29:22.123 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:29:22.123 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:29:22.123 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "58959", 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 6246, 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 6246 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:29:22.124 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 5863542 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 5863542 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.6246 192.168.0.1:0/5863542", 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/5863542", 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 29, 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 55, 2024-03-04T07:29:22.125 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:29:22.126 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:29:22.126 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:29:22.126 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:29:22.132 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:22.132 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:29:22.132 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:22.147 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:29:22.147 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:22.148 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:22.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:22.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_fragment_size_max 50 2024-03-04T07:29:22.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 Processor -- start 2024-03-04T07:29:22.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 -- start start 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fac241456e0 0x7fac240ceec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac24067980 0x7fac24066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fac2406f270 0x7fac24067240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fac2415d160 con 0x7fac2406f270 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fac240cd1f0 con 0x7fac241456e0 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.359+0000 7fac2a0d4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fac240cd4d0 con 0x7fac24067980 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fac2406f270 0x7fac24067240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fac2406f270 0x7fac24067240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39448/0 (socket says 172.21.15.64:39448) 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 -- 172.21.15.64:0/3737649450 learned_addr learned my addr 172.21.15.64:0/3737649450 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac237fe640 1 --2- 172.21.15.64:0/3737649450 >> v2:172.21.15.66:3300/0 conn(0x7fac241456e0 0x7fac240ceec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac22ffd640 1 --2- 172.21.15.64:0/3737649450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac24067980 0x7fac24066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 -- 172.21.15.64:0/3737649450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac24067980 msgr2=0x7fac24066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 --2- 172.21.15.64:0/3737649450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac24067980 0x7fac24066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 -- 172.21.15.64:0/3737649450 >> v2:172.21.15.66:3300/0 conn(0x7fac241456e0 msgr2=0x7fac240ceec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 --2- 172.21.15.64:0/3737649450 >> v2:172.21.15.66:3300/0 conn(0x7fac241456e0 0x7fac240ceec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac23fff640 1 -- 172.21.15.64:0/3737649450 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac240cd7b0 con 0x7fac2406f270 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.360+0000 7fac237fe640 1 --2- 172.21.15.64:0/3737649450 >> v2:172.21.15.66:3300/0 conn(0x7fac241456e0 0x7fac240ceec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac23fff640 1 --2- 172.21.15.64:0/3737649450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac2406f270 0x7fac24067240 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7fac2406f9f0 tx=0x7fac1c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=97dcec8c72eaad6e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac227fc640 1 -- 172.21.15.64:0/3737649450 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac1c02ca70 con 0x7fac2406f270 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac227fc640 1 -- 172.21.15.64:0/3737649450 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fac1c02cc10 con 0x7fac2406f270 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac227fc640 1 -- 172.21.15.64:0/3737649450 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac1c0343f0 con 0x7fac2406f270 2024-03-04T07:29:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 -- 172.21.15.64:0/3737649450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac2406f270 msgr2=0x7fac24067240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/3737649450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac2406f270 0x7fac24067240 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7fac2406f9f0 tx=0x7fac1c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 -- 172.21.15.64:0/3737649450 shutdown_connections 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/3737649450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac2406f270 0x7fac24067240 unknown :-1 s=CLOSED pgs=698 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/3737649450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac24067980 0x7fac24066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/3737649450 >> v2:172.21.15.66:3300/0 conn(0x7fac241456e0 0x7fac240ceec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 -- 172.21.15.64:0/3737649450 >> 172.21.15.64:0/3737649450 conn(0x7fac240b2720 msgr2=0x7fac240b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 -- 172.21.15.64:0/3737649450 shutdown_connections 2024-03-04T07:29:22.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.361+0000 7fac2a0d4640 1 -- 172.21.15.64:0/3737649450 wait complete. 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.362+0000 7fac2a0d4640 1 Processor -- start 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.363+0000 7fac2a0d4640 1 -- start start 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.363+0000 7fac2a0d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac24067980 0x7fac2415b570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.363+0000 7fac2a0d4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac2406f270 0x7fac24142a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.363+0000 7fac2a0d4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac241456e0 0x7fac240a95c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.363+0000 7fac2a0d4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac2415cee0 con 0x7fac24067980 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.363+0000 7fac2a0d4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac2415cd60 con 0x7fac2406f270 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.363+0000 7fac2a0d4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fac24071d20 con 0x7fac241456e0 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac24067980 0x7fac2415b570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac24067980 0x7fac2415b570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39464/0 (socket says 172.21.15.64:39464) 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 -- 172.21.15.64:0/1781667501 learned_addr learned my addr 172.21.15.64:0/1781667501 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac22ffd640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac2406f270 0x7fac24142a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac23fff640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac241456e0 0x7fac240a95c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 -- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac241456e0 msgr2=0x7fac240a95c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac241456e0 0x7fac240a95c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 -- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac2406f270 msgr2=0x7fac24142a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac2406f270 0x7fac24142a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 -- 172.21.15.64:0/1781667501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac24143440 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac237fe640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac24067980 0x7fac2415b570 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7fac1400d7d0 tx=0x7fac1400dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac0bfff640 1 -- 172.21.15.64:0/1781667501 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac14004030 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac0bfff640 1 -- 172.21.15.64:0/1781667501 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fac140041d0 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fac24143720 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac0bfff640 1 -- 172.21.15.64:0/1781667501 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac14009640 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.364+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fac241f4780 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.365+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fac240733a0 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.366+0000 7fac0bfff640 1 -- 172.21.15.64:0/1781667501 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fac140097e0 con 0x7fac24067980 2024-03-04T07:29:22.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.366+0000 7fac0bfff640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac1006b4d0 0x7fac1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.366+0000 7fac0bfff640 1 -- 172.21.15.64:0/1781667501 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fac1408a2c0 con 0x7fac24067980 2024-03-04T07:29:22.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.366+0000 7fac22ffd640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac1006b4d0 0x7fac1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.367+0000 7fac22ffd640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac1006b4d0 0x7fac1006d990 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7fac241e7070 tx=0x7fac18008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:22.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.370+0000 7fac0bfff640 1 -- 172.21.15.64:0/1781667501 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fac14058af0 con 0x7fac24067980 2024-03-04T07:29:22.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.568+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mds_bal_fragment_size_max}] v 0) v1 -- 0x7fac2404f410 con 0x7fac24067980 2024-03-04T07:29:22.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.570+0000 7fac0bfff640 1 -- 172.21.15.64:0/1781667501 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_fragment_size_max}]=0 v17)=0 v17) v1 ==== 124+0+0 (secure 0 0 0) 0x7fac1405cb90 con 0x7fac24067980 2024-03-04T07:29:22.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac1006b4d0 msgr2=0x7fac1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac1006b4d0 0x7fac1006d990 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7fac241e7070 tx=0x7fac18008040 comp rx=0 tx=0).stop 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac24067980 msgr2=0x7fac2415b570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac24067980 0x7fac2415b570 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7fac1400d7d0 tx=0x7fac1400dc90 comp rx=0 tx=0).stop 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 shutdown_connections 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac1006b4d0 0x7fac1006d990 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac241456e0 0x7fac240a95c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac2406f270 0x7fac24142a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 --2- 172.21.15.64:0/1781667501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac24067980 0x7fac2415b570 unknown :-1 s=CLOSED pgs=699 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 >> 172.21.15.64:0/1781667501 conn(0x7fac240b2720 msgr2=0x7fac24066d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 shutdown_connections 2024-03-04T07:29:22.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.573+0000 7fac2a0d4640 1 -- 172.21.15.64:0/1781667501 wait complete. 2024-03-04T07:29:22.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_merge_size 1 2024-03-04T07:29:22.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.688+0000 7fd79ac47640 1 Processor -- start 2024-03-04T07:29:22.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.688+0000 7fd79ac47640 1 -- start start 2024-03-04T07:29:22.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd79ac47640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd79415dd90 0x7fd7940ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd79ac47640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd7940d29d0 0x7fd7940d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd79ac47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940b6d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd79ac47640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7940d3400 con 0x7fd7940d29d0 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd79ac47640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7940d3740 con 0x7fd79415dd90 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd79ac47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7940d3120 con 0x7fd7940d4720 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd7940d29d0 0x7fd7940d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd7940d29d0 0x7fd7940d41e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39468/0 (socket says 172.21.15.64:39468) 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 -- 172.21.15.64:0/945415038 learned_addr learned my addr 172.21.15.64:0/945415038 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd7989e3640 1 --2- 172.21.15.64:0/945415038 >> v2:172.21.15.66:3300/0 conn(0x7fd79415dd90 0x7fd7940ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd7991e4640 1 --2- 172.21.15.64:0/945415038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940b6d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 -- 172.21.15.64:0/945415038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 msgr2=0x7fd7940b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 --2- 172.21.15.64:0/945415038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940b6d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 -- 172.21.15.64:0/945415038 >> v2:172.21.15.66:3300/0 conn(0x7fd79415dd90 msgr2=0x7fd7940ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:22.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 --2- 172.21.15.64:0/945415038 >> v2:172.21.15.66:3300/0 conn(0x7fd79415dd90 0x7fd7940ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.689+0000 7fd793fff640 1 -- 172.21.15.64:0/945415038 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7940d39e0 con 0x7fd7940d29d0 2024-03-04T07:29:22.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.690+0000 7fd793fff640 1 --2- 172.21.15.64:0/945415038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7940d29d0 0x7fd7940d41e0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7fd7940d1f90 tx=0x7fd78002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=108be94660d0d481 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:22.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.690+0000 7fd7937fe640 1 -- 172.21.15.64:0/945415038 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd780034070 con 0x7fd7940d29d0 2024-03-04T07:29:22.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.690+0000 7fd7937fe640 1 -- 172.21.15.64:0/945415038 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd780030860 con 0x7fd7940d29d0 2024-03-04T07:29:22.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.690+0000 7fd7937fe640 1 -- 172.21.15.64:0/945415038 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd780030c30 con 0x7fd7940d29d0 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 -- 172.21.15.64:0/945415038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7940d29d0 msgr2=0x7fd7940d41e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 --2- 172.21.15.64:0/945415038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7940d29d0 0x7fd7940d41e0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7fd7940d1f90 tx=0x7fd78002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 -- 172.21.15.64:0/945415038 shutdown_connections 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 --2- 172.21.15.64:0/945415038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940b6d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 --2- 172.21.15.64:0/945415038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7940d29d0 0x7fd7940d41e0 unknown :-1 s=CLOSED pgs=700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 --2- 172.21.15.64:0/945415038 >> v2:172.21.15.66:3300/0 conn(0x7fd79415dd90 0x7fd7940ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 -- 172.21.15.64:0/945415038 >> 172.21.15.64:0/945415038 conn(0x7fd794071960 msgr2=0x7fd794071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 -- 172.21.15.64:0/945415038 shutdown_connections 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.691+0000 7fd79ac47640 1 -- 172.21.15.64:0/945415038 wait complete. 2024-03-04T07:29:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.692+0000 7fd79ac47640 1 Processor -- start 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.692+0000 7fd79ac47640 1 -- start start 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd79ac47640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7940d29d0 0x7fd7941569d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd79ac47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940cb660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd79ac47640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd79415dd90 0x7fd7940b8420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd79ac47640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7940d2df0 con 0x7fd79415dd90 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd79ac47640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd79415e170 con 0x7fd7940d29d0 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd79ac47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7940b7250 con 0x7fd7940d4720 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd79415dd90 0x7fd7940b8420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd79415dd90 0x7fd7940b8420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:39480/0 (socket says 172.21.15.64:39480) 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 -- 172.21.15.64:0/447268992 learned_addr learned my addr 172.21.15.64:0/447268992 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7989e3640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7940d29d0 0x7fd7941569d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd793fff640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940cb660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 -- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 msgr2=0x7fd7940cb660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940cb660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 -- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7940d29d0 msgr2=0x7fd7941569d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7940d29d0 0x7fd7941569d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 -- 172.21.15.64:0/447268992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd79415f550 con 0x7fd79415dd90 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.693+0000 7fd7991e4640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd79415dd90 0x7fd7940b8420 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7fd78c00ccf0 tx=0x7fd78c00a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.694+0000 7fd7917fa640 1 -- 172.21.15.64:0/447268992 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd78c003b90 con 0x7fd79415dd90 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.694+0000 7fd7917fa640 1 -- 172.21.15.64:0/447268992 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd78c003d30 con 0x7fd79415dd90 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.694+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd79415f830 con 0x7fd79415dd90 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.694+0000 7fd7917fa640 1 -- 172.21.15.64:0/447268992 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd78c004240 con 0x7fd79415dd90 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.694+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd79415fdf0 con 0x7fd79415dd90 2024-03-04T07:29:22.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.695+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd794072a30 con 0x7fd79415dd90 2024-03-04T07:29:22.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.695+0000 7fd7917fa640 1 -- 172.21.15.64:0/447268992 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd78c0043e0 con 0x7fd79415dd90 2024-03-04T07:29:22.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.695+0000 7fd7917fa640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd77806b4d0 0x7fd77806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:22.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.696+0000 7fd7917fa640 1 -- 172.21.15.64:0/447268992 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd78c08ab90 con 0x7fd79415dd90 2024-03-04T07:29:22.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.696+0000 7fd7989e3640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd77806b4d0 0x7fd77806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.696+0000 7fd7989e3640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd77806b4d0 0x7fd77806d990 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7fd7880079a0 tx=0x7fd788008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.700+0000 7fd7917fa640 1 -- 172.21.15.64:0/447268992 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd78c059330 con 0x7fd79415dd90 2024-03-04T07:29:22.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.881+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mds_bal_merge_size}] v 0) v1 -- 0x7fd79404f410 con 0x7fd79415dd90 2024-03-04T07:29:22.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.884+0000 7fd7917fa640 1 -- 172.21.15.64:0/447268992 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_merge_size}]=0 v18)=0 v18) v1 ==== 116+0+0 (secure 0 0 0) 0x7fd78c05d3d0 con 0x7fd79415dd90 2024-03-04T07:29:22.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.887+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd77806b4d0 msgr2=0x7fd77806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.887+0000 7fd79ac47640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd77806b4d0 0x7fd77806d990 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7fd7880079a0 tx=0x7fd788008040 comp rx=0 tx=0).stop 2024-03-04T07:29:22.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.887+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd79415dd90 msgr2=0x7fd7940b8420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.887+0000 7fd79ac47640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd79415dd90 0x7fd7940b8420 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7fd78c00ccf0 tx=0x7fd78c00a660 comp rx=0 tx=0).stop 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 shutdown_connections 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd77806b4d0 0x7fd77806d990 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd79415dd90 0x7fd7940b8420 unknown :-1 s=CLOSED pgs=701 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd7940d4720 0x7fd7940cb660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 --2- 172.21.15.64:0/447268992 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7940d29d0 0x7fd7941569d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 >> 172.21.15.64:0/447268992 conn(0x7fd794071960 msgr2=0x7fd7940ce380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 shutdown_connections 2024-03-04T07:29:22.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:22.888+0000 7fd79ac47640 1 -- 172.21.15.64:0/447268992 wait complete. 2024-03-04T07:29:32.899 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:32.900 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:32.915 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:29:32.915 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:29:32.915 DEBUG:teuthology.orchestra.run.smithi064:> import uuid 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> n = 50 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = False 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> if 0 > 0: 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = True 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> dpath = os.path.dirname(path) 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> fnameprefix = os.path.basename(path) 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> os.makedirs(dpath, exist_ok=True) 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> try: 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:32.916 DEBUG:teuthology.orchestra.run.smithi064:> for i in range(n): 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> with open(fpath, '"'"'w'"'"') as f: 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> f.write(f"{i}") 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> f.flush() 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(f.fileno()) 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> os.unlink(fpath) 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> if create_hard_links: 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> for j in range(0): 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> if True: 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> finally: 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> os.close(dirfd) 2024-03-04T07:29:32.917 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:29:33.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:29:33.136 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:33.136 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:33.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:33.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.332+0000 7fa0eb7e2640 1 Processor -- start 2024-03-04T07:29:33.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.332+0000 7fa0eb7e2640 1 -- start start 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0eb7e2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa0e415df00 0x7fa0e40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0eb7e2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa0e40b95a0 0x7fa0e40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0eb7e2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa0e40cea30 con 0x7fa0e40b9a30 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0eb7e2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa0e40d2af0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0eb7e2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa0e40cf3c0 con 0x7fa0e415df00 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0e8d7d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa0e40b95a0 0x7fa0e40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0e8d7d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa0e40b95a0 0x7fa0e40b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48516/0 (socket says 172.21.15.64:48516) 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0e8d7d640 1 -- 172.21.15.64:0/1284924611 learned_addr learned my addr 172.21.15.64:0/1284924611 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.333+0000 7fa0e957e640 1 --2- 172.21.15.64:0/1284924611 >> v2:172.21.15.66:3300/0 conn(0x7fa0e415df00 0x7fa0e40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.334+0000 7fa0e8d7d640 1 -- 172.21.15.64:0/1284924611 >> v2:172.21.15.66:3300/0 conn(0x7fa0e415df00 msgr2=0x7fa0e40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.334+0000 7fa0e8d7d640 1 --2- 172.21.15.64:0/1284924611 >> v2:172.21.15.66:3300/0 conn(0x7fa0e415df00 0x7fa0e40d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.334+0000 7fa0e8d7d640 1 -- 172.21.15.64:0/1284924611 >> v1:172.21.15.66:6789/0 conn(0x7fa0e40b9a30 legacy=0x7fa0e40d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.334+0000 7fa0e8d7d640 1 -- 172.21.15.64:0/1284924611 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa0e40cf0e0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.334+0000 7fa0e8d7d640 1 --2- 172.21.15.64:0/1284924611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e40b5700 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7fa0e40d11b0 tx=0x7fa0d802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1eb06b13a5ca90ea server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0e3fff640 1 -- 172.21.15.64:0/1284924611 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d8034070 con 0x7fa0e40b95a0 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0e3fff640 1 -- 172.21.15.64:0/1284924611 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa0d8030860 con 0x7fa0e40b95a0 2024-03-04T07:29:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0e3fff640 1 -- 172.21.15.64:0/1284924611 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d8030c30 con 0x7fa0e40b95a0 2024-03-04T07:29:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/1284924611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 msgr2=0x7fa0e40b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/1284924611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e40b5700 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7fa0e40d11b0 tx=0x7fa0d802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/1284924611 shutdown_connections 2024-03-04T07:29:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/1284924611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e40b5700 unknown :-1 s=CLOSED pgs=702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/1284924611 >> v2:172.21.15.66:3300/0 conn(0x7fa0e415df00 0x7fa0e40d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/1284924611 >> 172.21.15.64:0/1284924611 conn(0x7fa0e4071960 msgr2=0x7fa0e4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/1284924611 shutdown_connections 2024-03-04T07:29:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.335+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/1284924611 wait complete. 2024-03-04T07:29:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.336+0000 7fa0eb7e2640 1 Processor -- start 2024-03-04T07:29:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.336+0000 7fa0eb7e2640 1 -- start start 2024-03-04T07:29:33.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0eb7e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e41550a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0eb7e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e415df00 0x7fa0e40cd250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0eb7e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e40cd790 0x7fa0e415f2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0eb7e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e40cec70 con 0x7fa0e40b95a0 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0eb7e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e40ce750 con 0x7fa0e415df00 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0eb7e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e40cedf0 con 0x7fa0e40cd790 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e41550a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e41550a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48526/0 (socket says 172.21.15.64:48526) 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 -- 172.21.15.64:0/4108041999 learned_addr learned my addr 172.21.15.64:0/4108041999 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 -- 172.21.15.64:0/4108041999 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e40cd790 msgr2=0x7fa0e415f2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e40cd790 0x7fa0e415f2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 -- 172.21.15.64:0/4108041999 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e415df00 msgr2=0x7fa0e40cd250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e8d7d640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e415df00 0x7fa0e40cd250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e415df00 0x7fa0e40cd250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.337+0000 7fa0e957e640 1 -- 172.21.15.64:0/4108041999 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa0e4160010 con 0x7fa0e40b95a0 2024-03-04T07:29:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.338+0000 7fa0e957e640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e41550a0 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7fa0d400d600 tx=0x7fa0d400dac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.338+0000 7fa0e1ffb640 1 -- 172.21.15.64:0/4108041999 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d40125f0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.338+0000 7fa0e1ffb640 1 -- 172.21.15.64:0/4108041999 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa0d4019070 con 0x7fa0e40b95a0 2024-03-04T07:29:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.338+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa0e41555e0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.338+0000 7fa0e1ffb640 1 -- 172.21.15.64:0/4108041999 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d40129f0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.338+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa0e40cb0d0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.339+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa0e4067ff0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.340+0000 7fa0e1ffb640 1 -- 172.21.15.64:0/4108041999 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa0d4012b90 con 0x7fa0e40b95a0 2024-03-04T07:29:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.340+0000 7fa0e1ffb640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0d006b500 0x7fa0d006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.340+0000 7fa0e1ffb640 1 -- 172.21.15.64:0/4108041999 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa0d408a670 con 0x7fa0e40b95a0 2024-03-04T07:29:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.340+0000 7fa0e8d7d640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0d006b500 0x7fa0d006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.341+0000 7fa0e8d7d640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0d006b500 0x7fa0d006d9c0 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7fa0d8036000 tx=0x7fa0d803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:33.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.345+0000 7fa0e1ffb640 1 -- 172.21.15.64:0/4108041999 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa0d4059470 con 0x7fa0e40b95a0 2024-03-04T07:29:33.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.527+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa0e40b5ce0 con 0x7fa0e40b95a0 2024-03-04T07:29:33.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.528+0000 7fa0e1ffb640 1 -- 172.21.15.64:0/4108041999 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v55) v1 ==== 74+0+16617 (secure 0 0 0) 0x7fa0d4058d70 con 0x7fa0e40b95a0 2024-03-04T07:29:33.631 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:12.504021+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0d006b500 msgr2=0x7fa0d006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0d006b500 0x7fa0d006d9c0 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7fa0d8036000 tx=0x7fa0d803b040 comp rx=0 tx=0).stop 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 msgr2=0x7fa0e41550a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e41550a0 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7fa0d400d600 tx=0x7fa0d400dac0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 shutdown_connections 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0d006b500 0x7fa0d006d9c0 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e40cd790 0x7fa0e415f2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e415df00 0x7fa0e40cd250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 --2- 172.21.15.64:0/4108041999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e40b95a0 0x7fa0e41550a0 unknown :-1 s=CLOSED pgs=703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.530+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 >> 172.21.15.64:0/4108041999 conn(0x7fa0e4071960 msgr2=0x7fa0e4071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:33.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.531+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 shutdown_connections 2024-03-04T07:29:33.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.531+0000 7fa0eb7e2640 1 -- 172.21.15.64:0/4108041999 wait complete. 2024-03-04T07:29:33.636 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 8, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:29:09.132151+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '54', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 9, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:29:09.825115+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '55', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:29:33.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:29:33.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.741+0000 7fc7570a4640 1 Processor -- start 2024-03-04T07:29:33.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.741+0000 7fc7570a4640 1 -- start start 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc7570a4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc7500ceea0 0x7fc7500d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc7570a4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc7500b6d10 0x7fc7500b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc7570a4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc7500d3740 con 0x7fc75015dd90 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc7570a4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc7500d3400 con 0x7fc7500b6d10 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc7570a4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc7500d3120 con 0x7fc7500ceea0 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc755641640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc7500b6d10 0x7fc7500b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc755641640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc7500b6d10 0x7fc7500b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36786/0 (socket says 172.21.15.64:36786) 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc755641640 1 -- 172.21.15.64:0/2050805051 learned_addr learned my addr 172.21.15.64:0/2050805051 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc755641640 1 -- 172.21.15.64:0/2050805051 >> v2:172.21.15.66:3300/0 conn(0x7fc7500ceea0 msgr2=0x7fc7500d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc755641640 1 --2- 172.21.15.64:0/2050805051 >> v2:172.21.15.66:3300/0 conn(0x7fc7500ceea0 0x7fc7500d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc755641640 1 -- 172.21.15.64:0/2050805051 >> v1:172.21.15.66:6789/0 conn(0x7fc75015dd90 legacy=0x7fc7500d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:33.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.742+0000 7fc755641640 1 -- 172.21.15.64:0/2050805051 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7500d39e0 con 0x7fc7500b6d10 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.743+0000 7fc755641640 1 --2- 172.21.15.64:0/2050805051 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7500b6d10 0x7fc7500b71a0 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7fc7500724d0 tx=0x7fc74802bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cea8000e349a9283 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.743+0000 7fc74f7fe640 1 -- 172.21.15.64:0/2050805051 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc74802cb20 con 0x7fc7500b6d10 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.743+0000 7fc74f7fe640 1 -- 172.21.15.64:0/2050805051 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc74802ccc0 con 0x7fc7500b6d10 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.743+0000 7fc74f7fe640 1 -- 172.21.15.64:0/2050805051 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc748034610 con 0x7fc7500b6d10 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 -- 172.21.15.64:0/2050805051 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7500b6d10 msgr2=0x7fc7500b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 --2- 172.21.15.64:0/2050805051 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7500b6d10 0x7fc7500b71a0 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7fc7500724d0 tx=0x7fc74802bf80 comp rx=0 tx=0).stop 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 -- 172.21.15.64:0/2050805051 shutdown_connections 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 --2- 172.21.15.64:0/2050805051 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7500b6d10 0x7fc7500b71a0 unknown :-1 s=CLOSED pgs=704 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 --2- 172.21.15.64:0/2050805051 >> v2:172.21.15.66:3300/0 conn(0x7fc7500ceea0 0x7fc7500d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 -- 172.21.15.64:0/2050805051 >> 172.21.15.64:0/2050805051 conn(0x7fc750071960 msgr2=0x7fc750071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 -- 172.21.15.64:0/2050805051 shutdown_connections 2024-03-04T07:29:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.744+0000 7fc7570a4640 1 -- 172.21.15.64:0/2050805051 wait complete. 2024-03-04T07:29:33.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.745+0000 7fc7570a4640 1 Processor -- start 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.745+0000 7fc7570a4640 1 -- start start 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc7570a4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7500b6d10 0x7fc75014a230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc7570a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7500ceea0 0x7fc75015e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc7570a4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc75015dd90 0x7fc7500c8800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc7570a4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7500d48f0 con 0x7fc75015dd90 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc7570a4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc75015e170 con 0x7fc7500b6d10 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc7570a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc7500d4a70 con 0x7fc7500ceea0 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc75015dd90 0x7fc7500c8800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc75015dd90 0x7fc7500c8800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36798/0 (socket says 172.21.15.64:36798) 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 -- 172.21.15.64:0/4264852529 learned_addr learned my addr 172.21.15.64:0/4264852529 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc754e40640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7500b6d10 0x7fc75014a230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 -- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7500ceea0 msgr2=0x7fc75015e6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7500ceea0 0x7fc75015e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 -- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7500b6d10 msgr2=0x7fc75014a230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7500b6d10 0x7fc75014a230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 -- 172.21.15.64:0/4264852529 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7500cafb0 con 0x7fc75015dd90 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc754e40640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7500b6d10 0x7fc75014a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc755641640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc75015dd90 0x7fc7500c8800 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7fc748036040 tx=0x7fc748004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc74d7fa640 1 -- 172.21.15.64:0/4264852529 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc748042020 con 0x7fc75015dd90 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc74d7fa640 1 -- 172.21.15.64:0/4264852529 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc74803e450 con 0x7fc75015dd90 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc7500cb290 con 0x7fc75015dd90 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.746+0000 7fc74d7fa640 1 -- 172.21.15.64:0/4264852529 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc74803e840 con 0x7fc75015dd90 2024-03-04T07:29:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.747+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc7500cb850 con 0x7fc75015dd90 2024-03-04T07:29:33.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.748+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc750072a30 con 0x7fc75015dd90 2024-03-04T07:29:33.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.748+0000 7fc74d7fa640 1 -- 172.21.15.64:0/4264852529 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc748045020 con 0x7fc75015dd90 2024-03-04T07:29:33.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.748+0000 7fc74d7fa640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc73406b4d0 0x7fc73406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:33.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.749+0000 7fc754e40640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc73406b4d0 0x7fc73406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:33.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.749+0000 7fc74d7fa640 1 -- 172.21.15.64:0/4264852529 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc7480aca80 con 0x7fc75015dd90 2024-03-04T07:29:33.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.749+0000 7fc754e40640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc73406b4d0 0x7fc73406d990 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7fc74400a810 tx=0x7fc744008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:33.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.753+0000 7fc74d7fa640 1 -- 172.21.15.64:0/4264852529 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc74804c030 con 0x7fc75015dd90 2024-03-04T07:29:33.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.937+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fc7500c7570 con 0x7fc75015dd90 2024-03-04T07:29:33.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.937+0000 7fc74d7fa640 1 -- 172.21.15.64:0/4264852529 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v55) v1 ==== 93+0+10 (secure 0 0 0) 0x7fc74807b900 con 0x7fc75015dd90 2024-03-04T07:29:33.937 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:29:33.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.939+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc73406b4d0 msgr2=0x7fc73406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.939+0000 7fc7570a4640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc73406b4d0 0x7fc73406d990 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7fc74400a810 tx=0x7fc744008040 comp rx=0 tx=0).stop 2024-03-04T07:29:33.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.939+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc75015dd90 msgr2=0x7fc7500c8800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.939+0000 7fc7570a4640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc75015dd90 0x7fc7500c8800 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7fc748036040 tx=0x7fc748004a40 comp rx=0 tx=0).stop 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.939+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 shutdown_connections 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.940+0000 7fc7570a4640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc73406b4d0 0x7fc73406d990 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.940+0000 7fc7570a4640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc75015dd90 0x7fc7500c8800 unknown :-1 s=CLOSED pgs=705 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.940+0000 7fc7570a4640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc7500ceea0 0x7fc75015e6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.940+0000 7fc7570a4640 1 --2- 172.21.15.64:0/4264852529 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7500b6d10 0x7fc75014a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.940+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 >> 172.21.15.64:0/4264852529 conn(0x7fc750071960 msgr2=0x7fc7500b7320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.940+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 shutdown_connections 2024-03-04T07:29:33.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:33.940+0000 7fc7570a4640 1 -- 172.21.15.64:0/4264852529 wait complete. 2024-03-04T07:29:33.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:29:34.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.058+0000 7f4a7251e640 1 Processor -- start 2024-03-04T07:29:34.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.058+0000 7f4a7251e640 1 -- start start 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.058+0000 7f4a7251e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4a6c15dbd0 0x7f4a6c15ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a7251e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4a6c15f3b0 con 0x7f4a6c0b5840 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a7251e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4a6c06f070 con 0x7f4a6c0b40c0 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a7251e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4a6c071940 con 0x7f4a6c15dbd0 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a6bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4a6c15dbd0 0x7f4a6c15ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a6b7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4a6c0b5840 0x7f4a6c0b39b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34514/0 (socket says 172.21.15.64:34514) 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a6b7fe640 1 -- 172.21.15.64:0/666785445 learned_addr learned my addr 172.21.15.64:0/666785445 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a6bfff640 1 -- 172.21.15.64:0/666785445 >> v1:172.21.15.66:6789/0 conn(0x7f4a6c0b40c0 legacy=0x7f4a6c066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a6bfff640 1 -- 172.21.15.64:0/666785445 >> v1:172.21.15.64:6789/0 conn(0x7f4a6c0b5840 legacy=0x7f4a6c0b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a6affd640 1 -- 172.21.15.64:0/666785445 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4025593992 0 0) 0x7f4a6c15f3b0 con 0x7f4a6c0b5840 2024-03-04T07:29:34.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.059+0000 7f4a6bfff640 1 -- 172.21.15.64:0/666785445 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4a6c0b5db0 con 0x7f4a6c15dbd0 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a6bfff640 1 --2- 172.21.15.64:0/666785445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c15ee70 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f4a6c06f240 tx=0x7f4a5c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fc4f6cf693bbb958 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a6affd640 1 -- 172.21.15.64:0/666785445 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a5c035020 con 0x7f4a6c15dbd0 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a6affd640 1 -- 172.21.15.64:0/666785445 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4a5c031830 con 0x7f4a6c15dbd0 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a6affd640 1 -- 172.21.15.64:0/666785445 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a5c031c00 con 0x7f4a6c15dbd0 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a7251e640 1 -- 172.21.15.64:0/666785445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c15dbd0 msgr2=0x7f4a6c15ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a7251e640 1 --2- 172.21.15.64:0/666785445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c15ee70 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f4a6c06f240 tx=0x7f4a5c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a7251e640 1 -- 172.21.15.64:0/666785445 shutdown_connections 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a7251e640 1 --2- 172.21.15.64:0/666785445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c15ee70 unknown :-1 s=CLOSED pgs=707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a7251e640 1 -- 172.21.15.64:0/666785445 >> 172.21.15.64:0/666785445 conn(0x7f4a6c0b26b0 msgr2=0x7f4a6c0b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:34.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.060+0000 7f4a7251e640 1 -- 172.21.15.64:0/666785445 shutdown_connections 2024-03-04T07:29:34.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.061+0000 7f4a7251e640 1 -- 172.21.15.64:0/666785445 wait complete. 2024-03-04T07:29:34.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.062+0000 7f4a7251e640 1 Processor -- start 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.062+0000 7f4a7251e640 1 -- start start 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.062+0000 7f4a7251e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a6c0b40c0 0x7f4a6c0c8140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a7251e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c0b5840 0x7f4a6c14f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a7251e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c14f640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a7251e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a6c0721c0 con 0x7f4a6c0b5840 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a7251e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a6c071ee0 con 0x7f4a6c15dbd0 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a7251e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4a6c0726e0 con 0x7f4a6c0b40c0 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c0b5840 0x7f4a6c14f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c0b5840 0x7f4a6c14f100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36814/0 (socket says 172.21.15.64:36814) 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 -- 172.21.15.64:0/1920007658 learned_addr learned my addr 172.21.15.64:0/1920007658 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6bfff640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a6c0b40c0 0x7f4a6c0c8140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a70abb640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c14f640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 -- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a6c0b40c0 msgr2=0x7f4a6c0c8140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a6c0b40c0 0x7f4a6c0c8140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 -- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a6c15dbd0 msgr2=0x7f4a6c14f640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c14f640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 -- 172.21.15.64:0/1920007658 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4a6c15fd60 con 0x7f4a6c0b5840 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a70abb640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c14f640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a6b7fe640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c0b5840 0x7f4a6c14f100 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7f4a6000c4d0 tx=0x7f4a6000c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a68ff9640 1 -- 172.21.15.64:0/1920007658 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a60018070 con 0x7f4a6c0b5840 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a68ff9640 1 -- 172.21.15.64:0/1920007658 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4a60007500 con 0x7f4a6c0b5840 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4a6c160040 con 0x7f4a6c0b5840 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a68ff9640 1 -- 172.21.15.64:0/1920007658 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a60004a20 con 0x7f4a6c0b5840 2024-03-04T07:29:34.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.063+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4a6c160600 con 0x7f4a6c0b5840 2024-03-04T07:29:34.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.065+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4a6c073380 con 0x7f4a6c0b5840 2024-03-04T07:29:34.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.065+0000 7f4a68ff9640 1 -- 172.21.15.64:0/1920007658 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4a60004bc0 con 0x7f4a6c0b5840 2024-03-04T07:29:34.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.066+0000 7f4a68ff9640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a5806b4d0 0x7f4a5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.066+0000 7f4a68ff9640 1 -- 172.21.15.64:0/1920007658 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4a60089620 con 0x7f4a6c0b5840 2024-03-04T07:29:34.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.066+0000 7f4a6bfff640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a5806b4d0 0x7f4a5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.066+0000 7f4a6bfff640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a5806b4d0 0x7f4a5806d990 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f4a5c036000 tx=0x7f4a5c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.070+0000 7f4a68ff9640 1 -- 172.21.15.64:0/1920007658 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4a60057d40 con 0x7f4a6c0b5840 2024-03-04T07:29:34.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.250+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f4a6c04f410 con 0x7f4a6c0b5840 2024-03-04T07:29:34.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.251+0000 7f4a68ff9640 1 -- 172.21.15.64:0/1920007658 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v55) v1 ==== 104+0+11 (secure 0 0 0) 0x7f4a6005bde0 con 0x7f4a6c0b5840 2024-03-04T07:29:34.252 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a5806b4d0 msgr2=0x7f4a5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a5806b4d0 0x7f4a5806d990 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f4a5c036000 tx=0x7f4a5c03c040 comp rx=0 tx=0).stop 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c0b5840 msgr2=0x7f4a6c14f100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c0b5840 0x7f4a6c14f100 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7f4a6000c4d0 tx=0x7f4a6000c990 comp rx=0 tx=0).stop 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 shutdown_connections 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a5806b4d0 0x7f4a5806d990 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a6c15dbd0 0x7f4a6c14f640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a6c0b5840 0x7f4a6c14f100 unknown :-1 s=CLOSED pgs=708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 --2- 172.21.15.64:0/1920007658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a6c0b40c0 0x7f4a6c0c8140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 >> 172.21.15.64:0/1920007658 conn(0x7f4a6c0b26b0 msgr2=0x7f4a6c15b990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:34.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 shutdown_connections 2024-03-04T07:29:34.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.253+0000 7f4a7251e640 1 -- 172.21.15.64:0/1920007658 wait complete. 2024-03-04T07:29:34.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:29:34.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.368+0000 7f97ada4e640 1 Processor -- start 2024-03-04T07:29:34.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.368+0000 7f97ada4e640 1 -- start start 2024-03-04T07:29:34.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97ada4e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f97a815dd90 0x7f97a80ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97ada4e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f97a80d29d0 0x7f97a80d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97ada4e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f97a80d3740 con 0x7f97a80d4720 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97ada4e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97a80d3400 con 0x7f97a80d29d0 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97ada4e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97a80d3120 con 0x7f97a815dd90 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97a67fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f97a80d29d0 0x7f97a80d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97a77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f97a80d4720 0x7f97a80b6d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34522/0 (socket says 172.21.15.64:34522) 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97a67fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f97a80d29d0 0x7f97a80d41e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36820/0 (socket says 172.21.15.64:36820) 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.369+0000 7f97a77fe640 1 -- 172.21.15.64:0/21071945 learned_addr learned my addr 172.21.15.64:0/21071945 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.370+0000 7f97a6ffd640 1 --2- 172.21.15.64:0/21071945 >> v2:172.21.15.66:3300/0 conn(0x7f97a815dd90 0x7f97a80ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.370+0000 7f97a67fc640 1 -- 172.21.15.64:0/21071945 >> v2:172.21.15.66:3300/0 conn(0x7f97a815dd90 msgr2=0x7f97a80ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.370+0000 7f97a67fc640 1 --2- 172.21.15.64:0/21071945 >> v2:172.21.15.66:3300/0 conn(0x7f97a815dd90 0x7f97a80ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.370+0000 7f97a67fc640 1 -- 172.21.15.64:0/21071945 >> v1:172.21.15.64:6789/0 conn(0x7f97a80d4720 legacy=0x7f97a80b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.370+0000 7f97a67fc640 1 -- 172.21.15.64:0/21071945 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97a80d39e0 con 0x7f97a80d29d0 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.370+0000 7f97a5ffb640 1 -- 172.21.15.64:0/21071945 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3711348156 0 0) 0x7f97a80d3740 con 0x7f97a80d4720 2024-03-04T07:29:34.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.370+0000 7f97a6ffd640 1 --2- 172.21.15.64:0/21071945 >> v2:172.21.15.66:3300/0 conn(0x7f97a815dd90 0x7f97a80ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97a67fc640 1 --2- 172.21.15.64:0/21071945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d29d0 0x7f97a80d41e0 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7f97a80d1f90 tx=0x7f979c02b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f5457761722fb7c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97a5ffb640 1 -- 172.21.15.64:0/21071945 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c034070 con 0x7f97a80d29d0 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97a5ffb640 1 -- 172.21.15.64:0/21071945 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f979c030610 con 0x7f97a80d29d0 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97a5ffb640 1 -- 172.21.15.64:0/21071945 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c0309e0 con 0x7f97a80d29d0 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97ada4e640 1 -- 172.21.15.64:0/21071945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d29d0 msgr2=0x7f97a80d41e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97ada4e640 1 --2- 172.21.15.64:0/21071945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d29d0 0x7f97a80d41e0 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7f97a80d1f90 tx=0x7f979c02b960 comp rx=0 tx=0).stop 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97ada4e640 1 -- 172.21.15.64:0/21071945 shutdown_connections 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97ada4e640 1 --2- 172.21.15.64:0/21071945 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d29d0 0x7f97a80d41e0 unknown :-1 s=CLOSED pgs=710 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97ada4e640 1 --2- 172.21.15.64:0/21071945 >> v2:172.21.15.66:3300/0 conn(0x7f97a815dd90 0x7f97a80ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.371+0000 7f97ada4e640 1 -- 172.21.15.64:0/21071945 >> 172.21.15.64:0/21071945 conn(0x7f97a8071960 msgr2=0x7f97a8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.372+0000 7f97ada4e640 1 -- 172.21.15.64:0/21071945 shutdown_connections 2024-03-04T07:29:34.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.372+0000 7f97ada4e640 1 -- 172.21.15.64:0/21071945 wait complete. 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.373+0000 7f97ada4e640 1 Processor -- start 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.373+0000 7f97ada4e640 1 -- start start 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.373+0000 7f97ada4e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a80d29d0 0x7f97a80cad70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.373+0000 7f97ada4e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d4720 0x7f97a80cb2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97ada4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a815dd90 0x7f97a81f3820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97ada4e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97a80d2df0 con 0x7f97a80d4720 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97ada4e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97a815e170 con 0x7f97a80d29d0 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97ada4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97a80b7420 con 0x7f97a815dd90 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d4720 0x7f97a80cb2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d4720 0x7f97a80cb2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36826/0 (socket says 172.21.15.64:36826) 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 -- 172.21.15.64:0/504546568 learned_addr learned my addr 172.21.15.64:0/504546568 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:34.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a6ffd640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a80d29d0 0x7f97a80cad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a77fe640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a815dd90 0x7f97a81f3820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 -- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a815dd90 msgr2=0x7f97a81f3820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a815dd90 0x7f97a81f3820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 -- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a80d29d0 msgr2=0x7f97a80cad70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a80d29d0 0x7f97a80cad70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 -- 172.21.15.64:0/504546568 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97a80c9710 con 0x7f97a80d4720 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a77fe640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a815dd90 0x7f97a81f3820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97a67fc640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d4720 0x7f97a80cb2b0 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f979c009890 tx=0x7f979c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f978ffff640 1 -- 172.21.15.64:0/504546568 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c046020 con 0x7f97a80d4720 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f978ffff640 1 -- 172.21.15.64:0/504546568 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f979c039a10 con 0x7f97a80d4720 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97a80c99f0 con 0x7f97a80d4720 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f978ffff640 1 -- 172.21.15.64:0/504546568 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c034040 con 0x7f97a80d4720 2024-03-04T07:29:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.374+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f97a80c9fb0 con 0x7f97a80d4720 2024-03-04T07:29:34.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.375+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f97a80ca1f0 con 0x7f97a80d4720 2024-03-04T07:29:34.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.376+0000 7f978ffff640 1 -- 172.21.15.64:0/504546568 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f979c037070 con 0x7f97a80d4720 2024-03-04T07:29:34.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.376+0000 7f978ffff640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f979406b4d0 0x7f979406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.376+0000 7f978ffff640 1 -- 172.21.15.64:0/504546568 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f979c0abf50 con 0x7f97a80d4720 2024-03-04T07:29:34.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.376+0000 7f97a6ffd640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f979406b4d0 0x7f979406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.377+0000 7f97a6ffd640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f979406b4d0 0x7f979406d990 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f9798006fd0 tx=0x7f9798008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.385+0000 7f978ffff640 1 -- 172.21.15.64:0/504546568 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f979c050020 con 0x7f97a80d4720 2024-03-04T07:29:34.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.566+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f97a8072a30 con 0x7f97a80d4720 2024-03-04T07:29:34.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.566+0000 7f978ffff640 1 -- 172.21.15.64:0/504546568 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v55) v1 ==== 100+0+11 (secure 0 0 0) 0x7f979c039e20 con 0x7f97a80d4720 2024-03-04T07:29:34.566 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f979406b4d0 msgr2=0x7f979406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f979406b4d0 0x7f979406d990 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f9798006fd0 tx=0x7f9798008040 comp rx=0 tx=0).stop 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d4720 msgr2=0x7f97a80cb2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d4720 0x7f97a80cb2b0 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f979c009890 tx=0x7f979c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 shutdown_connections 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f979406b4d0 0x7f979406d990 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a815dd90 0x7f97a81f3820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a80d4720 0x7f97a80cb2b0 unknown :-1 s=CLOSED pgs=711 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 --2- 172.21.15.64:0/504546568 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a80d29d0 0x7f97a80cad70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 >> 172.21.15.64:0/504546568 conn(0x7f97a8071960 msgr2=0x7f97a80b36c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:34.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 shutdown_connections 2024-03-04T07:29:34.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.568+0000 7f97ada4e640 1 -- 172.21.15.64:0/504546568 wait complete. 2024-03-04T07:29:34.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:29:34.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.680+0000 7fca422fb640 1 Processor -- start 2024-03-04T07:29:34.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca422fb640 1 -- start start 2024-03-04T07:29:34.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca422fb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fca3c15df00 0x7fca3c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca422fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca422fb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fca3c0b9a30 0x7fca3c0b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca422fb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fca3c0cea30 con 0x7fca3c15df00 2024-03-04T07:29:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca422fb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fca3c0d2af0 con 0x7fca3c0b9a30 2024-03-04T07:29:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca422fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca3c0cf3c0 con 0x7fca3c0b95a0 2024-03-04T07:29:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca3bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fca3c15df00 0x7fca3c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca3bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fca3c15df00 0x7fca3c0d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36842/0 (socket says 172.21.15.64:36842) 2024-03-04T07:29:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.681+0000 7fca3bfff640 1 -- 172.21.15.64:0/3432692129 learned_addr learned my addr 172.21.15.64:0/3432692129 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca3b7fe640 1 --2- 172.21.15.64:0/3432692129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca40898640 1 --2- 172.21.15.64:0/3432692129 >> v2:172.21.15.66:3300/0 conn(0x7fca3c0b9a30 0x7fca3c0b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca3bfff640 1 -- 172.21.15.64:0/3432692129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 msgr2=0x7fca3c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca3bfff640 1 --2- 172.21.15.64:0/3432692129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca3bfff640 1 -- 172.21.15.64:0/3432692129 >> v2:172.21.15.66:3300/0 conn(0x7fca3c0b9a30 msgr2=0x7fca3c0b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca3bfff640 1 --2- 172.21.15.64:0/3432692129 >> v2:172.21.15.66:3300/0 conn(0x7fca3c0b9a30 0x7fca3c0b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca3bfff640 1 -- 172.21.15.64:0/3432692129 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca3c0cf0e0 con 0x7fca3c15df00 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.682+0000 7fca40898640 1 --2- 172.21.15.64:0/3432692129 >> v2:172.21.15.66:3300/0 conn(0x7fca3c0b9a30 0x7fca3c0b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca3bfff640 1 --2- 172.21.15.64:0/3432692129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c15df00 0x7fca3c0d25b0 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7fca3c0d1160 tx=0x7fca2c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=242e34479a1c5b94 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca3affd640 1 -- 172.21.15.64:0/3432692129 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca2c035020 con 0x7fca3c15df00 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca3affd640 1 -- 172.21.15.64:0/3432692129 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca2c031870 con 0x7fca3c15df00 2024-03-04T07:29:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca3affd640 1 -- 172.21.15.64:0/3432692129 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca2c031c40 con 0x7fca3c15df00 2024-03-04T07:29:34.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 -- 172.21.15.64:0/3432692129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c15df00 msgr2=0x7fca3c0d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 --2- 172.21.15.64:0/3432692129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c15df00 0x7fca3c0d25b0 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7fca3c0d1160 tx=0x7fca2c02b910 comp rx=0 tx=0).stop 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 -- 172.21.15.64:0/3432692129 shutdown_connections 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 --2- 172.21.15.64:0/3432692129 >> v2:172.21.15.66:3300/0 conn(0x7fca3c0b9a30 0x7fca3c0b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 --2- 172.21.15.64:0/3432692129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 --2- 172.21.15.64:0/3432692129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c15df00 0x7fca3c0d25b0 unknown :-1 s=CLOSED pgs=712 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 -- 172.21.15.64:0/3432692129 >> 172.21.15.64:0/3432692129 conn(0x7fca3c071960 msgr2=0x7fca3c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 -- 172.21.15.64:0/3432692129 shutdown_connections 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.683+0000 7fca422fb640 1 -- 172.21.15.64:0/3432692129 wait complete. 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.685+0000 7fca422fb640 1 Processor -- start 2024-03-04T07:29:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.685+0000 7fca422fb640 1 -- start start 2024-03-04T07:29:34.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca422fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b3350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca422fb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca3c15df00 0x7fca3c1603d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca422fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c0cd000 0x7fca3c1520b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca422fb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca3c0cec70 con 0x7fca3c0cd000 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca422fb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca3c0ce750 con 0x7fca3c15df00 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca422fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca3c0cedf0 con 0x7fca3c0b95a0 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c0cd000 0x7fca3c1520b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c0cd000 0x7fca3c1520b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36854/0 (socket says 172.21.15.64:36854) 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 -- 172.21.15.64:0/3426884902 learned_addr learned my addr 172.21.15.64:0/3426884902 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca3b7fe640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca3c15df00 0x7fca3c1603d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca3bfff640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b3350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 -- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 msgr2=0x7fca3c0b3350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b3350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 -- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca3c15df00 msgr2=0x7fca3c1603d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca3c15df00 0x7fca3c1603d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 -- 172.21.15.64:0/3426884902 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca3c0b39e0 con 0x7fca3c0cd000 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca40898640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c0cd000 0x7fca3c1520b0 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7fca34007f20 tx=0x7fca3400d560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca38ff9640 1 -- 172.21.15.64:0/3426884902 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca3401a070 con 0x7fca3c0cd000 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca38ff9640 1 -- 172.21.15.64:0/3426884902 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fca34015600 con 0x7fca3c0cd000 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca38ff9640 1 -- 172.21.15.64:0/3426884902 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca340159f0 con 0x7fca3c0cd000 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.687+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca3c0b3cc0 con 0x7fca3c0cd000 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.688+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca3c0b4280 con 0x7fca3c0cd000 2024-03-04T07:29:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.689+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca3c067ff0 con 0x7fca3c0cd000 2024-03-04T07:29:34.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.689+0000 7fca38ff9640 1 -- 172.21.15.64:0/3426884902 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fca3401d070 con 0x7fca3c0cd000 2024-03-04T07:29:34.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.690+0000 7fca38ff9640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca2806b4d0 0x7fca2806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:34.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.690+0000 7fca38ff9640 1 -- 172.21.15.64:0/3426884902 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fca3408a3d0 con 0x7fca3c0cd000 2024-03-04T07:29:34.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.690+0000 7fca3bfff640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca2806b4d0 0x7fca2806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:34.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.690+0000 7fca3bfff640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca2806b4d0 0x7fca2806d990 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7fca2c036000 tx=0x7fca2c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:34.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.698+0000 7fca38ff9640 1 -- 172.21.15.64:0/3426884902 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fca34010070 con 0x7fca3c0cd000 2024-03-04T07:29:34.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.894+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fca3c0b4720 con 0x7fca3c0cd000 2024-03-04T07:29:34.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.895+0000 7fca38ff9640 1 -- 172.21.15.64:0/3426884902 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v55) v1 ==== 80+0+0 (secure 0 0 0) 0x7fca34058c00 con 0x7fca3c0cd000 2024-03-04T07:29:34.895 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:29:34.896 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.896+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca2806b4d0 msgr2=0x7fca2806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.896+0000 7fca422fb640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca2806b4d0 0x7fca2806d990 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7fca2c036000 tx=0x7fca2c03c040 comp rx=0 tx=0).stop 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.896+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c0cd000 msgr2=0x7fca3c1520b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c0cd000 0x7fca3c1520b0 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7fca34007f20 tx=0x7fca3400d560 comp rx=0 tx=0).stop 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 shutdown_connections 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fca2806b4d0 0x7fca2806d990 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fca3c0cd000 0x7fca3c1520b0 unknown :-1 s=CLOSED pgs=713 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fca3c15df00 0x7fca3c1603d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 --2- 172.21.15.64:0/3426884902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fca3c0b95a0 0x7fca3c0b3350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 >> 172.21.15.64:0/3426884902 conn(0x7fca3c071960 msgr2=0x7fca3c15bbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 shutdown_connections 2024-03-04T07:29:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:34.897+0000 7fca422fb640 1 -- 172.21.15.64:0/3426884902 wait complete. 2024-03-04T07:29:34.907 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:34.907 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:34.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:35.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.063+0000 7f90f8177640 1 Processor -- start 2024-03-04T07:29:35.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.063+0000 7f90f8177640 1 -- start start 2024-03-04T07:29:35.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f8177640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f8177640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f90f00b9a10 0x7f90f00b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f8177640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f90f00cea30 con 0x7f90f015df00 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f8177640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f90f00cf3c0 con 0x7f90f00b9a10 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f8177640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f90f00cf0e0 con 0x7f90f00b95a0 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f5f13640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f90f015df00 0x7f90f00d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34528/0 (socket says 172.21.15.64:34528) 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f5f13640 1 -- 172.21.15.64:0/1992426125 learned_addr learned my addr 172.21.15.64:0/1992426125 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f6714640 1 --2- 172.21.15.64:0/1992426125 >> v2:172.21.15.66:3300/0 conn(0x7f90f00b9a10 0x7f90f00b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f5712640 1 --2- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f4f11640 1 -- 172.21.15.64:0/1992426125 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2615516314 0 0) 0x7f90f00cea30 con 0x7f90f015df00 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.064+0000 7f90f4f11640 1 -- 172.21.15.64:0/1992426125 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f90dc003520 con 0x7f90f015df00 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.065+0000 7f90f4f11640 1 -- 172.21.15.64:0/1992426125 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3642143506 0 0) 0x7f90dc003520 con 0x7f90f015df00 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.065+0000 7f90f6714640 1 -- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 msgr2=0x7f90f00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.065+0000 7f90f6714640 1 --2- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f00b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.065+0000 7f90f6714640 1 -- 172.21.15.64:0/1992426125 >> v1:172.21.15.64:6789/0 conn(0x7f90f015df00 legacy=0x7f90f00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.065+0000 7f90f6714640 1 -- 172.21.15.64:0/1992426125 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90f00ce470 con 0x7f90f00b9a10 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.065+0000 7f90f6714640 1 --2- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 0x7f90f00b5c40 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f90e8009d90 tx=0x7f90e802bd70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=caa4930fcad6d154 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f4f11640 1 -- 172.21.15.64:0/1992426125 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90e8039070 con 0x7f90f00b9a10 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f4f11640 1 -- 172.21.15.64:0/1992426125 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f90e802c8a0 con 0x7f90f00b9a10 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f4f11640 1 -- 172.21.15.64:0/1992426125 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90ec001670 con 0x7f90f00b9a10 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f8177640 1 -- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 msgr2=0x7f90f00b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f8177640 1 --2- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 0x7f90f00b5c40 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f90e8009d90 tx=0x7f90e802bd70 comp rx=0 tx=0).stop 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f8177640 1 -- 172.21.15.64:0/1992426125 shutdown_connections 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f8177640 1 --2- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 0x7f90f00b5c40 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f8177640 1 --2- 172.21.15.64:0/1992426125 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f00b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.066+0000 7f90f8177640 1 -- 172.21.15.64:0/1992426125 >> 172.21.15.64:0/1992426125 conn(0x7f90f0071960 msgr2=0x7f90f0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.067+0000 7f90f8177640 1 -- 172.21.15.64:0/1992426125 shutdown_connections 2024-03-04T07:29:35.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.067+0000 7f90f8177640 1 -- 172.21.15.64:0/1992426125 wait complete. 2024-03-04T07:29:35.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.068+0000 7f90f8177640 1 Processor -- start 2024-03-04T07:29:35.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.068+0000 7f90f8177640 1 -- start start 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.069+0000 7f90f8177640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f015a3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f8177640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 0x7f90f00c98c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f8177640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90f015df00 0x7f90f015e8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f8177640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90f00cedf0 con 0x7f90f015df00 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f8177640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90f00cec70 con 0x7f90f00b9a10 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f8177640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f90f00d3220 con 0x7f90f00b95a0 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90f015df00 0x7f90f015e8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90f015df00 0x7f90f015e8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36862/0 (socket says 172.21.15.64:36862) 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 -- 172.21.15.64:0/1710141552 learned_addr learned my addr 172.21.15.64:0/1710141552 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f5712640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 0x7f90f00c98c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f5f13640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f015a3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 -- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 msgr2=0x7f90f015a3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f015a3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 -- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 msgr2=0x7f90f00c98c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 0x7f90f00c98c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 -- 172.21.15.64:0/1710141552 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90f00cbb20 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f6714640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90f015df00 0x7f90f015e8d0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f90e8037040 tx=0x7f90e8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90e67fc640 1 -- 172.21.15.64:0/1710141552 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90e8039050 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90e67fc640 1 -- 172.21.15.64:0/1710141552 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f90e8045070 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90f00cbe00 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90e67fc640 1 -- 172.21.15.64:0/1710141552 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90e8009450 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.070+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90f00cc3c0 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.071+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f90f0067ff0 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.072+0000 7f90e67fc640 1 -- 172.21.15.64:0/1710141552 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f90e8040690 con 0x7f90f015df00 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.072+0000 7f90e67fc640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90d406b4d0 0x7f90d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.073+0000 7f90e67fc640 1 -- 172.21.15.64:0/1710141552 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f90e80ac720 con 0x7f90f015df00 2024-03-04T07:29:35.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.073+0000 7f90f5f13640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90d406b4d0 0x7f90d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.073+0000 7f90f5f13640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90d406b4d0 0x7f90d406d990 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f90ec001320 tx=0x7f90ec0012b0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.077+0000 7f90e67fc640 1 -- 172.21.15.64:0/1710141552 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f90e804f020 con 0x7f90f015df00 2024-03-04T07:29:35.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.258+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f90f00d4210 con 0x7f90f015df00 2024-03-04T07:29:35.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.259+0000 7f90e67fc640 1 -- 172.21.15.64:0/1710141552 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v55) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f90e807af40 con 0x7f90f015df00 2024-03-04T07:29:35.259 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:35.260 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:12.504021+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:35.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.261+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90d406b4d0 msgr2=0x7f90d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.261+0000 7f90f8177640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90d406b4d0 0x7f90d406d990 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f90ec001320 tx=0x7f90ec0012b0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.261+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90f015df00 msgr2=0x7f90f015e8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.261+0000 7f90f8177640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90f015df00 0x7f90f015e8d0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f90e8037040 tx=0x7f90e8004a40 comp rx=0 tx=0).stop 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 shutdown_connections 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f90d406b4d0 0x7f90d406d990 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f90f015df00 0x7f90f015e8d0 unknown :-1 s=CLOSED pgs=715 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f90f00b9a10 0x7f90f00c98c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 --2- 172.21.15.64:0/1710141552 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f90f00b95a0 0x7f90f015a3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 >> 172.21.15.64:0/1710141552 conn(0x7f90f0071960 msgr2=0x7f90f015d1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 shutdown_connections 2024-03-04T07:29:35.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.262+0000 7f90f8177640 1 -- 172.21.15.64:0/1710141552 wait complete. 2024-03-04T07:29:35.276 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 8, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:29:09.132151+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '54', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 9, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:29:09.825115+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '55', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:29:35.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:29:35.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.413+0000 7ff57804e640 1 Processor -- start 2024-03-04T07:29:35.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.413+0000 7ff57804e640 1 -- start start 2024-03-04T07:29:35.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff57804e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff5700ceea0 0x7ff5700d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff57804e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff5700d3740 con 0x7ff5700b6d10 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff57804e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff5700d3120 con 0x7ff57015dd90 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff57804e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff5700d3400 con 0x7ff5700ceea0 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff5755e9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff5700ceea0 0x7ff5700d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff5755e9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff5700ceea0 0x7ff5700d29d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36874/0 (socket says 172.21.15.64:36874) 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff5765eb640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff5700b6d10 0x7ff5700d0c30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34534/0 (socket says 172.21.15.64:34534) 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.414+0000 7ff5755e9640 1 -- 172.21.15.64:0/424680682 learned_addr learned my addr 172.21.15.64:0/424680682 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.415+0000 7ff5755e9640 1 -- 172.21.15.64:0/424680682 >> v1:172.21.15.66:6789/0 conn(0x7ff57015dd90 legacy=0x7ff5700d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.415+0000 7ff5755e9640 1 -- 172.21.15.64:0/424680682 >> v1:172.21.15.64:6789/0 conn(0x7ff5700b6d10 legacy=0x7ff5700d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.415+0000 7ff5755e9640 1 -- 172.21.15.64:0/424680682 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5700d39e0 con 0x7ff5700ceea0 2024-03-04T07:29:35.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.415+0000 7ff574de8640 1 -- 172.21.15.64:0/424680682 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 424798620 0 0) 0x7ff5700d3740 con 0x7ff5700b6d10 2024-03-04T07:29:35.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff5755e9640 1 --2- 172.21.15.64:0/424680682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700ceea0 0x7ff5700d29d0 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7ff560009870 tx=0x7ff56002bab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a942eb922bedb73 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff574de8640 1 -- 172.21.15.64:0/424680682 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff56002d870 con 0x7ff5700ceea0 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff574de8640 1 -- 172.21.15.64:0/424680682 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff56002da10 con 0x7ff5700ceea0 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff574de8640 1 -- 172.21.15.64:0/424680682 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff56002dde0 con 0x7ff5700ceea0 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff57804e640 1 -- 172.21.15.64:0/424680682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700ceea0 msgr2=0x7ff5700d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff57804e640 1 --2- 172.21.15.64:0/424680682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700ceea0 0x7ff5700d29d0 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7ff560009870 tx=0x7ff56002bab0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff57804e640 1 -- 172.21.15.64:0/424680682 shutdown_connections 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff57804e640 1 --2- 172.21.15.64:0/424680682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700ceea0 0x7ff5700d29d0 unknown :-1 s=CLOSED pgs=717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff57804e640 1 -- 172.21.15.64:0/424680682 >> 172.21.15.64:0/424680682 conn(0x7ff570071960 msgr2=0x7ff570071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff57804e640 1 -- 172.21.15.64:0/424680682 shutdown_connections 2024-03-04T07:29:35.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.416+0000 7ff57804e640 1 -- 172.21.15.64:0/424680682 wait complete. 2024-03-04T07:29:35.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.417+0000 7ff57804e640 1 Processor -- start 2024-03-04T07:29:35.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff57804e640 1 -- start start 2024-03-04T07:29:35.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff57804e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700b6d10 0x7ff5700c6740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff57804e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5700ceea0 0x7ff570148ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff57804e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff57015dd90 0x7ff5701491e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff57804e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5700d48f0 con 0x7ff5700b6d10 2024-03-04T07:29:35.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff57804e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff57015e170 con 0x7ff57015dd90 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff57804e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5700d4a70 con 0x7ff5700ceea0 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff575dea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700b6d10 0x7ff5700c6740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff575dea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700b6d10 0x7ff5700c6740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36886/0 (socket says 172.21.15.64:36886) 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff575dea640 1 -- 172.21.15.64:0/3859052014 learned_addr learned my addr 172.21.15.64:0/3859052014 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff5755e9640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5700ceea0 0x7ff570148ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.418+0000 7ff5765eb640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff57015dd90 0x7ff5701491e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff575dea640 1 -- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5700ceea0 msgr2=0x7ff570148ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff575dea640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5700ceea0 0x7ff570148ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff575dea640 1 -- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff57015dd90 msgr2=0x7ff5701491e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff575dea640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff57015dd90 0x7ff5701491e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff575dea640 1 -- 172.21.15.64:0/3859052014 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5700c9830 con 0x7ff5700b6d10 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff5755e9640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5700ceea0 0x7ff570148ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff575dea640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700b6d10 0x7ff5700c6740 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7ff56800c400 tx=0x7ff56800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff5667fc640 1 -- 172.21.15.64:0/3859052014 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff568017070 con 0x7ff5700b6d10 2024-03-04T07:29:35.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff5667fc640 1 -- 172.21.15.64:0/3859052014 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5680052d0 con 0x7ff5700b6d10 2024-03-04T07:29:35.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff5700c9b10 con 0x7ff5700b6d10 2024-03-04T07:29:35.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff5667fc640 1 -- 172.21.15.64:0/3859052014 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff568003830 con 0x7ff5700b6d10 2024-03-04T07:29:35.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.419+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff5700ca3b0 con 0x7ff5700b6d10 2024-03-04T07:29:35.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.420+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff5700c9ce0 con 0x7ff5700b6d10 2024-03-04T07:29:35.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.420+0000 7ff5667fc640 1 -- 172.21.15.64:0/3859052014 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff56801a070 con 0x7ff5700b6d10 2024-03-04T07:29:35.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.421+0000 7ff5667fc640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff55406b4d0 0x7ff55406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.421+0000 7ff5755e9640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff55406b4d0 0x7ff55406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.421+0000 7ff5667fc640 1 -- 172.21.15.64:0/3859052014 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff568089540 con 0x7ff5700b6d10 2024-03-04T07:29:35.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.422+0000 7ff5755e9640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff55406b4d0 0x7ff55406d990 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7ff560036000 tx=0x7ff560008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.425+0000 7ff5667fc640 1 -- 172.21.15.64:0/3859052014 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff568057ce0 con 0x7ff5700b6d10 2024-03-04T07:29:35.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.611+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7ff570072a30 con 0x7ff5700b6d10 2024-03-04T07:29:35.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.611+0000 7ff5667fc640 1 -- 172.21.15.64:0/3859052014 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v55) v1 ==== 93+0+10 (secure 0 0 0) 0x7ff56805bd80 con 0x7ff5700b6d10 2024-03-04T07:29:35.611 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff55406b4d0 msgr2=0x7ff55406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff55406b4d0 0x7ff55406d990 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7ff560036000 tx=0x7ff560008030 comp rx=0 tx=0).stop 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700b6d10 msgr2=0x7ff5700c6740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700b6d10 0x7ff5700c6740 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7ff56800c400 tx=0x7ff56800c8c0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 shutdown_connections 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff55406b4d0 0x7ff55406d990 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff57015dd90 0x7ff5701491e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5700ceea0 0x7ff570148ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 --2- 172.21.15.64:0/3859052014 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5700b6d10 0x7ff5700c6740 unknown :-1 s=CLOSED pgs=718 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 >> 172.21.15.64:0/3859052014 conn(0x7ff570071960 msgr2=0x7ff5700b3700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:35.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 shutdown_connections 2024-03-04T07:29:35.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.613+0000 7ff57804e640 1 -- 172.21.15.64:0/3859052014 wait complete. 2024-03-04T07:29:35.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:29:35.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.728+0000 7f75709f7640 1 Processor -- start 2024-03-04T07:29:35.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.728+0000 7f75709f7640 1 -- start start 2024-03-04T07:29:35.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f75709f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f756c0ceea0 0x7f756c0d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f75709f7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f756c0d3740 con 0x7f756c15dd90 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f75709f7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f756c0d3120 con 0x7f756c0b6d10 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f75709f7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f756c0d3400 con 0x7f756c0ceea0 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756a59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f756c15dd90 0x7f756c0d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34546/0 (socket says 172.21.15.64:34546) 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756a59c640 1 -- 172.21.15.64:0/951128627 learned_addr learned my addr 172.21.15.64:0/951128627 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f7569d9b640 1 --2- 172.21.15.64:0/951128627 >> v2:172.21.15.66:3300/0 conn(0x7f756c0ceea0 0x7f756c0d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3531789318 0 0) 0x7f756c0d3740 con 0x7f756c15dd90 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7554003520 con 0x7f756c15dd90 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1170324915 0 0) 0x7f7554003520 con 0x7f756c15dd90 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 >> v2:172.21.15.66:3300/0 conn(0x7f756c0ceea0 msgr2=0x7f756c0d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:35.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 --2- 172.21.15.64:0/951128627 >> v2:172.21.15.66:3300/0 conn(0x7f756c0ceea0 0x7f756c0d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 >> v1:172.21.15.66:6789/0 conn(0x7f756c0b6d10 legacy=0x7f756c0d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f756c0d39e0 con 0x7f756c15dd90 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.729+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f755c001670 con 0x7f756c15dd90 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 >> v1:172.21.15.64:6789/0 conn(0x7f756c15dd90 legacy=0x7f756c0d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756959a640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75540039b0 0x7f7554003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756959a640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7554004800 0x7f7554004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756959a640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75540051c0 0x7f755400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f756c0d3400 con 0x7f75540039b0 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f755400c070 con 0x7f75540051c0 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f755400c270 con 0x7f7554004800 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756a59c640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75540039b0 0x7f7554003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f7569d9b640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7554004800 0x7f7554004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756ad9d640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75540051c0 0x7f755400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756a59c640 1 -- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7554004800 msgr2=0x7f7554004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756a59c640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7554004800 0x7f7554004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756a59c640 1 -- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75540051c0 msgr2=0x7f755400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756a59c640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75540051c0 0x7f755400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756a59c640 1 -- 172.21.15.64:0/951128627 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7554003520 con 0x7f75540039b0 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756ad9d640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75540051c0 0x7f755400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.730+0000 7f756ad9d640 1 -- 172.21.15.64:0/951128627 reap_dead start 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f756a59c640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75540039b0 0x7f7554003dd0 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7f755c00c240 tx=0x7f7554005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=952677f5cc83f3a5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f755c007cf0 con 0x7f75540039b0 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f755c030070 con 0x7f75540039b0 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f756959a640 1 -- 172.21.15.64:0/951128627 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f755c005d30 con 0x7f75540039b0 2024-03-04T07:29:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f75709f7640 1 -- 172.21.15.64:0/951128627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75540039b0 msgr2=0x7f7554003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f75709f7640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75540039b0 0x7f7554003dd0 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7f755c00c240 tx=0x7f7554005830 comp rx=0 tx=0).stop 2024-03-04T07:29:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f75709f7640 1 -- 172.21.15.64:0/951128627 shutdown_connections 2024-03-04T07:29:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f75709f7640 1 --2- 172.21.15.64:0/951128627 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75540039b0 0x7f7554003dd0 unknown :-1 s=CLOSED pgs=720 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.731+0000 7f75709f7640 1 -- 172.21.15.64:0/951128627 >> 172.21.15.64:0/951128627 conn(0x7f756c071960 msgr2=0x7f756c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.732+0000 7f75709f7640 1 -- 172.21.15.64:0/951128627 shutdown_connections 2024-03-04T07:29:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.732+0000 7f75709f7640 1 -- 172.21.15.64:0/951128627 wait complete. 2024-03-04T07:29:35.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.733+0000 7f75709f7640 1 Processor -- start 2024-03-04T07:29:35.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.733+0000 7f75709f7640 1 -- start start 2024-03-04T07:29:35.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75540039b0 0x7f756c0cc1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f756c0cd590 0x7f756c1f7380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f756c0cda10 0x7f756c1f78c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f756c0b71a0 con 0x7f756c0cd590 2024-03-04T07:29:35.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f756c15e170 con 0x7f756c0cda10 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f756c0b7320 con 0x7f75540039b0 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f756c0cd590 0x7f756c1f7380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f756c0cd590 0x7f756c1f7380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36900/0 (socket says 172.21.15.64:36900) 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 -- 172.21.15.64:0/1879537760 learned_addr learned my addr 172.21.15.64:0/1879537760 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f756a59c640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75540039b0 0x7f756c0cc1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f756ad9d640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f756c0cda10 0x7f756c1f78c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 -- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75540039b0 msgr2=0x7f756c0cc1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75540039b0 0x7f756c0cc1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 -- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f756c0cda10 msgr2=0x7f756c1f78c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f756c0cda10 0x7f756c1f78c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 -- 172.21.15.64:0/1879537760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f756c0cb130 con 0x7f756c0cd590 2024-03-04T07:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f756ad9d640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f756c0cda10 0x7f756c1f78c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f7569d9b640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f756c0cd590 0x7f756c1f7380 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7f75600116d0 tx=0x7f7560011b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f755affd640 1 -- 172.21.15.64:0/1879537760 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f756000f9c0 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f755affd640 1 -- 172.21.15.64:0/1879537760 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f756000fb60 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f756c0cb410 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f755affd640 1 -- 172.21.15.64:0/1879537760 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7560015460 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.734+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f756c0cb9d0 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.736+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f756c072a30 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.736+0000 7f755affd640 1 -- 172.21.15.64:0/1879537760 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7560015600 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.736+0000 7f755affd640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f754c06b4d0 0x7f754c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.737+0000 7f756a59c640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f754c06b4d0 0x7f754c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.737+0000 7f755affd640 1 -- 172.21.15.64:0/1879537760 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f756008aa90 con 0x7f756c0cd590 2024-03-04T07:29:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.737+0000 7f756a59c640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f754c06b4d0 0x7f754c06d990 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f755c034000 tx=0x7f755c007000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:35.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.741+0000 7f755affd640 1 -- 172.21.15.64:0/1879537760 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7560019030 con 0x7f756c0cd590 2024-03-04T07:29:35.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.920+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f756c04f410 con 0x7f756c0cd590 2024-03-04T07:29:35.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.921+0000 7f755affd640 1 -- 172.21.15.64:0/1879537760 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v55) v1 ==== 104+0+11 (secure 0 0 0) 0x7f75600599a0 con 0x7f756c0cd590 2024-03-04T07:29:35.921 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.922+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f754c06b4d0 msgr2=0x7f754c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f754c06b4d0 0x7f754c06d990 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f755c034000 tx=0x7f755c007000 comp rx=0 tx=0).stop 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f756c0cd590 msgr2=0x7f756c1f7380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f756c0cd590 0x7f756c1f7380 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7f75600116d0 tx=0x7f7560011b90 comp rx=0 tx=0).stop 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 shutdown_connections 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f754c06b4d0 0x7f754c06d990 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f756c0cda10 0x7f756c1f78c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f756c0cd590 0x7f756c1f7380 unknown :-1 s=CLOSED pgs=721 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 --2- 172.21.15.64:0/1879537760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75540039b0 0x7f756c0cc1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 >> 172.21.15.64:0/1879537760 conn(0x7f756c071960 msgr2=0x7f756c0d0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 shutdown_connections 2024-03-04T07:29:35.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:35.923+0000 7f75709f7640 1 -- 172.21.15.64:0/1879537760 wait complete. 2024-03-04T07:29:35.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:29:36.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.038+0000 7f014c116640 1 Processor -- start 2024-03-04T07:29:36.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.038+0000 7f014c116640 1 -- start start 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014c116640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f014415b130 0x7f014415b5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014c116640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f01440ced70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014c116640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f014415bc40 con 0x7f0144144640 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014c116640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f01440cf2b0 con 0x7f014415b130 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014c116640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f014406df70 con 0x7f01440ce900 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f0149eb2640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0144144640 0x7f014406e400 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34556/0 (socket says 172.21.15.64:34556) 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f0149eb2640 1 -- 172.21.15.64:0/1608124204 learned_addr learned my addr 172.21.15.64:0/1608124204 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f01496b1640 1 --2- 172.21.15.64:0/1608124204 >> v2:172.21.15.66:3300/0 conn(0x7f014415b130 0x7f014415b5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014a6b3640 1 --2- 172.21.15.64:0/1608124204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f01440ced70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f0148eb0640 1 -- 172.21.15.64:0/1608124204 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3612337234 0 0) 0x7f014415bc40 con 0x7f0144144640 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f0148eb0640 1 -- 172.21.15.64:0/1608124204 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0130003520 con 0x7f0144144640 2024-03-04T07:29:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014a6b3640 1 -- 172.21.15.64:0/1608124204 >> v2:172.21.15.66:3300/0 conn(0x7f014415b130 msgr2=0x7f014415b5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014a6b3640 1 --2- 172.21.15.64:0/1608124204 >> v2:172.21.15.66:3300/0 conn(0x7f014415b130 0x7f014415b5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014a6b3640 1 -- 172.21.15.64:0/1608124204 >> v1:172.21.15.64:6789/0 conn(0x7f0144144640 legacy=0x7f014406e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f014a6b3640 1 -- 172.21.15.64:0/1608124204 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f014415bf20 con 0x7f01440ce900 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.039+0000 7f01496b1640 1 --2- 172.21.15.64:0/1608124204 >> v2:172.21.15.66:3300/0 conn(0x7f014415b130 0x7f014415b5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.040+0000 7f014a6b3640 1 --2- 172.21.15.64:0/1608124204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f01440ced70 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f013c009d90 tx=0x7f013c02bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6a2c6ed4ed90bdb9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.040+0000 7f0148eb0640 1 -- 172.21.15.64:0/1608124204 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f013c039070 con 0x7f01440ce900 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.040+0000 7f0148eb0640 1 -- 172.21.15.64:0/1608124204 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f013c02cc50 con 0x7f01440ce900 2024-03-04T07:29:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.040+0000 7f0148eb0640 1 -- 172.21.15.64:0/1608124204 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f013c034440 con 0x7f01440ce900 2024-03-04T07:29:36.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 -- 172.21.15.64:0/1608124204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 msgr2=0x7f01440ced70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 --2- 172.21.15.64:0/1608124204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f01440ced70 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f013c009d90 tx=0x7f013c02bea0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 -- 172.21.15.64:0/1608124204 shutdown_connections 2024-03-04T07:29:36.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 --2- 172.21.15.64:0/1608124204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f01440ced70 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 --2- 172.21.15.64:0/1608124204 >> v2:172.21.15.66:3300/0 conn(0x7f014415b130 0x7f014415b5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 -- 172.21.15.64:0/1608124204 >> 172.21.15.64:0/1608124204 conn(0x7f0144066d00 msgr2=0x7f0144067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:36.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 -- 172.21.15.64:0/1608124204 shutdown_connections 2024-03-04T07:29:36.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.041+0000 7f014c116640 1 -- 172.21.15.64:0/1608124204 wait complete. 2024-03-04T07:29:36.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.042+0000 7f014c116640 1 Processor -- start 2024-03-04T07:29:36.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.042+0000 7f014c116640 1 -- start start 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.043+0000 7f014c116640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f014413b000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014c116640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0144144640 0x7f01441e5350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014c116640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f014415b130 0x7f01441e5890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014c116640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0144070aa0 con 0x7f014415b130 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014c116640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f014406eb10 con 0x7f0144144640 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014c116640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0144070c20 con 0x7f01440ce900 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f014415b130 0x7f01441e5890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f014415b130 0x7f01441e5890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36914/0 (socket says 172.21.15.64:36914) 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 -- 172.21.15.64:0/3412813413 learned_addr learned my addr 172.21.15.64:0/3412813413 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f01496b1640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0144144640 0x7f01441e5350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f0149eb2640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f014413b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 -- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 msgr2=0x7f014413b000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f014413b000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 -- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0144144640 msgr2=0x7f01441e5350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0144144640 0x7f01441e5350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 -- 172.21.15.64:0/3412813413 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01441e5eb0 con 0x7f014415b130 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014a6b3640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f014415b130 0x7f01441e5890 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7f013c037040 tx=0x7f013c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f013a7fc640 1 -- 172.21.15.64:0/3412813413 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f013c042030 con 0x7f014415b130 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f013a7fc640 1 -- 172.21.15.64:0/3412813413 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f013c03a040 con 0x7f014415b130 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01441e6190 con 0x7f014415b130 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f013a7fc640 1 -- 172.21.15.64:0/3412813413 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f013c039070 con 0x7f014415b130 2024-03-04T07:29:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.044+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f014413bc20 con 0x7f014415b130 2024-03-04T07:29:36.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.046+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f01440733c0 con 0x7f014415b130 2024-03-04T07:29:36.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.046+0000 7f013a7fc640 1 -- 172.21.15.64:0/3412813413 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f013c045070 con 0x7f014415b130 2024-03-04T07:29:36.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.046+0000 7f013a7fc640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f012806b500 0x7f012806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.046+0000 7f013a7fc640 1 -- 172.21.15.64:0/3412813413 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f013c0abdf0 con 0x7f014415b130 2024-03-04T07:29:36.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.046+0000 7f0149eb2640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f012806b500 0x7f012806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.047+0000 7f0149eb2640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f012806b500 0x7f012806d9c0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f0134005880 tx=0x7f0134009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:36.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.055+0000 7f013a7fc640 1 -- 172.21.15.64:0/3412813413 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f013c04c070 con 0x7f014415b130 2024-03-04T07:29:36.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.237+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f014415ae10 con 0x7f014415b130 2024-03-04T07:29:36.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.238+0000 7f013a7fc640 1 -- 172.21.15.64:0/3412813413 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v55) v1 ==== 100+0+11 (secure 0 0 0) 0x7f013c07adc0 con 0x7f014415b130 2024-03-04T07:29:36.238 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f012806b500 msgr2=0x7f012806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f012806b500 0x7f012806d9c0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f0134005880 tx=0x7f0134009040 comp rx=0 tx=0).stop 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f014415b130 msgr2=0x7f01441e5890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f014415b130 0x7f01441e5890 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7f013c037040 tx=0x7f013c004a40 comp rx=0 tx=0).stop 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 shutdown_connections 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f012806b500 0x7f012806d9c0 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f014415b130 0x7f01441e5890 unknown :-1 s=CLOSED pgs=723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0144144640 0x7f01441e5350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 --2- 172.21.15.64:0/3412813413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01440ce900 0x7f014413b000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 >> 172.21.15.64:0/3412813413 conn(0x7f0144066d00 msgr2=0x7f014415d9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:36.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 shutdown_connections 2024-03-04T07:29:36.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.240+0000 7f014c116640 1 -- 172.21.15.64:0/3412813413 wait complete. 2024-03-04T07:29:36.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:29:36.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.353+0000 7f6e299a0640 1 Processor -- start 2024-03-04T07:29:36.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.353+0000 7f6e299a0640 1 -- start start 2024-03-04T07:29:36.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e299a0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6e2415f7e0 0x7f6e2415fbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e299a0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6e241605d0 con 0x7f6e2415cee0 2024-03-04T07:29:36.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e299a0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6e24160100 con 0x7f6e240cec00 2024-03-04T07:29:36.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e299a0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6e240cc960 con 0x7f6e2415f7e0 2024-03-04T07:29:36.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e2899e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6e2415f7e0 0x7f6e2415fbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e2919f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6e2415cee0 0x7f6e240cfbe0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34558/0 (socket says 172.21.15.64:34558) 2024-03-04T07:29:36.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e2899e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6e2415f7e0 0x7f6e2415fbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36922/0 (socket says 172.21.15.64:36922) 2024-03-04T07:29:36.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e2919f640 1 -- 172.21.15.64:0/2203493458 learned_addr learned my addr 172.21.15.64:0/2203493458 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:36.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e2899e640 1 -- 172.21.15.64:0/2203493458 >> v1:172.21.15.66:6789/0 conn(0x7f6e240cec00 legacy=0x7f6e240cf4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e2899e640 1 -- 172.21.15.64:0/2203493458 >> v1:172.21.15.64:6789/0 conn(0x7f6e2415cee0 legacy=0x7f6e240cfbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.354+0000 7f6e237fe640 1 -- 172.21.15.64:0/2203493458 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 923324101 0 0) 0x7f6e241605d0 con 0x7f6e2415cee0 2024-03-04T07:29:36.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.355+0000 7f6e2899e640 1 -- 172.21.15.64:0/2203493458 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e240ccc40 con 0x7f6e2415f7e0 2024-03-04T07:29:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.355+0000 7f6e2899e640 1 --2- 172.21.15.64:0/2203493458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415f7e0 0x7f6e2415fbc0 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f6e180097a0 tx=0x7f6e1802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=27022b80e2e3fff4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.355+0000 7f6e237fe640 1 -- 172.21.15.64:0/2203493458 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e18034070 con 0x7f6e2415f7e0 2024-03-04T07:29:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.355+0000 7f6e237fe640 1 -- 172.21.15.64:0/2203493458 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6e18030610 con 0x7f6e2415f7e0 2024-03-04T07:29:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.355+0000 7f6e237fe640 1 -- 172.21.15.64:0/2203493458 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e180309e0 con 0x7f6e2415f7e0 2024-03-04T07:29:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.356+0000 7f6e299a0640 1 -- 172.21.15.64:0/2203493458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415f7e0 msgr2=0x7f6e2415fbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.356+0000 7f6e299a0640 1 --2- 172.21.15.64:0/2203493458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415f7e0 0x7f6e2415fbc0 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f6e180097a0 tx=0x7f6e1802b960 comp rx=0 tx=0).stop 2024-03-04T07:29:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.356+0000 7f6e299a0640 1 -- 172.21.15.64:0/2203493458 shutdown_connections 2024-03-04T07:29:36.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.356+0000 7f6e299a0640 1 --2- 172.21.15.64:0/2203493458 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415f7e0 0x7f6e2415fbc0 unknown :-1 s=CLOSED pgs=725 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.356+0000 7f6e299a0640 1 -- 172.21.15.64:0/2203493458 >> 172.21.15.64:0/2203493458 conn(0x7f6e240b3430 msgr2=0x7f6e240b3840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:36.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.356+0000 7f6e299a0640 1 -- 172.21.15.64:0/2203493458 shutdown_connections 2024-03-04T07:29:36.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.356+0000 7f6e299a0640 1 -- 172.21.15.64:0/2203493458 wait complete. 2024-03-04T07:29:36.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.357+0000 7f6e299a0640 1 Processor -- start 2024-03-04T07:29:36.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.357+0000 7f6e299a0640 1 -- start start 2024-03-04T07:29:36.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e299a0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6e240cec00 0x7f6e2406c930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e299a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415cee0 0x7f6e241584e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e299a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6e2415f7e0 0x7f6e241590f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e299a0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e240cd4e0 con 0x7f6e2415cee0 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e299a0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e240cd200 con 0x7f6e240cec00 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e299a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6e240cc5c0 con 0x7f6e2415f7e0 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e23fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415cee0 0x7f6e241584e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e23fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415cee0 0x7f6e241584e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36934/0 (socket says 172.21.15.64:36934) 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.358+0000 7f6e23fff640 1 -- 172.21.15.64:0/3764766433 learned_addr learned my addr 172.21.15.64:0/3764766433 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e2899e640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6e240cec00 0x7f6e2406c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e2919f640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6e2415f7e0 0x7f6e241590f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e23fff640 1 -- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6e2415f7e0 msgr2=0x7f6e241590f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e23fff640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6e2415f7e0 0x7f6e241590f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e23fff640 1 -- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6e240cec00 msgr2=0x7f6e2406c930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e23fff640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6e240cec00 0x7f6e2406c930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e23fff640 1 -- 172.21.15.64:0/3764766433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e2406c460 con 0x7f6e2415cee0 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e23fff640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415cee0 0x7f6e241584e0 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f6e1000c3d0 tx=0x7f6e1000c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:36.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e217fa640 1 -- 172.21.15.64:0/3764766433 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e10017070 con 0x7f6e2415cee0 2024-03-04T07:29:36.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e217fa640 1 -- 172.21.15.64:0/3764766433 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6e100052f0 con 0x7f6e2415cee0 2024-03-04T07:29:36.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6e2406c180 con 0x7f6e2415cee0 2024-03-04T07:29:36.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e217fa640 1 -- 172.21.15.64:0/3764766433 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e10003640 con 0x7f6e2415cee0 2024-03-04T07:29:36.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.359+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6e24157fe0 con 0x7f6e2415cee0 2024-03-04T07:29:36.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.360+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6e2404f330 con 0x7f6e2415cee0 2024-03-04T07:29:36.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.361+0000 7f6e217fa640 1 -- 172.21.15.64:0/3764766433 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6e1001a070 con 0x7f6e2415cee0 2024-03-04T07:29:36.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.361+0000 7f6e217fa640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6e0806b4d0 0x7f6e0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:36.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.361+0000 7f6e217fa640 1 -- 172.21.15.64:0/3764766433 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6e10089530 con 0x7f6e2415cee0 2024-03-04T07:29:36.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.361+0000 7f6e2899e640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6e0806b4d0 0x7f6e0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:36.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.361+0000 7f6e2899e640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6e0806b4d0 0x7f6e0806d990 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f6e18036000 tx=0x7f6e18008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:36.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.365+0000 7f6e217fa640 1 -- 172.21.15.64:0/3764766433 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6e10058440 con 0x7f6e2415cee0 2024-03-04T07:29:36.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.562+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f6e240d1220 con 0x7f6e2415cee0 2024-03-04T07:29:36.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.562+0000 7f6e217fa640 1 -- 172.21.15.64:0/3764766433 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v55) v1 ==== 80+0+0 (secure 0 0 0) 0x7f6e10057d40 con 0x7f6e2415cee0 2024-03-04T07:29:36.562 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:29:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6e0806b4d0 msgr2=0x7f6e0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6e0806b4d0 0x7f6e0806d990 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f6e18036000 tx=0x7f6e18008030 comp rx=0 tx=0).stop 2024-03-04T07:29:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415cee0 msgr2=0x7f6e241584e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415cee0 0x7f6e241584e0 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f6e1000c3d0 tx=0x7f6e1000c890 comp rx=0 tx=0).stop 2024-03-04T07:29:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 shutdown_connections 2024-03-04T07:29:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6e0806b4d0 0x7f6e0806d990 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6e2415f7e0 0x7f6e241590f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6e2415cee0 0x7f6e241584e0 unknown :-1 s=CLOSED pgs=726 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 --2- 172.21.15.64:0/3764766433 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6e240cec00 0x7f6e2406c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 >> 172.21.15.64:0/3764766433 conn(0x7f6e240b3430 msgr2=0x7f6e240cfe00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 shutdown_connections 2024-03-04T07:29:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:36.565+0000 7f6e299a0640 1 -- 172.21.15.64:0/3764766433 wait complete. 2024-03-04T07:29:36.576 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:29:36.576 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:36.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:36.602 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[58959]: fuse finished with error 0 and tester_r 0 2024-03-04T07:29:36.604 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:42.705 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:29:42.705 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:42.706 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:29:42.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:42.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.865+0000 7ff8af1e2640 1 Processor -- start 2024-03-04T07:29:42.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.865+0000 7ff8af1e2640 1 -- start start 2024-03-04T07:29:42.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.865+0000 7ff8af1e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 0x7ff8a8165a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.865+0000 7ff8af1e2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8a8071240 con 0x7ff8a8158520 2024-03-04T07:29:42.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.865+0000 7ff8af1e2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8a8071520 con 0x7ff8a806df30 2024-03-04T07:29:42.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.865+0000 7ff8af1e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a8071b70 con 0x7ff8a80aee10 2024-03-04T07:29:42.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8acf7e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff8a8158520 0x7ff8a806d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34568/0 (socket says 172.21.15.64:34568) 2024-03-04T07:29:42.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8acf7e640 1 -- 172.21.15.64:0/1505096244 learned_addr learned my addr 172.21.15.64:0/1505096244 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3121883435 0 0) 0x7ff8a8071240 con 0x7ff8a8158520 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff898003520 con 0x7ff8a8158520 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8ad77f640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 0x7ff8a8165a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4006518750 0 0) 0x7ff898003520 con 0x7ff8a8158520 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 msgr2=0x7ff8a8165a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 0x7ff8a8165a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 >> v1:172.21.15.66:6789/0 conn(0x7ff8a806df30 legacy=0x7ff8a80ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8a8071800 con 0x7ff8a8158520 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7ff89c001670 con 0x7ff8a8158520 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 >> v1:172.21.15.64:6789/0 conn(0x7ff8a8158520 legacy=0x7ff8a806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8980039e0 0x7ff898003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff898004800 0x7ff898004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 0x7ff8980051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a8071b70 con 0x7ff8980039e0 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff89800ba70 con 0x7ff898004800 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff89800bc70 con 0x7ff8a80aee10 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8acf7e640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8980039e0 0x7ff898003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8ad77f640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 0x7ff8980051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.866+0000 7ff8a7fff640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff898004800 0x7ff898004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.867+0000 7ff8acf7e640 1 -- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 msgr2=0x7ff8980051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.867+0000 7ff8acf7e640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a80aee10 0x7ff8980051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.867+0000 7ff8acf7e640 1 -- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff898004800 msgr2=0x7ff898004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.867+0000 7ff8acf7e640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff898004800 0x7ff898004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.867+0000 7ff8acf7e640 1 -- 172.21.15.64:0/1505096244 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff898003520 con 0x7ff8980039e0 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.867+0000 7ff8acf7e640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8980039e0 0x7ff898003e00 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7ff89c00c210 tx=0x7ff898005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a2c3cef23f825c19 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:42.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff89c007cf0 con 0x7ff8980039e0 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff89c030070 con 0x7ff8980039e0 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8a77fe640 1 -- 172.21.15.64:0/1505096244 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff89c005d30 con 0x7ff8980039e0 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8af1e2640 1 -- 172.21.15.64:0/1505096244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8980039e0 msgr2=0x7ff898003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8af1e2640 1 --2- 172.21.15.64:0/1505096244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8980039e0 0x7ff898003e00 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7ff89c00c210 tx=0x7ff898005830 comp rx=0 tx=0).stop 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8ad77f640 1 -- 172.21.15.64:0/1505096244 reap_dead start 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8af1e2640 1 -- 172.21.15.64:0/1505096244 shutdown_connections 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8af1e2640 1 -- 172.21.15.64:0/1505096244 >> 172.21.15.64:0/1505096244 conn(0x7ff8a8066d00 msgr2=0x7ff8a80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8af1e2640 1 -- 172.21.15.64:0/1505096244 shutdown_connections 2024-03-04T07:29:42.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.868+0000 7ff8af1e2640 1 -- 172.21.15.64:0/1505096244 wait complete. 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.869+0000 7ff8af1e2640 1 Processor -- start 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.869+0000 7ff8af1e2640 1 -- start start 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8af1e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8980039e0 0x7ff8a81f4a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8af1e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a8156870 0x7ff8a81e43d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8af1e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8a806cab0 0x7ff8a806cf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8af1e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a8158900 con 0x7ff8a806cab0 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8af1e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a8072000 con 0x7ff8980039e0 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8af1e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a8158a80 con 0x7ff8a8156870 2024-03-04T07:29:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8ad77f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8a806cab0 0x7ff8a806cf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8ad77f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8a806cab0 0x7ff8a806cf30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36950/0 (socket says 172.21.15.64:36950) 2024-03-04T07:29:42.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8ad77f640 1 -- 172.21.15.64:0/247687297 learned_addr learned my addr 172.21.15.64:0/247687297 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:42.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8a7fff640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a8156870 0x7ff8a81e43d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.870+0000 7ff8acf7e640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8980039e0 0x7ff8a81f4a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8ad77f640 1 -- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a8156870 msgr2=0x7ff8a81e43d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:42.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8ad77f640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a8156870 0x7ff8a81e43d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:42.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8ad77f640 1 -- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8980039e0 msgr2=0x7ff8a81f4a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:42.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8ad77f640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8980039e0 0x7ff8a81f4a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:42.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8ad77f640 1 -- 172.21.15.64:0/247687297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8a8156da0 con 0x7ff8a806cab0 2024-03-04T07:29:42.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8a7fff640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a8156870 0x7ff8a81e43d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:42.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8ad77f640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8a806cab0 0x7ff8a806cf30 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7ff8a0011c80 tx=0x7ff8a000f580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:42.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8a57fa640 1 -- 172.21.15.64:0/247687297 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a0015430 con 0x7ff8a806cab0 2024-03-04T07:29:42.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8a57fa640 1 -- 172.21.15.64:0/247687297 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff8a001d070 con 0x7ff8a806cab0 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8a57fa640 1 -- 172.21.15.64:0/247687297 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a0015810 con 0x7ff8a806cab0 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8a8157080 con 0x7ff8a806cab0 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.871+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8a8157640 con 0x7ff8a806cab0 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.872+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff8a8067ff0 con 0x7ff8a806cab0 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.873+0000 7ff8a57fa640 1 -- 172.21.15.64:0/247687297 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff8a00159b0 con 0x7ff8a806cab0 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.873+0000 7ff8a57fa640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff88c06b4d0 0x7ff88c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.873+0000 7ff8a57fa640 1 -- 172.21.15.64:0/247687297 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff8a008aaa0 con 0x7ff8a806cab0 2024-03-04T07:29:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.873+0000 7ff8acf7e640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff88c06b4d0 0x7ff88c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:42.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.873+0000 7ff8acf7e640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff88c06b4d0 0x7ff88c06d990 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7ff89c034000 tx=0x7ff89c007000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:42.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:42.877+0000 7ff8a57fa640 1 -- 172.21.15.64:0/247687297 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff8a0010040 con 0x7ff8a806cab0 2024-03-04T07:29:43.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.218+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff8a80b31f0 con 0x7ff8a806cab0 2024-03-04T07:29:43.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.218+0000 7ff8a57fa640 1 -- 172.21.15.64:0/247687297 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7ff8a0059920 con 0x7ff8a806cab0 2024-03-04T07:29:43.218 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:43.218 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:43.219 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:43.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff88c06b4d0 msgr2=0x7ff88c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff88c06b4d0 0x7ff88c06d990 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7ff89c034000 tx=0x7ff89c007000 comp rx=0 tx=0).stop 2024-03-04T07:29:43.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8a806cab0 msgr2=0x7ff8a806cf30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8a806cab0 0x7ff8a806cf30 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7ff8a0011c80 tx=0x7ff8a000f580 comp rx=0 tx=0).stop 2024-03-04T07:29:43.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 shutdown_connections 2024-03-04T07:29:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff88c06b4d0 0x7ff88c06d990 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8a806cab0 0x7ff8a806cf30 unknown :-1 s=CLOSED pgs=729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8a8156870 0x7ff8a81e43d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 --2- 172.21.15.64:0/247687297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8980039e0 0x7ff8a81f4a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 >> 172.21.15.64:0/247687297 conn(0x7ff8a8066d00 msgr2=0x7ff8a81590f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 shutdown_connections 2024-03-04T07:29:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.220+0000 7ff8af1e2640 1 -- 172.21.15.64:0/247687297 wait complete. 2024-03-04T07:29:43.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json dirfrag split /subdir 0/0 1 2024-03-04T07:29:43.347 DEBUG:tasks.cephfs.filesystem:_json_asok output empty 2024-03-04T07:29:43.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:43.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.452+0000 7ffb58015640 1 Processor -- start 2024-03-04T07:29:43.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.452+0000 7ffb58015640 1 -- start start 2024-03-04T07:29:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.453+0000 7ffb58015640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ffb5015be60 0x7ffb500719b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.453+0000 7ffb58015640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffb5015fcc0 con 0x7ffb50066d00 2024-03-04T07:29:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.453+0000 7ffb58015640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffb5015ffa0 con 0x7ffb5015f740 2024-03-04T07:29:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.453+0000 7ffb58015640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ffb50160280 con 0x7ffb5015be60 2024-03-04T07:29:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.453+0000 7ffb565b2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ffb5015be60 0x7ffb500719b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.453+0000 7ffb55db1640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ffb50066d00 0x7ffb500670e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41266/0 (socket says 172.21.15.64:41266) 2024-03-04T07:29:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.453+0000 7ffb55db1640 1 -- 172.21.15.64:0/392468381 learned_addr learned my addr 172.21.15.64:0/392468381 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.454+0000 7ffb565b2640 1 -- 172.21.15.64:0/392468381 >> v1:172.21.15.66:6789/0 conn(0x7ffb5015f740 legacy=0x7ffb5006fdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.454+0000 7ffb565b2640 1 -- 172.21.15.64:0/392468381 >> v1:172.21.15.64:6789/0 conn(0x7ffb50066d00 legacy=0x7ffb500670e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.454+0000 7ffb54daf640 1 -- 172.21.15.64:0/392468381 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3227743422 0 0) 0x7ffb5015fcc0 con 0x7ffb50066d00 2024-03-04T07:29:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.454+0000 7ffb565b2640 1 -- 172.21.15.64:0/392468381 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb50160560 con 0x7ffb5015be60 2024-03-04T07:29:43.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.455+0000 7ffb565b2640 1 --2- 172.21.15.64:0/392468381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb5015be60 0x7ffb500719b0 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7ffb48009d90 tx=0x7ffb4802bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5508c58c11a11c4a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:43.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.455+0000 7ffb54daf640 1 -- 172.21.15.64:0/392468381 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb48039070 con 0x7ffb5015be60 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.455+0000 7ffb54daf640 1 -- 172.21.15.64:0/392468381 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffb4802cd00 con 0x7ffb5015be60 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.455+0000 7ffb54daf640 1 -- 172.21.15.64:0/392468381 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb48034660 con 0x7ffb5015be60 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.456+0000 7ffb58015640 1 -- 172.21.15.64:0/392468381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb5015be60 msgr2=0x7ffb500719b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.456+0000 7ffb58015640 1 --2- 172.21.15.64:0/392468381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb5015be60 0x7ffb500719b0 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7ffb48009d90 tx=0x7ffb4802bf50 comp rx=0 tx=0).stop 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.456+0000 7ffb58015640 1 -- 172.21.15.64:0/392468381 shutdown_connections 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.456+0000 7ffb58015640 1 --2- 172.21.15.64:0/392468381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb5015be60 0x7ffb500719b0 unknown :-1 s=CLOSED pgs=731 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.456+0000 7ffb58015640 1 -- 172.21.15.64:0/392468381 >> 172.21.15.64:0/392468381 conn(0x7ffb500d0990 msgr2=0x7ffb500d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.456+0000 7ffb58015640 1 -- 172.21.15.64:0/392468381 shutdown_connections 2024-03-04T07:29:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.456+0000 7ffb58015640 1 -- 172.21.15.64:0/392468381 wait complete. 2024-03-04T07:29:43.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.457+0000 7ffb58015640 1 Processor -- start 2024-03-04T07:29:43.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.457+0000 7ffb58015640 1 -- start start 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb50066d00 0x7ffb5015a710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffb5015be60 0x7ffb500ae300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffb5015f740 0x7ffb501f7510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb5015c6e0 con 0x7ffb50066d00 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb50072100 con 0x7ffb5015be60 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffb5015c860 con 0x7ffb5015f740 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb50066d00 0x7ffb5015a710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb50066d00 0x7ffb5015a710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40390/0 (socket says 172.21.15.64:40390) 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 -- 172.21.15.64:0/1929285455 learned_addr learned my addr 172.21.15.64:0/1929285455 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb555b0640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffb5015be60 0x7ffb500ae300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb565b2640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffb5015f740 0x7ffb501f7510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 -- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffb5015f740 msgr2=0x7ffb501f7510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffb5015f740 0x7ffb501f7510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 -- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffb5015be60 msgr2=0x7ffb500ae300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffb5015be60 0x7ffb500ae300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 -- 172.21.15.64:0/1929285455 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb501f7f90 con 0x7ffb50066d00 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb565b2640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffb5015f740 0x7ffb501f7510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb55db1640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb50066d00 0x7ffb5015a710 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c00c4a0 tx=0x7ffb4c00c960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb467fc640 1 -- 172.21.15.64:0/1929285455 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4c018070 con 0x7ffb50066d00 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb467fc640 1 -- 172.21.15.64:0/1929285455 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffb4c0074e0 con 0x7ffb50066d00 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffb501f8270 con 0x7ffb50066d00 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb467fc640 1 -- 172.21.15.64:0/1929285455 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4c004a20 con 0x7ffb50066d00 2024-03-04T07:29:43.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.458+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb500adde0 con 0x7ffb50066d00 2024-03-04T07:29:43.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.459+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffb50068700 con 0x7ffb50066d00 2024-03-04T07:29:43.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.460+0000 7ffb467fc640 1 -- 172.21.15.64:0/1929285455 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffb4c004bc0 con 0x7ffb50066d00 2024-03-04T07:29:43.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.460+0000 7ffb467fc640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffb3406b4d0 0x7ffb3406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.461+0000 7ffb555b0640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffb3406b4d0 0x7ffb3406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.461+0000 7ffb467fc640 1 -- 172.21.15.64:0/1929285455 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ffb4c089590 con 0x7ffb50066d00 2024-03-04T07:29:43.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.461+0000 7ffb555b0640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffb3406b4d0 0x7ffb3406d990 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7ffb5006fcb0 tx=0x7ffb40009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:43.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.468+0000 7ffb467fc640 1 -- 172.21.15.64:0/1929285455 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffb4c057d40 con 0x7ffb50066d00 2024-03-04T07:29:43.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.692+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ffb500d1790 con 0x7ffb50066d00 2024-03-04T07:29:43.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.692+0000 7ffb467fc640 1 -- 172.21.15.64:0/1929285455 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7ffb4c05bde0 con 0x7ffb50066d00 2024-03-04T07:29:43.692 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:43.693 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:43.693 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:43.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffb3406b4d0 msgr2=0x7ffb3406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffb3406b4d0 0x7ffb3406d990 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7ffb5006fcb0 tx=0x7ffb40009040 comp rx=0 tx=0).stop 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb50066d00 msgr2=0x7ffb5015a710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb50066d00 0x7ffb5015a710 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c00c4a0 tx=0x7ffb4c00c960 comp rx=0 tx=0).stop 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 shutdown_connections 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffb3406b4d0 0x7ffb3406d990 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffb5015f740 0x7ffb501f7510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffb5015be60 0x7ffb500ae300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 --2- 172.21.15.64:0/1929285455 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffb50066d00 0x7ffb5015a710 unknown :-1 s=CLOSED pgs=732 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 >> 172.21.15.64:0/1929285455 conn(0x7ffb500d0990 msgr2=0x7ffb5006af20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 shutdown_connections 2024-03-04T07:29:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.694+0000 7ffb58015640 1 -- 172.21.15.64:0/1929285455 wait complete. 2024-03-04T07:29:43.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-04T07:29:43.839 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:29:43.839 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:29:43.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:29:43.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.941+0000 7f481cc0e640 1 Processor -- start 2024-03-04T07:29:43.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.941+0000 7f481cc0e640 1 -- start start 2024-03-04T07:29:43.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f481cc0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f481cc0e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f48180b0e90 0x7f481806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f481cc0e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f48180b3340 con 0x7f48180ae9b0 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f481cc0e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f48180cfa00 con 0x7f48180b0e90 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f481cc0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f48180cee90 con 0x7f481806b5c0 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f480dd9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f48180b0e90 0x7f481806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f4816d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f48180ae9b0 0x7f48180d03a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41280/0 (socket says 172.21.15.64:41280) 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f4816d9d640 1 -- 172.21.15.64:0/1055409904 learned_addr learned my addr 172.21.15.64:0/1055409904 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f481659c640 1 --2- 172.21.15.64:0/1055409904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f4815d9b640 1 -- 172.21.15.64:0/1055409904 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1732974990 0 0) 0x7f48180b3340 con 0x7f48180ae9b0 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f4815d9b640 1 -- 172.21.15.64:0/1055409904 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4804003520 con 0x7f48180ae9b0 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f480dd9b640 1 -- 172.21.15.64:0/1055409904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 msgr2=0x7f48180b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f480dd9b640 1 --2- 172.21.15.64:0/1055409904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f480dd9b640 1 -- 172.21.15.64:0/1055409904 >> v1:172.21.15.64:6789/0 conn(0x7f48180ae9b0 legacy=0x7f48180d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.942+0000 7f480dd9b640 1 -- 172.21.15.64:0/1055409904 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48180cfc50 con 0x7f48180b0e90 2024-03-04T07:29:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.943+0000 7f480dd9b640 1 --2- 172.21.15.64:0/1055409904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180b0e90 0x7f481806b9a0 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7f48000097a0 tx=0x7f480002b820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8df868c4276cf1d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.943+0000 7f4815d9b640 1 -- 172.21.15.64:0/1055409904 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4800031780 con 0x7f48180b0e90 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.943+0000 7f4815d9b640 1 -- 172.21.15.64:0/1055409904 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4800037070 con 0x7f48180b0e90 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.943+0000 7f4815d9b640 1 -- 172.21.15.64:0/1055409904 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4800031b40 con 0x7f48180b0e90 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 -- 172.21.15.64:0/1055409904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180b0e90 msgr2=0x7f481806b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 --2- 172.21.15.64:0/1055409904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180b0e90 0x7f481806b9a0 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7f48000097a0 tx=0x7f480002b820 comp rx=0 tx=0).stop 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 -- 172.21.15.64:0/1055409904 shutdown_connections 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 --2- 172.21.15.64:0/1055409904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180b0e90 0x7f481806b9a0 unknown :-1 s=CLOSED pgs=734 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 --2- 172.21.15.64:0/1055409904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 -- 172.21.15.64:0/1055409904 >> 172.21.15.64:0/1055409904 conn(0x7f48180b36d0 msgr2=0x7f48180b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 -- 172.21.15.64:0/1055409904 shutdown_connections 2024-03-04T07:29:43.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.944+0000 7f481cc0e640 1 -- 172.21.15.64:0/1055409904 wait complete. 2024-03-04T07:29:43.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.945+0000 7f481cc0e640 1 Processor -- start 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.945+0000 7f481cc0e640 1 -- start start 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f481cc0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180cc190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f481cc0e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180ae9b0 0x7f48180c58a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f481cc0e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48180b0e90 0x7f48180c7b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f481cc0e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481806c790 con 0x7f48180ae9b0 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f481cc0e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481806bee0 con 0x7f48180b0e90 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f481cc0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f481806c910 con 0x7f481806b5c0 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180ae9b0 0x7f48180c58a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180ae9b0 0x7f48180c58a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40406/0 (socket says 172.21.15.64:40406) 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 -- 172.21.15.64:0/2214234137 learned_addr learned my addr 172.21.15.64:0/2214234137 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f481659c640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180cc190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f4816d9d640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48180b0e90 0x7f48180c7b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 -- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 msgr2=0x7f48180cc190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180cc190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 -- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48180b0e90 msgr2=0x7f48180c7b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48180b0e90 0x7f48180c7b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.946+0000 7f480dd9b640 1 -- 172.21.15.64:0/2214234137 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48180cc6d0 con 0x7f48180ae9b0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.947+0000 7f480dd9b640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180ae9b0 0x7f48180c58a0 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7f4800034040 tx=0x7f48000046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.947+0000 7f480f7fe640 1 -- 172.21.15.64:0/2214234137 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4800032610 con 0x7f48180ae9b0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.947+0000 7f480f7fe640 1 -- 172.21.15.64:0/2214234137 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f48000327b0 con 0x7f48180ae9b0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.947+0000 7f480f7fe640 1 -- 172.21.15.64:0/2214234137 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4800032ba0 con 0x7f48180ae9b0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.947+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f48180cc970 con 0x7f48180ae9b0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.947+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48180ccf30 con 0x7f48180ae9b0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.948+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4818068700 con 0x7f48180ae9b0 2024-03-04T07:29:43.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.948+0000 7f480f7fe640 1 -- 172.21.15.64:0/2214234137 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f480004e440 con 0x7f48180ae9b0 2024-03-04T07:29:43.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.949+0000 7f480f7fe640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47f806b4d0 0x7f47f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:43.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.949+0000 7f481659c640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47f806b4d0 0x7f47f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:43.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.949+0000 7f480f7fe640 1 -- 172.21.15.64:0/2214234137 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f48000abd60 con 0x7f48180ae9b0 2024-03-04T07:29:43.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.949+0000 7f481659c640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47f806b4d0 0x7f47f806d990 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f48080079a0 tx=0x7f4808008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:43.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:43.953+0000 7f480f7fe640 1 -- 172.21.15.64:0/2214234137 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4800040070 con 0x7f48180ae9b0 2024-03-04T07:29:44.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.172+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f481806c550 con 0x7f48180ae9b0 2024-03-04T07:29:44.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.172+0000 7f480f7fe640 1 -- 172.21.15.64:0/2214234137 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v29) v1 ==== 53+0+73 (secure 0 0 0) 0x7f480002f050 con 0x7f48180ae9b0 2024-03-04T07:29:44.172 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47f806b4d0 msgr2=0x7f47f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47f806b4d0 0x7f47f806d990 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f48080079a0 tx=0x7f4808008040 comp rx=0 tx=0).stop 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180ae9b0 msgr2=0x7f48180c58a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180ae9b0 0x7f48180c58a0 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7f4800034040 tx=0x7f48000046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 shutdown_connections 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47f806b4d0 0x7f47f806d990 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f48180b0e90 0x7f48180c7b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:44.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f48180ae9b0 0x7f48180c58a0 unknown :-1 s=CLOSED pgs=735 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:44.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 --2- 172.21.15.64:0/2214234137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f481806b5c0 0x7f48180cc190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:44.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 >> 172.21.15.64:0/2214234137 conn(0x7f48180b36d0 msgr2=0x7f481815e210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:44.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 shutdown_connections 2024-03-04T07:29:44.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:44.174+0000 7f481cc0e640 1 -- 172.21.15.64:0/2214234137 wait complete. 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:29:44.188 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:29:44.189 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.189 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:29:44.205 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:29:44.205 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:29:44.205 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:29:44.205 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:44.205 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:29:44.205 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:44.205 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42528sec preferred_lft 42528sec 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:29:44.206 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:29:44.207 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:29:44.208 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.208 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:29:44.270 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:29:44.272 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.272 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:29:44.327 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:29:44.327 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:29:44.327 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:29:44.327 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:29:44.327 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:29:44.328 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.328 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:29:44.328 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:29:44.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:29:44.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:29:44.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:29:44.328 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:29:44.422 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.423 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:29:44.437 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:29:44.437 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:29:44.437 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:29:44.437 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:29:44.504 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:29:44.504 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:29:44.504 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:44.559 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:29:44.559 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:29:44.559 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:44.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:29:44.682 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:29:44.737 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:29:44.738 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:29:44.739 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:29:44.740 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:29:44.740 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:29:44.740 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:44.740 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:44.740 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:44.740 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:44.740 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.740 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:29:44.792 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:29:44.792 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:29:44.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:29:44.862 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:29:44.917 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:29:44.918 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:29:44.919 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:29:44.920 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:44.920 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:29:44.925 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:29:44.925+0000 7f857ef67480 -1 init, newargv = 0x55dc3a0668f0 newargc=15 2024-03-04T07:29:44.925 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[60105]: starting ceph client 2024-03-04T07:29:44.930 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[60105]: starting fuse 2024-03-04T07:29:44.940 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:29:44.940 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:29:44.941 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:29:44.942 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:29:45.064 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.60105.asok 2024-03-04T07:29:45.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:45 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:29:45.074 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.60105.asok 2024-03-04T07:29:45.075 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:45.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.60105.asok status 2024-03-04T07:29:45.212 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:29:45.212 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:29:45.212 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:29:45.212 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:29:45.212 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:29:45.212 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "60105", 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 6360, 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 6360 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:29:45.213 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:29:45.214 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 937858182 2024-03-04T07:29:45.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:29:45.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:45.214 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:29:45.214 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:29:45.214 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:29:45.214 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 937858182 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.6360 192.168.0.1:0/937858182", 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/937858182", 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 29, 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 55, 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:29:45.215 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:29:45.220 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:45.221 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:29:45.221 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:45.236 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:29:45.236 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:45.236 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:45.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:45.303 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:45.303 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:45.358 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> import uuid 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> n = 75 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = False 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> if 0 > 0: 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = True 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> dpath = os.path.dirname(path) 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> fnameprefix = os.path.basename(path) 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> os.makedirs(dpath, exist_ok=True) 2024-03-04T07:29:45.359 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> try: 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> for i in range(n): 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> with open(fpath, '"'"'w'"'"') as f: 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> f.write(f"{i}") 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> f.flush() 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(f.fileno()) 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> os.unlink(fpath) 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:29:45.360 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> if create_hard_links: 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> for j in range(0): 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> if True: 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> finally: 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> os.close(dirfd) 2024-03-04T07:29:45.361 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:29:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:45 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:29:45.804 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:29:45.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:29:45.893 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:29:45.941 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:29:45.941 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:45.941 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:45.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.098+0000 7f7f3b6c9640 1 Processor -- start 2024-03-04T07:29:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.098+0000 7f7f3b6c9640 1 -- start start 2024-03-04T07:29:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f3b6c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7f3415df00 0x7f7f340d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f3b6c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7f340b95a0 0x7f7f340b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f3b6c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7f340cea30 con 0x7f7f340b9a30 2024-03-04T07:29:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f3b6c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7f340d2af0 con 0x7f7f340b95a0 2024-03-04T07:29:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f3b6c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7f340cf3c0 con 0x7f7f3415df00 2024-03-04T07:29:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f38c64640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7f340b95a0 0x7f7f340b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f38c64640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7f340b95a0 0x7f7f340b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40418/0 (socket says 172.21.15.64:40418) 2024-03-04T07:29:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f38c64640 1 -- 172.21.15.64:0/1976749735 learned_addr learned my addr 172.21.15.64:0/1976749735 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f39465640 1 --2- 172.21.15.64:0/1976749735 >> v2:172.21.15.66:3300/0 conn(0x7f7f3415df00 0x7f7f340d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f38c64640 1 -- 172.21.15.64:0/1976749735 >> v2:172.21.15.66:3300/0 conn(0x7f7f3415df00 msgr2=0x7f7f340d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f38c64640 1 --2- 172.21.15.64:0/1976749735 >> v2:172.21.15.66:3300/0 conn(0x7f7f3415df00 0x7f7f340d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f38c64640 1 -- 172.21.15.64:0/1976749735 >> v1:172.21.15.66:6789/0 conn(0x7f7f340b9a30 legacy=0x7f7f340d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:46.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.099+0000 7f7f38c64640 1 -- 172.21.15.64:0/1976749735 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f340cf0e0 con 0x7f7f340b95a0 2024-03-04T07:29:46.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.100+0000 7f7f38c64640 1 --2- 172.21.15.64:0/1976749735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f340b95a0 0x7f7f340b5700 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f7f340d11b0 tx=0x7f7f3002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be24b9f341c82b1e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.100+0000 7f7f2bfff640 1 -- 172.21.15.64:0/1976749735 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f30034070 con 0x7f7f340b95a0 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.100+0000 7f7f2bfff640 1 -- 172.21.15.64:0/1976749735 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f30030860 con 0x7f7f340b95a0 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.100+0000 7f7f2bfff640 1 -- 172.21.15.64:0/1976749735 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f30030c30 con 0x7f7f340b95a0 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1976749735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f340b95a0 msgr2=0x7f7f340b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1976749735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f340b95a0 0x7f7f340b5700 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f7f340d11b0 tx=0x7f7f3002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1976749735 shutdown_connections 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1976749735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f340b95a0 0x7f7f340b5700 unknown :-1 s=CLOSED pgs=739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1976749735 >> v2:172.21.15.66:3300/0 conn(0x7f7f3415df00 0x7f7f340d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1976749735 >> 172.21.15.64:0/1976749735 conn(0x7f7f34071960 msgr2=0x7f7f34071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1976749735 shutdown_connections 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.101+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1976749735 wait complete. 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.102+0000 7f7f3b6c9640 1 Processor -- start 2024-03-04T07:29:46.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.102+0000 7f7f3b6c9640 1 -- start start 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f3b6c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f340b95a0 0x7f7f341553e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f3b6c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f3415df00 0x7f7f340cb640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f3b6c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f340cc960 0x7f7f341f3930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f3b6c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f340cec70 con 0x7f7f3415df00 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f3b6c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f340ce750 con 0x7f7f340b95a0 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f3b6c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7f340cedf0 con 0x7f7f340cc960 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f3415df00 0x7f7f340cb640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f3415df00 0x7f7f340cb640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40424/0 (socket says 172.21.15.64:40424) 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 -- 172.21.15.64:0/1096822947 learned_addr learned my addr 172.21.15.64:0/1096822947 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f39465640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f340b95a0 0x7f7f341553e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f39c66640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f340cc960 0x7f7f341f3930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 -- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f340cc960 msgr2=0x7f7f341f3930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f340cc960 0x7f7f341f3930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 -- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f340b95a0 msgr2=0x7f7f341553e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f340b95a0 0x7f7f341553e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.103+0000 7f7f38c64640 1 -- 172.21.15.64:0/1096822947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f340cae80 con 0x7f7f3415df00 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.104+0000 7f7f38c64640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f3415df00 0x7f7f340cb640 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f7f3002b9e0 tx=0x7f7f30004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.104+0000 7f7f29ffb640 1 -- 172.21.15.64:0/1096822947 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f30048070 con 0x7f7f3415df00 2024-03-04T07:29:46.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.104+0000 7f7f29ffb640 1 -- 172.21.15.64:0/1096822947 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7f30030660 con 0x7f7f3415df00 2024-03-04T07:29:46.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.104+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7f341f48a0 con 0x7f7f3415df00 2024-03-04T07:29:46.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.104+0000 7f7f29ffb640 1 -- 172.21.15.64:0/1096822947 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f30034040 con 0x7f7f3415df00 2024-03-04T07:29:46.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.104+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7f341f45c0 con 0x7f7f3415df00 2024-03-04T07:29:46.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.105+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7f34067ff0 con 0x7f7f3415df00 2024-03-04T07:29:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.106+0000 7f7f29ffb640 1 -- 172.21.15.64:0/1096822947 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7f3004b070 con 0x7f7f3415df00 2024-03-04T07:29:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.106+0000 7f7f29ffb640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1806b4d0 0x7f7f1806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.106+0000 7f7f29ffb640 1 -- 172.21.15.64:0/1096822947 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7f300ac140 con 0x7f7f3415df00 2024-03-04T07:29:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.106+0000 7f7f39465640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1806b4d0 0x7f7f1806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.106+0000 7f7f39465640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1806b4d0 0x7f7f1806d990 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f7f240079a0 tx=0x7f7f24008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.110+0000 7f7f29ffb640 1 -- 172.21.15.64:0/1096822947 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7f3007aa00 con 0x7f7f3415df00 2024-03-04T07:29:46.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.293+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7f340b7cf0 con 0x7f7f3415df00 2024-03-04T07:29:46.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.294+0000 7f7f29ffb640 1 -- 172.21.15.64:0/1096822947 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v55) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f7f3007eaa0 con 0x7f7f3415df00 2024-03-04T07:29:46.294 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:46.294 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:12.504021+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:46.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.296+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1806b4d0 msgr2=0x7f7f1806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.296+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1806b4d0 0x7f7f1806d990 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f7f240079a0 tx=0x7f7f24008040 comp rx=0 tx=0).stop 2024-03-04T07:29:46.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.296+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f3415df00 msgr2=0x7f7f340cb640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.296+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f3415df00 0x7f7f340cb640 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f7f3002b9e0 tx=0x7f7f30004730 comp rx=0 tx=0).stop 2024-03-04T07:29:46.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.296+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 shutdown_connections 2024-03-04T07:29:46.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.296+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7f1806b4d0 0x7f7f1806d990 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.297+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7f340cc960 0x7f7f341f3930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.297+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7f3415df00 0x7f7f340cb640 unknown :-1 s=CLOSED pgs=740 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.297+0000 7f7f3b6c9640 1 --2- 172.21.15.64:0/1096822947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7f340b95a0 0x7f7f341553e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.297+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 >> 172.21.15.64:0/1096822947 conn(0x7f7f34071960 msgr2=0x7f7f340d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:46.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.297+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 shutdown_connections 2024-03-04T07:29:46.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.297+0000 7f7f3b6c9640 1 -- 172.21.15.64:0/1096822947 wait complete. 2024-03-04T07:29:46.308 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 8, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:29:09.132151+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '54', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 9, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:29:09.825115+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '55', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:29:46.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:29:46.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.414+0000 7f1035488640 1 Processor -- start 2024-03-04T07:29:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.414+0000 7f1035488640 1 -- start start 2024-03-04T07:29:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f1035488640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f103015dd90 0x7f10300ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f1035488640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f10300b6d10 0x7f10300b7150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f1035488640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f10300d3740 con 0x7f10300d29d0 2024-03-04T07:29:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f1035488640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f10300d3400 con 0x7f10300b6d10 2024-03-04T07:29:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f1035488640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f10300b7690 con 0x7f103015dd90 2024-03-04T07:29:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f10300b6d10 0x7f10300b7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102e7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f10300d29d0 0x7f10300d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41288/0 (socket says 172.21.15.64:41288) 2024-03-04T07:29:46.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102e7fc640 1 -- 172.21.15.64:0/867759067 learned_addr learned my addr 172.21.15.64:0/867759067 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:46.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102effd640 1 --2- 172.21.15.64:0/867759067 >> v2:172.21.15.66:3300/0 conn(0x7f103015dd90 0x7f10300ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102f7fe640 1 -- 172.21.15.64:0/867759067 >> v2:172.21.15.66:3300/0 conn(0x7f103015dd90 msgr2=0x7f10300ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:46.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102f7fe640 1 --2- 172.21.15.64:0/867759067 >> v2:172.21.15.66:3300/0 conn(0x7f103015dd90 0x7f10300ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102f7fe640 1 -- 172.21.15.64:0/867759067 >> v1:172.21.15.64:6789/0 conn(0x7f10300d29d0 legacy=0x7f10300d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102dffb640 1 -- 172.21.15.64:0/867759067 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3405775273 0 0) 0x7f10300d3740 con 0x7f10300d29d0 2024-03-04T07:29:46.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.415+0000 7f102f7fe640 1 -- 172.21.15.64:0/867759067 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10300d3120 con 0x7f10300b6d10 2024-03-04T07:29:46.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.416+0000 7f102f7fe640 1 --2- 172.21.15.64:0/867759067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300b6d10 0x7f10300b7150 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7f10300d1f90 tx=0x7f102802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bab7c43f028dc3ed server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.416+0000 7f102dffb640 1 -- 172.21.15.64:0/867759067 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1028034070 con 0x7f10300b6d10 2024-03-04T07:29:46.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.416+0000 7f102dffb640 1 -- 172.21.15.64:0/867759067 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1028030610 con 0x7f10300b6d10 2024-03-04T07:29:46.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.416+0000 7f102dffb640 1 -- 172.21.15.64:0/867759067 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10280309e0 con 0x7f10300b6d10 2024-03-04T07:29:46.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 -- 172.21.15.64:0/867759067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300b6d10 msgr2=0x7f10300b7150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 --2- 172.21.15.64:0/867759067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300b6d10 0x7f10300b7150 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7f10300d1f90 tx=0x7f102802b960 comp rx=0 tx=0).stop 2024-03-04T07:29:46.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 -- 172.21.15.64:0/867759067 shutdown_connections 2024-03-04T07:29:46.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 --2- 172.21.15.64:0/867759067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300b6d10 0x7f10300b7150 unknown :-1 s=CLOSED pgs=742 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 --2- 172.21.15.64:0/867759067 >> v2:172.21.15.66:3300/0 conn(0x7f103015dd90 0x7f10300ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 -- 172.21.15.64:0/867759067 >> 172.21.15.64:0/867759067 conn(0x7f1030071960 msgr2=0x7f1030071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:46.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 -- 172.21.15.64:0/867759067 shutdown_connections 2024-03-04T07:29:46.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.417+0000 7f1035488640 1 -- 172.21.15.64:0/867759067 wait complete. 2024-03-04T07:29:46.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.418+0000 7f1035488640 1 Processor -- start 2024-03-04T07:29:46.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.418+0000 7f1035488640 1 -- start start 2024-03-04T07:29:46.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f1035488640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10300b6d10 0x7f103015e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f1035488640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300d29d0 0x7f10300d43b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f1035488640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f103015dd90 0x7f103014fbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f1035488640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f103015e170 con 0x7f10300d29d0 2024-03-04T07:29:46.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f1035488640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10300d3cc0 con 0x7f10300b6d10 2024-03-04T07:29:46.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f1035488640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10300d2df0 con 0x7f103015dd90 2024-03-04T07:29:46.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300d29d0 0x7f10300d43b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300d29d0 0x7f10300d43b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40438/0 (socket says 172.21.15.64:40438) 2024-03-04T07:29:46.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 -- 172.21.15.64:0/159164179 learned_addr learned my addr 172.21.15.64:0/159164179 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:46.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102effd640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10300b6d10 0x7f103015e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102f7fe640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f103015dd90 0x7f103014fbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 -- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f103015dd90 msgr2=0x7f103014fbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f103015dd90 0x7f103014fbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 -- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10300b6d10 msgr2=0x7f103015e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10300b6d10 0x7f103015e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.419+0000 7f102e7fc640 1 -- 172.21.15.64:0/159164179 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f103015fd70 con 0x7f10300d29d0 2024-03-04T07:29:46.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.420+0000 7f102e7fc640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300d29d0 0x7f10300d43b0 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f102400c4d0 tx=0x7f102400c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.420+0000 7f10177fe640 1 -- 172.21.15.64:0/159164179 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1024018070 con 0x7f10300d29d0 2024-03-04T07:29:46.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.420+0000 7f10177fe640 1 -- 172.21.15.64:0/159164179 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1024007500 con 0x7f10300d29d0 2024-03-04T07:29:46.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.420+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10301f7240 con 0x7f10300d29d0 2024-03-04T07:29:46.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.420+0000 7f10177fe640 1 -- 172.21.15.64:0/159164179 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1024004a20 con 0x7f10300d29d0 2024-03-04T07:29:46.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.420+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10301f7ae0 con 0x7f10300d29d0 2024-03-04T07:29:46.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.421+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f103015ff40 con 0x7f10300d29d0 2024-03-04T07:29:46.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.422+0000 7f10177fe640 1 -- 172.21.15.64:0/159164179 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f102401b070 con 0x7f10300d29d0 2024-03-04T07:29:46.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.422+0000 7f10177fe640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f101c06b4d0 0x7f101c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.422+0000 7f10177fe640 1 -- 172.21.15.64:0/159164179 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f10240893c0 con 0x7f10300d29d0 2024-03-04T07:29:46.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.422+0000 7f102effd640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f101c06b4d0 0x7f101c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.423+0000 7f102effd640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f101c06b4d0 0x7f101c06d990 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f103015ce90 tx=0x7f102000b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.426+0000 7f10177fe640 1 -- 172.21.15.64:0/159164179 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1024057c80 con 0x7f10300d29d0 2024-03-04T07:29:46.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.700+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f10300b5540 con 0x7f10300d29d0 2024-03-04T07:29:46.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.701+0000 7f10177fe640 1 -- 172.21.15.64:0/159164179 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v55) v1 ==== 93+0+10 (secure 0 0 0) 0x7f10300b5540 con 0x7f10300d29d0 2024-03-04T07:29:46.700 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:29:46.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f101c06b4d0 msgr2=0x7f101c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f101c06b4d0 0x7f101c06d990 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f103015ce90 tx=0x7f102000b040 comp rx=0 tx=0).stop 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300d29d0 msgr2=0x7f10300d43b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300d29d0 0x7f10300d43b0 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f102400c4d0 tx=0x7f102400c990 comp rx=0 tx=0).stop 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 shutdown_connections 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f101c06b4d0 0x7f101c06d990 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f103015dd90 0x7f103014fbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10300d29d0 0x7f10300d43b0 unknown :-1 s=CLOSED pgs=743 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 --2- 172.21.15.64:0/159164179 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10300b6d10 0x7f103015e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.702+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 >> 172.21.15.64:0/159164179 conn(0x7f1030071960 msgr2=0x7f10300b96c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.703+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 shutdown_connections 2024-03-04T07:29:46.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.703+0000 7f1035488640 1 -- 172.21.15.64:0/159164179 wait complete. 2024-03-04T07:29:46.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:29:46.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.818+0000 7f9ebf993640 1 Processor -- start 2024-03-04T07:29:46.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.818+0000 7f9ebf993640 1 -- start start 2024-03-04T07:29:46.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebf993640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb815f500 0x7f9eb80ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebf993640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9eb80c5e30 0x7f9eb80c6280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebf993640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9eb80cd310 con 0x7f9eb80cf740 2024-03-04T07:29:46.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebf993640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9eb815f8e0 con 0x7f9eb80c5e30 2024-03-04T07:29:46.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebf993640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9eb80c56c0 con 0x7f9eb815f500 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebdf30640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9eb80c5e30 0x7f9eb80c6280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebdf30640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9eb80c5e30 0x7f9eb80c6280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40454/0 (socket says 172.21.15.64:40454) 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebdf30640 1 -- 172.21.15.64:0/2102972969 learned_addr learned my addr 172.21.15.64:0/2102972969 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebd72f640 1 --2- 172.21.15.64:0/2102972969 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb815f500 0x7f9eb80ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebdf30640 1 -- 172.21.15.64:0/2102972969 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb815f500 msgr2=0x7f9eb80ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebdf30640 1 --2- 172.21.15.64:0/2102972969 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb815f500 0x7f9eb80ce300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.819+0000 7f9ebdf30640 1 -- 172.21.15.64:0/2102972969 >> v1:172.21.15.66:6789/0 conn(0x7f9eb80cf740 legacy=0x7f9eb80ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.820+0000 7f9ebdf30640 1 -- 172.21.15.64:0/2102972969 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9eb80cd5b0 con 0x7f9eb80c5e30 2024-03-04T07:29:46.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.820+0000 7f9ebdf30640 1 --2- 172.21.15.64:0/2102972969 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb80c5e30 0x7f9eb80c6280 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f9eb0009e50 tx=0x7f9eb002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9460613bb5a03316 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.820+0000 7f9eaffff640 1 -- 172.21.15.64:0/2102972969 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9eb002ca70 con 0x7f9eb80c5e30 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.820+0000 7f9eaffff640 1 -- 172.21.15.64:0/2102972969 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9eb002cc10 con 0x7f9eb80c5e30 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.820+0000 7f9eaffff640 1 -- 172.21.15.64:0/2102972969 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9eb00343f0 con 0x7f9eb80c5e30 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 -- 172.21.15.64:0/2102972969 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb80c5e30 msgr2=0x7f9eb80c6280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 --2- 172.21.15.64:0/2102972969 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb80c5e30 0x7f9eb80c6280 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f9eb0009e50 tx=0x7f9eb002bed0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 -- 172.21.15.64:0/2102972969 shutdown_connections 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 --2- 172.21.15.64:0/2102972969 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb80c5e30 0x7f9eb80c6280 unknown :-1 s=CLOSED pgs=744 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 --2- 172.21.15.64:0/2102972969 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb815f500 0x7f9eb80ce300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 -- 172.21.15.64:0/2102972969 >> 172.21.15.64:0/2102972969 conn(0x7f9eb80b3700 msgr2=0x7f9eb80b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 -- 172.21.15.64:0/2102972969 shutdown_connections 2024-03-04T07:29:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.821+0000 7f9ebf993640 1 -- 172.21.15.64:0/2102972969 wait complete. 2024-03-04T07:29:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.823+0000 7f9ebf993640 1 Processor -- start 2024-03-04T07:29:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.823+0000 7f9ebf993640 1 -- start start 2024-03-04T07:29:46.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebf993640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb80c5e30 0x7f9eb8151bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebf993640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9eb80cf740 0x7f9eb8152dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebf993640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb815f500 0x7f9eb815db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebf993640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9eb80b43d0 con 0x7f9eb815f500 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebf993640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9eb80c59a0 con 0x7f9eb80cf740 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebf993640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9eb80cef50 con 0x7f9eb80c5e30 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebdf30640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb815f500 0x7f9eb815db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebdf30640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb815f500 0x7f9eb815db60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40456/0 (socket says 172.21.15.64:40456) 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.824+0000 7f9ebdf30640 1 -- 172.21.15.64:0/1067368268 learned_addr learned my addr 172.21.15.64:0/1067368268 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebd72f640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb80c5e30 0x7f9eb8151bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebcf2e640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9eb80cf740 0x7f9eb8152dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebdf30640 1 -- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb80c5e30 msgr2=0x7f9eb8151bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebdf30640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb80c5e30 0x7f9eb8151bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebdf30640 1 -- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9eb80cf740 msgr2=0x7f9eb8152dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebdf30640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9eb80cf740 0x7f9eb8152dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:46.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebdf30640 1 -- 172.21.15.64:0/1067368268 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9eb80ca880 con 0x7f9eb815f500 2024-03-04T07:29:46.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebdf30640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb815f500 0x7f9eb815db60 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f9eb0009e50 tx=0x7f9eb0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9eadffb640 1 -- 172.21.15.64:0/1067368268 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9eb0043020 con 0x7f9eb815f500 2024-03-04T07:29:46.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9eadffb640 1 -- 172.21.15.64:0/1067368268 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9eb003e450 con 0x7f9eb815f500 2024-03-04T07:29:46.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9eb80cab60 con 0x7f9eb815f500 2024-03-04T07:29:46.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9eadffb640 1 -- 172.21.15.64:0/1067368268 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9eb0009510 con 0x7f9eb815f500 2024-03-04T07:29:46.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.825+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9eb80cb120 con 0x7f9eb815f500 2024-03-04T07:29:46.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.826+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9eb809fcf0 con 0x7f9eb815f500 2024-03-04T07:29:46.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.827+0000 7f9eadffb640 1 -- 172.21.15.64:0/1067368268 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9eb0046020 con 0x7f9eb815f500 2024-03-04T07:29:46.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.827+0000 7f9eadffb640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e9c06b4d0 0x7f9e9c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:46.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.828+0000 7f9ebd72f640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e9c06b4d0 0x7f9e9c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:46.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.828+0000 7f9eadffb640 1 -- 172.21.15.64:0/1067368268 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9eb00ac8d0 con 0x7f9eb815f500 2024-03-04T07:29:46.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.828+0000 7f9ebd72f640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e9c06b4d0 0x7f9e9c06d990 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f9eb4009aa0 tx=0x7f9eb4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:46.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:46.832+0000 7f9eadffb640 1 -- 172.21.15.64:0/1067368268 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9eb004d030 con 0x7f9eb815f500 2024-03-04T07:29:47.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.013+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f9eb804f410 con 0x7f9eb815f500 2024-03-04T07:29:47.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.014+0000 7f9eadffb640 1 -- 172.21.15.64:0/1067368268 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v55) v1 ==== 104+0+11 (secure 0 0 0) 0x7f9eb007b070 con 0x7f9eb815f500 2024-03-04T07:29:47.014 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:29:47.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.016+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e9c06b4d0 msgr2=0x7f9e9c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.016+0000 7f9ebf993640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e9c06b4d0 0x7f9e9c06d990 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f9eb4009aa0 tx=0x7f9eb4009190 comp rx=0 tx=0).stop 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.016+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb815f500 msgr2=0x7f9eb815db60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.016+0000 7f9ebf993640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb815f500 0x7f9eb815db60 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f9eb0009e50 tx=0x7f9eb0004a40 comp rx=0 tx=0).stop 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 shutdown_connections 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9e9c06b4d0 0x7f9e9c06d990 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9eb815f500 0x7f9eb815db60 unknown :-1 s=CLOSED pgs=745 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9eb80cf740 0x7f9eb8152dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 --2- 172.21.15.64:0/1067368268 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9eb80c5e30 0x7f9eb8151bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 >> 172.21.15.64:0/1067368268 conn(0x7f9eb80b3700 msgr2=0x7f9eb80c69f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 shutdown_connections 2024-03-04T07:29:47.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.017+0000 7f9ebf993640 1 -- 172.21.15.64:0/1067368268 wait complete. 2024-03-04T07:29:47.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:29:47.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.135+0000 7ff4e16eb640 1 Processor -- start 2024-03-04T07:29:47.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.135+0000 7ff4e16eb640 1 -- start start 2024-03-04T07:29:47.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4e16eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc15dd50 0x7ff4dc0ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4e16eb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff4dc0cef20 0x7ff4dc071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4e16eb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4dc072cf0 con 0x7ff4dc0724b0 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4e16eb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff4dc0cf380 con 0x7ff4dc0cef20 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4e16eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4dc072fd0 con 0x7ff4dc15dd50 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4db7fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7ff4dc0724b0 0x7ff4dc161a00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:60642/0 (socket says 172.21.15.64:60642) 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4db7fe640 1 -- 172.21.15.64:0/3731605212 learned_addr learned my addr 172.21.15.64:0/3731605212 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4daffd640 1 --2- 172.21.15.64:0/3731605212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc15dd50 0x7ff4dc0ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.136+0000 7ff4d3fff640 1 --2- 172.21.15.64:0/3731605212 >> v2:172.21.15.66:3300/0 conn(0x7ff4dc0cef20 0x7ff4dc071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.137+0000 7ff4da7fc640 1 -- 172.21.15.64:0/3731605212 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 163587413 0 0) 0x7ff4dc072cf0 con 0x7ff4dc0724b0 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.137+0000 7ff4da7fc640 1 -- 172.21.15.64:0/3731605212 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff4c8003520 con 0x7ff4dc0724b0 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.137+0000 7ff4daffd640 1 -- 172.21.15.64:0/3731605212 >> v2:172.21.15.66:3300/0 conn(0x7ff4dc0cef20 msgr2=0x7ff4dc071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.137+0000 7ff4daffd640 1 --2- 172.21.15.64:0/3731605212 >> v2:172.21.15.66:3300/0 conn(0x7ff4dc0cef20 0x7ff4dc071f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.137+0000 7ff4daffd640 1 -- 172.21.15.64:0/3731605212 >> v1:172.21.15.66:6789/0 conn(0x7ff4dc0724b0 legacy=0x7ff4dc161a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.137+0000 7ff4daffd640 1 -- 172.21.15.64:0/3731605212 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4dc0d1f60 con 0x7ff4dc15dd50 2024-03-04T07:29:47.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.137+0000 7ff4daffd640 1 --2- 172.21.15.64:0/3731605212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc15dd50 0x7ff4dc0ce9e0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7ff4cc0097a0 tx=0x7ff4cc02b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c79ade5df365a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:47.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4da7fc640 1 -- 172.21.15.64:0/3731605212 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4cc0315a0 con 0x7ff4dc15dd50 2024-03-04T07:29:47.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4da7fc640 1 -- 172.21.15.64:0/3731605212 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4cc037070 con 0x7ff4dc15dd50 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4da7fc640 1 -- 172.21.15.64:0/3731605212 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4cc031940 con 0x7ff4dc15dd50 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 -- 172.21.15.64:0/3731605212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc15dd50 msgr2=0x7ff4dc0ce9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/3731605212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc15dd50 0x7ff4dc0ce9e0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7ff4cc0097a0 tx=0x7ff4cc02b840 comp rx=0 tx=0).stop 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 -- 172.21.15.64:0/3731605212 shutdown_connections 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/3731605212 >> v2:172.21.15.66:3300/0 conn(0x7ff4dc0cef20 0x7ff4dc071f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/3731605212 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc15dd50 0x7ff4dc0ce9e0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 -- 172.21.15.64:0/3731605212 >> 172.21.15.64:0/3731605212 conn(0x7ff4dc0d3ef0 msgr2=0x7ff4dc0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 -- 172.21.15.64:0/3731605212 shutdown_connections 2024-03-04T07:29:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.138+0000 7ff4e16eb640 1 -- 172.21.15.64:0/3731605212 wait complete. 2024-03-04T07:29:47.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.139+0000 7ff4e16eb640 1 Processor -- start 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.139+0000 7ff4e16eb640 1 -- start start 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4e16eb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4dc0724b0 0x7ff4dc15b320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4e16eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc0cef20 0x7ff4dc0cd220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4e16eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4dc15dd50 0x7ff4dc14fd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4e16eb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4dc072a80 con 0x7ff4dc15dd50 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4e16eb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4dc072900 con 0x7ff4dc0724b0 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4e16eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4dc15e130 con 0x7ff4dc0cef20 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4db7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4dc15dd50 0x7ff4dc14fd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4db7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4dc15dd50 0x7ff4dc14fd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40464/0 (socket says 172.21.15.64:40464) 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4db7fe640 1 -- 172.21.15.64:0/394805083 learned_addr learned my addr 172.21.15.64:0/394805083 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4d3fff640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc0cef20 0x7ff4dc0cd220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.140+0000 7ff4daffd640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4dc0724b0 0x7ff4dc15b320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d3fff640 1 -- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4dc0724b0 msgr2=0x7ff4dc15b320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d3fff640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4dc0724b0 0x7ff4dc15b320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d3fff640 1 -- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4dc15dd50 msgr2=0x7ff4dc14fd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d3fff640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4dc15dd50 0x7ff4dc14fd60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d3fff640 1 -- 172.21.15.64:0/394805083 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4dc15bb50 con 0x7ff4dc0cef20 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4db7fe640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4dc15dd50 0x7ff4dc14fd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d3fff640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc0cef20 0x7ff4dc0cd220 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7ff4c400e850 tx=0x7ff4c400ed10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d37fe640 1 -- 172.21.15.64:0/394805083 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4c4017070 con 0x7ff4dc0cef20 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d37fe640 1 -- 172.21.15.64:0/394805083 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4c400c940 con 0x7ff4dc0cef20 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4dc15be30 con 0x7ff4dc0cef20 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4d37fe640 1 -- 172.21.15.64:0/394805083 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4c400cd10 con 0x7ff4dc0cef20 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.141+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4dc15c6d0 con 0x7ff4dc0cef20 2024-03-04T07:29:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.142+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4dc15c000 con 0x7ff4dc0cef20 2024-03-04T07:29:47.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.143+0000 7ff4d37fe640 1 -- 172.21.15.64:0/394805083 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff4c401a070 con 0x7ff4dc0cef20 2024-03-04T07:29:47.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.143+0000 7ff4d37fe640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4bc06b500 0x7ff4bc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.143+0000 7ff4d37fe640 1 -- 172.21.15.64:0/394805083 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff4c408a740 con 0x7ff4dc0cef20 2024-03-04T07:29:47.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.143+0000 7ff4daffd640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4bc06b500 0x7ff4bc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.144+0000 7ff4daffd640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4bc06b500 0x7ff4bc06d9c0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7ff4cc036000 tx=0x7ff4cc008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:47.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.147+0000 7ff4d37fe640 1 -- 172.21.15.64:0/394805083 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff4c4058ec0 con 0x7ff4dc0cef20 2024-03-04T07:29:47.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.499+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7ff4dc06cdd0 con 0x7ff4dc0cef20 2024-03-04T07:29:47.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.499+0000 7ff4d37fe640 1 -- 172.21.15.64:0/394805083 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v55) v1 ==== 100+0+11 (secure 0 0 0) 0x7ff4c405cf60 con 0x7ff4dc0cef20 2024-03-04T07:29:47.499 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:29:47.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4bc06b500 msgr2=0x7ff4bc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4bc06b500 0x7ff4bc06d9c0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7ff4cc036000 tx=0x7ff4cc008030 comp rx=0 tx=0).stop 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc0cef20 msgr2=0x7ff4dc0cd220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc0cef20 0x7ff4dc0cd220 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7ff4c400e850 tx=0x7ff4c400ed10 comp rx=0 tx=0).stop 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 shutdown_connections 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4bc06b500 0x7ff4bc06d9c0 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4dc15dd50 0x7ff4dc14fd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4dc0cef20 0x7ff4dc0cd220 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 --2- 172.21.15.64:0/394805083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4dc0724b0 0x7ff4dc15b320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 >> 172.21.15.64:0/394805083 conn(0x7ff4dc0d3ef0 msgr2=0x7ff4dc0c95f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 shutdown_connections 2024-03-04T07:29:47.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.501+0000 7ff4e16eb640 1 -- 172.21.15.64:0/394805083 wait complete. 2024-03-04T07:29:47.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:29:47.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:29:47.594 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:29:47.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.618+0000 7f21a9eaf640 1 Processor -- start 2024-03-04T07:29:47.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.618+0000 7f21a9eaf640 1 -- start start 2024-03-04T07:29:47.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.619+0000 7f21a9eaf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a415e2e0 0x7f21a40af110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.619+0000 7f21a9eaf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f21a40b40e0 0x7f21a40b0330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.619+0000 7f21a9eaf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21a4074240 con 0x7f21a4072c10 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.619+0000 7f21a9eaf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21a415e6c0 con 0x7f21a40b40e0 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.619+0000 7f21a9eaf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21a4073ff0 con 0x7f21a415e2e0 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a3fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f21a40b40e0 0x7f21a40b0330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a37fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a415e2e0 0x7f21a40af110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a37fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a415e2e0 0x7f21a40af110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:36950/0 (socket says 172.21.15.64:36950) 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a37fe640 1 -- 172.21.15.64:0/1328379871 learned_addr learned my addr 172.21.15.64:0/1328379871 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a37fe640 1 -- 172.21.15.64:0/1328379871 >> v2:172.21.15.66:3300/0 conn(0x7f21a40b40e0 msgr2=0x7f21a40b0330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a37fe640 1 --2- 172.21.15.64:0/1328379871 >> v2:172.21.15.66:3300/0 conn(0x7f21a40b40e0 0x7f21a40b0330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a37fe640 1 -- 172.21.15.64:0/1328379871 >> v1:172.21.15.66:6789/0 conn(0x7f21a4072c10 legacy=0x7f21a40b39d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a37fe640 1 -- 172.21.15.64:0/1328379871 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21a4074520 con 0x7f21a415e2e0 2024-03-04T07:29:47.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.620+0000 7f21a3fff640 1 --2- 172.21.15.64:0/1328379871 >> v2:172.21.15.66:3300/0 conn(0x7f21a40b40e0 0x7f21a40b0330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.621+0000 7f21a37fe640 1 --2- 172.21.15.64:0/1328379871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a415e2e0 0x7f21a40af110 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f2194009e50 tx=0x7f219402be40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c5349adfec78bf86 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.621+0000 7f21a27fc640 1 -- 172.21.15.64:0/1328379871 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f219402c9e0 con 0x7f21a415e2e0 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.621+0000 7f21a27fc640 1 -- 172.21.15.64:0/1328379871 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f219402cb80 con 0x7f21a415e2e0 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.621+0000 7f21a27fc640 1 -- 172.21.15.64:0/1328379871 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21940343f0 con 0x7f21a415e2e0 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 -- 172.21.15.64:0/1328379871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a415e2e0 msgr2=0x7f21a40af110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/1328379871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a415e2e0 0x7f21a40af110 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f2194009e50 tx=0x7f219402be40 comp rx=0 tx=0).stop 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 -- 172.21.15.64:0/1328379871 shutdown_connections 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/1328379871 >> v2:172.21.15.66:3300/0 conn(0x7f21a40b40e0 0x7f21a40b0330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/1328379871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a415e2e0 0x7f21a40af110 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 -- 172.21.15.64:0/1328379871 >> 172.21.15.64:0/1328379871 conn(0x7f21a4071650 msgr2=0x7f21a4072500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:47.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 -- 172.21.15.64:0/1328379871 shutdown_connections 2024-03-04T07:29:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.622+0000 7f21a9eaf640 1 -- 172.21.15.64:0/1328379871 wait complete. 2024-03-04T07:29:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.623+0000 7f21a9eaf640 1 Processor -- start 2024-03-04T07:29:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.624+0000 7f21a9eaf640 1 -- start start 2024-03-04T07:29:47.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.624+0000 7f21a9eaf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4072c10 0x7f21a414e2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.624+0000 7f21a9eaf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40b40e0 0x7f21a414c9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.624+0000 7f21a9eaf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a415e2e0 0x7f21a415a1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.624+0000 7f21a9eaf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21a40b0870 con 0x7f21a415e2e0 2024-03-04T07:29:47.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.624+0000 7f21a9eaf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21a40afbc0 con 0x7f21a40b40e0 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.624+0000 7f21a9eaf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21a40b09f0 con 0x7f21a4072c10 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a415e2e0 0x7f21a415a1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a415e2e0 0x7f21a415a1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40472/0 (socket says 172.21.15.64:40472) 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 -- 172.21.15.64:0/387899192 learned_addr learned my addr 172.21.15.64:0/387899192 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a2ffd640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40b40e0 0x7f21a414c9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a37fe640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4072c10 0x7f21a414e2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 -- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4072c10 msgr2=0x7f21a414e2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4072c10 0x7f21a414e2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 -- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40b40e0 msgr2=0x7f21a414c9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40b40e0 0x7f21a414c9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 -- 172.21.15.64:0/387899192 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21a414e980 con 0x7f21a415e2e0 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a3fff640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a415e2e0 0x7f21a415a1f0 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f219c00e930 tx=0x7f219c00edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f218bfff640 1 -- 172.21.15.64:0/387899192 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f219c017070 con 0x7f21a415e2e0 2024-03-04T07:29:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f218bfff640 1 -- 172.21.15.64:0/387899192 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f219c00ca00 con 0x7f21a415e2e0 2024-03-04T07:29:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21a414ec60 con 0x7f21a415e2e0 2024-03-04T07:29:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f218bfff640 1 -- 172.21.15.64:0/387899192 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f219c00cdd0 con 0x7f21a415e2e0 2024-03-04T07:29:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.625+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21a414f220 con 0x7f21a415e2e0 2024-03-04T07:29:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.626+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f21a4068700 con 0x7f21a415e2e0 2024-03-04T07:29:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.627+0000 7f218bfff640 1 -- 172.21.15.64:0/387899192 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f219c013470 con 0x7f21a415e2e0 2024-03-04T07:29:47.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.627+0000 7f218bfff640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:47.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.627+0000 7f218bfff640 1 -- 172.21.15.64:0/387899192 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f219c089ff0 con 0x7f21a415e2e0 2024-03-04T07:29:47.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.627+0000 7f21a37fe640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:47.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.628+0000 7f21a37fe640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f2194035000 tx=0x7f2194030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:47.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.636+0000 7f218bfff640 1 -- 172.21.15.64:0/387899192 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f219c058970 con 0x7f21a415e2e0 2024-03-04T07:29:47.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.867+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f21a40af310 con 0x7f21a415e2e0 2024-03-04T07:29:47.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.867+0000 7f218bfff640 1 -- 172.21.15.64:0/387899192 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v55) v1 ==== 80+0+0 (secure 0 0 0) 0x7f219c05ca10 con 0x7f21a415e2e0 2024-03-04T07:29:47.867 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:29:47.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 msgr2=0x7f219006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f2194035000 tx=0x7f2194030000 comp rx=0 tx=0).stop 2024-03-04T07:29:47.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a415e2e0 msgr2=0x7f21a415a1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:47.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a415e2e0 0x7f21a415a1f0 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f219c00e930 tx=0x7f219c00edf0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 shutdown_connections 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f219006b4d0 0x7f219006d990 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21a415e2e0 0x7f21a415a1f0 unknown :-1 s=CLOSED pgs=746 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21a40b40e0 0x7f21a414c9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 --2- 172.21.15.64:0/387899192 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21a4072c10 0x7f21a414e2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.869+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 >> 172.21.15.64:0/387899192 conn(0x7f21a4071650 msgr2=0x7f21a4155670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.870+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 shutdown_connections 2024-03-04T07:29:47.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:47.870+0000 7f21a9eaf640 1 -- 172.21.15.64:0/387899192 wait complete. 2024-03-04T07:29:47.880 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:29:47.880 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:47.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:29:47.905 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[60105]: fuse finished with error 0 and tester_r 0 2024-03-04T07:29:47.907 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:29:47.907 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:47.907 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:29:48.089 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:29:48.089 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:48.108 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:29:48.109 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:29:48.109 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:48.109 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:29:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:29:48.167 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:29:48.167 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:48.167 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:48.182 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:29:48.182 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:29:48.182 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:48.182 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:29:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:29:48.237 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:29:48.237 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:29:48.238 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:48.255 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:29:48.255 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:29:48.256 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:29:48.256 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:29:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:29:48.315 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:29:48.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:48.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.417+0000 7fe64a5e9640 1 Processor -- start 2024-03-04T07:29:48.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.417+0000 7fe64a5e9640 1 -- start start 2024-03-04T07:29:48.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.417+0000 7fe64a5e9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe6440b95a0 0x7fe6440b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.417+0000 7fe64a5e9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe6440cea30 con 0x7fe6440b9a10 2024-03-04T07:29:48.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.417+0000 7fe64a5e9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe6440cf3c0 con 0x7fe64415df00 2024-03-04T07:29:48.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.417+0000 7fe64a5e9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe6440cf0e0 con 0x7fe6440b95a0 2024-03-04T07:29:48.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe648b86640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe6440b9a10 0x7fe6440d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41300/0 (socket says 172.21.15.64:41300) 2024-03-04T07:29:48.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe648b86640 1 -- 172.21.15.64:0/2752751810 learned_addr learned my addr 172.21.15.64:0/2752751810 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:48.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe6437fe640 1 --2- 172.21.15.64:0/2752751810 >> v2:172.21.15.66:3300/0 conn(0x7fe6440b95a0 0x7fe6440b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 209619903 0 0) 0x7fe6440cea30 con 0x7fe6440b9a10 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe62c003520 con 0x7fe6440b9a10 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3766727696 0 0) 0x7fe6440cf3c0 con 0x7fe64415df00 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe6440cea30 con 0x7fe64415df00 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe6437fe640 1 -- 172.21.15.64:0/2752751810 >> v1:172.21.15.66:6789/0 conn(0x7fe64415df00 legacy=0x7fe6440d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 608017203 0 0) 0x7fe62c003520 con 0x7fe6440b9a10 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe6437fe640 1 -- 172.21.15.64:0/2752751810 >> v1:172.21.15.64:6789/0 conn(0x7fe6440b9a10 legacy=0x7fe6440d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.418+0000 7fe6437fe640 1 -- 172.21.15.64:0/2752751810 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6440ce470 con 0x7fe6440b95a0 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.419+0000 7fe6437fe640 1 --2- 172.21.15.64:0/2752751810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 0x7fe6440b5700 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fe6440cf590 tx=0x7fe63802b810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=23de679e2d5075c3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.419+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe638039070 con 0x7fe6440b95a0 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.419+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6380314f0 con 0x7fe6440b95a0 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.419+0000 7fe642ffd640 1 -- 172.21.15.64:0/2752751810 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe63c0019f0 con 0x7fe6440b95a0 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.420+0000 7fe64a5e9640 1 -- 172.21.15.64:0/2752751810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 msgr2=0x7fe6440b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.420+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/2752751810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 0x7fe6440b5700 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fe6440cf590 tx=0x7fe63802b810 comp rx=0 tx=0).stop 2024-03-04T07:29:48.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.420+0000 7fe64a5e9640 1 -- 172.21.15.64:0/2752751810 shutdown_connections 2024-03-04T07:29:48.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.420+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/2752751810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 0x7fe6440b5700 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.420+0000 7fe64a5e9640 1 -- 172.21.15.64:0/2752751810 >> 172.21.15.64:0/2752751810 conn(0x7fe644071960 msgr2=0x7fe644071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:48.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.420+0000 7fe64a5e9640 1 -- 172.21.15.64:0/2752751810 shutdown_connections 2024-03-04T07:29:48.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.420+0000 7fe64a5e9640 1 -- 172.21.15.64:0/2752751810 wait complete. 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.421+0000 7fe64a5e9640 1 Processor -- start 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.421+0000 7fe64a5e9640 1 -- start start 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe64a5e9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 0x7fe64415e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe64a5e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6440b9a10 0x7fe64414c5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe64a5e9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe64415df00 0x7fe64414cb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe64a5e9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6440cedf0 con 0x7fe64415df00 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe64a5e9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6440cec70 con 0x7fe6440b95a0 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe64a5e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe6440b5c40 con 0x7fe6440b9a10 2024-03-04T07:29:48.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe64415df00 0x7fe64414cb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe64415df00 0x7fe64414cb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40484/0 (socket says 172.21.15.64:40484) 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 -- 172.21.15.64:0/109940260 learned_addr learned my addr 172.21.15.64:0/109940260 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe643fff640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 0x7fe64415e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe6437fe640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6440b9a10 0x7fe64414c5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 -- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6440b9a10 msgr2=0x7fe64414c5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6440b9a10 0x7fe64414c5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 -- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 msgr2=0x7fe64415e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 0x7fe64415e2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 -- 172.21.15.64:0/109940260 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe64414a210 con 0x7fe64415df00 2024-03-04T07:29:48.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.422+0000 7fe648b86640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe64415df00 0x7fe64414cb00 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7fe63c001fc0 tx=0x7fe63c00cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.423+0000 7fe640ff9640 1 -- 172.21.15.64:0/109940260 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe63c012400 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.423+0000 7fe640ff9640 1 -- 172.21.15.64:0/109940260 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe63c0125a0 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.423+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe6440cbda0 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.423+0000 7fe640ff9640 1 -- 172.21.15.64:0/109940260 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe63c012990 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.423+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe644160de0 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.424+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe6440cbf70 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.424+0000 7fe640ff9640 1 -- 172.21.15.64:0/109940260 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe63c012b30 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.424+0000 7fe640ff9640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe63006b4d0 0x7fe63006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.425+0000 7fe643fff640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe63006b4d0 0x7fe63006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.425+0000 7fe640ff9640 1 -- 172.21.15.64:0/109940260 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe63c08aa10 con 0x7fe64415df00 2024-03-04T07:29:48.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.425+0000 7fe643fff640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe63006b4d0 0x7fe63006d990 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7fe6340011e0 tx=0x7fe634009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:48.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.429+0000 7fe640ff9640 1 -- 172.21.15.64:0/109940260 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe63c012e20 con 0x7fe64415df00 2024-03-04T07:29:48.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.654+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe6440d27c0 con 0x7fe64415df00 2024-03-04T07:29:48.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.654+0000 7fe640ff9640 1 -- 172.21.15.64:0/109940260 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7fe63c0591c0 con 0x7fe64415df00 2024-03-04T07:29:48.654 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:48.655 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:48.655 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:48.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe63006b4d0 msgr2=0x7fe63006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe63006b4d0 0x7fe63006d990 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7fe6340011e0 tx=0x7fe634009040 comp rx=0 tx=0).stop 2024-03-04T07:29:48.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe64415df00 msgr2=0x7fe64414cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe64415df00 0x7fe64414cb00 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7fe63c001fc0 tx=0x7fe63c00cf90 comp rx=0 tx=0).stop 2024-03-04T07:29:48.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 shutdown_connections 2024-03-04T07:29:48.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe63006b4d0 0x7fe63006d990 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe64415df00 0x7fe64414cb00 unknown :-1 s=CLOSED pgs=748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6440b9a10 0x7fe64414c5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 --2- 172.21.15.64:0/109940260 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6440b95a0 0x7fe64415e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 >> 172.21.15.64:0/109940260 conn(0x7fe644071960 msgr2=0x7fe644071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 shutdown_connections 2024-03-04T07:29:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.656+0000 7fe64a5e9640 1 -- 172.21.15.64:0/109940260 wait complete. 2024-03-04T07:29:48.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:48.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.768+0000 7f29284b5640 1 Processor -- start 2024-03-04T07:29:48.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.768+0000 7f29284b5640 1 -- start start 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.769+0000 7f29284b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200b4260 0x7f29200cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.769+0000 7f29284b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29200b2320 con 0x7f29200d06c0 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.769+0000 7f29284b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29200b0dd0 con 0x7f29200d02e0 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.769+0000 7f29284b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29200b0b80 con 0x7f29200b4260 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.769+0000 7f2926a52640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f29200d06c0 0x7f29200b1c10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41302/0 (socket says 172.21.15.64:41302) 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.769+0000 7f2926a52640 1 -- 172.21.15.64:0/285244758 learned_addr learned my addr 172.21.15.64:0/285244758 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f2925a50640 1 --2- 172.21.15.64:0/285244758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200b4260 0x7f29200cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f292524f640 1 -- 172.21.15.64:0/285244758 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4277618153 0 0) 0x7f29200b2320 con 0x7f29200d06c0 2024-03-04T07:29:48.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f292524f640 1 -- 172.21.15.64:0/285244758 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2908003520 con 0x7f29200d06c0 2024-03-04T07:29:48.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f292524f640 1 -- 172.21.15.64:0/285244758 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 126410881 0 0) 0x7f29200b0dd0 con 0x7f29200d02e0 2024-03-04T07:29:48.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f292524f640 1 -- 172.21.15.64:0/285244758 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f29200b2320 con 0x7f29200d02e0 2024-03-04T07:29:48.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f2925a50640 1 -- 172.21.15.64:0/285244758 >> v1:172.21.15.66:6789/0 conn(0x7f29200d02e0 legacy=0x7f29200b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f2925a50640 1 -- 172.21.15.64:0/285244758 >> v1:172.21.15.64:6789/0 conn(0x7f29200d06c0 legacy=0x7f29200b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.770+0000 7f2925a50640 1 -- 172.21.15.64:0/285244758 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29200b10b0 con 0x7f29200b4260 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.771+0000 7f2925a50640 1 --2- 172.21.15.64:0/285244758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200b4260 0x7f29200cfa00 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f29140097d0 tx=0x7f291402b9d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=beed24208a0c2593 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.771+0000 7f292524f640 1 -- 172.21.15.64:0/285244758 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2914034070 con 0x7f29200b4260 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.771+0000 7f292524f640 1 -- 172.21.15.64:0/285244758 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2914030890 con 0x7f29200b4260 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.771+0000 7f292524f640 1 -- 172.21.15.64:0/285244758 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2914030c60 con 0x7f29200b4260 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.771+0000 7f29284b5640 1 -- 172.21.15.64:0/285244758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200b4260 msgr2=0x7f29200cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.771+0000 7f29284b5640 1 --2- 172.21.15.64:0/285244758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200b4260 0x7f29200cfa00 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f29140097d0 tx=0x7f291402b9d0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.772+0000 7f29284b5640 1 -- 172.21.15.64:0/285244758 shutdown_connections 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.772+0000 7f29284b5640 1 --2- 172.21.15.64:0/285244758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200b4260 0x7f29200cfa00 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.772+0000 7f29284b5640 1 -- 172.21.15.64:0/285244758 >> 172.21.15.64:0/285244758 conn(0x7f29200b36d0 msgr2=0x7f29200b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.772+0000 7f29284b5640 1 -- 172.21.15.64:0/285244758 shutdown_connections 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.772+0000 7f29284b5640 1 -- 172.21.15.64:0/285244758 wait complete. 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.773+0000 7f29284b5640 1 Processor -- start 2024-03-04T07:29:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.773+0000 7f29284b5640 1 -- start start 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.773+0000 7f29284b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29200b4260 0x7f292015f580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f29284b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200d02e0 0x7f292014e680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f29284b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29200d06c0 0x7f292014f9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f29284b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29200b0360 con 0x7f29200b4260 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f29284b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29200aeec0 con 0x7f29200d06c0 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f29284b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29200b04e0 con 0x7f29200d02e0 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29200b4260 0x7f292015f580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29200b4260 0x7f292015f580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40492/0 (socket says 172.21.15.64:40492) 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 -- 172.21.15.64:0/1539705115 learned_addr learned my addr 172.21.15.64:0/1539705115 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2925a50640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200d02e0 0x7f292014e680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926a52640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29200d06c0 0x7f292014f9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 -- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200d02e0 msgr2=0x7f292014e680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200d02e0 0x7f292014e680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 -- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29200d06c0 msgr2=0x7f292014f9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29200d06c0 0x7f292014f9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 -- 172.21.15.64:0/1539705115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f292015fac0 con 0x7f29200b4260 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f2926251640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29200b4260 0x7f292015f580 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7f291800c430 tx=0x7f291800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f291effd640 1 -- 172.21.15.64:0/1539705115 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2918017070 con 0x7f29200b4260 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f291effd640 1 -- 172.21.15.64:0/1539705115 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29180052f0 con 0x7f29200b4260 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f291effd640 1 -- 172.21.15.64:0/1539705115 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2918003830 con 0x7f29200b4260 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29201f19b0 con 0x7f29200b4260 2024-03-04T07:29:48.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.774+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f292015e7c0 con 0x7f29200b4260 2024-03-04T07:29:48.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.776+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2920068700 con 0x7f29200b4260 2024-03-04T07:29:48.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.776+0000 7f291effd640 1 -- 172.21.15.64:0/1539705115 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f291801a070 con 0x7f29200b4260 2024-03-04T07:29:48.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.776+0000 7f291effd640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f290c06b4d0 0x7f290c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:48.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.776+0000 7f291effd640 1 -- 172.21.15.64:0/1539705115 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2918089500 con 0x7f29200b4260 2024-03-04T07:29:48.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.776+0000 7f2925a50640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f290c06b4d0 0x7f290c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:48.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.777+0000 7f2925a50640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f290c06b4d0 0x7f290c06d990 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f2914036000 tx=0x7f291403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:48.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.781+0000 7f291effd640 1 -- 172.21.15.64:0/1539705115 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2918057d30 con 0x7f29200b4260 2024-03-04T07:29:48.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.961+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f29200cf0b0 con 0x7f29200b4260 2024-03-04T07:29:48.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.963+0000 7f291effd640 1 -- 172.21.15.64:0/1539705115 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v55) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f291805bdd0 con 0x7f29200b4260 2024-03-04T07:29:48.963 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:48.963 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:12.504021+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f290c06b4d0 msgr2=0x7f290c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f290c06b4d0 0x7f290c06d990 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f2914036000 tx=0x7f291403b040 comp rx=0 tx=0).stop 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29200b4260 msgr2=0x7f292015f580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29200b4260 0x7f292015f580 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7f291800c430 tx=0x7f291800c8f0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 shutdown_connections 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f290c06b4d0 0x7f290c06d990 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29200d06c0 0x7f292014f9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29200d02e0 0x7f292014e680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 --2- 172.21.15.64:0/1539705115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29200b4260 0x7f292015f580 unknown :-1 s=CLOSED pgs=750 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 >> 172.21.15.64:0/1539705115 conn(0x7f29200b36d0 msgr2=0x7f29200cd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 shutdown_connections 2024-03-04T07:29:48.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:48.965+0000 7f29284b5640 1 -- 172.21.15.64:0/1539705115 wait complete. 2024-03-04T07:29:48.975 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 8, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:29:09.132151+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '54', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 9, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:29:09.825115+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '55', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.6299999952316284, 'score_stable': 1.6299999952316284, 'optimal_score': 1, 'raw_score_acting': 1.6299999952316284, 'raw_score_stable': 1.6299999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:29:48.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:29:49.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.080+0000 7f6a1a281640 1 Processor -- start 2024-03-04T07:29:49.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.080+0000 7f6a1a281640 1 -- start start 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1a281640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6a140ceea0 0x7f6a140d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1a281640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6a140b6d10 0x7f6a140b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1a281640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a140d3740 con 0x7f6a1415dd90 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1a281640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6a140d3400 con 0x7f6a140b6d10 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1a281640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6a140d3120 con 0x7f6a140ceea0 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1881e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6a140b6d10 0x7f6a140b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1881e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6a140b6d10 0x7f6a140b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40506/0 (socket says 172.21.15.64:40506) 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1881e640 1 -- 172.21.15.64:0/2508204481 learned_addr learned my addr 172.21.15.64:0/2508204481 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a137fe640 1 --2- 172.21.15.64:0/2508204481 >> v2:172.21.15.66:3300/0 conn(0x7f6a140ceea0 0x7f6a140d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1881e640 1 -- 172.21.15.64:0/2508204481 >> v2:172.21.15.66:3300/0 conn(0x7f6a140ceea0 msgr2=0x7f6a140d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1881e640 1 --2- 172.21.15.64:0/2508204481 >> v2:172.21.15.66:3300/0 conn(0x7f6a140ceea0 0x7f6a140d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1881e640 1 -- 172.21.15.64:0/2508204481 >> v1:172.21.15.66:6789/0 conn(0x7f6a1415dd90 legacy=0x7f6a140d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a1881e640 1 -- 172.21.15.64:0/2508204481 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a140d39e0 con 0x7f6a140b6d10 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.081+0000 7f6a137fe640 1 --2- 172.21.15.64:0/2508204481 >> v2:172.21.15.66:3300/0 conn(0x7f6a140ceea0 0x7f6a140d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.082+0000 7f6a1881e640 1 --2- 172.21.15.64:0/2508204481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140b6d10 0x7f6a140b71a0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7f6a140724d0 tx=0x7f6a0c02c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8254f9e8a87487f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.082+0000 7f6a12ffd640 1 -- 172.21.15.64:0/2508204481 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a0c02cbb0 con 0x7f6a140b6d10 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.082+0000 7f6a12ffd640 1 -- 172.21.15.64:0/2508204481 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a0c02cd50 con 0x7f6a140b6d10 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.082+0000 7f6a12ffd640 1 -- 172.21.15.64:0/2508204481 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a0c034610 con 0x7f6a140b6d10 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 -- 172.21.15.64:0/2508204481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140b6d10 msgr2=0x7f6a140b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 --2- 172.21.15.64:0/2508204481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140b6d10 0x7f6a140b71a0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7f6a140724d0 tx=0x7f6a0c02c010 comp rx=0 tx=0).stop 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 -- 172.21.15.64:0/2508204481 shutdown_connections 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 --2- 172.21.15.64:0/2508204481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140b6d10 0x7f6a140b71a0 unknown :-1 s=CLOSED pgs=751 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 --2- 172.21.15.64:0/2508204481 >> v2:172.21.15.66:3300/0 conn(0x7f6a140ceea0 0x7f6a140d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 -- 172.21.15.64:0/2508204481 >> 172.21.15.64:0/2508204481 conn(0x7f6a14071960 msgr2=0x7f6a14071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:49.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 -- 172.21.15.64:0/2508204481 shutdown_connections 2024-03-04T07:29:49.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.083+0000 7f6a1a281640 1 -- 172.21.15.64:0/2508204481 wait complete. 2024-03-04T07:29:49.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.085+0000 7f6a1a281640 1 Processor -- start 2024-03-04T07:29:49.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.085+0000 7f6a1a281640 1 -- start start 2024-03-04T07:29:49.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1a281640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a140b6d10 0x7f6a141530c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1a281640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140ceea0 0x7f6a141601d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1a281640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a1415dd90 0x7f6a14160710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1a281640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a140d48f0 con 0x7f6a140ceea0 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1a281640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a1415e170 con 0x7f6a1415dd90 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1a281640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a140d4a70 con 0x7f6a140b6d10 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140ceea0 0x7f6a141601d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140ceea0 0x7f6a141601d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40510/0 (socket says 172.21.15.64:40510) 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 -- 172.21.15.64:0/304210834 learned_addr learned my addr 172.21.15.64:0/304210834 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1881e640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a1415dd90 0x7f6a14160710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a13fff640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a140b6d10 0x7f6a141530c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 -- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a140b6d10 msgr2=0x7f6a141530c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a140b6d10 0x7f6a141530c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 -- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a1415dd90 msgr2=0x7f6a14160710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a1415dd90 0x7f6a14160710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 -- 172.21.15.64:0/304210834 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a140c8260 con 0x7f6a140ceea0 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a1881e640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a1415dd90 0x7f6a14160710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a13fff640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a140b6d10 0x7f6a141530c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a137fe640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140ceea0 0x7f6a141601d0 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7f6a0800e7e0 tx=0x7f6a0800eca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.086+0000 7f6a10ff9640 1 -- 172.21.15.64:0/304210834 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a0800f040 con 0x7f6a140ceea0 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.087+0000 7f6a10ff9640 1 -- 172.21.15.64:0/304210834 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a08004030 con 0x7f6a140ceea0 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.087+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a140c8540 con 0x7f6a140ceea0 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.087+0000 7f6a10ff9640 1 -- 172.21.15.64:0/304210834 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a08004400 con 0x7f6a140ceea0 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.087+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a140c8b00 con 0x7f6a140ceea0 2024-03-04T07:29:49.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.088+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6a14072a30 con 0x7f6a140ceea0 2024-03-04T07:29:49.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.088+0000 7f6a10ff9640 1 -- 172.21.15.64:0/304210834 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6a0800c6d0 con 0x7f6a140ceea0 2024-03-04T07:29:49.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.089+0000 7f6a10ff9640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a0006b4d0 0x7f6a0006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.089+0000 7f6a13fff640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a0006b4d0 0x7f6a0006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.089+0000 7f6a10ff9640 1 -- 172.21.15.64:0/304210834 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6a0808a060 con 0x7f6a140ceea0 2024-03-04T07:29:49.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.089+0000 7f6a13fff640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a0006b4d0 0x7f6a0006d990 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f6a140b56d0 tx=0x7f6a04009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.093+0000 7f6a10ff9640 1 -- 172.21.15.64:0/304210834 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6a08017020 con 0x7f6a140ceea0 2024-03-04T07:29:49.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.273+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f6a1404f410 con 0x7f6a140ceea0 2024-03-04T07:29:49.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.273+0000 7f6a10ff9640 1 -- 172.21.15.64:0/304210834 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v55) v1 ==== 93+0+10 (secure 0 0 0) 0x7f6a08058ee0 con 0x7f6a140ceea0 2024-03-04T07:29:49.273 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:29:49.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a0006b4d0 msgr2=0x7f6a0006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a0006b4d0 0x7f6a0006d990 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f6a140b56d0 tx=0x7f6a04009040 comp rx=0 tx=0).stop 2024-03-04T07:29:49.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140ceea0 msgr2=0x7f6a141601d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140ceea0 0x7f6a141601d0 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7f6a0800e7e0 tx=0x7f6a0800eca0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 shutdown_connections 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a0006b4d0 0x7f6a0006d990 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a1415dd90 0x7f6a14160710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a140ceea0 0x7f6a141601d0 unknown :-1 s=CLOSED pgs=752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 --2- 172.21.15.64:0/304210834 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a140b6d10 0x7f6a141530c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 >> 172.21.15.64:0/304210834 conn(0x7f6a14071960 msgr2=0x7f6a140b36c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.275+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 shutdown_connections 2024-03-04T07:29:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.276+0000 7f6a1a281640 1 -- 172.21.15.64:0/304210834 wait complete. 2024-03-04T07:29:49.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:29:49.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.391+0000 7f998fd93640 1 Processor -- start 2024-03-04T07:29:49.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.391+0000 7f998fd93640 1 -- start start 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998fd93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99880cf740 0x7f99880cea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998fd93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99880c5e30 0x7f99880c6280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998fd93640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f99880cd310 con 0x7f998815f500 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998fd93640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f998815f8e0 con 0x7f99880cf740 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998fd93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99880c56c0 con 0x7f99880c5e30 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998d32e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99880cf740 0x7f99880cea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998d32e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99880cf740 0x7f99880cea10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40522/0 (socket says 172.21.15.64:40522) 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998d32e640 1 -- 172.21.15.64:0/601020751 learned_addr learned my addr 172.21.15.64:0/601020751 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998e330640 1 --2- 172.21.15.64:0/601020751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99880c5e30 0x7f99880c6280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998d32e640 1 -- 172.21.15.64:0/601020751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99880c5e30 msgr2=0x7f99880c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998d32e640 1 --2- 172.21.15.64:0/601020751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99880c5e30 0x7f99880c6280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998d32e640 1 -- 172.21.15.64:0/601020751 >> v1:172.21.15.66:6789/0 conn(0x7f998815f500 legacy=0x7f99880ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:49.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998d32e640 1 -- 172.21.15.64:0/601020751 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99880cd5b0 con 0x7f99880cf740 2024-03-04T07:29:49.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.392+0000 7f998e330640 1 --2- 172.21.15.64:0/601020751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99880c5e30 0x7f99880c6280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:49.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.393+0000 7f998d32e640 1 --2- 172.21.15.64:0/601020751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880cf740 0x7f99880cea10 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7f99880d1ab0 tx=0x7f997802b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b7ec6b33a53562e2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.393+0000 7f998cb2d640 1 -- 172.21.15.64:0/601020751 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99780315a0 con 0x7f99880cf740 2024-03-04T07:29:49.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.393+0000 7f998cb2d640 1 -- 172.21.15.64:0/601020751 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9978037070 con 0x7f99880cf740 2024-03-04T07:29:49.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.393+0000 7f998cb2d640 1 -- 172.21.15.64:0/601020751 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9978031960 con 0x7f99880cf740 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 -- 172.21.15.64:0/601020751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880cf740 msgr2=0x7f99880cea10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 --2- 172.21.15.64:0/601020751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880cf740 0x7f99880cea10 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7f99880d1ab0 tx=0x7f997802b840 comp rx=0 tx=0).stop 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 -- 172.21.15.64:0/601020751 shutdown_connections 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 --2- 172.21.15.64:0/601020751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99880c5e30 0x7f99880c6280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 --2- 172.21.15.64:0/601020751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880cf740 0x7f99880cea10 unknown :-1 s=CLOSED pgs=753 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 -- 172.21.15.64:0/601020751 >> 172.21.15.64:0/601020751 conn(0x7f99880b3700 msgr2=0x7f99880b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 -- 172.21.15.64:0/601020751 shutdown_connections 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.394+0000 7f998fd93640 1 -- 172.21.15.64:0/601020751 wait complete. 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.395+0000 7f998fd93640 1 Processor -- start 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.395+0000 7f998fd93640 1 -- start start 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998fd93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880c5e30 0x7f99880cb7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998fd93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99880cf740 0x7f998814b010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998fd93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998815f500 0x7f99880c8c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998fd93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99880b43d0 con 0x7f99880c5e30 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998fd93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99880c59a0 con 0x7f99880cf740 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998fd93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99880cef50 con 0x7f998815f500 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880c5e30 0x7f99880cb7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880c5e30 0x7f99880cb7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40528/0 (socket says 172.21.15.64:40528) 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 -- 172.21.15.64:0/917467790 learned_addr learned my addr 172.21.15.64:0/917467790 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 -- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998815f500 msgr2=0x7f99880c8c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998e330640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998815f500 0x7f99880c8c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998d32e640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99880cf740 0x7f998814b010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998815f500 0x7f99880c8c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 -- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99880cf740 msgr2=0x7f998814b010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99880cf740 0x7f998814b010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 -- 172.21.15.64:0/917467790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99880c9490 con 0x7f99880c5e30 2024-03-04T07:29:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998d32e640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99880cf740 0x7f998814b010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f998db2f640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880c5e30 0x7f99880cb7a0 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f998400b3f0 tx=0x7f998400b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.396+0000 7f997e7fc640 1 -- 172.21.15.64:0/917467790 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9984015020 con 0x7f99880c5e30 2024-03-04T07:29:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.397+0000 7f997e7fc640 1 -- 172.21.15.64:0/917467790 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9984011830 con 0x7f99880c5e30 2024-03-04T07:29:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.397+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99880c9770 con 0x7f99880c5e30 2024-03-04T07:29:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.397+0000 7f997e7fc640 1 -- 172.21.15.64:0/917467790 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9984011c20 con 0x7f99880c5e30 2024-03-04T07:29:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.397+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99880c9d30 con 0x7f99880c5e30 2024-03-04T07:29:49.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.398+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f998809fcf0 con 0x7f99880c5e30 2024-03-04T07:29:49.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.398+0000 7f997e7fc640 1 -- 172.21.15.64:0/917467790 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9984018020 con 0x7f99880c5e30 2024-03-04T07:29:49.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.398+0000 7f997e7fc640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f996c06b4d0 0x7f996c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.398+0000 7f998d32e640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f996c06b4d0 0x7f996c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.398+0000 7f997e7fc640 1 -- 172.21.15.64:0/917467790 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9984089e20 con 0x7f99880c5e30 2024-03-04T07:29:49.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.399+0000 7f998d32e640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f996c06b4d0 0x7f996c06d990 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f9978036000 tx=0x7f9978008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.403+0000 7f997e7fc640 1 -- 172.21.15.64:0/917467790 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f998401f030 con 0x7f99880c5e30 2024-03-04T07:29:49.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.639+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f99880cbde0 con 0x7f99880c5e30 2024-03-04T07:29:49.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.640+0000 7f997e7fc640 1 -- 172.21.15.64:0/917467790 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v55) v1 ==== 104+0+11 (secure 0 0 0) 0x7f99840172c0 con 0x7f99880c5e30 2024-03-04T07:29:49.639 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:29:49.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f996c06b4d0 msgr2=0x7f996c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f996c06b4d0 0x7f996c06d990 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f9978036000 tx=0x7f9978008030 comp rx=0 tx=0).stop 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880c5e30 msgr2=0x7f99880cb7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880c5e30 0x7f99880cb7a0 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f998400b3f0 tx=0x7f998400b8b0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 shutdown_connections 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f996c06b4d0 0x7f996c06d990 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f998815f500 0x7f99880c8c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99880cf740 0x7f998814b010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 --2- 172.21.15.64:0/917467790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99880c5e30 0x7f99880cb7a0 unknown :-1 s=CLOSED pgs=754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 >> 172.21.15.64:0/917467790 conn(0x7f99880b3700 msgr2=0x7f99880c6b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 shutdown_connections 2024-03-04T07:29:49.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.642+0000 7f998fd93640 1 -- 172.21.15.64:0/917467790 wait complete. 2024-03-04T07:29:49.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:29:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.758+0000 7f5e2f842640 1 Processor -- start 2024-03-04T07:29:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.758+0000 7f5e2f842640 1 -- start start 2024-03-04T07:29:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e2f842640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 0x7f5e28071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e2f842640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5e2815e4b0 con 0x7f5e280b3060 2024-03-04T07:29:49.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e2f842640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5e280b97e0 con 0x7f5e280b2b10 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e2f842640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5e280b7c70 con 0x7f5e280717b0 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e2cddd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5e280b3060 0x7f5e280b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41310/0 (socket says 172.21.15.64:41310) 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e2cddd640 1 -- 172.21.15.64:0/554546710 learned_addr learned my addr 172.21.15.64:0/554546710 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3615284811 0 0) 0x7f5e1c001110 con 0x7f5e280b3060 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5e10003520 con 0x7f5e280b3060 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e2d5de640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 0x7f5e28071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.759+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3277350102 0 0) 0x7f5e10003520 con 0x7f5e280b3060 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 msgr2=0x7f5e28071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 0x7f5e28071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 >> v1:172.21.15.66:6789/0 conn(0x7f5e280b2b10 legacy=0x7f5e280b7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5e280b9a80 con 0x7f5e280b3060 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f5e1c001840 con 0x7f5e280b3060 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 >> v1:172.21.15.64:6789/0 conn(0x7f5e280b3060 legacy=0x7f5e280b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e100039e0 0x7f5e10003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e10004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 0x7f5e10005200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e1000bac0 con 0x7f5e10004800 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e10014480 con 0x7f5e100039e0 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5e10014680 con 0x7f5e280717b0 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2cddd640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e10004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2d5de640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e100039e0 0x7f5e10003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2dddf640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 0x7f5e10005200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2cddd640 1 -- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 msgr2=0x7f5e10005200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2cddd640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e280717b0 0x7f5e10005200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2cddd640 1 -- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e100039e0 msgr2=0x7f5e10003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2cddd640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e100039e0 0x7f5e10003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.760+0000 7f5e2cddd640 1 -- 172.21.15.64:0/554546710 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5e10003520 con 0x7f5e10004800 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.761+0000 7f5e2cddd640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e10004c80 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7f5e1c00c4c0 tx=0x7f5e10005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a11dc1c04cc1f3f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.761+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e1c00c0e0 con 0x7f5e10004800 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.761+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5e1c030070 con 0x7f5e10004800 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.761+0000 7f5e27fff640 1 -- 172.21.15.64:0/554546710 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e1c0058d0 con 0x7f5e10004800 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.761+0000 7f5e2f842640 1 -- 172.21.15.64:0/554546710 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 msgr2=0x7f5e10004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.761+0000 7f5e2f842640 1 --2- 172.21.15.64:0/554546710 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e10004c80 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7f5e1c00c4c0 tx=0x7f5e10005830 comp rx=0 tx=0).stop 2024-03-04T07:29:49.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.761+0000 7f5e2dddf640 1 -- 172.21.15.64:0/554546710 reap_dead start 2024-03-04T07:29:49.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.762+0000 7f5e2f842640 1 -- 172.21.15.64:0/554546710 shutdown_connections 2024-03-04T07:29:49.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.762+0000 7f5e2f842640 1 -- 172.21.15.64:0/554546710 >> 172.21.15.64:0/554546710 conn(0x7f5e280d3d20 msgr2=0x7f5e280d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:49.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.762+0000 7f5e2f842640 1 -- 172.21.15.64:0/554546710 shutdown_connections 2024-03-04T07:29:49.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.762+0000 7f5e2f842640 1 -- 172.21.15.64:0/554546710 wait complete. 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.763+0000 7f5e2f842640 1 Processor -- start 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.764+0000 7f5e2f842640 1 -- start start 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2f842640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e2806f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2f842640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e2815c930 0x7f5e2806d940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2f842640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e2806de80 0x7f5e28156e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2f842640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e280d4bf0 con 0x7f5e10004800 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2f842640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e280d2e10 con 0x7f5e2806de80 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2f842640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5e28071b90 con 0x7f5e2815c930 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e2806f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e2806f5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40542/0 (socket says 172.21.15.64:40542) 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 -- 172.21.15.64:0/1511225144 learned_addr learned my addr 172.21.15.64:0/1511225144 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2cddd640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e2815c930 0x7f5e2806d940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2dddf640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e2806de80 0x7f5e28156e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 -- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e2815c930 msgr2=0x7f5e2806d940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e2815c930 0x7f5e2806d940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 -- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e2806de80 msgr2=0x7f5e28156e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e2806de80 0x7f5e28156e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 -- 172.21.15.64:0/1511225144 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5e280cd570 con 0x7f5e10004800 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2d5de640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e2806f5f0 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f5e2000b820 tx=0x7f5e2000bce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e25ffb640 1 -- 172.21.15.64:0/1511225144 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e20018070 con 0x7f5e10004800 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e25ffb640 1 -- 172.21.15.64:0/1511225144 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5e2000be10 con 0x7f5e10004800 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5e280cd850 con 0x7f5e10004800 2024-03-04T07:29:49.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.765+0000 7f5e25ffb640 1 -- 172.21.15.64:0/1511225144 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e20010c60 con 0x7f5e10004800 2024-03-04T07:29:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.766+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5e280cb0d0 con 0x7f5e10004800 2024-03-04T07:29:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.767+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5e280cdc40 con 0x7f5e10004800 2024-03-04T07:29:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.767+0000 7f5e25ffb640 1 -- 172.21.15.64:0/1511225144 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5e2001b070 con 0x7f5e10004800 2024-03-04T07:29:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.767+0000 7f5e25ffb640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e1406b500 0x7f5e1406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.767+0000 7f5e25ffb640 1 -- 172.21.15.64:0/1511225144 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5e2008a560 con 0x7f5e10004800 2024-03-04T07:29:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.768+0000 7f5e2cddd640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e1406b500 0x7f5e1406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.768+0000 7f5e2cddd640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e1406b500 0x7f5e1406d9c0 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f5e1c00c490 tx=0x7f5e1c009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:49.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.772+0000 7f5e25ffb640 1 -- 172.21.15.64:0/1511225144 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5e280cdc40 con 0x7f5e10004800 2024-03-04T07:29:49.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.969+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f5e2806ce40 con 0x7f5e10004800 2024-03-04T07:29:49.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.969+0000 7f5e25ffb640 1 -- 172.21.15.64:0/1511225144 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v55) v1 ==== 100+0+11 (secure 0 0 0) 0x7f5e20059470 con 0x7f5e10004800 2024-03-04T07:29:49.969 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e1406b500 msgr2=0x7f5e1406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e1406b500 0x7f5e1406d9c0 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f5e1c00c490 tx=0x7f5e1c009030 comp rx=0 tx=0).stop 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 msgr2=0x7f5e2806f5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e2806f5f0 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f5e2000b820 tx=0x7f5e2000bce0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 shutdown_connections 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5e1406b500 0x7f5e1406d9c0 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5e2806de80 0x7f5e28156e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5e2815c930 0x7f5e2806d940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 --2- 172.21.15.64:0/1511225144 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5e10004800 0x7f5e2806f5f0 unknown :-1 s=CLOSED pgs=757 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:49.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 >> 172.21.15.64:0/1511225144 conn(0x7f5e280d3d20 msgr2=0x7f5e28157ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:49.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 shutdown_connections 2024-03-04T07:29:49.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:49.971+0000 7f5e2f842640 1 -- 172.21.15.64:0/1511225144 wait complete. 2024-03-04T07:29:49.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:50.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.088+0000 7fb917cdf640 1 Processor -- start 2024-03-04T07:29:50.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.088+0000 7fb917cdf640 1 -- start start 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb917cdf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb91015df00 0x7fb9100d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb917cdf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb9100b9a30 0x7fb9100d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb917cdf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb9100cea30 con 0x7fb9100b95a0 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb917cdf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb9100d2af0 con 0x7fb91015df00 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb917cdf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb9100cf3c0 con 0x7fb9100b9a30 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb915a7b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb91015df00 0x7fb9100d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb915a7b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb91015df00 0x7fb9100d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40550/0 (socket says 172.21.15.64:40550) 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.089+0000 7fb915a7b640 1 -- 172.21.15.64:0/3266745548 learned_addr learned my addr 172.21.15.64:0/3266745548 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.090+0000 7fb915a7b640 1 -- 172.21.15.64:0/3266745548 >> v2:172.21.15.66:3300/0 conn(0x7fb9100b9a30 msgr2=0x7fb9100d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.090+0000 7fb91627c640 1 --2- 172.21.15.64:0/3266745548 >> v2:172.21.15.66:3300/0 conn(0x7fb9100b9a30 0x7fb9100d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.090+0000 7fb915a7b640 1 --2- 172.21.15.64:0/3266745548 >> v2:172.21.15.66:3300/0 conn(0x7fb9100b9a30 0x7fb9100d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.090+0000 7fb915a7b640 1 -- 172.21.15.64:0/3266745548 >> v1:172.21.15.66:6789/0 conn(0x7fb9100b95a0 legacy=0x7fb9100b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.090+0000 7fb915a7b640 1 -- 172.21.15.64:0/3266745548 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9100cf0e0 con 0x7fb91015df00 2024-03-04T07:29:50.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.090+0000 7fb91627c640 1 --2- 172.21.15.64:0/3266745548 >> v2:172.21.15.66:3300/0 conn(0x7fb9100b9a30 0x7fb9100d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:50.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.091+0000 7fb915a7b640 1 --2- 172.21.15.64:0/3266745548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb91015df00 0x7fb9100d25b0 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7fb9100d4d20 tx=0x7fb90c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=57e6c2a2fe4f0ce1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.091+0000 7fb914a79640 1 -- 172.21.15.64:0/3266745548 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c035020 con 0x7fb91015df00 2024-03-04T07:29:50.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.091+0000 7fb914a79640 1 -- 172.21.15.64:0/3266745548 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb90c031870 con 0x7fb91015df00 2024-03-04T07:29:50.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.091+0000 7fb914a79640 1 -- 172.21.15.64:0/3266745548 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c031c40 con 0x7fb91015df00 2024-03-04T07:29:50.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.091+0000 7fb917cdf640 1 -- 172.21.15.64:0/3266745548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb91015df00 msgr2=0x7fb9100d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.091+0000 7fb917cdf640 1 --2- 172.21.15.64:0/3266745548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb91015df00 0x7fb9100d25b0 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7fb9100d4d20 tx=0x7fb90c02b910 comp rx=0 tx=0).stop 2024-03-04T07:29:50.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.092+0000 7fb917cdf640 1 -- 172.21.15.64:0/3266745548 shutdown_connections 2024-03-04T07:29:50.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.092+0000 7fb917cdf640 1 --2- 172.21.15.64:0/3266745548 >> v2:172.21.15.66:3300/0 conn(0x7fb9100b9a30 0x7fb9100d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.092+0000 7fb917cdf640 1 --2- 172.21.15.64:0/3266745548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb91015df00 0x7fb9100d25b0 unknown :-1 s=CLOSED pgs=758 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.092+0000 7fb917cdf640 1 -- 172.21.15.64:0/3266745548 >> 172.21.15.64:0/3266745548 conn(0x7fb910071960 msgr2=0x7fb910071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:50.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.092+0000 7fb917cdf640 1 -- 172.21.15.64:0/3266745548 shutdown_connections 2024-03-04T07:29:50.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.092+0000 7fb917cdf640 1 -- 172.21.15.64:0/3266745548 wait complete. 2024-03-04T07:29:50.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 Processor -- start 2024-03-04T07:29:50.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 -- start start 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9100b95a0 0x7fb91015bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb91015df00 0x7fb91015e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb91015e820 0x7fb91015f8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9100cec70 con 0x7fb9100b95a0 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9100ce750 con 0x7fb91015df00 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.093+0000 7fb917cdf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb9100cedf0 con 0x7fb91015e820 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9100b95a0 0x7fb91015bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9100b95a0 0x7fb91015bde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40558/0 (socket says 172.21.15.64:40558) 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 -- 172.21.15.64:0/2702536279 learned_addr learned my addr 172.21.15.64:0/2702536279 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb91627c640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb91015e820 0x7fb91015f8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb91527a640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb91015df00 0x7fb91015e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 -- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb91015e820 msgr2=0x7fb91015f8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb91015e820 0x7fb91015f8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 -- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb91015df00 msgr2=0x7fb91015e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb91015df00 0x7fb91015e2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 -- 172.21.15.64:0/2702536279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9101605b0 con 0x7fb9100b95a0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb91627c640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb91015e820 0x7fb91015f8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb91527a640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb91015df00 0x7fb91015e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb915a7b640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9100b95a0 0x7fb91015bde0 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7fb90c02b950 tx=0x7fb90c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb9067fc640 1 -- 172.21.15.64:0/2702536279 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c040070 con 0x7fb9100b95a0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb9067fc640 1 -- 172.21.15.64:0/2702536279 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb90c03a420 con 0x7fb9100b95a0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb910160890 con 0x7fb9100b95a0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb9067fc640 1 -- 172.21.15.64:0/2702536279 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c035040 con 0x7fb9100b95a0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.094+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9100c7a00 con 0x7fb9100b95a0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.095+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb91015ec30 con 0x7fb9100b95a0 2024-03-04T07:29:50.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.096+0000 7fb9067fc640 1 -- 172.21.15.64:0/2702536279 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb90c038020 con 0x7fb9100b95a0 2024-03-04T07:29:50.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.096+0000 7fb9067fc640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb8f406b4d0 0x7fb8f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.097+0000 7fb9067fc640 1 -- 172.21.15.64:0/2702536279 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb90c0ac1f0 con 0x7fb9100b95a0 2024-03-04T07:29:50.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.097+0000 7fb91527a640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb8f406b4d0 0x7fb8f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.097+0000 7fb91527a640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb8f406b4d0 0x7fb8f406d990 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fb900003eb0 tx=0x7fb900009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.101+0000 7fb9067fc640 1 -- 172.21.15.64:0/2702536279 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb91015ec30 con 0x7fb9100b95a0 2024-03-04T07:29:50.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.319+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb910067ff0 con 0x7fb9100b95a0 2024-03-04T07:29:50.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.319+0000 7fb9067fc640 1 -- 172.21.15.64:0/2702536279 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7fb90c07b070 con 0x7fb9100b95a0 2024-03-04T07:29:50.319 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:50.320 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:50.320 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:50.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb8f406b4d0 msgr2=0x7fb8f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb8f406b4d0 0x7fb8f406d990 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fb900003eb0 tx=0x7fb900009040 comp rx=0 tx=0).stop 2024-03-04T07:29:50.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9100b95a0 msgr2=0x7fb91015bde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9100b95a0 0x7fb91015bde0 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7fb90c02b950 tx=0x7fb90c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 shutdown_connections 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb8f406b4d0 0x7fb8f406d990 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb91015e820 0x7fb91015f8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb91015df00 0x7fb91015e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 --2- 172.21.15.64:0/2702536279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9100b95a0 0x7fb91015bde0 unknown :-1 s=CLOSED pgs=759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.321+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 >> 172.21.15.64:0/2702536279 conn(0x7fb910071960 msgr2=0x7fb9100b6e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.322+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 shutdown_connections 2024-03-04T07:29:50.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.322+0000 7fb917cdf640 1 -- 172.21.15.64:0/2702536279 wait complete. 2024-03-04T07:29:50.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:50.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.433+0000 7f148fdc9640 1 Processor -- start 2024-03-04T07:29:50.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.434+0000 7f148fdc9640 1 -- start start 2024-03-04T07:29:50.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148fdc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 0x7f14880d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148fdc9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f14880b0ed0 0x7f14880ce360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148fdc9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14880ce8a0 0x7f14880cfc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148fdc9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f148806eb20 con 0x7f14880ce8a0 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148fdc9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14880b1330 con 0x7f14880b0ed0 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148fdc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14880cd350 con 0x7f148815bda0 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14880ce8a0 0x7f14880cfc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14880ce8a0 0x7f14880cfc00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40568/0 (socket says 172.21.15.64:40568) 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 -- 172.21.15.64:0/3160170916 learned_addr learned my addr 172.21.15.64:0/3160170916 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148d364640 1 --2- 172.21.15.64:0/3160170916 >> v2:172.21.15.66:3300/0 conn(0x7f14880b0ed0 0x7f14880ce360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148db65640 1 --2- 172.21.15.64:0/3160170916 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 0x7f14880d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 -- 172.21.15.64:0/3160170916 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 msgr2=0x7f14880d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 --2- 172.21.15.64:0/3160170916 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 0x7f14880d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 -- 172.21.15.64:0/3160170916 >> v2:172.21.15.66:3300/0 conn(0x7f14880b0ed0 msgr2=0x7f14880ce360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 --2- 172.21.15.64:0/3160170916 >> v2:172.21.15.66:3300/0 conn(0x7f14880b0ed0 0x7f14880ce360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.435+0000 7f148e366640 1 -- 172.21.15.64:0/3160170916 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14880cd630 con 0x7f14880ce8a0 2024-03-04T07:29:50.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.436+0000 7f148e366640 1 --2- 172.21.15.64:0/3160170916 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880ce8a0 0x7f14880cfc00 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f1488072110 tx=0x7f148002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ab507a1733bfcc90 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.436+0000 7f148cb63640 1 -- 172.21.15.64:0/3160170916 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f148002ca70 con 0x7f14880ce8a0 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.436+0000 7f148cb63640 1 -- 172.21.15.64:0/3160170916 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f148002cc10 con 0x7f14880ce8a0 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.436+0000 7f148cb63640 1 -- 172.21.15.64:0/3160170916 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14800343f0 con 0x7f14880ce8a0 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 -- 172.21.15.64:0/3160170916 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880ce8a0 msgr2=0x7f14880cfc00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 --2- 172.21.15.64:0/3160170916 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880ce8a0 0x7f14880cfc00 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f1488072110 tx=0x7f148002bed0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 -- 172.21.15.64:0/3160170916 shutdown_connections 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 --2- 172.21.15.64:0/3160170916 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880ce8a0 0x7f14880cfc00 unknown :-1 s=CLOSED pgs=760 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 --2- 172.21.15.64:0/3160170916 >> v2:172.21.15.66:3300/0 conn(0x7f14880b0ed0 0x7f14880ce360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 --2- 172.21.15.64:0/3160170916 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 0x7f14880d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 -- 172.21.15.64:0/3160170916 >> 172.21.15.64:0/3160170916 conn(0x7f14880d1a90 msgr2=0x7f14880d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 -- 172.21.15.64:0/3160170916 shutdown_connections 2024-03-04T07:29:50.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.437+0000 7f148fdc9640 1 -- 172.21.15.64:0/3160170916 wait complete. 2024-03-04T07:29:50.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.438+0000 7f148fdc9640 1 Processor -- start 2024-03-04T07:29:50.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.438+0000 7f148fdc9640 1 -- start start 2024-03-04T07:29:50.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148fdc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880b0ed0 0x7f1488158630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148fdc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14880ce8a0 0x7f148815cfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148fdc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 0x7f148815ed20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148fdc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f148815c180 con 0x7f14880b0ed0 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148fdc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f148806ed70 con 0x7f14880ce8a0 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148fdc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f148815c300 con 0x7f148815bda0 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880b0ed0 0x7f1488158630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880b0ed0 0x7f1488158630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40580/0 (socket says 172.21.15.64:40580) 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 -- 172.21.15.64:0/1282216904 learned_addr learned my addr 172.21.15.64:0/1282216904 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 -- 172.21.15.64:0/1282216904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 msgr2=0x7f148815ed20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 0x7f148815ed20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 -- 172.21.15.64:0/1282216904 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14880ce8a0 msgr2=0x7f148815cfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148d364640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14880ce8a0 0x7f148815cfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14880ce8a0 0x7f148815cfa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 -- 172.21.15.64:0/1282216904 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14880c6200 con 0x7f14880b0ed0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f148db65640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880b0ed0 0x7f1488158630 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f147800d5f0 tx=0x7f147800dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f147e7fc640 1 -- 172.21.15.64:0/1282216904 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14780125f0 con 0x7f14880b0ed0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.439+0000 7f147e7fc640 1 -- 172.21.15.64:0/1282216904 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1478019070 con 0x7f14880b0ed0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.440+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1488158cc0 con 0x7f14880b0ed0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.440+0000 7f147e7fc640 1 -- 172.21.15.64:0/1282216904 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14780129d0 con 0x7f14880b0ed0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.440+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14880c6a80 con 0x7f14880b0ed0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.441+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1488067ff0 con 0x7f14880b0ed0 2024-03-04T07:29:50.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.441+0000 7f147e7fc640 1 -- 172.21.15.64:0/1282216904 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1478012b70 con 0x7f14880b0ed0 2024-03-04T07:29:50.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.441+0000 7f147e7fc640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f146c06b4d0 0x7f146c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.442+0000 7f148d364640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f146c06b4d0 0x7f146c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.442+0000 7f147e7fc640 1 -- 172.21.15.64:0/1282216904 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f147808a570 con 0x7f14880b0ed0 2024-03-04T07:29:50.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.442+0000 7f148d364640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f146c06b4d0 0x7f146c06d990 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f1488158160 tx=0x7f1484008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.446+0000 7f147e7fc640 1 -- 172.21.15.64:0/1282216904 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f14780593f0 con 0x7f14880b0ed0 2024-03-04T07:29:50.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.630+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f14880d27d0 con 0x7f14880b0ed0 2024-03-04T07:29:50.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.631+0000 7f147e7fc640 1 -- 172.21.15.64:0/1282216904 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v55) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f1478058cf0 con 0x7f14880b0ed0 2024-03-04T07:29:50.632 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:50.632 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:12.504021+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:50.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f146c06b4d0 msgr2=0x7f146c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f146c06b4d0 0x7f146c06d990 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f1488158160 tx=0x7f1484008040 comp rx=0 tx=0).stop 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880b0ed0 msgr2=0x7f1488158630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880b0ed0 0x7f1488158630 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f147800d5f0 tx=0x7f147800dab0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 shutdown_connections 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f146c06b4d0 0x7f146c06d990 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f148815bda0 0x7f148815ed20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14880ce8a0 0x7f148815cfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 --2- 172.21.15.64:0/1282216904 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14880b0ed0 0x7f1488158630 unknown :-1 s=CLOSED pgs=761 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 >> 172.21.15.64:0/1282216904 conn(0x7f14880d1a90 msgr2=0x7f14880affd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 shutdown_connections 2024-03-04T07:29:50.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.633+0000 7f148fdc9640 1 -- 172.21.15.64:0/1282216904 wait complete. 2024-03-04T07:29:50.644 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:29:50.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:50.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 Processor -- start 2024-03-04T07:29:50.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 -- start start 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f74fc15df00 0x7f74fc0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc0b95a0 0x7f74fc0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f74fc0b9a30 0x7f74fc0b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f74fc0cea30 con 0x7f74fc15df00 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f74fc0d2af0 con 0x7f74fc0b9a30 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.746+0000 7f7501ef4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f74fc0cf3c0 con 0x7f74fc0b95a0 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f74fc15df00 0x7f74fc0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f74fc15df00 0x7f74fc0d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40596/0 (socket says 172.21.15.64:40596) 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 -- 172.21.15.64:0/2362300636 learned_addr learned my addr 172.21.15.64:0/2362300636 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:50.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fbfff640 1 --2- 172.21.15.64:0/2362300636 >> v2:172.21.15.66:3300/0 conn(0x7f74fc0b9a30 0x7f74fc0b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 -- 172.21.15.64:0/2362300636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc0b95a0 msgr2=0x7f74fc0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74faffd640 1 --2- 172.21.15.64:0/2362300636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc0b95a0 0x7f74fc0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 --2- 172.21.15.64:0/2362300636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc0b95a0 0x7f74fc0b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 -- 172.21.15.64:0/2362300636 >> v2:172.21.15.66:3300/0 conn(0x7f74fc0b9a30 msgr2=0x7f74fc0b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 --2- 172.21.15.64:0/2362300636 >> v2:172.21.15.66:3300/0 conn(0x7f74fc0b9a30 0x7f74fc0b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fb7fe640 1 -- 172.21.15.64:0/2362300636 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74fc0cf0e0 con 0x7f74fc15df00 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74fbfff640 1 --2- 172.21.15.64:0/2362300636 >> v2:172.21.15.66:3300/0 conn(0x7f74fc0b9a30 0x7f74fc0b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.747+0000 7f74faffd640 1 --2- 172.21.15.64:0/2362300636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc0b95a0 0x7f74fc0b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.748+0000 7f74fb7fe640 1 --2- 172.21.15.64:0/2362300636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc0d25b0 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f74fc0d1160 tx=0x7f74ec02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2f492d841dab55f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.748+0000 7f74fa7fc640 1 -- 172.21.15.64:0/2362300636 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74ec035020 con 0x7f74fc15df00 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.748+0000 7f74fa7fc640 1 -- 172.21.15.64:0/2362300636 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f74ec031870 con 0x7f74fc15df00 2024-03-04T07:29:50.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.748+0000 7f74fa7fc640 1 -- 172.21.15.64:0/2362300636 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74ec031c40 con 0x7f74fc15df00 2024-03-04T07:29:50.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 -- 172.21.15.64:0/2362300636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 msgr2=0x7f74fc0d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 --2- 172.21.15.64:0/2362300636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc0d25b0 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f74fc0d1160 tx=0x7f74ec02b910 comp rx=0 tx=0).stop 2024-03-04T07:29:50.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 -- 172.21.15.64:0/2362300636 shutdown_connections 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 --2- 172.21.15.64:0/2362300636 >> v2:172.21.15.66:3300/0 conn(0x7f74fc0b9a30 0x7f74fc0b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 --2- 172.21.15.64:0/2362300636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc0b95a0 0x7f74fc0b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 --2- 172.21.15.64:0/2362300636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc0d25b0 unknown :-1 s=CLOSED pgs=762 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 -- 172.21.15.64:0/2362300636 >> 172.21.15.64:0/2362300636 conn(0x7f74fc071960 msgr2=0x7f74fc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 -- 172.21.15.64:0/2362300636 shutdown_connections 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.749+0000 7f7501ef4640 1 -- 172.21.15.64:0/2362300636 wait complete. 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.751+0000 7f7501ef4640 1 Processor -- start 2024-03-04T07:29:50.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.751+0000 7f7501ef4640 1 -- start start 2024-03-04T07:29:50.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f7501ef4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74fc0b95a0 0x7f74fc0ccaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f7501ef4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc1e7830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f7501ef4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc1e7d70 0x7f74fc1f2e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f7501ef4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74fc0cec70 con 0x7f74fc15df00 2024-03-04T07:29:50.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f7501ef4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74fc0ce750 con 0x7f74fc0b95a0 2024-03-04T07:29:50.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f7501ef4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f74fc0cedf0 con 0x7f74fc1e7d70 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc1e7830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc1e7830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40608/0 (socket says 172.21.15.64:40608) 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74faffd640 1 -- 172.21.15.64:0/3597252583 learned_addr learned my addr 172.21.15.64:0/3597252583 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74fbfff640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc1e7d70 0x7f74fc1f2e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74fb7fe640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74fc0b95a0 0x7f74fc0ccaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74faffd640 1 -- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc1e7d70 msgr2=0x7f74fc1f2e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74faffd640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc1e7d70 0x7f74fc1f2e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74faffd640 1 -- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74fc0b95a0 msgr2=0x7f74fc0ccaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.752+0000 7f74faffd640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74fc0b95a0 0x7f74fc0ccaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.753+0000 7f74faffd640 1 -- 172.21.15.64:0/3597252583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74fc1e85a0 con 0x7f74fc15df00 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.753+0000 7f74faffd640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc1e7830 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7f74f000c7f0 tx=0x7f74f000cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.753+0000 7f74e3fff640 1 -- 172.21.15.64:0/3597252583 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74f0015070 con 0x7f74fc15df00 2024-03-04T07:29:50.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.753+0000 7f74e3fff640 1 -- 172.21.15.64:0/3597252583 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f74f0007a30 con 0x7f74fc15df00 2024-03-04T07:29:50.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.753+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f74fc0c7b00 con 0x7f74fc15df00 2024-03-04T07:29:50.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.753+0000 7f74e3fff640 1 -- 172.21.15.64:0/3597252583 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f74f0007e00 con 0x7f74fc15df00 2024-03-04T07:29:50.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.753+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f74fc0c77b0 con 0x7f74fc15df00 2024-03-04T07:29:50.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.754+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f74fc067ff0 con 0x7f74fc15df00 2024-03-04T07:29:50.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.754+0000 7f74e3fff640 1 -- 172.21.15.64:0/3597252583 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f74f0018070 con 0x7f74fc15df00 2024-03-04T07:29:50.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.755+0000 7f74e3fff640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f74e806b500 0x7f74e806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:50.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.755+0000 7f74e3fff640 1 -- 172.21.15.64:0/3597252583 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f74f008a150 con 0x7f74fc15df00 2024-03-04T07:29:50.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.755+0000 7f74fb7fe640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f74e806b500 0x7f74e806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:50.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.755+0000 7f74fb7fe640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f74e806b500 0x7f74e806d9c0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f74ec036000 tx=0x7f74ec03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:50.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.759+0000 7f74e3fff640 1 -- 172.21.15.64:0/3597252583 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f74f0058fd0 con 0x7f74fc15df00 2024-03-04T07:29:50.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.977+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f74fc15c860 con 0x7f74fc15df00 2024-03-04T07:29:50.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.978+0000 7f74e3fff640 1 -- 172.21.15.64:0/3597252583 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f74f00588d0 con 0x7f74fc15df00 2024-03-04T07:29:50.978 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:50.978 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:50.978 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:50.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f74e806b500 msgr2=0x7f74e806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f74e806b500 0x7f74e806d9c0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f74ec036000 tx=0x7f74ec03c040 comp rx=0 tx=0).stop 2024-03-04T07:29:50.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 msgr2=0x7f74fc1e7830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:50.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc1e7830 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7f74f000c7f0 tx=0x7f74f000cbf0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 shutdown_connections 2024-03-04T07:29:50.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f74e806b500 0x7f74e806d9c0 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f74fc1e7d70 0x7f74fc1f2e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74fc15df00 0x7f74fc1e7830 unknown :-1 s=CLOSED pgs=763 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 --2- 172.21.15.64:0/3597252583 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74fc0b95a0 0x7f74fc0ccaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:50.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 >> 172.21.15.64:0/3597252583 conn(0x7f74fc071960 msgr2=0x7f74fc1618b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:50.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 shutdown_connections 2024-03-04T07:29:50.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:50.980+0000 7f7501ef4640 1 -- 172.21.15.64:0/3597252583 wait complete. 2024-03-04T07:29:50.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:51.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.092+0000 7f1667755640 1 Processor -- start 2024-03-04T07:29:51.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.092+0000 7f1667755640 1 -- start start 2024-03-04T07:29:51.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f1667755640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f16600d02e0 0x7f16600cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f1667755640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600d06c0 0x7f16600b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f1667755640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16600b0dd0 con 0x7f16600b4260 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f1667755640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f16600b0b80 con 0x7f16600d02e0 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f1667755640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16600aeec0 con 0x7f16600d06c0 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f16654f1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f16600d02e0 0x7f16600cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f16654f1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f16600d02e0 0x7f16600cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40618/0 (socket says 172.21.15.64:40618) 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.093+0000 7f16654f1640 1 -- 172.21.15.64:0/176526647 learned_addr learned my addr 172.21.15.64:0/176526647 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.094+0000 7f1665cf2640 1 --2- 172.21.15.64:0/176526647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600d06c0 0x7f16600b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.094+0000 7f16654f1640 1 -- 172.21.15.64:0/176526647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600d06c0 msgr2=0x7f16600b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.094+0000 7f16654f1640 1 --2- 172.21.15.64:0/176526647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600d06c0 0x7f16600b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.094+0000 7f16654f1640 1 -- 172.21.15.64:0/176526647 >> v1:172.21.15.66:6789/0 conn(0x7f16600b4260 legacy=0x7f16600b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.094+0000 7f16654f1640 1 -- 172.21.15.64:0/176526647 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16600b10b0 con 0x7f16600d02e0 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.094+0000 7f16654f1640 1 --2- 172.21.15.64:0/176526647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16600cfa00 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7f165c009870 tx=0x7f165c02ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=19ae72a23116a678 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1657fff640 1 -- 172.21.15.64:0/176526647 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f165c02f840 con 0x7f16600d02e0 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1657fff640 1 -- 172.21.15.64:0/176526647 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f165c02f9e0 con 0x7f16600d02e0 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1657fff640 1 -- 172.21.15.64:0/176526647 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f165c02fdb0 con 0x7f16600d02e0 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 -- 172.21.15.64:0/176526647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 msgr2=0x7f16600cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 --2- 172.21.15.64:0/176526647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16600cfa00 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7f165c009870 tx=0x7f165c02ba20 comp rx=0 tx=0).stop 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 -- 172.21.15.64:0/176526647 shutdown_connections 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 --2- 172.21.15.64:0/176526647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600d06c0 0x7f16600b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 --2- 172.21.15.64:0/176526647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16600cfa00 unknown :-1 s=CLOSED pgs=764 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 -- 172.21.15.64:0/176526647 >> 172.21.15.64:0/176526647 conn(0x7f16600b36d0 msgr2=0x7f16600b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:51.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 -- 172.21.15.64:0/176526647 shutdown_connections 2024-03-04T07:29:51.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.095+0000 7f1667755640 1 -- 172.21.15.64:0/176526647 wait complete. 2024-03-04T07:29:51.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.096+0000 7f1667755640 1 Processor -- start 2024-03-04T07:29:51.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.097+0000 7f1667755640 1 -- start start 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1667755640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600b4260 0x7f1660153f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1667755640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16601544c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1667755640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16600d06c0 0x7f1660154a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1667755640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16600b22d0 con 0x7f16600d02e0 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1667755640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16600b2150 con 0x7f16600d06c0 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1667755640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16600af350 con 0x7f16600b4260 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16601544c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16601544c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40628/0 (socket says 172.21.15.64:40628) 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 -- 172.21.15.64:0/1011965711 learned_addr learned my addr 172.21.15.64:0/1011965711 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f16654f1640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600b4260 0x7f1660153f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1665cf2640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16600d06c0 0x7f1660154a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 -- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600b4260 msgr2=0x7f1660153f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600b4260 0x7f1660153f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 -- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16600d06c0 msgr2=0x7f1660154a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16600d06c0 0x7f1660154a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 -- 172.21.15.64:0/1011965711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16601f1b90 con 0x7f16600d02e0 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1664cf0640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16601544c0 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7f165000c430 tx=0x7f165000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.098+0000 7f1655ffb640 1 -- 172.21.15.64:0/1011965711 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16500052d0 con 0x7f16600d02e0 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.099+0000 7f1655ffb640 1 -- 172.21.15.64:0/1011965711 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1650005490 con 0x7f16600d02e0 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.099+0000 7f1655ffb640 1 -- 172.21.15.64:0/1011965711 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1650007260 con 0x7f16600d02e0 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.099+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16600ca320 con 0x7f16600d02e0 2024-03-04T07:29:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.099+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16600ca8e0 con 0x7f16600d02e0 2024-03-04T07:29:51.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.100+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1660072d90 con 0x7f16600d02e0 2024-03-04T07:29:51.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.100+0000 7f1655ffb640 1 -- 172.21.15.64:0/1011965711 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1650007400 con 0x7f16600d02e0 2024-03-04T07:29:51.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.100+0000 7f1655ffb640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164406b4d0 0x7f164406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.101+0000 7f16654f1640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164406b4d0 0x7f164406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.101+0000 7f1655ffb640 1 -- 172.21.15.64:0/1011965711 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1650089640 con 0x7f16600d02e0 2024-03-04T07:29:51.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.101+0000 7f16654f1640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164406b4d0 0x7f164406d990 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f165c036000 tx=0x7f165c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.105+0000 7f1655ffb640 1 -- 172.21.15.64:0/1011965711 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1660072d90 con 0x7f16600d02e0 2024-03-04T07:29:51.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.326+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1660068700 con 0x7f16600d02e0 2024-03-04T07:29:51.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.327+0000 7f1655ffb640 1 -- 172.21.15.64:0/1011965711 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) v1 ==== 94+0+4974 (secure 0 0 0) 0x7f1650057e70 con 0x7f16600d02e0 2024-03-04T07:29:51.327 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:51.327 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":29,"default_fscid":4,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26}],"filesystems":[{"mdsmap":{"epoch":29,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:29:10.861414+0000","modified":"2024-03-04T07:29:12.860001+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6069},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6069":{"gid":6069,"name":"c","rank":0,"incarnation":27,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/1811282816","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":1811282816},{"type":"v1","addr":"172.21.15.64:6837","nonce":1811282816}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[9],"metadata_pool":8,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6069,"qdb_cluster":[6069]},"id":4}]} 2024-03-04T07:29:51.327 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 29 2024-03-04T07:29:51.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.329+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164406b4d0 msgr2=0x7f164406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.329+0000 7f1667755640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164406b4d0 0x7f164406d990 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f165c036000 tx=0x7f165c008030 comp rx=0 tx=0).stop 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.329+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 msgr2=0x7f16601544c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.329+0000 7f1667755640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16601544c0 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7f165000c430 tx=0x7f165000c8f0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 shutdown_connections 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164406b4d0 0x7f164406d990 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16600d06c0 0x7f1660154a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16600d02e0 0x7f16601544c0 unknown :-1 s=CLOSED pgs=765 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 --2- 172.21.15.64:0/1011965711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16600b4260 0x7f1660153f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 >> 172.21.15.64:0/1011965711 conn(0x7f16600b36d0 msgr2=0x7f1660140840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 shutdown_connections 2024-03-04T07:29:51.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.330+0000 7f1667755640 1 -- 172.21.15.64:0/1011965711 wait complete. 2024-03-04T07:29:51.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:51.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.443+0000 7f013c881640 1 Processor -- start 2024-03-04T07:29:51.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.443+0000 7f013c881640 1 -- start start 2024-03-04T07:29:51.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013c881640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f013815df00 0x7f01380d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013c881640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f01380cea30 con 0x7f01380b95a0 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013c881640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f01380d2af0 con 0x7f01380b9a30 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013c881640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f01380cf3c0 con 0x7f013815df00 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f0135d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f01380b95a0 0x7f01380b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41318/0 (socket says 172.21.15.64:41318) 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f0135d9b640 1 -- 172.21.15.64:0/3687340977 learned_addr learned my addr 172.21.15.64:0/3687340977 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013659c640 1 --2- 172.21.15.64:0/3687340977 >> v2:172.21.15.66:3300/0 conn(0x7f013815df00 0x7f01380d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 441498244 0 0) 0x7f01380cea30 con 0x7f01380b95a0 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0120003520 con 0x7f01380b95a0 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 692219888 0 0) 0x7f0120003520 con 0x7f01380b95a0 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 >> v2:172.21.15.66:3300/0 conn(0x7f013815df00 msgr2=0x7f01380d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 --2- 172.21.15.64:0/3687340977 >> v2:172.21.15.66:3300/0 conn(0x7f013815df00 0x7f01380d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 >> v1:172.21.15.66:6789/0 conn(0x7f01380b9a30 legacy=0x7f01380d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01380cf0e0 con 0x7f01380b95a0 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f012c001670 con 0x7f01380b95a0 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.444+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 >> v1:172.21.15.64:6789/0 conn(0x7f01380b95a0 legacy=0x7f01380b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f013559a640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01200039b0 0x7f0120003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f013559a640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0120004800 0x7f0120004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f013559a640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01200051c0 0x7f012000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01380cf3c0 con 0x7f01200051c0 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f012000c060 con 0x7f0120004800 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f012000c260 con 0x7f01200039b0 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f0136d9d640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01200051c0 0x7f012000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f0135d9b640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0120004800 0x7f0120004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f013659c640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01200039b0 0x7f0120003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f0136d9d640 1 -- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01200039b0 msgr2=0x7f0120003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f0136d9d640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01200039b0 0x7f0120003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f0136d9d640 1 -- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0120004800 msgr2=0x7f0120004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f0136d9d640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0120004800 0x7f0120004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.445+0000 7f0136d9d640 1 -- 172.21.15.64:0/3687340977 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0120003520 con 0x7f01200051c0 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f0136d9d640 1 -- 172.21.15.64:0/3687340977 reap_dead start 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f0136d9d640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01200051c0 0x7f012000b9c0 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f0130001e50 tx=0x7f0120005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80458528273e44ae server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0130001a70 con 0x7f01200051c0 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0130003bb0 con 0x7f01200051c0 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013559a640 1 -- 172.21.15.64:0/3687340977 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f01300050d0 con 0x7f01200051c0 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013c881640 1 -- 172.21.15.64:0/3687340977 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01200051c0 msgr2=0x7f012000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013c881640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01200051c0 0x7f012000b9c0 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f0130001e50 tx=0x7f0120005830 comp rx=0 tx=0).stop 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013c881640 1 -- 172.21.15.64:0/3687340977 shutdown_connections 2024-03-04T07:29:51.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013c881640 1 --2- 172.21.15.64:0/3687340977 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01200051c0 0x7f012000b9c0 unknown :-1 s=CLOSED pgs=767 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.446+0000 7f013c881640 1 -- 172.21.15.64:0/3687340977 >> 172.21.15.64:0/3687340977 conn(0x7f0138071960 msgr2=0x7f0138071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:51.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.447+0000 7f013c881640 1 -- 172.21.15.64:0/3687340977 shutdown_connections 2024-03-04T07:29:51.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.447+0000 7f013c881640 1 -- 172.21.15.64:0/3687340977 wait complete. 2024-03-04T07:29:51.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.448+0000 7f013c881640 1 Processor -- start 2024-03-04T07:29:51.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.448+0000 7f013c881640 1 -- start start 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01200051c0 0x7f01380ca0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013815fe20 0x7f01381f6de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01381f7320 0x7f01380ccdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01380cec70 con 0x7f013815fe20 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01380ce750 con 0x7f01200051c0 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f01380cedf0 con 0x7f01381f7320 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013815fe20 0x7f01381f6de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013815fe20 0x7f01381f6de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40636/0 (socket says 172.21.15.64:40636) 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 -- 172.21.15.64:0/4259766484 learned_addr learned my addr 172.21.15.64:0/4259766484 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013659c640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01200051c0 0x7f01380ca0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0136d9d640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01381f7320 0x7f01380ccdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 -- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01381f7320 msgr2=0x7f01380ccdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01381f7320 0x7f01380ccdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 -- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01200051c0 msgr2=0x7f01380ca0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01200051c0 0x7f01380ca0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 -- 172.21.15.64:0/4259766484 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01380ca5e0 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0136d9d640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01381f7320 0x7f01380ccdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0135d9b640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013815fe20 0x7f01381f6de0 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7f012c007370 tx=0x7f012c007770 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0126ffd640 1 -- 172.21.15.64:0/4259766484 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f012c01a070 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0126ffd640 1 -- 172.21.15.64:0/4259766484 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f012c00f9c0 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01381602e0 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f0126ffd640 1 -- 172.21.15.64:0/4259766484 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f012c00fd70 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.449+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f01381608a0 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.451+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f01381f7730 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.451+0000 7f0126ffd640 1 -- 172.21.15.64:0/4259766484 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f012c01d070 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.451+0000 7f0126ffd640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011806b4d0 0x7f011806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.451+0000 7f0126ffd640 1 -- 172.21.15.64:0/4259766484 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f012c089c00 con 0x7f013815fe20 2024-03-04T07:29:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.451+0000 7f013659c640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011806b4d0 0x7f011806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.452+0000 7f013659c640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011806b4d0 0x7f011806d990 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f0128009b40 tx=0x7f0128009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.460+0000 7f0126ffd640 1 -- 172.21.15.64:0/4259766484 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f01381f7730 con 0x7f013815fe20 2024-03-04T07:29:51.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.650+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0138067ff0 con 0x7f013815fe20 2024-03-04T07:29:51.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.651+0000 7f0126ffd640 1 -- 172.21.15.64:0/4259766484 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v55) v1 ==== 74+0+16617 (secure 0 0 0) 0x7f012c090080 con 0x7f013815fe20 2024-03-04T07:29:51.651 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:51.651 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:12.504021+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":8,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:29:09.132151+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"54","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":9,"pool_name":"cephfs_data","create_time":"2024-03-04T07:29:09.825115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.6299999952316284,"score_stable":1.6299999952316284,"optimal_score":1,"raw_score_acting":1.6299999952316284,"raw_score_stable":1.6299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:51.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011806b4d0 msgr2=0x7f011806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011806b4d0 0x7f011806d990 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f0128009b40 tx=0x7f0128009190 comp rx=0 tx=0).stop 2024-03-04T07:29:51.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013815fe20 msgr2=0x7f01381f6de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013815fe20 0x7f01381f6de0 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7f012c007370 tx=0x7f012c007770 comp rx=0 tx=0).stop 2024-03-04T07:29:51.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 shutdown_connections 2024-03-04T07:29:51.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011806b4d0 0x7f011806d990 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f01381f7320 0x7f01380ccdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013815fe20 0x7f01381f6de0 unknown :-1 s=CLOSED pgs=768 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 --2- 172.21.15.64:0/4259766484 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01200051c0 0x7f01380ca0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 >> 172.21.15.64:0/4259766484 conn(0x7f0138071960 msgr2=0x7f01380b5830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:51.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 shutdown_connections 2024-03-04T07:29:51.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.653+0000 7f013c881640 1 -- 172.21.15.64:0/4259766484 wait complete. 2024-03-04T07:29:51.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:29:51.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.770+0000 7f326e20e640 1 Processor -- start 2024-03-04T07:29:51.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.770+0000 7f326e20e640 1 -- start start 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f326e20e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f32680b0360 0x7f32680cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f326e20e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f32680b2320 con 0x7f32680b4260 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f326e20e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f32680b0dd0 con 0x7f32680d02e0 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f326e20e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f32680b0b80 con 0x7f32680b0360 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f3267fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f32680b0360 0x7f32680cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f3267fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f32680b0360 0x7f32680cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40646/0 (socket says 172.21.15.64:40646) 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f3267fff640 1 -- 172.21.15.64:0/3193245615 learned_addr learned my addr 172.21.15.64:0/3193245615 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f3266ffd640 1 --1- 172.21.15.64:0/3193245615 >> v1:172.21.15.64:6789/0 conn(0x7f32680b4260 0x7f32680b1c10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41324/0 (socket says 172.21.15.64:41324) 2024-03-04T07:29:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f3267fff640 1 -- 172.21.15.64:0/3193245615 >> v1:172.21.15.66:6789/0 conn(0x7f32680d02e0 legacy=0x7f32680b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:51.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f3267fff640 1 -- 172.21.15.64:0/3193245615 >> v1:172.21.15.64:6789/0 conn(0x7f32680b4260 legacy=0x7f32680b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f32667fc640 1 -- 172.21.15.64:0/3193245615 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3844845387 0 0) 0x7f32680b2320 con 0x7f32680b4260 2024-03-04T07:29:51.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.772+0000 7f3267fff640 1 -- 172.21.15.64:0/3193245615 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32680b10b0 con 0x7f32680b0360 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.773+0000 7f3267fff640 1 --2- 172.21.15.64:0/3193245615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b0360 0x7f32680cfa00 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f3260009e50 tx=0x7f326002c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=869ee22fb7ae05dc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.773+0000 7f32667fc640 1 -- 172.21.15.64:0/3193245615 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f326002cbb0 con 0x7f32680b0360 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.773+0000 7f32667fc640 1 -- 172.21.15.64:0/3193245615 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f326002cd50 con 0x7f32680b0360 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.773+0000 7f32667fc640 1 -- 172.21.15.64:0/3193245615 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3260034610 con 0x7f32680b0360 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.774+0000 7f326e20e640 1 -- 172.21.15.64:0/3193245615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b0360 msgr2=0x7f32680cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.774+0000 7f326e20e640 1 --2- 172.21.15.64:0/3193245615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b0360 0x7f32680cfa00 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f3260009e50 tx=0x7f326002c010 comp rx=0 tx=0).stop 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.774+0000 7f326e20e640 1 -- 172.21.15.64:0/3193245615 shutdown_connections 2024-03-04T07:29:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.774+0000 7f326e20e640 1 --2- 172.21.15.64:0/3193245615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b0360 0x7f32680cfa00 unknown :-1 s=CLOSED pgs=770 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.774+0000 7f326e20e640 1 -- 172.21.15.64:0/3193245615 >> 172.21.15.64:0/3193245615 conn(0x7f32680b36d0 msgr2=0x7f32680b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:51.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.774+0000 7f326e20e640 1 -- 172.21.15.64:0/3193245615 shutdown_connections 2024-03-04T07:29:51.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.774+0000 7f326e20e640 1 -- 172.21.15.64:0/3193245615 wait complete. 2024-03-04T07:29:51.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.775+0000 7f326e20e640 1 Processor -- start 2024-03-04T07:29:51.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f326e20e640 1 -- start start 2024-03-04T07:29:51.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f326e20e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32680b0360 0x7f32680cb800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f326e20e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b4260 0x7f32681f4090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f326e20e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32680d02e0 0x7f32681f3120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f326e20e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32680d06c0 con 0x7f32680b4260 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f326e20e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32680aeec0 con 0x7f32680b0360 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f326e20e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32680d0840 con 0x7f32680d02e0 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f3266ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b4260 0x7f32681f4090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f3266ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b4260 0x7f32681f4090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40660/0 (socket says 172.21.15.64:40660) 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.776+0000 7f3266ffd640 1 -- 172.21.15.64:0/3179810767 learned_addr learned my addr 172.21.15.64:0/3179810767 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f32677fe640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32680b0360 0x7f32680cb800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3267fff640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32680d02e0 0x7f32681f3120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3266ffd640 1 -- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32680d02e0 msgr2=0x7f32681f3120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3266ffd640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32680d02e0 0x7f32681f3120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3266ffd640 1 -- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32680b0360 msgr2=0x7f32680cb800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3266ffd640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32680b0360 0x7f32680cb800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3266ffd640 1 -- 172.21.15.64:0/3179810767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32680c8690 con 0x7f32680b4260 2024-03-04T07:29:51.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f32677fe640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32680b0360 0x7f32680cb800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:51.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3267fff640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32680d02e0 0x7f32681f3120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:51.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f3266ffd640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b4260 0x7f32681f4090 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7f325000c430 tx=0x7f325000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f325ffff640 1 -- 172.21.15.64:0/3179810767 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32500074e0 con 0x7f32680b4260 2024-03-04T07:29:51.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f325ffff640 1 -- 172.21.15.64:0/3179810767 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f32500047e0 con 0x7f32680b4260 2024-03-04T07:29:51.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f325ffff640 1 -- 172.21.15.64:0/3179810767 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3250004bd0 con 0x7f32680b4260 2024-03-04T07:29:51.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32680c9640 con 0x7f32680b4260 2024-03-04T07:29:51.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.777+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f32680c9e80 con 0x7f32680b4260 2024-03-04T07:29:51.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.778+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f32680ca080 con 0x7f32680b4260 2024-03-04T07:29:51.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.779+0000 7f325ffff640 1 -- 172.21.15.64:0/3179810767 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3250011070 con 0x7f32680b4260 2024-03-04T07:29:51.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.779+0000 7f325ffff640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324806b4d0 0x7f324806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:51.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.779+0000 7f325ffff640 1 -- 172.21.15.64:0/3179810767 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(55..55 src has 1..55) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3250089ec0 con 0x7f32680b4260 2024-03-04T07:29:51.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.779+0000 7f32677fe640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324806b4d0 0x7f324806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:51.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.780+0000 7f32677fe640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324806b4d0 0x7f324806d990 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f3258003eb0 tx=0x7f3258009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:51.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:51.787+0000 7f325ffff640 1 -- 172.21.15.64:0/3179810767 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f32680ca080 con 0x7f32680b4260 2024-03-04T07:29:52.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.006+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f3268068700 con 0x7f32680b4260 2024-03-04T07:29:52.006 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:29:52.007+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:29:52.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.008+0000 7f325ffff640 1 -- 172.21.15.64:0/3179810767 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v30) v1 ==== 161+0+0 (secure 0 0 0) 0x7f32680c9640 con 0x7f32680b4260 2024-03-04T07:29:52.008 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:29:52.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324806b4d0 msgr2=0x7f324806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324806b4d0 0x7f324806d990 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f3258003eb0 tx=0x7f3258009040 comp rx=0 tx=0).stop 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b4260 msgr2=0x7f32681f4090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b4260 0x7f32681f4090 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7f325000c430 tx=0x7f325000c8f0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 shutdown_connections 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f324806b4d0 0x7f324806d990 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32680d02e0 0x7f32681f3120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32680b4260 0x7f32681f4090 unknown :-1 s=CLOSED pgs=771 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 --2- 172.21.15.64:0/3179810767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32680b0360 0x7f32680cb800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.011+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 >> 172.21.15.64:0/3179810767 conn(0x7f32680b36d0 msgr2=0x7f3268151d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:52.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.012+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 shutdown_connections 2024-03-04T07:29:52.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.012+0000 7f326e20e640 1 -- 172.21.15.64:0/3179810767 wait complete. 2024-03-04T07:29:52.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:29:52.048 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:29:52.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.130+0000 7f7a18365640 1 Processor -- start 2024-03-04T07:29:52.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.130+0000 7f7a18365640 1 -- start start 2024-03-04T07:29:52.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a18365640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 0x7f7a100d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a18365640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7a100b6d10 0x7f7a100b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a18365640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7a100d3740 con 0x7f7a1015dd90 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a18365640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7a100d3400 con 0x7f7a100b6d10 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a18365640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7a100d3120 con 0x7f7a100ceea0 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a16101640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7a1015dd90 0x7f7a100d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41336/0 (socket says 172.21.15.64:41336) 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a16101640 1 -- 172.21.15.64:0/4272793574 learned_addr learned my addr 172.21.15.64:0/4272793574 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a16902640 1 --2- 172.21.15.64:0/4272793574 >> v2:172.21.15.66:3300/0 conn(0x7f7a100b6d10 0x7f7a100b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.131+0000 7f7a15900640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 0x7f7a100d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4228333422 0 0) 0x7f7a100d3740 con 0x7f7a1015dd90 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f79fc003520 con 0x7f7a1015dd90 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2128717328 0 0) 0x7f79fc003520 con 0x7f7a1015dd90 2024-03-04T07:29:52.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 msgr2=0x7f7a100d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 0x7f7a100d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 >> v2:172.21.15.66:3300/0 conn(0x7f7a100b6d10 msgr2=0x7f7a100b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 --2- 172.21.15.64:0/4272793574 >> v2:172.21.15.66:3300/0 conn(0x7f7a100b6d10 0x7f7a100b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a100d39e0 con 0x7f7a1015dd90 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f7a040019f0 con 0x7f7a1015dd90 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 >> v1:172.21.15.64:6789/0 conn(0x7f7a1015dd90 legacy=0x7f7a100d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f79fc003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79fc004800 0x7f79fc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 0x7f79fc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a100d3400 con 0x7f79fc0039b0 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a100d3120 con 0x7f79fc004800 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79fc00b9c0 con 0x7f7a100ceea0 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16101640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f79fc003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a15900640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79fc004800 0x7f79fc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16902640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 0x7f79fc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16101640 1 -- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 msgr2=0x7f79fc0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16101640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 0x7f79fc0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16101640 1 -- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79fc004800 msgr2=0x7f79fc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16101640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79fc004800 0x7f79fc004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16101640 1 -- 172.21.15.64:0/4272793574 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79fc003520 con 0x7f79fc0039b0 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.132+0000 7f7a16902640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100ceea0 0x7f79fc0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.133+0000 7f7a16101640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f79fc003dd0 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f7a0400a860 tx=0x7f79fc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=90c6c72f91654247 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.133+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a0400a480 con 0x7f79fc0039b0 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.133+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a04034630 con 0x7f79fc0039b0 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.133+0000 7f7a150ff640 1 -- 172.21.15.64:0/4272793574 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a04034a40 con 0x7f79fc0039b0 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.134+0000 7f7a18365640 1 -- 172.21.15.64:0/4272793574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 msgr2=0x7f79fc003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:52.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.134+0000 7f7a18365640 1 --2- 172.21.15.64:0/4272793574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f79fc003dd0 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f7a0400a860 tx=0x7f79fc005830 comp rx=0 tx=0).stop 2024-03-04T07:29:52.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.134+0000 7f7a16902640 1 -- 172.21.15.64:0/4272793574 reap_dead start 2024-03-04T07:29:52.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.134+0000 7f7a18365640 1 -- 172.21.15.64:0/4272793574 shutdown_connections 2024-03-04T07:29:52.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.134+0000 7f7a18365640 1 -- 172.21.15.64:0/4272793574 >> 172.21.15.64:0/4272793574 conn(0x7f7a10071960 msgr2=0x7f7a10071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:52.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.134+0000 7f7a18365640 1 -- 172.21.15.64:0/4272793574 shutdown_connections 2024-03-04T07:29:52.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.134+0000 7f7a18365640 1 -- 172.21.15.64:0/4272793574 wait complete. 2024-03-04T07:29:52.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.135+0000 7f7a18365640 1 Processor -- start 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.135+0000 7f7a18365640 1 -- start start 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f7a100c8800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100c7390 0x7f7a100c8d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a100c9280 0x7f7a1015f6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a100d48f0 con 0x7f79fc0039b0 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a1015e170 con 0x7f7a100c9280 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7a100d4a70 con 0x7f7a100c7390 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f7a100c8800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f7a100c8800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40684/0 (socket says 172.21.15.64:40684) 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 -- 172.21.15.64:0/1859506720 learned_addr learned my addr 172.21.15.64:0/1859506720 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16902640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a100c9280 0x7f7a1015f6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a15900640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100c7390 0x7f7a100c8d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 -- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100c7390 msgr2=0x7f7a100c8d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100c7390 0x7f7a100c8d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 -- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a100c9280 msgr2=0x7f7a1015f6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a100c9280 0x7f7a1015f6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 -- 172.21.15.64:0/1859506720 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a1015e940 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a15900640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100c7390 0x7f7a100c8d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a16101640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f7a100c8800 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f7a0400a860 tx=0x7f7a100d3bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a02ffd640 1 -- 172.21.15.64:0/1859506720 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a04043020 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a02ffd640 1 -- 172.21.15.64:0/1859506720 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a04034430 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a1015ebe0 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a02ffd640 1 -- 172.21.15.64:0/1859506720 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a0400a480 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.136+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a10155830 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.137+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7a10072a30 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.138+0000 7f7a02ffd640 1 -- 172.21.15.64:0/1859506720 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7a04046020 con 0x7f79fc0039b0 2024-03-04T07:29:52.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.138+0000 7f7a02ffd640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79f406b500 0x7f79f406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:52.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.138+0000 7f7a02ffd640 1 -- 172.21.15.64:0/1859506720 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(56..56 src has 1..56) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f7a040ac650 con 0x7f79fc0039b0 2024-03-04T07:29:52.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.138+0000 7f7a15900640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79f406b500 0x7f79f406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:52.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.139+0000 7f7a15900640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79f406b500 0x7f79f406d9c0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f7a100ca7f0 tx=0x7f7a0c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:52.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.142+0000 7f7a02ffd640 1 -- 172.21.15.64:0/1859506720 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7a0404d030 con 0x7f79fc0039b0 2024-03-04T07:29:52.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:52.362+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f7a1015e680 con 0x7f79fc0039b0 2024-03-04T07:29:53.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.010+0000 7f7a02ffd640 1 -- 172.21.15.64:0/1859506720 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v31) v1 ==== 104+0+0 (secure 0 0 0) 0x7f7a04034600 con 0x7f79fc0039b0 2024-03-04T07:29:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.012+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79f406b500 msgr2=0x7f79f406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:53.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.012+0000 7f7a18365640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79f406b500 0x7f79f406d9c0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f7a100ca7f0 tx=0x7f7a0c008040 comp rx=0 tx=0).stop 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.012+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 msgr2=0x7f7a100c8800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.012+0000 7f7a18365640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f7a100c8800 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f7a0400a860 tx=0x7f7a100d3bb0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 shutdown_connections 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f79f406b500 0x7f79f406d9c0 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a100c9280 0x7f7a1015f6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a100c7390 0x7f7a100c8d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 --2- 172.21.15.64:0/1859506720 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79fc0039b0 0x7f7a100c8800 unknown :-1 s=CLOSED pgs=775 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 >> 172.21.15.64:0/1859506720 conn(0x7f7a10071960 msgr2=0x7f7a100d0c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 shutdown_connections 2024-03-04T07:29:53.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.013+0000 7f7a18365640 1 -- 172.21.15.64:0/1859506720 wait complete. 2024-03-04T07:29:53.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:29:53.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.179+0000 7f5725779640 1 Processor -- start 2024-03-04T07:29:53.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.179+0000 7f5725779640 1 -- start start 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f5725779640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5720071540 0x7f5720071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f5725779640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f57200b2d30 0x7f57200ce770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f5725779640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5720071e60 con 0x7f57200cecb0 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f5725779640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f57200d3230 con 0x7f5720071540 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f5725779640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f57200d4960 con 0x7f57200b2d30 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571effd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5720071540 0x7f5720071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571f7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f57200cecb0 0x7f57200b72e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41350/0 (socket says 172.21.15.64:41350) 2024-03-04T07:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571f7fe640 1 -- 172.21.15.64:0/4237091668 learned_addr learned my addr 172.21.15.64:0/4237091668 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:53.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571e7fc640 1 --2- 172.21.15.64:0/4237091668 >> v2:172.21.15.66:3300/0 conn(0x7f57200b2d30 0x7f57200ce770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:53.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571effd640 1 -- 172.21.15.64:0/4237091668 >> v2:172.21.15.66:3300/0 conn(0x7f57200b2d30 msgr2=0x7f57200ce770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:53.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571effd640 1 --2- 172.21.15.64:0/4237091668 >> v2:172.21.15.66:3300/0 conn(0x7f57200b2d30 0x7f57200ce770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571effd640 1 -- 172.21.15.64:0/4237091668 >> v1:172.21.15.64:6789/0 conn(0x7f57200cecb0 legacy=0x7f57200b72e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:53.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571effd640 1 -- 172.21.15.64:0/4237091668 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57200d36c0 con 0x7f5720071540 2024-03-04T07:29:53.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.180+0000 7f571e7fc640 1 --2- 172.21.15.64:0/4237091668 >> v2:172.21.15.66:3300/0 conn(0x7f57200b2d30 0x7f57200ce770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:53.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.181+0000 7f571effd640 1 --2- 172.21.15.64:0/4237091668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5720071540 0x7f5720071920 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f57200c7690 tx=0x7f571802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4295c29461dd45ad server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:53.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.181+0000 7f571dffb640 1 -- 172.21.15.64:0/4237091668 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5718035020 con 0x7f5720071540 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.181+0000 7f571dffb640 1 -- 172.21.15.64:0/4237091668 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5718031830 con 0x7f5720071540 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.181+0000 7f571dffb640 1 -- 172.21.15.64:0/4237091668 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5718031c00 con 0x7f5720071540 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 -- 172.21.15.64:0/4237091668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5720071540 msgr2=0x7f5720071920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 --2- 172.21.15.64:0/4237091668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5720071540 0x7f5720071920 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f57200c7690 tx=0x7f571802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 -- 172.21.15.64:0/4237091668 shutdown_connections 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 --2- 172.21.15.64:0/4237091668 >> v2:172.21.15.66:3300/0 conn(0x7f57200b2d30 0x7f57200ce770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 --2- 172.21.15.64:0/4237091668 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5720071540 0x7f5720071920 unknown :-1 s=CLOSED pgs=777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 -- 172.21.15.64:0/4237091668 >> 172.21.15.64:0/4237091668 conn(0x7f57200d3d60 msgr2=0x7f57200d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 -- 172.21.15.64:0/4237091668 shutdown_connections 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.182+0000 7f5725779640 1 -- 172.21.15.64:0/4237091668 wait complete. 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.183+0000 7f5725779640 1 Processor -- start 2024-03-04T07:29:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.183+0000 7f5725779640 1 -- start start 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f5725779640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5720071540 0x7f57201f5240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f5725779640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57200b2d30 0x7f572015a4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f5725779640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57200cecb0 0x7f57200b46d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f5725779640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57200d2be0 con 0x7f57200b2d30 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f5725779640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57200d2900 con 0x7f57200cecb0 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f5725779640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f57200d2ec0 con 0x7f5720071540 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57200b2d30 0x7f572015a4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57200b2d30 0x7f572015a4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40702/0 (socket says 172.21.15.64:40702) 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 -- 172.21.15.64:0/4240101294 learned_addr learned my addr 172.21.15.64:0/4240101294 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:53.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571f7fe640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57200cecb0 0x7f57200b46d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571effd640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5720071540 0x7f57201f5240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 -- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5720071540 msgr2=0x7f57201f5240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5720071540 0x7f57201f5240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 -- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57200cecb0 msgr2=0x7f57200b46d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57200cecb0 0x7f57200b46d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 -- 172.21.15.64:0/4240101294 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57200adb10 con 0x7f57200b2d30 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571effd640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5720071540 0x7f57201f5240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.184+0000 7f571e7fc640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57200b2d30 0x7f572015a4d0 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f570800c7f0 tx=0x7f570800ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.185+0000 7f57177fe640 1 -- 172.21.15.64:0/4240101294 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5708004230 con 0x7f57200b2d30 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.185+0000 7f57177fe640 1 -- 172.21.15.64:0/4240101294 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f57080043d0 con 0x7f57200b2d30 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.185+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f57200aaa80 con 0x7f57200b2d30 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.185+0000 7f57177fe640 1 -- 172.21.15.64:0/4240101294 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5708007a70 con 0x7f57200b2d30 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.185+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f57200ab040 con 0x7f57200b2d30 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.186+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5720072070 con 0x7f57200b2d30 2024-03-04T07:29:53.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.186+0000 7f57177fe640 1 -- 172.21.15.64:0/4240101294 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5708007c10 con 0x7f57200b2d30 2024-03-04T07:29:53.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.186+0000 7f57177fe640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570006b4d0 0x7f570006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:53.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.187+0000 7f571effd640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570006b4d0 0x7f570006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:53.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.187+0000 7f57177fe640 1 -- 172.21.15.64:0/4240101294 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(56..56 src has 1..56) v4 ==== 5795+0+0 (secure 0 0 0) 0x7f570808a370 con 0x7f57200b2d30 2024-03-04T07:29:53.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.187+0000 7f571effd640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570006b4d0 0x7f570006d990 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f5718036000 tx=0x7f5718008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:53.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.191+0000 7f57177fe640 1 -- 172.21.15.64:0/4240101294 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5708015020 con 0x7f57200b2d30 2024-03-04T07:29:53.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:53.371+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f572004f410 con 0x7f57200b2d30 2024-03-04T07:29:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.014+0000 7f57177fe640 1 -- 172.21.15.64:0/4240101294 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v57) v1 ==== 181+0+0 (secure 0 0 0) 0x7f5708059190 con 0x7f57200b2d30 2024-03-04T07:29:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570006b4d0 msgr2=0x7f570006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570006b4d0 0x7f570006d990 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f5718036000 tx=0x7f5718008030 comp rx=0 tx=0).stop 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57200b2d30 msgr2=0x7f572015a4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57200b2d30 0x7f572015a4d0 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f570800c7f0 tx=0x7f570800ccb0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 shutdown_connections 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570006b4d0 0x7f570006d990 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57200cecb0 0x7f57200b46d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57200b2d30 0x7f572015a4d0 unknown :-1 s=CLOSED pgs=778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 --2- 172.21.15.64:0/4240101294 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5720071540 0x7f57201f5240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 >> 172.21.15.64:0/4240101294 conn(0x7f57200d3d60 msgr2=0x7f572015d2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 shutdown_connections 2024-03-04T07:29:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.018+0000 7f5725779640 1 -- 172.21.15.64:0/4240101294 wait complete. 2024-03-04T07:29:54.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:29:54.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.138+0000 7fb2061a0640 1 Processor -- start 2024-03-04T07:29:54.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.138+0000 7fb2061a0640 1 -- start start 2024-03-04T07:29:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb2061a0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb200161280 0x7fb2000d3340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb2061a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 0x7fb2000d0ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb2061a0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb2000d17a0 con 0x7fb2000d0ff0 2024-03-04T07:29:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb2061a0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb2000d3920 con 0x7fb200161280 2024-03-04T07:29:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb2061a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb2000d1400 con 0x7fb2000b2820 2024-03-04T07:29:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb20599f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb2000d0ff0 0x7fb200161a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56822/0 (socket says 172.21.15.64:56822) 2024-03-04T07:29:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb20599f640 1 -- 172.21.15.64:0/194871504 learned_addr learned my addr 172.21.15.64:0/194871504 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 915540425 0 0) 0x7fb2000d17a0 con 0x7fb2000d0ff0 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb1e8003520 con 0x7fb2000d0ff0 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb20519e640 1 --2- 172.21.15.64:0/194871504 >> v2:172.21.15.66:3300/0 conn(0x7fb200161280 0x7fb2000d3340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.139+0000 7fb20499d640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 0x7fb2000d0ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1532184383 0 0) 0x7fb1e8003520 con 0x7fb2000d0ff0 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 msgr2=0x7fb2000d0ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 0x7fb2000d0ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 >> v2:172.21.15.66:3300/0 conn(0x7fb200161280 msgr2=0x7fb2000d3340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 --2- 172.21.15.64:0/194871504 >> v2:172.21.15.66:3300/0 conn(0x7fb200161280 0x7fb2000d3340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2000cec30 con 0x7fb2000d0ff0 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fb1f0001a10 con 0x7fb2000d0ff0 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20499d640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 0x7fb2000d0ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 >> v1:172.21.15.64:6789/0 conn(0x7fb2000d0ff0 legacy=0x7fb200161a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:54.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 0x7fb1e80039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb1e8003ef0 0x7fb1e8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb1e8004d40 0x7fb1e80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2000d3920 con 0x7fb1e8004d40 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2000d1400 con 0x7fb1e8003ef0 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb1e800bb20 con 0x7fb2000b2820 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20599f640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb1e8004d40 0x7fb1e80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20519e640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 0x7fb1e80039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20499d640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb1e8003ef0 0x7fb1e8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20599f640 1 -- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 msgr2=0x7fb1e80039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20599f640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000b2820 0x7fb1e80039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20599f640 1 -- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb1e8003ef0 msgr2=0x7fb1e8004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20599f640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb1e8003ef0 0x7fb1e8004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20599f640 1 -- 172.21.15.64:0/194871504 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb1e8003520 con 0x7fb1e8004d40 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.140+0000 7fb20499d640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb1e8003ef0 0x7fb1e8004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb20599f640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb1e8004d40 0x7fb1e80051c0 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7fb1f000a890 tx=0x7fb1e8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b0db84bf74652075 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb1f000a4e0 con 0x7fb1e8004d40 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb1f0034630 con 0x7fb1e8004d40 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb1fffff640 1 -- 172.21.15.64:0/194871504 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb1f0034a20 con 0x7fb1e8004d40 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb2061a0640 1 -- 172.21.15.64:0/194871504 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb1e8004d40 msgr2=0x7fb1e80051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb2061a0640 1 --2- 172.21.15.64:0/194871504 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb1e8004d40 0x7fb1e80051c0 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7fb1f000a890 tx=0x7fb1e8005830 comp rx=0 tx=0).stop 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb20599f640 1 -- 172.21.15.64:0/194871504 reap_dead start 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb2061a0640 1 -- 172.21.15.64:0/194871504 shutdown_connections 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.141+0000 7fb2061a0640 1 -- 172.21.15.64:0/194871504 >> 172.21.15.64:0/194871504 conn(0x7fb200067120 msgr2=0x7fb200067530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.142+0000 7fb2061a0640 1 -- 172.21.15.64:0/194871504 shutdown_connections 2024-03-04T07:29:54.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.142+0000 7fb2061a0640 1 -- 172.21.15.64:0/194871504 wait complete. 2024-03-04T07:29:54.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 Processor -- start 2024-03-04T07:29:54.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 -- start start 2024-03-04T07:29:54.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000c9d20 0x7fb20015a6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb20015abe0 0x7fb20015c510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb20015ca50 0x7fb2001f6fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2000cf370 con 0x7fb20015ca50 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2000cf1f0 con 0x7fb20015abe0 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.143+0000 7fb2061a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb2000b2e90 con 0x7fb2000c9d20 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb20015ca50 0x7fb2001f6fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb20015ca50 0x7fb2001f6fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43144/0 (socket says 172.21.15.64:43144) 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 -- 172.21.15.64:0/2170026833 learned_addr learned my addr 172.21.15.64:0/2170026833 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20519e640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000c9d20 0x7fb20015a6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20499d640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb20015abe0 0x7fb20015c510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 -- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000c9d20 msgr2=0x7fb20015a6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000c9d20 0x7fb20015a6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 -- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb20015abe0 msgr2=0x7fb20015c510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb20015abe0 0x7fb20015c510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:54.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 -- 172.21.15.64:0/2170026833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb200156780 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20519e640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000c9d20 0x7fb20015a6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb20599f640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb20015ca50 0x7fb2001f6fb0 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7fb1f000a9b0 tx=0x7fb1f0001de0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb1fdffb640 1 -- 172.21.15.64:0/2170026833 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb1f000a4e0 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb1fdffb640 1 -- 172.21.15.64:0/2170026833 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb1f0043070 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb20015ced0 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb1fdffb640 1 -- 172.21.15.64:0/2170026833 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb1f0002030 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.144+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb20015d460 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.145+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb20004f330 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.146+0000 7fb1fdffb640 1 -- 172.21.15.64:0/2170026833 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb1f00349f0 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.146+0000 7fb1fdffb640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb1ec06b500 0x7fb1ec06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.146+0000 7fb1fdffb640 1 -- 172.21.15.64:0/2170026833 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(57..57 src has 1..57) v4 ==== 5357+0+0 (secure 0 0 0) 0x7fb1f00abff0 con 0x7fb20015ca50 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.146+0000 7fb20519e640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb1ec06b500 0x7fb1ec06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:54.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.147+0000 7fb20519e640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb1ec06b500 0x7fb1ec06d9c0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7fb1f8009b30 tx=0x7fb1f8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:54.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.151+0000 7fb1fdffb640 1 -- 172.21.15.64:0/2170026833 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb1f007b180 con 0x7fb20015ca50 2024-03-04T07:29:54.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:54.339+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fb20015afc0 con 0x7fb20015ca50 2024-03-04T07:29:55.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.018+0000 7fb1fdffb640 1 -- 172.21.15.64:0/2170026833 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v58) v1 ==== 169+0+0 (secure 0 0 0) 0x7fb1f007aa80 con 0x7fb20015ca50 2024-03-04T07:29:55.018 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:29:55.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.020+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb1ec06b500 msgr2=0x7fb1ec06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb1ec06b500 0x7fb1ec06d9c0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7fb1f8009b30 tx=0x7fb1f8008040 comp rx=0 tx=0).stop 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb20015ca50 msgr2=0x7fb2001f6fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb20015ca50 0x7fb2001f6fb0 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7fb1f000a9b0 tx=0x7fb1f0001de0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 shutdown_connections 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb1ec06b500 0x7fb1ec06d9c0 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb20015ca50 0x7fb2001f6fb0 unknown :-1 s=CLOSED pgs=781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb20015abe0 0x7fb20015c510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 --2- 172.21.15.64:0/2170026833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2000c9d20 0x7fb20015a6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 >> 172.21.15.64:0/2170026833 conn(0x7fb200067120 msgr2=0x7fb2000b6e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 shutdown_connections 2024-03-04T07:29:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.021+0000 7fb2061a0640 1 -- 172.21.15.64:0/2170026833 wait complete. 2024-03-04T07:29:55.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:29:55.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.154+0000 7fd2021ac640 1 Processor -- start 2024-03-04T07:29:55.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.154+0000 7fd2021ac640 1 -- start start 2024-03-04T07:29:55.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd2021ac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd1fc0d3770 0x7fd1fc072680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd2021ac640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1fc072cf0 con 0x7fd1fc0cf0f0 2024-03-04T07:29:55.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd2021ac640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1fc072fd0 con 0x7fd1fc15dd50 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd2021ac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd1fc0d1f60 con 0x7fd1fc0d3770 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd1faffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd1fc0cf0f0 0x7fd1fc071f70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56836/0 (socket says 172.21.15.64:56836) 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd1faffd640 1 -- 172.21.15.64:0/2583772394 learned_addr learned my addr 172.21.15.64:0/2583772394 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd1fbfff640 1 --2- 172.21.15.64:0/2583772394 >> v2:172.21.15.66:3300/0 conn(0x7fd1fc0d3770 0x7fd1fc072680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3607211744 0 0) 0x7fd1fc072cf0 con 0x7fd1fc0cf0f0 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.155+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd1e4003520 con 0x7fd1fc0cf0f0 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4218497953 0 0) 0x7fd1e4003520 con 0x7fd1fc0cf0f0 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 >> v2:172.21.15.66:3300/0 conn(0x7fd1fc0d3770 msgr2=0x7fd1fc072680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 --2- 172.21.15.64:0/2583772394 >> v2:172.21.15.66:3300/0 conn(0x7fd1fc0d3770 0x7fd1fc072680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 >> v1:172.21.15.66:6789/0 conn(0x7fd1fc15dd50 legacy=0x7fd1fc0ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1fc0d2240 con 0x7fd1fc0cf0f0 2024-03-04T07:29:55.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd1f0001670 con 0x7fd1fc0cf0f0 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 >> v1:172.21.15.64:6789/0 conn(0x7fd1fc0cf0f0 legacy=0x7fd1fc071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40039b0 0x7fd1e4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1e4004800 0x7fd1e4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1e40052e0 0x7fd1e400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1fc0d1f60 con 0x7fd1e40052e0 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1e400c1b0 con 0x7fd1e40039b0 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1e400c3b0 con 0x7fd1e4004800 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fbfff640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1e40052e0 0x7fd1e400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fb7fe640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40039b0 0x7fd1e4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1faffd640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1e4004800 0x7fd1e4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fbfff640 1 -- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1e4004800 msgr2=0x7fd1e4004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fbfff640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1e4004800 0x7fd1e4004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fbfff640 1 -- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40039b0 msgr2=0x7fd1e4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fbfff640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40039b0 0x7fd1e4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.156+0000 7fd1fbfff640 1 -- 172.21.15.64:0/2583772394 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1e4003520 con 0x7fd1e40052e0 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.157+0000 7fd1fbfff640 1 -- 172.21.15.64:0/2583772394 reap_dead start 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.157+0000 7fd1fbfff640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1e40052e0 0x7fd1e400b9c0 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fd1f4009f90 tx=0x7fd1e4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=559fe3d391ce7410 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.157+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1f4009c10 con 0x7fd1e40052e0 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.157+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd1f402cc20 con 0x7fd1e40052e0 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.157+0000 7fd1fa7fc640 1 -- 172.21.15.64:0/2583772394 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1f4003990 con 0x7fd1e40052e0 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.158+0000 7fd2021ac640 1 -- 172.21.15.64:0/2583772394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1e40052e0 msgr2=0x7fd1e400b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.158+0000 7fd2021ac640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1e40052e0 0x7fd1e400b9c0 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fd1f4009f90 tx=0x7fd1e4005830 comp rx=0 tx=0).stop 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.158+0000 7fd2021ac640 1 -- 172.21.15.64:0/2583772394 shutdown_connections 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.158+0000 7fd2021ac640 1 --2- 172.21.15.64:0/2583772394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1e40052e0 0x7fd1e400b9c0 unknown :-1 s=CLOSED pgs=783 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.158+0000 7fd2021ac640 1 -- 172.21.15.64:0/2583772394 >> 172.21.15.64:0/2583772394 conn(0x7fd1fc0d3ef0 msgr2=0x7fd1fc0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.158+0000 7fd2021ac640 1 -- 172.21.15.64:0/2583772394 shutdown_connections 2024-03-04T07:29:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.158+0000 7fd2021ac640 1 -- 172.21.15.64:0/2583772394 wait complete. 2024-03-04T07:29:55.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.159+0000 7fd2021ac640 1 Processor -- start 2024-03-04T07:29:55.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.159+0000 7fd2021ac640 1 -- start start 2024-03-04T07:29:55.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd2021ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40052e0 0x7fd1fc0c9920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd2021ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1fc1e74b0 0x7fd1fc1f4050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd2021ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1fc1f4590 0x7fd1fc1f7620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd2021ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1fc0ce9e0 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd2021ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1fc15e340 con 0x7fd1e40052e0 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd2021ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1fc0ceb60 con 0x7fd1fc1f4590 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1fc1e74b0 0x7fd1fc1f4050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1fc1e74b0 0x7fd1fc1f4050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43160/0 (socket says 172.21.15.64:43160) 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 -- 172.21.15.64:0/1435652575 learned_addr learned my addr 172.21.15.64:0/1435652575 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1fb7fe640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40052e0 0x7fd1fc0c9920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 -- 172.21.15.64:0/1435652575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1fc1f4590 msgr2=0x7fd1fc1f7620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1fc1f4590 0x7fd1fc1f7620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 -- 172.21.15.64:0/1435652575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40052e0 msgr2=0x7fd1fc0c9920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40052e0 0x7fd1fc0c9920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 -- 172.21.15.64:0/1435652575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1fc1f4a10 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1faffd640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1fc1e74b0 0x7fd1fc1f4050 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7fd1f0004f70 tx=0x7fd1f00074f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.160+0000 7fd1e3fff640 1 -- 172.21.15.64:0/1435652575 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1f001a070 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.161+0000 7fd1e3fff640 1 -- 172.21.15.64:0/1435652575 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd1f0007740 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.161+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd1fc1e8850 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.161+0000 7fd1e3fff640 1 -- 172.21.15.64:0/1435652575 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1f000fba0 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.161+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd1fc1e8e10 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.162+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd1fc06cdd0 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.162+0000 7fd1e3fff640 1 -- 172.21.15.64:0/1435652575 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd1f001d070 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.162+0000 7fd1e3fff640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd1e806b4d0 0x7fd1e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.163+0000 7fd1e3fff640 1 -- 172.21.15.64:0/1435652575 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fd1f0089760 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.163+0000 7fd1fb7fe640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd1e806b4d0 0x7fd1e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.163+0000 7fd1fb7fe640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd1e806b4d0 0x7fd1e806d990 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7fd1ec000c00 tx=0x7fd1ec009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.167+0000 7fd1e3fff640 1 -- 172.21.15.64:0/1435652575 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd1fc1f4a10 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.348+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7fd1fc04f410 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.351+0000 7fd1e3fff640 1 -- 172.21.15.64:0/1435652575 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v19) v1 ==== 104+0+0 (secure 0 0 0) 0x7fd1f00588e0 con 0x7fd1fc1e74b0 2024-03-04T07:29:55.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd1e806b4d0 msgr2=0x7fd1e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd1e806b4d0 0x7fd1e806d990 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7fd1ec000c00 tx=0x7fd1ec009040 comp rx=0 tx=0).stop 2024-03-04T07:29:55.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1fc1e74b0 msgr2=0x7fd1fc1f4050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1fc1e74b0 0x7fd1fc1f4050 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7fd1f0004f70 tx=0x7fd1f00074f0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 shutdown_connections 2024-03-04T07:29:55.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd1e806b4d0 0x7fd1e806d990 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1fc1f4590 0x7fd1fc1f7620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1fc1e74b0 0x7fd1fc1f4050 unknown :-1 s=CLOSED pgs=784 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 --2- 172.21.15.64:0/1435652575 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1e40052e0 0x7fd1fc0c9920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 >> 172.21.15.64:0/1435652575 conn(0x7fd1fc0d3ef0 msgr2=0x7fd1fc0d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 shutdown_connections 2024-03-04T07:29:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.353+0000 7fd2021ac640 1 -- 172.21.15.64:0/1435652575 wait complete. 2024-03-04T07:29:55.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_merge_size 2024-03-04T07:29:55.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.467+0000 7f9908fc9640 1 Processor -- start 2024-03-04T07:29:55.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.467+0000 7f9908fc9640 1 -- start start 2024-03-04T07:29:55.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f9908fc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040717b0 0x7f9904071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f9908fc9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f99040b3060 0x7f99040d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f9908fc9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f990415e4b0 con 0x7f99040b2b10 2024-03-04T07:29:55.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f9908fc9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f99040d2e10 con 0x7f99040b3060 2024-03-04T07:29:55.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f9908fc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99040b97e0 con 0x7f99040717b0 2024-03-04T07:29:55.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f990259c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040717b0 0x7f9904071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f9901d9b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f99040b3060 0x7f99040d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f990259c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040717b0 0x7f9904071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:43930/0 (socket says 172.21.15.64:43930) 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.468+0000 7f990259c640 1 -- 172.21.15.64:0/114294764 learned_addr learned my addr 172.21.15.64:0/114294764 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.469+0000 7f990259c640 1 -- 172.21.15.64:0/114294764 >> v2:172.21.15.66:3300/0 conn(0x7f99040b3060 msgr2=0x7f99040d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.469+0000 7f990259c640 1 --2- 172.21.15.64:0/114294764 >> v2:172.21.15.66:3300/0 conn(0x7f99040b3060 0x7f99040d32a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.469+0000 7f990159a640 1 -- 172.21.15.64:0/114294764 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4064106741 0 0) 0x7f98f4001490 con 0x7f99040b2b10 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.469+0000 7f990259c640 1 -- 172.21.15.64:0/114294764 >> v1:172.21.15.66:6789/0 conn(0x7f99040b2b10 legacy=0x7f99040b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.469+0000 7f990259c640 1 -- 172.21.15.64:0/114294764 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99040b9a80 con 0x7f99040717b0 2024-03-04T07:29:55.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f990259c640 1 --2- 172.21.15.64:0/114294764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040717b0 0x7f9904071b90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f98fc009870 tx=0x7f98fc02b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2eb4f91a9bc9445e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f990159a640 1 -- 172.21.15.64:0/114294764 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98fc034070 con 0x7f99040717b0 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f990159a640 1 -- 172.21.15.64:0/114294764 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f98fc030840 con 0x7f99040717b0 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f990159a640 1 -- 172.21.15.64:0/114294764 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98fc030c10 con 0x7f99040717b0 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 -- 172.21.15.64:0/114294764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040717b0 msgr2=0x7f9904071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 --2- 172.21.15.64:0/114294764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040717b0 0x7f9904071b90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f98fc009870 tx=0x7f98fc02b980 comp rx=0 tx=0).stop 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 -- 172.21.15.64:0/114294764 shutdown_connections 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 --2- 172.21.15.64:0/114294764 >> v2:172.21.15.66:3300/0 conn(0x7f99040b3060 0x7f99040d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 --2- 172.21.15.64:0/114294764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040717b0 0x7f9904071b90 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 -- 172.21.15.64:0/114294764 >> 172.21.15.64:0/114294764 conn(0x7f99040d3d20 msgr2=0x7f99040d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 -- 172.21.15.64:0/114294764 shutdown_connections 2024-03-04T07:29:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.470+0000 7f9908fc9640 1 -- 172.21.15.64:0/114294764 wait complete. 2024-03-04T07:29:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 Processor -- start 2024-03-04T07:29:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 -- start start 2024-03-04T07:29:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99040717b0 0x7f99041e33a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99040b2b10 0x7f990415ac00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040b3060 0x7f990414d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99040b7560 con 0x7f99040717b0 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99040d4bf0 con 0x7f99040b2b10 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.472+0000 7f9908fc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99040b76e0 con 0x7f99040b3060 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99040717b0 0x7f99041e33a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99040717b0 0x7f99041e33a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43162/0 (socket says 172.21.15.64:43162) 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 -- 172.21.15.64:0/3148671974 learned_addr learned my addr 172.21.15.64:0/3148671974 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f9901d9b640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99040b2b10 0x7f990415ac00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f9902d9d640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040b3060 0x7f990414d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 -- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040b3060 msgr2=0x7f990414d9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040b3060 0x7f990414d9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 -- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99040b2b10 msgr2=0x7f990415ac00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99040b2b10 0x7f990415ac00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 -- 172.21.15.64:0/3148671974 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9904159eb0 con 0x7f99040717b0 2024-03-04T07:29:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f990259c640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99040717b0 0x7f99041e33a0 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7f98fc000c00 tx=0x7f98fc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f98faffd640 1 -- 172.21.15.64:0/3148671974 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98fc034040 con 0x7f99040717b0 2024-03-04T07:29:55.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f98faffd640 1 -- 172.21.15.64:0/3148671974 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f98fc030640 con 0x7f99040717b0 2024-03-04T07:29:55.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f98faffd640 1 -- 172.21.15.64:0/3148671974 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98fc039d40 con 0x7f99040717b0 2024-03-04T07:29:55.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f990414df00 con 0x7f99040717b0 2024-03-04T07:29:55.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.473+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f990415a400 con 0x7f99040717b0 2024-03-04T07:29:55.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.474+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f990406ce40 con 0x7f99040717b0 2024-03-04T07:29:55.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.475+0000 7f98faffd640 1 -- 172.21.15.64:0/3148671974 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f98fc04b060 con 0x7f99040717b0 2024-03-04T07:29:55.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.475+0000 7f98faffd640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98e406b500 0x7f98e406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.475+0000 7f98faffd640 1 -- 172.21.15.64:0/3148671974 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f98fc0aba70 con 0x7f99040717b0 2024-03-04T07:29:55.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.475+0000 7f9901d9b640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98e406b500 0x7f98e406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.476+0000 7f9901d9b640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98e406b500 0x7f98e406d9c0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f99041581f0 tx=0x7f98ec008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.480+0000 7f98faffd640 1 -- 172.21.15.64:0/3148671974 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f98fc07ac80 con 0x7f99040717b0 2024-03-04T07:29:55.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.663+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_merge_size"} v 0) v1 -- 0x7f990404f410 con 0x7f99040717b0 2024-03-04T07:29:55.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.666+0000 7f98faffd640 1 -- 172.21.15.64:0/3148671974 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_merge_size"}]=0 v20) v1 ==== 101+0+0 (secure 0 0 0) 0x7f98fc07a580 con 0x7f99040717b0 2024-03-04T07:29:55.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98e406b500 msgr2=0x7f98e406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98e406b500 0x7f98e406d9c0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f99041581f0 tx=0x7f98ec008040 comp rx=0 tx=0).stop 2024-03-04T07:29:55.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99040717b0 msgr2=0x7f99041e33a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99040717b0 0x7f99041e33a0 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7f98fc000c00 tx=0x7f98fc0046c0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 shutdown_connections 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f98e406b500 0x7f98e406d9c0 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99040b3060 0x7f990414d9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99040b2b10 0x7f990415ac00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 --2- 172.21.15.64:0/3148671974 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99040717b0 0x7f99041e33a0 unknown :-1 s=CLOSED pgs=785 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 >> 172.21.15.64:0/3148671974 conn(0x7f99040d3d20 msgr2=0x7f99040b7aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 shutdown_connections 2024-03-04T07:29:55.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.668+0000 7f9908fc9640 1 -- 172.21.15.64:0/3148671974 wait complete. 2024-03-04T07:29:55.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_fragment_size_max 2024-03-04T07:29:55.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.783+0000 7ff951c6f640 1 Processor -- start 2024-03-04T07:29:55.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.783+0000 7ff951c6f640 1 -- start start 2024-03-04T07:29:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff951c6f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff94c0b4830 0x7ff94c160280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff951c6f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff94c070980 con 0x7ff94c142170 2024-03-04T07:29:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff951c6f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff94c1607c0 con 0x7ff94c0b29a0 2024-03-04T07:29:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff951c6f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff94c1458e0 con 0x7ff94c0b4830 2024-03-04T07:29:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff94affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff94c0b4830 0x7ff94c160280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff94b7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff94c142170 0x7ff94c142a20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56852/0 (socket says 172.21.15.64:56852) 2024-03-04T07:29:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff94b7fe640 1 -- 172.21.15.64:0/138971891 learned_addr learned my addr 172.21.15.64:0/138971891 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:55.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff94affd640 1 -- 172.21.15.64:0/138971891 >> v1:172.21.15.66:6789/0 conn(0x7ff94c0b29a0 legacy=0x7ff94c0b2db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:55.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff94affd640 1 -- 172.21.15.64:0/138971891 >> v1:172.21.15.64:6789/0 conn(0x7ff94c142170 legacy=0x7ff94c142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff94a7fc640 1 -- 172.21.15.64:0/138971891 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1913517623 0 0) 0x7ff94c070980 con 0x7ff94c142170 2024-03-04T07:29:55.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.784+0000 7ff94affd640 1 -- 172.21.15.64:0/138971891 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff94c070bf0 con 0x7ff94c0b4830 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.785+0000 7ff94affd640 1 --2- 172.21.15.64:0/138971891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c0b4830 0x7ff94c160280 secure :-1 s=READY pgs=787 cs=0 l=1 rev1=1 crypto rx=0x7ff9400097a0 tx=0x7ff94002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=512bee6c71b5cab9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.785+0000 7ff94a7fc640 1 -- 172.21.15.64:0/138971891 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9400315a0 con 0x7ff94c0b4830 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.785+0000 7ff94a7fc640 1 -- 172.21.15.64:0/138971891 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff940037070 con 0x7ff94c0b4830 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.785+0000 7ff94a7fc640 1 -- 172.21.15.64:0/138971891 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff940031960 con 0x7ff94c0b4830 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.786+0000 7ff951c6f640 1 -- 172.21.15.64:0/138971891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c0b4830 msgr2=0x7ff94c160280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.786+0000 7ff951c6f640 1 --2- 172.21.15.64:0/138971891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c0b4830 0x7ff94c160280 secure :-1 s=READY pgs=787 cs=0 l=1 rev1=1 crypto rx=0x7ff9400097a0 tx=0x7ff94002b840 comp rx=0 tx=0).stop 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.786+0000 7ff951c6f640 1 -- 172.21.15.64:0/138971891 shutdown_connections 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.786+0000 7ff951c6f640 1 --2- 172.21.15.64:0/138971891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c0b4830 0x7ff94c160280 unknown :-1 s=CLOSED pgs=787 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.786+0000 7ff951c6f640 1 -- 172.21.15.64:0/138971891 >> 172.21.15.64:0/138971891 conn(0x7ff94c066d00 msgr2=0x7ff94c067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:55.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.786+0000 7ff951c6f640 1 -- 172.21.15.64:0/138971891 shutdown_connections 2024-03-04T07:29:55.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.786+0000 7ff951c6f640 1 -- 172.21.15.64:0/138971891 wait complete. 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.787+0000 7ff951c6f640 1 Processor -- start 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.787+0000 7ff951c6f640 1 -- start start 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff951c6f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff94c0b29a0 0x7ff94c06c940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff951c6f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff94c0b4830 0x7ff94c15b640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff951c6f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c142170 0x7ff94c0ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff951c6f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff94c0b4df0 con 0x7ff94c142170 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff951c6f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff94c0711b0 con 0x7ff94c0b29a0 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff951c6f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff94c142550 con 0x7ff94c0b4830 2024-03-04T07:29:55.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c142170 0x7ff94c0ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c142170 0x7ff94c0ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43176/0 (socket says 172.21.15.64:43176) 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 -- 172.21.15.64:0/1545120324 learned_addr learned my addr 172.21.15.64:0/1545120324 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94b7fe640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff94c0b29a0 0x7ff94c06c940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94affd640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff94c0b4830 0x7ff94c15b640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 -- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff94c0b4830 msgr2=0x7ff94c15b640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff94c0b4830 0x7ff94c15b640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 -- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff94c0b29a0 msgr2=0x7ff94c06c940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff94c0b29a0 0x7ff94c06c940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 -- 172.21.15.64:0/1545120324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff94c0adf70 con 0x7ff94c142170 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.788+0000 7ff94bfff640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c142170 0x7ff94c0ad8b0 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7ff94400c780 tx=0x7ff94400cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.789+0000 7ff933fff640 1 -- 172.21.15.64:0/1545120324 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff944004b60 con 0x7ff94c142170 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.789+0000 7ff933fff640 1 -- 172.21.15.64:0/1545120324 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff944004d00 con 0x7ff94c142170 2024-03-04T07:29:55.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.789+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff94c0ae250 con 0x7ff94c142170 2024-03-04T07:29:55.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.789+0000 7ff933fff640 1 -- 172.21.15.64:0/1545120324 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff94400aa90 con 0x7ff94c142170 2024-03-04T07:29:55.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.789+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff94c0afe80 con 0x7ff94c142170 2024-03-04T07:29:55.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.790+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff94c0733c0 con 0x7ff94c142170 2024-03-04T07:29:55.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.790+0000 7ff933fff640 1 -- 172.21.15.64:0/1545120324 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff94400ac30 con 0x7ff94c142170 2024-03-04T07:29:55.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.790+0000 7ff933fff640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff93806b4d0 0x7ff93806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:55.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.790+0000 7ff933fff640 1 -- 172.21.15.64:0/1545120324 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7ff94408a150 con 0x7ff94c142170 2024-03-04T07:29:55.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.790+0000 7ff94b7fe640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff93806b4d0 0x7ff93806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:55.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.791+0000 7ff94b7fe640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff93806b4d0 0x7ff93806d990 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7ff93c000f60 tx=0x7ff93c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:55.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.794+0000 7ff933fff640 1 -- 172.21.15.64:0/1545120324 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff944020070 con 0x7ff94c142170 2024-03-04T07:29:55.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.980+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"} v 0) v1 -- 0x7ff94c04f410 con 0x7ff94c142170 2024-03-04T07:29:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.983+0000 7ff933fff640 1 -- 172.21.15.64:0/1545120324 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"}]=0 v21) v1 ==== 108+0+0 (secure 0 0 0) 0x7ff944059360 con 0x7ff94c142170 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff93806b4d0 msgr2=0x7ff93806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff93806b4d0 0x7ff93806d990 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7ff93c000f60 tx=0x7ff93c009040 comp rx=0 tx=0).stop 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c142170 msgr2=0x7ff94c0ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c142170 0x7ff94c0ad8b0 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7ff94400c780 tx=0x7ff94400cc40 comp rx=0 tx=0).stop 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 shutdown_connections 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff93806b4d0 0x7ff93806d990 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff94c142170 0x7ff94c0ad8b0 unknown :-1 s=CLOSED pgs=788 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff94c0b4830 0x7ff94c15b640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 --2- 172.21.15.64:0/1545120324 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff94c0b29a0 0x7ff94c06c940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 >> 172.21.15.64:0/1545120324 conn(0x7ff94c066d00 msgr2=0x7ff94c15fe50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 shutdown_connections 2024-03-04T07:29:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:55.985+0000 7ff951c6f640 1 -- 172.21.15.64:0/1545120324 wait complete. 2024-03-04T07:29:55.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented' 2024-03-04T07:29:56.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.099+0000 7f1adf37a640 1 Processor -- start 2024-03-04T07:29:56.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.099+0000 7f1adf37a640 1 -- start start 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1adf37a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1ad815e060 0x7f1ad80ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1adf37a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ad80d0dd0 con 0x7f1ad806fe20 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1adf37a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ad80d10b0 con 0x7f1ad80d0880 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1adf37a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1ad80cc0e0 con 0x7f1ad815e060 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1adc915640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1ad806fe20 0x7f1ad80ce850 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56858/0 (socket says 172.21.15.64:56858) 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1adc915640 1 -- 172.21.15.64:0/1919200711 learned_addr learned my addr 172.21.15.64:0/1919200711 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1add116640 1 --2- 172.21.15.64:0/1919200711 >> v2:172.21.15.66:3300/0 conn(0x7f1ad815e060 0x7f1ad80ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2338901522 0 0) 0x7f1ad80d0dd0 con 0x7f1ad806fe20 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1ac4003520 con 0x7f1ad806fe20 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3982245166 0 0) 0x7f1ac4003520 con 0x7f1ad806fe20 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 >> v2:172.21.15.66:3300/0 conn(0x7f1ad815e060 msgr2=0x7f1ad80ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 --2- 172.21.15.64:0/1919200711 >> v2:172.21.15.66:3300/0 conn(0x7f1ad815e060 0x7f1ad80ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 >> v1:172.21.15.66:6789/0 conn(0x7f1ad80d0880 legacy=0x7f1ad80ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ad80cc3c0 con 0x7f1ad806fe20 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f1ac8001670 con 0x7f1ad806fe20 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 >> v1:172.21.15.64:6789/0 conn(0x7f1ad806fe20 legacy=0x7f1ad80ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac40039b0 0x7f1ac4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ac4004800 0x7f1ac4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ac40051c0 0x7f1ac400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ad80cc0e0 con 0x7f1ac4004800 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ac400c060 con 0x7f1ac40051c0 2024-03-04T07:29:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ac400c260 con 0x7f1ac40039b0 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.100+0000 7f1adc915640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ac4004800 0x7f1ac4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1add917640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ac40051c0 0x7f1ac400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1add116640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac40039b0 0x7f1ac4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1adc915640 1 -- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac40039b0 msgr2=0x7f1ac4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1adc915640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac40039b0 0x7f1ac4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1adc915640 1 -- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ac40051c0 msgr2=0x7f1ac400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1adc915640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ac40051c0 0x7f1ac400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1add917640 1 -- 172.21.15.64:0/1919200711 reap_dead start 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1adc915640 1 -- 172.21.15.64:0/1919200711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ac4003520 con 0x7f1ac4004800 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.101+0000 7f1adc915640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ac4004800 0x7f1ac4004c80 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7f1ac800c210 tx=0x7f1ac4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7a42d09f0813709d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ac8007d10 con 0x7f1ac4004800 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ac8030070 con 0x7f1ac4004800 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1acffff640 1 -- 172.21.15.64:0/1919200711 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ac8005d70 con 0x7f1ac4004800 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1adf37a640 1 -- 172.21.15.64:0/1919200711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ac4004800 msgr2=0x7f1ac4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1adf37a640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ac4004800 0x7f1ac4004c80 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7f1ac800c210 tx=0x7f1ac4005830 comp rx=0 tx=0).stop 2024-03-04T07:29:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1adf37a640 1 -- 172.21.15.64:0/1919200711 shutdown_connections 2024-03-04T07:29:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1adf37a640 1 --2- 172.21.15.64:0/1919200711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ac4004800 0x7f1ac4004c80 unknown :-1 s=CLOSED pgs=790 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1adf37a640 1 -- 172.21.15.64:0/1919200711 >> 172.21.15.64:0/1919200711 conn(0x7f1ad80d1a90 msgr2=0x7f1ad80d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1adf37a640 1 -- 172.21.15.64:0/1919200711 shutdown_connections 2024-03-04T07:29:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.102+0000 7f1adf37a640 1 -- 172.21.15.64:0/1919200711 wait complete. 2024-03-04T07:29:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.103+0000 7f1adf37a640 1 Processor -- start 2024-03-04T07:29:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.103+0000 7f1adf37a640 1 -- start start 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adf37a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac4004800 0x7f1ad80cb5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adf37a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ad80b52c0 0x7f1ad81f3050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adf37a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ad81f6dd0 0x7f1ad81f7250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adf37a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ad80ce310 con 0x7f1ad80b52c0 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adf37a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ad815e440 con 0x7f1ad81f6dd0 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adf37a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ad80ce490 con 0x7f1ac4004800 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adc915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ad80b52c0 0x7f1ad81f3050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adc915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ad80b52c0 0x7f1ad81f3050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43192/0 (socket says 172.21.15.64:43192) 2024-03-04T07:29:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.104+0000 7f1adc915640 1 -- 172.21.15.64:0/2321174717 learned_addr learned my addr 172.21.15.64:0/2321174717 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1add116640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac4004800 0x7f1ad80cb5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1add917640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ad81f6dd0 0x7f1ad81f7250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adc915640 1 -- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac4004800 msgr2=0x7f1ad80cb5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adc915640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac4004800 0x7f1ad80cb5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adc915640 1 -- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ad81f6dd0 msgr2=0x7f1ad81f7250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adc915640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ad81f6dd0 0x7f1ad81f7250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adc915640 1 -- 172.21.15.64:0/2321174717 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ad80b5840 con 0x7f1ad80b52c0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adc915640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ad80b52c0 0x7f1ad81f3050 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7f1ac8001970 tx=0x7f1ac802e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1acdffb640 1 -- 172.21.15.64:0/2321174717 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ac8007d10 con 0x7f1ad80b52c0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1acdffb640 1 -- 172.21.15.64:0/2321174717 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ac8004bf0 con 0x7f1ad80b52c0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ad80b5b20 con 0x7f1ad80b52c0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1acdffb640 1 -- 172.21.15.64:0/2321174717 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ac8007070 con 0x7f1ad80b52c0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.105+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ad81f3590 con 0x7f1ad80b52c0 2024-03-04T07:29:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.106+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1ad8067ff0 con 0x7f1ad80b52c0 2024-03-04T07:29:56.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.107+0000 7f1acdffb640 1 -- 172.21.15.64:0/2321174717 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1ac8048020 con 0x7f1ad80b52c0 2024-03-04T07:29:56.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.107+0000 7f1acdffb640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1abc06b4d0 0x7f1abc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:56.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.107+0000 7f1acdffb640 1 -- 172.21.15.64:0/2321174717 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f1ac80ab160 con 0x7f1ad80b52c0 2024-03-04T07:29:56.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.107+0000 7f1add116640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1abc06b4d0 0x7f1abc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:56.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.108+0000 7f1add116640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1abc06b4d0 0x7f1abc06d990 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f1ad4009c40 tx=0x7f1ad4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:56.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.111+0000 7f1acdffb640 1 -- 172.21.15.64:0/2321174717 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1ac807a490 con 0x7f1ad80b52c0 2024-03-04T07:29:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:56.367+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]} v 0) v1 -- 0x7f1ad80b5cf0 con 0x7f1ad80b52c0 2024-03-04T07:29:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.024+0000 7f1acdffb640 1 -- 172.21.15.64:0/2321174717 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_fragmented"]}]=0 v177) v1 ==== 144+0+0 (secure 0 0 0) 0x7f1ac8079d90 con 0x7f1ad80b52c0 2024-03-04T07:29:57.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.026+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1abc06b4d0 msgr2=0x7f1abc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.026+0000 7f1adf37a640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1abc06b4d0 0x7f1abc06d990 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f1ad4009c40 tx=0x7f1ad4009190 comp rx=0 tx=0).stop 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.026+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ad80b52c0 msgr2=0x7f1ad81f3050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.026+0000 7f1adf37a640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ad80b52c0 0x7f1ad81f3050 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7f1ac8001970 tx=0x7f1ac802e070 comp rx=0 tx=0).stop 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.026+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 shutdown_connections 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.026+0000 7f1adf37a640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1abc06b4d0 0x7f1abc06d990 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.026+0000 7f1adf37a640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ad81f6dd0 0x7f1ad81f7250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.027+0000 7f1adf37a640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ad80b52c0 0x7f1ad81f3050 unknown :-1 s=CLOSED pgs=791 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.027+0000 7f1adf37a640 1 --2- 172.21.15.64:0/2321174717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ac4004800 0x7f1ad80cb5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.027+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 >> 172.21.15.64:0/2321174717 conn(0x7f1ad80d1a90 msgr2=0x7f1ad806eb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.027+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 shutdown_connections 2024-03-04T07:29:57.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.027+0000 7f1adf37a640 1 -- 172.21.15.64:0/2321174717 wait complete. 2024-03-04T07:29:57.040 INFO:tasks.cephfs_test_runner:That fragmentation (forced) will allow more entries to be created. ... ok 2024-03-04T07:29:57.040 INFO:tasks.cephfs_test_runner:Starting test: test_dirfrag_limit_strays (tasks.cephfs.test_strays.TestStrays) That unlinking fails when the stray directory fragment becomes too 2024-03-04T07:29:57.040 INFO:tasks.cephfs_test_runner:test_dirfrag_limit_strays (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:29:57.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:29:57.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.149+0000 7f0bd67f6640 1 Processor -- start 2024-03-04T07:29:57.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.149+0000 7f0bd67f6640 1 -- start start 2024-03-04T07:29:57.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.149+0000 7f0bd67f6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0bd015bde0 0x7f0bd00b4cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.149+0000 7f0bd67f6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0bd00b2460 0x7f0bd015c1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.149+0000 7f0bd67f6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0bd00b3c70 con 0x7f0bd01451a0 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.149+0000 7f0bd67f6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0bd00b4840 con 0x7f0bd00b2460 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.149+0000 7f0bd67f6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0bd00afad0 con 0x7f0bd015bde0 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bd4d93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0bd00b2460 0x7f0bd015c1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bd4d93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0bd00b2460 0x7f0bd015c1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43208/0 (socket says 172.21.15.64:43208) 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bd4d93640 1 -- 172.21.15.64:0/1831047693 learned_addr learned my addr 172.21.15.64:0/1831047693 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bcf7fe640 1 --1- 172.21.15.64:0/1831047693 >> v1:172.21.15.64:6789/0 conn(0x7f0bd01451a0 0x7f0bd00b1d50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56868/0 (socket says 172.21.15.64:56868) 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bcffff640 1 --2- 172.21.15.64:0/1831047693 >> v2:172.21.15.66:3300/0 conn(0x7f0bd015bde0 0x7f0bd00b4cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bd4d93640 1 -- 172.21.15.64:0/1831047693 >> v2:172.21.15.66:3300/0 conn(0x7f0bd015bde0 msgr2=0x7f0bd00b4cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bd4d93640 1 --2- 172.21.15.64:0/1831047693 >> v2:172.21.15.66:3300/0 conn(0x7f0bd015bde0 0x7f0bd00b4cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bd4d93640 1 -- 172.21.15.64:0/1831047693 >> v1:172.21.15.64:6789/0 conn(0x7f0bd01451a0 legacy=0x7f0bd00b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bd4d93640 1 -- 172.21.15.64:0/1831047693 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bd00b3f10 con 0x7f0bd00b2460 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bceffd640 1 -- 172.21.15.64:0/1831047693 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1573863247 0 0) 0x7f0bd00b3c70 con 0x7f0bd01451a0 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.150+0000 7f0bcffff640 1 --2- 172.21.15.64:0/1831047693 >> v2:172.21.15.66:3300/0 conn(0x7f0bd015bde0 0x7f0bd00b4cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bd4d93640 1 --2- 172.21.15.64:0/1831047693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd00b2460 0x7f0bd015c1c0 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7f0bc8009e50 tx=0x7f0bc802bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e548942aead0511b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bceffd640 1 -- 172.21.15.64:0/1831047693 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bc802cb20 con 0x7f0bd00b2460 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bceffd640 1 -- 172.21.15.64:0/1831047693 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0bc802ccc0 con 0x7f0bd00b2460 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bceffd640 1 -- 172.21.15.64:0/1831047693 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bc8034610 con 0x7f0bd00b2460 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bd67f6640 1 -- 172.21.15.64:0/1831047693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd00b2460 msgr2=0x7f0bd015c1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/1831047693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd00b2460 0x7f0bd015c1c0 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7f0bc8009e50 tx=0x7f0bc802bf80 comp rx=0 tx=0).stop 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bd67f6640 1 -- 172.21.15.64:0/1831047693 shutdown_connections 2024-03-04T07:29:57.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/1831047693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd00b2460 0x7f0bd015c1c0 unknown :-1 s=CLOSED pgs=793 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/1831047693 >> v2:172.21.15.66:3300/0 conn(0x7f0bd015bde0 0x7f0bd00b4cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.151+0000 7f0bd67f6640 1 -- 172.21.15.64:0/1831047693 >> 172.21.15.64:0/1831047693 conn(0x7f0bd00b55e0 msgr2=0x7f0bd00b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:57.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.152+0000 7f0bd67f6640 1 -- 172.21.15.64:0/1831047693 shutdown_connections 2024-03-04T07:29:57.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.152+0000 7f0bd67f6640 1 -- 172.21.15.64:0/1831047693 wait complete. 2024-03-04T07:29:57.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.153+0000 7f0bd67f6640 1 Processor -- start 2024-03-04T07:29:57.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.153+0000 7f0bd67f6640 1 -- start start 2024-03-04T07:29:57.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bd00b2460 0x7f0bd009ac70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bd01451a0 0x7f0bd01f1470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd015bde0 0x7f0bd009a4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bd00b5210 con 0x7f0bd015bde0 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bd00b44d0 con 0x7f0bd01451a0 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0bd00b5390 con 0x7f0bd00b2460 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd015bde0 0x7f0bd009a4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd015bde0 0x7f0bd009a4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43218/0 (socket says 172.21.15.64:43218) 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 -- 172.21.15.64:0/3151827535 learned_addr learned my addr 172.21.15.64:0/3151827535 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bcf7fe640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bd01451a0 0x7f0bd01f1470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bcffff640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bd00b2460 0x7f0bd009ac70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 -- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bd00b2460 msgr2=0x7f0bd009ac70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bd00b2460 0x7f0bd009ac70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 -- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bd01451a0 msgr2=0x7f0bd01f1470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bd01451a0 0x7f0bd01f1470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 -- 172.21.15.64:0/3151827535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bd009b370 con 0x7f0bd015bde0 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bcf7fe640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bd01451a0 0x7f0bd01f1470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd4d93640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd015bde0 0x7f0bd009a4e0 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7f0bc8009e50 tx=0x7f0bc8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bccff9640 1 -- 172.21.15.64:0/3151827535 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bc8043030 con 0x7f0bd015bde0 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bccff9640 1 -- 172.21.15.64:0/3151827535 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0bc803e450 con 0x7f0bd015bde0 2024-03-04T07:29:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0bd009b650 con 0x7f0bd015bde0 2024-03-04T07:29:57.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bccff9640 1 -- 172.21.15.64:0/3151827535 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bc8009510 con 0x7f0bd015bde0 2024-03-04T07:29:57.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.154+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0bd014a1f0 con 0x7f0bd015bde0 2024-03-04T07:29:57.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.156+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0bd0067ff0 con 0x7f0bd015bde0 2024-03-04T07:29:57.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.156+0000 7f0bccff9640 1 -- 172.21.15.64:0/3151827535 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0bc8046070 con 0x7f0bd015bde0 2024-03-04T07:29:57.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.156+0000 7f0bccff9640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bbc06b500 0x7f0bbc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.156+0000 7f0bccff9640 1 -- 172.21.15.64:0/3151827535 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f0bc80ac390 con 0x7f0bd015bde0 2024-03-04T07:29:57.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.156+0000 7f0bcffff640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bbc06b500 0x7f0bbc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.157+0000 7f0bcffff640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bbc06b500 0x7f0bbc06d9c0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f0bc0006fd0 tx=0x7f0bc0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.161+0000 7f0bccff9640 1 -- 172.21.15.64:0/3151827535 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0bc804d070 con 0x7f0bd015bde0 2024-03-04T07:29:57.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.342+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0bd015b030 con 0x7f0bd015bde0 2024-03-04T07:29:57.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.343+0000 7f0bccff9640 1 -- 172.21.15.64:0/3151827535 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v58) v1 ==== 74+0+12972 (secure 0 0 0) 0x7f0bc80333a0 con 0x7f0bd015bde0 2024-03-04T07:29:57.343 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:57.343 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":58,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:29:55.015497+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/1811282816":"2024-03-05T07:29:52.007621+0000","172.21.15.64:6836/1811282816":"2024-03-05T07:29:52.007621+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:29:57.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bbc06b500 msgr2=0x7f0bbc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bbc06b500 0x7f0bbc06d9c0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f0bc0006fd0 tx=0x7f0bc0008040 comp rx=0 tx=0).stop 2024-03-04T07:29:57.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd015bde0 msgr2=0x7f0bd009a4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd015bde0 0x7f0bd009a4e0 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7f0bc8009e50 tx=0x7f0bc8004a40 comp rx=0 tx=0).stop 2024-03-04T07:29:57.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 shutdown_connections 2024-03-04T07:29:57.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0bbc06b500 0x7f0bbc06d9c0 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0bd015bde0 0x7f0bd009a4e0 unknown :-1 s=CLOSED pgs=794 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0bd01451a0 0x7f0bd01f1470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 --2- 172.21.15.64:0/3151827535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0bd00b2460 0x7f0bd009ac70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 >> 172.21.15.64:0/3151827535 conn(0x7f0bd00b55e0 msgr2=0x7f0bd015a350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:57.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 shutdown_connections 2024-03-04T07:29:57.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.345+0000 7f0bd67f6640 1 -- 172.21.15.64:0/3151827535 wait complete. 2024-03-04T07:29:57.356 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:29:57.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:29:57.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.461+0000 7f77cfd48640 1 Processor -- start 2024-03-04T07:29:57.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77cfd48640 1 -- start start 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77cfd48640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c815dd90 0x7f77c80ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77cfd48640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f77c80b6d10 0x7f77c80b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77cfd48640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f77c80d3740 con 0x7f77c80d29d0 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77cfd48640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f77c80d3400 con 0x7f77c80b6d10 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77cfd48640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f77c80d3120 con 0x7f77c815dd90 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77ce2e5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f77c80b6d10 0x7f77c80b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77ce2e5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f77c80b6d10 0x7f77c80b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43234/0 (socket says 172.21.15.64:43234) 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.462+0000 7f77ce2e5640 1 -- 172.21.15.64:0/1307733683 learned_addr learned my addr 172.21.15.64:0/1307733683 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.463+0000 7f77cdae4640 1 --2- 172.21.15.64:0/1307733683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c815dd90 0x7f77c80ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.463+0000 7f77ce2e5640 1 -- 172.21.15.64:0/1307733683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c815dd90 msgr2=0x7f77c80ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.463+0000 7f77ce2e5640 1 --2- 172.21.15.64:0/1307733683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c815dd90 0x7f77c80ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.463+0000 7f77ce2e5640 1 -- 172.21.15.64:0/1307733683 >> v1:172.21.15.66:6789/0 conn(0x7f77c80d29d0 legacy=0x7f77c80d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:57.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.463+0000 7f77ce2e5640 1 -- 172.21.15.64:0/1307733683 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77c80d39e0 con 0x7f77c80b6d10 2024-03-04T07:29:57.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.463+0000 7f77ce2e5640 1 --2- 172.21.15.64:0/1307733683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c80b6d10 0x7f77c80b71a0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7f77c0009e50 tx=0x7f77c002bff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9ee513ff27aac88 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77ccae2640 1 -- 172.21.15.64:0/1307733683 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77c002cb90 con 0x7f77c80b6d10 2024-03-04T07:29:57.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77ccae2640 1 -- 172.21.15.64:0/1307733683 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f77c002cd30 con 0x7f77c80b6d10 2024-03-04T07:29:57.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77ccae2640 1 -- 172.21.15.64:0/1307733683 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77c0034610 con 0x7f77c80b6d10 2024-03-04T07:29:57.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 -- 172.21.15.64:0/1307733683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c80b6d10 msgr2=0x7f77c80b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 --2- 172.21.15.64:0/1307733683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c80b6d10 0x7f77c80b71a0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7f77c0009e50 tx=0x7f77c002bff0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 -- 172.21.15.64:0/1307733683 shutdown_connections 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 --2- 172.21.15.64:0/1307733683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c80b6d10 0x7f77c80b71a0 unknown :-1 s=CLOSED pgs=795 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 --2- 172.21.15.64:0/1307733683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c815dd90 0x7f77c80ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 -- 172.21.15.64:0/1307733683 >> 172.21.15.64:0/1307733683 conn(0x7f77c8071960 msgr2=0x7f77c8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 -- 172.21.15.64:0/1307733683 shutdown_connections 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.464+0000 7f77cfd48640 1 -- 172.21.15.64:0/1307733683 wait complete. 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.465+0000 7f77cfd48640 1 Processor -- start 2024-03-04T07:29:57.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.465+0000 7f77cfd48640 1 -- start start 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cfd48640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f77c80b6d10 0x7f77c815c550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cfd48640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c80d29d0 0x7f77c80cd630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cfd48640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c815dd90 0x7f77c80cdb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cfd48640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77c80d2df0 con 0x7f77c815dd90 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cfd48640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77c815e170 con 0x7f77c80b6d10 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cfd48640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f77c80d48f0 con 0x7f77c80d29d0 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77ce2e5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c815dd90 0x7f77c80cdb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77ce2e5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c815dd90 0x7f77c80cdb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43246/0 (socket says 172.21.15.64:43246) 2024-03-04T07:29:57.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77ce2e5640 1 -- 172.21.15.64:0/713950754 learned_addr learned my addr 172.21.15.64:0/713950754 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cdae4640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f77c80b6d10 0x7f77c815c550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77cd2e3640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c80d29d0 0x7f77c80cd630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77ce2e5640 1 -- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c80d29d0 msgr2=0x7f77c80cd630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77ce2e5640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c80d29d0 0x7f77c80cd630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.466+0000 7f77ce2e5640 1 -- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f77c80b6d10 msgr2=0x7f77c815c550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77ce2e5640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f77c80b6d10 0x7f77c815c550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77ce2e5640 1 -- 172.21.15.64:0/713950754 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77c815d000 con 0x7f77c815dd90 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77ce2e5640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c815dd90 0x7f77c80cdb70 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7f77c0009e50 tx=0x7f77c0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77be7fc640 1 -- 172.21.15.64:0/713950754 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77c0043020 con 0x7f77c815dd90 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77be7fc640 1 -- 172.21.15.64:0/713950754 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f77c003e450 con 0x7f77c815dd90 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f77c815ca90 con 0x7f77c815dd90 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77be7fc640 1 -- 172.21.15.64:0/713950754 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77c0009510 con 0x7f77c815dd90 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.467+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f77c80cbe60 con 0x7f77c815dd90 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.468+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f77c8072a30 con 0x7f77c815dd90 2024-03-04T07:29:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.468+0000 7f77be7fc640 1 -- 172.21.15.64:0/713950754 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f77c0046020 con 0x7f77c815dd90 2024-03-04T07:29:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.469+0000 7f77be7fc640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77ac06b4d0 0x7f77ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.469+0000 7f77be7fc640 1 -- 172.21.15.64:0/713950754 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f77c00ac580 con 0x7f77c815dd90 2024-03-04T07:29:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.469+0000 7f77cdae4640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77ac06b4d0 0x7f77ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.469+0000 7f77cdae4640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77ac06b4d0 0x7f77ac06d990 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f77c4006fd0 tx=0x7f77c4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.473+0000 7f77be7fc640 1 -- 172.21.15.64:0/713950754 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f77c004d030 con 0x7f77c815dd90 2024-03-04T07:29:57.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.653+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f77c804f410 con 0x7f77c815dd90 2024-03-04T07:29:57.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.654+0000 7f77be7fc640 1 -- 172.21.15.64:0/713950754 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v58) v1 ==== 93+0+10 (secure 0 0 0) 0x7f77c007b700 con 0x7f77c815dd90 2024-03-04T07:29:57.654 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:29:57.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77ac06b4d0 msgr2=0x7f77ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77ac06b4d0 0x7f77ac06d990 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f77c4006fd0 tx=0x7f77c4008040 comp rx=0 tx=0).stop 2024-03-04T07:29:57.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c815dd90 msgr2=0x7f77c80cdb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c815dd90 0x7f77c80cdb70 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7f77c0009e50 tx=0x7f77c0004a40 comp rx=0 tx=0).stop 2024-03-04T07:29:57.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 shutdown_connections 2024-03-04T07:29:57.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77ac06b4d0 0x7f77ac06d990 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f77c815dd90 0x7f77c80cdb70 unknown :-1 s=CLOSED pgs=796 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f77c80d29d0 0x7f77c80cd630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 --2- 172.21.15.64:0/713950754 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f77c80b6d10 0x7f77c815c550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 >> 172.21.15.64:0/713950754 conn(0x7f77c8071960 msgr2=0x7f77c81616d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:57.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 shutdown_connections 2024-03-04T07:29:57.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.656+0000 7f77cfd48640 1 -- 172.21.15.64:0/713950754 wait complete. 2024-03-04T07:29:57.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays' 2024-03-04T07:29:57.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.769+0000 7f9f75847640 1 Processor -- start 2024-03-04T07:29:57.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.769+0000 7f9f75847640 1 -- start start 2024-03-04T07:29:57.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.769+0000 7f9f75847640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9f7015df70 0x7f9f700d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.769+0000 7f9f75847640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9f700d18c0 0x7f9f70071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f75847640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f70071a80 0x7f9f700b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f75847640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f70071ef0 con 0x7f9f7015df70 2024-03-04T07:29:57.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f75847640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f700b6f20 con 0x7f9f700d18c0 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f75847640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f700d3030 con 0x7f9f70071a80 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9f7015df70 0x7f9f700d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9f7015df70 0x7f9f700d27b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43260/0 (socket says 172.21.15.64:43260) 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 -- 172.21.15.64:0/3281574602 learned_addr learned my addr 172.21.15.64:0/3281574602 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6e7fc640 1 --2- 172.21.15.64:0/3281574602 >> v2:172.21.15.66:3300/0 conn(0x7f9f700d18c0 0x7f9f70071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6f7fe640 1 --2- 172.21.15.64:0/3281574602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f70071a80 0x7f9f700b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 -- 172.21.15.64:0/3281574602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f70071a80 msgr2=0x7f9f700b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 --2- 172.21.15.64:0/3281574602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f70071a80 0x7f9f700b2700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 -- 172.21.15.64:0/3281574602 >> v2:172.21.15.66:3300/0 conn(0x7f9f700d18c0 msgr2=0x7f9f70071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 --2- 172.21.15.64:0/3281574602 >> v2:172.21.15.66:3300/0 conn(0x7f9f700d18c0 0x7f9f70071540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.770+0000 7f9f6effd640 1 -- 172.21.15.64:0/3281574602 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f700d3310 con 0x7f9f7015df70 2024-03-04T07:29:57.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.771+0000 7f9f6effd640 1 --2- 172.21.15.64:0/3281574602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f7015df70 0x7f9f700d27b0 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7f9f70161b30 tx=0x7f9f6002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d20ae8074be6509c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.771+0000 7f9f6dffb640 1 -- 172.21.15.64:0/3281574602 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f60034070 con 0x7f9f7015df70 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.771+0000 7f9f6dffb640 1 -- 172.21.15.64:0/3281574602 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f60030860 con 0x7f9f7015df70 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.771+0000 7f9f6dffb640 1 -- 172.21.15.64:0/3281574602 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f60030c30 con 0x7f9f7015df70 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 -- 172.21.15.64:0/3281574602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f7015df70 msgr2=0x7f9f700d27b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 --2- 172.21.15.64:0/3281574602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f7015df70 0x7f9f700d27b0 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7f9f70161b30 tx=0x7f9f6002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 -- 172.21.15.64:0/3281574602 shutdown_connections 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 --2- 172.21.15.64:0/3281574602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f70071a80 0x7f9f700b2700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 --2- 172.21.15.64:0/3281574602 >> v2:172.21.15.66:3300/0 conn(0x7f9f700d18c0 0x7f9f70071540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 --2- 172.21.15.64:0/3281574602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f7015df70 0x7f9f700d27b0 unknown :-1 s=CLOSED pgs=797 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 -- 172.21.15.64:0/3281574602 >> 172.21.15.64:0/3281574602 conn(0x7f9f700d3cf0 msgr2=0x7f9f700d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 -- 172.21.15.64:0/3281574602 shutdown_connections 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.772+0000 7f9f75847640 1 -- 172.21.15.64:0/3281574602 wait complete. 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.773+0000 7f9f75847640 1 Processor -- start 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.773+0000 7f9f75847640 1 -- start start 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f75847640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f70071a80 0x7f9f700aa680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f75847640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f700d18c0 0x7f9f70152fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f75847640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f7015df70 0x7f9f701f7150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f75847640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f700d2cf0 con 0x7f9f70071a80 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f75847640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f700d38d0 con 0x7f9f7015df70 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f75847640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f7015e350 con 0x7f9f700d18c0 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f70071a80 0x7f9f700aa680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f70071a80 0x7f9f700aa680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43264/0 (socket says 172.21.15.64:43264) 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 -- 172.21.15.64:0/2155953859 learned_addr learned my addr 172.21.15.64:0/2155953859 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6f7fe640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f7015df70 0x7f9f701f7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6e7fc640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f700d18c0 0x7f9f70152fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 -- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f700d18c0 msgr2=0x7f9f70152fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f700d18c0 0x7f9f70152fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 -- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f7015df70 msgr2=0x7f9f701f7150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f7015df70 0x7f9f701f7150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 -- 172.21.15.64:0/2155953859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f700aabc0 con 0x7f9f70071a80 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6e7fc640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f700d18c0 0x7f9f70152fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6f7fe640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f7015df70 0x7f9f701f7150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f6effd640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f70071a80 0x7f9f700aa680 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7f9f6002b9e0 tx=0x7f9f60004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f57fff640 1 -- 172.21.15.64:0/2155953859 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f60048070 con 0x7f9f70071a80 2024-03-04T07:29:57.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f57fff640 1 -- 172.21.15.64:0/2155953859 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f60030660 con 0x7f9f70071a80 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f700ce970 con 0x7f9f70071a80 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.774+0000 7f9f57fff640 1 -- 172.21.15.64:0/2155953859 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f60034040 con 0x7f9f70071a80 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.775+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f700cc230 con 0x7f9f70071a80 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.775+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f700cc670 con 0x7f9f70071a80 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.776+0000 7f9f57fff640 1 -- 172.21.15.64:0/2155953859 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9f6004b070 con 0x7f9f70071a80 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.776+0000 7f9f57fff640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f5c06b500 0x7f9f5c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.776+0000 7f9f57fff640 1 -- 172.21.15.64:0/2155953859 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f9f600abce0 con 0x7f9f70071a80 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.776+0000 7f9f6e7fc640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f5c06b500 0x7f9f5c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:57.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.777+0000 7f9f6e7fc640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f5c06b500 0x7f9f5c06d9c0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f9f701f70b0 tx=0x7f9f64008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:57.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:57.781+0000 7f9f57fff640 1 -- 172.21.15.64:0/2155953859 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9f60030830 con 0x7f9f70071a80 2024-03-04T07:29:58.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:58.032+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]} v 0) v1 -- 0x7f9f70067ff0 con 0x7f9f70071a80 2024-03-04T07:29:59.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.029+0000 7f9f57fff640 1 -- 172.21.15.64:0/2155953859 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]}]=0 v179) v1 ==== 143+0+0 (secure 0 0 0) 0x7f9f6007aef0 con 0x7f9f70071a80 2024-03-04T07:29:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.032+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f5c06b500 msgr2=0x7f9f5c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.032+0000 7f9f75847640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f5c06b500 0x7f9f5c06d9c0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f9f701f70b0 tx=0x7f9f64008040 comp rx=0 tx=0).stop 2024-03-04T07:29:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.032+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f70071a80 msgr2=0x7f9f700aa680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.032+0000 7f9f75847640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f70071a80 0x7f9f700aa680 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7f9f6002b9e0 tx=0x7f9f60004730 comp rx=0 tx=0).stop 2024-03-04T07:29:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.032+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 shutdown_connections 2024-03-04T07:29:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.032+0000 7f9f75847640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f5c06b500 0x7f9f5c06d9c0 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.033+0000 7f9f75847640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f7015df70 0x7f9f701f7150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.033+0000 7f9f75847640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f700d18c0 0x7f9f70152fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.033+0000 7f9f75847640 1 --2- 172.21.15.64:0/2155953859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f70071a80 0x7f9f700aa680 unknown :-1 s=CLOSED pgs=798 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.033+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 >> 172.21.15.64:0/2155953859 conn(0x7f9f700d3cf0 msgr2=0x7f9f7015e650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:59.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.033+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 shutdown_connections 2024-03-04T07:29:59.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.033+0000 7f9f75847640 1 -- 172.21.15.64:0/2155953859 wait complete. 2024-03-04T07:29:59.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:29:59.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.157+0000 7f3776fe9640 1 Processor -- start 2024-03-04T07:29:59.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.157+0000 7f3776fe9640 1 -- start start 2024-03-04T07:29:59.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f3776fe9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 0x7f37700ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f3776fe9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f37700c5e30 0x7f37700c6280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f3776fe9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f37700cd310 con 0x7f37700cf740 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f3776fe9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f377015f8e0 con 0x7f37700c5e30 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f3776fe9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f37700c56c0 con 0x7f377015f500 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f376ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f37700cf740 0x7f37700ce840 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56870/0 (socket says 172.21.15.64:56870) 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f376ffff640 1 -- 172.21.15.64:0/1055692585 learned_addr learned my addr 172.21.15.64:0/1055692585 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f3775586640 1 --2- 172.21.15.64:0/1055692585 >> v2:172.21.15.66:3300/0 conn(0x7f37700c5e30 0x7f37700c6280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f3774d85640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 0x7f37700ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1802318626 0 0) 0x7f37700cd310 con 0x7f37700cf740 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.158+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f375c003520 con 0x7f37700cf740 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 642358059 0 0) 0x7f375c003520 con 0x7f37700cf740 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 msgr2=0x7f37700ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 0x7f37700ce300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 >> v2:172.21.15.66:3300/0 conn(0x7f37700c5e30 msgr2=0x7f37700c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 --2- 172.21.15.64:0/1055692585 >> v2:172.21.15.66:3300/0 conn(0x7f37700c5e30 0x7f37700c6280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37700cd5b0 con 0x7f37700cf740 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f3760001a10 con 0x7f37700cf740 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 >> v1:172.21.15.64:6789/0 conn(0x7f37700cf740 legacy=0x7f37700ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 0x7f375c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f375c003f80 0x7f375c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f375c004d40 0x7f375c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f377015f8e0 con 0x7f375c004d40 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37700c56c0 con 0x7f375c003f80 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f375c00ba70 con 0x7f377015f500 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f3775586640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f375c004d40 0x7f375c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f3774d85640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 0x7f375c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f376ffff640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f375c003f80 0x7f375c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f3775586640 1 -- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 msgr2=0x7f375c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f3775586640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f377015f500 0x7f375c0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f3775586640 1 -- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f375c003f80 msgr2=0x7f375c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f3775586640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f375c003f80 0x7f375c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.159+0000 7f3775586640 1 -- 172.21.15.64:0/1055692585 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f375c003520 con 0x7f375c004d40 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.160+0000 7f3775586640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f375c004d40 0x7f375c0051c0 secure :-1 s=READY pgs=800 cs=0 l=1 rev1=1 crypto rx=0x7f3758009ac0 tx=0x7f375c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7a4cd27629e93f73 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.160+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3758039070 con 0x7f375c004d40 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.160+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f375802c710 con 0x7f375c004d40 2024-03-04T07:29:59.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.160+0000 7f376f7fe640 1 -- 172.21.15.64:0/1055692585 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3758009690 con 0x7f375c004d40 2024-03-04T07:29:59.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.161+0000 7f3776fe9640 1 -- 172.21.15.64:0/1055692585 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f375c004d40 msgr2=0x7f375c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.161+0000 7f3776fe9640 1 --2- 172.21.15.64:0/1055692585 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f375c004d40 0x7f375c0051c0 secure :-1 s=READY pgs=800 cs=0 l=1 rev1=1 crypto rx=0x7f3758009ac0 tx=0x7f375c005830 comp rx=0 tx=0).stop 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.161+0000 7f3775586640 1 -- 172.21.15.64:0/1055692585 reap_dead start 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.161+0000 7f3776fe9640 1 -- 172.21.15.64:0/1055692585 shutdown_connections 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.161+0000 7f3776fe9640 1 -- 172.21.15.64:0/1055692585 >> 172.21.15.64:0/1055692585 conn(0x7f37700b3700 msgr2=0x7f37700b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.161+0000 7f3776fe9640 1 -- 172.21.15.64:0/1055692585 shutdown_connections 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.161+0000 7f3776fe9640 1 -- 172.21.15.64:0/1055692585 wait complete. 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.162+0000 7f3776fe9640 1 Processor -- start 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.162+0000 7f3776fe9640 1 -- start start 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3776fe9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f375c004d40 0x7f37700b5c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3776fe9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37700b72d0 0x7f37700c8c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3776fe9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3770148ef0 0x7f377015d650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3776fe9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37700b43d0 con 0x7f37700b72d0 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3776fe9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37700c59a0 con 0x7f3770148ef0 2024-03-04T07:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3776fe9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f37700cef50 con 0x7f375c004d40 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37700b72d0 0x7f37700c8c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37700b72d0 0x7f37700c8c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43278/0 (socket says 172.21.15.64:43278) 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 -- 172.21.15.64:0/2719961325 learned_addr learned my addr 172.21.15.64:0/2719961325 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3775586640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3770148ef0 0x7f377015d650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f3774d85640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f375c004d40 0x7f37700b5c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 -- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f375c004d40 msgr2=0x7f37700b5c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f375c004d40 0x7f37700b5c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 -- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3770148ef0 msgr2=0x7f377015d650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3770148ef0 0x7f377015d650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 -- 172.21.15.64:0/2719961325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37700c9180 con 0x7f37700b72d0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.163+0000 7f376ffff640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37700b72d0 0x7f37700c8c40 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7f3760007970 tx=0x7f376000a6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.164+0000 7f376d7fa640 1 -- 172.21.15.64:0/2719961325 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3760017020 con 0x7f37700b72d0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.164+0000 7f376d7fa640 1 -- 172.21.15.64:0/2719961325 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f376000ae10 con 0x7f37700b72d0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.164+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f37700b6230 con 0x7f37700b72d0 2024-03-04T07:29:59.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.164+0000 7f376d7fa640 1 -- 172.21.15.64:0/2719961325 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3760013610 con 0x7f37700b72d0 2024-03-04T07:29:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.164+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f37700b67f0 con 0x7f37700b72d0 2024-03-04T07:29:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.165+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f377009fcf0 con 0x7f37700b72d0 2024-03-04T07:29:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.165+0000 7f376d7fa640 1 -- 172.21.15.64:0/2719961325 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f376001a020 con 0x7f37700b72d0 2024-03-04T07:29:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.165+0000 7f376d7fa640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f375006b500 0x7f375006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.166+0000 7f376d7fa640 1 -- 172.21.15.64:0/2719961325 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f376008a850 con 0x7f37700b72d0 2024-03-04T07:29:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.166+0000 7f3774d85640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f375006b500 0x7f375006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.166+0000 7f3774d85640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f375006b500 0x7f375006d9c0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f3768009bc0 tx=0x7f3768009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:59.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.170+0000 7f376d7fa640 1 -- 172.21.15.64:0/2719961325 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3760021020 con 0x7f37700b72d0 2024-03-04T07:29:59.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.351+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f377004f410 con 0x7f37700b72d0 2024-03-04T07:29:59.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.354+0000 7f376d7fa640 1 -- 172.21.15.64:0/2719961325 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v22)=0 v22) v1 ==== 122+0+0 (secure 0 0 0) 0x7f37600599d0 con 0x7f37700b72d0 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.356+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f375006b500 msgr2=0x7f375006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f375006b500 0x7f375006d9c0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f3768009bc0 tx=0x7f3768009190 comp rx=0 tx=0).stop 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37700b72d0 msgr2=0x7f37700c8c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37700b72d0 0x7f37700c8c40 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7f3760007970 tx=0x7f376000a6a0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 shutdown_connections 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f375006b500 0x7f375006d9c0 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3770148ef0 0x7f377015d650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37700b72d0 0x7f37700c8c40 unknown :-1 s=CLOSED pgs=801 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 --2- 172.21.15.64:0/2719961325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f375c004d40 0x7f37700b5c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 >> 172.21.15.64:0/2719961325 conn(0x7f37700b3700 msgr2=0x7f37700c6630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:59.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 shutdown_connections 2024-03-04T07:29:59.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.357+0000 7f3776fe9640 1 -- 172.21.15.64:0/2719961325 wait complete. 2024-03-04T07:29:59.367 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:29:59.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:29:59.416 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:29:59.466 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:29:59.466 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:59.466 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:59.481 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:29:59.481 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:29:59.482 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:59.500 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:29:59.500 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:29:59.500 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:59.536 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:29:59.536 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:29:59.536 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:29:59.558 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:29:59.558 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:29:59.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:29:59.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.679+0000 7fe7a275c640 1 Processor -- start 2024-03-04T07:29:59.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.679+0000 7fe7a275c640 1 -- start start 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.679+0000 7fe7a275c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 0x7fe79c06af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.679+0000 7fe7a275c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe79c0cc0e0 con 0x7fe79c0b1020 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.679+0000 7fe7a275c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe79c0cc3c0 con 0x7fe79c0cd890 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.679+0000 7fe7a275c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe79c0d28b0 con 0x7fe79c06ed10 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe79c0b1020 0x7fe79c0cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56880/0 (socket says 172.21.15.64:56880) 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79bfff640 1 -- 172.21.15.64:0/4146056468 learned_addr learned my addr 172.21.15.64:0/4146056468 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 0x7fe79c06af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3456512103 0 0) 0x7fe79c0cc0e0 con 0x7fe79c0b1020 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe788003520 con 0x7fe79c0b1020 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3234946073 0 0) 0x7fe788003520 con 0x7fe79c0b1020 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 msgr2=0x7fe79c06af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 0x7fe79c06af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 >> v1:172.21.15.66:6789/0 conn(0x7fe79c0cd890 legacy=0x7fe79c15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe79c0cc6a0 con 0x7fe79c0b1020 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fe784001670 con 0x7fe79c0b1020 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 >> v1:172.21.15.64:6789/0 conn(0x7fe79c0b1020 legacy=0x7fe79c0cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 0x7fe7880039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe788003f60 0x7fe788004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe788004d40 0x7fe7880051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe79c0d28b0 con 0x7fe788003f60 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe78800baa0 con 0x7fe788004d40 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe78800bca0 con 0x7fe79c06ed10 2024-03-04T07:29:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe788003f60 0x7fe788004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe79bfff640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 0x7fe7880039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.680+0000 7fe7a0cf9640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe788004d40 0x7fe7880051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79b7fe640 1 -- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 msgr2=0x7fe7880039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe79c06ed10 0x7fe7880039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79b7fe640 1 -- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe788004d40 msgr2=0x7fe7880051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe788004d40 0x7fe7880051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79b7fe640 1 -- 172.21.15.64:0/4146056468 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe788003520 con 0x7fe788003f60 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe788003f60 0x7fe788004800 secure :-1 s=READY pgs=803 cs=0 l=1 rev1=1 crypto rx=0x7fe78c009ac0 tx=0x7fe788005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4183790136edc716 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe78c039070 con 0x7fe788003f60 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe78c02c6b0 con 0x7fe788003f60 2024-03-04T07:29:59.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.681+0000 7fe79affd640 1 -- 172.21.15.64:0/4146056468 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe78c009690 con 0x7fe788003f60 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.682+0000 7fe7a275c640 1 -- 172.21.15.64:0/4146056468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe788003f60 msgr2=0x7fe788004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.682+0000 7fe7a275c640 1 --2- 172.21.15.64:0/4146056468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe788003f60 0x7fe788004800 secure :-1 s=READY pgs=803 cs=0 l=1 rev1=1 crypto rx=0x7fe78c009ac0 tx=0x7fe788005830 comp rx=0 tx=0).stop 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.682+0000 7fe7a0cf9640 1 -- 172.21.15.64:0/4146056468 reap_dead start 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.682+0000 7fe7a275c640 1 -- 172.21.15.64:0/4146056468 shutdown_connections 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.682+0000 7fe7a275c640 1 -- 172.21.15.64:0/4146056468 >> 172.21.15.64:0/4146056468 conn(0x7fe79c0d1a90 msgr2=0x7fe79c0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.682+0000 7fe7a275c640 1 -- 172.21.15.64:0/4146056468 shutdown_connections 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.682+0000 7fe7a275c640 1 -- 172.21.15.64:0/4146056468 wait complete. 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.683+0000 7fe7a275c640 1 Processor -- start 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.683+0000 7fe7a275c640 1 -- start start 2024-03-04T07:29:59.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a275c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe788003f60 0x7fe79c15e200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a275c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe79c1f7a00 0x7fe79c0b61e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a275c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe79c1f7e60 0x7fe79c0b6720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a275c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe79c0ccc60 con 0x7fe79c1f7a00 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a275c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe79c0cf320 con 0x7fe79c1f7e60 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a275c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe79c0cdce0 con 0x7fe788003f60 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe79c1f7a00 0x7fe79c0b61e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe79c1f7a00 0x7fe79c0b61e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43306/0 (socket says 172.21.15.64:43306) 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 -- 172.21.15.64:0/65032087 learned_addr learned my addr 172.21.15.64:0/65032087 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79bfff640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe788003f60 0x7fe79c15e200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a0cf9640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe79c1f7e60 0x7fe79c0b6720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 -- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe788003f60 msgr2=0x7fe79c15e200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe788003f60 0x7fe79c15e200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 -- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe79c1f7e60 msgr2=0x7fe79c0b6720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe79c1f7e60 0x7fe79c0b6720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79b7fe640 1 -- 172.21.15.64:0/65032087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe79c0cac10 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe7a0cf9640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe79c1f7e60 0x7fe79c0b6720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.684+0000 7fe79bfff640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe788003f60 0x7fe79c15e200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.685+0000 7fe79b7fe640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe79c1f7a00 0x7fe79c0b61e0 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7fe78c009ac0 tx=0x7fe78c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.685+0000 7fe798ff9640 1 -- 172.21.15.64:0/65032087 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe78c039050 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.685+0000 7fe798ff9640 1 -- 172.21.15.64:0/65032087 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe78c045070 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.685+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe79c0caef0 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.685+0000 7fe798ff9640 1 -- 172.21.15.64:0/65032087 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe78c0044e0 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.685+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe79c0cb4b0 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.686+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe79c067ff0 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.686+0000 7fe798ff9640 1 -- 172.21.15.64:0/65032087 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe78c003850 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.687+0000 7fe798ff9640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe77c06b4d0 0x7fe77c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.687+0000 7fe798ff9640 1 -- 172.21.15.64:0/65032087 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fe78c0aba70 con 0x7fe79c1f7a00 2024-03-04T07:29:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.687+0000 7fe79bfff640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe77c06b4d0 0x7fe77c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:29:59.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.687+0000 7fe79bfff640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe77c06b4d0 0x7fe77c06d990 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fe784001eb0 tx=0x7fe784009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:29:59.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.691+0000 7fe798ff9640 1 -- 172.21.15.64:0/65032087 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe78c04e020 con 0x7fe79c1f7a00 2024-03-04T07:29:59.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.912+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe79c06f2f0 con 0x7fe79c1f7a00 2024-03-04T07:29:59.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.913+0000 7fe798ff9640 1 -- 172.21.15.64:0/65032087 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 31 v31) v1 ==== 94+0+3693 (secure 0 0 0) 0x7fe78c07ac80 con 0x7fe79c1f7a00 2024-03-04T07:29:59.913 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:29:59.913 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":31,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5671,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/104584754","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":104584754},{"type":"v1","addr":"172.21.15.64:6835","nonce":104584754}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5686,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1280811776","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1280811776},{"type":"v1","addr":"172.21.15.66:6834","nonce":1280811776}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":5689,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2282933732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2282933732},{"type":"v1","addr":"172.21.15.66:6835","nonce":2282933732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":26},{"gid":6459,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/658320226","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":658320226},{"type":"v1","addr":"172.21.15.64:6837","nonce":658320226}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":31}],"filesystems":[]} 2024-03-04T07:29:59.913 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 31 2024-03-04T07:29:59.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe77c06b4d0 msgr2=0x7fe77c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe77c06b4d0 0x7fe77c06d990 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fe784001eb0 tx=0x7fe784009040 comp rx=0 tx=0).stop 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe79c1f7a00 msgr2=0x7fe79c0b61e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe79c1f7a00 0x7fe79c0b61e0 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7fe78c009ac0 tx=0x7fe78c004730 comp rx=0 tx=0).stop 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 shutdown_connections 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe77c06b4d0 0x7fe77c06d990 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe79c1f7e60 0x7fe79c0b6720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe79c1f7a00 0x7fe79c0b61e0 unknown :-1 s=CLOSED pgs=804 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 --2- 172.21.15.64:0/65032087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe788003f60 0x7fe79c15e200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.914+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 >> 172.21.15.64:0/65032087 conn(0x7fe79c0d1a90 msgr2=0x7fe79c0afaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.915+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 shutdown_connections 2024-03-04T07:29:59.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:29:59.915+0000 7fe7a275c640 1 -- 172.21.15.64:0/65032087 wait complete. 2024-03-04T07:29:59.925 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:29:59.925 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:29:59.925 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:29:59.926 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:59.926 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:29:59.926 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:29:59.926 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:29:59.926 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:59.926 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:29:59.926 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:29:59.927 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:29:59.927 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:59.927 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:29:59.927 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:29:59.927 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:29:59.927 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:29:59.935 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:29:59.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:29:59.936 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:29:59.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:29:59.939 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:29:59.939 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:29:59.939 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:29:59.940 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:29:59.978 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:29:59.979 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:59.979 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:29:59.980 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:59.983 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:29:59.983 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:59.984 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:29:59.984 DEBUG:teuthology.parallel:result is None 2024-03-04T07:29:59.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:30:00.058 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:30:00.063 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:30:00.075 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:30:00.079 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:30:00.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.099+0000 7f7185083640 1 Processor -- start 2024-03-04T07:30:00.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.100+0000 7f7185083640 1 -- start start 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.100+0000 7f7185083640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f718015df00 0x7f71800d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.100+0000 7f7185083640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f71800b9a30 0x7f71800d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.100+0000 7f7185083640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71800cea30 con 0x7f71800b95a0 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.100+0000 7f7185083640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f71800d2af0 con 0x7f71800b9a30 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.100+0000 7f7185083640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f71800cf3c0 con 0x7f718015df00 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.100+0000 7f717f59e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f71800b9a30 0x7f71800d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717e59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f71800b95a0 0x7f71800b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56882/0 (socket says 172.21.15.64:56882) 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717e59c640 1 -- 172.21.15.64:0/1273449216 learned_addr learned my addr 172.21.15.64:0/1273449216 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717ed9d640 1 --2- 172.21.15.64:0/1273449216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f718015df00 0x7f71800d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717f59e640 1 -- 172.21.15.64:0/1273449216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f718015df00 msgr2=0x7f71800d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717f59e640 1 --2- 172.21.15.64:0/1273449216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f718015df00 0x7f71800d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:00.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717f59e640 1 -- 172.21.15.64:0/1273449216 >> v1:172.21.15.64:6789/0 conn(0x7f71800b95a0 legacy=0x7f71800b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:00.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717dd9b640 1 -- 172.21.15.64:0/1273449216 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3106718528 0 0) 0x7f71800cea30 con 0x7f71800b95a0 2024-03-04T07:30:00.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717f59e640 1 -- 172.21.15.64:0/1273449216 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71800cf0e0 con 0x7f71800b9a30 2024-03-04T07:30:00.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.101+0000 7f717ed9d640 1 --2- 172.21.15.64:0/1273449216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f718015df00 0x7f71800d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.102+0000 7f717f59e640 1 --2- 172.21.15.64:0/1273449216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800b9a30 0x7f71800d3220 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7f7178009d90 tx=0x7f717802bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=17b6ff19ff70657e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.102+0000 7f717dd9b640 1 -- 172.21.15.64:0/1273449216 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7178039070 con 0x7f71800b9a30 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.102+0000 7f717dd9b640 1 -- 172.21.15.64:0/1273449216 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f717802cc50 con 0x7f71800b9a30 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.102+0000 7f717dd9b640 1 -- 172.21.15.64:0/1273449216 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7178034440 con 0x7f71800b9a30 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.102+0000 7f7185083640 1 -- 172.21.15.64:0/1273449216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800b9a30 msgr2=0x7f71800d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.102+0000 7f7185083640 1 --2- 172.21.15.64:0/1273449216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800b9a30 0x7f71800d3220 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7f7178009d90 tx=0x7f717802bea0 comp rx=0 tx=0).stop 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.102+0000 7f7185083640 1 -- 172.21.15.64:0/1273449216 shutdown_connections 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.103+0000 7f7185083640 1 --2- 172.21.15.64:0/1273449216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800b9a30 0x7f71800d3220 unknown :-1 s=CLOSED pgs=810 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.103+0000 7f7185083640 1 --2- 172.21.15.64:0/1273449216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f718015df00 0x7f71800d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.103+0000 7f7185083640 1 -- 172.21.15.64:0/1273449216 >> 172.21.15.64:0/1273449216 conn(0x7f7180071960 msgr2=0x7f7180071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:00.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.103+0000 7f7185083640 1 -- 172.21.15.64:0/1273449216 shutdown_connections 2024-03-04T07:30:00.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.103+0000 7f7185083640 1 -- 172.21.15.64:0/1273449216 wait complete. 2024-03-04T07:30:00.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.104+0000 7f7185083640 1 Processor -- start 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.104+0000 7f7185083640 1 -- start start 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f7185083640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f71800b95a0 0x7f718015f330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f7185083640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f718015df00 0x7f71800c77b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f7185083640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800c7cf0 0x7f71800ca140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f7185083640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71800cec70 con 0x7f71800c7cf0 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f7185083640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71800ce750 con 0x7f718015df00 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f7185083640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f71800cedf0 con 0x7f71800b95a0 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717f59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800c7cf0 0x7f71800ca140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717f59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800c7cf0 0x7f71800ca140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43358/0 (socket says 172.21.15.64:43358) 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717f59e640 1 -- 172.21.15.64:0/1237418635 learned_addr learned my addr 172.21.15.64:0/1237418635 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717e59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f718015df00 0x7f71800c77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717ed9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f71800b95a0 0x7f718015f330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:00.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717ed9d640 1 -- 172.21.15.64:0/1237418635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f718015df00 msgr2=0x7f71800c77b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717ed9d640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f718015df00 0x7f71800c77b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717ed9d640 1 -- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800c7cf0 msgr2=0x7f71800ca140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717ed9d640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800c7cf0 0x7f71800ca140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.105+0000 7f717ed9d640 1 -- 172.21.15.64:0/1237418635 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f718015fca0 con 0x7f71800b95a0 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.106+0000 7f717ed9d640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f71800b95a0 0x7f718015f330 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f717400e870 tx=0x7f717400ed30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.106+0000 7f71677fe640 1 -- 172.21.15.64:0/1237418635 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7174017070 con 0x7f71800b95a0 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.106+0000 7f71677fe640 1 -- 172.21.15.64:0/1237418635 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f717400c940 con 0x7f71800b95a0 2024-03-04T07:30:00.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.106+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f718015ff80 con 0x7f71800b95a0 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.106+0000 7f71677fe640 1 -- 172.21.15.64:0/1237418635 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f717400cd10 con 0x7f71800b95a0 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.106+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7180160540 con 0x7f71800b95a0 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.107+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f71800c8100 con 0x7f71800b95a0 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.107+0000 7f71677fe640 1 -- 172.21.15.64:0/1237418635 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f717401a070 con 0x7f71800b95a0 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.108+0000 7f71677fe640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f716c06b500 0x7f716c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.108+0000 7f71677fe640 1 -- 172.21.15.64:0/1237418635 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(58..58 src has 1..58) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f7174089ff0 con 0x7f71800b95a0 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.108+0000 7f717e59c640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f716c06b500 0x7f716c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:00.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.108+0000 7f717e59c640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f716c06b500 0x7f716c06d9c0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f71801f7060 tx=0x7f7170007d50 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:00.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.113+0000 7f71677fe640 1 -- 172.21.15.64:0/1237418635 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f71800c8100 con 0x7f71800b95a0 2024-03-04T07:30:00.140 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.b is failed for ~0s 2024-03-04T07:30:00.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:00.307+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f7180067ff0 con 0x7f71800b95a0 2024-03-04T07:30:01.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.049+0000 7f71677fe640 1 -- 172.21.15.64:0/1237418635 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v60) v1 ==== 97+0+0 (secure 0 0 0) 0x7f7174058b20 con 0x7f71800b95a0 2024-03-04T07:30:01.049 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:30:01.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.051+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f716c06b500 msgr2=0x7f716c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.051+0000 7f7185083640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f716c06b500 0x7f716c06d9c0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f71801f7060 tx=0x7f7170007d50 comp rx=0 tx=0).stop 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.051+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f71800b95a0 msgr2=0x7f718015f330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.051+0000 7f7185083640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f71800b95a0 0x7f718015f330 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f717400e870 tx=0x7f717400ed30 comp rx=0 tx=0).stop 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 shutdown_connections 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f716c06b500 0x7f716c06d9c0 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71800c7cf0 0x7f71800ca140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f718015df00 0x7f71800c77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 --2- 172.21.15.64:0/1237418635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f71800b95a0 0x7f718015f330 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 >> 172.21.15.64:0/1237418635 conn(0x7f7180071960 msgr2=0x7f71800b8b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 shutdown_connections 2024-03-04T07:30:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.052+0000 7f7185083640 1 -- 172.21.15.64:0/1237418635 wait complete. 2024-03-04T07:30:01.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:30:01.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.175+0000 7faee7d93640 1 Processor -- start 2024-03-04T07:30:01.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.175+0000 7faee7d93640 1 -- start start 2024-03-04T07:30:01.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee7d93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee00b3940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee7d93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faee00720c0 0x7faee00cdf90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee7d93640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7faee00b3e80 con 0x7faee00afb30 2024-03-04T07:30:01.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee7d93640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faee00b4120 con 0x7faee00720c0 2024-03-04T07:30:01.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee7d93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faee00b4400 con 0x7faee015ac30 2024-03-04T07:30:01.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee6330640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faee00720c0 0x7faee00cdf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee6330640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faee00720c0 0x7faee00cdf90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43370/0 (socket says 172.21.15.64:43370) 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee6330640 1 -- 172.21.15.64:0/1533392156 learned_addr learned my addr 172.21.15.64:0/1533392156 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.176+0000 7faee532e640 1 --1- 172.21.15.64:0/1533392156 >> v1:172.21.15.64:6789/0 conn(0x7faee00afb30 0x7faee00719b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56886/0 (socket says 172.21.15.64:56886) 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.177+0000 7faee5b2f640 1 --2- 172.21.15.64:0/1533392156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee00b3940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.177+0000 7faee6330640 1 -- 172.21.15.64:0/1533392156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 msgr2=0x7faee00b3940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.177+0000 7faee6330640 1 --2- 172.21.15.64:0/1533392156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee00b3940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.177+0000 7faee6330640 1 -- 172.21.15.64:0/1533392156 >> v1:172.21.15.64:6789/0 conn(0x7faee00afb30 legacy=0x7faee00719b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.177+0000 7faee6330640 1 -- 172.21.15.64:0/1533392156 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faee00b46e0 con 0x7faee00720c0 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.177+0000 7faee4b2d640 1 -- 172.21.15.64:0/1533392156 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2477249110 0 0) 0x7faee00b3e80 con 0x7faee00afb30 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.177+0000 7faee6330640 1 --2- 172.21.15.64:0/1533392156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee00cdf90 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7faed8009e50 tx=0x7faed802bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=75d3b7506602efe0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee4b2d640 1 -- 172.21.15.64:0/1533392156 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faed802cb00 con 0x7faee00720c0 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee4b2d640 1 -- 172.21.15.64:0/1533392156 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faed802cca0 con 0x7faee00720c0 2024-03-04T07:30:01.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee4b2d640 1 -- 172.21.15.64:0/1533392156 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faed8034610 con 0x7faee00720c0 2024-03-04T07:30:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 -- 172.21.15.64:0/1533392156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 msgr2=0x7faee00cdf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 --2- 172.21.15.64:0/1533392156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee00cdf90 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7faed8009e50 tx=0x7faed802bf60 comp rx=0 tx=0).stop 2024-03-04T07:30:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 -- 172.21.15.64:0/1533392156 shutdown_connections 2024-03-04T07:30:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 --2- 172.21.15.64:0/1533392156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee00cdf90 unknown :-1 s=CLOSED pgs=812 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 --2- 172.21.15.64:0/1533392156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee00b3940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 -- 172.21.15.64:0/1533392156 >> 172.21.15.64:0/1533392156 conn(0x7faee00d0990 msgr2=0x7faee00d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:01.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 -- 172.21.15.64:0/1533392156 shutdown_connections 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.178+0000 7faee7d93640 1 -- 172.21.15.64:0/1533392156 wait complete. 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.179+0000 7faee7d93640 1 Processor -- start 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee7d93640 1 -- start start 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee7d93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee014ab30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee7d93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faee00afb30 0x7faee0159230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee7d93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee0159770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee7d93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faee00ce4d0 con 0x7faee00720c0 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee7d93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faee00b4ca0 con 0x7faee00afb30 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee7d93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faee00ce650 con 0x7faee015ac30 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee5b2f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee014ab30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee5b2f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee014ab30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43380/0 (socket says 172.21.15.64:43380) 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.180+0000 7faee5b2f640 1 -- 172.21.15.64:0/198082531 learned_addr learned my addr 172.21.15.64:0/198082531 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee6330640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee0159770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee5b2f640 1 -- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 msgr2=0x7faee0159770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee5b2f640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee0159770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee5b2f640 1 -- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faee00afb30 msgr2=0x7faee0159230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee532e640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faee00afb30 0x7faee0159230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee5b2f640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faee00afb30 0x7faee0159230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee5b2f640 1 -- 172.21.15.64:0/198082531 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faee00ade50 con 0x7faee00720c0 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee5b2f640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee014ab30 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7faedc00e870 tx=0x7faedc00ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faed67fc640 1 -- 172.21.15.64:0/198082531 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faedc017070 con 0x7faee00720c0 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faed67fc640 1 -- 172.21.15.64:0/198082531 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faedc00c940 con 0x7faee00720c0 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faee00ae130 con 0x7faee00720c0 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faed67fc640 1 -- 172.21.15.64:0/198082531 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faedc00cd30 con 0x7faee00720c0 2024-03-04T07:30:01.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.181+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faee00ae6f0 con 0x7faee00720c0 2024-03-04T07:30:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.182+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faee0068700 con 0x7faee00720c0 2024-03-04T07:30:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.182+0000 7faed67fc640 1 -- 172.21.15.64:0/198082531 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faedc01a070 con 0x7faee00720c0 2024-03-04T07:30:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.183+0000 7faed67fc640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.183+0000 7faed67fc640 1 -- 172.21.15.64:0/198082531 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(62..62 src has 1..62) v4 ==== 4857+0+0 (secure 0 0 0) 0x7faedc08a040 con 0x7faee00720c0 2024-03-04T07:30:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.183+0000 7faee532e640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.183+0000 7faee532e640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7faee0167820 tx=0x7faed0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.187+0000 7faed67fc640 1 -- 172.21.15.64:0/198082531 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faedc0592a0 con 0x7faee00720c0 2024-03-04T07:30:01.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.434+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7faee00aea90 con 0x7faee00720c0 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.435+0000 7faed67fc640 1 -- 172.21.15.64:0/198082531 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v17) v1 ==== 80+0+5304 (secure 0 0 0) 0x7faedc058ba0 con 0x7faee00720c0 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:01.435 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.436 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:01.437 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:30:01.438 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.440 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.441 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.442 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow r", 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:30:01.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:30:01.449 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:01.449 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.449 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:01.449 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:30:01.449 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:30:01.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 msgr2=0x7faec406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7faee0167820 tx=0x7faed0009040 comp rx=0 tx=0).stop 2024-03-04T07:30:01.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 msgr2=0x7faee014ab30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee014ab30 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7faedc00e870 tx=0x7faedc00ed30 comp rx=0 tx=0).stop 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 shutdown_connections 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faec406b4d0 0x7faec406d990 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faee015ac30 0x7faee0159770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faee00afb30 0x7faee0159230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 --2- 172.21.15.64:0/198082531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faee00720c0 0x7faee014ab30 unknown :-1 s=CLOSED pgs=813 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 >> 172.21.15.64:0/198082531 conn(0x7faee00d0990 msgr2=0x7faee00719b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 shutdown_connections 2024-03-04T07:30:01.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.437+0000 7faee7d93640 1 -- 172.21.15.64:0/198082531 wait complete. 2024-03-04T07:30:01.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:01.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.553+0000 7f16b64c9640 1 Processor -- start 2024-03-04T07:30:01.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.554+0000 7f16b64c9640 1 -- start start 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.554+0000 7f16b64c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 0x7f16b00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.554+0000 7f16b64c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16b00cea30 con 0x7f16b00b9a10 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.554+0000 7f16b64c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16b00cf3c0 con 0x7f16b015df00 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.554+0000 7f16b64c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16b00cf0e0 con 0x7f16b00b95a0 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.554+0000 7f16b4a66640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f16b00b9a10 0x7f16b00d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56902/0 (socket says 172.21.15.64:56902) 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.554+0000 7f16b4a66640 1 -- 172.21.15.64:0/3736757411 learned_addr learned my addr 172.21.15.64:0/3736757411 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16af7fe640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 0x7f16b00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2657592358 0 0) 0x7f16b00cea30 con 0x7f16b00b9a10 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1698003520 con 0x7f16b00b9a10 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1692755200 0 0) 0x7f1698003520 con 0x7f16b00b9a10 2024-03-04T07:30:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 msgr2=0x7f16b00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 0x7f16b00b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 >> v1:172.21.15.66:6789/0 conn(0x7f16b015df00 legacy=0x7f16b00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16b00ce470 con 0x7f16b00b9a10 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f16a80019f0 con 0x7f16b00b9a10 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 >> v1:172.21.15.64:6789/0 conn(0x7f16b00b9a10 legacy=0x7f16b00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 0x7f16980039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1698003f60 0x7f1698004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698004d40 0x7f16980051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16b00cf0e0 con 0x7f1698003f60 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f169800baa0 con 0x7f1698004d40 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f169800bca0 con 0x7f16b00b95a0 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16af7fe640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1698003f60 0x7f1698004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16affff640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 0x7f16980039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16b4a66640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698004d40 0x7f16980051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16af7fe640 1 -- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 msgr2=0x7f16980039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16af7fe640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00b95a0 0x7f16980039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16af7fe640 1 -- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698004d40 msgr2=0x7f16980051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16af7fe640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698004d40 0x7f16980051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16af7fe640 1 -- 172.21.15.64:0/3736757411 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1698003520 con 0x7f1698003f60 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.555+0000 7f16b4a66640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698004d40 0x7f16980051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.556+0000 7f16af7fe640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1698003f60 0x7f1698004800 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f16a0009ac0 tx=0x7f1698005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=811b7f3a640fc4bd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.556+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16a0039070 con 0x7f1698003f60 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.556+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16a002c6b0 con 0x7f1698003f60 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.556+0000 7f16aeffd640 1 -- 172.21.15.64:0/3736757411 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16a0009690 con 0x7f1698003f60 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.557+0000 7f16b64c9640 1 -- 172.21.15.64:0/3736757411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1698003f60 msgr2=0x7f1698004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.557+0000 7f16b64c9640 1 --2- 172.21.15.64:0/3736757411 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1698003f60 0x7f1698004800 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f16a0009ac0 tx=0x7f1698005830 comp rx=0 tx=0).stop 2024-03-04T07:30:01.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.557+0000 7f16b4a66640 1 -- 172.21.15.64:0/3736757411 reap_dead start 2024-03-04T07:30:01.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.557+0000 7f16b64c9640 1 -- 172.21.15.64:0/3736757411 shutdown_connections 2024-03-04T07:30:01.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.557+0000 7f16b64c9640 1 -- 172.21.15.64:0/3736757411 >> 172.21.15.64:0/3736757411 conn(0x7f16b0071960 msgr2=0x7f16b0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:01.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.557+0000 7f16b64c9640 1 -- 172.21.15.64:0/3736757411 shutdown_connections 2024-03-04T07:30:01.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.557+0000 7f16b64c9640 1 -- 172.21.15.64:0/3736757411 wait complete. 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.558+0000 7f16b64c9640 1 Processor -- start 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.558+0000 7f16b64c9640 1 -- start start 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b64c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698003f60 0x7f16b00c8f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b64c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00c79c0 0x7f16b01550c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b64c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b00cd530 0x7f16b00b4c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b64c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16b00cedf0 con 0x7f16b00cd530 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b64c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16b00cec70 con 0x7f1698003f60 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b64c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16b00d3930 con 0x7f16b00c79c0 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b00cd530 0x7f16b00b4c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b00cd530 0x7f16b00b4c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43394/0 (socket says 172.21.15.64:43394) 2024-03-04T07:30:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 -- 172.21.15.64:0/2510952494 learned_addr learned my addr 172.21.15.64:0/2510952494 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16affff640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698003f60 0x7f16b00c8f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16af7fe640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00c79c0 0x7f16b01550c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 -- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00c79c0 msgr2=0x7f16b01550c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00c79c0 0x7f16b01550c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 -- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698003f60 msgr2=0x7f16b00c8f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698003f60 0x7f16b00c8f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.559+0000 7f16b4a66640 1 -- 172.21.15.64:0/2510952494 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16b00c7ea0 con 0x7f16b00cd530 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.560+0000 7f16b4a66640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b00cd530 0x7f16b00b4c80 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f16b00d0f40 tx=0x7f16a800e530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.560+0000 7f16acff9640 1 -- 172.21.15.64:0/2510952494 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16a800ed40 con 0x7f16b00cd530 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.560+0000 7f16acff9640 1 -- 172.21.15.64:0/2510952494 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16a8014430 con 0x7f16b00cd530 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.560+0000 7f16acff9640 1 -- 172.21.15.64:0/2510952494 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16a8014820 con 0x7f16b00cd530 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.560+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16b00c8180 con 0x7f16b00cd530 2024-03-04T07:30:01.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.560+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16b00c8740 con 0x7f16b00cd530 2024-03-04T07:30:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.561+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16b0067ff0 con 0x7f16b00cd530 2024-03-04T07:30:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.561+0000 7f16acff9640 1 -- 172.21.15.64:0/2510952494 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f16a80149c0 con 0x7f16b00cd530 2024-03-04T07:30:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.562+0000 7f16acff9640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f169c06b500 0x7f169c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.562+0000 7f16acff9640 1 -- 172.21.15.64:0/2510952494 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(62..62 src has 1..62) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f16a8089de0 con 0x7f16b00cd530 2024-03-04T07:30:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.562+0000 7f16affff640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f169c06b500 0x7f169c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.562+0000 7f16affff640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f169c06b500 0x7f169c06d9c0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f16a40035f0 tx=0x7f16a4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.566+0000 7f16acff9640 1 -- 172.21.15.64:0/2510952494 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f16a8058fb0 con 0x7f16b00cd530 2024-03-04T07:30:01.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.750+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f16b0071ea0 con 0x7f16b00cd530 2024-03-04T07:30:01.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.751+0000 7f16acff9640 1 -- 172.21.15.64:0/2510952494 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v62) v1 ==== 74+0+12843 (secure 0 0 0) 0x7f16a80588b0 con 0x7f16b00cd530 2024-03-04T07:30:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:01.750 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":62,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:01.054242+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":9,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:01.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.752+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f169c06b500 msgr2=0x7f169c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.752+0000 7f16b64c9640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f169c06b500 0x7f169c06d9c0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f16a40035f0 tx=0x7f16a4009040 comp rx=0 tx=0).stop 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.752+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b00cd530 msgr2=0x7f16b00b4c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.752+0000 7f16b64c9640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b00cd530 0x7f16b00b4c80 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f16b00d0f40 tx=0x7f16a800e530 comp rx=0 tx=0).stop 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.752+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 shutdown_connections 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.753+0000 7f16b64c9640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f169c06b500 0x7f169c06d9c0 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.753+0000 7f16b64c9640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b00cd530 0x7f16b00b4c80 unknown :-1 s=CLOSED pgs=816 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.753+0000 7f16b64c9640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b00c79c0 0x7f16b01550c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.753+0000 7f16b64c9640 1 --2- 172.21.15.64:0/2510952494 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1698003f60 0x7f16b00c8f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.753+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 >> 172.21.15.64:0/2510952494 conn(0x7f16b0071960 msgr2=0x7f16b00b79c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.753+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 shutdown_connections 2024-03-04T07:30:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.753+0000 7f16b64c9640 1 -- 172.21.15.64:0/2510952494 wait complete. 2024-03-04T07:30:01.763 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:30:01.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:30:01.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.869+0000 7efecd4c9640 1 Processor -- start 2024-03-04T07:30:01.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.869+0000 7efecd4c9640 1 -- start start 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efecd4c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efec80b4830 0x7efec8160280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efecd4c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efec80b29a0 0x7efec80b2db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efecd4c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efec8070980 con 0x7efec8142170 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efecd4c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efec81458e0 con 0x7efec80b29a0 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efecd4c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efec81607c0 con 0x7efec80b4830 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec77fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efec80b29a0 0x7efec80b2db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec77fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efec80b29a0 0x7efec80b2db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43400/0 (socket says 172.21.15.64:43400) 2024-03-04T07:30:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec77fe640 1 -- 172.21.15.64:0/3205795280 learned_addr learned my addr 172.21.15.64:0/3205795280 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec67fc640 1 --2- 172.21.15.64:0/3205795280 >> v2:172.21.15.66:3300/0 conn(0x7efec80b4830 0x7efec8160280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec77fe640 1 -- 172.21.15.64:0/3205795280 >> v2:172.21.15.66:3300/0 conn(0x7efec80b4830 msgr2=0x7efec8160280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec77fe640 1 --2- 172.21.15.64:0/3205795280 >> v2:172.21.15.66:3300/0 conn(0x7efec80b4830 0x7efec8160280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec77fe640 1 -- 172.21.15.64:0/3205795280 >> v1:172.21.15.66:6789/0 conn(0x7efec8142170 legacy=0x7efec8142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.870+0000 7efec77fe640 1 -- 172.21.15.64:0/3205795280 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efec80b32f0 con 0x7efec80b29a0 2024-03-04T07:30:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.871+0000 7efec77fe640 1 --2- 172.21.15.64:0/3205795280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec80b2db0 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7efec80af240 tx=0x7efec002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d5619e50a251188 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.871+0000 7efec5ffb640 1 -- 172.21.15.64:0/3205795280 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec002ca80 con 0x7efec80b29a0 2024-03-04T07:30:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.871+0000 7efec5ffb640 1 -- 172.21.15.64:0/3205795280 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efec002cc20 con 0x7efec80b29a0 2024-03-04T07:30:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.871+0000 7efec5ffb640 1 -- 172.21.15.64:0/3205795280 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec00343f0 con 0x7efec80b29a0 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.871+0000 7efecd4c9640 1 -- 172.21.15.64:0/3205795280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 msgr2=0x7efec80b2db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.871+0000 7efecd4c9640 1 --2- 172.21.15.64:0/3205795280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec80b2db0 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7efec80af240 tx=0x7efec002bee0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.872+0000 7efecd4c9640 1 -- 172.21.15.64:0/3205795280 shutdown_connections 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.872+0000 7efecd4c9640 1 --2- 172.21.15.64:0/3205795280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec80b2db0 unknown :-1 s=CLOSED pgs=817 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.872+0000 7efecd4c9640 1 --2- 172.21.15.64:0/3205795280 >> v2:172.21.15.66:3300/0 conn(0x7efec80b4830 0x7efec8160280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.872+0000 7efecd4c9640 1 -- 172.21.15.64:0/3205795280 >> 172.21.15.64:0/3205795280 conn(0x7efec8066d00 msgr2=0x7efec8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.872+0000 7efecd4c9640 1 -- 172.21.15.64:0/3205795280 shutdown_connections 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.872+0000 7efecd4c9640 1 -- 172.21.15.64:0/3205795280 wait complete. 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.873+0000 7efecd4c9640 1 Processor -- start 2024-03-04T07:30:01.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.873+0000 7efecd4c9640 1 -- start start 2024-03-04T07:30:01.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec815e560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efec80b4830 0x7efec815b0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efec8142170 0x7efec815b5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec80711b0 con 0x7efec80b29a0 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec8070ed0 con 0x7efec80b4830 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efec80b4df0 con 0x7efec8142170 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec815e560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec815e560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43412/0 (socket says 172.21.15.64:43412) 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 -- 172.21.15.64:0/2938730216 learned_addr learned my addr 172.21.15.64:0/2938730216 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec67fc640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efec80b4830 0x7efec815b0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec77fe640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efec8142170 0x7efec815b5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 -- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efec8142170 msgr2=0x7efec815b5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efec8142170 0x7efec815b5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 -- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efec80b4830 msgr2=0x7efec815b0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efec80b4830 0x7efec815b0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 -- 172.21.15.64:0/2938730216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efec81f27c0 con 0x7efec80b29a0 2024-03-04T07:30:01.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efec6ffd640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec815e560 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7efeb800c3c0 tx=0x7efeb800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efeaf7fe640 1 -- 172.21.15.64:0/2938730216 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb80052d0 con 0x7efec80b29a0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efeaf7fe640 1 -- 172.21.15.64:0/2938730216 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efeb8005490 con 0x7efec80b29a0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efec81f2aa0 con 0x7efec80b29a0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efeaf7fe640 1 -- 172.21.15.64:0/2938730216 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb8007260 con 0x7efec80b29a0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.874+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efec81f3060 con 0x7efec80b29a0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.875+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efec80733c0 con 0x7efec80b29a0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.876+0000 7efeaf7fe640 1 -- 172.21.15.64:0/2938730216 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efeb8007400 con 0x7efec80b29a0 2024-03-04T07:30:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.876+0000 7efeaf7fe640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efeb406b500 0x7efeb406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.876+0000 7efeaf7fe640 1 -- 172.21.15.64:0/2938730216 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(62..62 src has 1..62) v4 ==== 4857+0+0 (secure 0 0 0) 0x7efeb8089340 con 0x7efec80b29a0 2024-03-04T07:30:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.876+0000 7efec67fc640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efeb406b500 0x7efeb406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.877+0000 7efec67fc640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efeb406b500 0x7efeb406d9c0 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7efec80afaa0 tx=0x7efebc008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:01.880+0000 7efeaf7fe640 1 -- 172.21.15.64:0/2938730216 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efeb8058510 con 0x7efec80b29a0 2024-03-04T07:30:02.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.064+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7efec8067420 con 0x7efec80b29a0 2024-03-04T07:30:02.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.064+0000 7efeaf7fe640 1 -- 172.21.15.64:0/2938730216 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v62) v1 ==== 93+0+10 (secure 0 0 0) 0x7efeb8057e10 con 0x7efec80b29a0 2024-03-04T07:30:02.064 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efeb406b500 msgr2=0x7efeb406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efeb406b500 0x7efeb406d9c0 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7efec80afaa0 tx=0x7efebc008040 comp rx=0 tx=0).stop 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 msgr2=0x7efec815e560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec815e560 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7efeb800c3c0 tx=0x7efeb800c880 comp rx=0 tx=0).stop 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 shutdown_connections 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efeb406b500 0x7efeb406d9c0 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efec8142170 0x7efec815b5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efec80b4830 0x7efec815b0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 --2- 172.21.15.64:0/2938730216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efec80b29a0 0x7efec815e560 unknown :-1 s=CLOSED pgs=818 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 >> 172.21.15.64:0/2938730216 conn(0x7efec8066d00 msgr2=0x7efec8144460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 shutdown_connections 2024-03-04T07:30:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.067+0000 7efecd4c9640 1 -- 172.21.15.64:0/2938730216 wait complete. 2024-03-04T07:30:02.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:30:02.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.179+0000 7f522893e640 1 Processor -- start 2024-03-04T07:30:02.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.179+0000 7f522893e640 1 -- start start 2024-03-04T07:30:02.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.180+0000 7f522893e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f522415dca0 0x7f522406fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.180+0000 7f522893e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240b00c0 0x7f52240b3ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.180+0000 7f522893e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5224161c20 con 0x7f522415e080 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.180+0000 7f522893e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f52241619d0 con 0x7f522415dca0 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.180+0000 7f522893e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5224161f00 con 0x7f52240b00c0 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5222d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240b00c0 0x7f52240b3ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f522259c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f522415dca0 0x7f522406fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5221d9b640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f522415e080 0x7f5224066d00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:33674/0 (socket says 172.21.15.64:33674) 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5222d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240b00c0 0x7f52240b3ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:44188/0 (socket says 172.21.15.64:44188) 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5221d9b640 1 -- 172.21.15.64:0/598371518 learned_addr learned my addr 172.21.15.64:0/598371518 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f522259c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f522415dca0 0x7f522406fd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:57576/0 (socket says 172.21.15.64:57576) 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5222d9d640 1 -- 172.21.15.64:0/598371518 >> v2:172.21.15.66:3300/0 conn(0x7f522415dca0 msgr2=0x7f522406fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5222d9d640 1 --2- 172.21.15.64:0/598371518 >> v2:172.21.15.66:3300/0 conn(0x7f522415dca0 0x7f522406fd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5222d9d640 1 -- 172.21.15.64:0/598371518 >> v1:172.21.15.66:6789/0 conn(0x7f522415e080 legacy=0x7f5224066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f5222d9d640 1 -- 172.21.15.64:0/598371518 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f52241621e0 con 0x7f52240b00c0 2024-03-04T07:30:02.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.181+0000 7f522259c640 1 --2- 172.21.15.64:0/598371518 >> v2:172.21.15.66:3300/0 conn(0x7f522415dca0 0x7f522406fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f5222d9d640 1 --2- 172.21.15.64:0/598371518 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240b00c0 0x7f52240b3ca0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f5214009d90 tx=0x7f521402bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7f451ea528ab573 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522159a640 1 -- 172.21.15.64:0/598371518 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5214039070 con 0x7f52240b00c0 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522159a640 1 -- 172.21.15.64:0/598371518 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f521402cc50 con 0x7f52240b00c0 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522159a640 1 -- 172.21.15.64:0/598371518 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5214034440 con 0x7f52240b00c0 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522893e640 1 -- 172.21.15.64:0/598371518 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240b00c0 msgr2=0x7f52240b3ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522893e640 1 --2- 172.21.15.64:0/598371518 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240b00c0 0x7f52240b3ca0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f5214009d90 tx=0x7f521402bea0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522893e640 1 -- 172.21.15.64:0/598371518 shutdown_connections 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522893e640 1 --2- 172.21.15.64:0/598371518 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240b00c0 0x7f52240b3ca0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522893e640 1 --2- 172.21.15.64:0/598371518 >> v2:172.21.15.66:3300/0 conn(0x7f522415dca0 0x7f522406fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.182+0000 7f522893e640 1 -- 172.21.15.64:0/598371518 >> 172.21.15.64:0/598371518 conn(0x7f52240d0990 msgr2=0x7f52240d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.183+0000 7f522893e640 1 -- 172.21.15.64:0/598371518 shutdown_connections 2024-03-04T07:30:02.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.183+0000 7f522893e640 1 -- 172.21.15.64:0/598371518 wait complete. 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.184+0000 7f522893e640 1 Processor -- start 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.184+0000 7f522893e640 1 -- start start 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.184+0000 7f522893e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52240b00c0 0x7f5224157a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522893e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f522415dca0 0x7f5224159d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522893e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240cd730 0x7f522415ffd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522893e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f52240703d0 con 0x7f52240b00c0 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522893e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5224070250 con 0x7f522415dca0 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522893e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5224067410 con 0x7f52240cd730 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52240b00c0 0x7f5224157a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52240b00c0 0x7f5224157a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43426/0 (socket says 172.21.15.64:43426) 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 -- 172.21.15.64:0/2447080123 learned_addr learned my addr 172.21.15.64:0/2447080123 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:02.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f5221d9b640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f522415dca0 0x7f5224159d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f5222d9d640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240cd730 0x7f522415ffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 -- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240cd730 msgr2=0x7f522415ffd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240cd730 0x7f522415ffd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 -- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f522415dca0 msgr2=0x7f5224159d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f522415dca0 0x7f5224159d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 -- 172.21.15.64:0/2447080123 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f522415f560 con 0x7f52240b00c0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f5222d9d640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240cd730 0x7f522415ffd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f5221d9b640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f522415dca0 0x7f5224159d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522259c640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52240b00c0 0x7f5224157a90 secure :-1 s=READY pgs=819 cs=0 l=1 rev1=1 crypto rx=0x7f521c00ec10 tx=0x7f521c00c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f521affd640 1 -- 172.21.15.64:0/2447080123 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f521c017070 con 0x7f52240b00c0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f521affd640 1 -- 172.21.15.64:0/2447080123 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f521c00cc30 con 0x7f52240b00c0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f522415a630 con 0x7f52240b00c0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f521affd640 1 -- 172.21.15.64:0/2447080123 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f521c013470 con 0x7f52240b00c0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.185+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f522415a910 con 0x7f52240b00c0 2024-03-04T07:30:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.186+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5224068700 con 0x7f52240b00c0 2024-03-04T07:30:02.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.187+0000 7f521affd640 1 -- 172.21.15.64:0/2447080123 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f521c01a070 con 0x7f52240b00c0 2024-03-04T07:30:02.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.187+0000 7f521affd640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f520406b4d0 0x7f520406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.187+0000 7f521affd640 1 -- 172.21.15.64:0/2447080123 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(62..62 src has 1..62) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f521c089e30 con 0x7f52240b00c0 2024-03-04T07:30:02.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.187+0000 7f5221d9b640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f520406b4d0 0x7f520406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.188+0000 7f5221d9b640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f520406b4d0 0x7f520406d990 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f5224161270 tx=0x7f520c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.191+0000 7f521affd640 1 -- 172.21.15.64:0/2447080123 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f521c059000 con 0x7f52240b00c0 2024-03-04T07:30:02.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.376+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f522406fe00 con 0x7f52240b00c0 2024-03-04T07:30:02.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.376+0000 7f521affd640 1 -- 172.21.15.64:0/2447080123 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v62) v1 ==== 84+0+65 (secure 0 0 0) 0x7f521c058900 con 0x7f52240b00c0 2024-03-04T07:30:02.376 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:02.376 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:30:02.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:02.376 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:30:02.376 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:30:02.376 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:02.376 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f520406b4d0 msgr2=0x7f520406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f520406b4d0 0x7f520406d990 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f5224161270 tx=0x7f520c009040 comp rx=0 tx=0).stop 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52240b00c0 msgr2=0x7f5224157a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52240b00c0 0x7f5224157a90 secure :-1 s=READY pgs=819 cs=0 l=1 rev1=1 crypto rx=0x7f521c00ec10 tx=0x7f521c00c520 comp rx=0 tx=0).stop 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 shutdown_connections 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f520406b4d0 0x7f520406d990 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f52240cd730 0x7f522415ffd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f522415dca0 0x7f5224159d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 --2- 172.21.15.64:0/2447080123 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52240b00c0 0x7f5224157a90 unknown :-1 s=CLOSED pgs=819 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 >> 172.21.15.64:0/2447080123 conn(0x7f52240d0990 msgr2=0x7f52240ad8b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:02.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 shutdown_connections 2024-03-04T07:30:02.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.378+0000 7f522893e640 1 -- 172.21.15.64:0/2447080123 wait complete. 2024-03-04T07:30:02.389 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:30:02.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:30:02.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.494+0000 7fd3f5f5e640 1 Processor -- start 2024-03-04T07:30:02.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.494+0000 7fd3f5f5e640 1 -- start start 2024-03-04T07:30:02.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3f5f5e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd3f01618e0 0x7fd3f006b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3f5f5e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd3f00ce960 0x7fd3f0162450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3f5f5e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd3f015f730 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3f5f5e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3f00cfce0 con 0x7fd3f01618e0 2024-03-04T07:30:02.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3f5f5e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3f015ebb0 con 0x7fd3f00ce960 2024-03-04T07:30:02.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3ef7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd3f01618e0 0x7fd3f006b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3eeffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd3f00ce4f0 0x7fd3f0161d40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56906/0 (socket says 172.21.15.64:56906) 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3ef7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd3f01618e0 0x7fd3f006b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43434/0 (socket says 172.21.15.64:43434) 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3eeffd640 1 -- 172.21.15.64:0/3647969193 learned_addr learned my addr 172.21.15.64:0/3647969193 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.495+0000 7fd3effff640 1 --2- 172.21.15.64:0/3647969193 >> v2:172.21.15.66:3300/0 conn(0x7fd3f00ce960 0x7fd3f0162450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ef7fe640 1 -- 172.21.15.64:0/3647969193 >> v2:172.21.15.66:3300/0 conn(0x7fd3f00ce960 msgr2=0x7fd3f0162450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ef7fe640 1 --2- 172.21.15.64:0/3647969193 >> v2:172.21.15.66:3300/0 conn(0x7fd3f00ce960 0x7fd3f0162450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ef7fe640 1 -- 172.21.15.64:0/3647969193 >> v1:172.21.15.64:6789/0 conn(0x7fd3f00ce4f0 legacy=0x7fd3f0161d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ef7fe640 1 -- 172.21.15.64:0/3647969193 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3f015ee90 con 0x7fd3f01618e0 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ee7fc640 1 -- 172.21.15.64:0/3647969193 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 988114144 0 0) 0x7fd3f015f730 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ef7fe640 1 --2- 172.21.15.64:0/3647969193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f01618e0 0x7fd3f006b130 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7fd3f00723c0 tx=0x7fd3e002b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2c19e0fafd9c2a17 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ee7fc640 1 -- 172.21.15.64:0/3647969193 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e0034070 con 0x7fd3f01618e0 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ee7fc640 1 -- 172.21.15.64:0/3647969193 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3e0030610 con 0x7fd3f01618e0 2024-03-04T07:30:02.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.496+0000 7fd3ee7fc640 1 -- 172.21.15.64:0/3647969193 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e00309e0 con 0x7fd3f01618e0 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/3647969193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f01618e0 msgr2=0x7fd3f006b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/3647969193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f01618e0 0x7fd3f006b130 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7fd3f00723c0 tx=0x7fd3e002b960 comp rx=0 tx=0).stop 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/3647969193 shutdown_connections 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/3647969193 >> v2:172.21.15.66:3300/0 conn(0x7fd3f00ce960 0x7fd3f0162450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/3647969193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f01618e0 0x7fd3f006b130 unknown :-1 s=CLOSED pgs=821 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/3647969193 >> 172.21.15.64:0/3647969193 conn(0x7fd3f00b3660 msgr2=0x7fd3f00b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/3647969193 shutdown_connections 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.497+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/3647969193 wait complete. 2024-03-04T07:30:02.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.498+0000 7fd3f5f5e640 1 Processor -- start 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.498+0000 7fd3f5f5e640 1 -- start start 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3f5f5e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f00ce4f0 0x7fd3f0151ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3f5f5e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd3f00ce960 0x7fd3f00c6c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3f5f5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3f00c71a0 0x7fd3f00b4770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3f5f5e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f006b670 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3f5f5e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f015f450 con 0x7fd3f00ce960 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3f5f5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3f00cedb0 con 0x7fd3f00c71a0 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3ef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f00ce4f0 0x7fd3f0151ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3ef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f00ce4f0 0x7fd3f0151ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43442/0 (socket says 172.21.15.64:43442) 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3ef7fe640 1 -- 172.21.15.64:0/215264877 learned_addr learned my addr 172.21.15.64:0/215264877 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3eeffd640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd3f00ce960 0x7fd3f00c6c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3ef7fe640 1 -- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3f00c71a0 msgr2=0x7fd3f00b4770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.499+0000 7fd3effff640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3f00c71a0 0x7fd3f00b4770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3ef7fe640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3f00c71a0 0x7fd3f00b4770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3ef7fe640 1 -- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd3f00ce960 msgr2=0x7fd3f00c6c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3ef7fe640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd3f00ce960 0x7fd3f00c6c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3ef7fe640 1 -- 172.21.15.64:0/215264877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3f00c7630 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3eeffd640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd3f00ce960 0x7fd3f00c6c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3ef7fe640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f00ce4f0 0x7fd3f0151ec0 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7fd3e0000c00 tx=0x7fd3e00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3d7fff640 1 -- 172.21.15.64:0/215264877 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e0034040 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3d7fff640 1 -- 172.21.15.64:0/215264877 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd3e003aaa0 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3f0152400 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3d7fff640 1 -- 172.21.15.64:0/215264877 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd3e00443f0 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.500+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3f00c7b90 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.501+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd3f009fc50 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.501+0000 7fd3d7fff640 1 -- 172.21.15.64:0/215264877 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd3e0037070 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.502+0000 7fd3d7fff640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd3dc06b4d0 0x7fd3dc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:02.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.502+0000 7fd3d7fff640 1 -- 172.21.15.64:0/215264877 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(62..62 src has 1..62) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fd3e00abec0 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.502+0000 7fd3eeffd640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd3dc06b4d0 0x7fd3dc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:02.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.502+0000 7fd3eeffd640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd3dc06b4d0 0x7fd3dc06d990 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7fd3f00cbaf0 tx=0x7fd3e4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:02.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.506+0000 7fd3d7fff640 1 -- 172.21.15.64:0/215264877 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd3f00c7630 con 0x7fd3f00ce4f0 2024-03-04T07:30:02.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:02.687+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7fd3f0072cb0 con 0x7fd3f00ce4f0 2024-03-04T07:30:03.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.057+0000 7fd3d7fff640 1 -- 172.21.15.64:0/215264877 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v63) v1 ==== 138+0+0 (secure 0 0 0) 0x7fd3e003f070 con 0x7fd3f00ce4f0 2024-03-04T07:30:03.057 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd3dc06b4d0 msgr2=0x7fd3dc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd3dc06b4d0 0x7fd3dc06d990 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7fd3f00cbaf0 tx=0x7fd3e4009040 comp rx=0 tx=0).stop 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f00ce4f0 msgr2=0x7fd3f0151ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f00ce4f0 0x7fd3f0151ec0 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7fd3e0000c00 tx=0x7fd3e00046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 shutdown_connections 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd3dc06b4d0 0x7fd3dc06d990 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3f00c71a0 0x7fd3f00b4770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd3f00ce960 0x7fd3f00c6c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 --2- 172.21.15.64:0/215264877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3f00ce4f0 0x7fd3f0151ec0 unknown :-1 s=CLOSED pgs=822 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 >> 172.21.15.64:0/215264877 conn(0x7fd3f00b3660 msgr2=0x7fd3f00cc870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:03.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 shutdown_connections 2024-03-04T07:30:03.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.060+0000 7fd3f5f5e640 1 -- 172.21.15.64:0/215264877 wait complete. 2024-03-04T07:30:03.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:30:03.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.183+0000 7f8e16da2640 1 Processor -- start 2024-03-04T07:30:03.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.183+0000 7f8e16da2640 1 -- start start 2024-03-04T07:30:03.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e16da2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100b6bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:03.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e16da2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8e1015df90 0x7f8e100b93e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:03.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e16da2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8e100b2850 0x7f8e100d0e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:03.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e16da2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e100b9ae0 con 0x7f8e100b2850 2024-03-04T07:30:03.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e16da2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e100b30c0 con 0x7f8e1015df90 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e16da2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e100d1c60 con 0x7f8e100d0a50 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8e100b2850 0x7f8e100d0e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8e100b2850 0x7f8e100d0e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43450/0 (socket says 172.21.15.64:43450) 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 -- 172.21.15.64:0/554035204 learned_addr learned my addr 172.21.15.64:0/554035204 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e1559f640 1 --2- 172.21.15.64:0/554035204 >> v2:172.21.15.66:3300/0 conn(0x7f8e1015df90 0x7f8e100b93e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e15da0640 1 --2- 172.21.15.64:0/554035204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100b6bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 -- 172.21.15.64:0/554035204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 msgr2=0x7f8e100b6bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 --2- 172.21.15.64:0/554035204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100b6bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 -- 172.21.15.64:0/554035204 >> v2:172.21.15.66:3300/0 conn(0x7f8e1015df90 msgr2=0x7f8e100b93e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 --2- 172.21.15.64:0/554035204 >> v2:172.21.15.66:3300/0 conn(0x7f8e1015df90 0x7f8e100b93e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e165a1640 1 -- 172.21.15.64:0/554035204 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e100d1370 con 0x7f8e100b2850 2024-03-04T07:30:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.184+0000 7f8e1559f640 1 --2- 172.21.15.64:0/554035204 >> v2:172.21.15.66:3300/0 conn(0x7f8e1015df90 0x7f8e100b93e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.185+0000 7f8e165a1640 1 --2- 172.21.15.64:0/554035204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e100d0e30 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7f8e1006b220 tx=0x7f8e0002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3b16d347c2b7aab9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.185+0000 7f8e14d9e640 1 -- 172.21.15.64:0/554035204 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e0002ca70 con 0x7f8e100b2850 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.185+0000 7f8e14d9e640 1 -- 172.21.15.64:0/554035204 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e0002cc10 con 0x7f8e100b2850 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.185+0000 7f8e14d9e640 1 -- 172.21.15.64:0/554035204 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e000343f0 con 0x7f8e100b2850 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 -- 172.21.15.64:0/554035204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 msgr2=0x7f8e100d0e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 --2- 172.21.15.64:0/554035204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e100d0e30 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7f8e1006b220 tx=0x7f8e0002bed0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 -- 172.21.15.64:0/554035204 shutdown_connections 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 --2- 172.21.15.64:0/554035204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e100d0e30 unknown :-1 s=CLOSED pgs=823 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 --2- 172.21.15.64:0/554035204 >> v2:172.21.15.66:3300/0 conn(0x7f8e1015df90 0x7f8e100b93e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 --2- 172.21.15.64:0/554035204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100b6bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 -- 172.21.15.64:0/554035204 >> 172.21.15.64:0/554035204 conn(0x7f8e10071960 msgr2=0x7f8e10071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 -- 172.21.15.64:0/554035204 shutdown_connections 2024-03-04T07:30:03.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.186+0000 7f8e16da2640 1 -- 172.21.15.64:0/554035204 wait complete. 2024-03-04T07:30:03.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.187+0000 7f8e16da2640 1 Processor -- start 2024-03-04T07:30:03.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.187+0000 7f8e16da2640 1 -- start start 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e16da2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e10151030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e16da2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100cbc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e16da2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e1015df90 0x7f8e100cd8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e16da2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e100d24c0 con 0x7f8e100b2850 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e16da2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e100d21e0 con 0x7f8e1015df90 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e16da2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e100d3a50 con 0x7f8e100d0a50 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e10151030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:03.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e10151030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43462/0 (socket says 172.21.15.64:43462) 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 -- 172.21.15.64:0/2823082687 learned_addr learned my addr 172.21.15.64:0/2823082687 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e165a1640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e1015df90 0x7f8e100cd8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e1559f640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100cbc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 -- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 msgr2=0x7f8e100cbc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100cbc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 -- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e1015df90 msgr2=0x7f8e100cd8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e1015df90 0x7f8e100cd8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 -- 172.21.15.64:0/2823082687 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e10151570 con 0x7f8e100b2850 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e1559f640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100cbc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.188+0000 7f8e15da0640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e10151030 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7f8e0800c820 tx=0x7f8e0800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.189+0000 7f8e0e7fc640 1 -- 172.21.15.64:0/2823082687 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e08015070 con 0x7f8e100b2850 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.189+0000 7f8e0e7fc640 1 -- 172.21.15.64:0/2823082687 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e08004240 con 0x7f8e100b2850 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.189+0000 7f8e0e7fc640 1 -- 172.21.15.64:0/2823082687 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e0800ce10 con 0x7f8e100b2850 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.189+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e100cc1b0 con 0x7f8e100b2850 2024-03-04T07:30:03.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.189+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e100cd0b0 con 0x7f8e100b2850 2024-03-04T07:30:03.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.190+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e1004f330 con 0x7f8e100b2850 2024-03-04T07:30:03.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.190+0000 7f8e0e7fc640 1 -- 172.21.15.64:0/2823082687 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8e080077d0 con 0x7f8e100b2850 2024-03-04T07:30:03.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.190+0000 7f8e0e7fc640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dfc06b4d0 0x7f8dfc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:03.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.191+0000 7f8e0e7fc640 1 -- 172.21.15.64:0/2823082687 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(63..63 src has 1..63) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f8e08089e40 con 0x7f8e100b2850 2024-03-04T07:30:03.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.191+0000 7f8e1559f640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dfc06b4d0 0x7f8dfc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:03.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.191+0000 7f8e1559f640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dfc06b4d0 0x7f8dfc06d990 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f8e04006fd0 tx=0x7f8e04008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:03.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.195+0000 7f8e0e7fc640 1 -- 172.21.15.64:0/2823082687 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8e08058f30 con 0x7f8e100b2850 2024-03-04T07:30:03.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:03.376+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f8e100b32d0 con 0x7f8e100b2850 2024-03-04T07:30:04.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.064+0000 7f8e0e7fc640 1 -- 172.21.15.64:0/2823082687 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v64) v1 ==== 170+0+0 (secure 0 0 0) 0x7f8e08058830 con 0x7f8e100b2850 2024-03-04T07:30:04.064 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:30:04.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dfc06b4d0 msgr2=0x7f8dfc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:04.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dfc06b4d0 0x7f8dfc06d990 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f8e04006fd0 tx=0x7f8e04008040 comp rx=0 tx=0).stop 2024-03-04T07:30:04.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 msgr2=0x7f8e10151030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:04.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e10151030 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7f8e0800c820 tx=0x7f8e0800cce0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 shutdown_connections 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dfc06b4d0 0x7f8dfc06d990 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e1015df90 0x7f8e100cd8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e100d0a50 0x7f8e100cbc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 --2- 172.21.15.64:0/2823082687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e100b2850 0x7f8e10151030 unknown :-1 s=CLOSED pgs=824 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 >> 172.21.15.64:0/2823082687 conn(0x7f8e10071960 msgr2=0x7f8e100ce500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 shutdown_connections 2024-03-04T07:30:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.067+0000 7f8e16da2640 1 -- 172.21.15.64:0/2823082687 wait complete. 2024-03-04T07:30:04.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:30:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.201+0000 7fd861839640 1 Processor -- start 2024-03-04T07:30:04.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.201+0000 7fd861839640 1 -- start start 2024-03-04T07:30:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd861839640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd85c15f0b0 0x7fd85c072100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd861839640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd85c143670 con 0x7fd85c066d00 2024-03-04T07:30:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd861839640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd85c1438e0 con 0x7fd85c0b3a90 2024-03-04T07:30:04.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd861839640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd85c143bc0 con 0x7fd85c15f0b0 2024-03-04T07:30:04.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd85b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd85c15f0b0 0x7fd85c072100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:04.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd85affd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd85c066d00 0x7fd85c0670e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45674/0 (socket says 172.21.15.64:45674) 2024-03-04T07:30:04.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd85affd640 1 -- 172.21.15.64:0/1712406407 learned_addr learned my addr 172.21.15.64:0/1712406407 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:04.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.202+0000 7fd85b7fe640 1 --2- 172.21.15.64:0/1712406407 >> v2:172.21.15.64:3300/0 conn(0x7fd85c15f0b0 0x7fd85c072100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49212/0 (socket says 172.21.15.64:49212) 2024-03-04T07:30:04.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.203+0000 7fd85b7fe640 1 -- 172.21.15.64:0/1712406407 >> v1:172.21.15.66:6789/0 conn(0x7fd85c0b3a90 legacy=0x7fd85c0719f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:04.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.203+0000 7fd85b7fe640 1 -- 172.21.15.64:0/1712406407 >> v1:172.21.15.64:6789/0 conn(0x7fd85c066d00 legacy=0x7fd85c0670e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:04.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.203+0000 7fd85b7fe640 1 -- 172.21.15.64:0/1712406407 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd85c143ea0 con 0x7fd85c15f0b0 2024-03-04T07:30:04.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.203+0000 7fd85b7fe640 1 --2- 172.21.15.64:0/1712406407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c072100 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fd84c009d90 tx=0x7fd84c02bc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb6d3f2cf4eedfd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:04.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd859ffb640 1 -- 172.21.15.64:0/1712406407 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd84c039070 con 0x7fd85c15f0b0 2024-03-04T07:30:04.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd859ffb640 1 -- 172.21.15.64:0/1712406407 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd84c02c990 con 0x7fd85c15f0b0 2024-03-04T07:30:04.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd859ffb640 1 -- 172.21.15.64:0/1712406407 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd84c02cd60 con 0x7fd85c15f0b0 2024-03-04T07:30:04.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd861839640 1 -- 172.21.15.64:0/1712406407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 msgr2=0x7fd85c072100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:04.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd861839640 1 --2- 172.21.15.64:0/1712406407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c072100 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fd84c009d90 tx=0x7fd84c02bc60 comp rx=0 tx=0).stop 2024-03-04T07:30:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd861839640 1 -- 172.21.15.64:0/1712406407 shutdown_connections 2024-03-04T07:30:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd861839640 1 --2- 172.21.15.64:0/1712406407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c072100 unknown :-1 s=CLOSED pgs=826 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd861839640 1 -- 172.21.15.64:0/1712406407 >> 172.21.15.64:0/1712406407 conn(0x7fd85c0d0a60 msgr2=0x7fd85c0d0e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd861839640 1 -- 172.21.15.64:0/1712406407 shutdown_connections 2024-03-04T07:30:04.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.204+0000 7fd861839640 1 -- 172.21.15.64:0/1712406407 wait complete. 2024-03-04T07:30:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.205+0000 7fd861839640 1 Processor -- start 2024-03-04T07:30:04.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd861839640 1 -- start start 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd861839640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd85c066d00 0x7fd85c154e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd861839640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd85c0b3a90 0x7fd85c15e820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd861839640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c0b1710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd861839640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd85c144540 con 0x7fd85c15f0b0 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd861839640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd85c1443c0 con 0x7fd85c066d00 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd861839640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd85c0b4120 con 0x7fd85c0b3a90 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd85b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c0b1710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd85b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c0b1710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49216/0 (socket says 172.21.15.64:49216) 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.206+0000 7fd85b7fe640 1 -- 172.21.15.64:0/765453957 learned_addr learned my addr 172.21.15.64:0/765453957 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85affd640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd85c066d00 0x7fd85c154e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85a7fc640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd85c0b3a90 0x7fd85c15e820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85b7fe640 1 -- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd85c0b3a90 msgr2=0x7fd85c15e820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:04.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85b7fe640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd85c0b3a90 0x7fd85c15e820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85b7fe640 1 -- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd85c066d00 msgr2=0x7fd85c154e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85b7fe640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd85c066d00 0x7fd85c154e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85b7fe640 1 -- 172.21.15.64:0/765453957 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd85c1553a0 con 0x7fd85c15f0b0 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85affd640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd85c066d00 0x7fd85c154e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd85b7fe640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c0b1710 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7fd84c009eb0 tx=0x7fd84c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd853fff640 1 -- 172.21.15.64:0/765453957 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd84c044020 con 0x7fd85c15f0b0 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd853fff640 1 -- 172.21.15.64:0/765453957 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd84c034cf0 con 0x7fd85c15f0b0 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd85c1e65f0 con 0x7fd85c15f0b0 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd853fff640 1 -- 172.21.15.64:0/765453957 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd84c039040 con 0x7fd85c15f0b0 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.207+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd85c1e6bb0 con 0x7fd85c15f0b0 2024-03-04T07:30:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.208+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd85c073430 con 0x7fd85c15f0b0 2024-03-04T07:30:04.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.209+0000 7fd853fff640 1 -- 172.21.15.64:0/765453957 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd84c047020 con 0x7fd85c15f0b0 2024-03-04T07:30:04.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.209+0000 7fd853fff640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd83c06b4d0 0x7fd83c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:04.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.209+0000 7fd853fff640 1 -- 172.21.15.64:0/765453957 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(64..64 src has 1..64) v4 ==== 5609+0+0 (secure 0 0 0) 0x7fd84c0ac8f0 con 0x7fd85c15f0b0 2024-03-04T07:30:04.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.209+0000 7fd85affd640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd83c06b4d0 0x7fd83c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:04.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.210+0000 7fd85affd640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd83c06b4d0 0x7fd83c06d990 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fd854000f60 tx=0x7fd854007d20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:04.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.214+0000 7fd853fff640 1 -- 172.21.15.64:0/765453957 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd84c04e030 con 0x7fd85c15f0b0 2024-03-04T07:30:04.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:04.436+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7fd85c073430 con 0x7fd85c15f0b0 2024-03-04T07:30:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.065+0000 7fd853fff640 1 -- 172.21.15.64:0/765453957 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 10 and data pool 11 v34) v1 ==== 174+0+0 (secure 0 0 0) 0x7fd84c07b240 con 0x7fd85c15f0b0 2024-03-04T07:30:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 10 and data pool 11 2024-03-04T07:30:05.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd83c06b4d0 msgr2=0x7fd83c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd83c06b4d0 0x7fd83c06d990 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fd854000f60 tx=0x7fd854007d20 comp rx=0 tx=0).stop 2024-03-04T07:30:05.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 msgr2=0x7fd85c0b1710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c0b1710 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7fd84c009eb0 tx=0x7fd84c004a40 comp rx=0 tx=0).stop 2024-03-04T07:30:05.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 shutdown_connections 2024-03-04T07:30:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd83c06b4d0 0x7fd83c06d990 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd85c15f0b0 0x7fd85c0b1710 unknown :-1 s=CLOSED pgs=827 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd85c0b3a90 0x7fd85c15e820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 --2- 172.21.15.64:0/765453957 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd85c066d00 0x7fd85c154e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 >> 172.21.15.64:0/765453957 conn(0x7fd85c0d0a60 msgr2=0x7fd85c06e770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.070+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 shutdown_connections 2024-03-04T07:30:05.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.071+0000 7fd861839640 1 -- 172.21.15.64:0/765453957 wait complete. 2024-03-04T07:30:05.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.202+0000 7f8f02284640 1 Processor -- start 2024-03-04T07:30:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.202+0000 7f8f02284640 1 -- start start 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8f02284640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8efc15e030 0x7f8efc06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8f02284640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc06ef70 0x7f8efc0b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8f02284640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8efc15e410 con 0x7f8efc0b2290 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8f02284640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8efc162700 con 0x7f8efc15e030 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8f02284640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8efc161af0 con 0x7f8efc06ef70 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8efb7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc06ef70 0x7f8efc0b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8efbfff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8efc15e030 0x7f8efc06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8f00821640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f8efc0b2290 0x7f8efc0d0040 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:57092/0 (socket says 172.21.15.64:57092) 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8efb7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc06ef70 0x7f8efc0b1d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:35474/0 (socket says 172.21.15.64:35474) 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.203+0000 7f8f00821640 1 -- 172.21.15.64:0/3766605635 learned_addr learned my addr 172.21.15.64:0/3766605635 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.204+0000 7f8efaffd640 1 -- 172.21.15.64:0/3766605635 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 315314301 0 0) 0x7f8ef4001490 con 0x7f8efc0b2290 2024-03-04T07:30:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.204+0000 7f8efaffd640 1 -- 172.21.15.64:0/3766605635 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8ee4003520 con 0x7f8efc0b2290 2024-03-04T07:30:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.204+0000 7f8efb7fe640 1 -- 172.21.15.64:0/3766605635 >> v2:172.21.15.66:3300/0 conn(0x7f8efc15e030 msgr2=0x7f8efc06ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.204+0000 7f8efb7fe640 1 --2- 172.21.15.64:0/3766605635 >> v2:172.21.15.66:3300/0 conn(0x7f8efc15e030 0x7f8efc06ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.204+0000 7f8efb7fe640 1 -- 172.21.15.64:0/3766605635 >> v1:172.21.15.66:6789/0 conn(0x7f8efc0b2290 legacy=0x7f8efc0d0040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.204+0000 7f8efb7fe640 1 -- 172.21.15.64:0/3766605635 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8efc161dd0 con 0x7f8efc06ef70 2024-03-04T07:30:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8efb7fe640 1 --2- 172.21.15.64:0/3766605635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc06ef70 0x7f8efc0b1d50 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f8eec0097a0 tx=0x7f8eec02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=40f911ca3c1acba2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8efaffd640 1 -- 172.21.15.64:0/3766605635 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8eec035020 con 0x7f8efc06ef70 2024-03-04T07:30:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8efaffd640 1 -- 172.21.15.64:0/3766605635 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8eec031830 con 0x7f8efc06ef70 2024-03-04T07:30:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8efaffd640 1 -- 172.21.15.64:0/3766605635 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8eec031be0 con 0x7f8efc06ef70 2024-03-04T07:30:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 -- 172.21.15.64:0/3766605635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc06ef70 msgr2=0x7f8efc0b1d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 --2- 172.21.15.64:0/3766605635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc06ef70 0x7f8efc0b1d50 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f8eec0097a0 tx=0x7f8eec02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 -- 172.21.15.64:0/3766605635 shutdown_connections 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 --2- 172.21.15.64:0/3766605635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc06ef70 0x7f8efc0b1d50 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 --2- 172.21.15.64:0/3766605635 >> v2:172.21.15.66:3300/0 conn(0x7f8efc15e030 0x7f8efc06ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 -- 172.21.15.64:0/3766605635 >> 172.21.15.64:0/3766605635 conn(0x7f8efc0b55e0 msgr2=0x7f8efc0b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 -- 172.21.15.64:0/3766605635 shutdown_connections 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.205+0000 7f8f02284640 1 -- 172.21.15.64:0/3766605635 wait complete. 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.206+0000 7f8f02284640 1 Processor -- start 2024-03-04T07:30:05.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8f02284640 1 -- start start 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8f02284640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8efc06ef70 0x7f8efc09cb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8f02284640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8efc0b2290 0x7f8efc09a1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8f02284640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc15e030 0x7f8efc0afc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8f02284640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8efc06e220 con 0x7f8efc0b2290 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8f02284640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8efc162390 con 0x7f8efc06ef70 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8f02284640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8efc06e3a0 con 0x7f8efc15e030 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8efb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8efc0b2290 0x7f8efc09a1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.207+0000 7f8efbfff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8efc06ef70 0x7f8efc09cb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8f00821640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc15e030 0x7f8efc0afc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8efc0b2290 0x7f8efc09a1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49222/0 (socket says 172.21.15.64:49222) 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efb7fe640 1 -- 172.21.15.64:0/319946632 learned_addr learned my addr 172.21.15.64:0/319946632 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efbfff640 1 -- 172.21.15.64:0/319946632 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc15e030 msgr2=0x7f8efc0afc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efbfff640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc15e030 0x7f8efc0afc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efbfff640 1 -- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8efc0b2290 msgr2=0x7f8efc09a1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efbfff640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8efc0b2290 0x7f8efc09a1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efbfff640 1 -- 172.21.15.64:0/319946632 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8efc09e1f0 con 0x7f8efc06ef70 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efb7fe640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8efc0b2290 0x7f8efc09a1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8efbfff640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8efc06ef70 0x7f8efc09cb00 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f8ef000e870 tx=0x7f8ef000ed30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8ef8ff9640 1 -- 172.21.15.64:0/319946632 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ef0017070 con 0x7f8efc06ef70 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8ef8ff9640 1 -- 172.21.15.64:0/319946632 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8ef000ee60 con 0x7f8efc06ef70 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8efc09d150 con 0x7f8efc06ef70 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8ef8ff9640 1 -- 172.21.15.64:0/319946632 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ef000cb60 con 0x7f8efc06ef70 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.208+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8efc09d9f0 con 0x7f8efc06ef70 2024-03-04T07:30:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.209+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8efc09d540 con 0x7f8efc06ef70 2024-03-04T07:30:05.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.210+0000 7f8ef8ff9640 1 -- 172.21.15.64:0/319946632 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8ef001a070 con 0x7f8efc06ef70 2024-03-04T07:30:05.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.210+0000 7f8ef8ff9640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ee806b4d0 0x7f8ee806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.210+0000 7f8ef8ff9640 1 -- 172.21.15.64:0/319946632 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(65..65 src has 1..65) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8ef0089f80 con 0x7f8efc06ef70 2024-03-04T07:30:05.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.210+0000 7f8efb7fe640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ee806b4d0 0x7f8ee806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.211+0000 7f8efb7fe640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ee806b4d0 0x7f8ee806d990 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f8eec02b910 tx=0x7f8eec008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.215+0000 7f8ef8ff9640 1 -- 172.21.15.64:0/319946632 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8ef00587a0 con 0x7f8efc06ef70 2024-03-04T07:30:05.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.396+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f8efc067ff0 con 0x7f8efc06ef70 2024-03-04T07:30:05.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.397+0000 7f8ef8ff9640 1 -- 172.21.15.64:0/319946632 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v65) v1 ==== 74+0+16633 (secure 0 0 0) 0x7f8ef005c840 con 0x7f8efc06ef70 2024-03-04T07:30:05.398 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:05.398 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":65,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:04.437689+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:02.706399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:03.395544+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:05.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ee806b4d0 msgr2=0x7f8ee806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ee806b4d0 0x7f8ee806d990 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f8eec02b910 tx=0x7f8eec008030 comp rx=0 tx=0).stop 2024-03-04T07:30:05.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8efc06ef70 msgr2=0x7f8efc09cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8efc06ef70 0x7f8efc09cb00 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f8ef000e870 tx=0x7f8ef000ed30 comp rx=0 tx=0).stop 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 shutdown_connections 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8ee806b4d0 0x7f8ee806d990 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8efc15e030 0x7f8efc0afc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8efc0b2290 0x7f8efc09a1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 --2- 172.21.15.64:0/319946632 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8efc06ef70 0x7f8efc09cb00 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.399+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 >> 172.21.15.64:0/319946632 conn(0x7f8efc0b55e0 msgr2=0x7f8efc0adbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.400+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 shutdown_connections 2024-03-04T07:30:05.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.400+0000 7f8f02284640 1 -- 172.21.15.64:0/319946632 wait complete. 2024-03-04T07:30:05.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:05.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.512+0000 7f1853647640 1 Processor -- start 2024-03-04T07:30:05.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.512+0000 7f1853647640 1 -- start start 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1853647640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f184c15e060 0x7f184c0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1853647640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f184c0d0dd0 con 0x7f184c0d0880 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1853647640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f184c0d10b0 con 0x7f184c06fe20 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1853647640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f184c0cc0e0 con 0x7f184c15e060 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1851be4640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f184c0d0880 0x7f184c0ccbc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45688/0 (socket says 172.21.15.64:45688) 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1851be4640 1 -- 172.21.15.64:0/2969476735 learned_addr learned my addr 172.21.15.64:0/2969476735 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2217720960 0 0) 0x7f184c0d0dd0 con 0x7f184c0d0880 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1838003520 con 0x7f184c0d0880 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f18513e3640 1 --2- 172.21.15.64:0/2969476735 >> v2:172.21.15.66:3300/0 conn(0x7f184c15e060 0x7f184c0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 48268259 0 0) 0x7f1838003520 con 0x7f184c0d0880 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 >> v2:172.21.15.66:3300/0 conn(0x7f184c15e060 msgr2=0x7f184c0ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 --2- 172.21.15.64:0/2969476735 >> v2:172.21.15.66:3300/0 conn(0x7f184c15e060 0x7f184c0ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 >> v1:172.21.15.66:6789/0 conn(0x7f184c06fe20 legacy=0x7f184c0ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f184c0cc3c0 con 0x7f184c0d0880 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f18440019f0 con 0x7f184c0d0880 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 >> v1:172.21.15.64:6789/0 conn(0x7f184c0d0880 legacy=0x7f184c0ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18380039b0 0x7f1838003db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.513+0000 7f1843fff640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f1838004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1843fff640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18380051c0 0x7f183800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f184c0cc0e0 con 0x7f1838004800 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f183800c0f0 con 0x7f18380039b0 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f183800c2f0 con 0x7f18380051c0 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f18513e3640 1 --2- 172.21.15.64:0/2969476735 >> v2:172.21.15.66:3300/0 conn(0x7f184c15e060 0x7f184c0ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1850be2640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f1838004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1851be4640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18380051c0 0x7f183800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f18513e3640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18380039b0 0x7f1838003db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1850be2640 1 -- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18380051c0 msgr2=0x7f183800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1850be2640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18380051c0 0x7f183800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1850be2640 1 -- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18380039b0 msgr2=0x7f1838003db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1850be2640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18380039b0 0x7f1838003db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1850be2640 1 -- 172.21.15.64:0/2969476735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1838003520 con 0x7f1838004800 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.514+0000 7f1851be4640 1 -- 172.21.15.64:0/2969476735 reap_dead start 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1850be2640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f1838004c80 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7f184c070270 tx=0x7f1838005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9b14ea4d07f38be0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f183c037070 con 0x7f1838004800 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f183c032600 con 0x7f1838004800 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1843fff640 1 -- 172.21.15.64:0/2969476735 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f183c003980 con 0x7f1838004800 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1853647640 1 -- 172.21.15.64:0/2969476735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 msgr2=0x7f1838004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1853647640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f1838004c80 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7f184c070270 tx=0x7f1838005830 comp rx=0 tx=0).stop 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1853647640 1 -- 172.21.15.64:0/2969476735 shutdown_connections 2024-03-04T07:30:05.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1853647640 1 --2- 172.21.15.64:0/2969476735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f1838004c80 unknown :-1 s=CLOSED pgs=829 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1853647640 1 -- 172.21.15.64:0/2969476735 >> 172.21.15.64:0/2969476735 conn(0x7f184c0d1a90 msgr2=0x7f184c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:05.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1853647640 1 -- 172.21.15.64:0/2969476735 shutdown_connections 2024-03-04T07:30:05.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.515+0000 7f1853647640 1 -- 172.21.15.64:0/2969476735 wait complete. 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 Processor -- start 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 -- start start 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f184c14edd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15f420 0x7f184c0b2130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b2670 0x7f184c0afe90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f184c0ce310 con 0x7f1838004800 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f184c15e440 con 0x7f184c15f420 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f1853647640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f184c0ce490 con 0x7f184c0b2670 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f18513e3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f184c14edd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f18513e3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f184c14edd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49246/0 (socket says 172.21.15.64:49246) 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.517+0000 7f18513e3640 1 -- 172.21.15.64:0/1729193564 learned_addr learned my addr 172.21.15.64:0/1729193564 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1850be2640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15f420 0x7f184c0b2130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1851be4640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b2670 0x7f184c0afe90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f18513e3640 1 -- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b2670 msgr2=0x7f184c0afe90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f18513e3640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b2670 0x7f184c0afe90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f18513e3640 1 -- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15f420 msgr2=0x7f184c0b2130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f18513e3640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15f420 0x7f184c0b2130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f18513e3640 1 -- 172.21.15.64:0/1729193564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f184c0b0bc0 con 0x7f1838004800 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1851be4640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b2670 0x7f184c0afe90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f18513e3640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f184c14edd0 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f1848010840 tx=0x7f1848010d00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1841ffb640 1 -- 172.21.15.64:0/1729193564 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f184800e030 con 0x7f1838004800 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1841ffb640 1 -- 172.21.15.64:0/1729193564 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1848004230 con 0x7f1838004800 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f184c15e660 con 0x7f1838004800 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1841ffb640 1 -- 172.21.15.64:0/1729193564 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f184800f040 con 0x7f1838004800 2024-03-04T07:30:05.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.518+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f184c15ec20 con 0x7f1838004800 2024-03-04T07:30:05.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.519+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f184c067ff0 con 0x7f1838004800 2024-03-04T07:30:05.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.520+0000 7f1841ffb640 1 -- 172.21.15.64:0/1729193564 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f184800c730 con 0x7f1838004800 2024-03-04T07:30:05.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.520+0000 7f1841ffb640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.520+0000 7f1841ffb640 1 -- 172.21.15.64:0/1729193564 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(65..65 src has 1..65) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f184808a780 con 0x7f1838004800 2024-03-04T07:30:05.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.520+0000 7f1850be2640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.521+0000 7f1850be2640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f183c034000 tx=0x7f183c009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.524+0000 7f1841ffb640 1 -- 172.21.15.64:0/1729193564 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1848059600 con 0x7f1838004800 2024-03-04T07:30:05.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.708+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f184c06f560 con 0x7f1838004800 2024-03-04T07:30:05.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.709+0000 7f1841ffb640 1 -- 172.21.15.64:0/1729193564 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v65) v1 ==== 74+0+16633 (secure 0 0 0) 0x7f1848058f00 con 0x7f1838004800 2024-03-04T07:30:05.709 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:05.709 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":65,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:04.437689+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:02.706399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:03.395544+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 msgr2=0x7f183006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f183c034000 tx=0x7f183c009030 comp rx=0 tx=0).stop 2024-03-04T07:30:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 msgr2=0x7f184c14edd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f184c14edd0 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f1848010840 tx=0x7f1848010d00 comp rx=0 tx=0).stop 2024-03-04T07:30:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 shutdown_connections 2024-03-04T07:30:05.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f183006b4d0 0x7f183006d990 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f184c0b2670 0x7f184c0afe90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f184c15f420 0x7f184c0b2130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 --2- 172.21.15.64:0/1729193564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1838004800 0x7f184c14edd0 unknown :-1 s=CLOSED pgs=830 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 >> 172.21.15.64:0/1729193564 conn(0x7f184c0d1a90 msgr2=0x7f184c06be00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:05.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 shutdown_connections 2024-03-04T07:30:05.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.711+0000 7f1853647640 1 -- 172.21.15.64:0/1729193564 wait complete. 2024-03-04T07:30:05.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:30:05.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.826+0000 7f96da49d640 1 Processor -- start 2024-03-04T07:30:05.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.826+0000 7f96da49d640 1 -- start start 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96da49d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f96d415e2f0 0x7f96d40b41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96da49d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96d406b4e0 0x7f96d406b940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96da49d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d406be80 0x7f96d40cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96da49d640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96d415e6d0 con 0x7f96d406b4e0 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96da49d640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96d40ccd00 con 0x7f96d415e2f0 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96da49d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96d40ccfe0 con 0x7f96d406be80 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96d37fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96d406b4e0 0x7f96d406b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96d37fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96d406b4e0 0x7f96d406b940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49254/0 (socket says 172.21.15.64:49254) 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96d37fe640 1 -- 172.21.15.64:0/1111979413 learned_addr learned my addr 172.21.15.64:0/1111979413 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96d3fff640 1 --2- 172.21.15.64:0/1111979413 >> v2:172.21.15.66:3300/0 conn(0x7f96d415e2f0 0x7f96d40b41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.827+0000 7f96d8a3a640 1 --2- 172.21.15.64:0/1111979413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d406be80 0x7f96d40cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d37fe640 1 -- 172.21.15.64:0/1111979413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d406be80 msgr2=0x7f96d40cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d37fe640 1 --2- 172.21.15.64:0/1111979413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d406be80 0x7f96d40cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d37fe640 1 -- 172.21.15.64:0/1111979413 >> v2:172.21.15.66:3300/0 conn(0x7f96d415e2f0 msgr2=0x7f96d40b41f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d37fe640 1 --2- 172.21.15.64:0/1111979413 >> v2:172.21.15.66:3300/0 conn(0x7f96d415e2f0 0x7f96d40b41f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d37fe640 1 -- 172.21.15.64:0/1111979413 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96d40cd2c0 con 0x7f96d406b4e0 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d3fff640 1 --2- 172.21.15.64:0/1111979413 >> v2:172.21.15.66:3300/0 conn(0x7f96d415e2f0 0x7f96d40b41f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d37fe640 1 --2- 172.21.15.64:0/1111979413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d406b940 secure :-1 s=READY pgs=831 cs=0 l=1 rev1=1 crypto rx=0x7f96d4071dc0 tx=0x7f96c402b7f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=296a9e0807b07418 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d2ffd640 1 -- 172.21.15.64:0/1111979413 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96c4038070 con 0x7f96d406b4e0 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d2ffd640 1 -- 172.21.15.64:0/1111979413 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f96c402f050 con 0x7f96d406b4e0 2024-03-04T07:30:05.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.828+0000 7f96d2ffd640 1 -- 172.21.15.64:0/1111979413 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96c4031970 con 0x7f96d406b4e0 2024-03-04T07:30:05.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 -- 172.21.15.64:0/1111979413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 msgr2=0x7f96d406b940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 --2- 172.21.15.64:0/1111979413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d406b940 secure :-1 s=READY pgs=831 cs=0 l=1 rev1=1 crypto rx=0x7f96d4071dc0 tx=0x7f96c402b7f0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 -- 172.21.15.64:0/1111979413 shutdown_connections 2024-03-04T07:30:05.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 --2- 172.21.15.64:0/1111979413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d406be80 0x7f96d40cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 --2- 172.21.15.64:0/1111979413 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d406b940 unknown :-1 s=CLOSED pgs=831 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 --2- 172.21.15.64:0/1111979413 >> v2:172.21.15.66:3300/0 conn(0x7f96d415e2f0 0x7f96d40b41f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 -- 172.21.15.64:0/1111979413 >> 172.21.15.64:0/1111979413 conn(0x7f96d40b36d0 msgr2=0x7f96d40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 -- 172.21.15.64:0/1111979413 shutdown_connections 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.829+0000 7f96da49d640 1 -- 172.21.15.64:0/1111979413 wait complete. 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.830+0000 7f96da49d640 1 Processor -- start 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.830+0000 7f96da49d640 1 -- start start 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96da49d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d4157620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96da49d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96d406be80 0x7f96d4157e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96da49d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d415e2f0 0x7f96d40b5600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96da49d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96d40cff40 con 0x7f96d406b4e0 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96da49d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96d406c310 con 0x7f96d406be80 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96da49d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96d40d00c0 con 0x7f96d415e2f0 2024-03-04T07:30:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d4157620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d4157620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49258/0 (socket says 172.21.15.64:49258) 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d3fff640 1 -- 172.21.15.64:0/997450855 learned_addr learned my addr 172.21.15.64:0/997450855 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d37fe640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96d406be80 0x7f96d4157e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d3fff640 1 -- 172.21.15.64:0/997450855 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d415e2f0 msgr2=0x7f96d40b5600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d3fff640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d415e2f0 0x7f96d40b5600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d3fff640 1 -- 172.21.15.64:0/997450855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96d406be80 msgr2=0x7f96d4157e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.831+0000 7f96d3fff640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96d406be80 0x7f96d4157e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.832+0000 7f96d3fff640 1 -- 172.21.15.64:0/997450855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96d40ca170 con 0x7f96d406b4e0 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.832+0000 7f96d3fff640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d4157620 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f96c800d7d0 tx=0x7f96c800dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.832+0000 7f96d0ff9640 1 -- 172.21.15.64:0/997450855 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96c8004030 con 0x7f96d406b4e0 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.832+0000 7f96d0ff9640 1 -- 172.21.15.64:0/997450855 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f96c80041d0 con 0x7f96d406b4e0 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.832+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96d40ca450 con 0x7f96d406b4e0 2024-03-04T07:30:05.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.832+0000 7f96d0ff9640 1 -- 172.21.15.64:0/997450855 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96c8009620 con 0x7f96d406b4e0 2024-03-04T07:30:05.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.832+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96d40caa10 con 0x7f96d406b4e0 2024-03-04T07:30:05.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.833+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f96d409fd30 con 0x7f96d406b4e0 2024-03-04T07:30:05.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.833+0000 7f96d0ff9640 1 -- 172.21.15.64:0/997450855 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f96c80097c0 con 0x7f96d406b4e0 2024-03-04T07:30:05.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.834+0000 7f96d0ff9640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f96c006b4d0 0x7f96c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:05.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.834+0000 7f96d37fe640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f96c006b4d0 0x7f96c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:05.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.834+0000 7f96d0ff9640 1 -- 172.21.15.64:0/997450855 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(65..65 src has 1..65) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f96c808a470 con 0x7f96d406b4e0 2024-03-04T07:30:05.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.834+0000 7f96d37fe640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f96c006b4d0 0x7f96c006d990 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f96c4035000 tx=0x7f96c4008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:05.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:05.838+0000 7f96d0ff9640 1 -- 172.21.15.64:0/997450855 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f96c80592f0 con 0x7f96d406b4e0 2024-03-04T07:30:06.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.058+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f96d415b280 con 0x7f96d406b4e0 2024-03-04T07:30:06.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.083+0000 7f96d0ff9640 1 -- 172.21.15.64:0/997450855 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v35) v1 ==== 118+0+0 (secure 0 0 0) 0x7f96c8058bf0 con 0x7f96d406b4e0 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f96c006b4d0 msgr2=0x7f96c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f96c006b4d0 0x7f96c006d990 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f96c4035000 tx=0x7f96c4008030 comp rx=0 tx=0).stop 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 msgr2=0x7f96d4157620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d4157620 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f96c800d7d0 tx=0x7f96c800dc90 comp rx=0 tx=0).stop 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 shutdown_connections 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f96c006b4d0 0x7f96c006d990 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96d415e2f0 0x7f96d40b5600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96d406be80 0x7f96d4157e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 --2- 172.21.15.64:0/997450855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96d406b4e0 0x7f96d4157620 unknown :-1 s=CLOSED pgs=832 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 >> 172.21.15.64:0/997450855 conn(0x7f96d40b36d0 msgr2=0x7f96d40cdcf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 shutdown_connections 2024-03-04T07:30:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.086+0000 7f96da49d640 1 -- 172.21.15.64:0/997450855 wait complete. 2024-03-04T07:30:06.097 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:30:06.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:30:06.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.203+0000 7f24ebddd640 1 Processor -- start 2024-03-04T07:30:06.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.203+0000 7f24ebddd640 1 -- start start 2024-03-04T07:30:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ebddd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40b52f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ebddd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f24e415e880 0x7f24e40719a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ebddd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f24e4071ee0 0x7f24e415b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ebddd640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f24e4072330 con 0x7f24e4071ee0 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ebddd640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f24e415f0d0 con 0x7f24e415e880 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ebddd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f24e415f3b0 con 0x7f24e4144370 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ea37a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f24e4071ee0 0x7f24e415b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ea37a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f24e4071ee0 0x7f24e415b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49274/0 (socket says 172.21.15.64:49274) 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24ea37a640 1 -- 172.21.15.64:0/1118949132 learned_addr learned my addr 172.21.15.64:0/1118949132 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.204+0000 7f24e9b79640 1 --2- 172.21.15.64:0/1118949132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40b52f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24e9378640 1 --2- 172.21.15.64:0/1118949132 >> v2:172.21.15.66:3300/0 conn(0x7f24e415e880 0x7f24e40719a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24ea37a640 1 -- 172.21.15.64:0/1118949132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 msgr2=0x7f24e40b52f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24ea37a640 1 --2- 172.21.15.64:0/1118949132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40b52f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24ea37a640 1 -- 172.21.15.64:0/1118949132 >> v2:172.21.15.66:3300/0 conn(0x7f24e415e880 msgr2=0x7f24e40719a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24ea37a640 1 --2- 172.21.15.64:0/1118949132 >> v2:172.21.15.66:3300/0 conn(0x7f24e415e880 0x7f24e40719a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24ea37a640 1 -- 172.21.15.64:0/1118949132 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f24e415f690 con 0x7f24e4071ee0 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24e9378640 1 --2- 172.21.15.64:0/1118949132 >> v2:172.21.15.66:3300/0 conn(0x7f24e415e880 0x7f24e40719a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24ea37a640 1 --2- 172.21.15.64:0/1118949132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e415b260 secure :-1 s=READY pgs=833 cs=0 l=1 rev1=1 crypto rx=0x7f24e40715c0 tx=0x7f24dc02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c5accecc11c98639 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:06.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.205+0000 7f24e8b77640 1 -- 172.21.15.64:0/1118949132 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24dc02ca70 con 0x7f24e4071ee0 2024-03-04T07:30:06.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24e8b77640 1 -- 172.21.15.64:0/1118949132 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f24dc02cc10 con 0x7f24e4071ee0 2024-03-04T07:30:06.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24e8b77640 1 -- 172.21.15.64:0/1118949132 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24dc0343f0 con 0x7f24e4071ee0 2024-03-04T07:30:06.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 -- 172.21.15.64:0/1118949132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 msgr2=0x7f24e415b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:06.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1118949132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e415b260 secure :-1 s=READY pgs=833 cs=0 l=1 rev1=1 crypto rx=0x7f24e40715c0 tx=0x7f24dc02bed0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 -- 172.21.15.64:0/1118949132 shutdown_connections 2024-03-04T07:30:06.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1118949132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e415b260 unknown :-1 s=CLOSED pgs=833 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1118949132 >> v2:172.21.15.66:3300/0 conn(0x7f24e415e880 0x7f24e40719a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1118949132 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40b52f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 -- 172.21.15.64:0/1118949132 >> 172.21.15.64:0/1118949132 conn(0x7f24e40b2790 msgr2=0x7f24e40b2ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:06.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 -- 172.21.15.64:0/1118949132 shutdown_connections 2024-03-04T07:30:06.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.206+0000 7f24ebddd640 1 -- 172.21.15.64:0/1118949132 wait complete. 2024-03-04T07:30:06.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.208+0000 7f24ebddd640 1 Processor -- start 2024-03-04T07:30:06.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.208+0000 7f24ebddd640 1 -- start start 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24ebddd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e41f1c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24ebddd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40d0460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24ebddd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f24e415e880 0x7f24e4140350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24ebddd640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f24e415ee80 con 0x7f24e4071ee0 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24ebddd640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f24e415ed00 con 0x7f24e415e880 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24ebddd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f24e4144750 con 0x7f24e4144370 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24e9b79640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e41f1c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24e9b79640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e41f1c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49276/0 (socket says 172.21.15.64:49276) 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.209+0000 7f24e9b79640 1 -- 172.21.15.64:0/1918781194 learned_addr learned my addr 172.21.15.64:0/1918781194 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24e9378640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40d0460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:06.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24ea37a640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f24e415e880 0x7f24e4140350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24e9b79640 1 -- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 msgr2=0x7f24e40d0460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24e9b79640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40d0460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24e9b79640 1 -- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f24e415e880 msgr2=0x7f24e4140350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24e9b79640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f24e415e880 0x7f24e4140350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24e9b79640 1 -- 172.21.15.64:0/1918781194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f24e413b960 con 0x7f24e4071ee0 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24e9b79640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e41f1c80 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f24e000d800 tx=0x7f24e000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24da7fc640 1 -- 172.21.15.64:0/1918781194 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24e0018020 con 0x7f24e4071ee0 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24da7fc640 1 -- 172.21.15.64:0/1918781194 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f24e00096a0 con 0x7f24e4071ee0 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f24e413bc40 con 0x7f24e4071ee0 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24da7fc640 1 -- 172.21.15.64:0/1918781194 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24e0009a70 con 0x7f24e4071ee0 2024-03-04T07:30:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.210+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f24e413c200 con 0x7f24e4071ee0 2024-03-04T07:30:06.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.211+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f24e404f410 con 0x7f24e4071ee0 2024-03-04T07:30:06.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.211+0000 7f24da7fc640 1 -- 172.21.15.64:0/1918781194 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f24e001b020 con 0x7f24e4071ee0 2024-03-04T07:30:06.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.212+0000 7f24da7fc640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f24c806b4d0 0x7f24c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:06.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.212+0000 7f24da7fc640 1 -- 172.21.15.64:0/1918781194 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f24e008a460 con 0x7f24e4071ee0 2024-03-04T07:30:06.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.212+0000 7f24e9378640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f24c806b4d0 0x7f24c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:06.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.212+0000 7f24e9378640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f24c806b4d0 0x7f24c806d990 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f24e40d11b0 tx=0x7f24d4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:06.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.216+0000 7f24da7fc640 1 -- 172.21.15.64:0/1918781194 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f24e0022030 con 0x7f24e4071ee0 2024-03-04T07:30:06.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:06.495+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f24e40725a0 con 0x7f24e4071ee0 2024-03-04T07:30:06.852 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.b has been restored 2024-03-04T07:30:07.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.087+0000 7f24da7fc640 1 -- 172.21.15.64:0/1918781194 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v36) v1 ==== 122+0+0 (secure 0 0 0) 0x7f24e00592f0 con 0x7f24e4071ee0 2024-03-04T07:30:07.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f24c806b4d0 msgr2=0x7f24c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f24c806b4d0 0x7f24c806d990 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f24e40d11b0 tx=0x7f24d4008040 comp rx=0 tx=0).stop 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 msgr2=0x7f24e41f1c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e41f1c80 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f24e000d800 tx=0x7f24e000dcc0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 shutdown_connections 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f24c806b4d0 0x7f24c806d990 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f24e415e880 0x7f24e4140350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24e4144370 0x7f24e40d0460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 --2- 172.21.15.64:0/1918781194 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24e4071ee0 0x7f24e41f1c80 unknown :-1 s=CLOSED pgs=834 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 >> 172.21.15.64:0/1918781194 conn(0x7f24e40b2790 msgr2=0x7f24e40b1d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.089+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 shutdown_connections 2024-03-04T07:30:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.090+0000 7f24ebddd640 1 -- 172.21.15.64:0/1918781194 wait complete. 2024-03-04T07:30:07.101 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:30:07.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:07.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.202+0000 7fb5f98ed640 1 Processor -- start 2024-03-04T07:30:07.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.202+0000 7fb5f98ed640 1 -- start start 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f98ed640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb5f415dd70 0x7fb5f40b9650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f98ed640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb5f40b5700 0x7fb5f40b5b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f98ed640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5f40b83c0 con 0x7fb5f40d1e50 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f98ed640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5f40b60b0 con 0x7fb5f415dd70 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f98ed640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5f40b8fd0 con 0x7fb5f40b5700 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb5f415dd70 0x7fb5f40b9650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb5f415dd70 0x7fb5f40b9650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49292/0 (socket says 172.21.15.64:49292) 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.203+0000 7fb5f2ffd640 1 -- 172.21.15.64:0/3382712020 learned_addr learned my addr 172.21.15.64:0/3382712020 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.204+0000 7fb5ea7fc640 1 --2- 172.21.15.64:0/3382712020 >> v2:172.21.15.66:3300/0 conn(0x7fb5f40b5700 0x7fb5f40b5b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.204+0000 7fb5f2ffd640 1 -- 172.21.15.64:0/3382712020 >> v2:172.21.15.66:3300/0 conn(0x7fb5f40b5700 msgr2=0x7fb5f40b5b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:07.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.204+0000 7fb5f2ffd640 1 --2- 172.21.15.64:0/3382712020 >> v2:172.21.15.66:3300/0 conn(0x7fb5f40b5700 0x7fb5f40b5b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.204+0000 7fb5f2ffd640 1 -- 172.21.15.64:0/3382712020 >> v1:172.21.15.66:6789/0 conn(0x7fb5f40d1e50 legacy=0x7fb5f40d22c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.204+0000 7fb5f2ffd640 1 -- 172.21.15.64:0/3382712020 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5f40b86a0 con 0x7fb5f415dd70 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.205+0000 7fb5f2ffd640 1 --2- 172.21.15.64:0/3382712020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f415dd70 0x7fb5f40b9650 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7fb5f408ce80 tx=0x7fb5e402ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a54f23984819c8b1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.205+0000 7fb5f27fc640 1 -- 172.21.15.64:0/3382712020 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5e402f850 con 0x7fb5f415dd70 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.205+0000 7fb5f27fc640 1 -- 172.21.15.64:0/3382712020 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5e402f9f0 con 0x7fb5f415dd70 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.205+0000 7fb5f27fc640 1 -- 172.21.15.64:0/3382712020 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5e402fdc0 con 0x7fb5f415dd70 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 -- 172.21.15.64:0/3382712020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f415dd70 msgr2=0x7fb5f40b9650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/3382712020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f415dd70 0x7fb5f40b9650 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7fb5f408ce80 tx=0x7fb5e402ba30 comp rx=0 tx=0).stop 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 -- 172.21.15.64:0/3382712020 shutdown_connections 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/3382712020 >> v2:172.21.15.66:3300/0 conn(0x7fb5f40b5700 0x7fb5f40b5b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/3382712020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f415dd70 0x7fb5f40b9650 unknown :-1 s=CLOSED pgs=835 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 -- 172.21.15.64:0/3382712020 >> 172.21.15.64:0/3382712020 conn(0x7fb5f4071960 msgr2=0x7fb5f4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:07.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 -- 172.21.15.64:0/3382712020 shutdown_connections 2024-03-04T07:30:07.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.206+0000 7fb5f98ed640 1 -- 172.21.15.64:0/3382712020 wait complete. 2024-03-04T07:30:07.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.207+0000 7fb5f98ed640 1 Processor -- start 2024-03-04T07:30:07.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.207+0000 7fb5f98ed640 1 -- start start 2024-03-04T07:30:07.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5f98ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5f40b5700 0x7fb5f414af90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5f98ed640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f40d1e50 0x7fb5f40c9c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5f98ed640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5f415dd70 0x7fb5f40ca170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5f98ed640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5f40b9b90 con 0x7fb5f40d1e50 2024-03-04T07:30:07.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5f98ed640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5f40b8c60 con 0x7fb5f415dd70 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5f98ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5f415e150 con 0x7fb5f40b5700 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f40d1e50 0x7fb5f40c9c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f40d1e50 0x7fb5f40c9c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49294/0 (socket says 172.21.15.64:49294) 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 -- 172.21.15.64:0/1021932550 learned_addr learned my addr 172.21.15.64:0/1021932550 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 -- 172.21.15.64:0/1021932550 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5f40b5700 msgr2=0x7fb5f414af90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5f40b5700 0x7fb5f414af90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 -- 172.21.15.64:0/1021932550 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5f415dd70 msgr2=0x7fb5f40ca170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5f37fe640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5f415dd70 0x7fb5f40ca170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5f415dd70 0x7fb5f40ca170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 -- 172.21.15.64:0/1021932550 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5f4158d00 con 0x7fb5f40d1e50 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.208+0000 7fb5ea7fc640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f40d1e50 0x7fb5f40c9c30 secure :-1 s=READY pgs=836 cs=0 l=1 rev1=1 crypto rx=0x7fb5dc00e9e0 tx=0x7fb5dc00eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.209+0000 7fb5f0ff9640 1 -- 172.21.15.64:0/1021932550 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5dc017070 con 0x7fb5f40d1e50 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.209+0000 7fb5f0ff9640 1 -- 172.21.15.64:0/1021932550 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5dc00c9d0 con 0x7fb5f40d1e50 2024-03-04T07:30:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.209+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5f4158fe0 con 0x7fb5f40d1e50 2024-03-04T07:30:07.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.209+0000 7fb5f0ff9640 1 -- 172.21.15.64:0/1021932550 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5dc00cdc0 con 0x7fb5f40d1e50 2024-03-04T07:30:07.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.209+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5f40c7a00 con 0x7fb5f40d1e50 2024-03-04T07:30:07.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.210+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5f4161400 con 0x7fb5f40d1e50 2024-03-04T07:30:07.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.210+0000 7fb5f0ff9640 1 -- 172.21.15.64:0/1021932550 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb5dc01a070 con 0x7fb5f40d1e50 2024-03-04T07:30:07.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.210+0000 7fb5f0ff9640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5d406b4d0 0x7fb5d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.210+0000 7fb5f0ff9640 1 -- 172.21.15.64:0/1021932550 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb5dc08a520 con 0x7fb5f40d1e50 2024-03-04T07:30:07.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.210+0000 7fb5f2ffd640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5d406b4d0 0x7fb5d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.211+0000 7fb5f2ffd640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5d406b4d0 0x7fb5d406d990 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7fb5e4036000 tx=0x7fb5e4039040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.215+0000 7fb5f0ff9640 1 -- 172.21.15.64:0/1021932550 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb5dc059430 con 0x7fb5f40d1e50 2024-03-04T07:30:07.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.433+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb5f4067ff0 con 0x7fb5f40d1e50 2024-03-04T07:30:07.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.434+0000 7fb5f0ff9640 1 -- 172.21.15.64:0/1021932550 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7fb5dc058d30 con 0x7fb5f40d1e50 2024-03-04T07:30:07.435 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:07.435 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:07.435 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5d406b4d0 msgr2=0x7fb5d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5d406b4d0 0x7fb5d406d990 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7fb5e4036000 tx=0x7fb5e4039040 comp rx=0 tx=0).stop 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f40d1e50 msgr2=0x7fb5f40c9c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f40d1e50 0x7fb5f40c9c30 secure :-1 s=READY pgs=836 cs=0 l=1 rev1=1 crypto rx=0x7fb5dc00e9e0 tx=0x7fb5dc00eea0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 shutdown_connections 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5d406b4d0 0x7fb5d406d990 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5f415dd70 0x7fb5f40ca170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5f40d1e50 0x7fb5f40c9c30 unknown :-1 s=CLOSED pgs=836 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 --2- 172.21.15.64:0/1021932550 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5f40b5700 0x7fb5f414af90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 >> 172.21.15.64:0/1021932550 conn(0x7fb5f4071960 msgr2=0x7fb5f415b240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 shutdown_connections 2024-03-04T07:30:07.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.436+0000 7fb5f98ed640 1 -- 172.21.15.64:0/1021932550 wait complete. 2024-03-04T07:30:07.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:07.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.548+0000 7fe45f2eb640 1 Processor -- start 2024-03-04T07:30:07.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.548+0000 7fe45f2eb640 1 -- start start 2024-03-04T07:30:07.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe45f2eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 0x7fe458162390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe45f2eb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4580d1090 con 0x7fe4580cebb0 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe45f2eb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe45806b0a0 con 0x7fe45806ef80 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe45f2eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe45806b380 con 0x7fe4580d0940 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe45d087640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe4580cebb0 0x7fe458161c80 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45700/0 (socket says 172.21.15.64:45700) 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe45d087640 1 -- 172.21.15.64:0/4097430901 learned_addr learned my addr 172.21.15.64:0/4097430901 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe45c886640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 0x7fe458162390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3870372006 0 0) 0x7fe4580d1090 con 0x7fe4580cebb0 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fe444003520 con 0x7fe4580cebb0 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1291427968 0 0) 0x7fe444003520 con 0x7fe4580cebb0 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 msgr2=0x7fe458162390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 0x7fe458162390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 >> v1:172.21.15.66:6789/0 conn(0x7fe45806ef80 legacy=0x7fe45806f410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe45806b660 con 0x7fe4580cebb0 2024-03-04T07:30:07.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fe448001670 con 0x7fe4580cebb0 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.550+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 >> v1:172.21.15.64:6789/0 conn(0x7fe4580cebb0 legacy=0x7fe458161c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe44ffff640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4440039e0 0x7fe444003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe44ffff640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe444004800 0x7fe444004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe44ffff640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 0x7fe4440051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe45806b380 con 0x7fe444004800 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe44400ba70 con 0x7fe4440039e0 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe44400bc70 con 0x7fe4580d0940 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45c886640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe444004800 0x7fe444004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45d087640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4440039e0 0x7fe444003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45d888640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 0x7fe4440051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45c886640 1 -- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 msgr2=0x7fe4440051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45c886640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4580d0940 0x7fe4440051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45c886640 1 -- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4440039e0 msgr2=0x7fe444003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45c886640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4440039e0 0x7fe444003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.551+0000 7fe45c886640 1 -- 172.21.15.64:0/4097430901 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe444003520 con 0x7fe444004800 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe45c886640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe444004800 0x7fe444004c80 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7fe454009a90 tx=0x7fe444005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b698f2cf8c7926f8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe454039070 con 0x7fe444004800 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe45402c680 con 0x7fe444004800 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe44ffff640 1 -- 172.21.15.64:0/4097430901 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe454009690 con 0x7fe444004800 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe45f2eb640 1 -- 172.21.15.64:0/4097430901 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe444004800 msgr2=0x7fe444004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe45f2eb640 1 --2- 172.21.15.64:0/4097430901 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe444004800 0x7fe444004c80 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7fe454009a90 tx=0x7fe444005830 comp rx=0 tx=0).stop 2024-03-04T07:30:07.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe45d888640 1 -- 172.21.15.64:0/4097430901 reap_dead start 2024-03-04T07:30:07.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe45f2eb640 1 -- 172.21.15.64:0/4097430901 shutdown_connections 2024-03-04T07:30:07.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe45f2eb640 1 -- 172.21.15.64:0/4097430901 >> 172.21.15.64:0/4097430901 conn(0x7fe4580b35d0 msgr2=0x7fe4580b39e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:07.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.552+0000 7fe45f2eb640 1 -- 172.21.15.64:0/4097430901 shutdown_connections 2024-03-04T07:30:07.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.553+0000 7fe45f2eb640 1 -- 172.21.15.64:0/4097430901 wait complete. 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 Processor -- start 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 -- start start 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe444004800 0x7fe4581f7690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe458154020 0x7fe4581e7fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe458160430 0x7fe4581608b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe45806cd10 con 0x7fe458154020 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe45806cb90 con 0x7fe444004800 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.554+0000 7fe45f2eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe45806ce90 con 0x7fe458160430 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe458154020 0x7fe4581e7fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe458154020 0x7fe4581e7fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49322/0 (socket says 172.21.15.64:49322) 2024-03-04T07:30:07.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 -- 172.21.15.64:0/1836472889 learned_addr learned my addr 172.21.15.64:0/1836472889 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45d087640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe444004800 0x7fe4581f7690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45d888640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe458160430 0x7fe4581608b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 -- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe458160430 msgr2=0x7fe4581608b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe458160430 0x7fe4581608b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 -- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe444004800 msgr2=0x7fe4581f7690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe444004800 0x7fe4581f7690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 -- 172.21.15.64:0/1836472889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4581f7d90 con 0x7fe458154020 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45d888640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe458160430 0x7fe4581608b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45c886640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe458154020 0x7fe4581e7fb0 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7fe454037040 tx=0x7fe454004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe44dffb640 1 -- 172.21.15.64:0/1836472889 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe454039050 con 0x7fe458154020 2024-03-04T07:30:07.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe44dffb640 1 -- 172.21.15.64:0/1836472889 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe454045070 con 0x7fe458154020 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4581f8070 con 0x7fe458154020 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe44dffb640 1 -- 172.21.15.64:0/1836472889 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4540044e0 con 0x7fe458154020 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.555+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4581e84f0 con 0x7fe458154020 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.556+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe4580c88e0 con 0x7fe458154020 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.557+0000 7fe44dffb640 1 -- 172.21.15.64:0/1836472889 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe454003830 con 0x7fe458154020 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.557+0000 7fe44dffb640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe43c06b4d0 0x7fe43c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.557+0000 7fe44dffb640 1 -- 172.21.15.64:0/1836472889 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe4540abbc0 con 0x7fe458154020 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.557+0000 7fe45d087640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe43c06b4d0 0x7fe43c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.558+0000 7fe45d087640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe43c06b4d0 0x7fe43c06d990 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7fe448001050 tx=0x7fe4480078f0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.562+0000 7fe44dffb640 1 -- 172.21.15.64:0/1836472889 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe4580c88e0 con 0x7fe458154020 2024-03-04T07:30:07.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.743+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe458068700 con 0x7fe458154020 2024-03-04T07:30:07.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.744+0000 7fe44dffb640 1 -- 172.21.15.64:0/1836472889 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v66) v1 ==== 74+0+16620 (secure 0 0 0) 0x7fe45404e020 con 0x7fe458154020 2024-03-04T07:30:07.745 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:07.745 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":66,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:06.066460+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:02.706399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:03.395544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:07.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe43c06b4d0 msgr2=0x7fe43c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe43c06b4d0 0x7fe43c06d990 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7fe448001050 tx=0x7fe4480078f0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe458154020 msgr2=0x7fe4581e7fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe458154020 0x7fe4581e7fb0 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7fe454037040 tx=0x7fe454004730 comp rx=0 tx=0).stop 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 shutdown_connections 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe43c06b4d0 0x7fe43c06d990 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe458160430 0x7fe4581608b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe458154020 0x7fe4581e7fb0 unknown :-1 s=CLOSED pgs=839 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 --2- 172.21.15.64:0/1836472889 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe444004800 0x7fe4581f7690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 >> 172.21.15.64:0/1836472889 conn(0x7fe4580b35d0 msgr2=0x7fe4580b1400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 shutdown_connections 2024-03-04T07:30:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.746+0000 7fe45f2eb640 1 -- 172.21.15.64:0/1836472889 wait complete. 2024-03-04T07:30:07.758 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:30:07.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:30:07.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.863+0000 7fc85dfac640 1 Processor -- start 2024-03-04T07:30:07.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.863+0000 7fc85dfac640 1 -- start start 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc85dfac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8580b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc85dfac640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc8580b0e90 0x7fc85806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc85dfac640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc8580b3340 con 0x7fc8580ae9b0 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc85dfac640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc8580cfa00 con 0x7fc8580b0e90 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc85dfac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc8580cee90 con 0x7fc85806b5c0 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc856ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc8580b0e90 0x7fc85806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc856ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc8580b0e90 0x7fc85806b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49334/0 (socket says 172.21.15.64:49334) 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc856ffd640 1 -- 172.21.15.64:0/2171102859 learned_addr learned my addr 172.21.15.64:0/2171102859 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc8577fe640 1 --2- 172.21.15.64:0/2171102859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8580b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc856ffd640 1 -- 172.21.15.64:0/2171102859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 msgr2=0x7fc8580b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc856ffd640 1 --2- 172.21.15.64:0/2171102859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8580b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc856ffd640 1 -- 172.21.15.64:0/2171102859 >> v1:172.21.15.66:6789/0 conn(0x7fc8580ae9b0 legacy=0x7fc8580d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc856ffd640 1 -- 172.21.15.64:0/2171102859 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc8580cfc50 con 0x7fc8580b0e90 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.864+0000 7fc8567fc640 1 -- 172.21.15.64:0/2171102859 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1407228186 0 0) 0x7fc8580b3340 con 0x7fc8580ae9b0 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.865+0000 7fc856ffd640 1 --2- 172.21.15.64:0/2171102859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580b0e90 0x7fc85806b9a0 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7fc84c009870 tx=0x7fc84c02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fdd9b2e9477f0a85 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.865+0000 7fc8567fc640 1 -- 172.21.15.64:0/2171102859 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc84c035020 con 0x7fc8580b0e90 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.865+0000 7fc8567fc640 1 -- 172.21.15.64:0/2171102859 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc84c031640 con 0x7fc8580b0e90 2024-03-04T07:30:07.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.865+0000 7fc8567fc640 1 -- 172.21.15.64:0/2171102859 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc84c02d040 con 0x7fc8580b0e90 2024-03-04T07:30:07.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 -- 172.21.15.64:0/2171102859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580b0e90 msgr2=0x7fc85806b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 --2- 172.21.15.64:0/2171102859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580b0e90 0x7fc85806b9a0 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7fc84c009870 tx=0x7fc84c02b8f0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 -- 172.21.15.64:0/2171102859 shutdown_connections 2024-03-04T07:30:07.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 --2- 172.21.15.64:0/2171102859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580b0e90 0x7fc85806b9a0 unknown :-1 s=CLOSED pgs=840 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 --2- 172.21.15.64:0/2171102859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8580b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 -- 172.21.15.64:0/2171102859 >> 172.21.15.64:0/2171102859 conn(0x7fc8580b36d0 msgr2=0x7fc8580b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:07.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 -- 172.21.15.64:0/2171102859 shutdown_connections 2024-03-04T07:30:07.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.866+0000 7fc85dfac640 1 -- 172.21.15.64:0/2171102859 wait complete. 2024-03-04T07:30:07.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.868+0000 7fc85dfac640 1 Processor -- start 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.868+0000 7fc85dfac640 1 -- start start 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.868+0000 7fc85dfac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8581e5c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc85dfac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580ae9b0 0x7fc8581e4a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc85dfac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8580b0e90 0x7fc8580cbf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc85dfac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc85806c790 con 0x7fc8580ae9b0 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc85dfac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc85806bee0 con 0x7fc8580b0e90 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc85dfac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc85806c910 con 0x7fc85806b5c0 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580ae9b0 0x7fc8581e4a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580ae9b0 0x7fc8581e4a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49336/0 (socket says 172.21.15.64:49336) 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 -- 172.21.15.64:0/4293721726 learned_addr learned my addr 172.21.15.64:0/4293721726 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc857fff640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8580b0e90 0x7fc8580cbf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc8577fe640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8581e5c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 -- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 msgr2=0x7fc8581e5c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8581e5c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 -- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8580b0e90 msgr2=0x7fc8580cbf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8580b0e90 0x7fc8580cbf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 -- 172.21.15.64:0/4293721726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc8580cc9f0 con 0x7fc8580ae9b0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc8577fe640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8581e5c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc856ffd640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580ae9b0 0x7fc8581e4a00 secure :-1 s=READY pgs=841 cs=0 l=1 rev1=1 crypto rx=0x7fc84c0090b0 tx=0x7fc84c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc83ffff640 1 -- 172.21.15.64:0/4293721726 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc84c040070 con 0x7fc8580ae9b0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc83ffff640 1 -- 172.21.15.64:0/4293721726 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc84c0399f0 con 0x7fc8580ae9b0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc8580cccd0 con 0x7fc8580ae9b0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc83ffff640 1 -- 172.21.15.64:0/4293721726 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc84c035040 con 0x7fc8580ae9b0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.869+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc8580cd290 con 0x7fc8580ae9b0 2024-03-04T07:30:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.870+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc858068700 con 0x7fc8580ae9b0 2024-03-04T07:30:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.871+0000 7fc83ffff640 1 -- 172.21.15.64:0/4293721726 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc84c038020 con 0x7fc8580ae9b0 2024-03-04T07:30:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.871+0000 7fc83ffff640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc84406b500 0x7fc84406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.871+0000 7fc83ffff640 1 -- 172.21.15.64:0/4293721726 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc84c0aba20 con 0x7fc8580ae9b0 2024-03-04T07:30:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.871+0000 7fc8577fe640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc84406b500 0x7fc84406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.872+0000 7fc8577fe640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc84406b500 0x7fc84406d9c0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7fc8581e5820 tx=0x7fc84800b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:07.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:07.875+0000 7fc83ffff640 1 -- 172.21.15.64:0/4293721726 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc84c07a930 con 0x7fc8580ae9b0 2024-03-04T07:30:08.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.206+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fc8580b5470 con 0x7fc84406b500 2024-03-04T07:30:08.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.212+0000 7fc83ffff640 1 -- 172.21.15.64:0/4293721726 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252049 (secure 0 0 0) 0x7fc8580b5470 con 0x7fc84406b500 2024-03-04T07:30:08.216 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:08.217 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc84406b500 msgr2=0x7fc84406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc84406b500 0x7fc84406d9c0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7fc8581e5820 tx=0x7fc84800b040 comp rx=0 tx=0).stop 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580ae9b0 msgr2=0x7fc8581e4a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580ae9b0 0x7fc8581e4a00 secure :-1 s=READY pgs=841 cs=0 l=1 rev1=1 crypto rx=0x7fc84c0090b0 tx=0x7fc84c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 shutdown_connections 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc84406b500 0x7fc84406d9c0 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc8580b0e90 0x7fc8580cbf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8580ae9b0 0x7fc8581e4a00 unknown :-1 s=CLOSED pgs=841 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 --2- 172.21.15.64:0/4293721726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc85806b5c0 0x7fc8581e5c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 >> 172.21.15.64:0/4293721726 conn(0x7fc8580b36d0 msgr2=0x7fc85806f550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:08.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 shutdown_connections 2024-03-04T07:30:08.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.215+0000 7fc85dfac640 1 -- 172.21.15.64:0/4293721726 wait complete. 2024-03-04T07:30:08.227 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":162,"stamp":"2024-03-04T07:30:07.177068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":142,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":256892,"kb_used_data":7400,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754717828,"statfs":{"total":773094113280,"available":772831055872,"internally_reserved":0,"allocated":7577600,"data_stored":3824692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":22,"apply_latency_ms":22,"commit_latency_ns":22000000,"apply_latency_ns":22000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.118114"},"pg_stats":[{"pgid":"11.3f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.091343+0000","last_change":"2024-03-04T07:30:05.080757+0000","last_active":"2024-03-04T07:30:05.091343+0000","last_peered":"2024-03-04T07:30:05.091343+0000","last_clean":"2024-03-04T07:30:05.091343+0000","last_became_active":"2024-03-04T07:30:05.080481+0000","last_became_peered":"2024-03-04T07:30:05.080481+0000","last_unstale":"2024-03-04T07:30:05.091343+0000","last_undegraded":"2024-03-04T07:30:05.091343+0000","last_fullsized":"2024-03-04T07:30:05.091343+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:09:31.879681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.3e","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.092691+0000","last_change":"2024-03-04T07:30:05.084660+0000","last_active":"2024-03-04T07:30:05.092691+0000","last_peered":"2024-03-04T07:30:05.092691+0000","last_clean":"2024-03-04T07:30:05.092691+0000","last_became_active":"2024-03-04T07:30:05.084351+0000","last_became_peered":"2024-03-04T07:30:05.084351+0000","last_unstale":"2024-03-04T07:30:05.092691+0000","last_undegraded":"2024-03-04T07:30:05.092691+0000","last_fullsized":"2024-03-04T07:30:05.092691+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:36:04.963795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.3d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.081867+0000","last_change":"2024-03-04T07:30:05.079727+0000","last_active":"2024-03-04T07:30:05.081867+0000","last_peered":"2024-03-04T07:30:05.081867+0000","last_clean":"2024-03-04T07:30:05.081867+0000","last_became_active":"2024-03-04T07:30:05.079439+0000","last_became_peered":"2024-03-04T07:30:05.079439+0000","last_unstale":"2024-03-04T07:30:05.081867+0000","last_undegraded":"2024-03-04T07:30:05.081867+0000","last_fullsized":"2024-03-04T07:30:05.081867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:26:54.073772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.3c","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080088+0000","last_change":"2024-03-04T07:30:05.098463+0000","last_active":"2024-03-04T07:30:06.080088+0000","last_peered":"2024-03-04T07:30:06.080088+0000","last_clean":"2024-03-04T07:30:06.080088+0000","last_became_active":"2024-03-04T07:30:05.098181+0000","last_became_peered":"2024-03-04T07:30:05.098181+0000","last_unstale":"2024-03-04T07:30:06.080088+0000","last_undegraded":"2024-03-04T07:30:06.080088+0000","last_fullsized":"2024-03-04T07:30:06.080088+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:55.863787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3b","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.084094+0000","last_change":"2024-03-04T07:30:05.082411+0000","last_active":"2024-03-04T07:30:05.084094+0000","last_peered":"2024-03-04T07:30:05.084094+0000","last_clean":"2024-03-04T07:30:05.084094+0000","last_became_active":"2024-03-04T07:30:05.082028+0000","last_became_peered":"2024-03-04T07:30:05.082028+0000","last_unstale":"2024-03-04T07:30:05.084094+0000","last_undegraded":"2024-03-04T07:30:05.084094+0000","last_fullsized":"2024-03-04T07:30:05.084094+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:50:02.482536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3a","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099512+0000","last_change":"2024-03-04T07:30:05.097621+0000","last_active":"2024-03-04T07:30:05.099512+0000","last_peered":"2024-03-04T07:30:05.099512+0000","last_clean":"2024-03-04T07:30:05.099512+0000","last_became_active":"2024-03-04T07:30:05.095412+0000","last_became_peered":"2024-03-04T07:30:05.095412+0000","last_unstale":"2024-03-04T07:30:05.099512+0000","last_undegraded":"2024-03-04T07:30:05.099512+0000","last_fullsized":"2024-03-04T07:30:05.099512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:00:36.612527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.39","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070514+0000","last_change":"2024-03-04T07:30:05.089188+0000","last_active":"2024-03-04T07:30:06.070514+0000","last_peered":"2024-03-04T07:30:06.070514+0000","last_clean":"2024-03-04T07:30:06.070514+0000","last_became_active":"2024-03-04T07:30:05.088884+0000","last_became_peered":"2024-03-04T07:30:05.088884+0000","last_unstale":"2024-03-04T07:30:06.070514+0000","last_undegraded":"2024-03-04T07:30:06.070514+0000","last_fullsized":"2024-03-04T07:30:06.070514+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:52:38.600368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.38","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099361+0000","last_change":"2024-03-04T07:30:05.094825+0000","last_active":"2024-03-04T07:30:05.099361+0000","last_peered":"2024-03-04T07:30:05.099361+0000","last_clean":"2024-03-04T07:30:05.099361+0000","last_became_active":"2024-03-04T07:30:05.094499+0000","last_became_peered":"2024-03-04T07:30:05.094499+0000","last_unstale":"2024-03-04T07:30:05.099361+0000","last_undegraded":"2024-03-04T07:30:05.099361+0000","last_fullsized":"2024-03-04T07:30:05.099361+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:55.863787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.37","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090255+0000","last_change":"2024-03-04T07:30:05.088738+0000","last_active":"2024-03-04T07:30:05.090255+0000","last_peered":"2024-03-04T07:30:05.090255+0000","last_clean":"2024-03-04T07:30:05.090255+0000","last_became_active":"2024-03-04T07:30:05.087436+0000","last_became_peered":"2024-03-04T07:30:05.087436+0000","last_unstale":"2024-03-04T07:30:05.090255+0000","last_undegraded":"2024-03-04T07:30:05.090255+0000","last_fullsized":"2024-03-04T07:30:05.090255+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:48.648887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.36","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071596+0000","last_change":"2024-03-04T07:30:05.077566+0000","last_active":"2024-03-04T07:30:06.071596+0000","last_peered":"2024-03-04T07:30:06.071596+0000","last_clean":"2024-03-04T07:30:06.071596+0000","last_became_active":"2024-03-04T07:30:05.077146+0000","last_became_peered":"2024-03-04T07:30:05.077146+0000","last_unstale":"2024-03-04T07:30:06.071596+0000","last_undegraded":"2024-03-04T07:30:06.071596+0000","last_fullsized":"2024-03-04T07:30:06.071596+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:09.457728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.35","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093052+0000","last_change":"2024-03-04T07:30:05.085914+0000","last_active":"2024-03-04T07:30:05.093052+0000","last_peered":"2024-03-04T07:30:05.093052+0000","last_clean":"2024-03-04T07:30:05.093052+0000","last_became_active":"2024-03-04T07:30:05.085617+0000","last_became_peered":"2024-03-04T07:30:05.085617+0000","last_unstale":"2024-03-04T07:30:05.093052+0000","last_undegraded":"2024-03-04T07:30:05.093052+0000","last_fullsized":"2024-03-04T07:30:05.093052+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:04:11.464737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.34","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080137+0000","last_change":"2024-03-04T07:30:05.091184+0000","last_active":"2024-03-04T07:30:06.080137+0000","last_peered":"2024-03-04T07:30:06.080137+0000","last_clean":"2024-03-04T07:30:06.080137+0000","last_became_active":"2024-03-04T07:30:05.090901+0000","last_became_peered":"2024-03-04T07:30:05.090901+0000","last_unstale":"2024-03-04T07:30:06.080137+0000","last_undegraded":"2024-03-04T07:30:06.080137+0000","last_fullsized":"2024-03-04T07:30:06.080137+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:44:35.285031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.33","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071824+0000","last_change":"2024-03-04T07:30:05.093386+0000","last_active":"2024-03-04T07:30:06.071824+0000","last_peered":"2024-03-04T07:30:06.071824+0000","last_clean":"2024-03-04T07:30:06.071824+0000","last_became_active":"2024-03-04T07:30:05.093080+0000","last_became_peered":"2024-03-04T07:30:05.093080+0000","last_unstale":"2024-03-04T07:30:06.071824+0000","last_undegraded":"2024-03-04T07:30:06.071824+0000","last_fullsized":"2024-03-04T07:30:06.071824+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:41.093583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.32","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.079973+0000","last_change":"2024-03-04T07:30:05.078876+0000","last_active":"2024-03-04T07:30:05.079973+0000","last_peered":"2024-03-04T07:30:05.079973+0000","last_clean":"2024-03-04T07:30:05.079973+0000","last_became_active":"2024-03-04T07:30:05.078470+0000","last_became_peered":"2024-03-04T07:30:05.078470+0000","last_unstale":"2024-03-04T07:30:05.079973+0000","last_undegraded":"2024-03-04T07:30:05.079973+0000","last_fullsized":"2024-03-04T07:30:05.079973+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:04:48.717451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.31","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071623+0000","last_change":"2024-03-04T07:30:05.080376+0000","last_active":"2024-03-04T07:30:06.071623+0000","last_peered":"2024-03-04T07:30:06.071623+0000","last_clean":"2024-03-04T07:30:06.071623+0000","last_became_active":"2024-03-04T07:30:05.080094+0000","last_became_peered":"2024-03-04T07:30:05.080094+0000","last_unstale":"2024-03-04T07:30:06.071623+0000","last_undegraded":"2024-03-04T07:30:06.071623+0000","last_fullsized":"2024-03-04T07:30:06.071623+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:28:21.513469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.30","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070519+0000","last_change":"2024-03-04T07:30:05.096512+0000","last_active":"2024-03-04T07:30:06.070519+0000","last_peered":"2024-03-04T07:30:06.070519+0000","last_clean":"2024-03-04T07:30:06.070519+0000","last_became_active":"2024-03-04T07:30:05.096133+0000","last_became_peered":"2024-03-04T07:30:05.096133+0000","last_unstale":"2024-03-04T07:30:06.070519+0000","last_undegraded":"2024-03-04T07:30:06.070519+0000","last_fullsized":"2024-03-04T07:30:06.070519+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:44:54.507339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093046+0000","last_change":"2024-03-04T07:30:05.088696+0000","last_active":"2024-03-04T07:30:05.093046+0000","last_peered":"2024-03-04T07:30:05.093046+0000","last_clean":"2024-03-04T07:30:05.093046+0000","last_became_active":"2024-03-04T07:30:05.087106+0000","last_became_peered":"2024-03-04T07:30:05.087106+0000","last_unstale":"2024-03-04T07:30:05.093046+0000","last_undegraded":"2024-03-04T07:30:05.093046+0000","last_fullsized":"2024-03-04T07:30:05.093046+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:58.076549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.2e","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080040+0000","last_change":"2024-03-04T07:30:05.098425+0000","last_active":"2024-03-04T07:30:06.080040+0000","last_peered":"2024-03-04T07:30:06.080040+0000","last_clean":"2024-03-04T07:30:06.080040+0000","last_became_active":"2024-03-04T07:30:05.098130+0000","last_became_peered":"2024-03-04T07:30:05.098130+0000","last_unstale":"2024-03-04T07:30:06.080040+0000","last_undegraded":"2024-03-04T07:30:06.080040+0000","last_fullsized":"2024-03-04T07:30:06.080040+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:04.703704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.2d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098601+0000","last_change":"2024-03-04T07:30:05.094019+0000","last_active":"2024-03-04T07:30:05.098601+0000","last_peered":"2024-03-04T07:30:05.098601+0000","last_clean":"2024-03-04T07:30:05.098601+0000","last_became_active":"2024-03-04T07:30:05.093273+0000","last_became_peered":"2024-03-04T07:30:05.093273+0000","last_unstale":"2024-03-04T07:30:05.098601+0000","last_undegraded":"2024-03-04T07:30:05.098601+0000","last_fullsized":"2024-03-04T07:30:05.098601+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:04.703704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.2c","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071755+0000","last_change":"2024-03-04T07:30:05.077646+0000","last_active":"2024-03-04T07:30:06.071755+0000","last_peered":"2024-03-04T07:30:06.071755+0000","last_clean":"2024-03-04T07:30:06.071755+0000","last_became_active":"2024-03-04T07:30:05.077329+0000","last_became_peered":"2024-03-04T07:30:05.077329+0000","last_unstale":"2024-03-04T07:30:06.071755+0000","last_undegraded":"2024-03-04T07:30:06.071755+0000","last_fullsized":"2024-03-04T07:30:06.071755+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:00.798986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2b","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070715+0000","last_change":"2024-03-04T07:30:05.080907+0000","last_active":"2024-03-04T07:30:06.070715+0000","last_peered":"2024-03-04T07:30:06.070715+0000","last_clean":"2024-03-04T07:30:06.070715+0000","last_became_active":"2024-03-04T07:30:05.080641+0000","last_became_peered":"2024-03-04T07:30:05.080641+0000","last_unstale":"2024-03-04T07:30:06.070715+0000","last_undegraded":"2024-03-04T07:30:06.070715+0000","last_fullsized":"2024-03-04T07:30:06.070715+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:09:31.879681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2a","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071329+0000","last_change":"2024-03-04T07:30:05.079488+0000","last_active":"2024-03-04T07:30:06.071329+0000","last_peered":"2024-03-04T07:30:06.071329+0000","last_clean":"2024-03-04T07:30:06.071329+0000","last_became_active":"2024-03-04T07:30:05.079182+0000","last_became_peered":"2024-03-04T07:30:05.079182+0000","last_unstale":"2024-03-04T07:30:06.071329+0000","last_undegraded":"2024-03-04T07:30:06.071329+0000","last_fullsized":"2024-03-04T07:30:06.071329+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:00:04.416316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.29","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090834+0000","last_change":"2024-03-04T07:30:05.078651+0000","last_active":"2024-03-04T07:30:05.090834+0000","last_peered":"2024-03-04T07:30:05.090834+0000","last_clean":"2024-03-04T07:30:05.090834+0000","last_became_active":"2024-03-04T07:30:05.078341+0000","last_became_peered":"2024-03-04T07:30:05.078341+0000","last_unstale":"2024-03-04T07:30:05.090834+0000","last_undegraded":"2024-03-04T07:30:05.090834+0000","last_fullsized":"2024-03-04T07:30:05.090834+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:08:39.102228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.28","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099105+0000","last_change":"2024-03-04T07:30:05.096767+0000","last_active":"2024-03-04T07:30:05.099105+0000","last_peered":"2024-03-04T07:30:05.099105+0000","last_clean":"2024-03-04T07:30:05.099105+0000","last_became_active":"2024-03-04T07:30:05.096069+0000","last_became_peered":"2024-03-04T07:30:05.096069+0000","last_unstale":"2024-03-04T07:30:05.099105+0000","last_undegraded":"2024-03-04T07:30:05.099105+0000","last_fullsized":"2024-03-04T07:30:05.099105+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:17:54.586600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.27","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070836+0000","last_change":"2024-03-04T07:30:05.085832+0000","last_active":"2024-03-04T07:30:06.070836+0000","last_peered":"2024-03-04T07:30:06.070836+0000","last_clean":"2024-03-04T07:30:06.070836+0000","last_became_active":"2024-03-04T07:30:05.085508+0000","last_became_peered":"2024-03-04T07:30:05.085508+0000","last_unstale":"2024-03-04T07:30:06.070836+0000","last_undegraded":"2024-03-04T07:30:06.070836+0000","last_fullsized":"2024-03-04T07:30:06.070836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:33:51.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.26","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071089+0000","last_change":"2024-03-04T07:30:05.096149+0000","last_active":"2024-03-04T07:30:06.071089+0000","last_peered":"2024-03-04T07:30:06.071089+0000","last_clean":"2024-03-04T07:30:06.071089+0000","last_became_active":"2024-03-04T07:30:05.095699+0000","last_became_peered":"2024-03-04T07:30:05.095699+0000","last_unstale":"2024-03-04T07:30:06.071089+0000","last_undegraded":"2024-03-04T07:30:06.071089+0000","last_fullsized":"2024-03-04T07:30:06.071089+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:41.093583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.25","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090810+0000","last_change":"2024-03-04T07:30:05.088843+0000","last_active":"2024-03-04T07:30:05.090810+0000","last_peered":"2024-03-04T07:30:05.090810+0000","last_clean":"2024-03-04T07:30:05.090810+0000","last_became_active":"2024-03-04T07:30:05.088208+0000","last_became_peered":"2024-03-04T07:30:05.088208+0000","last_unstale":"2024-03-04T07:30:05.090810+0000","last_undegraded":"2024-03-04T07:30:05.090810+0000","last_fullsized":"2024-03-04T07:30:05.090810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:19.737517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.24","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071646+0000","last_change":"2024-03-04T07:30:05.079614+0000","last_active":"2024-03-04T07:30:06.071646+0000","last_peered":"2024-03-04T07:30:06.071646+0000","last_clean":"2024-03-04T07:30:06.071646+0000","last_became_active":"2024-03-04T07:30:05.079273+0000","last_became_peered":"2024-03-04T07:30:05.079273+0000","last_unstale":"2024-03-04T07:30:06.071646+0000","last_undegraded":"2024-03-04T07:30:06.071646+0000","last_fullsized":"2024-03-04T07:30:06.071646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.23","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.082872+0000","last_change":"2024-03-04T07:30:05.080645+0000","last_active":"2024-03-04T07:30:05.082872+0000","last_peered":"2024-03-04T07:30:05.082872+0000","last_clean":"2024-03-04T07:30:05.082872+0000","last_became_active":"2024-03-04T07:30:05.080390+0000","last_became_peered":"2024-03-04T07:30:05.080390+0000","last_unstale":"2024-03-04T07:30:05.082872+0000","last_undegraded":"2024-03-04T07:30:05.082872+0000","last_fullsized":"2024-03-04T07:30:05.082872+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:41:10.488917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.22","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070493+0000","last_change":"2024-03-04T07:30:05.094084+0000","last_active":"2024-03-04T07:30:06.070493+0000","last_peered":"2024-03-04T07:30:06.070493+0000","last_clean":"2024-03-04T07:30:06.070493+0000","last_became_active":"2024-03-04T07:30:05.093772+0000","last_became_peered":"2024-03-04T07:30:05.093772+0000","last_unstale":"2024-03-04T07:30:06.070493+0000","last_undegraded":"2024-03-04T07:30:06.070493+0000","last_fullsized":"2024-03-04T07:30:06.070493+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:09.457728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.21","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070809+0000","last_change":"2024-03-04T07:30:05.080374+0000","last_active":"2024-03-04T07:30:06.070809+0000","last_peered":"2024-03-04T07:30:06.070809+0000","last_clean":"2024-03-04T07:30:06.070809+0000","last_became_active":"2024-03-04T07:30:05.080075+0000","last_became_peered":"2024-03-04T07:30:05.080075+0000","last_unstale":"2024-03-04T07:30:06.070809+0000","last_undegraded":"2024-03-04T07:30:06.070809+0000","last_fullsized":"2024-03-04T07:30:06.070809+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:28:21.513469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.20","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099498+0000","last_change":"2024-03-04T07:30:05.095504+0000","last_active":"2024-03-04T07:30:05.099498+0000","last_peered":"2024-03-04T07:30:05.099498+0000","last_clean":"2024-03-04T07:30:05.099498+0000","last_became_active":"2024-03-04T07:30:05.095188+0000","last_became_peered":"2024-03-04T07:30:05.095188+0000","last_unstale":"2024-03-04T07:30:05.099498+0000","last_undegraded":"2024-03-04T07:30:05.099498+0000","last_fullsized":"2024-03-04T07:30:05.099498+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:52.747064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.1e","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099058+0000","last_change":"2024-03-04T07:30:05.096047+0000","last_active":"2024-03-04T07:30:05.099058+0000","last_peered":"2024-03-04T07:30:05.099058+0000","last_clean":"2024-03-04T07:30:05.099058+0000","last_became_active":"2024-03-04T07:30:05.095751+0000","last_became_peered":"2024-03-04T07:30:05.095751+0000","last_unstale":"2024-03-04T07:30:05.099058+0000","last_undegraded":"2024-03-04T07:30:05.099058+0000","last_fullsized":"2024-03-04T07:30:05.099058+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:52:38.600368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.1f","version":"65'4","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.096236+0000","last_change":"2024-03-04T07:30:04.081175+0000","last_active":"2024-03-04T07:30:05.096236+0000","last_peered":"2024-03-04T07:30:05.096236+0000","last_clean":"2024-03-04T07:30:05.096236+0000","last_became_active":"2024-03-04T07:30:04.080863+0000","last_became_peered":"2024-03-04T07:30:04.080863+0000","last_unstale":"2024-03-04T07:30:05.096236+0000","last_undegraded":"2024-03-04T07:30:05.096236+0000","last_fullsized":"2024-03-04T07:30:05.096236+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:38:53.665426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.1f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089351+0000","last_change":"2024-03-04T07:30:05.088078+0000","last_active":"2024-03-04T07:30:05.089351+0000","last_peered":"2024-03-04T07:30:05.089351+0000","last_clean":"2024-03-04T07:30:05.089351+0000","last_became_active":"2024-03-04T07:30:05.086783+0000","last_became_peered":"2024-03-04T07:30:05.086783+0000","last_unstale":"2024-03-04T07:30:05.089351+0000","last_undegraded":"2024-03-04T07:30:05.089351+0000","last_fullsized":"2024-03-04T07:30:05.089351+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:14:25.100757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.1e","version":"65'1","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071927+0000","last_change":"2024-03-04T07:30:04.082721+0000","last_active":"2024-03-04T07:30:06.071927+0000","last_peered":"2024-03-04T07:30:06.071927+0000","last_clean":"2024-03-04T07:30:06.071927+0000","last_became_active":"2024-03-04T07:30:04.082417+0000","last_became_peered":"2024-03-04T07:30:04.082417+0000","last_unstale":"2024-03-04T07:30:06.071927+0000","last_undegraded":"2024-03-04T07:30:06.071927+0000","last_fullsized":"2024-03-04T07:30:06.071927+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:22.005661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.1c","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099482+0000","last_change":"2024-03-04T07:30:05.097528+0000","last_active":"2024-03-04T07:30:05.099482+0000","last_peered":"2024-03-04T07:30:05.099482+0000","last_clean":"2024-03-04T07:30:05.099482+0000","last_became_active":"2024-03-04T07:30:05.095278+0000","last_became_peered":"2024-03-04T07:30:05.095278+0000","last_unstale":"2024-03-04T07:30:05.099482+0000","last_undegraded":"2024-03-04T07:30:05.099482+0000","last_fullsized":"2024-03-04T07:30:05.099482+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:40:03.427093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1d","version":"65'2","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071298+0000","last_change":"2024-03-04T07:30:04.083247+0000","last_active":"2024-03-04T07:30:06.071298+0000","last_peered":"2024-03-04T07:30:06.071298+0000","last_clean":"2024-03-04T07:30:06.071298+0000","last_became_active":"2024-03-04T07:30:04.082936+0000","last_became_peered":"2024-03-04T07:30:04.082936+0000","last_unstale":"2024-03-04T07:30:06.071298+0000","last_undegraded":"2024-03-04T07:30:06.071298+0000","last_fullsized":"2024-03-04T07:30:06.071298+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:44:20.200529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.091158+0000","last_change":"2024-03-04T07:30:05.083224+0000","last_active":"2024-03-04T07:30:05.091158+0000","last_peered":"2024-03-04T07:30:05.091158+0000","last_clean":"2024-03-04T07:30:05.091158+0000","last_became_active":"2024-03-04T07:30:05.082955+0000","last_became_peered":"2024-03-04T07:30:05.082955+0000","last_unstale":"2024-03-04T07:30:05.091158+0000","last_undegraded":"2024-03-04T07:30:05.091158+0000","last_fullsized":"2024-03-04T07:30:05.091158+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:33:51.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.1c","version":"65'2","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.100492+0000","last_change":"2024-03-04T07:30:04.087388+0000","last_active":"2024-03-04T07:30:05.100492+0000","last_peered":"2024-03-04T07:30:05.100492+0000","last_clean":"2024-03-04T07:30:05.100492+0000","last_became_active":"2024-03-04T07:30:04.087100+0000","last_became_peered":"2024-03-04T07:30:04.087100+0000","last_unstale":"2024-03-04T07:30:05.100492+0000","last_undegraded":"2024-03-04T07:30:05.100492+0000","last_fullsized":"2024-03-04T07:30:05.100492+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:42:56.438271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.1a","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070701+0000","last_change":"2024-03-04T07:30:05.083234+0000","last_active":"2024-03-04T07:30:06.070701+0000","last_peered":"2024-03-04T07:30:06.070701+0000","last_clean":"2024-03-04T07:30:06.070701+0000","last_became_active":"2024-03-04T07:30:05.082948+0000","last_became_peered":"2024-03-04T07:30:05.082948+0000","last_unstale":"2024-03-04T07:30:06.070701+0000","last_undegraded":"2024-03-04T07:30:06.070701+0000","last_fullsized":"2024-03-04T07:30:06.070701+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.1b","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093702+0000","last_change":"2024-03-04T07:30:04.076239+0000","last_active":"2024-03-04T07:30:05.093702+0000","last_peered":"2024-03-04T07:30:05.093702+0000","last_clean":"2024-03-04T07:30:05.093702+0000","last_became_active":"2024-03-04T07:30:04.075758+0000","last_became_peered":"2024-03-04T07:30:04.075758+0000","last_unstale":"2024-03-04T07:30:05.093702+0000","last_undegraded":"2024-03-04T07:30:05.093702+0000","last_fullsized":"2024-03-04T07:30:05.093702+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:35:01.014832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1b","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070894+0000","last_change":"2024-03-04T07:30:05.087307+0000","last_active":"2024-03-04T07:30:06.070894+0000","last_peered":"2024-03-04T07:30:06.070894+0000","last_clean":"2024-03-04T07:30:06.070894+0000","last_became_active":"2024-03-04T07:30:05.085836+0000","last_became_peered":"2024-03-04T07:30:05.085836+0000","last_unstale":"2024-03-04T07:30:06.070894+0000","last_undegraded":"2024-03-04T07:30:06.070894+0000","last_fullsized":"2024-03-04T07:30:06.070894+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:04:11.464737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1a","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.097530+0000","last_change":"2024-03-04T07:30:04.084488+0000","last_active":"2024-03-04T07:30:05.097530+0000","last_peered":"2024-03-04T07:30:05.097530+0000","last_clean":"2024-03-04T07:30:05.097530+0000","last_became_active":"2024-03-04T07:30:04.083989+0000","last_became_peered":"2024-03-04T07:30:04.083989+0000","last_unstale":"2024-03-04T07:30:05.097530+0000","last_undegraded":"2024-03-04T07:30:05.097530+0000","last_fullsized":"2024-03-04T07:30:05.097530+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:47:31.326424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.18","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.088818+0000","last_change":"2024-03-04T07:30:05.084596+0000","last_active":"2024-03-04T07:30:05.088818+0000","last_peered":"2024-03-04T07:30:05.088818+0000","last_clean":"2024-03-04T07:30:05.088818+0000","last_became_active":"2024-03-04T07:30:05.084281+0000","last_became_peered":"2024-03-04T07:30:05.084281+0000","last_unstale":"2024-03-04T07:30:05.088818+0000","last_undegraded":"2024-03-04T07:30:05.088818+0000","last_fullsized":"2024-03-04T07:30:05.088818+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:34:52.102333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.19","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079844+0000","last_change":"2024-03-04T07:30:04.075017+0000","last_active":"2024-03-04T07:30:06.079844+0000","last_peered":"2024-03-04T07:30:06.079844+0000","last_clean":"2024-03-04T07:30:06.079844+0000","last_became_active":"2024-03-04T07:30:04.074485+0000","last_became_peered":"2024-03-04T07:30:04.074485+0000","last_unstale":"2024-03-04T07:30:06.079844+0000","last_undegraded":"2024-03-04T07:30:06.079844+0000","last_fullsized":"2024-03-04T07:30:06.079844+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:55.834473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.19","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071671+0000","last_change":"2024-03-04T07:30:05.080173+0000","last_active":"2024-03-04T07:30:06.071671+0000","last_peered":"2024-03-04T07:30:06.071671+0000","last_clean":"2024-03-04T07:30:06.071671+0000","last_became_active":"2024-03-04T07:30:05.079907+0000","last_became_peered":"2024-03-04T07:30:05.079907+0000","last_unstale":"2024-03-04T07:30:06.071671+0000","last_undegraded":"2024-03-04T07:30:06.071671+0000","last_fullsized":"2024-03-04T07:30:06.071671+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:17:54.586600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.18","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.073846+0000","last_change":"2024-03-04T07:30:04.075015+0000","last_active":"2024-03-04T07:30:05.073846+0000","last_peered":"2024-03-04T07:30:05.073846+0000","last_clean":"2024-03-04T07:30:05.073846+0000","last_became_active":"2024-03-04T07:30:04.074699+0000","last_became_peered":"2024-03-04T07:30:04.074699+0000","last_unstale":"2024-03-04T07:30:05.073846+0000","last_undegraded":"2024-03-04T07:30:05.073846+0000","last_fullsized":"2024-03-04T07:30:05.073846+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:53.436832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.16","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071240+0000","last_change":"2024-03-04T07:30:05.078927+0000","last_active":"2024-03-04T07:30:06.071240+0000","last_peered":"2024-03-04T07:30:06.071240+0000","last_clean":"2024-03-04T07:30:06.071240+0000","last_became_active":"2024-03-04T07:30:05.078602+0000","last_became_peered":"2024-03-04T07:30:05.078602+0000","last_unstale":"2024-03-04T07:30:06.071240+0000","last_undegraded":"2024-03-04T07:30:06.071240+0000","last_fullsized":"2024-03-04T07:30:06.071240+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:55.231331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.17","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070945+0000","last_change":"2024-03-04T07:30:04.078857+0000","last_active":"2024-03-04T07:30:06.070945+0000","last_peered":"2024-03-04T07:30:06.070945+0000","last_clean":"2024-03-04T07:30:06.070945+0000","last_became_active":"2024-03-04T07:30:04.078578+0000","last_became_peered":"2024-03-04T07:30:04.078578+0000","last_unstale":"2024-03-04T07:30:06.070945+0000","last_undegraded":"2024-03-04T07:30:06.070945+0000","last_fullsized":"2024-03-04T07:30:06.070945+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:45:03.481435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.17","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099062+0000","last_change":"2024-03-04T07:30:05.096381+0000","last_active":"2024-03-04T07:30:05.099062+0000","last_peered":"2024-03-04T07:30:05.099062+0000","last_clean":"2024-03-04T07:30:05.099062+0000","last_became_active":"2024-03-04T07:30:05.095758+0000","last_became_peered":"2024-03-04T07:30:05.095758+0000","last_unstale":"2024-03-04T07:30:05.099062+0000","last_undegraded":"2024-03-04T07:30:05.099062+0000","last_fullsized":"2024-03-04T07:30:05.099062+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089996+0000","last_change":"2024-03-04T07:30:04.083596+0000","last_active":"2024-03-04T07:30:05.089996+0000","last_peered":"2024-03-04T07:30:05.089996+0000","last_clean":"2024-03-04T07:30:05.089996+0000","last_became_active":"2024-03-04T07:30:04.083258+0000","last_became_peered":"2024-03-04T07:30:04.083258+0000","last_unstale":"2024-03-04T07:30:05.089996+0000","last_undegraded":"2024-03-04T07:30:05.089996+0000","last_fullsized":"2024-03-04T07:30:05.089996+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:02:06.058460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.14","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099402+0000","last_change":"2024-03-04T07:30:05.095347+0000","last_active":"2024-03-04T07:30:05.099402+0000","last_peered":"2024-03-04T07:30:05.099402+0000","last_clean":"2024-03-04T07:30:05.099402+0000","last_became_active":"2024-03-04T07:30:05.094728+0000","last_became_peered":"2024-03-04T07:30:05.094728+0000","last_unstale":"2024-03-04T07:30:05.099402+0000","last_undegraded":"2024-03-04T07:30:05.099402+0000","last_fullsized":"2024-03-04T07:30:05.099402+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:44:35.285031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.15","version":"65'3","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089052+0000","last_change":"2024-03-04T07:30:04.079117+0000","last_active":"2024-03-04T07:30:05.089052+0000","last_peered":"2024-03-04T07:30:05.089052+0000","last_clean":"2024-03-04T07:30:05.089052+0000","last_became_active":"2024-03-04T07:30:04.078754+0000","last_became_peered":"2024-03-04T07:30:04.078754+0000","last_unstale":"2024-03-04T07:30:05.089052+0000","last_undegraded":"2024-03-04T07:30:05.089052+0000","last_fullsized":"2024-03-04T07:30:05.089052+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:35:44.355784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.095438+0000","last_change":"2024-03-04T07:30:05.091558+0000","last_active":"2024-03-04T07:30:05.095438+0000","last_peered":"2024-03-04T07:30:05.095438+0000","last_clean":"2024-03-04T07:30:05.095438+0000","last_became_active":"2024-03-04T07:30:05.091290+0000","last_became_peered":"2024-03-04T07:30:05.091290+0000","last_unstale":"2024-03-04T07:30:05.095438+0000","last_undegraded":"2024-03-04T07:30:05.095438+0000","last_fullsized":"2024-03-04T07:30:05.095438+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:54.753453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.14","version":"65'6","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080230+0000","last_change":"2024-03-04T07:30:04.083002+0000","last_active":"2024-03-04T07:30:06.080230+0000","last_peered":"2024-03-04T07:30:06.080230+0000","last_clean":"2024-03-04T07:30:06.080230+0000","last_became_active":"2024-03-04T07:30:04.082223+0000","last_became_peered":"2024-03-04T07:30:04.082223+0000","last_unstale":"2024-03-04T07:30:06.080230+0000","last_undegraded":"2024-03-04T07:30:06.080230+0000","last_fullsized":"2024-03-04T07:30:06.080230+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:20:35.593676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090944+0000","last_change":"2024-03-04T07:30:05.080091+0000","last_active":"2024-03-04T07:30:05.090944+0000","last_peered":"2024-03-04T07:30:05.090944+0000","last_clean":"2024-03-04T07:30:05.090944+0000","last_became_active":"2024-03-04T07:30:05.079837+0000","last_became_peered":"2024-03-04T07:30:05.079837+0000","last_unstale":"2024-03-04T07:30:05.090944+0000","last_undegraded":"2024-03-04T07:30:05.090944+0000","last_fullsized":"2024-03-04T07:30:05.090944+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:44:54.507339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.13","version":"65'2","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079974+0000","last_change":"2024-03-04T07:30:04.080925+0000","last_active":"2024-03-04T07:30:06.079974+0000","last_peered":"2024-03-04T07:30:06.079974+0000","last_clean":"2024-03-04T07:30:06.079974+0000","last_became_active":"2024-03-04T07:30:04.080597+0000","last_became_peered":"2024-03-04T07:30:04.080597+0000","last_unstale":"2024-03-04T07:30:06.079974+0000","last_undegraded":"2024-03-04T07:30:06.079974+0000","last_fullsized":"2024-03-04T07:30:06.079974+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:47:31.326424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.13","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070966+0000","last_change":"2024-03-04T07:30:05.074092+0000","last_active":"2024-03-04T07:30:06.070966+0000","last_peered":"2024-03-04T07:30:06.070966+0000","last_clean":"2024-03-04T07:30:06.070966+0000","last_became_active":"2024-03-04T07:30:05.073786+0000","last_became_peered":"2024-03-04T07:30:05.073786+0000","last_unstale":"2024-03-04T07:30:06.070966+0000","last_undegraded":"2024-03-04T07:30:06.070966+0000","last_fullsized":"2024-03-04T07:30:06.070966+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:00.798986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.087930+0000","last_change":"2024-03-04T07:30:04.082134+0000","last_active":"2024-03-04T07:30:05.087930+0000","last_peered":"2024-03-04T07:30:05.087930+0000","last_clean":"2024-03-04T07:30:05.087930+0000","last_became_active":"2024-03-04T07:30:04.081835+0000","last_became_peered":"2024-03-04T07:30:04.081835+0000","last_unstale":"2024-03-04T07:30:05.087930+0000","last_undegraded":"2024-03-04T07:30:05.087930+0000","last_fullsized":"2024-03-04T07:30:05.087930+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:49:20.014438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.10","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071622+0000","last_change":"2024-03-04T07:30:05.097365+0000","last_active":"2024-03-04T07:30:06.071622+0000","last_peered":"2024-03-04T07:30:06.071622+0000","last_clean":"2024-03-04T07:30:06.071622+0000","last_became_active":"2024-03-04T07:30:05.097140+0000","last_became_peered":"2024-03-04T07:30:05.097140+0000","last_unstale":"2024-03-04T07:30:06.071622+0000","last_undegraded":"2024-03-04T07:30:06.071622+0000","last_fullsized":"2024-03-04T07:30:06.071622+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070639+0000","last_change":"2024-03-04T07:30:04.085183+0000","last_active":"2024-03-04T07:30:06.070639+0000","last_peered":"2024-03-04T07:30:06.070639+0000","last_clean":"2024-03-04T07:30:06.070639+0000","last_became_active":"2024-03-04T07:30:04.084915+0000","last_became_peered":"2024-03-04T07:30:04.084915+0000","last_unstale":"2024-03-04T07:30:06.070639+0000","last_undegraded":"2024-03-04T07:30:06.070639+0000","last_fullsized":"2024-03-04T07:30:06.070639+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:45:54.708994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.11","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089287+0000","last_change":"2024-03-04T07:30:05.086586+0000","last_active":"2024-03-04T07:30:05.089287+0000","last_peered":"2024-03-04T07:30:05.089287+0000","last_clean":"2024-03-04T07:30:05.089287+0000","last_became_active":"2024-03-04T07:30:05.086320+0000","last_became_peered":"2024-03-04T07:30:05.086320+0000","last_unstale":"2024-03-04T07:30:05.089287+0000","last_undegraded":"2024-03-04T07:30:05.089287+0000","last_fullsized":"2024-03-04T07:30:05.089287+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:46:05.386449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.10","version":"65'8","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070650+0000","last_change":"2024-03-04T07:30:04.077819+0000","last_active":"2024-03-04T07:30:06.070650+0000","last_peered":"2024-03-04T07:30:06.070650+0000","last_clean":"2024-03-04T07:30:06.070650+0000","last_became_active":"2024-03-04T07:30:04.077507+0000","last_became_peered":"2024-03-04T07:30:04.077507+0000","last_unstale":"2024-03-04T07:30:06.070650+0000","last_undegraded":"2024-03-04T07:30:06.070650+0000","last_fullsized":"2024-03-04T07:30:06.070650+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:19:12.786935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071645+0000","last_change":"2024-03-04T07:30:05.089680+0000","last_active":"2024-03-04T07:30:06.071645+0000","last_peered":"2024-03-04T07:30:06.071645+0000","last_clean":"2024-03-04T07:30:06.071645+0000","last_became_active":"2024-03-04T07:30:05.089403+0000","last_became_peered":"2024-03-04T07:30:05.089403+0000","last_unstale":"2024-03-04T07:30:06.071645+0000","last_undegraded":"2024-03-04T07:30:06.071645+0000","last_fullsized":"2024-03-04T07:30:06.071645+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:52:38.600368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.f","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098212+0000","last_change":"2024-03-04T07:30:04.085909+0000","last_active":"2024-03-04T07:30:05.098212+0000","last_peered":"2024-03-04T07:30:05.098212+0000","last_clean":"2024-03-04T07:30:05.098212+0000","last_became_active":"2024-03-04T07:30:04.085611+0000","last_became_peered":"2024-03-04T07:30:04.085611+0000","last_unstale":"2024-03-04T07:30:05.098212+0000","last_undegraded":"2024-03-04T07:30:05.098212+0000","last_fullsized":"2024-03-04T07:30:05.098212+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:18:30.192631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093562+0000","last_change":"2024-03-04T07:30:05.091451+0000","last_active":"2024-03-04T07:30:05.093562+0000","last_peered":"2024-03-04T07:30:05.093562+0000","last_clean":"2024-03-04T07:30:05.093562+0000","last_became_active":"2024-03-04T07:30:05.091169+0000","last_became_peered":"2024-03-04T07:30:05.091169+0000","last_unstale":"2024-03-04T07:30:05.093562+0000","last_undegraded":"2024-03-04T07:30:05.093562+0000","last_fullsized":"2024-03-04T07:30:05.093562+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:00:04.416316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"65'2","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071686+0000","last_change":"2024-03-04T07:30:04.084651+0000","last_active":"2024-03-04T07:30:06.071686+0000","last_peered":"2024-03-04T07:30:06.071686+0000","last_clean":"2024-03-04T07:30:06.071686+0000","last_became_active":"2024-03-04T07:30:04.084108+0000","last_became_peered":"2024-03-04T07:30:04.084108+0000","last_unstale":"2024-03-04T07:30:06.071686+0000","last_undegraded":"2024-03-04T07:30:06.071686+0000","last_fullsized":"2024-03-04T07:30:06.071686+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:35:28.188240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080190+0000","last_change":"2024-03-04T07:30:05.090055+0000","last_active":"2024-03-04T07:30:06.080190+0000","last_peered":"2024-03-04T07:30:06.080190+0000","last_clean":"2024-03-04T07:30:06.080190+0000","last_became_active":"2024-03-04T07:30:05.089654+0000","last_became_peered":"2024-03-04T07:30:05.089654+0000","last_unstale":"2024-03-04T07:30:06.080190+0000","last_undegraded":"2024-03-04T07:30:06.080190+0000","last_fullsized":"2024-03-04T07:30:06.080190+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:52.747064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.097403+0000","last_change":"2024-03-04T07:30:04.088337+0000","last_active":"2024-03-04T07:30:05.097403+0000","last_peered":"2024-03-04T07:30:05.097403+0000","last_clean":"2024-03-04T07:30:05.097403+0000","last_became_active":"2024-03-04T07:30:04.088004+0000","last_became_peered":"2024-03-04T07:30:04.088004+0000","last_unstale":"2024-03-04T07:30:05.097403+0000","last_undegraded":"2024-03-04T07:30:05.097403+0000","last_fullsized":"2024-03-04T07:30:05.097403+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:44:15.112781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.088517+0000","last_change":"2024-03-04T07:30:05.084439+0000","last_active":"2024-03-04T07:30:05.088517+0000","last_peered":"2024-03-04T07:30:05.088517+0000","last_clean":"2024-03-04T07:30:05.088517+0000","last_became_active":"2024-03-04T07:30:05.084151+0000","last_became_peered":"2024-03-04T07:30:05.084151+0000","last_unstale":"2024-03-04T07:30:05.088517+0000","last_undegraded":"2024-03-04T07:30:05.088517+0000","last_fullsized":"2024-03-04T07:30:05.088517+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:37:15.704917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.074624+0000","last_change":"2024-03-04T07:30:04.088232+0000","last_active":"2024-03-04T07:30:05.074624+0000","last_peered":"2024-03-04T07:30:05.074624+0000","last_clean":"2024-03-04T07:30:05.074624+0000","last_became_active":"2024-03-04T07:30:04.087775+0000","last_became_peered":"2024-03-04T07:30:04.087775+0000","last_unstale":"2024-03-04T07:30:05.074624+0000","last_undegraded":"2024-03-04T07:30:05.074624+0000","last_fullsized":"2024-03-04T07:30:05.074624+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:10:21.100819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.095590+0000","last_change":"2024-03-04T07:30:05.092123+0000","last_active":"2024-03-04T07:30:05.095590+0000","last_peered":"2024-03-04T07:30:05.095590+0000","last_clean":"2024-03-04T07:30:05.095590+0000","last_became_active":"2024-03-04T07:30:05.091828+0000","last_became_peered":"2024-03-04T07:30:05.091828+0000","last_unstale":"2024-03-04T07:30:05.095590+0000","last_undegraded":"2024-03-04T07:30:05.095590+0000","last_fullsized":"2024-03-04T07:30:05.095590+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:44:37.388344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.088174+0000","last_change":"2024-03-04T07:30:04.084269+0000","last_active":"2024-03-04T07:30:05.088174+0000","last_peered":"2024-03-04T07:30:05.088174+0000","last_clean":"2024-03-04T07:30:05.088174+0000","last_became_active":"2024-03-04T07:30:04.083967+0000","last_became_peered":"2024-03-04T07:30:04.083967+0000","last_unstale":"2024-03-04T07:30:05.088174+0000","last_undegraded":"2024-03-04T07:30:05.088174+0000","last_fullsized":"2024-03-04T07:30:05.088174+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:21.237751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.100273+0000","last_change":"2024-03-04T07:30:05.096068+0000","last_active":"2024-03-04T07:30:05.100273+0000","last_peered":"2024-03-04T07:30:05.100273+0000","last_clean":"2024-03-04T07:30:05.100273+0000","last_became_active":"2024-03-04T07:30:05.095805+0000","last_became_peered":"2024-03-04T07:30:05.095805+0000","last_unstale":"2024-03-04T07:30:05.100273+0000","last_undegraded":"2024-03-04T07:30:05.100273+0000","last_fullsized":"2024-03-04T07:30:05.100273+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:03.769544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070921+0000","last_change":"2024-03-04T07:30:04.080719+0000","last_active":"2024-03-04T07:30:06.070921+0000","last_peered":"2024-03-04T07:30:06.070921+0000","last_clean":"2024-03-04T07:30:06.070921+0000","last_became_active":"2024-03-04T07:30:04.080390+0000","last_became_peered":"2024-03-04T07:30:04.080390+0000","last_unstale":"2024-03-04T07:30:06.070921+0000","last_undegraded":"2024-03-04T07:30:06.070921+0000","last_fullsized":"2024-03-04T07:30:06.070921+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:33:29.956468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.094927+0000","last_change":"2024-03-04T07:30:05.090639+0000","last_active":"2024-03-04T07:30:05.094927+0000","last_peered":"2024-03-04T07:30:05.094927+0000","last_clean":"2024-03-04T07:30:05.094927+0000","last_became_active":"2024-03-04T07:30:05.090378+0000","last_became_peered":"2024-03-04T07:30:05.090378+0000","last_unstale":"2024-03-04T07:30:05.094927+0000","last_undegraded":"2024-03-04T07:30:05.094927+0000","last_fullsized":"2024-03-04T07:30:05.094927+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.725745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071552+0000","last_change":"2024-03-04T07:30:04.083013+0000","last_active":"2024-03-04T07:30:06.071552+0000","last_peered":"2024-03-04T07:30:06.071552+0000","last_clean":"2024-03-04T07:30:06.071552+0000","last_became_active":"2024-03-04T07:30:04.082697+0000","last_became_peered":"2024-03-04T07:30:04.082697+0000","last_unstale":"2024-03-04T07:30:06.071552+0000","last_undegraded":"2024-03-04T07:30:06.071552+0000","last_fullsized":"2024-03-04T07:30:06.071552+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:38:53.665426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071924+0000","last_change":"2024-03-04T07:30:05.079174+0000","last_active":"2024-03-04T07:30:06.071924+0000","last_peered":"2024-03-04T07:30:06.071924+0000","last_clean":"2024-03-04T07:30:06.071924+0000","last_became_active":"2024-03-04T07:30:05.078886+0000","last_became_peered":"2024-03-04T07:30:05.078886+0000","last_unstale":"2024-03-04T07:30:06.071924+0000","last_undegraded":"2024-03-04T07:30:06.071924+0000","last_fullsized":"2024-03-04T07:30:06.071924+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:55.231331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.7","version":"65'2","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098729+0000","last_change":"2024-03-04T07:30:04.085269+0000","last_active":"2024-03-04T07:30:05.098729+0000","last_peered":"2024-03-04T07:30:05.098729+0000","last_clean":"2024-03-04T07:30:05.098729+0000","last_became_active":"2024-03-04T07:30:04.084644+0000","last_became_peered":"2024-03-04T07:30:04.084644+0000","last_unstale":"2024-03-04T07:30:05.098729+0000","last_undegraded":"2024-03-04T07:30:05.098729+0000","last_fullsized":"2024-03-04T07:30:05.098729+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:55:19.043273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070827+0000","last_change":"2024-03-04T07:30:05.089201+0000","last_active":"2024-03-04T07:30:06.070827+0000","last_peered":"2024-03-04T07:30:06.070827+0000","last_clean":"2024-03-04T07:30:06.070827+0000","last_became_active":"2024-03-04T07:30:05.088914+0000","last_became_peered":"2024-03-04T07:30:05.088914+0000","last_unstale":"2024-03-04T07:30:06.070827+0000","last_undegraded":"2024-03-04T07:30:06.070827+0000","last_fullsized":"2024-03-04T07:30:06.070827+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:17:54.586600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071335+0000","last_change":"2024-03-04T07:30:04.084243+0000","last_active":"2024-03-04T07:30:06.071335+0000","last_peered":"2024-03-04T07:30:06.071335+0000","last_clean":"2024-03-04T07:30:06.071335+0000","last_became_active":"2024-03-04T07:30:04.083988+0000","last_became_peered":"2024-03-04T07:30:04.083988+0000","last_unstale":"2024-03-04T07:30:06.071335+0000","last_undegraded":"2024-03-04T07:30:06.071335+0000","last_fullsized":"2024-03-04T07:30:06.071335+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:32.049332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093715+0000","last_change":"2024-03-04T07:30:05.092852+0000","last_active":"2024-03-04T07:30:05.093715+0000","last_peered":"2024-03-04T07:30:05.093715+0000","last_clean":"2024-03-04T07:30:05.093715+0000","last_became_active":"2024-03-04T07:30:05.092547+0000","last_became_peered":"2024-03-04T07:30:05.092547+0000","last_unstale":"2024-03-04T07:30:05.093715+0000","last_undegraded":"2024-03-04T07:30:05.093715+0000","last_fullsized":"2024-03-04T07:30:05.093715+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:09.457728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.073700+0000","last_change":"2024-03-04T07:30:04.085522+0000","last_active":"2024-03-04T07:30:05.073700+0000","last_peered":"2024-03-04T07:30:05.073700+0000","last_clean":"2024-03-04T07:30:05.073700+0000","last_became_active":"2024-03-04T07:30:04.085239+0000","last_became_peered":"2024-03-04T07:30:04.085239+0000","last_unstale":"2024-03-04T07:30:05.073700+0000","last_undegraded":"2024-03-04T07:30:05.073700+0000","last_fullsized":"2024-03-04T07:30:05.073700+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:06:51.907642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.091135+0000","last_change":"2024-03-04T07:30:05.079345+0000","last_active":"2024-03-04T07:30:05.091135+0000","last_peered":"2024-03-04T07:30:05.091135+0000","last_clean":"2024-03-04T07:30:05.091135+0000","last_became_active":"2024-03-04T07:30:05.078981+0000","last_became_peered":"2024-03-04T07:30:05.078981+0000","last_unstale":"2024-03-04T07:30:05.091135+0000","last_undegraded":"2024-03-04T07:30:05.091135+0000","last_fullsized":"2024-03-04T07:30:05.091135+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:23.007594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071861+0000","last_change":"2024-03-04T07:30:04.088197+0000","last_active":"2024-03-04T07:30:06.071861+0000","last_peered":"2024-03-04T07:30:06.071861+0000","last_clean":"2024-03-04T07:30:06.071861+0000","last_became_active":"2024-03-04T07:30:04.087898+0000","last_became_peered":"2024-03-04T07:30:04.087898+0000","last_unstale":"2024-03-04T07:30:06.071861+0000","last_undegraded":"2024-03-04T07:30:06.071861+0000","last_fullsized":"2024-03-04T07:30:06.071861+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:08:38.100295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089324+0000","last_change":"2024-03-04T07:30:05.086424+0000","last_active":"2024-03-04T07:30:05.089324+0000","last_peered":"2024-03-04T07:30:05.089324+0000","last_clean":"2024-03-04T07:30:05.089324+0000","last_became_active":"2024-03-04T07:30:05.086129+0000","last_became_peered":"2024-03-04T07:30:05.086129+0000","last_unstale":"2024-03-04T07:30:05.089324+0000","last_undegraded":"2024-03-04T07:30:05.089324+0000","last_fullsized":"2024-03-04T07:30:05.089324+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:07:11.622279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.075994+0000","last_change":"2024-03-04T07:30:04.079841+0000","last_active":"2024-03-04T07:30:05.075994+0000","last_peered":"2024-03-04T07:30:05.075994+0000","last_clean":"2024-03-04T07:30:05.075994+0000","last_became_active":"2024-03-04T07:30:04.079576+0000","last_became_peered":"2024-03-04T07:30:04.079576+0000","last_unstale":"2024-03-04T07:30:05.075994+0000","last_undegraded":"2024-03-04T07:30:05.075994+0000","last_fullsized":"2024-03-04T07:30:05.075994+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:23:10.581374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098643+0000","last_change":"2024-03-04T07:30:05.093742+0000","last_active":"2024-03-04T07:30:05.098643+0000","last_peered":"2024-03-04T07:30:05.098643+0000","last_clean":"2024-03-04T07:30:05.098643+0000","last_became_active":"2024-03-04T07:30:05.093437+0000","last_became_peered":"2024-03-04T07:30:05.093437+0000","last_unstale":"2024-03-04T07:30:05.098643+0000","last_undegraded":"2024-03-04T07:30:05.098643+0000","last_fullsized":"2024-03-04T07:30:05.098643+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:37:23.639067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079887+0000","last_change":"2024-03-04T07:30:04.078433+0000","last_active":"2024-03-04T07:30:06.079887+0000","last_peered":"2024-03-04T07:30:06.079887+0000","last_clean":"2024-03-04T07:30:06.079887+0000","last_became_active":"2024-03-04T07:30:04.078130+0000","last_became_peered":"2024-03-04T07:30:04.078130+0000","last_unstale":"2024-03-04T07:30:06.079887+0000","last_undegraded":"2024-03-04T07:30:06.079887+0000","last_fullsized":"2024-03-04T07:30:06.079887+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:49:20.014438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079946+0000","last_change":"2024-03-04T07:30:05.086188+0000","last_active":"2024-03-04T07:30:06.079946+0000","last_peered":"2024-03-04T07:30:06.079946+0000","last_clean":"2024-03-04T07:30:06.079946+0000","last_became_active":"2024-03-04T07:30:05.085626+0000","last_became_peered":"2024-03-04T07:30:05.085626+0000","last_unstale":"2024-03-04T07:30:06.079946+0000","last_undegraded":"2024-03-04T07:30:06.079946+0000","last_fullsized":"2024-03-04T07:30:06.079946+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:37:23.639067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071208+0000","last_change":"2024-03-04T07:30:04.077165+0000","last_active":"2024-03-04T07:30:06.071208+0000","last_peered":"2024-03-04T07:30:06.071208+0000","last_clean":"2024-03-04T07:30:06.071208+0000","last_became_active":"2024-03-04T07:30:04.076855+0000","last_became_peered":"2024-03-04T07:30:04.076855+0000","last_unstale":"2024-03-04T07:30:06.071208+0000","last_undegraded":"2024-03-04T07:30:06.071208+0000","last_fullsized":"2024-03-04T07:30:06.071208+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:37:56.442757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093376+0000","last_change":"2024-03-04T07:30:05.087930+0000","last_active":"2024-03-04T07:30:05.093376+0000","last_peered":"2024-03-04T07:30:05.093376+0000","last_clean":"2024-03-04T07:30:05.093376+0000","last_became_active":"2024-03-04T07:30:05.087569+0000","last_became_peered":"2024-03-04T07:30:05.087569+0000","last_unstale":"2024-03-04T07:30:05.093376+0000","last_undegraded":"2024-03-04T07:30:05.093376+0000","last_fullsized":"2024-03-04T07:30:05.093376+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:20:36.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.085786+0000","last_change":"2024-03-04T07:30:04.090076+0000","last_active":"2024-03-04T07:30:05.085786+0000","last_peered":"2024-03-04T07:30:05.085786+0000","last_clean":"2024-03-04T07:30:05.085786+0000","last_became_active":"2024-03-04T07:30:04.089818+0000","last_became_peered":"2024-03-04T07:30:04.089818+0000","last_unstale":"2024-03-04T07:30:05.085786+0000","last_undegraded":"2024-03-04T07:30:05.085786+0000","last_fullsized":"2024-03-04T07:30:05.085786+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:36:02.896980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.100264+0000","last_change":"2024-03-04T07:30:05.095682+0000","last_active":"2024-03-04T07:30:05.100264+0000","last_peered":"2024-03-04T07:30:05.100264+0000","last_clean":"2024-03-04T07:30:05.100264+0000","last_became_active":"2024-03-04T07:30:05.095372+0000","last_became_peered":"2024-03-04T07:30:05.095372+0000","last_unstale":"2024-03-04T07:30:05.100264+0000","last_undegraded":"2024-03-04T07:30:05.100264+0000","last_fullsized":"2024-03-04T07:30:05.100264+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:48.729808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"65'2","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099688+0000","last_change":"2024-03-04T07:30:04.090065+0000","last_active":"2024-03-04T07:30:05.099688+0000","last_peered":"2024-03-04T07:30:05.099688+0000","last_clean":"2024-03-04T07:30:05.099688+0000","last_became_active":"2024-03-04T07:30:04.089768+0000","last_became_peered":"2024-03-04T07:30:04.089768+0000","last_unstale":"2024-03-04T07:30:05.099688+0000","last_undegraded":"2024-03-04T07:30:05.099688+0000","last_fullsized":"2024-03-04T07:30:05.099688+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:55.834473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":251,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.073930+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:30:05.073930+0000","last_peered":"2024-03-04T07:30:05.073930+0000","last_clean":"2024-03-04T07:30:05.073930+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:30:05.073930+0000","last_undegraded":"2024-03-04T07:30:05.073930+0000","last_fullsized":"2024-03-04T07:30:05.073930+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":10,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574894,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32548,"kb_used_data":1360,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339292,"statfs":{"total":96636764160,"available":96603435008,"internally_reserved":0,"allocated":1392640,"data_stored":920876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574893,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31968,"kb_used_data":780,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339872,"statfs":{"total":96636764160,"available":96604028928,"internally_reserved":0,"allocated":798720,"data_stored":329984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574893,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31964,"kb_used_data":776,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339876,"statfs":{"total":96636764160,"available":96604033024,"internally_reserved":0,"allocated":794624,"data_stored":329982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574894,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31960,"kb_used_data":776,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339880,"statfs":{"total":96636764160,"available":96604037120,"internally_reserved":0,"allocated":794624,"data_stored":329962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574893,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31968,"kb_used_data":780,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339872,"statfs":{"total":96636764160,"available":96604028928,"internally_reserved":0,"allocated":798720,"data_stored":330004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574893,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31972,"kb_used_data":784,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339868,"statfs":{"total":96636764160,"available":96604024832,"internally_reserved":0,"allocated":802816,"data_stored":331504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31980,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339860,"statfs":{"total":96636764160,"available":96604016640,"internally_reserved":0,"allocated":811008,"data_stored":332120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574893,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32532,"kb_used_data":1352,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339308,"statfs":{"total":96636764160,"available":96603451392,"internally_reserved":0,"allocated":1384448,"data_stored":920260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":92,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:30:08.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:30:08.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.369+0000 7f052593e640 1 Processor -- start 2024-03-04T07:30:08.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.369+0000 7f052593e640 1 -- start start 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f052593e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f05200b4260 0x7f05200cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f052593e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f05200b2320 con 0x7f05200d02e0 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f052593e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f05200b0dd0 con 0x7f05200d06c0 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f052593e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f05200b0b80 con 0x7f05200b4260 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f0517fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f05200b4260 0x7f05200cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f0517fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f05200b4260 0x7f05200cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49350/0 (socket says 172.21.15.64:49350) 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f051effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f05200d02e0 0x7f05200b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45702/0 (socket says 172.21.15.64:45702) 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.370+0000 7f0517fff640 1 -- 172.21.15.64:0/1786331680 learned_addr learned my addr 172.21.15.64:0/1786331680 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.371+0000 7f0517fff640 1 -- 172.21.15.64:0/1786331680 >> v1:172.21.15.66:6789/0 conn(0x7f05200d06c0 legacy=0x7f05200b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.371+0000 7f0517fff640 1 -- 172.21.15.64:0/1786331680 >> v1:172.21.15.64:6789/0 conn(0x7f05200d02e0 legacy=0x7f05200b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.371+0000 7f0517fff640 1 -- 172.21.15.64:0/1786331680 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05200b10b0 con 0x7f05200b4260 2024-03-04T07:30:08.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.371+0000 7f0517fff640 1 --2- 172.21.15.64:0/1786331680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200b4260 0x7f05200cfa00 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7f0508009870 tx=0x7f050802bab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e3c83febd09bb85b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f051e7fc640 1 -- 172.21.15.64:0/1786331680 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f050802d870 con 0x7f05200b4260 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f051e7fc640 1 -- 172.21.15.64:0/1786331680 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f050802da10 con 0x7f05200b4260 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f051e7fc640 1 -- 172.21.15.64:0/1786331680 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f050802dde0 con 0x7f05200b4260 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f052593e640 1 -- 172.21.15.64:0/1786331680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200b4260 msgr2=0x7f05200cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f052593e640 1 --2- 172.21.15.64:0/1786331680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200b4260 0x7f05200cfa00 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7f0508009870 tx=0x7f050802bab0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f052593e640 1 -- 172.21.15.64:0/1786331680 shutdown_connections 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f052593e640 1 --2- 172.21.15.64:0/1786331680 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200b4260 0x7f05200cfa00 unknown :-1 s=CLOSED pgs=843 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f052593e640 1 -- 172.21.15.64:0/1786331680 >> 172.21.15.64:0/1786331680 conn(0x7f05200b36d0 msgr2=0x7f05200b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f052593e640 1 -- 172.21.15.64:0/1786331680 shutdown_connections 2024-03-04T07:30:08.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.372+0000 7f052593e640 1 -- 172.21.15.64:0/1786331680 wait complete. 2024-03-04T07:30:08.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 Processor -- start 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 -- start start 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05200b4260 0x7f05200c8150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05200d02e0 0x7f05201569b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200d06c0 0x7f0520156ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05200b0360 con 0x7f05200d06c0 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05200aeec0 con 0x7f05200d02e0 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f052593e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05200b04e0 con 0x7f05200b4260 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f051f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200d06c0 0x7f0520156ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f051f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200d06c0 0x7f0520156ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49362/0 (socket says 172.21.15.64:49362) 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.374+0000 7f051f7fe640 1 -- 172.21.15.64:0/2747737884 learned_addr learned my addr 172.21.15.64:0/2747737884 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f0517fff640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05200d02e0 0x7f05201569b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051f7fe640 1 -- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05200b4260 msgr2=0x7f05200c8150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051effd640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05200b4260 0x7f05200c8150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051f7fe640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05200b4260 0x7f05200c8150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051f7fe640 1 -- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05200d02e0 msgr2=0x7f05201569b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051f7fe640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05200d02e0 0x7f05201569b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051f7fe640 1 -- 172.21.15.64:0/2747737884 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05201e5ad0 con 0x7f05200d06c0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051f7fe640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200d06c0 0x7f0520156ef0 secure :-1 s=READY pgs=844 cs=0 l=1 rev1=1 crypto rx=0x7f051800c7d0 tx=0x7f051800cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051cff9640 1 -- 172.21.15.64:0/2747737884 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0518003b70 con 0x7f05200d06c0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051cff9640 1 -- 172.21.15.64:0/2747737884 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0518003d30 con 0x7f05200d06c0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f051cff9640 1 -- 172.21.15.64:0/2747737884 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f051800aa90 con 0x7f05200d06c0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f05201e5db0 con 0x7f05200d06c0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.375+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f05201e6370 con 0x7f05200d06c0 2024-03-04T07:30:08.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.376+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0520068700 con 0x7f05200d06c0 2024-03-04T07:30:08.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.376+0000 7f051cff9640 1 -- 172.21.15.64:0/2747737884 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f051800ac30 con 0x7f05200d06c0 2024-03-04T07:30:08.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.377+0000 7f051cff9640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f050006b4d0 0x7f050006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.377+0000 7f051cff9640 1 -- 172.21.15.64:0/2747737884 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f051808a130 con 0x7f05200d06c0 2024-03-04T07:30:08.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.377+0000 7f051effd640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f050006b4d0 0x7f050006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.377+0000 7f051effd640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f050006b4d0 0x7f050006d990 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f0510007fd0 tx=0x7f0510009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:08.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.381+0000 7f051cff9640 1 -- 172.21.15.64:0/2747737884 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0518058fb0 con 0x7f05200d06c0 2024-03-04T07:30:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.559+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f05201e75f0 con 0x7f050006b4d0 2024-03-04T07:30:08.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.571+0000 7f051cff9640 1 -- 172.21.15.64:0/2747737884 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252049 (secure 0 0 0) 0x7f05201e75f0 con 0x7f050006b4d0 2024-03-04T07:30:08.575 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:08.576 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:30:08.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f050006b4d0 msgr2=0x7f050006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f050006b4d0 0x7f050006d990 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f0510007fd0 tx=0x7f0510009040 comp rx=0 tx=0).stop 2024-03-04T07:30:08.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200d06c0 msgr2=0x7f0520156ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200d06c0 0x7f0520156ef0 secure :-1 s=READY pgs=844 cs=0 l=1 rev1=1 crypto rx=0x7f051800c7d0 tx=0x7f051800cc90 comp rx=0 tx=0).stop 2024-03-04T07:30:08.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 shutdown_connections 2024-03-04T07:30:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f050006b4d0 0x7f050006d990 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05200d06c0 0x7f0520156ef0 unknown :-1 s=CLOSED pgs=844 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f05200d02e0 0x7f05201569b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 --2- 172.21.15.64:0/2747737884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05200b4260 0x7f05200c8150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 >> 172.21.15.64:0/2747737884 conn(0x7f05200b36d0 msgr2=0x7f05200ccd60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 shutdown_connections 2024-03-04T07:30:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.575+0000 7f052593e640 1 -- 172.21.15.64:0/2747737884 wait complete. 2024-03-04T07:30:08.587 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":162,"stamp":"2024-03-04T07:30:07.177068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":23,"num_object_clones":0,"num_object_copies":46,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":203,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155,"ondisk_log_size":155,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":142,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":256892,"kb_used_data":7400,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754717828,"statfs":{"total":773094113280,"available":772831055872,"internally_reserved":0,"allocated":7577600,"data_stored":3824692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":22,"apply_latency_ms":22,"commit_latency_ns":22000000,"apply_latency_ns":22000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.118114"},"pg_stats":[{"pgid":"11.3f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.091343+0000","last_change":"2024-03-04T07:30:05.080757+0000","last_active":"2024-03-04T07:30:05.091343+0000","last_peered":"2024-03-04T07:30:05.091343+0000","last_clean":"2024-03-04T07:30:05.091343+0000","last_became_active":"2024-03-04T07:30:05.080481+0000","last_became_peered":"2024-03-04T07:30:05.080481+0000","last_unstale":"2024-03-04T07:30:05.091343+0000","last_undegraded":"2024-03-04T07:30:05.091343+0000","last_fullsized":"2024-03-04T07:30:05.091343+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:09:31.879681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.3e","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.092691+0000","last_change":"2024-03-04T07:30:05.084660+0000","last_active":"2024-03-04T07:30:05.092691+0000","last_peered":"2024-03-04T07:30:05.092691+0000","last_clean":"2024-03-04T07:30:05.092691+0000","last_became_active":"2024-03-04T07:30:05.084351+0000","last_became_peered":"2024-03-04T07:30:05.084351+0000","last_unstale":"2024-03-04T07:30:05.092691+0000","last_undegraded":"2024-03-04T07:30:05.092691+0000","last_fullsized":"2024-03-04T07:30:05.092691+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:36:04.963795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.3d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.081867+0000","last_change":"2024-03-04T07:30:05.079727+0000","last_active":"2024-03-04T07:30:05.081867+0000","last_peered":"2024-03-04T07:30:05.081867+0000","last_clean":"2024-03-04T07:30:05.081867+0000","last_became_active":"2024-03-04T07:30:05.079439+0000","last_became_peered":"2024-03-04T07:30:05.079439+0000","last_unstale":"2024-03-04T07:30:05.081867+0000","last_undegraded":"2024-03-04T07:30:05.081867+0000","last_fullsized":"2024-03-04T07:30:05.081867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:26:54.073772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.3c","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080088+0000","last_change":"2024-03-04T07:30:05.098463+0000","last_active":"2024-03-04T07:30:06.080088+0000","last_peered":"2024-03-04T07:30:06.080088+0000","last_clean":"2024-03-04T07:30:06.080088+0000","last_became_active":"2024-03-04T07:30:05.098181+0000","last_became_peered":"2024-03-04T07:30:05.098181+0000","last_unstale":"2024-03-04T07:30:06.080088+0000","last_undegraded":"2024-03-04T07:30:06.080088+0000","last_fullsized":"2024-03-04T07:30:06.080088+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:55.863787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3b","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.084094+0000","last_change":"2024-03-04T07:30:05.082411+0000","last_active":"2024-03-04T07:30:05.084094+0000","last_peered":"2024-03-04T07:30:05.084094+0000","last_clean":"2024-03-04T07:30:05.084094+0000","last_became_active":"2024-03-04T07:30:05.082028+0000","last_became_peered":"2024-03-04T07:30:05.082028+0000","last_unstale":"2024-03-04T07:30:05.084094+0000","last_undegraded":"2024-03-04T07:30:05.084094+0000","last_fullsized":"2024-03-04T07:30:05.084094+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:50:02.482536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3a","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099512+0000","last_change":"2024-03-04T07:30:05.097621+0000","last_active":"2024-03-04T07:30:05.099512+0000","last_peered":"2024-03-04T07:30:05.099512+0000","last_clean":"2024-03-04T07:30:05.099512+0000","last_became_active":"2024-03-04T07:30:05.095412+0000","last_became_peered":"2024-03-04T07:30:05.095412+0000","last_unstale":"2024-03-04T07:30:05.099512+0000","last_undegraded":"2024-03-04T07:30:05.099512+0000","last_fullsized":"2024-03-04T07:30:05.099512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:00:36.612527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.39","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070514+0000","last_change":"2024-03-04T07:30:05.089188+0000","last_active":"2024-03-04T07:30:06.070514+0000","last_peered":"2024-03-04T07:30:06.070514+0000","last_clean":"2024-03-04T07:30:06.070514+0000","last_became_active":"2024-03-04T07:30:05.088884+0000","last_became_peered":"2024-03-04T07:30:05.088884+0000","last_unstale":"2024-03-04T07:30:06.070514+0000","last_undegraded":"2024-03-04T07:30:06.070514+0000","last_fullsized":"2024-03-04T07:30:06.070514+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:52:38.600368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.38","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099361+0000","last_change":"2024-03-04T07:30:05.094825+0000","last_active":"2024-03-04T07:30:05.099361+0000","last_peered":"2024-03-04T07:30:05.099361+0000","last_clean":"2024-03-04T07:30:05.099361+0000","last_became_active":"2024-03-04T07:30:05.094499+0000","last_became_peered":"2024-03-04T07:30:05.094499+0000","last_unstale":"2024-03-04T07:30:05.099361+0000","last_undegraded":"2024-03-04T07:30:05.099361+0000","last_fullsized":"2024-03-04T07:30:05.099361+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:55.863787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.37","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090255+0000","last_change":"2024-03-04T07:30:05.088738+0000","last_active":"2024-03-04T07:30:05.090255+0000","last_peered":"2024-03-04T07:30:05.090255+0000","last_clean":"2024-03-04T07:30:05.090255+0000","last_became_active":"2024-03-04T07:30:05.087436+0000","last_became_peered":"2024-03-04T07:30:05.087436+0000","last_unstale":"2024-03-04T07:30:05.090255+0000","last_undegraded":"2024-03-04T07:30:05.090255+0000","last_fullsized":"2024-03-04T07:30:05.090255+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:48.648887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.36","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071596+0000","last_change":"2024-03-04T07:30:05.077566+0000","last_active":"2024-03-04T07:30:06.071596+0000","last_peered":"2024-03-04T07:30:06.071596+0000","last_clean":"2024-03-04T07:30:06.071596+0000","last_became_active":"2024-03-04T07:30:05.077146+0000","last_became_peered":"2024-03-04T07:30:05.077146+0000","last_unstale":"2024-03-04T07:30:06.071596+0000","last_undegraded":"2024-03-04T07:30:06.071596+0000","last_fullsized":"2024-03-04T07:30:06.071596+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:09.457728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.35","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093052+0000","last_change":"2024-03-04T07:30:05.085914+0000","last_active":"2024-03-04T07:30:05.093052+0000","last_peered":"2024-03-04T07:30:05.093052+0000","last_clean":"2024-03-04T07:30:05.093052+0000","last_became_active":"2024-03-04T07:30:05.085617+0000","last_became_peered":"2024-03-04T07:30:05.085617+0000","last_unstale":"2024-03-04T07:30:05.093052+0000","last_undegraded":"2024-03-04T07:30:05.093052+0000","last_fullsized":"2024-03-04T07:30:05.093052+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:04:11.464737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.34","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080137+0000","last_change":"2024-03-04T07:30:05.091184+0000","last_active":"2024-03-04T07:30:06.080137+0000","last_peered":"2024-03-04T07:30:06.080137+0000","last_clean":"2024-03-04T07:30:06.080137+0000","last_became_active":"2024-03-04T07:30:05.090901+0000","last_became_peered":"2024-03-04T07:30:05.090901+0000","last_unstale":"2024-03-04T07:30:06.080137+0000","last_undegraded":"2024-03-04T07:30:06.080137+0000","last_fullsized":"2024-03-04T07:30:06.080137+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:44:35.285031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.33","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071824+0000","last_change":"2024-03-04T07:30:05.093386+0000","last_active":"2024-03-04T07:30:06.071824+0000","last_peered":"2024-03-04T07:30:06.071824+0000","last_clean":"2024-03-04T07:30:06.071824+0000","last_became_active":"2024-03-04T07:30:05.093080+0000","last_became_peered":"2024-03-04T07:30:05.093080+0000","last_unstale":"2024-03-04T07:30:06.071824+0000","last_undegraded":"2024-03-04T07:30:06.071824+0000","last_fullsized":"2024-03-04T07:30:06.071824+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:41.093583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.32","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.079973+0000","last_change":"2024-03-04T07:30:05.078876+0000","last_active":"2024-03-04T07:30:05.079973+0000","last_peered":"2024-03-04T07:30:05.079973+0000","last_clean":"2024-03-04T07:30:05.079973+0000","last_became_active":"2024-03-04T07:30:05.078470+0000","last_became_peered":"2024-03-04T07:30:05.078470+0000","last_unstale":"2024-03-04T07:30:05.079973+0000","last_undegraded":"2024-03-04T07:30:05.079973+0000","last_fullsized":"2024-03-04T07:30:05.079973+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:04:48.717451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.31","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071623+0000","last_change":"2024-03-04T07:30:05.080376+0000","last_active":"2024-03-04T07:30:06.071623+0000","last_peered":"2024-03-04T07:30:06.071623+0000","last_clean":"2024-03-04T07:30:06.071623+0000","last_became_active":"2024-03-04T07:30:05.080094+0000","last_became_peered":"2024-03-04T07:30:05.080094+0000","last_unstale":"2024-03-04T07:30:06.071623+0000","last_undegraded":"2024-03-04T07:30:06.071623+0000","last_fullsized":"2024-03-04T07:30:06.071623+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:28:21.513469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.30","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070519+0000","last_change":"2024-03-04T07:30:05.096512+0000","last_active":"2024-03-04T07:30:06.070519+0000","last_peered":"2024-03-04T07:30:06.070519+0000","last_clean":"2024-03-04T07:30:06.070519+0000","last_became_active":"2024-03-04T07:30:05.096133+0000","last_became_peered":"2024-03-04T07:30:05.096133+0000","last_unstale":"2024-03-04T07:30:06.070519+0000","last_undegraded":"2024-03-04T07:30:06.070519+0000","last_fullsized":"2024-03-04T07:30:06.070519+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:44:54.507339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093046+0000","last_change":"2024-03-04T07:30:05.088696+0000","last_active":"2024-03-04T07:30:05.093046+0000","last_peered":"2024-03-04T07:30:05.093046+0000","last_clean":"2024-03-04T07:30:05.093046+0000","last_became_active":"2024-03-04T07:30:05.087106+0000","last_became_peered":"2024-03-04T07:30:05.087106+0000","last_unstale":"2024-03-04T07:30:05.093046+0000","last_undegraded":"2024-03-04T07:30:05.093046+0000","last_fullsized":"2024-03-04T07:30:05.093046+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:27:58.076549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.2e","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080040+0000","last_change":"2024-03-04T07:30:05.098425+0000","last_active":"2024-03-04T07:30:06.080040+0000","last_peered":"2024-03-04T07:30:06.080040+0000","last_clean":"2024-03-04T07:30:06.080040+0000","last_became_active":"2024-03-04T07:30:05.098130+0000","last_became_peered":"2024-03-04T07:30:05.098130+0000","last_unstale":"2024-03-04T07:30:06.080040+0000","last_undegraded":"2024-03-04T07:30:06.080040+0000","last_fullsized":"2024-03-04T07:30:06.080040+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:04.703704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.2d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098601+0000","last_change":"2024-03-04T07:30:05.094019+0000","last_active":"2024-03-04T07:30:05.098601+0000","last_peered":"2024-03-04T07:30:05.098601+0000","last_clean":"2024-03-04T07:30:05.098601+0000","last_became_active":"2024-03-04T07:30:05.093273+0000","last_became_peered":"2024-03-04T07:30:05.093273+0000","last_unstale":"2024-03-04T07:30:05.098601+0000","last_undegraded":"2024-03-04T07:30:05.098601+0000","last_fullsized":"2024-03-04T07:30:05.098601+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:30:04.703704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.2c","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071755+0000","last_change":"2024-03-04T07:30:05.077646+0000","last_active":"2024-03-04T07:30:06.071755+0000","last_peered":"2024-03-04T07:30:06.071755+0000","last_clean":"2024-03-04T07:30:06.071755+0000","last_became_active":"2024-03-04T07:30:05.077329+0000","last_became_peered":"2024-03-04T07:30:05.077329+0000","last_unstale":"2024-03-04T07:30:06.071755+0000","last_undegraded":"2024-03-04T07:30:06.071755+0000","last_fullsized":"2024-03-04T07:30:06.071755+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:00.798986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2b","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070715+0000","last_change":"2024-03-04T07:30:05.080907+0000","last_active":"2024-03-04T07:30:06.070715+0000","last_peered":"2024-03-04T07:30:06.070715+0000","last_clean":"2024-03-04T07:30:06.070715+0000","last_became_active":"2024-03-04T07:30:05.080641+0000","last_became_peered":"2024-03-04T07:30:05.080641+0000","last_unstale":"2024-03-04T07:30:06.070715+0000","last_undegraded":"2024-03-04T07:30:06.070715+0000","last_fullsized":"2024-03-04T07:30:06.070715+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:09:31.879681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2a","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071329+0000","last_change":"2024-03-04T07:30:05.079488+0000","last_active":"2024-03-04T07:30:06.071329+0000","last_peered":"2024-03-04T07:30:06.071329+0000","last_clean":"2024-03-04T07:30:06.071329+0000","last_became_active":"2024-03-04T07:30:05.079182+0000","last_became_peered":"2024-03-04T07:30:05.079182+0000","last_unstale":"2024-03-04T07:30:06.071329+0000","last_undegraded":"2024-03-04T07:30:06.071329+0000","last_fullsized":"2024-03-04T07:30:06.071329+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:00:04.416316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.29","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090834+0000","last_change":"2024-03-04T07:30:05.078651+0000","last_active":"2024-03-04T07:30:05.090834+0000","last_peered":"2024-03-04T07:30:05.090834+0000","last_clean":"2024-03-04T07:30:05.090834+0000","last_became_active":"2024-03-04T07:30:05.078341+0000","last_became_peered":"2024-03-04T07:30:05.078341+0000","last_unstale":"2024-03-04T07:30:05.090834+0000","last_undegraded":"2024-03-04T07:30:05.090834+0000","last_fullsized":"2024-03-04T07:30:05.090834+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:08:39.102228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.28","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099105+0000","last_change":"2024-03-04T07:30:05.096767+0000","last_active":"2024-03-04T07:30:05.099105+0000","last_peered":"2024-03-04T07:30:05.099105+0000","last_clean":"2024-03-04T07:30:05.099105+0000","last_became_active":"2024-03-04T07:30:05.096069+0000","last_became_peered":"2024-03-04T07:30:05.096069+0000","last_unstale":"2024-03-04T07:30:05.099105+0000","last_undegraded":"2024-03-04T07:30:05.099105+0000","last_fullsized":"2024-03-04T07:30:05.099105+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:17:54.586600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.27","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070836+0000","last_change":"2024-03-04T07:30:05.085832+0000","last_active":"2024-03-04T07:30:06.070836+0000","last_peered":"2024-03-04T07:30:06.070836+0000","last_clean":"2024-03-04T07:30:06.070836+0000","last_became_active":"2024-03-04T07:30:05.085508+0000","last_became_peered":"2024-03-04T07:30:05.085508+0000","last_unstale":"2024-03-04T07:30:06.070836+0000","last_undegraded":"2024-03-04T07:30:06.070836+0000","last_fullsized":"2024-03-04T07:30:06.070836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:33:51.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.26","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071089+0000","last_change":"2024-03-04T07:30:05.096149+0000","last_active":"2024-03-04T07:30:06.071089+0000","last_peered":"2024-03-04T07:30:06.071089+0000","last_clean":"2024-03-04T07:30:06.071089+0000","last_became_active":"2024-03-04T07:30:05.095699+0000","last_became_peered":"2024-03-04T07:30:05.095699+0000","last_unstale":"2024-03-04T07:30:06.071089+0000","last_undegraded":"2024-03-04T07:30:06.071089+0000","last_fullsized":"2024-03-04T07:30:06.071089+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:35:41.093583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.25","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090810+0000","last_change":"2024-03-04T07:30:05.088843+0000","last_active":"2024-03-04T07:30:05.090810+0000","last_peered":"2024-03-04T07:30:05.090810+0000","last_clean":"2024-03-04T07:30:05.090810+0000","last_became_active":"2024-03-04T07:30:05.088208+0000","last_became_peered":"2024-03-04T07:30:05.088208+0000","last_unstale":"2024-03-04T07:30:05.090810+0000","last_undegraded":"2024-03-04T07:30:05.090810+0000","last_fullsized":"2024-03-04T07:30:05.090810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:19.737517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.24","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071646+0000","last_change":"2024-03-04T07:30:05.079614+0000","last_active":"2024-03-04T07:30:06.071646+0000","last_peered":"2024-03-04T07:30:06.071646+0000","last_clean":"2024-03-04T07:30:06.071646+0000","last_became_active":"2024-03-04T07:30:05.079273+0000","last_became_peered":"2024-03-04T07:30:05.079273+0000","last_unstale":"2024-03-04T07:30:06.071646+0000","last_undegraded":"2024-03-04T07:30:06.071646+0000","last_fullsized":"2024-03-04T07:30:06.071646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.23","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.082872+0000","last_change":"2024-03-04T07:30:05.080645+0000","last_active":"2024-03-04T07:30:05.082872+0000","last_peered":"2024-03-04T07:30:05.082872+0000","last_clean":"2024-03-04T07:30:05.082872+0000","last_became_active":"2024-03-04T07:30:05.080390+0000","last_became_peered":"2024-03-04T07:30:05.080390+0000","last_unstale":"2024-03-04T07:30:05.082872+0000","last_undegraded":"2024-03-04T07:30:05.082872+0000","last_fullsized":"2024-03-04T07:30:05.082872+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:41:10.488917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.22","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070493+0000","last_change":"2024-03-04T07:30:05.094084+0000","last_active":"2024-03-04T07:30:06.070493+0000","last_peered":"2024-03-04T07:30:06.070493+0000","last_clean":"2024-03-04T07:30:06.070493+0000","last_became_active":"2024-03-04T07:30:05.093772+0000","last_became_peered":"2024-03-04T07:30:05.093772+0000","last_unstale":"2024-03-04T07:30:06.070493+0000","last_undegraded":"2024-03-04T07:30:06.070493+0000","last_fullsized":"2024-03-04T07:30:06.070493+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:09.457728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.21","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070809+0000","last_change":"2024-03-04T07:30:05.080374+0000","last_active":"2024-03-04T07:30:06.070809+0000","last_peered":"2024-03-04T07:30:06.070809+0000","last_clean":"2024-03-04T07:30:06.070809+0000","last_became_active":"2024-03-04T07:30:05.080075+0000","last_became_peered":"2024-03-04T07:30:05.080075+0000","last_unstale":"2024-03-04T07:30:06.070809+0000","last_undegraded":"2024-03-04T07:30:06.070809+0000","last_fullsized":"2024-03-04T07:30:06.070809+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:28:21.513469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.20","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099498+0000","last_change":"2024-03-04T07:30:05.095504+0000","last_active":"2024-03-04T07:30:05.099498+0000","last_peered":"2024-03-04T07:30:05.099498+0000","last_clean":"2024-03-04T07:30:05.099498+0000","last_became_active":"2024-03-04T07:30:05.095188+0000","last_became_peered":"2024-03-04T07:30:05.095188+0000","last_unstale":"2024-03-04T07:30:05.099498+0000","last_undegraded":"2024-03-04T07:30:05.099498+0000","last_fullsized":"2024-03-04T07:30:05.099498+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:52.747064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.1e","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099058+0000","last_change":"2024-03-04T07:30:05.096047+0000","last_active":"2024-03-04T07:30:05.099058+0000","last_peered":"2024-03-04T07:30:05.099058+0000","last_clean":"2024-03-04T07:30:05.099058+0000","last_became_active":"2024-03-04T07:30:05.095751+0000","last_became_peered":"2024-03-04T07:30:05.095751+0000","last_unstale":"2024-03-04T07:30:05.099058+0000","last_undegraded":"2024-03-04T07:30:05.099058+0000","last_fullsized":"2024-03-04T07:30:05.099058+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:52:38.600368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.1f","version":"65'4","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.096236+0000","last_change":"2024-03-04T07:30:04.081175+0000","last_active":"2024-03-04T07:30:05.096236+0000","last_peered":"2024-03-04T07:30:05.096236+0000","last_clean":"2024-03-04T07:30:05.096236+0000","last_became_active":"2024-03-04T07:30:04.080863+0000","last_became_peered":"2024-03-04T07:30:04.080863+0000","last_unstale":"2024-03-04T07:30:05.096236+0000","last_undegraded":"2024-03-04T07:30:05.096236+0000","last_fullsized":"2024-03-04T07:30:05.096236+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:38:53.665426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.1f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089351+0000","last_change":"2024-03-04T07:30:05.088078+0000","last_active":"2024-03-04T07:30:05.089351+0000","last_peered":"2024-03-04T07:30:05.089351+0000","last_clean":"2024-03-04T07:30:05.089351+0000","last_became_active":"2024-03-04T07:30:05.086783+0000","last_became_peered":"2024-03-04T07:30:05.086783+0000","last_unstale":"2024-03-04T07:30:05.089351+0000","last_undegraded":"2024-03-04T07:30:05.089351+0000","last_fullsized":"2024-03-04T07:30:05.089351+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:14:25.100757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.1e","version":"65'1","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071927+0000","last_change":"2024-03-04T07:30:04.082721+0000","last_active":"2024-03-04T07:30:06.071927+0000","last_peered":"2024-03-04T07:30:06.071927+0000","last_clean":"2024-03-04T07:30:06.071927+0000","last_became_active":"2024-03-04T07:30:04.082417+0000","last_became_peered":"2024-03-04T07:30:04.082417+0000","last_unstale":"2024-03-04T07:30:06.071927+0000","last_undegraded":"2024-03-04T07:30:06.071927+0000","last_fullsized":"2024-03-04T07:30:06.071927+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:22.005661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.1c","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099482+0000","last_change":"2024-03-04T07:30:05.097528+0000","last_active":"2024-03-04T07:30:05.099482+0000","last_peered":"2024-03-04T07:30:05.099482+0000","last_clean":"2024-03-04T07:30:05.099482+0000","last_became_active":"2024-03-04T07:30:05.095278+0000","last_became_peered":"2024-03-04T07:30:05.095278+0000","last_unstale":"2024-03-04T07:30:05.099482+0000","last_undegraded":"2024-03-04T07:30:05.099482+0000","last_fullsized":"2024-03-04T07:30:05.099482+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:40:03.427093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1d","version":"65'2","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071298+0000","last_change":"2024-03-04T07:30:04.083247+0000","last_active":"2024-03-04T07:30:06.071298+0000","last_peered":"2024-03-04T07:30:06.071298+0000","last_clean":"2024-03-04T07:30:06.071298+0000","last_became_active":"2024-03-04T07:30:04.082936+0000","last_became_peered":"2024-03-04T07:30:04.082936+0000","last_unstale":"2024-03-04T07:30:06.071298+0000","last_undegraded":"2024-03-04T07:30:06.071298+0000","last_fullsized":"2024-03-04T07:30:06.071298+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:44:20.200529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.091158+0000","last_change":"2024-03-04T07:30:05.083224+0000","last_active":"2024-03-04T07:30:05.091158+0000","last_peered":"2024-03-04T07:30:05.091158+0000","last_clean":"2024-03-04T07:30:05.091158+0000","last_became_active":"2024-03-04T07:30:05.082955+0000","last_became_peered":"2024-03-04T07:30:05.082955+0000","last_unstale":"2024-03-04T07:30:05.091158+0000","last_undegraded":"2024-03-04T07:30:05.091158+0000","last_fullsized":"2024-03-04T07:30:05.091158+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:33:51.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.1c","version":"65'2","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.100492+0000","last_change":"2024-03-04T07:30:04.087388+0000","last_active":"2024-03-04T07:30:05.100492+0000","last_peered":"2024-03-04T07:30:05.100492+0000","last_clean":"2024-03-04T07:30:05.100492+0000","last_became_active":"2024-03-04T07:30:04.087100+0000","last_became_peered":"2024-03-04T07:30:04.087100+0000","last_unstale":"2024-03-04T07:30:05.100492+0000","last_undegraded":"2024-03-04T07:30:05.100492+0000","last_fullsized":"2024-03-04T07:30:05.100492+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:42:56.438271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.1a","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070701+0000","last_change":"2024-03-04T07:30:05.083234+0000","last_active":"2024-03-04T07:30:06.070701+0000","last_peered":"2024-03-04T07:30:06.070701+0000","last_clean":"2024-03-04T07:30:06.070701+0000","last_became_active":"2024-03-04T07:30:05.082948+0000","last_became_peered":"2024-03-04T07:30:05.082948+0000","last_unstale":"2024-03-04T07:30:06.070701+0000","last_undegraded":"2024-03-04T07:30:06.070701+0000","last_fullsized":"2024-03-04T07:30:06.070701+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.1b","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093702+0000","last_change":"2024-03-04T07:30:04.076239+0000","last_active":"2024-03-04T07:30:05.093702+0000","last_peered":"2024-03-04T07:30:05.093702+0000","last_clean":"2024-03-04T07:30:05.093702+0000","last_became_active":"2024-03-04T07:30:04.075758+0000","last_became_peered":"2024-03-04T07:30:04.075758+0000","last_unstale":"2024-03-04T07:30:05.093702+0000","last_undegraded":"2024-03-04T07:30:05.093702+0000","last_fullsized":"2024-03-04T07:30:05.093702+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:35:01.014832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1b","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070894+0000","last_change":"2024-03-04T07:30:05.087307+0000","last_active":"2024-03-04T07:30:06.070894+0000","last_peered":"2024-03-04T07:30:06.070894+0000","last_clean":"2024-03-04T07:30:06.070894+0000","last_became_active":"2024-03-04T07:30:05.085836+0000","last_became_peered":"2024-03-04T07:30:05.085836+0000","last_unstale":"2024-03-04T07:30:06.070894+0000","last_undegraded":"2024-03-04T07:30:06.070894+0000","last_fullsized":"2024-03-04T07:30:06.070894+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:04:11.464737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1a","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.097530+0000","last_change":"2024-03-04T07:30:04.084488+0000","last_active":"2024-03-04T07:30:05.097530+0000","last_peered":"2024-03-04T07:30:05.097530+0000","last_clean":"2024-03-04T07:30:05.097530+0000","last_became_active":"2024-03-04T07:30:04.083989+0000","last_became_peered":"2024-03-04T07:30:04.083989+0000","last_unstale":"2024-03-04T07:30:05.097530+0000","last_undegraded":"2024-03-04T07:30:05.097530+0000","last_fullsized":"2024-03-04T07:30:05.097530+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:47:31.326424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.18","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.088818+0000","last_change":"2024-03-04T07:30:05.084596+0000","last_active":"2024-03-04T07:30:05.088818+0000","last_peered":"2024-03-04T07:30:05.088818+0000","last_clean":"2024-03-04T07:30:05.088818+0000","last_became_active":"2024-03-04T07:30:05.084281+0000","last_became_peered":"2024-03-04T07:30:05.084281+0000","last_unstale":"2024-03-04T07:30:05.088818+0000","last_undegraded":"2024-03-04T07:30:05.088818+0000","last_fullsized":"2024-03-04T07:30:05.088818+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:34:52.102333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.19","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079844+0000","last_change":"2024-03-04T07:30:04.075017+0000","last_active":"2024-03-04T07:30:06.079844+0000","last_peered":"2024-03-04T07:30:06.079844+0000","last_clean":"2024-03-04T07:30:06.079844+0000","last_became_active":"2024-03-04T07:30:04.074485+0000","last_became_peered":"2024-03-04T07:30:04.074485+0000","last_unstale":"2024-03-04T07:30:06.079844+0000","last_undegraded":"2024-03-04T07:30:06.079844+0000","last_fullsized":"2024-03-04T07:30:06.079844+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:55.834473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.19","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071671+0000","last_change":"2024-03-04T07:30:05.080173+0000","last_active":"2024-03-04T07:30:06.071671+0000","last_peered":"2024-03-04T07:30:06.071671+0000","last_clean":"2024-03-04T07:30:06.071671+0000","last_became_active":"2024-03-04T07:30:05.079907+0000","last_became_peered":"2024-03-04T07:30:05.079907+0000","last_unstale":"2024-03-04T07:30:06.071671+0000","last_undegraded":"2024-03-04T07:30:06.071671+0000","last_fullsized":"2024-03-04T07:30:06.071671+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:17:54.586600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.18","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.073846+0000","last_change":"2024-03-04T07:30:04.075015+0000","last_active":"2024-03-04T07:30:05.073846+0000","last_peered":"2024-03-04T07:30:05.073846+0000","last_clean":"2024-03-04T07:30:05.073846+0000","last_became_active":"2024-03-04T07:30:04.074699+0000","last_became_peered":"2024-03-04T07:30:04.074699+0000","last_unstale":"2024-03-04T07:30:05.073846+0000","last_undegraded":"2024-03-04T07:30:05.073846+0000","last_fullsized":"2024-03-04T07:30:05.073846+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:53.436832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.16","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071240+0000","last_change":"2024-03-04T07:30:05.078927+0000","last_active":"2024-03-04T07:30:06.071240+0000","last_peered":"2024-03-04T07:30:06.071240+0000","last_clean":"2024-03-04T07:30:06.071240+0000","last_became_active":"2024-03-04T07:30:05.078602+0000","last_became_peered":"2024-03-04T07:30:05.078602+0000","last_unstale":"2024-03-04T07:30:06.071240+0000","last_undegraded":"2024-03-04T07:30:06.071240+0000","last_fullsized":"2024-03-04T07:30:06.071240+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:55.231331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.17","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070945+0000","last_change":"2024-03-04T07:30:04.078857+0000","last_active":"2024-03-04T07:30:06.070945+0000","last_peered":"2024-03-04T07:30:06.070945+0000","last_clean":"2024-03-04T07:30:06.070945+0000","last_became_active":"2024-03-04T07:30:04.078578+0000","last_became_peered":"2024-03-04T07:30:04.078578+0000","last_unstale":"2024-03-04T07:30:06.070945+0000","last_undegraded":"2024-03-04T07:30:06.070945+0000","last_fullsized":"2024-03-04T07:30:06.070945+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:45:03.481435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.17","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099062+0000","last_change":"2024-03-04T07:30:05.096381+0000","last_active":"2024-03-04T07:30:05.099062+0000","last_peered":"2024-03-04T07:30:05.099062+0000","last_clean":"2024-03-04T07:30:05.099062+0000","last_became_active":"2024-03-04T07:30:05.095758+0000","last_became_peered":"2024-03-04T07:30:05.095758+0000","last_unstale":"2024-03-04T07:30:05.099062+0000","last_undegraded":"2024-03-04T07:30:05.099062+0000","last_fullsized":"2024-03-04T07:30:05.099062+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089996+0000","last_change":"2024-03-04T07:30:04.083596+0000","last_active":"2024-03-04T07:30:05.089996+0000","last_peered":"2024-03-04T07:30:05.089996+0000","last_clean":"2024-03-04T07:30:05.089996+0000","last_became_active":"2024-03-04T07:30:04.083258+0000","last_became_peered":"2024-03-04T07:30:04.083258+0000","last_unstale":"2024-03-04T07:30:05.089996+0000","last_undegraded":"2024-03-04T07:30:05.089996+0000","last_fullsized":"2024-03-04T07:30:05.089996+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:02:06.058460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.14","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099402+0000","last_change":"2024-03-04T07:30:05.095347+0000","last_active":"2024-03-04T07:30:05.099402+0000","last_peered":"2024-03-04T07:30:05.099402+0000","last_clean":"2024-03-04T07:30:05.099402+0000","last_became_active":"2024-03-04T07:30:05.094728+0000","last_became_peered":"2024-03-04T07:30:05.094728+0000","last_unstale":"2024-03-04T07:30:05.099402+0000","last_undegraded":"2024-03-04T07:30:05.099402+0000","last_fullsized":"2024-03-04T07:30:05.099402+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:44:35.285031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.15","version":"65'3","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089052+0000","last_change":"2024-03-04T07:30:04.079117+0000","last_active":"2024-03-04T07:30:05.089052+0000","last_peered":"2024-03-04T07:30:05.089052+0000","last_clean":"2024-03-04T07:30:05.089052+0000","last_became_active":"2024-03-04T07:30:04.078754+0000","last_became_peered":"2024-03-04T07:30:04.078754+0000","last_unstale":"2024-03-04T07:30:05.089052+0000","last_undegraded":"2024-03-04T07:30:05.089052+0000","last_fullsized":"2024-03-04T07:30:05.089052+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:35:44.355784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.095438+0000","last_change":"2024-03-04T07:30:05.091558+0000","last_active":"2024-03-04T07:30:05.095438+0000","last_peered":"2024-03-04T07:30:05.095438+0000","last_clean":"2024-03-04T07:30:05.095438+0000","last_became_active":"2024-03-04T07:30:05.091290+0000","last_became_peered":"2024-03-04T07:30:05.091290+0000","last_unstale":"2024-03-04T07:30:05.095438+0000","last_undegraded":"2024-03-04T07:30:05.095438+0000","last_fullsized":"2024-03-04T07:30:05.095438+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:54.753453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.14","version":"65'6","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080230+0000","last_change":"2024-03-04T07:30:04.083002+0000","last_active":"2024-03-04T07:30:06.080230+0000","last_peered":"2024-03-04T07:30:06.080230+0000","last_clean":"2024-03-04T07:30:06.080230+0000","last_became_active":"2024-03-04T07:30:04.082223+0000","last_became_peered":"2024-03-04T07:30:04.082223+0000","last_unstale":"2024-03-04T07:30:06.080230+0000","last_undegraded":"2024-03-04T07:30:06.080230+0000","last_fullsized":"2024-03-04T07:30:06.080230+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:20:35.593676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.090944+0000","last_change":"2024-03-04T07:30:05.080091+0000","last_active":"2024-03-04T07:30:05.090944+0000","last_peered":"2024-03-04T07:30:05.090944+0000","last_clean":"2024-03-04T07:30:05.090944+0000","last_became_active":"2024-03-04T07:30:05.079837+0000","last_became_peered":"2024-03-04T07:30:05.079837+0000","last_unstale":"2024-03-04T07:30:05.090944+0000","last_undegraded":"2024-03-04T07:30:05.090944+0000","last_fullsized":"2024-03-04T07:30:05.090944+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:44:54.507339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.13","version":"65'2","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079974+0000","last_change":"2024-03-04T07:30:04.080925+0000","last_active":"2024-03-04T07:30:06.079974+0000","last_peered":"2024-03-04T07:30:06.079974+0000","last_clean":"2024-03-04T07:30:06.079974+0000","last_became_active":"2024-03-04T07:30:04.080597+0000","last_became_peered":"2024-03-04T07:30:04.080597+0000","last_unstale":"2024-03-04T07:30:06.079974+0000","last_undegraded":"2024-03-04T07:30:06.079974+0000","last_fullsized":"2024-03-04T07:30:06.079974+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:47:31.326424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.13","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070966+0000","last_change":"2024-03-04T07:30:05.074092+0000","last_active":"2024-03-04T07:30:06.070966+0000","last_peered":"2024-03-04T07:30:06.070966+0000","last_clean":"2024-03-04T07:30:06.070966+0000","last_became_active":"2024-03-04T07:30:05.073786+0000","last_became_peered":"2024-03-04T07:30:05.073786+0000","last_unstale":"2024-03-04T07:30:06.070966+0000","last_undegraded":"2024-03-04T07:30:06.070966+0000","last_fullsized":"2024-03-04T07:30:06.070966+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:45:00.798986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.087930+0000","last_change":"2024-03-04T07:30:04.082134+0000","last_active":"2024-03-04T07:30:05.087930+0000","last_peered":"2024-03-04T07:30:05.087930+0000","last_clean":"2024-03-04T07:30:05.087930+0000","last_became_active":"2024-03-04T07:30:04.081835+0000","last_became_peered":"2024-03-04T07:30:04.081835+0000","last_unstale":"2024-03-04T07:30:05.087930+0000","last_undegraded":"2024-03-04T07:30:05.087930+0000","last_fullsized":"2024-03-04T07:30:05.087930+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:49:20.014438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.10","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071622+0000","last_change":"2024-03-04T07:30:05.097365+0000","last_active":"2024-03-04T07:30:06.071622+0000","last_peered":"2024-03-04T07:30:06.071622+0000","last_clean":"2024-03-04T07:30:06.071622+0000","last_became_active":"2024-03-04T07:30:05.097140+0000","last_became_peered":"2024-03-04T07:30:05.097140+0000","last_unstale":"2024-03-04T07:30:06.071622+0000","last_undegraded":"2024-03-04T07:30:06.071622+0000","last_fullsized":"2024-03-04T07:30:06.071622+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:07.417755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070639+0000","last_change":"2024-03-04T07:30:04.085183+0000","last_active":"2024-03-04T07:30:06.070639+0000","last_peered":"2024-03-04T07:30:06.070639+0000","last_clean":"2024-03-04T07:30:06.070639+0000","last_became_active":"2024-03-04T07:30:04.084915+0000","last_became_peered":"2024-03-04T07:30:04.084915+0000","last_unstale":"2024-03-04T07:30:06.070639+0000","last_undegraded":"2024-03-04T07:30:06.070639+0000","last_fullsized":"2024-03-04T07:30:06.070639+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:45:54.708994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.11","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089287+0000","last_change":"2024-03-04T07:30:05.086586+0000","last_active":"2024-03-04T07:30:05.089287+0000","last_peered":"2024-03-04T07:30:05.089287+0000","last_clean":"2024-03-04T07:30:05.089287+0000","last_became_active":"2024-03-04T07:30:05.086320+0000","last_became_peered":"2024-03-04T07:30:05.086320+0000","last_unstale":"2024-03-04T07:30:05.089287+0000","last_undegraded":"2024-03-04T07:30:05.089287+0000","last_fullsized":"2024-03-04T07:30:05.089287+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:46:05.386449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.10","version":"65'8","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070650+0000","last_change":"2024-03-04T07:30:04.077819+0000","last_active":"2024-03-04T07:30:06.070650+0000","last_peered":"2024-03-04T07:30:06.070650+0000","last_clean":"2024-03-04T07:30:06.070650+0000","last_became_active":"2024-03-04T07:30:04.077507+0000","last_became_peered":"2024-03-04T07:30:04.077507+0000","last_unstale":"2024-03-04T07:30:06.070650+0000","last_undegraded":"2024-03-04T07:30:06.070650+0000","last_fullsized":"2024-03-04T07:30:06.070650+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:19:12.786935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071645+0000","last_change":"2024-03-04T07:30:05.089680+0000","last_active":"2024-03-04T07:30:06.071645+0000","last_peered":"2024-03-04T07:30:06.071645+0000","last_clean":"2024-03-04T07:30:06.071645+0000","last_became_active":"2024-03-04T07:30:05.089403+0000","last_became_peered":"2024-03-04T07:30:05.089403+0000","last_unstale":"2024-03-04T07:30:06.071645+0000","last_undegraded":"2024-03-04T07:30:06.071645+0000","last_fullsized":"2024-03-04T07:30:06.071645+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:52:38.600368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.f","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098212+0000","last_change":"2024-03-04T07:30:04.085909+0000","last_active":"2024-03-04T07:30:05.098212+0000","last_peered":"2024-03-04T07:30:05.098212+0000","last_clean":"2024-03-04T07:30:05.098212+0000","last_became_active":"2024-03-04T07:30:04.085611+0000","last_became_peered":"2024-03-04T07:30:04.085611+0000","last_unstale":"2024-03-04T07:30:05.098212+0000","last_undegraded":"2024-03-04T07:30:05.098212+0000","last_fullsized":"2024-03-04T07:30:05.098212+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:18:30.192631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093562+0000","last_change":"2024-03-04T07:30:05.091451+0000","last_active":"2024-03-04T07:30:05.093562+0000","last_peered":"2024-03-04T07:30:05.093562+0000","last_clean":"2024-03-04T07:30:05.093562+0000","last_became_active":"2024-03-04T07:30:05.091169+0000","last_became_peered":"2024-03-04T07:30:05.091169+0000","last_unstale":"2024-03-04T07:30:05.093562+0000","last_undegraded":"2024-03-04T07:30:05.093562+0000","last_fullsized":"2024-03-04T07:30:05.093562+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:00:04.416316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"65'2","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071686+0000","last_change":"2024-03-04T07:30:04.084651+0000","last_active":"2024-03-04T07:30:06.071686+0000","last_peered":"2024-03-04T07:30:06.071686+0000","last_clean":"2024-03-04T07:30:06.071686+0000","last_became_active":"2024-03-04T07:30:04.084108+0000","last_became_peered":"2024-03-04T07:30:04.084108+0000","last_unstale":"2024-03-04T07:30:06.071686+0000","last_undegraded":"2024-03-04T07:30:06.071686+0000","last_fullsized":"2024-03-04T07:30:06.071686+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:35:28.188240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.080190+0000","last_change":"2024-03-04T07:30:05.090055+0000","last_active":"2024-03-04T07:30:06.080190+0000","last_peered":"2024-03-04T07:30:06.080190+0000","last_clean":"2024-03-04T07:30:06.080190+0000","last_became_active":"2024-03-04T07:30:05.089654+0000","last_became_peered":"2024-03-04T07:30:05.089654+0000","last_unstale":"2024-03-04T07:30:06.080190+0000","last_undegraded":"2024-03-04T07:30:06.080190+0000","last_fullsized":"2024-03-04T07:30:06.080190+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:52.747064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"65'1","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.097403+0000","last_change":"2024-03-04T07:30:04.088337+0000","last_active":"2024-03-04T07:30:05.097403+0000","last_peered":"2024-03-04T07:30:05.097403+0000","last_clean":"2024-03-04T07:30:05.097403+0000","last_became_active":"2024-03-04T07:30:04.088004+0000","last_became_peered":"2024-03-04T07:30:04.088004+0000","last_unstale":"2024-03-04T07:30:05.097403+0000","last_undegraded":"2024-03-04T07:30:05.097403+0000","last_fullsized":"2024-03-04T07:30:05.097403+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:44:15.112781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.088517+0000","last_change":"2024-03-04T07:30:05.084439+0000","last_active":"2024-03-04T07:30:05.088517+0000","last_peered":"2024-03-04T07:30:05.088517+0000","last_clean":"2024-03-04T07:30:05.088517+0000","last_became_active":"2024-03-04T07:30:05.084151+0000","last_became_peered":"2024-03-04T07:30:05.084151+0000","last_unstale":"2024-03-04T07:30:05.088517+0000","last_undegraded":"2024-03-04T07:30:05.088517+0000","last_fullsized":"2024-03-04T07:30:05.088517+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:37:15.704917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.074624+0000","last_change":"2024-03-04T07:30:04.088232+0000","last_active":"2024-03-04T07:30:05.074624+0000","last_peered":"2024-03-04T07:30:05.074624+0000","last_clean":"2024-03-04T07:30:05.074624+0000","last_became_active":"2024-03-04T07:30:04.087775+0000","last_became_peered":"2024-03-04T07:30:04.087775+0000","last_unstale":"2024-03-04T07:30:05.074624+0000","last_undegraded":"2024-03-04T07:30:05.074624+0000","last_fullsized":"2024-03-04T07:30:05.074624+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:10:21.100819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.095590+0000","last_change":"2024-03-04T07:30:05.092123+0000","last_active":"2024-03-04T07:30:05.095590+0000","last_peered":"2024-03-04T07:30:05.095590+0000","last_clean":"2024-03-04T07:30:05.095590+0000","last_became_active":"2024-03-04T07:30:05.091828+0000","last_became_peered":"2024-03-04T07:30:05.091828+0000","last_unstale":"2024-03-04T07:30:05.095590+0000","last_undegraded":"2024-03-04T07:30:05.095590+0000","last_fullsized":"2024-03-04T07:30:05.095590+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:44:37.388344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.088174+0000","last_change":"2024-03-04T07:30:04.084269+0000","last_active":"2024-03-04T07:30:05.088174+0000","last_peered":"2024-03-04T07:30:05.088174+0000","last_clean":"2024-03-04T07:30:05.088174+0000","last_became_active":"2024-03-04T07:30:04.083967+0000","last_became_peered":"2024-03-04T07:30:04.083967+0000","last_unstale":"2024-03-04T07:30:05.088174+0000","last_undegraded":"2024-03-04T07:30:05.088174+0000","last_fullsized":"2024-03-04T07:30:05.088174+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:10:21.237751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.100273+0000","last_change":"2024-03-04T07:30:05.096068+0000","last_active":"2024-03-04T07:30:05.100273+0000","last_peered":"2024-03-04T07:30:05.100273+0000","last_clean":"2024-03-04T07:30:05.100273+0000","last_became_active":"2024-03-04T07:30:05.095805+0000","last_became_peered":"2024-03-04T07:30:05.095805+0000","last_unstale":"2024-03-04T07:30:05.100273+0000","last_undegraded":"2024-03-04T07:30:05.100273+0000","last_fullsized":"2024-03-04T07:30:05.100273+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:03.769544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070921+0000","last_change":"2024-03-04T07:30:04.080719+0000","last_active":"2024-03-04T07:30:06.070921+0000","last_peered":"2024-03-04T07:30:06.070921+0000","last_clean":"2024-03-04T07:30:06.070921+0000","last_became_active":"2024-03-04T07:30:04.080390+0000","last_became_peered":"2024-03-04T07:30:04.080390+0000","last_unstale":"2024-03-04T07:30:06.070921+0000","last_undegraded":"2024-03-04T07:30:06.070921+0000","last_fullsized":"2024-03-04T07:30:06.070921+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:33:29.956468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.094927+0000","last_change":"2024-03-04T07:30:05.090639+0000","last_active":"2024-03-04T07:30:05.094927+0000","last_peered":"2024-03-04T07:30:05.094927+0000","last_clean":"2024-03-04T07:30:05.094927+0000","last_became_active":"2024-03-04T07:30:05.090378+0000","last_became_peered":"2024-03-04T07:30:05.090378+0000","last_unstale":"2024-03-04T07:30:05.094927+0000","last_undegraded":"2024-03-04T07:30:05.094927+0000","last_fullsized":"2024-03-04T07:30:05.094927+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:42:45.725745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071552+0000","last_change":"2024-03-04T07:30:04.083013+0000","last_active":"2024-03-04T07:30:06.071552+0000","last_peered":"2024-03-04T07:30:06.071552+0000","last_clean":"2024-03-04T07:30:06.071552+0000","last_became_active":"2024-03-04T07:30:04.082697+0000","last_became_peered":"2024-03-04T07:30:04.082697+0000","last_unstale":"2024-03-04T07:30:06.071552+0000","last_undegraded":"2024-03-04T07:30:06.071552+0000","last_fullsized":"2024-03-04T07:30:06.071552+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:38:53.665426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071924+0000","last_change":"2024-03-04T07:30:05.079174+0000","last_active":"2024-03-04T07:30:06.071924+0000","last_peered":"2024-03-04T07:30:06.071924+0000","last_clean":"2024-03-04T07:30:06.071924+0000","last_became_active":"2024-03-04T07:30:05.078886+0000","last_became_peered":"2024-03-04T07:30:05.078886+0000","last_unstale":"2024-03-04T07:30:06.071924+0000","last_undegraded":"2024-03-04T07:30:06.071924+0000","last_fullsized":"2024-03-04T07:30:06.071924+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:55.231331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.7","version":"65'2","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098729+0000","last_change":"2024-03-04T07:30:04.085269+0000","last_active":"2024-03-04T07:30:05.098729+0000","last_peered":"2024-03-04T07:30:05.098729+0000","last_clean":"2024-03-04T07:30:05.098729+0000","last_became_active":"2024-03-04T07:30:04.084644+0000","last_became_peered":"2024-03-04T07:30:04.084644+0000","last_unstale":"2024-03-04T07:30:05.098729+0000","last_undegraded":"2024-03-04T07:30:05.098729+0000","last_fullsized":"2024-03-04T07:30:05.098729+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:55:19.043273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.070827+0000","last_change":"2024-03-04T07:30:05.089201+0000","last_active":"2024-03-04T07:30:06.070827+0000","last_peered":"2024-03-04T07:30:06.070827+0000","last_clean":"2024-03-04T07:30:06.070827+0000","last_became_active":"2024-03-04T07:30:05.088914+0000","last_became_peered":"2024-03-04T07:30:05.088914+0000","last_unstale":"2024-03-04T07:30:06.070827+0000","last_undegraded":"2024-03-04T07:30:06.070827+0000","last_fullsized":"2024-03-04T07:30:06.070827+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:17:54.586600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071335+0000","last_change":"2024-03-04T07:30:04.084243+0000","last_active":"2024-03-04T07:30:06.071335+0000","last_peered":"2024-03-04T07:30:06.071335+0000","last_clean":"2024-03-04T07:30:06.071335+0000","last_became_active":"2024-03-04T07:30:04.083988+0000","last_became_peered":"2024-03-04T07:30:04.083988+0000","last_unstale":"2024-03-04T07:30:06.071335+0000","last_undegraded":"2024-03-04T07:30:06.071335+0000","last_fullsized":"2024-03-04T07:30:06.071335+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:32.049332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093715+0000","last_change":"2024-03-04T07:30:05.092852+0000","last_active":"2024-03-04T07:30:05.093715+0000","last_peered":"2024-03-04T07:30:05.093715+0000","last_clean":"2024-03-04T07:30:05.093715+0000","last_became_active":"2024-03-04T07:30:05.092547+0000","last_became_peered":"2024-03-04T07:30:05.092547+0000","last_unstale":"2024-03-04T07:30:05.093715+0000","last_undegraded":"2024-03-04T07:30:05.093715+0000","last_fullsized":"2024-03-04T07:30:05.093715+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:09.457728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.073700+0000","last_change":"2024-03-04T07:30:04.085522+0000","last_active":"2024-03-04T07:30:05.073700+0000","last_peered":"2024-03-04T07:30:05.073700+0000","last_clean":"2024-03-04T07:30:05.073700+0000","last_became_active":"2024-03-04T07:30:04.085239+0000","last_became_peered":"2024-03-04T07:30:04.085239+0000","last_unstale":"2024-03-04T07:30:05.073700+0000","last_undegraded":"2024-03-04T07:30:05.073700+0000","last_fullsized":"2024-03-04T07:30:05.073700+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:06:51.907642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.091135+0000","last_change":"2024-03-04T07:30:05.079345+0000","last_active":"2024-03-04T07:30:05.091135+0000","last_peered":"2024-03-04T07:30:05.091135+0000","last_clean":"2024-03-04T07:30:05.091135+0000","last_became_active":"2024-03-04T07:30:05.078981+0000","last_became_peered":"2024-03-04T07:30:05.078981+0000","last_unstale":"2024-03-04T07:30:05.091135+0000","last_undegraded":"2024-03-04T07:30:05.091135+0000","last_fullsized":"2024-03-04T07:30:05.091135+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:23.007594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071861+0000","last_change":"2024-03-04T07:30:04.088197+0000","last_active":"2024-03-04T07:30:06.071861+0000","last_peered":"2024-03-04T07:30:06.071861+0000","last_clean":"2024-03-04T07:30:06.071861+0000","last_became_active":"2024-03-04T07:30:04.087898+0000","last_became_peered":"2024-03-04T07:30:04.087898+0000","last_unstale":"2024-03-04T07:30:06.071861+0000","last_undegraded":"2024-03-04T07:30:06.071861+0000","last_fullsized":"2024-03-04T07:30:06.071861+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:08:38.100295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.089324+0000","last_change":"2024-03-04T07:30:05.086424+0000","last_active":"2024-03-04T07:30:05.089324+0000","last_peered":"2024-03-04T07:30:05.089324+0000","last_clean":"2024-03-04T07:30:05.089324+0000","last_became_active":"2024-03-04T07:30:05.086129+0000","last_became_peered":"2024-03-04T07:30:05.086129+0000","last_unstale":"2024-03-04T07:30:05.089324+0000","last_undegraded":"2024-03-04T07:30:05.089324+0000","last_fullsized":"2024-03-04T07:30:05.089324+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:07:11.622279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.075994+0000","last_change":"2024-03-04T07:30:04.079841+0000","last_active":"2024-03-04T07:30:05.075994+0000","last_peered":"2024-03-04T07:30:05.075994+0000","last_clean":"2024-03-04T07:30:05.075994+0000","last_became_active":"2024-03-04T07:30:04.079576+0000","last_became_peered":"2024-03-04T07:30:04.079576+0000","last_unstale":"2024-03-04T07:30:05.075994+0000","last_undegraded":"2024-03-04T07:30:05.075994+0000","last_fullsized":"2024-03-04T07:30:05.075994+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:23:10.581374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.098643+0000","last_change":"2024-03-04T07:30:05.093742+0000","last_active":"2024-03-04T07:30:05.098643+0000","last_peered":"2024-03-04T07:30:05.098643+0000","last_clean":"2024-03-04T07:30:05.098643+0000","last_became_active":"2024-03-04T07:30:05.093437+0000","last_became_peered":"2024-03-04T07:30:05.093437+0000","last_unstale":"2024-03-04T07:30:05.098643+0000","last_undegraded":"2024-03-04T07:30:05.098643+0000","last_fullsized":"2024-03-04T07:30:05.098643+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:37:23.639067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079887+0000","last_change":"2024-03-04T07:30:04.078433+0000","last_active":"2024-03-04T07:30:06.079887+0000","last_peered":"2024-03-04T07:30:06.079887+0000","last_clean":"2024-03-04T07:30:06.079887+0000","last_became_active":"2024-03-04T07:30:04.078130+0000","last_became_peered":"2024-03-04T07:30:04.078130+0000","last_unstale":"2024-03-04T07:30:06.079887+0000","last_undegraded":"2024-03-04T07:30:06.079887+0000","last_fullsized":"2024-03-04T07:30:06.079887+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:49:20.014438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":12,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.079946+0000","last_change":"2024-03-04T07:30:05.086188+0000","last_active":"2024-03-04T07:30:06.079946+0000","last_peered":"2024-03-04T07:30:06.079946+0000","last_clean":"2024-03-04T07:30:06.079946+0000","last_became_active":"2024-03-04T07:30:05.085626+0000","last_became_peered":"2024-03-04T07:30:05.085626+0000","last_unstale":"2024-03-04T07:30:06.079946+0000","last_undegraded":"2024-03-04T07:30:06.079946+0000","last_fullsized":"2024-03-04T07:30:06.079946+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:37:23.639067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-03-04T07:30:06.071208+0000","last_change":"2024-03-04T07:30:04.077165+0000","last_active":"2024-03-04T07:30:06.071208+0000","last_peered":"2024-03-04T07:30:06.071208+0000","last_clean":"2024-03-04T07:30:06.071208+0000","last_became_active":"2024-03-04T07:30:04.076855+0000","last_became_peered":"2024-03-04T07:30:04.076855+0000","last_unstale":"2024-03-04T07:30:06.071208+0000","last_undegraded":"2024-03-04T07:30:06.071208+0000","last_fullsized":"2024-03-04T07:30:06.071208+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:37:56.442757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.093376+0000","last_change":"2024-03-04T07:30:05.087930+0000","last_active":"2024-03-04T07:30:05.093376+0000","last_peered":"2024-03-04T07:30:05.093376+0000","last_clean":"2024-03-04T07:30:05.093376+0000","last_became_active":"2024-03-04T07:30:05.087569+0000","last_became_peered":"2024-03-04T07:30:05.087569+0000","last_unstale":"2024-03-04T07:30:05.093376+0000","last_undegraded":"2024-03-04T07:30:05.093376+0000","last_fullsized":"2024-03-04T07:30:05.093376+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:20:36.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":12,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.085786+0000","last_change":"2024-03-04T07:30:04.090076+0000","last_active":"2024-03-04T07:30:05.085786+0000","last_peered":"2024-03-04T07:30:05.085786+0000","last_clean":"2024-03-04T07:30:05.085786+0000","last_became_active":"2024-03-04T07:30:04.089818+0000","last_became_peered":"2024-03-04T07:30:04.089818+0000","last_unstale":"2024-03-04T07:30:05.085786+0000","last_undegraded":"2024-03-04T07:30:05.085786+0000","last_fullsized":"2024-03-04T07:30:05.085786+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:36:02.896980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":9,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.100264+0000","last_change":"2024-03-04T07:30:05.095682+0000","last_active":"2024-03-04T07:30:05.100264+0000","last_peered":"2024-03-04T07:30:05.100264+0000","last_clean":"2024-03-04T07:30:05.100264+0000","last_became_active":"2024-03-04T07:30:05.095372+0000","last_became_peered":"2024-03-04T07:30:05.095372+0000","last_unstale":"2024-03-04T07:30:05.100264+0000","last_undegraded":"2024-03-04T07:30:05.100264+0000","last_fullsized":"2024-03-04T07:30:05.100264+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:04.058953+0000","last_clean_scrub_stamp":"2024-03-04T07:30:04.058953+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:55:48.729808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"65'2","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.099688+0000","last_change":"2024-03-04T07:30:04.090065+0000","last_active":"2024-03-04T07:30:05.099688+0000","last_peered":"2024-03-04T07:30:05.099688+0000","last_clean":"2024-03-04T07:30:05.099688+0000","last_became_active":"2024-03-04T07:30:04.089768+0000","last_became_peered":"2024-03-04T07:30:04.089768+0000","last_unstale":"2024-03-04T07:30:05.099688+0000","last_undegraded":"2024-03-04T07:30:05.099688+0000","last_fullsized":"2024-03-04T07:30:05.099688+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:03.057020+0000","last_clean_scrub_stamp":"2024-03-04T07:30:03.057020+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:55.834473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":251,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-03-04T07:30:05.073930+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:30:05.073930+0000","last_peered":"2024-03-04T07:30:05.073930+0000","last_clean":"2024-03-04T07:30:05.073930+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:30:05.073930+0000","last_undegraded":"2024-03-04T07:30:05.073930+0000","last_fullsized":"2024-03-04T07:30:05.073930+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":10,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":21,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":44,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":13,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38,"ondisk_log_size":38,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574894,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32548,"kb_used_data":1360,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339292,"statfs":{"total":96636764160,"available":96603435008,"internally_reserved":0,"allocated":1392640,"data_stored":920876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574893,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31968,"kb_used_data":780,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339872,"statfs":{"total":96636764160,"available":96604028928,"internally_reserved":0,"allocated":798720,"data_stored":329984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574893,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31964,"kb_used_data":776,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339876,"statfs":{"total":96636764160,"available":96604033024,"internally_reserved":0,"allocated":794624,"data_stored":329982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574894,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31960,"kb_used_data":776,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339880,"statfs":{"total":96636764160,"available":96604037120,"internally_reserved":0,"allocated":794624,"data_stored":329962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574893,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31968,"kb_used_data":780,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339872,"statfs":{"total":96636764160,"available":96604028928,"internally_reserved":0,"allocated":798720,"data_stored":330004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574893,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31972,"kb_used_data":784,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339868,"statfs":{"total":96636764160,"available":96604024832,"internally_reserved":0,"allocated":802816,"data_stored":331504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31980,"kb_used_data":792,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339860,"statfs":{"total":96636764160,"available":96604016640,"internally_reserved":0,"allocated":811008,"data_stored":332120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574893,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32532,"kb_used_data":1352,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339308,"statfs":{"total":96636764160,"available":96603451392,"internally_reserved":0,"allocated":1384448,"data_stored":920260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":92,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:30:08.591 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:30:08.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:30:08.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.735+0000 7fb97c25c640 1 Processor -- start 2024-03-04T07:30:08.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.735+0000 7fb97c25c640 1 -- start start 2024-03-04T07:30:08.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97c25c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb97415b590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97c25c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb9740ce900 0x7fb9740ced90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97c25c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb97415bc40 con 0x7fb974144640 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97c25c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb97406df70 con 0x7fb9740ce900 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97c25c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb97415bf20 con 0x7fb97415b130 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97a7f9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb9740ce900 0x7fb9740ced90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97a7f9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb9740ce900 0x7fb9740ced90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49370/0 (socket says 172.21.15.64:49370) 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb97a7f9640 1 -- 172.21.15.64:0/2417898647 learned_addr learned my addr 172.21.15.64:0/2417898647 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.736+0000 7fb9797f7640 1 --2- 172.21.15.64:0/2417898647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb97415b590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb97a7f9640 1 -- 172.21.15.64:0/2417898647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 msgr2=0x7fb97415b590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb97a7f9640 1 --2- 172.21.15.64:0/2417898647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb97415b590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb97a7f9640 1 -- 172.21.15.64:0/2417898647 >> v1:172.21.15.66:6789/0 conn(0x7fb974144640 legacy=0x7fb97406e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb97a7f9640 1 -- 172.21.15.64:0/2417898647 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb97406dc00 con 0x7fb9740ce900 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb97a7f9640 1 --2- 172.21.15.64:0/2417898647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9740ce900 0x7fb9740ced90 secure :-1 s=READY pgs=845 cs=0 l=1 rev1=1 crypto rx=0x7fb96c009e50 tx=0x7fb96c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98696bb7009a2fc7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb978ff6640 1 -- 172.21.15.64:0/2417898647 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb96c02ca70 con 0x7fb9740ce900 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb978ff6640 1 -- 172.21.15.64:0/2417898647 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb96c02cc10 con 0x7fb9740ce900 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.737+0000 7fb978ff6640 1 -- 172.21.15.64:0/2417898647 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb96c0343f0 con 0x7fb9740ce900 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 -- 172.21.15.64:0/2417898647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9740ce900 msgr2=0x7fb9740ced90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 --2- 172.21.15.64:0/2417898647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9740ce900 0x7fb9740ced90 secure :-1 s=READY pgs=845 cs=0 l=1 rev1=1 crypto rx=0x7fb96c009e50 tx=0x7fb96c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 -- 172.21.15.64:0/2417898647 shutdown_connections 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 --2- 172.21.15.64:0/2417898647 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9740ce900 0x7fb9740ced90 unknown :-1 s=CLOSED pgs=845 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 --2- 172.21.15.64:0/2417898647 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb97415b590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 -- 172.21.15.64:0/2417898647 >> 172.21.15.64:0/2417898647 conn(0x7fb974066d00 msgr2=0x7fb974067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:08.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 -- 172.21.15.64:0/2417898647 shutdown_connections 2024-03-04T07:30:08.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.738+0000 7fb97c25c640 1 -- 172.21.15.64:0/2417898647 wait complete. 2024-03-04T07:30:08.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.739+0000 7fb97c25c640 1 Processor -- start 2024-03-04T07:30:08.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.739+0000 7fb97c25c640 1 -- start start 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb97c25c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9740ce900 0x7fb97414d390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb97c25c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb974144640 0x7fb97413ca30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb97c25c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb9741e4f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb97c25c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb974070c20 con 0x7fb974144640 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb97c25c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb974070aa0 con 0x7fb9740ce900 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb97c25c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb9740b36d0 con 0x7fb97415b130 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb974144640 0x7fb97413ca30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb974144640 0x7fb97413ca30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49378/0 (socket says 172.21.15.64:49378) 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 -- 172.21.15.64:0/1433956388 learned_addr learned my addr 172.21.15.64:0/1433956388 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb979ff8640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9740ce900 0x7fb97414d390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb97a7f9640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb9741e4f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 -- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 msgr2=0x7fb9741e4f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb9741e4f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 -- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9740ce900 msgr2=0x7fb97414d390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9740ce900 0x7fb97414d390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.740+0000 7fb9797f7640 1 -- 172.21.15.64:0/1433956388 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb97413d3a0 con 0x7fb974144640 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.741+0000 7fb9797f7640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb974144640 0x7fb97413ca30 secure :-1 s=READY pgs=846 cs=0 l=1 rev1=1 crypto rx=0x7fb96400c7f0 tx=0x7fb96400ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.741+0000 7fb96a7fc640 1 -- 172.21.15.64:0/1433956388 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb964015070 con 0x7fb974144640 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.741+0000 7fb96a7fc640 1 -- 172.21.15.64:0/1433956388 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb964004220 con 0x7fb974144640 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.741+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb97413d680 con 0x7fb974144640 2024-03-04T07:30:08.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.741+0000 7fb96a7fc640 1 -- 172.21.15.64:0/1433956388 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb964007620 con 0x7fb974144640 2024-03-04T07:30:08.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.741+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb97413dc40 con 0x7fb974144640 2024-03-04T07:30:08.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.742+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb9740733c0 con 0x7fb974144640 2024-03-04T07:30:08.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.742+0000 7fb96a7fc640 1 -- 172.21.15.64:0/1433956388 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb9640077c0 con 0x7fb974144640 2024-03-04T07:30:08.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.743+0000 7fb96a7fc640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb95806b4d0 0x7fb95806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:08.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.743+0000 7fb979ff8640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb95806b4d0 0x7fb95806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:08.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.743+0000 7fb96a7fc640 1 -- 172.21.15.64:0/1433956388 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb96408a330 con 0x7fb974144640 2024-03-04T07:30:08.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.743+0000 7fb979ff8640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb95806b4d0 0x7fb95806d990 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fb970003d90 tx=0x7fb970009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:08.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.747+0000 7fb96a7fc640 1 -- 172.21.15.64:0/1433956388 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb9640591b0 con 0x7fb974144640 2024-03-04T07:30:09.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:08.999+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7fb974143e40 con 0x7fb974144640 2024-03-04T07:30:09.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.002+0000 7fb96a7fc640 1 -- 172.21.15.64:0/1433956388 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v18) v1 ==== 190+0+147 (secure 0 0 0) 0x7fb964058ab0 con 0x7fb974144640 2024-03-04T07:30:09.003 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:30:09.003 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:30:09.003 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:30:09.003 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:30:09.003 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb95806b4d0 msgr2=0x7fb95806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb95806b4d0 0x7fb95806d990 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fb970003d90 tx=0x7fb970009040 comp rx=0 tx=0).stop 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb974144640 msgr2=0x7fb97413ca30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb974144640 0x7fb97413ca30 secure :-1 s=READY pgs=846 cs=0 l=1 rev1=1 crypto rx=0x7fb96400c7f0 tx=0x7fb96400ccb0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 shutdown_connections 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb95806b4d0 0x7fb95806d990 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb97415b130 0x7fb9741e4f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb974144640 0x7fb97413ca30 unknown :-1 s=CLOSED pgs=846 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 --2- 172.21.15.64:0/1433956388 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9740ce900 0x7fb97414d390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 >> 172.21.15.64:0/1433956388 conn(0x7fb974066d00 msgr2=0x7fb9740b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 shutdown_connections 2024-03-04T07:30:09.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.004+0000 7fb97c25c640 1 -- 172.21.15.64:0/1433956388 wait complete. 2024-03-04T07:30:09.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:09.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.116+0000 7f40dbb5c640 1 Processor -- start 2024-03-04T07:30:09.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.116+0000 7f40dbb5c640 1 -- start start 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.117+0000 7f40dbb5c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f40d40b4260 0x7f40d40cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.117+0000 7f40dbb5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40d06c0 0x7f40d40b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.117+0000 7f40dbb5c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f40d40b0dd0 con 0x7f40d40d02e0 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.117+0000 7f40dbb5c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f40d40b0b80 con 0x7f40d40b4260 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.117+0000 7f40dbb5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f40d40aeec0 con 0x7f40d40d06c0 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40da0f9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40d06c0 0x7f40d40b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40d90f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f40d40b4260 0x7f40d40cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40d98f8640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f40d40d02e0 0x7f40d40b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:57154/0 (socket says 172.21.15.64:57154) 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40d98f8640 1 -- 172.21.15.64:0/1507642172 learned_addr learned my addr 172.21.15.64:0/1507642172 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40d90f7640 1 -- 172.21.15.64:0/1507642172 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40d06c0 msgr2=0x7f40d40b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40d90f7640 1 --2- 172.21.15.64:0/1507642172 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40d06c0 0x7f40d40b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40d90f7640 1 -- 172.21.15.64:0/1507642172 >> v1:172.21.15.66:6789/0 conn(0x7f40d40d02e0 legacy=0x7f40d40b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.118+0000 7f40d90f7640 1 -- 172.21.15.64:0/1507642172 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40d40b10b0 con 0x7f40d40b4260 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40d90f7640 1 --2- 172.21.15.64:0/1507642172 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40b4260 0x7f40d40cfa00 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f40c80097a0 tx=0x7f40c802b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=31cbd036b076c9a1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40d88f6640 1 -- 172.21.15.64:0/1507642172 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40c8034070 con 0x7f40d40b4260 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40d88f6640 1 -- 172.21.15.64:0/1507642172 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f40c8030610 con 0x7f40d40b4260 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40d88f6640 1 -- 172.21.15.64:0/1507642172 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40c80309e0 con 0x7f40d40b4260 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40dbb5c640 1 -- 172.21.15.64:0/1507642172 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40b4260 msgr2=0x7f40d40cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/1507642172 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40b4260 0x7f40d40cfa00 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f40c80097a0 tx=0x7f40c802b960 comp rx=0 tx=0).stop 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40dbb5c640 1 -- 172.21.15.64:0/1507642172 shutdown_connections 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/1507642172 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40d06c0 0x7f40d40b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/1507642172 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40b4260 0x7f40d40cfa00 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.119+0000 7f40dbb5c640 1 -- 172.21.15.64:0/1507642172 >> 172.21.15.64:0/1507642172 conn(0x7f40d40b36d0 msgr2=0x7f40d40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.120+0000 7f40dbb5c640 1 -- 172.21.15.64:0/1507642172 shutdown_connections 2024-03-04T07:30:09.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.120+0000 7f40dbb5c640 1 -- 172.21.15.64:0/1507642172 wait complete. 2024-03-04T07:30:09.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 Processor -- start 2024-03-04T07:30:09.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 -- start start 2024-03-04T07:30:09.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40b4260 0x7f40d41f5c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:09.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f40d40d02e0 0x7f40d415ef20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:09.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40d06c0 0x7f40d415f460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40d40b22d0 con 0x7f40d40d02e0 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40d40b2150 con 0x7f40d40d06c0 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40dbb5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f40d40af350 con 0x7f40d40b4260 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.121+0000 7f40d90f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f40d40d02e0 0x7f40d415ef20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40d90f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f40d40d02e0 0x7f40d415ef20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49382/0 (socket says 172.21.15.64:49382) 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40d90f7640 1 -- 172.21.15.64:0/3113325653 learned_addr learned my addr 172.21.15.64:0/3113325653 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40da0f9640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40d06c0 0x7f40d415f460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40d98f8640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40b4260 0x7f40d41f5c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40da0f9640 1 -- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40b4260 msgr2=0x7f40d41f5c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40da0f9640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40b4260 0x7f40d41f5c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40da0f9640 1 -- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f40d40d02e0 msgr2=0x7f40d415ef20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40da0f9640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f40d40d02e0 0x7f40d415ef20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40da0f9640 1 -- 172.21.15.64:0/3113325653 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40d4144c70 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40d90f7640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f40d40d02e0 0x7f40d415ef20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40da0f9640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40d06c0 0x7f40d415f460 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f40cc00e820 tx=0x7f40cc00ece0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40d27fc640 1 -- 172.21.15.64:0/3113325653 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40cc017070 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40d27fc640 1 -- 172.21.15.64:0/3113325653 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f40cc00c730 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f40d415d860 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40d27fc640 1 -- 172.21.15.64:0/3113325653 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f40cc00cb40 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.122+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f40d415cf30 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.124+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f40d4068700 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.124+0000 7f40d27fc640 1 -- 172.21.15.64:0/3113325653 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f40cc01a070 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.124+0000 7f40d27fc640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f40c006b4d0 0x7f40c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.124+0000 7f40d27fc640 1 -- 172.21.15.64:0/3113325653 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f40cc089ad0 con 0x7f40d40d06c0 2024-03-04T07:30:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.124+0000 7f40d98f8640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f40c006b4d0 0x7f40c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:09.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.125+0000 7f40d98f8640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f40c006b4d0 0x7f40c006d990 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f40c40012e0 tx=0x7f40c4001a30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:09.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.129+0000 7f40d27fc640 1 -- 172.21.15.64:0/3113325653 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f40cc058390 con 0x7f40d40d06c0 2024-03-04T07:30:09.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.494+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f40d40ce450 con 0x7f40d40d06c0 2024-03-04T07:30:09.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.495+0000 7f40d27fc640 1 -- 172.21.15.64:0/3113325653 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f40cc05c430 con 0x7f40d40d06c0 2024-03-04T07:30:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:09.496 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:09.496 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:09.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f40c006b4d0 msgr2=0x7f40c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f40c006b4d0 0x7f40c006d990 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f40c40012e0 tx=0x7f40c4001a30 comp rx=0 tx=0).stop 2024-03-04T07:30:09.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40d06c0 msgr2=0x7f40d415f460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:09.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40d06c0 0x7f40d415f460 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f40cc00e820 tx=0x7f40cc00ece0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 shutdown_connections 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f40c006b4d0 0x7f40c006d990 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f40d40d06c0 0x7f40d415f460 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f40d40d02e0 0x7f40d415ef20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 --2- 172.21.15.64:0/3113325653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f40d40b4260 0x7f40d41f5c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 >> 172.21.15.64:0/3113325653 conn(0x7f40d40b36d0 msgr2=0x7f40d40b1520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 shutdown_connections 2024-03-04T07:30:09.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:09.497+0000 7f40dbb5c640 1 -- 172.21.15.64:0/3113325653 wait complete. 2024-03-04T07:30:09.912 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 36, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:30:04.437991+0000', 'modified': '2024-03-04T07:30:06.496433+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 6546}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6546': {'gid': 6546, 'name': 'a', 'rank': 0, 'incarnation': 34, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.64:6837/2304674412', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6836', 'nonce': 2304674412}, {'type': 'v1', 'addr': '172.21.15.64:6837', 'nonce': 2304674412}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [11], 'metadata_pool': 10, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 6546, 'qdb_cluster': [6546]} 2024-03-04T07:30:09.912 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:30:09.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.a status 2024-03-04T07:30:10.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.014+0000 7f37e490f640 1 Processor -- start 2024-03-04T07:30:10.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.014+0000 7f37e490f640 1 -- start start 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.015+0000 7f37e490f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 0x7f37e00ced50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.015+0000 7f37e490f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f37e0067b40 con 0x7f37e00cda40 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.015+0000 7f37e490f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f37e00d0070 con 0x7f37e006c040 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.015+0000 7f37e490f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f37e00cfe40 con 0x7f37e006b130 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.015+0000 7f37de59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f37e00cda40 0x7f37e006b930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45710/0 (socket says 172.21.15.64:45710) 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.015+0000 7f37de59c640 1 -- 172.21.15.64:0/1711568764 learned_addr learned my addr 172.21.15.64:0/1711568764 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 0x7f37e00ced50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3759467588 0 0) 0x7f37e0067b40 con 0x7f37e00cda40 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f37cc003520 con 0x7f37e00cda40 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4048585302 0 0) 0x7f37cc003520 con 0x7f37e00cda40 2024-03-04T07:30:10.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 msgr2=0x7f37e00ced50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 0x7f37e00ced50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 >> v1:172.21.15.66:6789/0 conn(0x7f37e006c040 legacy=0x7f37e015d6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37e00d0310 con 0x7f37e00cda40 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f37d0001670 con 0x7f37e00cda40 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 >> v1:172.21.15.64:6789/0 conn(0x7f37e00cda40 legacy=0x7f37e006b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 0x7f37cc0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37cc003f20 0x7f37cc004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37cc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37cc00baa0 con 0x7f37cc004d40 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37cc00bc50 con 0x7f37cc003f20 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f37cc00be50 con 0x7f37e006b130 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37cc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37d7fff640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37cc003f20 0x7f37cc004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37de59c640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 0x7f37cc0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ded9d640 1 -- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 msgr2=0x7f37cc0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e006b130 0x7f37cc0039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ded9d640 1 -- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37cc003f20 msgr2=0x7f37cc004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37cc003f20 0x7f37cc004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.016+0000 7f37ded9d640 1 -- 172.21.15.64:0/1711568764 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37cc003520 con 0x7f37cc004d40 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.017+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37cc0051c0 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7f37d8009fc0 tx=0x7f37cc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5bcf6269294fd3c5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.017+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d8039070 con 0x7f37cc004d40 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.017+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f37d802cc20 con 0x7f37cc004d40 2024-03-04T07:30:10.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.017+0000 7f37ddd9b640 1 -- 172.21.15.64:0/1711568764 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d8009c10 con 0x7f37cc004d40 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.018+0000 7f37e490f640 1 -- 172.21.15.64:0/1711568764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 msgr2=0x7f37cc0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.018+0000 7f37e490f640 1 --2- 172.21.15.64:0/1711568764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37cc0051c0 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7f37d8009fc0 tx=0x7f37cc005830 comp rx=0 tx=0).stop 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.018+0000 7f37ded9d640 1 -- 172.21.15.64:0/1711568764 reap_dead start 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.018+0000 7f37e490f640 1 -- 172.21.15.64:0/1711568764 shutdown_connections 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.018+0000 7f37e490f640 1 -- 172.21.15.64:0/1711568764 >> 172.21.15.64:0/1711568764 conn(0x7f37e00d1a40 msgr2=0x7f37e00d1e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.018+0000 7f37e490f640 1 -- 172.21.15.64:0/1711568764 shutdown_connections 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.018+0000 7f37e490f640 1 -- 172.21.15.64:0/1711568764 wait complete. 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.019+0000 7f37e490f640 1 Processor -- start 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.019+0000 7f37e490f640 1 -- start start 2024-03-04T07:30:10.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37e490f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37e01f63b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37e490f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37e00c6a90 0x7f37e00c6f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37e490f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e015f9a0 0x7f37e015fe20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37e490f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37e00cd0a0 con 0x7f37cc004d40 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37e490f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37e00d0c40 con 0x7f37e00c6a90 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37e490f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f37e00d08d0 con 0x7f37e015f9a0 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37e01f63b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37e01f63b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49408/0 (socket says 172.21.15.64:49408) 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 -- 172.21.15.64:0/3394585145 learned_addr learned my addr 172.21.15.64:0/3394585145 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37d7fff640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37e00c6a90 0x7f37e00c6f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37ded9d640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e015f9a0 0x7f37e015fe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 -- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e015f9a0 msgr2=0x7f37e015fe20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e015f9a0 0x7f37e015fe20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 -- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37e00c6a90 msgr2=0x7f37e00c6f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37e00c6a90 0x7f37e00c6f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37de59c640 1 -- 172.21.15.64:0/3394585145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37e00c7550 con 0x7f37cc004d40 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.020+0000 7f37ded9d640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e015f9a0 0x7f37e015fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.021+0000 7f37de59c640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37e01f63b0 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7f37d0007310 tx=0x7f37d00077d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.021+0000 7f37d6ffd640 1 -- 172.21.15.64:0/3394585145 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d000f960 con 0x7f37cc004d40 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.021+0000 7f37d6ffd640 1 -- 172.21.15.64:0/3394585145 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f37d0018070 con 0x7f37cc004d40 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.021+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f37e00c7830 con 0x7f37cc004d40 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.021+0000 7f37d6ffd640 1 -- 172.21.15.64:0/3394585145 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d000fd60 con 0x7f37cc004d40 2024-03-04T07:30:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.021+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f37e00c7df0 con 0x7f37cc004d40 2024-03-04T07:30:10.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.023+0000 7f37d6ffd640 1 -- 172.21.15.64:0/3394585145 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f37d001d460 con 0x7f37cc004d40 2024-03-04T07:30:10.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.023+0000 7f37d6ffd640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f37c006b500 0x7f37c006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.023+0000 7f37d6ffd640 1 -- 172.21.15.64:0/3394585145 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f37d008a280 con 0x7f37cc004d40 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.023+0000 7f37d7fff640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f37c006b500 0x7f37c006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.024+0000 7f37d7fff640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f37c006b500 0x7f37c006d9c0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f37e015b380 tx=0x7f37c8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.024+0000 7f37e75cb740 1 Processor -- start 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.024+0000 7f37e75cb740 1 -- start start 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.025+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8eb4870 0x5562d8eb4c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.025+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8eb88c0 0x5562d8eba360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.025+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee4a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37e75cb740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562d8e1e6b0 con 0x5562d8ee4600 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562d8dc7a60 con 0x5562d8eb88c0 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5562d8e1a750 con 0x5562d8eb4870 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee4a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee4a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49412/0 (socket says 172.21.15.64:49412) 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 -- 172.21.15.64:0/1559217012 learned_addr learned my addr 172.21.15.64:0/1559217012 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:10.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37de59c640 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8eb4870 0x5562d8eb4c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37d7fff640 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8eb88c0 0x5562d8eba360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 -- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8eb4870 msgr2=0x5562d8eb4c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8eb4870 0x5562d8eb4c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 -- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8eb88c0 msgr2=0x5562d8eba360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8eb88c0 0x5562d8eba360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 -- 172.21.15.64:0/1559217012 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5562d8de5f30 con 0x5562d8ee4600 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37d7fff640 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8eb88c0 0x5562d8eba360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37ded9d640 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee4a80 secure :-1 s=READY pgs=850 cs=0 l=1 rev1=1 crypto rx=0x7f37d800e200 tx=0x7f37d802e070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b618ba9c4aa6ceb7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37bffff640 1 -- 172.21.15.64:0/1559217012 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d8043070 con 0x5562d8ee4600 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37bffff640 1 -- 172.21.15.64:0/1559217012 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f37d802cc20 con 0x5562d8ee4600 2024-03-04T07:30:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.026+0000 7f37bffff640 1 -- 172.21.15.64:0/1559217012 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d8039040 con 0x5562d8ee4600 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 -- 172.21.15.64:0/1559217012 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 msgr2=0x5562d8ee4a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee4a80 secure :-1 s=READY pgs=850 cs=0 l=1 rev1=1 crypto rx=0x7f37d800e200 tx=0x7f37d802e070 comp rx=0 tx=0).stop 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 -- 172.21.15.64:0/1559217012 shutdown_connections 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee4a80 unknown :-1 s=CLOSED pgs=850 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8eb88c0 0x5562d8eba360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 --2- 172.21.15.64:0/1559217012 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8eb4870 0x5562d8eb4c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 -- 172.21.15.64:0/1559217012 >> 172.21.15.64:0/1559217012 conn(0x5562d8eb68f0 msgr2=0x5562d8eb81b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 -- 172.21.15.64:0/1559217012 shutdown_connections 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.027+0000 7f37e75cb740 1 -- 172.21.15.64:0/1559217012 wait complete. 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.028+0000 7f37e75cb740 1 -- start start 2024-03-04T07:30:10.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.028+0000 7f37e75cb740 1 Processor -- start 2024-03-04T07:30:10.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8ee4600 0x5562d8ede980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee77b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edefd0 0x5562d8ee7cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562d8eb8d50 con 0x5562d8edefd0 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562d8ddaea0 con 0x5562d8edf520 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5562d8eb9030 con 0x5562d8ee4600 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 10 client.0 _ll_register_callbacks cb 0x5562d8dda870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edefd0 0x5562d8ee7cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edefd0 0x5562d8ee7cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49424/0 (socket says 172.21.15.64:49424) 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 -- 172.21.15.64:0/2985339763 learned_addr learned my addr 172.21.15.64:0/2985339763 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37de59c640 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8ee4600 0x5562d8ede980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37d7fff640 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 -- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8ee4600 msgr2=0x5562d8ede980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8ee4600 0x5562d8ede980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 -- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 msgr2=0x5562d8ee77b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee77b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 -- 172.21.15.64:0/2985339763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5562d8ddb270 con 0x5562d8edefd0 2024-03-04T07:30:10.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37ded9d640 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edefd0 0x5562d8ee7cf0 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f37d8037040 tx=0x7f37d8004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a7e0a3997aa99043 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37bd7fa640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37bd7fa640 1 -- 172.21.15.64:0/2985339763 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d8039070 con 0x5562d8edefd0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37bd7fa640 1 -- 172.21.15.64:0/2985339763 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f37d8004030 con 0x5562d8edefd0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.029+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f37d802cc20 con 0x5562d8edefd0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37bd7fa640 1 -- 172.21.15.64:0/2985339763 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=36) v2 ==== 24+0+0 (secure 0 0 0) 0x7f37d8052050 con 0x5562d8edefd0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 10 client.6687 fetch_fsmap learned FSMap version 36 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=36,monmap=2+}) v3 -- 0x5562d8b332d0 con 0x5562d8edefd0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37bd7fa640 1 -- 172.21.15.64:0/2985339763 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f37d802cc20 con 0x5562d8edefd0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37bd7fa640 1 -- 172.21.15.64:0/2985339763 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 36) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f37d8055020 con 0x5562d8edefd0 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 10 client.6687 fetch_fsmap finished waiting for FSMap version 36 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 10 client.6687 resolve_mds: resolved name 'a' to daemon mds.a 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 20 client.6687 populate_metadata read hostname 'smithi064' 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee5e50 0x5562d8ee6230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 4 client.6687 mds_command: new command op to 6546 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 --> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5562d8e2bbf0 con 0x5562d8ee5e50 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37de59c640 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee5e50 0x5562d8ee6230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37bd7fa640 10 client.6687 ms_handle_connect on v2:172.21.15.64:6836/2304674412 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.030+0000 7f37de59c640 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee5e50 0x5562d8ee6230 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8ec81a6f2b60cf16 server_cookie=bd0476c6ec14f999 in_seq=0 out_seq=0 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.031+0000 7f37bd7fa640 1 -- 172.21.15.64:0/2985339763 <== mds.0 v2:172.21.15.64:6836/2304674412 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x5562d8e2bbf0 con 0x5562d8ee5e50 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.031+0000 7f37bd7fa640 10 client.6687 handle_command_reply: tid=0 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.031+0000 7f37e75cb740 1 client.6687 shutdown 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.031+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edefd0 msgr2=0x5562d8ee7cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.031+0000 7f37e75cb740 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edefd0 0x5562d8ee7cf0 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f37d8037040 tx=0x7f37d8004ab0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 shutdown_connections 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee5e50 0x5562d8ee6230 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edefd0 0x5562d8ee7cf0 unknown :-1 s=CLOSED pgs=851 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 --2- 172.21.15.64:0/2985339763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8ee4600 0x5562d8ede980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37bd7fa640 0 client.6687 ms_handle_reset on v2:172.21.15.64:6836/2304674412 2024-03-04T07:30:10.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 >> 172.21.15.64:0/2985339763 conn(0x5562d8eb4870 msgr2=0x5562d8ee25c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 shutdown_connections 2024-03-04T07:30:10.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 1 -- 172.21.15.64:0/2985339763 wait complete. 2024-03-04T07:30:10.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.032+0000 7f37e75cb740 20 client.6687 trim_cache size 0 max 16384 2024-03-04T07:30:10.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 Processor -- start 2024-03-04T07:30:10.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 -- start start 2024-03-04T07:30:10.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8edd890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edf520 0x5562d8edddd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8ee4600 0x5562d8ee0190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37e01ff4e0 con 0x5562d8edf520 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37e01f4630 con 0x5562d8ee4600 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f37e01fef10 con 0x5562d8edefd0 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37d7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edf520 0x5562d8edddd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37d7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edf520 0x5562d8edddd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49438/0 (socket says 172.21.15.64:49438) 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.049+0000 7f37d7fff640 1 -- 172.21.15.64:0/4234941898 learned_addr learned my addr 172.21.15.64:0/4234941898 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37ded9d640 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8ee4600 0x5562d8ee0190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37de59c640 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8edd890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37d7fff640 1 -- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 msgr2=0x5562d8edd890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37d7fff640 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8edd890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37d7fff640 1 -- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8ee4600 msgr2=0x5562d8ee0190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37d7fff640 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8ee4600 0x5562d8ee0190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37d7fff640 1 -- 172.21.15.64:0/4234941898 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5562d8bfd6b0 con 0x5562d8edf520 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37d7fff640 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edf520 0x5562d8edddd0 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7f37c80019b0 tx=0x7f37c8014000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f8cceddfc8fa3418 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37bf7fe640 1 -- 172.21.15.64:0/4234941898 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37c801b030 con 0x5562d8edf520 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37bf7fe640 1 -- 172.21.15.64:0/4234941898 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f37c8029070 con 0x5562d8edf520 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.050+0000 7f37bf7fe640 1 -- 172.21.15.64:0/4234941898 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37c8024480 con 0x5562d8edf520 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 -- 172.21.15.64:0/4234941898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edf520 msgr2=0x5562d8edddd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edf520 0x5562d8edddd0 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7f37c80019b0 tx=0x7f37c8014000 comp rx=0 tx=0).stop 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 -- 172.21.15.64:0/4234941898 shutdown_connections 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8ee4600 0x5562d8ee0190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8edf520 0x5562d8edddd0 unknown :-1 s=CLOSED pgs=852 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 --2- 172.21.15.64:0/4234941898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8edd890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 -- 172.21.15.64:0/4234941898 >> 172.21.15.64:0/4234941898 conn(0x5562d8eb4870 msgr2=0x5562d8eb8f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 -- 172.21.15.64:0/4234941898 shutdown_connections 2024-03-04T07:30:10.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 -- 172.21.15.64:0/4234941898 wait complete. 2024-03-04T07:30:10.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.051+0000 7f37e75cb740 1 -- start start 2024-03-04T07:30:10.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.052+0000 7f37e75cb740 1 Processor -- start 2024-03-04T07:30:10.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8f71f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee0190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37e75cb740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee06d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37e75cb740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562d8eb8d50 con 0x5562d8ee4600 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562d8ddaea0 con 0x5562d8edf520 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37e75cb740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5562d8ee6900 con 0x5562d8edefd0 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37e75cb740 10 client.0 _ll_register_callbacks cb 0x7f37cc0044f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee06d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee06d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49446/0 (socket says 172.21.15.64:49446) 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 -- 172.21.15.64:0/3451006164 learned_addr learned my addr 172.21.15.64:0/3451006164 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37d7fff640 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee0190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37de59c640 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8f71f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 -- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 msgr2=0x5562d8f71f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8f71f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 -- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 msgr2=0x5562d8ee0190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee0190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 -- 172.21.15.64:0/3451006164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5562d8ed8970 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37ded9d640 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee06d0 secure :-1 s=READY pgs=853 cs=0 l=1 rev1=1 crypto rx=0x7f37d8037040 tx=0x7f37d802e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6747134c396d2a26 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.053+0000 7f37bd7fa640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37bd7fa640 1 -- 172.21.15.64:0/3451006164 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37d8039040 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37bd7fa640 1 -- 172.21.15.64:0/3451006164 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f37d8004440 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f37c8029070 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37bd7fa640 1 -- 172.21.15.64:0/3451006164 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=36) v2 ==== 24+0+0 (secure 0 0 0) 0x7f37d8043070 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 10 client.6693 fetch_fsmap learned FSMap version 36 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=36,monmap=2+}) v3 -- 0x5562d8efc440 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37bd7fa640 1 -- 172.21.15.64:0/3451006164 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f37c8029070 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37bd7fa640 1 -- 172.21.15.64:0/3451006164 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 36) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f37d8055050 con 0x5562d8ee4600 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 10 client.6693 fetch_fsmap finished waiting for FSMap version 36 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 10 client.6693 resolve_mds: resolved name 'a' to daemon mds.a 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 20 client.6693 populate_metadata read hostname 'smithi064' 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee3f50 0x5562d8f72610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 4 client.6693 mds_command: new command op to 6546 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 --> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5562d8eb5e90 con 0x5562d8ee3f50 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.054+0000 7f37de59c640 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee3f50 0x5562d8f72610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37bd7fa640 10 client.6693 ms_handle_connect on v2:172.21.15.64:6836/2304674412 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37de59c640 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee3f50 0x5562d8f72610 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7d1fcfc672866b78 server_cookie=1288aacbcae817a2 in_seq=0 out_seq=0 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37bd7fa640 1 -- 172.21.15.64:0/3451006164 <== mds.0 v2:172.21.15.64:6836/2304674412 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+324 (crc 0 0 0) 0x5562d8eb5e90 con 0x5562d8ee3f50 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37bd7fa640 10 client.6693 handle_command_reply: tid=0 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 client.6693 shutdown 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 msgr2=0x5562d8ee06d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee06d0 secure :-1 s=READY pgs=853 cs=0 l=1 rev1=1 crypto rx=0x7f37d8037040 tx=0x7f37d802e000 comp rx=0 tx=0).stop 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 shutdown_connections 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:6836/2304674412,v1:172.21.15.64:6837/2304674412] conn(0x5562d8ee3f50 0x5562d8f72610 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5562d8ee4600 0x5562d8ee06d0 unknown :-1 s=CLOSED pgs=853 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5562d8edf520 0x5562d8ee0190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 --2- 172.21.15.64:0/3451006164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5562d8edefd0 0x5562d8f71f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37bd7fa640 0 client.6693 ms_handle_reset on v2:172.21.15.64:6836/2304674412 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 >> 172.21.15.64:0/3451006164 conn(0x5562d8eb4870 msgr2=0x5562d8ef9ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 shutdown_connections 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 1 -- 172.21.15.64:0/3451006164 wait complete. 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.055+0000 7f37e75cb740 20 client.6693 trim_cache size 0 max 16384 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 6546, 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:30:10.061 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 4.989944833, 2024-03-04T07:30:10.062 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 36, 2024-03-04T07:30:10.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 66, 2024-03-04T07:30:10.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 66, 2024-03-04T07:30:10.062 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 9.9909972880000009 2024-03-04T07:30:10.062 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:30:10.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f37c006b500 msgr2=0x7f37c006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f37c006b500 0x7f37c006d9c0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f37e015b380 tx=0x7f37c8009040 comp rx=0 tx=0).stop 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 msgr2=0x7f37e01f63b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37e01f63b0 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7f37d0007310 tx=0x7f37d00077d0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 shutdown_connections 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f37c006b500 0x7f37c006d9c0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f37e015f9a0 0x7f37e015fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37e00c6a90 0x7f37e00c6f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 --2- 172.21.15.64:0/3394585145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f37cc004d40 0x7f37e01f63b0 unknown :-1 s=CLOSED pgs=849 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 >> 172.21.15.64:0/3394585145 conn(0x7f37e00d1a40 msgr2=0x7f37e006ead0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 shutdown_connections 2024-03-04T07:30:10.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.056+0000 7f37e490f640 1 -- 172.21.15.64:0/3394585145 wait complete. 2024-03-04T07:30:10.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:30:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.168+0000 7feea389a640 1 Processor -- start 2024-03-04T07:30:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.168+0000 7feea389a640 1 -- start start 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea389a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fee9c0d02e0 0x7fee9c0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea389a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0b4260 0x7fee9c0b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea389a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fee9c0b0dd0 con 0x7fee9c0d06c0 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea389a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fee9c0b0b80 con 0x7fee9c0d02e0 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea389a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fee9c0aeec0 con 0x7fee9c0b4260 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea1e37640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fee9c0d06c0 0x7fee9c0b1a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45722/0 (socket says 172.21.15.64:45722) 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea1e37640 1 -- 172.21.15.64:0/1726931799 learned_addr learned my addr 172.21.15.64:0/1726931799 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea0e35640 1 --2- 172.21.15.64:0/1726931799 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0b4260 0x7fee9c0b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea1636640 1 --2- 172.21.15.64:0/1726931799 >> v2:172.21.15.66:3300/0 conn(0x7fee9c0d02e0 0x7fee9c0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7fee93fff640 1 -- 172.21.15.64:0/1726931799 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 821132434 0 0) 0x7fee9c0b0dd0 con 0x7fee9c0d06c0 2024-03-04T07:30:10.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7fee93fff640 1 -- 172.21.15.64:0/1726931799 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fee88003520 con 0x7fee9c0d06c0 2024-03-04T07:30:10.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea0e35640 1 -- 172.21.15.64:0/1726931799 >> v2:172.21.15.66:3300/0 conn(0x7fee9c0d02e0 msgr2=0x7fee9c0cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:10.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea0e35640 1 --2- 172.21.15.64:0/1726931799 >> v2:172.21.15.66:3300/0 conn(0x7fee9c0d02e0 0x7fee9c0cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea0e35640 1 -- 172.21.15.64:0/1726931799 >> v1:172.21.15.64:6789/0 conn(0x7fee9c0d06c0 legacy=0x7fee9c0b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea0e35640 1 -- 172.21.15.64:0/1726931799 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fee9c0b10b0 con 0x7fee9c0b4260 2024-03-04T07:30:10.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.170+0000 7feea1636640 1 --2- 172.21.15.64:0/1726931799 >> v2:172.21.15.66:3300/0 conn(0x7fee9c0d02e0 0x7fee9c0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.171+0000 7feea0e35640 1 --2- 172.21.15.64:0/1726931799 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0b4260 0x7fee9c0b1500 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fee8c0097a0 tx=0x7fee8c02b960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b4b5db7b4c005e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.171+0000 7fee93fff640 1 -- 172.21.15.64:0/1726931799 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee8c034070 con 0x7fee9c0b4260 2024-03-04T07:30:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.171+0000 7fee93fff640 1 -- 172.21.15.64:0/1726931799 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fee8c030610 con 0x7fee9c0b4260 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.171+0000 7fee93fff640 1 -- 172.21.15.64:0/1726931799 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee8c0309c0 con 0x7fee9c0b4260 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 -- 172.21.15.64:0/1726931799 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0b4260 msgr2=0x7fee9c0b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 --2- 172.21.15.64:0/1726931799 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0b4260 0x7fee9c0b1500 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fee8c0097a0 tx=0x7fee8c02b960 comp rx=0 tx=0).stop 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 -- 172.21.15.64:0/1726931799 shutdown_connections 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 --2- 172.21.15.64:0/1726931799 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0b4260 0x7fee9c0b1500 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 --2- 172.21.15.64:0/1726931799 >> v2:172.21.15.66:3300/0 conn(0x7fee9c0d02e0 0x7fee9c0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 -- 172.21.15.64:0/1726931799 >> 172.21.15.64:0/1726931799 conn(0x7fee9c0b36d0 msgr2=0x7fee9c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 -- 172.21.15.64:0/1726931799 shutdown_connections 2024-03-04T07:30:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.172+0000 7feea389a640 1 -- 172.21.15.64:0/1726931799 wait complete. 2024-03-04T07:30:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.174+0000 7feea389a640 1 Processor -- start 2024-03-04T07:30:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.174+0000 7feea389a640 1 -- start start 2024-03-04T07:30:10.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea389a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fee9c0b4260 0x7fee9c0ca630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea389a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fee9c0d02e0 0x7fee9c0cab70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea389a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0d06c0 0x7fee9c1f51a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea389a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee9c0b22d0 con 0x7fee9c0d02e0 2024-03-04T07:30:10.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea389a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee9c0b2150 con 0x7fee9c0b4260 2024-03-04T07:30:10.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea389a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fee9c0af350 con 0x7fee9c0d06c0 2024-03-04T07:30:10.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea0e35640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fee9c0d02e0 0x7fee9c0cab70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea0e35640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fee9c0d02e0 0x7fee9c0cab70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49456/0 (socket says 172.21.15.64:49456) 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.175+0000 7feea0e35640 1 -- 172.21.15.64:0/2368687574 learned_addr learned my addr 172.21.15.64:0/2368687574 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea1e37640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0d06c0 0x7fee9c1f51a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea1636640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fee9c0b4260 0x7fee9c0ca630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea0e35640 1 -- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0d06c0 msgr2=0x7fee9c1f51a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea0e35640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0d06c0 0x7fee9c1f51a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea0e35640 1 -- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fee9c0b4260 msgr2=0x7fee9c0ca630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea0e35640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fee9c0b4260 0x7fee9c0ca630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea0e35640 1 -- 172.21.15.64:0/2368687574 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fee9c0cb1b0 con 0x7fee9c0d02e0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea0e35640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fee9c0d02e0 0x7fee9c0cab70 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7fee8c02b9a0 tx=0x7fee8c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7fee91ffb640 1 -- 172.21.15.64:0/2368687574 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee8c048070 con 0x7fee9c0d02e0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7fee91ffb640 1 -- 172.21.15.64:0/2368687574 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fee8c0044c0 con 0x7fee9c0d02e0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fee9c1574b0 con 0x7fee9c0d02e0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7fee91ffb640 1 -- 172.21.15.64:0/2368687574 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee8c034050 con 0x7fee9c0d02e0 2024-03-04T07:30:10.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.176+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fee9c155dc0 con 0x7fee9c0d02e0 2024-03-04T07:30:10.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.177+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fee9c068700 con 0x7fee9c0d02e0 2024-03-04T07:30:10.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.178+0000 7fee91ffb640 1 -- 172.21.15.64:0/2368687574 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fee8c04b070 con 0x7fee9c0d02e0 2024-03-04T07:30:10.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.178+0000 7fee91ffb640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fee8006b4d0 0x7fee8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:10.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.178+0000 7fee91ffb640 1 -- 172.21.15.64:0/2368687574 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fee8c0abda0 con 0x7fee9c0d02e0 2024-03-04T07:30:10.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.178+0000 7feea1636640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fee8006b4d0 0x7fee8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:10.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.179+0000 7feea1636640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fee8006b4d0 0x7fee8006d990 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7fee98009560 tx=0x7fee98008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:10.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.186+0000 7fee91ffb640 1 -- 172.21.15.64:0/2368687574 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fee8c07ac20 con 0x7fee9c0d02e0 2024-03-04T07:30:10.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.405+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fee9c0b1f30 con 0x7fee9c0d02e0 2024-03-04T07:30:10.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.405+0000 7fee91ffb640 1 -- 172.21.15.64:0/2368687574 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v36) v1 ==== 53+0+73 (secure 0 0 0) 0x7fee8c07a520 con 0x7fee9c0d02e0 2024-03-04T07:30:10.406 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:30:10.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fee8006b4d0 msgr2=0x7fee8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fee8006b4d0 0x7fee8006d990 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7fee98009560 tx=0x7fee98008040 comp rx=0 tx=0).stop 2024-03-04T07:30:10.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fee9c0d02e0 msgr2=0x7fee9c0cab70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:10.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fee9c0d02e0 0x7fee9c0cab70 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7fee8c02b9a0 tx=0x7fee8c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 shutdown_connections 2024-03-04T07:30:10.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fee8006b4d0 0x7fee8006d990 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fee9c0d06c0 0x7fee9c1f51a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fee9c0d02e0 0x7fee9c0cab70 unknown :-1 s=CLOSED pgs=855 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 --2- 172.21.15.64:0/2368687574 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fee9c0b4260 0x7fee9c0ca630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:10.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 >> 172.21.15.64:0/2368687574 conn(0x7fee9c0b36d0 msgr2=0x7fee9c0d2200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:10.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 shutdown_connections 2024-03-04T07:30:10.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:10.408+0000 7feea389a640 1 -- 172.21.15.64:0/2368687574 wait complete. 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:30:10.420 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:30:10.420 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:10.420 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:30:10.436 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42502sec preferred_lft 42502sec 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:30:10.437 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:10.438 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:10.438 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:30:10.502 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:30:10.503 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:10.503 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:30:10.558 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:30:10.558 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:30:10.558 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:30:10.558 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:30:10.558 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:30:10.559 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:10.559 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:30:10.559 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:30:10.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:30:10.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:30:10.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:30:10.559 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:30:10.653 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:10.653 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:30:10.667 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:30:10.667 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:30:10.668 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:30:10.668 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:10.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:30:10.734 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:30:10.734 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:30:10.734 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:10.789 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:30:10.790 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:30:10.790 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:10.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:30:10.910 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:30:10.965 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:30:10.965 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:30:10.966 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:30:10.967 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:30:10.968 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:30:10.968 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:10.968 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:10.968 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:10.968 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:10.968 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:10.968 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:30:11.020 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:30:11.020 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:30:11.048 INFO:tasks.ceph.mds.a.smithi064.stderr:2024-03-04T07:30:11.048+0000 7fa96940e640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:30:11.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:30:11.077 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:30:11.134 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:30:11.134 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:30:11.135 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:11.136 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:11.137 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:11.137 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:11.137 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:11.137 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:30:11.138 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:30:11.137+0000 7f865e83f480 -1 init, newargv = 0x56097c8a37b0 newargc=15 2024-03-04T07:30:11.138 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[62844]: starting ceph client 2024-03-04T07:30:11.142 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[62844]: starting fuse 2024-03-04T07:30:11.153 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:30:11.154 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:30:11.154 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:30:11.155 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:30:11.156 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:11.156 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:30:11.156 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:30:11.274 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.62844.asok 2024-03-04T07:30:11.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:30:11.284 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.62844.asok 2024-03-04T07:30:11.284 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:11.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.62844.asok status 2024-03-04T07:30:11.422 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:30:11.422 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:30:11.422 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:30:11.422 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "62844", 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 6705, 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:30:11.423 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 6705 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 3486769189 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:30:11.424 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 3486769189 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.6705 192.168.0.1:0/3486769189", 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/3486769189", 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 36, 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 66, 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:30:11.425 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:30:11.431 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:11.431 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:30:11.431 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:11.446 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:30:11.447 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:11.447 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:11.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:11.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mds mds_bal_fragment_size_max 10 2024-03-04T07:30:11.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.657+0000 7f16d357a640 1 Processor -- start 2024-03-04T07:30:11.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.658+0000 7f16d357a640 1 -- start start 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16d357a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f16cc0cf740 0x7f16cc0cea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16d357a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16cc0cd310 con 0x7f16cc15f500 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16d357a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16cc0cd5b0 con 0x7f16cc0c5e30 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16d357a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f16cc15f8e0 con 0x7f16cc0cf740 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16d1316640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f16cc15f500 0x7f16cc0ce300 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45734/0 (socket says 172.21.15.64:45734) 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16d1316640 1 -- 172.21.15.64:0/1906752127 learned_addr learned my addr 172.21.15.64:0/1906752127 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1515680934 0 0) 0x7f16cc0cd310 con 0x7f16cc15f500 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.659+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f16b8003520 con 0x7f16cc15f500 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d0b15640 1 --2- 172.21.15.64:0/1906752127 >> v2:172.21.15.66:3300/0 conn(0x7f16cc0cf740 0x7f16cc0cea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 873939129 0 0) 0x7f16b8003520 con 0x7f16cc15f500 2024-03-04T07:30:11.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 >> v2:172.21.15.66:3300/0 conn(0x7f16cc0cf740 msgr2=0x7f16cc0cea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 --2- 172.21.15.64:0/1906752127 >> v2:172.21.15.66:3300/0 conn(0x7f16cc0cf740 0x7f16cc0cea10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 >> v1:172.21.15.66:6789/0 conn(0x7f16cc0c5e30 legacy=0x7f16cc0c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16cc0cd890 con 0x7f16cc15f500 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f16c8001690 con 0x7f16cc15f500 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 >> v1:172.21.15.64:6789/0 conn(0x7f16cc15f500 legacy=0x7f16cc0ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b8003a00 0x7f16b8003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8004800 0x7f16b8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16b80052e0 0x7f16b800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16cc15f8e0 con 0x7f16b8003a00 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16b800c200 con 0x7f16b80052e0 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16b800c400 con 0x7f16b8004800 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1316640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b8003a00 0x7f16b8003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1b17640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16b80052e0 0x7f16b800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d0b15640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8004800 0x7f16b8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1316640 1 -- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8004800 msgr2=0x7f16b8004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:11.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1316640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8004800 0x7f16b8004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1316640 1 -- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16b80052e0 msgr2=0x7f16b800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:11.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1316640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16b80052e0 0x7f16b800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1b17640 1 -- 172.21.15.64:0/1906752127 reap_dead start 2024-03-04T07:30:11.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.660+0000 7f16d1316640 1 -- 172.21.15.64:0/1906752127 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16b8003520 con 0x7f16b8003a00 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.661+0000 7f16d1316640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b8003a00 0x7f16b8003e20 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7f16c800c160 tx=0x7f16b8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1df1fcbb7c476111 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c8007c60 con 0x7f16b8003a00 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16c8030070 con 0x7f16b8003a00 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16c3fff640 1 -- 172.21.15.64:0/1906752127 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c8005d10 con 0x7f16b8003a00 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16d357a640 1 -- 172.21.15.64:0/1906752127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b8003a00 msgr2=0x7f16b8003e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16d357a640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b8003a00 0x7f16b8003e20 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7f16c800c160 tx=0x7f16b8005830 comp rx=0 tx=0).stop 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16d357a640 1 -- 172.21.15.64:0/1906752127 shutdown_connections 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16d357a640 1 --2- 172.21.15.64:0/1906752127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16b8003a00 0x7f16b8003e20 unknown :-1 s=CLOSED pgs=860 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16d357a640 1 -- 172.21.15.64:0/1906752127 >> 172.21.15.64:0/1906752127 conn(0x7f16cc0b3700 msgr2=0x7f16cc0b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16d357a640 1 -- 172.21.15.64:0/1906752127 shutdown_connections 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.662+0000 7f16d357a640 1 -- 172.21.15.64:0/1906752127 wait complete. 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.664+0000 7f16d357a640 1 Processor -- start 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.664+0000 7f16d357a640 1 -- start start 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d357a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8003a00 0x7f16cc151990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d357a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16cc0ca0c0 0x7f16cc0b6890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d357a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16cc0b6dd0 0x7f16cc0cb100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d357a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16cc0c59a0 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d357a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16cc0c56c0 con 0x7f16cc0ca0c0 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d357a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16cc0ce300 con 0x7f16b8003a00 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16cc0b6dd0 0x7f16cc0cb100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16cc0b6dd0 0x7f16cc0cb100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49466/0 (socket says 172.21.15.64:49466) 2024-03-04T07:30:11.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 -- 172.21.15.64:0/4200996368 learned_addr learned my addr 172.21.15.64:0/4200996368 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d0b15640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16cc0ca0c0 0x7f16cc0b6890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1316640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8003a00 0x7f16cc151990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 -- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8003a00 msgr2=0x7f16cc151990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8003a00 0x7f16cc151990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 -- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16cc0ca0c0 msgr2=0x7f16cc0b6890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16cc0ca0c0 0x7f16cc0b6890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d1b17640 1 -- 172.21.15.64:0/4200996368 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16cc0c9020 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.665+0000 7f16d0b15640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16cc0ca0c0 0x7f16cc0b6890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.666+0000 7f16d1b17640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16cc0b6dd0 0x7f16cc0cb100 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7f16cc0cf6d0 tx=0x7f16c400fc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.666+0000 7f16c1ffb640 1 -- 172.21.15.64:0/4200996368 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c4003b80 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.666+0000 7f16c1ffb640 1 -- 172.21.15.64:0/4200996368 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f16c4018070 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.666+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16cc0c9300 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.666+0000 7f16c1ffb640 1 -- 172.21.15.64:0/4200996368 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16c40050d0 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.666+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16cc0c98c0 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.667+0000 7f16c1ffb640 1 -- 172.21.15.64:0/4200996368 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f16c4005270 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.667+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16cc09fcf0 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.668+0000 7f16c1ffb640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f16b006b500 0x7f16b006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:11.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.668+0000 7f16d1316640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f16b006b500 0x7f16b006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:11.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.668+0000 7f16c1ffb640 1 -- 172.21.15.64:0/4200996368 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f16c4089a70 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.668+0000 7f16d1316640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f16b006b500 0x7f16b006d9c0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f16c8034000 tx=0x7f16c8009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:11.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.672+0000 7f16c1ffb640 1 -- 172.21.15.64:0/4200996368 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f16c4058a60 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.870+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mds_bal_fragment_size_max}] v 0) v1 -- 0x7f16cc151ed0 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.873+0000 7f16c1ffb640 1 -- 172.21.15.64:0/4200996368 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_bal_fragment_size_max}]=0 v23)=0 v23) v1 ==== 124+0+0 (secure 0 0 0) 0x7f16c4058360 con 0x7f16cc0b6dd0 2024-03-04T07:30:11.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f16b006b500 msgr2=0x7f16b006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:11.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f16b006b500 0x7f16b006d9c0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f16c8034000 tx=0x7f16c8009030 comp rx=0 tx=0).stop 2024-03-04T07:30:11.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16cc0b6dd0 msgr2=0x7f16cc0cb100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:11.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16cc0b6dd0 0x7f16cc0cb100 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7f16cc0cf6d0 tx=0x7f16c400fc00 comp rx=0 tx=0).stop 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 shutdown_connections 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f16b006b500 0x7f16b006d9c0 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16cc0b6dd0 0x7f16cc0cb100 unknown :-1 s=CLOSED pgs=861 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16cc0ca0c0 0x7f16cc0b6890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 --2- 172.21.15.64:0/4200996368 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16b8003a00 0x7f16cc151990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 >> 172.21.15.64:0/4200996368 conn(0x7f16cc0b3700 msgr2=0x7f16cc0d0670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 shutdown_connections 2024-03-04T07:30:11.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:11.876+0000 7f16d357a640 1 -- 172.21.15.64:0/4200996368 wait complete. 2024-03-04T07:30:17.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:30:17.659 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:30:21.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.004+0000 7f04643bf640 1 Processor -- start 2024-03-04T07:30:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.004+0000 7f04643bf640 1 -- start start 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.005+0000 7f04643bf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f045c15df00 0x7f045c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.005+0000 7f04643bf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f045c0b95a0 0x7f045c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.005+0000 7f04643bf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f045c0cea30 con 0x7f045c0b9a30 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.005+0000 7f04643bf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f045c0d2af0 con 0x7f045c0b95a0 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.005+0000 7f04643bf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f045c0cf3c0 con 0x7f045c15df00 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.005+0000 7f046195a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f045c0b95a0 0x7f045c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f046295c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f045c0b9a30 0x7f045c0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59092/0 (socket says 172.21.15.64:59092) 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f046295c640 1 -- 172.21.15.64:0/651250249 learned_addr learned my addr 172.21.15.64:0/651250249 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f046215b640 1 --2- 172.21.15.64:0/651250249 >> v2:172.21.15.66:3300/0 conn(0x7f045c15df00 0x7f045c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f046195a640 1 -- 172.21.15.64:0/651250249 >> v2:172.21.15.66:3300/0 conn(0x7f045c15df00 msgr2=0x7f045c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f046195a640 1 --2- 172.21.15.64:0/651250249 >> v2:172.21.15.66:3300/0 conn(0x7f045c15df00 0x7f045c0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f046195a640 1 -- 172.21.15.64:0/651250249 >> v1:172.21.15.64:6789/0 conn(0x7f045c0b9a30 legacy=0x7f045c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f046195a640 1 -- 172.21.15.64:0/651250249 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f045c0cf0e0 con 0x7f045c0b95a0 2024-03-04T07:30:22.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.006+0000 7f0461159640 1 -- 172.21.15.64:0/651250249 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3220821902 0 0) 0x7f045c0cea30 con 0x7f045c0b9a30 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.007+0000 7f046195a640 1 --2- 172.21.15.64:0/651250249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c0b95a0 0x7f045c0b5700 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7f045c0d11b0 tx=0x7f045002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=805e49dd5b0fc019 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.007+0000 7f0461159640 1 -- 172.21.15.64:0/651250249 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450035020 con 0x7f045c0b95a0 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.007+0000 7f0461159640 1 -- 172.21.15.64:0/651250249 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0450031830 con 0x7f045c0b95a0 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.007+0000 7f0461159640 1 -- 172.21.15.64:0/651250249 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450031c00 con 0x7f045c0b95a0 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 -- 172.21.15.64:0/651250249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c0b95a0 msgr2=0x7f045c0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 --2- 172.21.15.64:0/651250249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c0b95a0 0x7f045c0b5700 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7f045c0d11b0 tx=0x7f045002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 -- 172.21.15.64:0/651250249 shutdown_connections 2024-03-04T07:30:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 --2- 172.21.15.64:0/651250249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c0b95a0 0x7f045c0b5700 unknown :-1 s=CLOSED pgs=863 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 --2- 172.21.15.64:0/651250249 >> v2:172.21.15.66:3300/0 conn(0x7f045c15df00 0x7f045c0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 -- 172.21.15.64:0/651250249 >> 172.21.15.64:0/651250249 conn(0x7f045c071960 msgr2=0x7f045c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:22.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 -- 172.21.15.64:0/651250249 shutdown_connections 2024-03-04T07:30:22.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.008+0000 7f04643bf640 1 -- 172.21.15.64:0/651250249 wait complete. 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.009+0000 7f04643bf640 1 Processor -- start 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.009+0000 7f04643bf640 1 -- start start 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f04643bf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f045c0b95a0 0x7f045c15a860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f04643bf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c15df00 0x7f045c1f78d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f04643bf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f045c15ae40 0x7f045c161060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f04643bf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f045c0cec70 con 0x7f045c15df00 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f04643bf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f045c0ce750 con 0x7f045c15ae40 2024-03-04T07:30:22.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f04643bf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f045c0cedf0 con 0x7f045c0b95a0 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c15df00 0x7f045c1f78d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c15df00 0x7f045c1f78d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42688/0 (socket says 172.21.15.64:42688) 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 -- 172.21.15.64:0/3201790930 learned_addr learned my addr 172.21.15.64:0/3201790930 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046295c640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f045c15ae40 0x7f045c161060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046215b640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f045c0b95a0 0x7f045c15a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 -- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f045c0b95a0 msgr2=0x7f045c15a860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f045c0b95a0 0x7f045c15a860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 -- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f045c15ae40 msgr2=0x7f045c161060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f045c15ae40 0x7f045c161060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 -- 172.21.15.64:0/3201790930 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f045c1f6e30 con 0x7f045c15df00 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.010+0000 7f046195a640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c15df00 0x7f045c1f78d0 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f0450000c00 tx=0x7f04500046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.011+0000 7f045affd640 1 -- 172.21.15.64:0/3201790930 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0450035070 con 0x7f045c15df00 2024-03-04T07:30:22.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.011+0000 7f045affd640 1 -- 172.21.15.64:0/3201790930 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0450004250 con 0x7f045c15df00 2024-03-04T07:30:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.011+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f045c1e8db0 con 0x7f045c15df00 2024-03-04T07:30:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.011+0000 7f045affd640 1 -- 172.21.15.64:0/3201790930 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f045003a630 con 0x7f045c15df00 2024-03-04T07:30:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.011+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f045c1f73b0 con 0x7f045c15df00 2024-03-04T07:30:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.011+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f045c067ff0 con 0x7f045c15df00 2024-03-04T07:30:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.012+0000 7f045affd640 1 -- 172.21.15.64:0/3201790930 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0450038020 con 0x7f045c15df00 2024-03-04T07:30:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.013+0000 7f045affd640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044806b4d0 0x7f044806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.013+0000 7f045affd640 1 -- 172.21.15.64:0/3201790930 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f04500abd20 con 0x7f045c15df00 2024-03-04T07:30:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.013+0000 7f046215b640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044806b4d0 0x7f044806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.013+0000 7f046215b640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044806b4d0 0x7f044806d990 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f0454009aa0 tx=0x7f0454009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:22.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.017+0000 7f045affd640 1 -- 172.21.15.64:0/3201790930 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0450040070 con 0x7f045c15df00 2024-03-04T07:30:22.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.235+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f045c0ce0a0 con 0x7f045c15df00 2024-03-04T07:30:22.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.235+0000 7f045affd640 1 -- 172.21.15.64:0/3201790930 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f045007a550 con 0x7f045c15df00 2024-03-04T07:30:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:22.235 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:22.236 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:22.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044806b4d0 msgr2=0x7f044806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044806b4d0 0x7f044806d990 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f0454009aa0 tx=0x7f0454009190 comp rx=0 tx=0).stop 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c15df00 msgr2=0x7f045c1f78d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c15df00 0x7f045c1f78d0 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f0450000c00 tx=0x7f04500046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 shutdown_connections 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f044806b4d0 0x7f044806d990 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f045c15ae40 0x7f045c161060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f045c15df00 0x7f045c1f78d0 unknown :-1 s=CLOSED pgs=864 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 --2- 172.21.15.64:0/3201790930 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f045c0b95a0 0x7f045c15a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 >> 172.21.15.64:0/3201790930 conn(0x7f045c071960 msgr2=0x7f045c0b8730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.237+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 shutdown_connections 2024-03-04T07:30:22.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.238+0000 7f04643bf640 1 -- 172.21.15.64:0/3201790930 wait complete. 2024-03-04T07:30:22.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_created 2024-03-04T07:30:22.365 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_created":0}} 2024-03-04T07:30:22.366 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 0 } } 2024-03-04T07:30:22.366 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:22.366 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:22.381 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:30:22.381 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:30:22.381 DEBUG:teuthology.orchestra.run.smithi064:> import uuid 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> n = 100 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = False 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> if 0 > 0: 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = True 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> dpath = os.path.dirname(path) 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> fnameprefix = os.path.basename(path) 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> os.makedirs(dpath, exist_ok=True) 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> try: 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> for i in range(n): 2024-03-04T07:30:22.382 DEBUG:teuthology.orchestra.run.smithi064:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> with open(fpath, '"'"'w'"'"') as f: 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> f.write(f"{i}") 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> f.flush() 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(f.fileno()) 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> if True: 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> os.unlink(fpath) 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> if create_hard_links: 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> for j in range(0): 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> if True: 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> finally: 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> os.close(dirfd) 2024-03-04T07:30:22.383 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:30:22.532 INFO:teuthology.orchestra.run.smithi064.stderr:Traceback (most recent call last): 2024-03-04T07:30:22.533 INFO:teuthology.orchestra.run.smithi064.stderr: File "", line 26, in 2024-03-04T07:30:22.533 INFO:teuthology.orchestra.run.smithi064.stderr:OSError: [Errno 28] No space left on device: '/home/ubuntu/cephtest/mnt.0/subdir/file_10' 2024-03-04T07:30:22.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22 stdin-killer INFO: command exited with status 1: exiting normally with same code! 2024-03-04T07:30:22.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:30:22.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:22.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.689+0000 7f075bbb6640 1 Processor -- start 2024-03-04T07:30:22.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.689+0000 7f075bbb6640 1 -- start start 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.690+0000 7f075bbb6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 0x7f07540d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.690+0000 7f075bbb6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f07540b95a0 0x7f07540b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.690+0000 7f075bbb6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f07540cea30 con 0x7f07540b9a30 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.690+0000 7f075bbb6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f07540d2af0 con 0x7f07540b95a0 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.690+0000 7f075bbb6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f07540cf3c0 con 0x7f075415df00 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f075a153640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f07540b9a30 0x7f07540d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59102/0 (socket says 172.21.15.64:59102) 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f075a153640 1 -- 172.21.15.64:0/2850655264 learned_addr learned my addr 172.21.15.64:0/2850655264 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:22.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0759151640 1 --2- 172.21.15.64:0/2850655264 >> v2:172.21.15.66:3300/0 conn(0x7f07540b95a0 0x7f07540b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0759952640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 0x7f07540d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1377691256 0 0) 0x7f07540cea30 con 0x7f07540b9a30 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0740003520 con 0x7f07540b9a30 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1886866052 0 0) 0x7f0740003520 con 0x7f07540b9a30 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 msgr2=0x7f07540d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 0x7f07540d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 >> v2:172.21.15.66:3300/0 conn(0x7f07540b95a0 msgr2=0x7f07540b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 --2- 172.21.15.64:0/2850655264 >> v2:172.21.15.66:3300/0 conn(0x7f07540b95a0 0x7f07540b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f07540cf0e0 con 0x7f07540b9a30 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f074c0019f0 con 0x7f07540b9a30 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0759151640 1 --2- 172.21.15.64:0/2850655264 >> v2:172.21.15.66:3300/0 conn(0x7f07540b95a0 0x7f07540b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 >> v1:172.21.15.64:6789/0 conn(0x7f07540b9a30 legacy=0x7f07540d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 0x7f07400039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f0740004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0740004d40 0x7f07400051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07540d2af0 con 0x7f0740003f30 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07540cf3c0 con 0x7f0740004d40 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f074000b9c0 con 0x7f075415df00 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0759151640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f0740004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.691+0000 7f0759952640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 0x7f07400039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.692+0000 7f075a153640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0740004d40 0x7f07400051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.692+0000 7f0759151640 1 -- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 msgr2=0x7f07400039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.692+0000 7f0759151640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075415df00 0x7f07400039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.692+0000 7f0759151640 1 -- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0740004d40 msgr2=0x7f07400051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.692+0000 7f0759151640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0740004d40 0x7f07400051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.692+0000 7f0759151640 1 -- 172.21.15.64:0/2850655264 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0740003520 con 0x7f0740003f30 2024-03-04T07:30:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.692+0000 7f0759151640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f0740004800 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7f0744009c90 tx=0x7f0740005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dfbe9012b0d2d47d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0744039070 con 0x7f0740003f30 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f074402c980 con 0x7f0740003f30 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f0758950640 1 -- 172.21.15.64:0/2850655264 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0744009860 con 0x7f0740003f30 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f075bbb6640 1 -- 172.21.15.64:0/2850655264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 msgr2=0x7f0740004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f075bbb6640 1 --2- 172.21.15.64:0/2850655264 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f0740004800 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7f0744009c90 tx=0x7f0740005830 comp rx=0 tx=0).stop 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f075a153640 1 -- 172.21.15.64:0/2850655264 reap_dead start 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f075bbb6640 1 -- 172.21.15.64:0/2850655264 shutdown_connections 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f075bbb6640 1 -- 172.21.15.64:0/2850655264 >> 172.21.15.64:0/2850655264 conn(0x7f0754071960 msgr2=0x7f0754071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f075bbb6640 1 -- 172.21.15.64:0/2850655264 shutdown_connections 2024-03-04T07:30:22.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.693+0000 7f075bbb6640 1 -- 172.21.15.64:0/2850655264 wait complete. 2024-03-04T07:30:22.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.694+0000 7f075bbb6640 1 Processor -- start 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.694+0000 7f075bbb6640 1 -- start start 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f075bbb6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f07540b4710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f075bbb6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075414b620 0x7f0754149b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f075bbb6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f075414a070 0x7f07541f7010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f075bbb6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07540cec70 con 0x7f0740003f30 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f075bbb6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07540ce750 con 0x7f075414a070 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f075bbb6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f07540cedf0 con 0x7f075414b620 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f0759952640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f07540b4710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f0759952640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f07540b4710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42708/0 (socket says 172.21.15.64:42708) 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f0759952640 1 -- 172.21.15.64:0/1381289386 learned_addr learned my addr 172.21.15.64:0/1381289386 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f075a153640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f075414a070 0x7f07541f7010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.695+0000 7f0759151640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075414b620 0x7f0754149b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f0759952640 1 -- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075414b620 msgr2=0x7f0754149b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f0759952640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075414b620 0x7f0754149b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f0759952640 1 -- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f075414a070 msgr2=0x7f07541f7010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f0759952640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f075414a070 0x7f07541f7010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f0759952640 1 -- 172.21.15.64:0/1381289386 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f075414a580 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f075a153640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f075414a070 0x7f07541f7010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f0759151640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075414b620 0x7f0754149b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f0759952640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f07540b4710 secure :-1 s=READY pgs=867 cs=0 l=1 rev1=1 crypto rx=0x7f0750011a00 tx=0x7f0750011ec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f074a7fc640 1 -- 172.21.15.64:0/1381289386 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f075000fb50 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f074a7fc640 1 -- 172.21.15.64:0/1381289386 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f075000fcf0 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f075414a860 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f074a7fc640 1 -- 172.21.15.64:0/1381289386 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f075000fea0 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.696+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f075414ae20 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.697+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0754067ff0 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.698+0000 7f074a7fc640 1 -- 172.21.15.64:0/1381289386 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0750012040 con 0x7f0740003f30 2024-03-04T07:30:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.698+0000 7f074a7fc640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f073806b500 0x7f073806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:22.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.698+0000 7f0759151640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f073806b500 0x7f073806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:22.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.698+0000 7f074a7fc640 1 -- 172.21.15.64:0/1381289386 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f075008a780 con 0x7f0740003f30 2024-03-04T07:30:22.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.699+0000 7f0759151640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f073806b500 0x7f073806d9c0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f0744036000 tx=0x7f0744030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:22.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.703+0000 7f074a7fc640 1 -- 172.21.15.64:0/1381289386 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0750019030 con 0x7f0740003f30 2024-03-04T07:30:22.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.922+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f07540d2ee0 con 0x7f0740003f30 2024-03-04T07:30:22.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.922+0000 7f074a7fc640 1 -- 172.21.15.64:0/1381289386 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f0750059690 con 0x7f0740003f30 2024-03-04T07:30:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:22.923 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:22.923 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f073806b500 msgr2=0x7f073806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f073806b500 0x7f073806d9c0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f0744036000 tx=0x7f0744030000 comp rx=0 tx=0).stop 2024-03-04T07:30:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 msgr2=0x7f07540b4710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f07540b4710 secure :-1 s=READY pgs=867 cs=0 l=1 rev1=1 crypto rx=0x7f0750011a00 tx=0x7f0750011ec0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 shutdown_connections 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f073806b500 0x7f073806d9c0 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f075414a070 0x7f07541f7010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f075414b620 0x7f0754149b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 --2- 172.21.15.64:0/1381289386 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0740003f30 0x7f07540b4710 unknown :-1 s=CLOSED pgs=867 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 >> 172.21.15.64:0/1381289386 conn(0x7f0754071960 msgr2=0x7f075415bfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 shutdown_connections 2024-03-04T07:30:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:22.924+0000 7f075bbb6640 1 -- 172.21.15.64:0/1381289386 wait complete. 2024-03-04T07:30:22.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_created 2024-03-04T07:30:23.052 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_created":10}} 2024-03-04T07:30:23.052 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 10 } } 2024-03-04T07:30:23.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:23.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.155+0000 7f3ebf77a640 1 Processor -- start 2024-03-04T07:30:23.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.155+0000 7f3ebf77a640 1 -- start start 2024-03-04T07:30:23.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3ebf77a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 0x7f3eb80ced50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3ebf77a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3eb8067b40 con 0x7f3eb80cda40 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3ebf77a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3eb80d0070 con 0x7f3eb806c040 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3ebf77a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3eb80cfe40 con 0x7f3eb806b130 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3ebd516640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3eb80cda40 0x7f3eb806b930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59114/0 (socket says 172.21.15.64:59114) 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3ebd516640 1 -- 172.21.15.64:0/4195203717 learned_addr learned my addr 172.21.15.64:0/4195203717 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3ebdd17640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 0x7f3eb80ced50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3372881260 0 0) 0x7f3eb8067b40 con 0x7f3eb80cda40 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3ea4003520 con 0x7f3eb80cda40 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3590532411 0 0) 0x7f3ea4003520 con 0x7f3eb80cda40 2024-03-04T07:30:23.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 msgr2=0x7f3eb80ced50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3eabfff640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 0x7f3eb80ced50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.156+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 >> v1:172.21.15.66:6789/0 conn(0x7f3eb806c040 legacy=0x7f3eb815d6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3eb80d0310 con 0x7f3eb80cda40 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f3eac001670 con 0x7f3eb80cda40 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 >> v1:172.21.15.64:6789/0 conn(0x7f3eb80cda40 legacy=0x7f3eb806b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ea40039e0 0x7f3ea4003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 0x7f3ea4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ea4004d40 0x7f3ea40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ea400ba70 con 0x7f3ea40039e0 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ea400bbf0 con 0x7f3ea4004d40 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ea400bdf0 con 0x7f3eb806b130 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebd516640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ea40039e0 0x7f3ea4003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebcd15640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 0x7f3ea4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebdd17640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ea4004d40 0x7f3ea40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebd516640 1 -- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 msgr2=0x7f3ea4004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebd516640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3eb806b130 0x7f3ea4004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebd516640 1 -- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ea4004d40 msgr2=0x7f3ea40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebd516640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ea4004d40 0x7f3ea40051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.157+0000 7f3ebd516640 1 -- 172.21.15.64:0/4195203717 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ea4003520 con 0x7f3ea40039e0 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.158+0000 7f3ebd516640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ea40039e0 0x7f3ea4003dc0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7f3eac00c290 tx=0x7f3ea4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c75f13894894aeb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.158+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3eac007d50 con 0x7f3ea40039e0 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.158+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3eac030070 con 0x7f3ea40039e0 2024-03-04T07:30:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.158+0000 7f3eabfff640 1 -- 172.21.15.64:0/4195203717 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3eac02cd80 con 0x7f3ea40039e0 2024-03-04T07:30:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.159+0000 7f3ebf77a640 1 -- 172.21.15.64:0/4195203717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ea40039e0 msgr2=0x7f3ea4003dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.159+0000 7f3ebf77a640 1 --2- 172.21.15.64:0/4195203717 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ea40039e0 0x7f3ea4003dc0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7f3eac00c290 tx=0x7f3ea4005830 comp rx=0 tx=0).stop 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.159+0000 7f3ebdd17640 1 -- 172.21.15.64:0/4195203717 reap_dead start 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.159+0000 7f3ebf77a640 1 -- 172.21.15.64:0/4195203717 shutdown_connections 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.159+0000 7f3ebf77a640 1 -- 172.21.15.64:0/4195203717 >> 172.21.15.64:0/4195203717 conn(0x7f3eb80d1a40 msgr2=0x7f3eb80d1e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.159+0000 7f3ebf77a640 1 -- 172.21.15.64:0/4195203717 shutdown_connections 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.159+0000 7f3ebf77a640 1 -- 172.21.15.64:0/4195203717 wait complete. 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.160+0000 7f3ebf77a640 1 Processor -- start 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.160+0000 7f3ebf77a640 1 -- start start 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebf77a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ea40039e0 0x7f3eb80c5570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebf77a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3eb81572c0 0x7f3eb80b65f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebf77a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3eb815fe50 0x7f3eb81602d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebf77a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3eb80cd0a0 con 0x7f3eb81572c0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebf77a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3eb80d0c40 con 0x7f3eb815fe50 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebf77a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3eb80d08d0 con 0x7f3ea40039e0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3eb81572c0 0x7f3eb80b65f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3eb81572c0 0x7f3eb80b65f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42734/0 (socket says 172.21.15.64:42734) 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 -- 172.21.15.64:0/2652496020 learned_addr learned my addr 172.21.15.64:0/2652496020 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebd516640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ea40039e0 0x7f3eb80c5570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebdd17640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3eb815fe50 0x7f3eb81602d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 -- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ea40039e0 msgr2=0x7f3eb80c5570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ea40039e0 0x7f3eb80c5570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 -- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3eb815fe50 msgr2=0x7f3eb81602d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3eb815fe50 0x7f3eb81602d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 -- 172.21.15.64:0/2652496020 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3eb8157740 con 0x7f3eb81572c0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebcd15640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3eb81572c0 0x7f3eb80b65f0 secure :-1 s=READY pgs=870 cs=0 l=1 rev1=1 crypto rx=0x7f3eb400f860 tx=0x7f3eb400fd20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ea9ffb640 1 -- 172.21.15.64:0/2652496020 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3eb40052e0 con 0x7f3eb81572c0 2024-03-04T07:30:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ea9ffb640 1 -- 172.21.15.64:0/2652496020 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3eb4018070 con 0x7f3eb81572c0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ea9ffb640 1 -- 172.21.15.64:0/2652496020 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3eb400d910 con 0x7f3eb81572c0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.161+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3eb80c5be0 con 0x7f3eb81572c0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.162+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3eb80c61a0 con 0x7f3eb81572c0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.162+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3eb8068700 con 0x7f3eb81572c0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.163+0000 7f3ea9ffb640 1 -- 172.21.15.64:0/2652496020 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3eb400dab0 con 0x7f3eb81572c0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.163+0000 7f3ea9ffb640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e9c06b500 0x7f3e9c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.163+0000 7f3ebd516640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e9c06b500 0x7f3e9c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.163+0000 7f3ea9ffb640 1 -- 172.21.15.64:0/2652496020 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3eb4089f80 con 0x7f3eb81572c0 2024-03-04T07:30:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.164+0000 7f3ebd516640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e9c06b500 0x7f3e9c06d9c0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f3eac034000 tx=0x7f3eac009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:23.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.168+0000 7f3ea9ffb640 1 -- 172.21.15.64:0/2652496020 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3eb4058840 con 0x7f3eb81572c0 2024-03-04T07:30:23.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.386+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3eb80b79c0 con 0x7f3eb81572c0 2024-03-04T07:30:23.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.386+0000 7f3ea9ffb640 1 -- 172.21.15.64:0/2652496020 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f3eb405c8e0 con 0x7f3eb81572c0 2024-03-04T07:30:23.388 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:23.388 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:23.389 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:23.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e9c06b500 msgr2=0x7f3e9c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:23.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e9c06b500 0x7f3e9c06d9c0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f3eac034000 tx=0x7f3eac009030 comp rx=0 tx=0).stop 2024-03-04T07:30:23.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3eb81572c0 msgr2=0x7f3eb80b65f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3eb81572c0 0x7f3eb80b65f0 secure :-1 s=READY pgs=870 cs=0 l=1 rev1=1 crypto rx=0x7f3eb400f860 tx=0x7f3eb400fd20 comp rx=0 tx=0).stop 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 shutdown_connections 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e9c06b500 0x7f3e9c06d9c0 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3eb815fe50 0x7f3eb81602d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3eb81572c0 0x7f3eb80b65f0 unknown :-1 s=CLOSED pgs=870 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 --2- 172.21.15.64:0/2652496020 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ea40039e0 0x7f3eb80c5570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 >> 172.21.15.64:0/2652496020 conn(0x7f3eb80d1a40 msgr2=0x7f3eb80b13a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 shutdown_connections 2024-03-04T07:30:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:23.389+0000 7f3ebf77a640 1 -- 172.21.15.64:0/2652496020 wait complete. 2024-03-04T07:30:23.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:30:23.520 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-04T07:30:23.521 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-04T07:30:23.521 DEBUG:tasks.ceph_test_case:wait_until_equal: 0 != 10, waiting (timeout=60)... 2024-03-04T07:30:28.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:28.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.628+0000 7fbc3d088640 1 Processor -- start 2024-03-04T07:30:28.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.628+0000 7fbc3d088640 1 -- start start 2024-03-04T07:30:28.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc3d088640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbc38157f60 0x7fbc3815b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:28.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc3d088640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc38070b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:28.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc3d088640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbc3806c7b0 con 0x7fbc38157b80 2024-03-04T07:30:28.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc3d088640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbc3806ca90 con 0x7fbc38157f60 2024-03-04T07:30:28.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc3d088640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbc3806cd70 con 0x7fbc380cbab0 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc36d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbc38157b80 0x7fbc3815b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35298/0 (socket says 172.21.15.64:35298) 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc2ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbc38157f60 0x7fbc3815b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc36d9d640 1 -- 172.21.15.64:0/2901453767 learned_addr learned my addr 172.21.15.64:0/2901453767 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc3759e640 1 --2- 172.21.15.64:0/2901453767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc38070b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc2ffff640 1 -- 172.21.15.64:0/2901453767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 msgr2=0x7fbc38070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc2ffff640 1 --2- 172.21.15.64:0/2901453767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc38070b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc2ffff640 1 -- 172.21.15.64:0/2901453767 >> v1:172.21.15.64:6789/0 conn(0x7fbc38157b80 legacy=0x7fbc3815b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc3659c640 1 -- 172.21.15.64:0/2901453767 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1321671412 0 0) 0x7fbc3806c7b0 con 0x7fbc38157b80 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.629+0000 7fbc2ffff640 1 -- 172.21.15.64:0/2901453767 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc3806d050 con 0x7fbc38157f60 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.630+0000 7fbc2ffff640 1 --2- 172.21.15.64:0/2901453767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157f60 0x7fbc3815b260 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7fbc20009800 tx=0x7fbc2002dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=14c53cdeef2a4e2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3659c640 1 -- 172.21.15.64:0/2901453767 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc2002f8a0 con 0x7fbc38157f60 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3659c640 1 -- 172.21.15.64:0/2901453767 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc2002fa40 con 0x7fbc38157f60 2024-03-04T07:30:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3659c640 1 -- 172.21.15.64:0/2901453767 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc2002fe10 con 0x7fbc38157f60 2024-03-04T07:30:28.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 -- 172.21.15.64:0/2901453767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157f60 msgr2=0x7fbc3815b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:28.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 --2- 172.21.15.64:0/2901453767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157f60 0x7fbc3815b260 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7fbc20009800 tx=0x7fbc2002dae0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 -- 172.21.15.64:0/2901453767 shutdown_connections 2024-03-04T07:30:28.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 --2- 172.21.15.64:0/2901453767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc38070b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 --2- 172.21.15.64:0/2901453767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157f60 0x7fbc3815b260 unknown :-1 s=CLOSED pgs=872 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 -- 172.21.15.64:0/2901453767 >> 172.21.15.64:0/2901453767 conn(0x7fbc38066d00 msgr2=0x7fbc380670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:28.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 -- 172.21.15.64:0/2901453767 shutdown_connections 2024-03-04T07:30:28.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.631+0000 7fbc3d088640 1 -- 172.21.15.64:0/2901453767 wait complete. 2024-03-04T07:30:28.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.633+0000 7fbc3d088640 1 Processor -- start 2024-03-04T07:30:28.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.633+0000 7fbc3d088640 1 -- start start 2024-03-04T07:30:28.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.633+0000 7fbc3d088640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc380b46d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:28.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.633+0000 7fbc3d088640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157b80 0x7fbc380b3df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:28.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.633+0000 7fbc3d088640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc38157f60 0x7fbc380ce670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:28.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc3d088640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc38071080 con 0x7fbc38157b80 2024-03-04T07:30:28.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc3d088640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc380cc260 con 0x7fbc38157f60 2024-03-04T07:30:28.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc3d088640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbc38071200 con 0x7fbc380cbab0 2024-03-04T07:30:28.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157b80 0x7fbc380b3df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:28.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157b80 0x7fbc380b3df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41510/0 (socket says 172.21.15.64:41510) 2024-03-04T07:30:28.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 -- 172.21.15.64:0/417365722 learned_addr learned my addr 172.21.15.64:0/417365722 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:28.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc36d9d640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc380b46d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:28.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc3759e640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc38157f60 0x7fbc380ce670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:28.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 -- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 msgr2=0x7fbc380b46d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:28.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc380b46d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 -- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc38157f60 msgr2=0x7fbc380ce670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:28.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc38157f60 0x7fbc380ce670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 -- 172.21.15.64:0/417365722 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc38154b80 con 0x7fbc38157b80 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc36d9d640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc380b46d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc3759e640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc38157f60 0x7fbc380ce670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2ffff640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157b80 0x7fbc380b3df0 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7fbc200098f0 tx=0x7fbc20004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2f7fe640 1 -- 172.21.15.64:0/417365722 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc20048070 con 0x7fbc38157b80 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2f7fe640 1 -- 172.21.15.64:0/417365722 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc20043440 con 0x7fbc38157b80 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc38154e60 con 0x7fbc38157b80 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc2f7fe640 1 -- 172.21.15.64:0/417365722 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc20043830 con 0x7fbc38157b80 2024-03-04T07:30:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.634+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc38153b20 con 0x7fbc38157b80 2024-03-04T07:30:28.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.635+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc38067ff0 con 0x7fbc38157b80 2024-03-04T07:30:28.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.636+0000 7fbc2f7fe640 1 -- 172.21.15.64:0/417365722 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbc200423e0 con 0x7fbc38157b80 2024-03-04T07:30:28.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.636+0000 7fbc2f7fe640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:28.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.637+0000 7fbc36d9d640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:28.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.637+0000 7fbc2f7fe640 1 -- 172.21.15.64:0/417365722 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbc200adfa0 con 0x7fbc38157b80 2024-03-04T07:30:28.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.637+0000 7fbc36d9d640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7fbc28000f60 tx=0x7fbc28007c10 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:28.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.641+0000 7fbc2f7fe640 1 -- 172.21.15.64:0/417365722 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbc20035030 con 0x7fbc38157b80 2024-03-04T07:30:28.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.862+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbc380d07a0 con 0x7fbc38157b80 2024-03-04T07:30:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.862+0000 7fbc2f7fe640 1 -- 172.21.15.64:0/417365722 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7fbc2007ce20 con 0x7fbc38157b80 2024-03-04T07:30:28.862 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:28.862 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 msgr2=0x7fbc1806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7fbc28000f60 tx=0x7fbc28007c10 comp rx=0 tx=0).stop 2024-03-04T07:30:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157b80 msgr2=0x7fbc380b3df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157b80 0x7fbc380b3df0 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7fbc200098f0 tx=0x7fbc20004730 comp rx=0 tx=0).stop 2024-03-04T07:30:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 shutdown_connections 2024-03-04T07:30:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc38157f60 0x7fbc380ce670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc38157b80 0x7fbc380b3df0 unknown :-1 s=CLOSED pgs=873 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 --2- 172.21.15.64:0/417365722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc380cbab0 0x7fbc380b46d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 >> 172.21.15.64:0/417365722 conn(0x7fbc38066d00 msgr2=0x7fbc380cf830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 shutdown_connections 2024-03-04T07:30:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:28.864+0000 7fbc3d088640 1 -- 172.21.15.64:0/417365722 wait complete. 2024-03-04T07:30:28.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:30:28.991 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":10}} 2024-03-04T07:30:28.992 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 10 } } 2024-03-04T07:30:28.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:29.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.100+0000 7fd657064640 1 Processor -- start 2024-03-04T07:30:29.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.100+0000 7fd657064640 1 -- start start 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.100+0000 7fd657064640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd6500cfe40 0x7fd6500d0220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.100+0000 7fd657064640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd65006c040 con 0x7fd6500b48c0 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.100+0000 7fd657064640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd6500d0760 con 0x7fd6500ce3c0 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.100+0000 7fd657064640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd6500b33b0 con 0x7fd6500cfe40 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.101+0000 7fd654e00640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd6500cfe40 0x7fd6500d0220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.101+0000 7fd64ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd6500b48c0 0x7fd65006b930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35306/0 (socket says 172.21.15.64:35306) 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.101+0000 7fd64ffff640 1 -- 172.21.15.64:0/3927358551 learned_addr learned my addr 172.21.15.64:0/3927358551 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.101+0000 7fd654e00640 1 -- 172.21.15.64:0/3927358551 >> v1:172.21.15.66:6789/0 conn(0x7fd6500ce3c0 legacy=0x7fd6500ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.101+0000 7fd654e00640 1 -- 172.21.15.64:0/3927358551 >> v1:172.21.15.64:6789/0 conn(0x7fd6500b48c0 legacy=0x7fd65006b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.101+0000 7fd64f7fe640 1 -- 172.21.15.64:0/3927358551 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3240810798 0 0) 0x7fd65006c040 con 0x7fd6500b48c0 2024-03-04T07:30:29.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.101+0000 7fd654e00640 1 -- 172.21.15.64:0/3927358551 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6500cd320 con 0x7fd6500cfe40 2024-03-04T07:30:29.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.102+0000 7fd654e00640 1 --2- 172.21.15.64:0/3927358551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500cfe40 0x7fd6500d0220 secure :-1 s=READY pgs=875 cs=0 l=1 rev1=1 crypto rx=0x7fd6440097d0 tx=0x7fd64402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b9ec0252036b6d9a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:29.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.102+0000 7fd64f7fe640 1 -- 172.21.15.64:0/3927358551 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd644034070 con 0x7fd6500cfe40 2024-03-04T07:30:29.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.102+0000 7fd64f7fe640 1 -- 172.21.15.64:0/3927358551 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd644030610 con 0x7fd6500cfe40 2024-03-04T07:30:29.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.102+0000 7fd64f7fe640 1 -- 172.21.15.64:0/3927358551 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd6440309e0 con 0x7fd6500cfe40 2024-03-04T07:30:29.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.103+0000 7fd657064640 1 -- 172.21.15.64:0/3927358551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500cfe40 msgr2=0x7fd6500d0220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:29.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.103+0000 7fd657064640 1 --2- 172.21.15.64:0/3927358551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500cfe40 0x7fd6500d0220 secure :-1 s=READY pgs=875 cs=0 l=1 rev1=1 crypto rx=0x7fd6440097d0 tx=0x7fd64402b960 comp rx=0 tx=0).stop 2024-03-04T07:30:29.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.103+0000 7fd657064640 1 -- 172.21.15.64:0/3927358551 shutdown_connections 2024-03-04T07:30:29.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.103+0000 7fd657064640 1 --2- 172.21.15.64:0/3927358551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500cfe40 0x7fd6500d0220 unknown :-1 s=CLOSED pgs=875 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.103+0000 7fd657064640 1 -- 172.21.15.64:0/3927358551 >> 172.21.15.64:0/3927358551 conn(0x7fd6500d1a70 msgr2=0x7fd6500d1e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:29.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.103+0000 7fd657064640 1 -- 172.21.15.64:0/3927358551 shutdown_connections 2024-03-04T07:30:29.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.103+0000 7fd657064640 1 -- 172.21.15.64:0/3927358551 wait complete. 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.104+0000 7fd657064640 1 Processor -- start 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.104+0000 7fd657064640 1 -- start start 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd657064640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500b48c0 0x7fd6501f4600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd657064640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6500ce3c0 0x7fd6501f4d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd657064640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6500cfe40 0x7fd6500c73f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd657064640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6500cd8e0 con 0x7fd6500b48c0 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd657064640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6500cd0d0 con 0x7fd6500cfe40 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd657064640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd6500cdbc0 con 0x7fd6500ce3c0 2024-03-04T07:30:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500b48c0 0x7fd6501f4600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500b48c0 0x7fd6501f4600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41530/0 (socket says 172.21.15.64:41530) 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 -- 172.21.15.64:0/4229885661 learned_addr learned my addr 172.21.15.64:0/4229885661 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd64ffff640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6500ce3c0 0x7fd6501f4d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd655601640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6500cfe40 0x7fd6500c73f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 -- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6500ce3c0 msgr2=0x7fd6501f4d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6500ce3c0 0x7fd6501f4d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 -- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6500cfe40 msgr2=0x7fd6500c73f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6500cfe40 0x7fd6500c73f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.105+0000 7fd654e00640 1 -- 172.21.15.64:0/4229885661 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6501dea10 con 0x7fd6500b48c0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.106+0000 7fd654e00640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500b48c0 0x7fd6501f4600 secure :-1 s=READY pgs=876 cs=0 l=1 rev1=1 crypto rx=0x7fd6440097d0 tx=0x7fd6440046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.106+0000 7fd64d7fa640 1 -- 172.21.15.64:0/4229885661 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd644034040 con 0x7fd6500b48c0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.106+0000 7fd64d7fa640 1 -- 172.21.15.64:0/4229885661 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd644004030 con 0x7fd6500b48c0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.106+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd6501decb0 con 0x7fd6500b48c0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.106+0000 7fd64d7fa640 1 -- 172.21.15.64:0/4229885661 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd644004420 con 0x7fd6500b48c0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.106+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6501f5240 con 0x7fd6500b48c0 2024-03-04T07:30:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.107+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd6500c7ee0 con 0x7fd6500b48c0 2024-03-04T07:30:29.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.108+0000 7fd64d7fa640 1 -- 172.21.15.64:0/4229885661 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd644037070 con 0x7fd6500b48c0 2024-03-04T07:30:29.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.108+0000 7fd64d7fa640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd63406b4d0 0x7fd63406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:29.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.108+0000 7fd64d7fa640 1 -- 172.21.15.64:0/4229885661 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd6440abda0 con 0x7fd6500b48c0 2024-03-04T07:30:29.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.108+0000 7fd64ffff640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd63406b4d0 0x7fd63406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:29.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.109+0000 7fd64ffff640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd63406b4d0 0x7fd63406d990 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7fd6500ccfc0 tx=0x7fd63c007e40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:29.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.117+0000 7fd64d7fa640 1 -- 172.21.15.64:0/4229885661 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd6500c7ee0 con 0x7fd6500b48c0 2024-03-04T07:30:29.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.399+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd650067ff0 con 0x7fd6500b48c0 2024-03-04T07:30:29.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.399+0000 7fd64d7fa640 1 -- 172.21.15.64:0/4229885661 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7fd64407a5d0 con 0x7fd6500b48c0 2024-03-04T07:30:29.399 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:29.400 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:29.400 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:29.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd63406b4d0 msgr2=0x7fd63406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:29.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd63406b4d0 0x7fd63406d990 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7fd6500ccfc0 tx=0x7fd63c007e40 comp rx=0 tx=0).stop 2024-03-04T07:30:29.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500b48c0 msgr2=0x7fd6501f4600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500b48c0 0x7fd6501f4600 secure :-1 s=READY pgs=876 cs=0 l=1 rev1=1 crypto rx=0x7fd6440097d0 tx=0x7fd6440046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 shutdown_connections 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd63406b4d0 0x7fd63406d990 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd6500cfe40 0x7fd6500c73f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd6500ce3c0 0x7fd6501f4d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 --2- 172.21.15.64:0/4229885661 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd6500b48c0 0x7fd6501f4600 unknown :-1 s=CLOSED pgs=876 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 >> 172.21.15.64:0/4229885661 conn(0x7fd6500d1a70 msgr2=0x7fd65006f860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 shutdown_connections 2024-03-04T07:30:29.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29.401+0000 7fd657064640 1 -- 172.21.15.64:0/4229885661 wait complete. 2024-03-04T07:30:29.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.a.asok --format=json perf dump purge_queue pq_executed 2024-03-04T07:30:29.531 INFO:teuthology.orchestra.run.smithi064.stdout:{"purge_queue":{"pq_executed":10}} 2024-03-04T07:30:29.531 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 10 } } 2024-03-04T07:30:29.532 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:29.532 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:29.546 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> import uuid 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> n = 10 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = False 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> if 0 > 0: 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> create_hard_links = True 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> path = "/home/ubuntu/cephtest/mnt.0/subdir/file" 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> dpath = os.path.dirname(path) 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> fnameprefix = os.path.basename(path) 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> os.makedirs(dpath, exist_ok=True) 2024-03-04T07:30:29.547 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> try: 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> dirfd = os.open(dpath, os.O_DIRECTORY) 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> for i in range(n): 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> fpath = os.path.join(dpath, f"{fnameprefix}_{i}") 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> with open(fpath, '"'"'w'"'"') as f: 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> f.write(f"{i}") 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> f.flush() 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(f.fileno()) 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> if True: 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> os.unlink(fpath) 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> if True: 2024-03-04T07:30:29.548 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> if create_hard_links: 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> for j in range(0): 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> os.system(f"ln {fpath} {dpath}/{fnameprefix}_{i}_{uuid.uuid4()}") 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> if False: 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> os.fsync(dirfd) 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> finally: 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> os.close(dirfd) 2024-03-04T07:30:29.549 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:30:29.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:29 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:30:29.733 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:30:29.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:30:29.813 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:30:29.866 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:30:29.867 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:29.867 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:29.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:30.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.031+0000 7f387a484640 1 Processor -- start 2024-03-04T07:30:30.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.031+0000 7f387a484640 1 -- start start 2024-03-04T07:30:30.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f387a484640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f387415df00 0x7f38740d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f387a484640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740b9a30 0x7f38740d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f387a484640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f38740cea30 con 0x7f38740b95a0 2024-03-04T07:30:30.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f387a484640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38740d2af0 con 0x7f38740b9a30 2024-03-04T07:30:30.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f387a484640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38740cf3c0 con 0x7f387415df00 2024-03-04T07:30:30.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3878a21640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740b9a30 0x7f38740d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f38737fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f38740b95a0 0x7f38740b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35310/0 (socket says 172.21.15.64:35310) 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f38737fe640 1 -- 172.21.15.64:0/3577252159 learned_addr learned my addr 172.21.15.64:0/3577252159 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3873fff640 1 --2- 172.21.15.64:0/3577252159 >> v2:172.21.15.66:3300/0 conn(0x7f387415df00 0x7f38740d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3872ffd640 1 -- 172.21.15.64:0/3577252159 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 71632023 0 0) 0x7f38740cea30 con 0x7f38740b95a0 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3872ffd640 1 -- 172.21.15.64:0/3577252159 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f385c003520 con 0x7f38740b95a0 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3878a21640 1 -- 172.21.15.64:0/3577252159 >> v2:172.21.15.66:3300/0 conn(0x7f387415df00 msgr2=0x7f38740d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3878a21640 1 --2- 172.21.15.64:0/3577252159 >> v2:172.21.15.66:3300/0 conn(0x7f387415df00 0x7f38740d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3878a21640 1 -- 172.21.15.64:0/3577252159 >> v1:172.21.15.64:6789/0 conn(0x7f38740b95a0 legacy=0x7f38740b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.032+0000 7f3878a21640 1 -- 172.21.15.64:0/3577252159 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38740cf0e0 con 0x7f38740b9a30 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.033+0000 7f3878a21640 1 --2- 172.21.15.64:0/3577252159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b9a30 0x7f38740d3220 secure :-1 s=READY pgs=878 cs=0 l=1 rev1=1 crypto rx=0x7f38740d11b0 tx=0x7f386c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=281fb30ad1c8ccfd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.033+0000 7f3872ffd640 1 -- 172.21.15.64:0/3577252159 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c039070 con 0x7f38740b9a30 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.033+0000 7f3872ffd640 1 -- 172.21.15.64:0/3577252159 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f386c02cc80 con 0x7f38740b9a30 2024-03-04T07:30:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.033+0000 7f3872ffd640 1 -- 172.21.15.64:0/3577252159 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c034660 con 0x7f38740b9a30 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 -- 172.21.15.64:0/3577252159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b9a30 msgr2=0x7f38740d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 --2- 172.21.15.64:0/3577252159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b9a30 0x7f38740d3220 secure :-1 s=READY pgs=878 cs=0 l=1 rev1=1 crypto rx=0x7f38740d11b0 tx=0x7f386c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 -- 172.21.15.64:0/3577252159 shutdown_connections 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 --2- 172.21.15.64:0/3577252159 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b9a30 0x7f38740d3220 unknown :-1 s=CLOSED pgs=878 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 --2- 172.21.15.64:0/3577252159 >> v2:172.21.15.66:3300/0 conn(0x7f387415df00 0x7f38740d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 -- 172.21.15.64:0/3577252159 >> 172.21.15.64:0/3577252159 conn(0x7f3874071960 msgr2=0x7f3874071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 -- 172.21.15.64:0/3577252159 shutdown_connections 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.034+0000 7f387a484640 1 -- 172.21.15.64:0/3577252159 wait complete. 2024-03-04T07:30:30.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.035+0000 7f387a484640 1 Processor -- start 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.035+0000 7f387a484640 1 -- start start 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f387a484640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b95a0 0x7f38741608b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f387a484640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415df00 0x7f38740cd450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f387a484640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740c9ce0 0x7f38741f7460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f387a484640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38740cec70 con 0x7f38740b95a0 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f387a484640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38740ce750 con 0x7f38740c9ce0 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f387a484640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38740cedf0 con 0x7f387415df00 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f3873fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b95a0 0x7f38741608b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f3873fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b95a0 0x7f38741608b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41560/0 (socket says 172.21.15.64:41560) 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.036+0000 7f3873fff640 1 -- 172.21.15.64:0/3951355737 learned_addr learned my addr 172.21.15.64:0/3951355737 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3873fff640 1 -- 172.21.15.64:0/3951355737 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415df00 msgr2=0x7f38740cd450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3873fff640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415df00 0x7f38740cd450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3873fff640 1 -- 172.21.15.64:0/3951355737 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740c9ce0 msgr2=0x7f38741f7460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3873fff640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740c9ce0 0x7f38741f7460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3873fff640 1 -- 172.21.15.64:0/3951355737 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38740cb800 con 0x7f38740b95a0 2024-03-04T07:30:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3873fff640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b95a0 0x7f38741608b0 secure :-1 s=READY pgs=879 cs=0 l=1 rev1=1 crypto rx=0x7f386400d5f0 tx=0x7f386400dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3870ff9640 1 -- 172.21.15.64:0/3951355737 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38640125f0 con 0x7f38740b95a0 2024-03-04T07:30:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3870ff9640 1 -- 172.21.15.64:0/3951355737 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3864019070 con 0x7f38740b95a0 2024-03-04T07:30:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f38740cbaa0 con 0x7f38740b95a0 2024-03-04T07:30:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f3870ff9640 1 -- 172.21.15.64:0/3951355737 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38640129d0 con 0x7f38740b95a0 2024-03-04T07:30:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.037+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f38740cc340 con 0x7f38740b95a0 2024-03-04T07:30:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.038+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f38740d01a0 con 0x7f38740b95a0 2024-03-04T07:30:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.038+0000 7f3870ff9640 1 -- 172.21.15.64:0/3951355737 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3864012b70 con 0x7f38740b95a0 2024-03-04T07:30:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.039+0000 7f3870ff9640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.039+0000 7f3870ff9640 1 -- 172.21.15.64:0/3951355737 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f386408a5a0 con 0x7f38740b95a0 2024-03-04T07:30:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.039+0000 7f38737fe640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.040+0000 7f38737fe640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f3868000f60 tx=0x7f38680078f0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.047+0000 7f3870ff9640 1 -- 172.21.15.64:0/3951355737 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f38740d01a0 con 0x7f38740b95a0 2024-03-04T07:30:30.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.232+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f38740d1400 con 0x7f38740b95a0 2024-03-04T07:30:30.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.233+0000 7f3870ff9640 1 -- 172.21.15.64:0/3951355737 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v66) v1 ==== 74+0+16620 (secure 0 0 0) 0x7f38740d1400 con 0x7f38740b95a0 2024-03-04T07:30:30.233 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:30.233 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":66,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:06.066460+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:02.706399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:03.395544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 msgr2=0x7f386006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f3868000f60 tx=0x7f38680078f0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b95a0 msgr2=0x7f38741608b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b95a0 0x7f38741608b0 secure :-1 s=READY pgs=879 cs=0 l=1 rev1=1 crypto rx=0x7f386400d5f0 tx=0x7f386400dab0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 shutdown_connections 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740c9ce0 0x7f38741f7460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f387415df00 0x7f38740cd450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 --2- 172.21.15.64:0/3951355737 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740b95a0 0x7f38741608b0 unknown :-1 s=CLOSED pgs=879 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 >> 172.21.15.64:0/3951355737 conn(0x7f3874071960 msgr2=0x7f38740b8370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:30.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 shutdown_connections 2024-03-04T07:30:30.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.235+0000 7f387a484640 1 -- 172.21.15.64:0/3951355737 wait complete. 2024-03-04T07:30:30.246 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 10, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:30:02.706399+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '65', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 11, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:30:03.395544+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '66', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:30:30.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:30:30.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.365+0000 7f2218fa2640 1 Processor -- start 2024-03-04T07:30:30.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.365+0000 7f2218fa2640 1 -- start start 2024-03-04T07:30:30.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.366+0000 7f2218fa2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f221415bb90 0x7f22140cd230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.366+0000 7f2218fa2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2214071bb0 con 0x7f2214067410 2024-03-04T07:30:30.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.366+0000 7f2218fa2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2214072a20 con 0x7f22140afbc0 2024-03-04T07:30:30.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.366+0000 7f2218fa2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2214072460 con 0x7f221415bb90 2024-03-04T07:30:30.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.366+0000 7f2213fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2214067410 0x7f22140cead0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35322/0 (socket says 172.21.15.64:35322) 2024-03-04T07:30:30.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.366+0000 7f2213fff640 1 -- 172.21.15.64:0/1969456083 learned_addr learned my addr 172.21.15.64:0/1969456083 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.366+0000 7f22137fe640 1 --2- 172.21.15.64:0/1969456083 >> v2:172.21.15.66:3300/0 conn(0x7f221415bb90 0x7f22140cd230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4185658411 0 0) 0x7f2214071bb0 con 0x7f2214067410 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f21fc003520 con 0x7f2214067410 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 491343524 0 0) 0x7f21fc003520 con 0x7f2214067410 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 >> v2:172.21.15.66:3300/0 conn(0x7f221415bb90 msgr2=0x7f22140cd230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 --2- 172.21.15.64:0/1969456083 >> v2:172.21.15.66:3300/0 conn(0x7f221415bb90 0x7f22140cd230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 >> v1:172.21.15.66:6789/0 conn(0x7f22140afbc0 legacy=0x7f2214066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2214072740 con 0x7f2214067410 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f220c0019f0 con 0x7f2214067410 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 >> v1:172.21.15.64:6789/0 conn(0x7f2214067410 legacy=0x7f22140cead0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21fc0039b0 0x7f21fc003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc004800 0x7f21fc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21fc0051c0 0x7f21fc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2214072460 con 0x7f21fc0039b0 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21fc00c160 con 0x7f21fc0051c0 2024-03-04T07:30:30.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21fc00c360 con 0x7f21fc004800 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22137fe640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21fc0039b0 0x7f21fc003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f2213fff640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21fc0051c0 0x7f21fc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f2212ffd640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc004800 0x7f21fc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22137fe640 1 -- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc004800 msgr2=0x7f21fc004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22137fe640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc004800 0x7f21fc004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22137fe640 1 -- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21fc0051c0 msgr2=0x7f21fc00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22137fe640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21fc0051c0 0x7f21fc00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f22137fe640 1 -- 172.21.15.64:0/1969456083 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21fc003520 con 0x7f21fc0039b0 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f2213fff640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21fc0051c0 0x7f21fc00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f2212ffd640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc004800 0x7f21fc004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.367+0000 7f2213fff640 1 -- 172.21.15.64:0/1969456083 reap_dead start 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f22137fe640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21fc0039b0 0x7f21fc003dd0 secure :-1 s=READY pgs=881 cs=0 l=1 rev1=1 crypto rx=0x7f2208009ac0 tx=0x7f21fc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6cfb258684ce6469 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2208039070 con 0x7f21fc0039b0 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f220802c6b0 con 0x7f21fc0039b0 2024-03-04T07:30:30.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f22127fc640 1 -- 172.21.15.64:0/1969456083 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2208009690 con 0x7f21fc0039b0 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f2218fa2640 1 -- 172.21.15.64:0/1969456083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21fc0039b0 msgr2=0x7f21fc003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21fc0039b0 0x7f21fc003dd0 secure :-1 s=READY pgs=881 cs=0 l=1 rev1=1 crypto rx=0x7f2208009ac0 tx=0x7f21fc005830 comp rx=0 tx=0).stop 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f2218fa2640 1 -- 172.21.15.64:0/1969456083 shutdown_connections 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1969456083 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21fc0039b0 0x7f21fc003dd0 unknown :-1 s=CLOSED pgs=881 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f2218fa2640 1 -- 172.21.15.64:0/1969456083 >> 172.21.15.64:0/1969456083 conn(0x7f22140b2560 msgr2=0x7f22140b2940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f2218fa2640 1 -- 172.21.15.64:0/1969456083 shutdown_connections 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.369+0000 7f2218fa2640 1 -- 172.21.15.64:0/1969456083 wait complete. 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.370+0000 7f2218fa2640 1 Processor -- start 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2218fa2640 1 -- start start 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2218fa2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc0039b0 0x7f22140b44d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2218fa2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22140b4a10 0x7f2214159000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2218fa2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f221415bf70 0x7f2214159540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2218fa2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22140cd230 con 0x7f221415bf70 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2218fa2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f221406b330 con 0x7f22140b4a10 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2218fa2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f22140cd3b0 con 0x7f21fc0039b0 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2213fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f221415bf70 0x7f2214159540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2213fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f221415bf70 0x7f2214159540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41576/0 (socket says 172.21.15.64:41576) 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2213fff640 1 -- 172.21.15.64:0/1835982226 learned_addr learned my addr 172.21.15.64:0/1835982226 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.371+0000 7f2212ffd640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22140b4a10 0x7f2214159000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f22137fe640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc0039b0 0x7f22140b44d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2213fff640 1 -- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc0039b0 msgr2=0x7f22140b44d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2213fff640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc0039b0 0x7f22140b44d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2213fff640 1 -- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22140b4a10 msgr2=0x7f2214159000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2213fff640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22140b4a10 0x7f2214159000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2213fff640 1 -- 172.21.15.64:0/1835982226 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f221415cd80 con 0x7f221415bf70 2024-03-04T07:30:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2213fff640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f221415bf70 0x7f2214159540 secure :-1 s=READY pgs=882 cs=0 l=1 rev1=1 crypto rx=0x7f220c007960 tx=0x7f220c00e530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f21fbfff640 1 -- 172.21.15.64:0/1835982226 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f220c019070 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f21fbfff640 1 -- 172.21.15.64:0/1835982226 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f220c00eb60 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f221415c4e0 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f21fbfff640 1 -- 172.21.15.64:0/1835982226 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f220c014480 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.372+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f221415caa0 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.373+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f221404f330 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.374+0000 7f21fbfff640 1 -- 172.21.15.64:0/1835982226 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f220c01c070 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.374+0000 7f21fbfff640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f220006b500 0x7f220006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.374+0000 7f21fbfff640 1 -- 172.21.15.64:0/1835982226 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f220c08a0e0 con 0x7f221415bf70 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.374+0000 7f22137fe640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f220006b500 0x7f220006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.374+0000 7f22137fe640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f220006b500 0x7f220006d9c0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f2208036000 tx=0x7f2208030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.378+0000 7f21fbfff640 1 -- 172.21.15.64:0/1835982226 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f220c058910 con 0x7f221415bf70 2024-03-04T07:30:30.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.569+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f22140b18f0 con 0x7f221415bf70 2024-03-04T07:30:30.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.569+0000 7f21fbfff640 1 -- 172.21.15.64:0/1835982226 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v66) v1 ==== 93+0+10 (secure 0 0 0) 0x7f220c05c9b0 con 0x7f221415bf70 2024-03-04T07:30:30.569 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.571+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f220006b500 msgr2=0x7f220006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f220006b500 0x7f220006d9c0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f2208036000 tx=0x7f2208030000 comp rx=0 tx=0).stop 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f221415bf70 msgr2=0x7f2214159540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f221415bf70 0x7f2214159540 secure :-1 s=READY pgs=882 cs=0 l=1 rev1=1 crypto rx=0x7f220c007960 tx=0x7f220c00e530 comp rx=0 tx=0).stop 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 shutdown_connections 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f220006b500 0x7f220006d9c0 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f221415bf70 0x7f2214159540 unknown :-1 s=CLOSED pgs=882 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22140b4a10 0x7f2214159000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 --2- 172.21.15.64:0/1835982226 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21fc0039b0 0x7f22140b44d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 >> 172.21.15.64:0/1835982226 conn(0x7f22140b2560 msgr2=0x7f2214066d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 shutdown_connections 2024-03-04T07:30:30.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.572+0000 7f2218fa2640 1 -- 172.21.15.64:0/1835982226 wait complete. 2024-03-04T07:30:30.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:30:30.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.689+0000 7f4eb5f8a640 1 Processor -- start 2024-03-04T07:30:30.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eb5f8a640 1 -- start start 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eb5f8a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4eb0072a30 0x7f4eb01624c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eb5f8a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4eb00cf9f0 0x7f4eb00d00b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eb5f8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb015d660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eb5f8a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4eb006dba0 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eb5f8a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4eb00d0a60 con 0x7f4eb0072a30 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eb5f8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4eb006d4d0 con 0x7f4eb00d05f0 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.690+0000 7f4eaeffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4eb00cf9f0 0x7f4eb00d00b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaeffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4eb00cf9f0 0x7f4eb00d00b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41578/0 (socket says 172.21.15.64:41578) 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaeffd640 1 -- 172.21.15.64:0/1871547550 learned_addr learned my addr 172.21.15.64:0/1871547550 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:30.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaf7fe640 1 --2- 172.21.15.64:0/1871547550 >> v2:172.21.15.66:3300/0 conn(0x7f4eb0072a30 0x7f4eb01624c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaffff640 1 --2- 172.21.15.64:0/1871547550 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb015d660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaeffd640 1 -- 172.21.15.64:0/1871547550 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 msgr2=0x7f4eb015d660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:30.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaeffd640 1 --2- 172.21.15.64:0/1871547550 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb015d660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaeffd640 1 -- 172.21.15.64:0/1871547550 >> v2:172.21.15.66:3300/0 conn(0x7f4eb0072a30 msgr2=0x7f4eb01624c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:30.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaeffd640 1 --2- 172.21.15.64:0/1871547550 >> v2:172.21.15.66:3300/0 conn(0x7f4eb0072a30 0x7f4eb01624c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaeffd640 1 -- 172.21.15.64:0/1871547550 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4eb006d7b0 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.691+0000 7f4eaf7fe640 1 --2- 172.21.15.64:0/1871547550 >> v2:172.21.15.66:3300/0 conn(0x7f4eb0072a30 0x7f4eb01624c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.692+0000 7f4eaeffd640 1 --2- 172.21.15.64:0/1871547550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00d00b0 secure :-1 s=READY pgs=883 cs=0 l=1 rev1=1 crypto rx=0x7f4eb0074550 tx=0x7f4ea402b7f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=863547f78f3cc940 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.692+0000 7f4eae7fc640 1 -- 172.21.15.64:0/1871547550 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ea4038070 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.692+0000 7f4eae7fc640 1 -- 172.21.15.64:0/1871547550 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ea402f050 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.692+0000 7f4eae7fc640 1 -- 172.21.15.64:0/1871547550 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ea4031970 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/1871547550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 msgr2=0x7f4eb00d00b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/1871547550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00d00b0 secure :-1 s=READY pgs=883 cs=0 l=1 rev1=1 crypto rx=0x7f4eb0074550 tx=0x7f4ea402b7f0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/1871547550 shutdown_connections 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/1871547550 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb015d660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/1871547550 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00d00b0 unknown :-1 s=CLOSED pgs=883 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/1871547550 >> v2:172.21.15.66:3300/0 conn(0x7f4eb0072a30 0x7f4eb01624c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/1871547550 >> 172.21.15.64:0/1871547550 conn(0x7f4eb006af20 msgr2=0x7f4eb006b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:30.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/1871547550 shutdown_connections 2024-03-04T07:30:30.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.693+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/1871547550 wait complete. 2024-03-04T07:30:30.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.695+0000 7f4eb5f8a640 1 Processor -- start 2024-03-04T07:30:30.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.695+0000 7f4eb5f8a640 1 -- start start 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eb5f8a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4eb0072a30 0x7f4eb00cba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eb5f8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00cbf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eb5f8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb00acfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eb5f8a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4eb0072e10 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eb5f8a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4eb0071130 con 0x7f4eb0072a30 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eb5f8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4eb0072f90 con 0x7f4eb00d05f0 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eaeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00cbf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eaeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00cbf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41580/0 (socket says 172.21.15.64:41580) 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eaeffd640 1 -- 172.21.15.64:0/2793257612 learned_addr learned my addr 172.21.15.64:0/2793257612 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eaf7fe640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4eb0072a30 0x7f4eb00cba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.696+0000 7f4eaffff640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb00acfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eaeffd640 1 -- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 msgr2=0x7f4eb00acfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eaeffd640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb00acfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eaeffd640 1 -- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4eb0072a30 msgr2=0x7f4eb00cba40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eaeffd640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4eb0072a30 0x7f4eb00cba40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eaeffd640 1 -- 172.21.15.64:0/2793257612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4eb0150e30 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eaeffd640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00cbf80 secure :-1 s=READY pgs=884 cs=0 l=1 rev1=1 crypto rx=0x7f4ea4000c00 tx=0x7f4ea40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4e97fff640 1 -- 172.21.15.64:0/2793257612 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ea4042070 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4e97fff640 1 -- 172.21.15.64:0/2793257612 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ea403d680 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4eb0151110 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4e97fff640 1 -- 172.21.15.64:0/2793257612 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ea4038040 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.697+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4eb01516d0 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.698+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4eb006b300 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.699+0000 7f4e97fff640 1 -- 172.21.15.64:0/2793257612 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4ea4045070 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.699+0000 7f4e97fff640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e9c06b4d0 0x7f4e9c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:30.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.699+0000 7f4eaf7fe640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e9c06b4d0 0x7f4e9c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:30.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.699+0000 7f4e97fff640 1 -- 172.21.15.64:0/2793257612 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4ea40abbd0 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.700+0000 7f4eaf7fe640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e9c06b4d0 0x7f4e9c06d990 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f4ea0009bd0 tx=0x7f4ea000b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:30.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.703+0000 7f4e97fff640 1 -- 172.21.15.64:0/2793257612 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4eb006b300 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.888+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f4eb009e320 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.888+0000 7f4e97fff640 1 -- 172.21.15.64:0/2793257612 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v66) v1 ==== 104+0+11 (secure 0 0 0) 0x7f4ea407a490 con 0x7f4eb00cf9f0 2024-03-04T07:30:30.889 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:30:30.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e9c06b4d0 msgr2=0x7f4e9c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e9c06b4d0 0x7f4e9c06d990 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f4ea0009bd0 tx=0x7f4ea000b040 comp rx=0 tx=0).stop 2024-03-04T07:30:30.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 msgr2=0x7f4eb00cbf80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00cbf80 secure :-1 s=READY pgs=884 cs=0 l=1 rev1=1 crypto rx=0x7f4ea4000c00 tx=0x7f4ea40046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 shutdown_connections 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e9c06b4d0 0x7f4e9c06d990 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4eb00d05f0 0x7f4eb00acfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4eb00cf9f0 0x7f4eb00cbf80 unknown :-1 s=CLOSED pgs=884 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 --2- 172.21.15.64:0/2793257612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4eb0072a30 0x7f4eb00cba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.890+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 >> 172.21.15.64:0/2793257612 conn(0x7f4eb006af20 msgr2=0x7f4eb00b28b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.891+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 shutdown_connections 2024-03-04T07:30:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:30.891+0000 7f4eb5f8a640 1 -- 172.21.15.64:0/2793257612 wait complete. 2024-03-04T07:30:30.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:30:31.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.005+0000 7f8d8e75e640 1 Processor -- start 2024-03-04T07:30:31.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.005+0000 7f8d8e75e640 1 -- start start 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d8e75e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8d8815dd90 0x7f8d880ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d8e75e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8d880b6d10 0x7f8d880b7150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d8e75e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8d880d3740 con 0x7f8d880d29d0 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d8e75e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8d880d3400 con 0x7f8d880b6d10 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d8e75e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8d880b7690 con 0x7f8d8815dd90 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d8ccfb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8d880b6d10 0x7f8d880b7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d877fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8d880d29d0 0x7f8d880d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35332/0 (socket says 172.21.15.64:35332) 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d877fe640 1 -- 172.21.15.64:0/855083565 learned_addr learned my addr 172.21.15.64:0/855083565 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d86ffd640 1 -- 172.21.15.64:0/855083565 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1768976871 0 0) 0x7f8d880d3740 con 0x7f8d880d29d0 2024-03-04T07:30:31.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d87fff640 1 --2- 172.21.15.64:0/855083565 >> v2:172.21.15.66:3300/0 conn(0x7f8d8815dd90 0x7f8d880ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d86ffd640 1 -- 172.21.15.64:0/855083565 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8d70003520 con 0x7f8d880d29d0 2024-03-04T07:30:31.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.006+0000 7f8d8ccfb640 1 -- 172.21.15.64:0/855083565 >> v2:172.21.15.66:3300/0 conn(0x7f8d8815dd90 msgr2=0x7f8d880ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:31.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.007+0000 7f8d8ccfb640 1 --2- 172.21.15.64:0/855083565 >> v2:172.21.15.66:3300/0 conn(0x7f8d8815dd90 0x7f8d880ceea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.007+0000 7f8d8ccfb640 1 -- 172.21.15.64:0/855083565 >> v1:172.21.15.64:6789/0 conn(0x7f8d880d29d0 legacy=0x7f8d880d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.007+0000 7f8d8ccfb640 1 -- 172.21.15.64:0/855083565 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d880d3120 con 0x7f8d880b6d10 2024-03-04T07:30:31.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8ccfb640 1 --2- 172.21.15.64:0/855083565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d880b6d10 0x7f8d880b7150 secure :-1 s=READY pgs=886 cs=0 l=1 rev1=1 crypto rx=0x7f8d880d1f90 tx=0x7f8d8002bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=88a714f40007423f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:31.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d86ffd640 1 -- 172.21.15.64:0/855083565 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d80039070 con 0x7f8d880b6d10 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d86ffd640 1 -- 172.21.15.64:0/855083565 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8d8002cd10 con 0x7f8d880b6d10 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d86ffd640 1 -- 172.21.15.64:0/855083565 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d80034660 con 0x7f8d880b6d10 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 -- 172.21.15.64:0/855083565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d880b6d10 msgr2=0x7f8d880b7150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/855083565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d880b6d10 0x7f8d880b7150 secure :-1 s=READY pgs=886 cs=0 l=1 rev1=1 crypto rx=0x7f8d880d1f90 tx=0x7f8d8002bf60 comp rx=0 tx=0).stop 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 -- 172.21.15.64:0/855083565 shutdown_connections 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/855083565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d880b6d10 0x7f8d880b7150 unknown :-1 s=CLOSED pgs=886 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/855083565 >> v2:172.21.15.66:3300/0 conn(0x7f8d8815dd90 0x7f8d880ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 -- 172.21.15.64:0/855083565 >> 172.21.15.64:0/855083565 conn(0x7f8d88071960 msgr2=0x7f8d88071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 -- 172.21.15.64:0/855083565 shutdown_connections 2024-03-04T07:30:31.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.008+0000 7f8d8e75e640 1 -- 172.21.15.64:0/855083565 wait complete. 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.009+0000 7f8d8e75e640 1 Processor -- start 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.010+0000 7f8d8e75e640 1 -- start start 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.010+0000 7f8d8e75e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8d880b6d10 0x7f8d880cb480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.010+0000 7f8d8e75e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8d880d29d0 0x7f8d88160e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.010+0000 7f8d8e75e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d8815dd90 0x7f8d880cbdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.010+0000 7f8d8e75e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d8815e170 con 0x7f8d8815dd90 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.010+0000 7f8d8e75e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d880d3cc0 con 0x7f8d880b6d10 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.010+0000 7f8d8e75e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8d880d2df0 con 0x7f8d880d29d0 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d8815dd90 0x7f8d880cbdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d8815dd90 0x7f8d880cbdb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41604/0 (socket says 172.21.15.64:41604) 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 -- 172.21.15.64:0/170616325 learned_addr learned my addr 172.21.15.64:0/170616325 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d87fff640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8d880b6d10 0x7f8d880cb480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d877fe640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8d880d29d0 0x7f8d88160e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 -- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8d880d29d0 msgr2=0x7f8d88160e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8d880d29d0 0x7f8d88160e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 -- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8d880b6d10 msgr2=0x7f8d880cb480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8d880b6d10 0x7f8d880cb480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 -- 172.21.15.64:0/170616325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d880c8930 con 0x7f8d8815dd90 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d87fff640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8d880b6d10 0x7f8d880cb480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8ccfb640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d8815dd90 0x7f8d880cbdb0 secure :-1 s=READY pgs=887 cs=0 l=1 rev1=1 crypto rx=0x7f8d80009e80 tx=0x7f8d80004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d84ff9640 1 -- 172.21.15.64:0/170616325 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d80039050 con 0x7f8d8815dd90 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d84ff9640 1 -- 172.21.15.64:0/170616325 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8d80045070 con 0x7f8d8815dd90 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8d880c8c10 con 0x7f8d8815dd90 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d84ff9640 1 -- 172.21.15.64:0/170616325 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d80009450 con 0x7f8d8815dd90 2024-03-04T07:30:31.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.011+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8d880c91d0 con 0x7f8d8815dd90 2024-03-04T07:30:31.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.012+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8d88072a30 con 0x7f8d8815dd90 2024-03-04T07:30:31.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.013+0000 7f8d84ff9640 1 -- 172.21.15.64:0/170616325 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8d8003e850 con 0x7f8d8815dd90 2024-03-04T07:30:31.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.013+0000 7f8d84ff9640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8d7406b500 0x7f8d7406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.013+0000 7f8d87fff640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8d7406b500 0x7f8d7406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.013+0000 7f8d84ff9640 1 -- 172.21.15.64:0/170616325 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8d800ac850 con 0x7f8d8815dd90 2024-03-04T07:30:31.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.014+0000 7f8d87fff640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8d7406b500 0x7f8d7406d9c0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f8d7c006fd0 tx=0x7f8d7c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:31.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.018+0000 7f8d84ff9640 1 -- 172.21.15.64:0/170616325 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8d8004e020 con 0x7f8d8815dd90 2024-03-04T07:30:31.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.200+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f8d880b2750 con 0x7f8d8815dd90 2024-03-04T07:30:31.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.200+0000 7f8d84ff9640 1 -- 172.21.15.64:0/170616325 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v66) v1 ==== 100+0+11 (secure 0 0 0) 0x7f8d8007b760 con 0x7f8d8815dd90 2024-03-04T07:30:31.200 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:30:31.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8d7406b500 msgr2=0x7f8d7406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8d7406b500 0x7f8d7406d9c0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f8d7c006fd0 tx=0x7f8d7c008040 comp rx=0 tx=0).stop 2024-03-04T07:30:31.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d8815dd90 msgr2=0x7f8d880cbdb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d8815dd90 0x7f8d880cbdb0 secure :-1 s=READY pgs=887 cs=0 l=1 rev1=1 crypto rx=0x7f8d80009e80 tx=0x7f8d80004a40 comp rx=0 tx=0).stop 2024-03-04T07:30:31.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 shutdown_connections 2024-03-04T07:30:31.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8d7406b500 0x7f8d7406d9c0 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8d8815dd90 0x7f8d880cbdb0 unknown :-1 s=CLOSED pgs=887 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8d880d29d0 0x7f8d88160e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 --2- 172.21.15.64:0/170616325 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8d880b6d10 0x7f8d880cb480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 >> 172.21.15.64:0/170616325 conn(0x7f8d88071960 msgr2=0x7f8d880b36c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:31.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 shutdown_connections 2024-03-04T07:30:31.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.202+0000 7f8d8e75e640 1 -- 172.21.15.64:0/170616325 wait complete. 2024-03-04T07:30:31.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:30:31.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.316+0000 7fb43a37a640 1 Processor -- start 2024-03-04T07:30:31.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.316+0000 7fb43a37a640 1 -- start start 2024-03-04T07:30:31.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb43a37a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 0x7fb4340d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb43a37a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb4340b9a30 0x7fb4340d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb43a37a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb4340cea30 con 0x7fb4340b95a0 2024-03-04T07:30:31.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb43a37a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4340d2af0 con 0x7fb4340b9a30 2024-03-04T07:30:31.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb43a37a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4340cf3c0 con 0x7fb43415df00 2024-03-04T07:30:31.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb4337fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb4340b95a0 0x7fb4340b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35334/0 (socket says 172.21.15.64:35334) 2024-03-04T07:30:31.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb4337fe640 1 -- 172.21.15.64:0/3366116323 learned_addr learned my addr 172.21.15.64:0/3366116323 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb438917640 1 --2- 172.21.15.64:0/3366116323 >> v2:172.21.15.66:3300/0 conn(0x7fb4340b9a30 0x7fb4340d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb433fff640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 0x7fb4340d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2100043474 0 0) 0x7fb4340cea30 con 0x7fb4340b95a0 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb41c003520 con 0x7fb4340b95a0 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3184433516 0 0) 0x7fb41c003520 con 0x7fb4340b95a0 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 msgr2=0x7fb4340d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 0x7fb4340d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 >> v2:172.21.15.66:3300/0 conn(0x7fb4340b9a30 msgr2=0x7fb4340d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 --2- 172.21.15.64:0/3366116323 >> v2:172.21.15.66:3300/0 conn(0x7fb4340b9a30 0x7fb4340d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4340cf0e0 con 0x7fb4340b95a0 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fb428001670 con 0x7fb4340b95a0 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 >> v1:172.21.15.64:6789/0 conn(0x7fb4340b95a0 legacy=0x7fb4340b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb41c0039b0 0x7fb41c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 0x7fb41c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb41c004d40 0x7fb41c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4340d2af0 con 0x7fb41c004d40 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4340cf3c0 con 0x7fb41c0039b0 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb41c00b9c0 con 0x7fb43415df00 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.318+0000 7fb438917640 1 --2- 172.21.15.64:0/3366116323 >> v2:172.21.15.66:3300/0 conn(0x7fb4340b9a30 0x7fb4340d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb438917640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb41c004d40 0x7fb41c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb433fff640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb41c0039b0 0x7fb41c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb4337fe640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 0x7fb41c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb438917640 1 -- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 msgr2=0x7fb41c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb438917640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb43415df00 0x7fb41c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb438917640 1 -- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb41c0039b0 msgr2=0x7fb41c003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb438917640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb41c0039b0 0x7fb41c003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb438917640 1 -- 172.21.15.64:0/3366116323 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb41c003520 con 0x7fb41c004d40 2024-03-04T07:30:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.319+0000 7fb433fff640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb41c0039b0 0x7fb41c003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.320+0000 7fb438917640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb41c004d40 0x7fb41c0051c0 secure :-1 s=READY pgs=889 cs=0 l=1 rev1=1 crypto rx=0x7fb42c00e320 tx=0x7fb41c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=88c2746909c9b92d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.320+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb42c039070 con 0x7fb41c004d40 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.320+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb42c02cde0 con 0x7fb41c004d40 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.320+0000 7fb432ffd640 1 -- 172.21.15.64:0/3366116323 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb42c009de0 con 0x7fb41c004d40 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.321+0000 7fb43a37a640 1 -- 172.21.15.64:0/3366116323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb41c004d40 msgr2=0x7fb41c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.321+0000 7fb43a37a640 1 --2- 172.21.15.64:0/3366116323 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb41c004d40 0x7fb41c0051c0 secure :-1 s=READY pgs=889 cs=0 l=1 rev1=1 crypto rx=0x7fb42c00e320 tx=0x7fb41c005830 comp rx=0 tx=0).stop 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.321+0000 7fb438917640 1 -- 172.21.15.64:0/3366116323 reap_dead start 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.321+0000 7fb43a37a640 1 -- 172.21.15.64:0/3366116323 shutdown_connections 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.321+0000 7fb43a37a640 1 -- 172.21.15.64:0/3366116323 >> 172.21.15.64:0/3366116323 conn(0x7fb434071960 msgr2=0x7fb434071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.321+0000 7fb43a37a640 1 -- 172.21.15.64:0/3366116323 shutdown_connections 2024-03-04T07:30:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.321+0000 7fb43a37a640 1 -- 172.21.15.64:0/3366116323 wait complete. 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.322+0000 7fb43a37a640 1 Processor -- start 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.322+0000 7fb43a37a640 1 -- start start 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb43a37a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb43415bd20 0x7fb4341493b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb43a37a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb43415c100 0x7fb4341498f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb43a37a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4340c9940 0x7fb4340c9dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb43a37a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4340cec70 con 0x7fb43415c100 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb43a37a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4340ce750 con 0x7fb43415bd20 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb43a37a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4340cedf0 con 0x7fb4340c9940 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb43415c100 0x7fb4341498f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb43415c100 0x7fb4341498f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41612/0 (socket says 172.21.15.64:41612) 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 -- 172.21.15.64:0/875568545 learned_addr learned my addr 172.21.15.64:0/875568545 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb433fff640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb43415bd20 0x7fb4341493b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb438917640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4340c9940 0x7fb4340c9dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 -- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4340c9940 msgr2=0x7fb4340c9dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4340c9940 0x7fb4340c9dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 -- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb43415bd20 msgr2=0x7fb4341493b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb43415bd20 0x7fb4341493b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 -- 172.21.15.64:0/875568545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb43414add0 con 0x7fb43415c100 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb438917640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4340c9940 0x7fb4340c9dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.323+0000 7fb4337fe640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb43415c100 0x7fb4341498f0 secure :-1 s=READY pgs=890 cs=0 l=1 rev1=1 crypto rx=0x7fb428007320 tx=0x7fb428007720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.324+0000 7fb430ff9640 1 -- 172.21.15.64:0/875568545 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb42801a070 con 0x7fb43415c100 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.324+0000 7fb430ff9640 1 -- 172.21.15.64:0/875568545 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb42800f9c0 con 0x7fb43415c100 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.324+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb43414a010 con 0x7fb43415c100 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.324+0000 7fb430ff9640 1 -- 172.21.15.64:0/875568545 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb42800fd90 con 0x7fb43415c100 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.324+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb43414a5d0 con 0x7fb43415c100 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.325+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb434067ff0 con 0x7fb43415c100 2024-03-04T07:30:31.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.325+0000 7fb430ff9640 1 -- 172.21.15.64:0/875568545 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb42801d360 con 0x7fb43415c100 2024-03-04T07:30:31.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.326+0000 7fb430ff9640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb42006b4d0 0x7fb42006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:31.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.326+0000 7fb430ff9640 1 -- 172.21.15.64:0/875568545 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb428089d30 con 0x7fb43415c100 2024-03-04T07:30:31.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.326+0000 7fb433fff640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb42006b4d0 0x7fb42006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:31.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.326+0000 7fb433fff640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb42006b4d0 0x7fb42006d990 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7fb424009ce0 tx=0x7fb424009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:31.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.330+0000 7fb430ff9640 1 -- 172.21.15.64:0/875568545 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb428058b60 con 0x7fb43415c100 2024-03-04T07:30:31.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.523+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fb4340cdb60 con 0x7fb43415c100 2024-03-04T07:30:31.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.524+0000 7fb430ff9640 1 -- 172.21.15.64:0/875568545 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v66) v1 ==== 80+0+0 (secure 0 0 0) 0x7fb428077080 con 0x7fb43415c100 2024-03-04T07:30:31.523 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:30:31.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.525+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb42006b4d0 msgr2=0x7fb42006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.525+0000 7fb43a37a640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb42006b4d0 0x7fb42006d990 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7fb424009ce0 tx=0x7fb424009190 comp rx=0 tx=0).stop 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb43415c100 msgr2=0x7fb4341498f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb43415c100 0x7fb4341498f0 secure :-1 s=READY pgs=890 cs=0 l=1 rev1=1 crypto rx=0x7fb428007320 tx=0x7fb428007720 comp rx=0 tx=0).stop 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 shutdown_connections 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb42006b4d0 0x7fb42006d990 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4340c9940 0x7fb4340c9dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb43415c100 0x7fb4341498f0 unknown :-1 s=CLOSED pgs=890 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 --2- 172.21.15.64:0/875568545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb43415bd20 0x7fb4341493b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 >> 172.21.15.64:0/875568545 conn(0x7fb434071960 msgr2=0x7fb43415d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 shutdown_connections 2024-03-04T07:30:31.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:31.526+0000 7fb43a37a640 1 -- 172.21.15.64:0/875568545 wait complete. 2024-03-04T07:30:31.539 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:30:31.539 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:31.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:31.565 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[62844]: fuse finished with error 0 and tester_r 0 2024-03-04T07:30:31.567 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:30:31.567 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:31.567 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:30:31.734 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:30:31.734 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:31.751 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:30:31.751 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:30:31.751 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:31.752 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:30:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:30:31.808 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:30:31.808 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:31.808 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:31.823 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:30:31.823 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:30:31.823 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:31.823 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:30:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:30:31.878 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:30:31.878 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:30:31.878 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:31.895 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:30:31.895 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:30:31.895 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:31.895 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:30:31.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:30:31.952 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:30:31.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:32.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.055+0000 7ff4bf2b5640 1 Processor -- start 2024-03-04T07:30:32.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.055+0000 7ff4bf2b5640 1 -- start start 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bf2b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff4b815e080 0x7ff4b8066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bf2b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4b8161c20 con 0x7ff4b80b00c0 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bf2b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4b8161f00 con 0x7ff4b815dca0 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bf2b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff4b81619d0 con 0x7ff4b815e080 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bc850640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff4b815e080 0x7ff4b8066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bd852640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff4b80b00c0 0x7ff4b80b3ca0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35348/0 (socket says 172.21.15.64:35348) 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bc850640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff4b815e080 0x7ff4b8066d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41626/0 (socket says 172.21.15.64:41626) 2024-03-04T07:30:32.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bd852640 1 -- 172.21.15.64:0/2825586838 learned_addr learned my addr 172.21.15.64:0/2825586838 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:32.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bc850640 1 -- 172.21.15.64:0/2825586838 >> v1:172.21.15.66:6789/0 conn(0x7ff4b815dca0 legacy=0x7ff4b806fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bc850640 1 -- 172.21.15.64:0/2825586838 >> v1:172.21.15.64:6789/0 conn(0x7ff4b80b00c0 legacy=0x7ff4b80b3ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4affff640 1 -- 172.21.15.64:0/2825586838 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1516203989 0 0) 0x7ff4b8161c20 con 0x7ff4b80b00c0 2024-03-04T07:30:32.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.056+0000 7ff4bc850640 1 -- 172.21.15.64:0/2825586838 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4b81621e0 con 0x7ff4b815e080 2024-03-04T07:30:32.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.057+0000 7ff4bc850640 1 --2- 172.21.15.64:0/2825586838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b815e080 0x7ff4b8066d00 secure :-1 s=READY pgs=892 cs=0 l=1 rev1=1 crypto rx=0x7ff4a80097a0 tx=0x7ff4a802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=77e9db8e54f525f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.057+0000 7ff4affff640 1 -- 172.21.15.64:0/2825586838 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4a8035020 con 0x7ff4b815e080 2024-03-04T07:30:32.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.057+0000 7ff4affff640 1 -- 172.21.15.64:0/2825586838 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4a8031830 con 0x7ff4b815e080 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.057+0000 7ff4affff640 1 -- 172.21.15.64:0/2825586838 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4a8031c00 con 0x7ff4b815e080 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.058+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/2825586838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b815e080 msgr2=0x7ff4b8066d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.058+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/2825586838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b815e080 0x7ff4b8066d00 secure :-1 s=READY pgs=892 cs=0 l=1 rev1=1 crypto rx=0x7ff4a80097a0 tx=0x7ff4a802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.058+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/2825586838 shutdown_connections 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.058+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/2825586838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b815e080 0x7ff4b8066d00 unknown :-1 s=CLOSED pgs=892 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.058+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/2825586838 >> 172.21.15.64:0/2825586838 conn(0x7ff4b80d0990 msgr2=0x7ff4b80d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.058+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/2825586838 shutdown_connections 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.058+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/2825586838 wait complete. 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.059+0000 7ff4bf2b5640 1 Processor -- start 2024-03-04T07:30:32.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.059+0000 7ff4bf2b5640 1 -- start start 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bf2b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b80b00c0 0x7ff4b8142600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bf2b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4b815dca0 0x7ff4b80cf600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bf2b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4b80cfb40 0x7ff4b80ae250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bf2b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4b80673c0 con 0x7ff4b80b00c0 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bf2b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4b8067240 con 0x7ff4b80cfb40 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bf2b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4b8067540 con 0x7ff4b815dca0 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b80b00c0 0x7ff4b8142600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b80b00c0 0x7ff4b8142600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41634/0 (socket says 172.21.15.64:41634) 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 -- 172.21.15.64:0/3400332542 learned_addr learned my addr 172.21.15.64:0/3400332542 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd852640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4b80cfb40 0x7ff4b80ae250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bc850640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4b815dca0 0x7ff4b80cf600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 -- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4b815dca0 msgr2=0x7ff4b80cf600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4b815dca0 0x7ff4b80cf600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 -- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4b80cfb40 msgr2=0x7ff4b80ae250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4b80cfb40 0x7ff4b80ae250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 -- 172.21.15.64:0/3400332542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4b80d00a0 con 0x7ff4b80b00c0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd852640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4b80cfb40 0x7ff4b80ae250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bd051640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b80b00c0 0x7ff4b8142600 secure :-1 s=READY pgs=893 cs=0 l=1 rev1=1 crypto rx=0x7ff4b400c450 tx=0x7ff4b400c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4adffb640 1 -- 172.21.15.64:0/3400332542 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4b40052d0 con 0x7ff4b80b00c0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4adffb640 1 -- 172.21.15.64:0/3400332542 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4b4005470 con 0x7ff4b80b00c0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4adffb640 1 -- 172.21.15.64:0/3400332542 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4b4012870 con 0x7ff4b80b00c0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.060+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4b81401b0 con 0x7ff4b80b00c0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.061+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4b8140770 con 0x7ff4b80b00c0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.061+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4b8067b50 con 0x7ff4b80b00c0 2024-03-04T07:30:32.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.062+0000 7ff4adffb640 1 -- 172.21.15.64:0/3400332542 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff4b4012a10 con 0x7ff4b80b00c0 2024-03-04T07:30:32.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.062+0000 7ff4adffb640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff49c06b4d0 0x7ff49c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.062+0000 7ff4adffb640 1 -- 172.21.15.64:0/3400332542 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff4b4089e40 con 0x7ff4b80b00c0 2024-03-04T07:30:32.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.062+0000 7ff4bc850640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff49c06b4d0 0x7ff49c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.063+0000 7ff4bc850640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff49c06b4d0 0x7ff49c06d990 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7ff4a8036000 tx=0x7ff4a803c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.067+0000 7ff4adffb640 1 -- 172.21.15.64:0/3400332542 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff4b8067b50 con 0x7ff4b80b00c0 2024-03-04T07:30:32.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.289+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff4b8068700 con 0x7ff4b80b00c0 2024-03-04T07:30:32.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.289+0000 7ff4adffb640 1 -- 172.21.15.64:0/3400332542 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7ff4b4058cc0 con 0x7ff4b80b00c0 2024-03-04T07:30:32.289 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:32.289 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:32.289 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:32.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff49c06b4d0 msgr2=0x7ff49c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff49c06b4d0 0x7ff49c06d990 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7ff4a8036000 tx=0x7ff4a803c040 comp rx=0 tx=0).stop 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b80b00c0 msgr2=0x7ff4b8142600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b80b00c0 0x7ff4b8142600 secure :-1 s=READY pgs=893 cs=0 l=1 rev1=1 crypto rx=0x7ff4b400c450 tx=0x7ff4b400c910 comp rx=0 tx=0).stop 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 shutdown_connections 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff49c06b4d0 0x7ff49c06d990 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4b80cfb40 0x7ff4b80ae250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4b815dca0 0x7ff4b80cf600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 --2- 172.21.15.64:0/3400332542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4b80b00c0 0x7ff4b8142600 unknown :-1 s=CLOSED pgs=893 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 >> 172.21.15.64:0/3400332542 conn(0x7ff4b80d0990 msgr2=0x7ff4b806fcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 shutdown_connections 2024-03-04T07:30:32.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.291+0000 7ff4bf2b5640 1 -- 172.21.15.64:0/3400332542 wait complete. 2024-03-04T07:30:32.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:32.404 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.404+0000 7f13c03c9640 1 Processor -- start 2024-03-04T07:30:32.404 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.404+0000 7f13c03c9640 1 -- start start 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.405+0000 7f13c03c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b815df00 0x7f13b80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.405+0000 7f13c03c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f13b80b95a0 0x7f13b80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.405+0000 7f13c03c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f13b80cea30 con 0x7f13b80b9a30 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.405+0000 7f13c03c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f13b80d2af0 con 0x7f13b80b95a0 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.405+0000 7f13c03c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f13b80cf3c0 con 0x7f13b815df00 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13bd964640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f13b80b95a0 0x7f13b80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13bd964640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f13b80b95a0 0x7f13b80b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41642/0 (socket says 172.21.15.64:41642) 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13bd964640 1 -- 172.21.15.64:0/281910394 learned_addr learned my addr 172.21.15.64:0/281910394 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13be165640 1 --2- 172.21.15.64:0/281910394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b815df00 0x7f13b80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13bd964640 1 -- 172.21.15.64:0/281910394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b815df00 msgr2=0x7f13b80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13bd964640 1 --2- 172.21.15.64:0/281910394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b815df00 0x7f13b80d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13bd964640 1 -- 172.21.15.64:0/281910394 >> v1:172.21.15.66:6789/0 conn(0x7f13b80b9a30 legacy=0x7f13b80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13bd964640 1 -- 172.21.15.64:0/281910394 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13b80cf0e0 con 0x7f13b80b95a0 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.406+0000 7f13be165640 1 --2- 172.21.15.64:0/281910394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b815df00 0x7f13b80d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13bd964640 1 --2- 172.21.15.64:0/281910394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b80b95a0 0x7f13b80b5700 secure :-1 s=READY pgs=894 cs=0 l=1 rev1=1 crypto rx=0x7f13a8009870 tx=0x7f13a802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=242bd013602486e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13bd163640 1 -- 172.21.15.64:0/281910394 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13a8034070 con 0x7f13b80b95a0 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13bd163640 1 -- 172.21.15.64:0/281910394 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f13a8030840 con 0x7f13b80b95a0 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13bd163640 1 -- 172.21.15.64:0/281910394 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13a8030c10 con 0x7f13b80b95a0 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13c03c9640 1 -- 172.21.15.64:0/281910394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b80b95a0 msgr2=0x7f13b80b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13c03c9640 1 --2- 172.21.15.64:0/281910394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b80b95a0 0x7f13b80b5700 secure :-1 s=READY pgs=894 cs=0 l=1 rev1=1 crypto rx=0x7f13a8009870 tx=0x7f13a802b980 comp rx=0 tx=0).stop 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13c03c9640 1 -- 172.21.15.64:0/281910394 shutdown_connections 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13c03c9640 1 --2- 172.21.15.64:0/281910394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b80b95a0 0x7f13b80b5700 unknown :-1 s=CLOSED pgs=894 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13c03c9640 1 --2- 172.21.15.64:0/281910394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b815df00 0x7f13b80d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.407+0000 7f13c03c9640 1 -- 172.21.15.64:0/281910394 >> 172.21.15.64:0/281910394 conn(0x7f13b8071960 msgr2=0x7f13b8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.408+0000 7f13c03c9640 1 -- 172.21.15.64:0/281910394 shutdown_connections 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.408+0000 7f13c03c9640 1 -- 172.21.15.64:0/281910394 wait complete. 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 Processor -- start 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 -- start start 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b80b95a0 0x7f13b80b59e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b815df00 0x7f13b80cc810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f13b80b4480 0x7f13b80c8b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13b80cec70 con 0x7f13b815df00 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13b80ce750 con 0x7f13b80b4480 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13c03c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f13b80cedf0 con 0x7f13b80b95a0 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13bd964640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b815df00 0x7f13b80cc810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13bd964640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b815df00 0x7f13b80cc810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41648/0 (socket says 172.21.15.64:41648) 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.409+0000 7f13bd964640 1 -- 172.21.15.64:0/3863884674 learned_addr learned my addr 172.21.15.64:0/3863884674 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:32.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13be165640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b80b95a0 0x7f13b80b59e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13be966640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f13b80b4480 0x7f13b80c8b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13bd964640 1 -- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b80b95a0 msgr2=0x7f13b80b59e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13bd964640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b80b95a0 0x7f13b80b59e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13bd964640 1 -- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f13b80b4480 msgr2=0x7f13b80c8b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13bd964640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f13b80b4480 0x7f13b80c8b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13bd964640 1 -- 172.21.15.64:0/3863884674 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13b80b4940 con 0x7f13b815df00 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13bd964640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b815df00 0x7f13b80cc810 secure :-1 s=READY pgs=895 cs=0 l=1 rev1=1 crypto rx=0x7f13a8000c00 tx=0x7f13a8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13aeffd640 1 -- 172.21.15.64:0/3863884674 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13a8048070 con 0x7f13b815df00 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13aeffd640 1 -- 172.21.15.64:0/3863884674 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f13a8030640 con 0x7f13b815df00 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f13b80b4c20 con 0x7f13b815df00 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13aeffd640 1 -- 172.21.15.64:0/3863884674 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13a8034040 con 0x7f13b815df00 2024-03-04T07:30:32.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.410+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13b80b51e0 con 0x7f13b815df00 2024-03-04T07:30:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.411+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f13b80caae0 con 0x7f13b815df00 2024-03-04T07:30:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.412+0000 7f13aeffd640 1 -- 172.21.15.64:0/3863884674 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f13a804b070 con 0x7f13b815df00 2024-03-04T07:30:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.412+0000 7f13aeffd640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f139c06b4d0 0x7f139c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.412+0000 7f13aeffd640 1 -- 172.21.15.64:0/3863884674 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f13a80ac220 con 0x7f13b815df00 2024-03-04T07:30:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.412+0000 7f13be165640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f139c06b4d0 0x7f139c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.413+0000 7f13be165640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f139c06b4d0 0x7f139c06d990 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f13b8160ec0 tx=0x7f13b4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.417+0000 7f13aeffd640 1 -- 172.21.15.64:0/3863884674 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f13b80caae0 con 0x7f13b815df00 2024-03-04T07:30:32.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.601+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f13b8067ff0 con 0x7f13b815df00 2024-03-04T07:30:32.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.602+0000 7f13aeffd640 1 -- 172.21.15.64:0/3863884674 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v66) v1 ==== 74+0+16620 (secure 0 0 0) 0x7f13a807ab70 con 0x7f13b815df00 2024-03-04T07:30:32.602 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:32.602 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":66,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:06.066460+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:02.706399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:03.395544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:32.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f139c06b4d0 msgr2=0x7f139c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f139c06b4d0 0x7f139c06d990 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f13b8160ec0 tx=0x7f13b4008040 comp rx=0 tx=0).stop 2024-03-04T07:30:32.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b815df00 msgr2=0x7f13b80cc810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b815df00 0x7f13b80cc810 secure :-1 s=READY pgs=895 cs=0 l=1 rev1=1 crypto rx=0x7f13a8000c00 tx=0x7f13a8004730 comp rx=0 tx=0).stop 2024-03-04T07:30:32.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 shutdown_connections 2024-03-04T07:30:32.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f139c06b4d0 0x7f139c06d990 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f13b80b4480 0x7f13b80c8b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f13b815df00 0x7f13b80cc810 unknown :-1 s=CLOSED pgs=895 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 --2- 172.21.15.64:0/3863884674 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f13b80b95a0 0x7f13b80b59e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 >> 172.21.15.64:0/3863884674 conn(0x7f13b8071960 msgr2=0x7f13b80b8860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:32.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 shutdown_connections 2024-03-04T07:30:32.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.604+0000 7f13c03c9640 1 -- 172.21.15.64:0/3863884674 wait complete. 2024-03-04T07:30:32.615 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 10, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:30:02.706399+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '65', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 11, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:30:03.395544+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '66', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:30:32.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:30:32.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.720+0000 7fe5fd066640 1 Processor -- start 2024-03-04T07:30:32.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.720+0000 7fe5fd066640 1 -- start start 2024-03-04T07:30:32.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5fd066640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe5f80cf010 0x7fe5f815b5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5fd066640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe5f815bc40 con 0x7fe5f815b130 2024-03-04T07:30:32.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5fd066640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe5f815bf20 con 0x7fe5f8144640 2024-03-04T07:30:32.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5fd066640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe5f806df70 con 0x7fe5f80cf010 2024-03-04T07:30:32.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe5f80cf010 0x7fe5f815b5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe5f80cf010 0x7fe5f815b5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41660/0 (socket says 172.21.15.64:41660) 2024-03-04T07:30:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f759e640 1 -- 172.21.15.64:0/2706837024 learned_addr learned my addr 172.21.15.64:0/2706837024 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f659c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe5f815b130 0x7fe5f80ce900 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35350/0 (socket says 172.21.15.64:35350) 2024-03-04T07:30:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f759e640 1 -- 172.21.15.64:0/2706837024 >> v1:172.21.15.66:6789/0 conn(0x7fe5f8144640 legacy=0x7fe5f806e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f759e640 1 -- 172.21.15.64:0/2706837024 >> v1:172.21.15.64:6789/0 conn(0x7fe5f815b130 legacy=0x7fe5f80ce900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f759e640 1 -- 172.21.15.64:0/2706837024 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5f806dc00 con 0x7fe5f80cf010 2024-03-04T07:30:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.721+0000 7fe5f5d9b640 1 -- 172.21.15.64:0/2706837024 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 688169633 0 0) 0x7fe5f815bc40 con 0x7fe5f815b130 2024-03-04T07:30:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.722+0000 7fe5f759e640 1 --2- 172.21.15.64:0/2706837024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f815b5c0 secure :-1 s=READY pgs=897 cs=0 l=1 rev1=1 crypto rx=0x7fe5f0009e50 tx=0x7fe5f002c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c186658c36d35577 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.722+0000 7fe5f5d9b640 1 -- 172.21.15.64:0/2706837024 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5f002cbb0 con 0x7fe5f80cf010 2024-03-04T07:30:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.722+0000 7fe5f5d9b640 1 -- 172.21.15.64:0/2706837024 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5f002cd50 con 0x7fe5f80cf010 2024-03-04T07:30:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.722+0000 7fe5f5d9b640 1 -- 172.21.15.64:0/2706837024 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5f0034610 con 0x7fe5f80cf010 2024-03-04T07:30:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.723+0000 7fe5fd066640 1 -- 172.21.15.64:0/2706837024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 msgr2=0x7fe5f815b5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.723+0000 7fe5fd066640 1 --2- 172.21.15.64:0/2706837024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f815b5c0 secure :-1 s=READY pgs=897 cs=0 l=1 rev1=1 crypto rx=0x7fe5f0009e50 tx=0x7fe5f002c010 comp rx=0 tx=0).stop 2024-03-04T07:30:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.723+0000 7fe5fd066640 1 -- 172.21.15.64:0/2706837024 shutdown_connections 2024-03-04T07:30:32.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.723+0000 7fe5fd066640 1 --2- 172.21.15.64:0/2706837024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f815b5c0 unknown :-1 s=CLOSED pgs=897 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.723+0000 7fe5fd066640 1 -- 172.21.15.64:0/2706837024 >> 172.21.15.64:0/2706837024 conn(0x7fe5f8066d00 msgr2=0x7fe5f8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:32.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.723+0000 7fe5fd066640 1 -- 172.21.15.64:0/2706837024 shutdown_connections 2024-03-04T07:30:32.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.723+0000 7fe5fd066640 1 -- 172.21.15.64:0/2706837024 wait complete. 2024-03-04T07:30:32.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.724+0000 7fe5fd066640 1 Processor -- start 2024-03-04T07:30:32.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.724+0000 7fe5fd066640 1 -- start start 2024-03-04T07:30:32.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5fd066640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f8159e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5fd066640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5f8144640 0x7fe5f81e3e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5fd066640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5f815b130 0x7fe5f813cd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5fd066640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f8070c20 con 0x7fe5f80cf010 2024-03-04T07:30:32.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5fd066640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f8070aa0 con 0x7fe5f8144640 2024-03-04T07:30:32.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5fd066640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f80b36d0 con 0x7fe5f815b130 2024-03-04T07:30:32.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f8159e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f8159e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41670/0 (socket says 172.21.15.64:41670) 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 -- 172.21.15.64:0/3872247777 learned_addr learned my addr 172.21.15.64:0/3872247777 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f659c640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5f8144640 0x7fe5f81e3e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f759e640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5f815b130 0x7fe5f813cd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 -- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5f815b130 msgr2=0x7fe5f813cd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5f815b130 0x7fe5f813cd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 -- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5f8144640 msgr2=0x7fe5f81e3e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5f8144640 0x7fe5f81e3e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 -- 172.21.15.64:0/3872247777 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5f81e4360 con 0x7fe5f80cf010 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.725+0000 7fe5f6d9d640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f8159e50 secure :-1 s=READY pgs=898 cs=0 l=1 rev1=1 crypto rx=0x7fe5ec00c480 tx=0x7fe5ec00c940 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.726+0000 7fe5df7fe640 1 -- 172.21.15.64:0/3872247777 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5ec0052d0 con 0x7fe5f80cf010 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.726+0000 7fe5df7fe640 1 -- 172.21.15.64:0/3872247777 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe5ec005470 con 0x7fe5f80cf010 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.726+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe5f81e4630 con 0x7fe5f80cf010 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.726+0000 7fe5df7fe640 1 -- 172.21.15.64:0/3872247777 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5ec012870 con 0x7fe5f80cf010 2024-03-04T07:30:32.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.726+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5f81e5140 con 0x7fe5f80cf010 2024-03-04T07:30:32.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.727+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe5f80733c0 con 0x7fe5f80cf010 2024-03-04T07:30:32.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.727+0000 7fe5df7fe640 1 -- 172.21.15.64:0/3872247777 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe5ec012a10 con 0x7fe5f80cf010 2024-03-04T07:30:32.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.727+0000 7fe5df7fe640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e406b4d0 0x7fe5e406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:32.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.728+0000 7fe5df7fe640 1 -- 172.21.15.64:0/3872247777 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe5ec089d70 con 0x7fe5f80cf010 2024-03-04T07:30:32.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.728+0000 7fe5f659c640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e406b4d0 0x7fe5e406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:32.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.728+0000 7fe5f659c640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e406b4d0 0x7fe5e406d990 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fe5f813fe60 tx=0x7fe5e8007e40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:32.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:32.732+0000 7fe5df7fe640 1 -- 172.21.15.64:0/3872247777 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe5ec058c80 con 0x7fe5f80cf010 2024-03-04T07:30:33.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.030+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fe5f8139170 con 0x7fe5f80cf010 2024-03-04T07:30:33.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.030+0000 7fe5df7fe640 1 -- 172.21.15.64:0/3872247777 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v66) v1 ==== 93+0+10 (secure 0 0 0) 0x7fe5ec058580 con 0x7fe5f80cf010 2024-03-04T07:30:33.030 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e406b4d0 msgr2=0x7fe5e406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e406b4d0 0x7fe5e406d990 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fe5f813fe60 tx=0x7fe5e8007e40 comp rx=0 tx=0).stop 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 msgr2=0x7fe5f8159e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f8159e50 secure :-1 s=READY pgs=898 cs=0 l=1 rev1=1 crypto rx=0x7fe5ec00c480 tx=0x7fe5ec00c940 comp rx=0 tx=0).stop 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 shutdown_connections 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe5e406b4d0 0x7fe5e406d990 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe5f815b130 0x7fe5f813cd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe5f8144640 0x7fe5f81e3e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 --2- 172.21.15.64:0/3872247777 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe5f80cf010 0x7fe5f8159e50 unknown :-1 s=CLOSED pgs=898 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 >> 172.21.15.64:0/3872247777 conn(0x7fe5f8066d00 msgr2=0x7fe5f81420c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 shutdown_connections 2024-03-04T07:30:33.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.033+0000 7fe5fd066640 1 -- 172.21.15.64:0/3872247777 wait complete. 2024-03-04T07:30:33.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:30:33.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.177+0000 7f577655e640 1 Processor -- start 2024-03-04T07:30:33.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.177+0000 7f577655e640 1 -- start start 2024-03-04T07:30:33.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.177+0000 7f577655e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f577015dd90 0x7f57700ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.177+0000 7f577655e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f57700d29d0 0x7f57700d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f577655e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f57700d3740 con 0x7f57700d4720 2024-03-04T07:30:33.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f577655e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f57700d3400 con 0x7f577015dd90 2024-03-04T07:30:33.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f577655e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f57700d3120 con 0x7f57700d29d0 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f5774afb640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f57700d4720 0x7f57700b6d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35364/0 (socket says 172.21.15.64:35364) 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f5774afb640 1 -- 172.21.15.64:0/2828661452 learned_addr learned my addr 172.21.15.64:0/2828661452 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576ffff640 1 --2- 172.21.15.64:0/2828661452 >> v2:172.21.15.66:3300/0 conn(0x7f577015dd90 0x7f57700ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576f7fe640 1 --2- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f57700d29d0 0x7f57700d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576effd640 1 -- 172.21.15.64:0/2828661452 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1577553804 0 0) 0x7f57700d3740 con 0x7f57700d4720 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576effd640 1 -- 172.21.15.64:0/2828661452 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5758003520 con 0x7f57700d4720 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576effd640 1 -- 172.21.15.64:0/2828661452 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 734167897 0 0) 0x7f5758003520 con 0x7f57700d4720 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576ffff640 1 -- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f57700d29d0 msgr2=0x7f57700d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576ffff640 1 --2- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f57700d29d0 0x7f57700d41e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576ffff640 1 -- 172.21.15.64:0/2828661452 >> v1:172.21.15.64:6789/0 conn(0x7f57700d4720 legacy=0x7f57700b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.178+0000 7f576ffff640 1 -- 172.21.15.64:0/2828661452 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57700d39e0 con 0x7f577015dd90 2024-03-04T07:30:33.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.179+0000 7f576ffff640 1 --2- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f577015dd90 0x7f57700ceea0 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f57600097a0 tx=0x7f576002b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=92cd840c19fcb198 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.179+0000 7f576effd640 1 -- 172.21.15.64:0/2828661452 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5760038070 con 0x7f577015dd90 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.179+0000 7f576effd640 1 -- 172.21.15.64:0/2828661452 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5760031500 con 0x7f577015dd90 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.179+0000 7f576effd640 1 -- 172.21.15.64:0/2828661452 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57680019f0 con 0x7f577015dd90 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 -- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f577015dd90 msgr2=0x7f57700ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 --2- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f577015dd90 0x7f57700ceea0 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f57600097a0 tx=0x7f576002b820 comp rx=0 tx=0).stop 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 -- 172.21.15.64:0/2828661452 shutdown_connections 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 --2- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f57700d29d0 0x7f57700d41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 --2- 172.21.15.64:0/2828661452 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f577015dd90 0x7f57700ceea0 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 -- 172.21.15.64:0/2828661452 >> 172.21.15.64:0/2828661452 conn(0x7f5770071960 msgr2=0x7f5770071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 -- 172.21.15.64:0/2828661452 shutdown_connections 2024-03-04T07:30:33.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.180+0000 7f577655e640 1 -- 172.21.15.64:0/2828661452 wait complete. 2024-03-04T07:30:33.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.182+0000 7f577655e640 1 Processor -- start 2024-03-04T07:30:33.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.182+0000 7f577655e640 1 -- start start 2024-03-04T07:30:33.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f577655e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57700d29d0 0x7f577015f700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f577655e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57700d4720 0x7f577015cc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f577655e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f577015dd90 0x7f57700ccd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f577655e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57700d2df0 con 0x7f57700d4720 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f577655e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f577015e170 con 0x7f57700d29d0 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f577655e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f57700b7420 con 0x7f577015dd90 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f576f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57700d4720 0x7f577015cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f576f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57700d4720 0x7f577015cc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41676/0 (socket says 172.21.15.64:41676) 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.183+0000 7f576f7fe640 1 -- 172.21.15.64:0/2826006526 learned_addr learned my addr 172.21.15.64:0/2826006526 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576ffff640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57700d29d0 0x7f577015f700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f5774afb640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f577015dd90 0x7f57700ccd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576f7fe640 1 -- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f577015dd90 msgr2=0x7f57700ccd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576f7fe640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f577015dd90 0x7f57700ccd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576f7fe640 1 -- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57700d29d0 msgr2=0x7f577015f700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576f7fe640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57700d29d0 0x7f577015f700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576f7fe640 1 -- 172.21.15.64:0/2826006526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f577015e660 con 0x7f57700d4720 2024-03-04T07:30:33.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576f7fe640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57700d4720 0x7f577015cc60 secure :-1 s=READY pgs=900 cs=0 l=1 rev1=1 crypto rx=0x7f576400e9b0 tx=0x7f576400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576cff9640 1 -- 172.21.15.64:0/2826006526 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5764017070 con 0x7f57700d4720 2024-03-04T07:30:33.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576cff9640 1 -- 172.21.15.64:0/2826006526 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f576400c9a0 con 0x7f57700d4720 2024-03-04T07:30:33.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f577015e940 con 0x7f57700d4720 2024-03-04T07:30:33.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f576cff9640 1 -- 172.21.15.64:0/2826006526 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f576400cd90 con 0x7f57700d4720 2024-03-04T07:30:33.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.184+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f577015ef00 con 0x7f57700d4720 2024-03-04T07:30:33.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.185+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f57700b7b40 con 0x7f57700d4720 2024-03-04T07:30:33.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.186+0000 7f576cff9640 1 -- 172.21.15.64:0/2826006526 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f576401a070 con 0x7f57700d4720 2024-03-04T07:30:33.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.186+0000 7f576cff9640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f575c06b4d0 0x7f575c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.186+0000 7f576cff9640 1 -- 172.21.15.64:0/2826006526 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f576408a4a0 con 0x7f57700d4720 2024-03-04T07:30:33.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.186+0000 7f576ffff640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f575c06b4d0 0x7f575c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.187+0000 7f576ffff640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f575c06b4d0 0x7f575c06d990 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f5760035000 tx=0x7f5760008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.190+0000 7f576cff9640 1 -- 172.21.15.64:0/2826006526 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f57700b7b40 con 0x7f57700d4720 2024-03-04T07:30:33.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.412+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f5770072a30 con 0x7f57700d4720 2024-03-04T07:30:33.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.412+0000 7f576cff9640 1 -- 172.21.15.64:0/2826006526 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v66) v1 ==== 104+0+11 (secure 0 0 0) 0x7f5764058d60 con 0x7f57700d4720 2024-03-04T07:30:33.412 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:30:33.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f575c06b4d0 msgr2=0x7f575c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f575c06b4d0 0x7f575c06d990 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f5760035000 tx=0x7f5760008030 comp rx=0 tx=0).stop 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57700d4720 msgr2=0x7f577015cc60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57700d4720 0x7f577015cc60 secure :-1 s=READY pgs=900 cs=0 l=1 rev1=1 crypto rx=0x7f576400e9b0 tx=0x7f576400ee70 comp rx=0 tx=0).stop 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 shutdown_connections 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f575c06b4d0 0x7f575c06d990 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f577015dd90 0x7f57700ccd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f57700d4720 0x7f577015cc60 unknown :-1 s=CLOSED pgs=900 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 --2- 172.21.15.64:0/2826006526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f57700d29d0 0x7f577015f700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 >> 172.21.15.64:0/2826006526 conn(0x7f5770071960 msgr2=0x7f57700ce280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 shutdown_connections 2024-03-04T07:30:33.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.414+0000 7f577655e640 1 -- 172.21.15.64:0/2826006526 wait complete. 2024-03-04T07:30:33.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:30:33.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.530+0000 7fdb7b95e640 1 Processor -- start 2024-03-04T07:30:33.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.531+0000 7fdb7b95e640 1 -- start start 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.531+0000 7fdb7b95e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb740b3ce0 0x7fdb7406dc90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.531+0000 7fdb7b95e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fdb74067410 0x7fdb7406e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.531+0000 7fdb7b95e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdb7406e710 con 0x7fdb7415df40 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.531+0000 7fdb7b95e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdb7415f000 con 0x7fdb74067410 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.531+0000 7fdb7b95e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdb7415e6d0 con 0x7fdb740b3ce0 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb78ef9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb740b3ce0 0x7fdb7406dc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb79efb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fdb74067410 0x7fdb7406e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb796fa640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fdb7415df40 0x7fdb74066d00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49778/0 (socket says 172.21.15.64:49778) 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb796fa640 1 -- 172.21.15.64:0/3554384252 learned_addr learned my addr 172.21.15.64:0/3554384252 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb79efb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fdb74067410 0x7fdb7406e1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:55788/0 (socket says 172.21.15.64:55788) 2024-03-04T07:30:33.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb79efb640 1 -- 172.21.15.64:0/3554384252 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb740b3ce0 msgr2=0x7fdb7406dc90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:33.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb79efb640 1 --2- 172.21.15.64:0/3554384252 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb740b3ce0 0x7fdb7406dc90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb79efb640 1 -- 172.21.15.64:0/3554384252 >> v1:172.21.15.66:6789/0 conn(0x7fdb7415df40 legacy=0x7fdb74066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb79efb640 1 -- 172.21.15.64:0/3554384252 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb7415e9b0 con 0x7fdb74067410 2024-03-04T07:30:33.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.532+0000 7fdb6bfff640 1 -- 172.21.15.64:0/3554384252 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2765332609 0 0) 0x7fdb7406e710 con 0x7fdb7415df40 2024-03-04T07:30:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.533+0000 7fdb79efb640 1 --2- 172.21.15.64:0/3554384252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb74067410 0x7fdb7406e1d0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fdb6c009d90 tx=0x7fdb6c02bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=24ec9c384056795c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.533+0000 7fdb6bfff640 1 -- 172.21.15.64:0/3554384252 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb6c039070 con 0x7fdb74067410 2024-03-04T07:30:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.533+0000 7fdb6bfff640 1 -- 172.21.15.64:0/3554384252 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdb6c02cc50 con 0x7fdb74067410 2024-03-04T07:30:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.533+0000 7fdb6bfff640 1 -- 172.21.15.64:0/3554384252 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb6c034440 con 0x7fdb74067410 2024-03-04T07:30:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3554384252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb74067410 msgr2=0x7fdb7406e1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3554384252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb74067410 0x7fdb7406e1d0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fdb6c009d90 tx=0x7fdb6c02bea0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3554384252 shutdown_connections 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3554384252 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb74067410 0x7fdb7406e1d0 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3554384252 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb740b3ce0 0x7fdb7406dc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3554384252 >> 172.21.15.64:0/3554384252 conn(0x7fdb740d09c0 msgr2=0x7fdb740d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3554384252 shutdown_connections 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.534+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3554384252 wait complete. 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.536+0000 7fdb7b95e640 1 Processor -- start 2024-03-04T07:30:33.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.536+0000 7fdb7b95e640 1 -- start start 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb7b95e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb74067410 0x7fdb7415f490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb7b95e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb740b3ce0 0x7fdb741611a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb7b95e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb7415df40 0x7fdb740ae050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb7b95e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb7415e320 con 0x7fdb74067410 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb7b95e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb74161950 con 0x7fdb740b3ce0 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb7b95e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdb741616e0 con 0x7fdb7415df40 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb74067410 0x7fdb7415f490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb74067410 0x7fdb7415f490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42514/0 (socket says 172.21.15.64:42514) 2024-03-04T07:30:33.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 -- 172.21.15.64:0/3587578055 learned_addr learned my addr 172.21.15.64:0/3587578055 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb78ef9640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb740b3ce0 0x7fdb741611a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb79efb640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb7415df40 0x7fdb740ae050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 -- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb7415df40 msgr2=0x7fdb740ae050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb7415df40 0x7fdb740ae050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 -- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb740b3ce0 msgr2=0x7fdb741611a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb740b3ce0 0x7fdb741611a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.537+0000 7fdb796fa640 1 -- 172.21.15.64:0/3587578055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb7415fab0 con 0x7fdb74067410 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.538+0000 7fdb796fa640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb74067410 0x7fdb7415f490 secure :-1 s=READY pgs=901 cs=0 l=1 rev1=1 crypto rx=0x7fdb7000c430 tx=0x7fdb7000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.538+0000 7fdb69ffb640 1 -- 172.21.15.64:0/3587578055 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb700074e0 con 0x7fdb74067410 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.538+0000 7fdb69ffb640 1 -- 172.21.15.64:0/3587578055 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdb700047e0 con 0x7fdb74067410 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.538+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdb7415fd90 con 0x7fdb74067410 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.538+0000 7fdb69ffb640 1 -- 172.21.15.64:0/3587578055 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb70004bd0 con 0x7fdb74067410 2024-03-04T07:30:33.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.538+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdb74160350 con 0x7fdb74067410 2024-03-04T07:30:33.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.539+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdb740733d0 con 0x7fdb74067410 2024-03-04T07:30:33.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.539+0000 7fdb69ffb640 1 -- 172.21.15.64:0/3587578055 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdb70011070 con 0x7fdb74067410 2024-03-04T07:30:33.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.540+0000 7fdb69ffb640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb5806b4d0 0x7fdb5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.540+0000 7fdb69ffb640 1 -- 172.21.15.64:0/3587578055 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fdb70089ca0 con 0x7fdb74067410 2024-03-04T07:30:33.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.540+0000 7fdb78ef9640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb5806b4d0 0x7fdb5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.540+0000 7fdb78ef9640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb5806b4d0 0x7fdb5806d990 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7fdb7406e450 tx=0x7fdb64009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.544+0000 7fdb69ffb640 1 -- 172.21.15.64:0/3587578055 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdb70058560 con 0x7fdb74067410 2024-03-04T07:30:33.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.728+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fdb740b5710 con 0x7fdb74067410 2024-03-04T07:30:33.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.729+0000 7fdb69ffb640 1 -- 172.21.15.64:0/3587578055 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v66) v1 ==== 100+0+11 (secure 0 0 0) 0x7fdb7005c600 con 0x7fdb74067410 2024-03-04T07:30:33.728 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.730+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb5806b4d0 msgr2=0x7fdb5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.730+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb5806b4d0 0x7fdb5806d990 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7fdb7406e450 tx=0x7fdb64009190 comp rx=0 tx=0).stop 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.730+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb74067410 msgr2=0x7fdb7415f490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.730+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb74067410 0x7fdb7415f490 secure :-1 s=READY pgs=901 cs=0 l=1 rev1=1 crypto rx=0x7fdb7000c430 tx=0x7fdb7000c8f0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 shutdown_connections 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdb5806b4d0 0x7fdb5806d990 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb7415df40 0x7fdb740ae050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb740b3ce0 0x7fdb741611a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 --2- 172.21.15.64:0/3587578055 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb74067410 0x7fdb7415f490 unknown :-1 s=CLOSED pgs=901 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 >> 172.21.15.64:0/3587578055 conn(0x7fdb740d09c0 msgr2=0x7fdb74066d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 shutdown_connections 2024-03-04T07:30:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.731+0000 7fdb7b95e640 1 -- 172.21.15.64:0/3587578055 wait complete. 2024-03-04T07:30:33.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:33.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.843+0000 7f5caf95e640 1 Processor -- start 2024-03-04T07:30:33.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.843+0000 7f5caf95e640 1 -- start start 2024-03-04T07:30:33.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5caf95e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d02e0 0x7f5ca80cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5caf95e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5ca80b4260 0x7f5ca80b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5caf95e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5ca80b0dd0 con 0x7f5ca80d06c0 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5caf95e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5ca80b0b80 con 0x7f5ca80b4260 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5caf95e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5ca80aeec0 con 0x7f5ca80d02e0 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cacef9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5ca80b4260 0x7f5ca80b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cacef9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5ca80b4260 0x7f5ca80b1500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42528/0 (socket says 172.21.15.64:42528) 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cacef9640 1 -- 172.21.15.64:0/3494263721 learned_addr learned my addr 172.21.15.64:0/3494263721 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cad6fa640 1 --2- 172.21.15.64:0/3494263721 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d02e0 0x7f5ca80cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cacef9640 1 -- 172.21.15.64:0/3494263721 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d02e0 msgr2=0x7f5ca80cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cacef9640 1 --2- 172.21.15.64:0/3494263721 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d02e0 0x7f5ca80cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cacef9640 1 -- 172.21.15.64:0/3494263721 >> v1:172.21.15.66:6789/0 conn(0x7f5ca80d06c0 legacy=0x7f5ca80b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.844+0000 7f5cacef9640 1 -- 172.21.15.64:0/3494263721 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ca80b10b0 con 0x7f5ca80b4260 2024-03-04T07:30:33.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.845+0000 7f5cacef9640 1 --2- 172.21.15.64:0/3494263721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca80b1500 secure :-1 s=READY pgs=902 cs=0 l=1 rev1=1 crypto rx=0x7f5c98009870 tx=0x7f5c9802ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7044a5ed14c1f8e9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.845+0000 7f5c9ffff640 1 -- 172.21.15.64:0/3494263721 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c9802f840 con 0x7f5ca80b4260 2024-03-04T07:30:33.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.845+0000 7f5c9ffff640 1 -- 172.21.15.64:0/3494263721 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c9802f9e0 con 0x7f5ca80b4260 2024-03-04T07:30:33.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.845+0000 7f5c9ffff640 1 -- 172.21.15.64:0/3494263721 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c9802fdb0 con 0x7f5ca80b4260 2024-03-04T07:30:33.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 -- 172.21.15.64:0/3494263721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 msgr2=0x7f5ca80b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3494263721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca80b1500 secure :-1 s=READY pgs=902 cs=0 l=1 rev1=1 crypto rx=0x7f5c98009870 tx=0x7f5c9802ba20 comp rx=0 tx=0).stop 2024-03-04T07:30:33.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 -- 172.21.15.64:0/3494263721 shutdown_connections 2024-03-04T07:30:33.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3494263721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca80b1500 unknown :-1 s=CLOSED pgs=902 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3494263721 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d02e0 0x7f5ca80cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 -- 172.21.15.64:0/3494263721 >> 172.21.15.64:0/3494263721 conn(0x7f5ca80b36d0 msgr2=0x7f5ca80b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:33.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 -- 172.21.15.64:0/3494263721 shutdown_connections 2024-03-04T07:30:33.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.846+0000 7f5caf95e640 1 -- 172.21.15.64:0/3494263721 wait complete. 2024-03-04T07:30:33.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.848+0000 7f5caf95e640 1 Processor -- start 2024-03-04T07:30:33.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.848+0000 7f5caf95e640 1 -- start start 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5caf95e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca8152850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5caf95e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ca80d02e0 0x7f5ca8141b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5caf95e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d06c0 0x7f5ca81f8400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5caf95e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ca80b22d0 con 0x7f5ca80b4260 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5caf95e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ca80b2150 con 0x7f5ca80d02e0 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5caf95e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5ca80af350 con 0x7f5ca80d06c0 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca8152850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca8152850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42540/0 (socket says 172.21.15.64:42540) 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 -- 172.21.15.64:0/3545357070 learned_addr learned my addr 172.21.15.64:0/3545357070 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cacef9640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ca80d02e0 0x7f5ca8141b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cadefb640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d06c0 0x7f5ca81f8400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 -- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d06c0 msgr2=0x7f5ca81f8400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d06c0 0x7f5ca81f8400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 -- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ca80d02e0 msgr2=0x7f5ca8141b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ca80d02e0 0x7f5ca8141b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 -- 172.21.15.64:0/3545357070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ca8155290 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cacef9640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ca80d02e0 0x7f5ca8141b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.849+0000 7f5cad6fa640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca8152850 secure :-1 s=READY pgs=903 cs=0 l=1 rev1=1 crypto rx=0x7f5ca400d800 tx=0x7f5ca400dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.850+0000 7f5c9dffb640 1 -- 172.21.15.64:0/3545357070 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ca400b6c0 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.850+0000 7f5c9dffb640 1 -- 172.21.15.64:0/3545357070 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5ca4019070 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.850+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5ca8155570 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.850+0000 7f5c9dffb640 1 -- 172.21.15.64:0/3545357070 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ca400ba80 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.850+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ca8143a50 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.851+0000 7f5c9dffb640 1 -- 172.21.15.64:0/3545357070 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5ca400bc20 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.851+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5ca80743c0 con 0x7f5ca80b4260 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.852+0000 7f5c9dffb640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c8c06b4d0 0x7f5c8c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:33.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.852+0000 7f5cacef9640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c8c06b4d0 0x7f5c8c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:33.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.852+0000 7f5c9dffb640 1 -- 172.21.15.64:0/3545357070 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5ca408a6f0 con 0x7f5ca80b4260 2024-03-04T07:30:33.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.852+0000 7f5cacef9640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c8c06b4d0 0x7f5c8c06d990 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f5c98036000 tx=0x7f5c98008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:33.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:33.856+0000 7f5c9dffb640 1 -- 172.21.15.64:0/3545357070 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5ca80743c0 con 0x7f5ca80b4260 2024-03-04T07:30:34.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.088+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5ca80725f0 con 0x7f5ca80b4260 2024-03-04T07:30:34.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.088+0000 7f5c9dffb640 1 -- 172.21.15.64:0/3545357070 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f5ca80725f0 con 0x7f5ca80b4260 2024-03-04T07:30:34.088 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:34.088 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:34.089 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:34.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c8c06b4d0 msgr2=0x7f5c8c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c8c06b4d0 0x7f5c8c06d990 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f5c98036000 tx=0x7f5c98008030 comp rx=0 tx=0).stop 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 msgr2=0x7f5ca8152850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca8152850 secure :-1 s=READY pgs=903 cs=0 l=1 rev1=1 crypto rx=0x7f5ca400d800 tx=0x7f5ca400dcc0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 shutdown_connections 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c8c06b4d0 0x7f5c8c06d990 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5ca80d06c0 0x7f5ca81f8400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5ca80d02e0 0x7f5ca8141b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.091+0000 7f5caf95e640 1 --2- 172.21.15.64:0/3545357070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5ca80b4260 0x7f5ca8152850 unknown :-1 s=CLOSED pgs=903 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.092+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 >> 172.21.15.64:0/3545357070 conn(0x7f5ca80b36d0 msgr2=0x7f5ca80b1740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.092+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 shutdown_connections 2024-03-04T07:30:34.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.092+0000 7f5caf95e640 1 -- 172.21.15.64:0/3545357070 wait complete. 2024-03-04T07:30:34.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.211+0000 7fc56deed640 1 Processor -- start 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.211+0000 7fc56deed640 1 -- start start 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.211+0000 7fc56deed640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc5680b95a0 0x7fc5680b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.211+0000 7fc56deed640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5680cea30 con 0x7fc5680b9a10 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.211+0000 7fc56deed640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5680cf3c0 con 0x7fc56815df00 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.211+0000 7fc56deed640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5680cf0e0 con 0x7fc5680b95a0 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.211+0000 7fc567fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc5680b9a10 0x7fc5680d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50414/0 (socket says 172.21.15.64:50414) 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc567fff640 1 -- 172.21.15.64:0/2003348682 learned_addr learned my addr 172.21.15.64:0/2003348682 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc566ffd640 1 --2- 172.21.15.64:0/2003348682 >> v2:172.21.15.66:3300/0 conn(0x7fc5680b95a0 0x7fc5680b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc5667fc640 1 -- 172.21.15.64:0/2003348682 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3605356351 0 0) 0x7fc5680cea30 con 0x7fc5680b9a10 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc5667fc640 1 -- 172.21.15.64:0/2003348682 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc550003520 con 0x7fc5680b9a10 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc566ffd640 1 -- 172.21.15.64:0/2003348682 >> v1:172.21.15.66:6789/0 conn(0x7fc56815df00 legacy=0x7fc5680d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc566ffd640 1 -- 172.21.15.64:0/2003348682 >> v1:172.21.15.64:6789/0 conn(0x7fc5680b9a10 legacy=0x7fc5680d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc5667fc640 1 -- 172.21.15.64:0/2003348682 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2676442225 0 0) 0x7fc550003520 con 0x7fc5680b9a10 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.212+0000 7fc566ffd640 1 -- 172.21.15.64:0/2003348682 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5680ce470 con 0x7fc5680b95a0 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.213+0000 7fc566ffd640 1 --2- 172.21.15.64:0/2003348682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc5680b5700 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7fc5680cf590 tx=0x7fc55c02b810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e301271b355b983 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.213+0000 7fc5667fc640 1 -- 172.21.15.64:0/2003348682 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc55c039070 con 0x7fc5680b95a0 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.213+0000 7fc5667fc640 1 -- 172.21.15.64:0/2003348682 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc55c0314f0 con 0x7fc5680b95a0 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.213+0000 7fc5667fc640 1 -- 172.21.15.64:0/2003348682 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5600019f0 con 0x7fc5680b95a0 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.213+0000 7fc56deed640 1 -- 172.21.15.64:0/2003348682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 msgr2=0x7fc5680b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.214+0000 7fc56deed640 1 --2- 172.21.15.64:0/2003348682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc5680b5700 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7fc5680cf590 tx=0x7fc55c02b810 comp rx=0 tx=0).stop 2024-03-04T07:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.214+0000 7fc56deed640 1 -- 172.21.15.64:0/2003348682 shutdown_connections 2024-03-04T07:30:34.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.214+0000 7fc56deed640 1 --2- 172.21.15.64:0/2003348682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc5680b5700 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.214+0000 7fc56deed640 1 -- 172.21.15.64:0/2003348682 >> 172.21.15.64:0/2003348682 conn(0x7fc568071960 msgr2=0x7fc568071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:34.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.214+0000 7fc56deed640 1 -- 172.21.15.64:0/2003348682 shutdown_connections 2024-03-04T07:30:34.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.214+0000 7fc56deed640 1 -- 172.21.15.64:0/2003348682 wait complete. 2024-03-04T07:30:34.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.215+0000 7fc56deed640 1 Processor -- start 2024-03-04T07:30:34.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.215+0000 7fc56deed640 1 -- start start 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc56deed640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc56815bba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc56deed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5680b9a10 0x7fc5680c77b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc56deed640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc56815df00 0x7fc56815e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc56deed640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5680cedf0 con 0x7fc56815df00 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc56deed640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5680cec70 con 0x7fc5680b95a0 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc56deed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5680b5c40 con 0x7fc5680b9a10 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc56815df00 0x7fc56815e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc56815df00 0x7fc56815e2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42542/0 (socket says 172.21.15.64:42542) 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 -- 172.21.15.64:0/2238468049 learned_addr learned my addr 172.21.15.64:0/2238468049 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc5677fe640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc56815bba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc566ffd640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5680b9a10 0x7fc5680c77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 -- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5680b9a10 msgr2=0x7fc5680c77b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5680b9a10 0x7fc5680c77b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 -- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 msgr2=0x7fc56815bba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc56815bba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 -- 172.21.15.64:0/2238468049 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc56815ee50 con 0x7fc56815df00 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc5677fe640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc56815bba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc566ffd640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5680b9a10 0x7fc5680c77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc567fff640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc56815df00 0x7fc56815e2e0 secure :-1 s=READY pgs=905 cs=0 l=1 rev1=1 crypto rx=0x7fc560001890 tx=0x7fc56000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc54ffff640 1 -- 172.21.15.64:0/2238468049 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc560012400 con 0x7fc56815df00 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc54ffff640 1 -- 172.21.15.64:0/2238468049 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5600125c0 con 0x7fc56815df00 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc56815f130 con 0x7fc56815df00 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.216+0000 7fc54ffff640 1 -- 172.21.15.64:0/2238468049 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5600129b0 con 0x7fc56815df00 2024-03-04T07:30:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.217+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5681600f0 con 0x7fc56815df00 2024-03-04T07:30:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.218+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5680b7d80 con 0x7fc56815df00 2024-03-04T07:30:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.218+0000 7fc54ffff640 1 -- 172.21.15.64:0/2238468049 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc560012b50 con 0x7fc56815df00 2024-03-04T07:30:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.218+0000 7fc54ffff640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc55406b4d0 0x7fc55406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.218+0000 7fc54ffff640 1 -- 172.21.15.64:0/2238468049 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(66..66 src has 1..66) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc56008aa80 con 0x7fc56815df00 2024-03-04T07:30:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.218+0000 7fc5677fe640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc55406b4d0 0x7fc55406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.219+0000 7fc5677fe640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc55406b4d0 0x7fc55406d990 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7fc558000f60 tx=0x7fc558009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.223+0000 7fc54ffff640 1 -- 172.21.15.64:0/2238468049 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc5680b7d80 con 0x7fc56815df00 2024-03-04T07:30:34.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.407+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc568068ad0 con 0x7fc56815df00 2024-03-04T07:30:34.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.408+0000 7fc54ffff640 1 -- 172.21.15.64:0/2238468049 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v67) v1 ==== 74+0+16717 (secure 0 0 0) 0x7fc568068ad0 con 0x7fc56815df00 2024-03-04T07:30:34.408 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:34.409 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":67,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:34.268776+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:02.706399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:03.395544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"11.4","mappings":[{"from":6,"to":0}]},{"pgid":"11.2e","mappings":[{"from":6,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:34.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc55406b4d0 msgr2=0x7fc55406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc55406b4d0 0x7fc55406d990 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7fc558000f60 tx=0x7fc558009040 comp rx=0 tx=0).stop 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc56815df00 msgr2=0x7fc56815e2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc56815df00 0x7fc56815e2e0 secure :-1 s=READY pgs=905 cs=0 l=1 rev1=1 crypto rx=0x7fc560001890 tx=0x7fc56000a430 comp rx=0 tx=0).stop 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 shutdown_connections 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc55406b4d0 0x7fc55406d990 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc56815df00 0x7fc56815e2e0 unknown :-1 s=CLOSED pgs=905 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5680b9a10 0x7fc5680c77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 --2- 172.21.15.64:0/2238468049 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5680b95a0 0x7fc56815bba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.410+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 >> 172.21.15.64:0/2238468049 conn(0x7fc568071960 msgr2=0x7fc568071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.411+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 shutdown_connections 2024-03-04T07:30:34.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.411+0000 7fc56deed640 1 -- 172.21.15.64:0/2238468049 wait complete. 2024-03-04T07:30:34.421 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:30:34.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:34.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.527+0000 7f032ec15640 1 Processor -- start 2024-03-04T07:30:34.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.527+0000 7f032ec15640 1 -- start start 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.528+0000 7f032ec15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0328158520 0x7f032806d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.528+0000 7f032ec15640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0328071240 con 0x7f032806dd60 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.528+0000 7f032ec15640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0328071520 con 0x7f03280aee10 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.528+0000 7f032ec15640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f032806e1c0 con 0x7f0328158520 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.528+0000 7f0327fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f032806dd60 0x7f03280ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50420/0 (socket says 172.21.15.64:50420) 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.528+0000 7f0327fff640 1 -- 172.21.15.64:0/3127443538 learned_addr learned my addr 172.21.15.64:0/3127443538 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.528+0000 7f032c9b1640 1 --2- 172.21.15.64:0/3127443538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0328158520 0x7f032806d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1513578163 0 0) 0x7f0328071240 con 0x7f032806dd60 2024-03-04T07:30:34.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0314003520 con 0x7f032806dd60 2024-03-04T07:30:34.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3568976524 0 0) 0x7f0328071520 con 0x7f03280aee10 2024-03-04T07:30:34.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0328071240 con 0x7f03280aee10 2024-03-04T07:30:34.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4013801891 0 0) 0x7f0314003520 con 0x7f032806dd60 2024-03-04T07:30:34.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f032c9b1640 1 -- 172.21.15.64:0/3127443538 >> v1:172.21.15.66:6789/0 conn(0x7f03280aee10 legacy=0x7f03280b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f032c9b1640 1 -- 172.21.15.64:0/3127443538 >> v1:172.21.15.64:6789/0 conn(0x7f032806dd60 legacy=0x7f03280ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.529+0000 7f032c9b1640 1 -- 172.21.15.64:0/3127443538 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0328071800 con 0x7f0328158520 2024-03-04T07:30:34.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032c9b1640 1 --2- 172.21.15.64:0/3127443538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0328158520 0x7f032806d820 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0320009dc0 tx=0x7f032002bde0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c136337b1606293b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0320039070 con 0x7f0328158520 2024-03-04T07:30:34.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f032002c910 con 0x7f0328158520 2024-03-04T07:30:34.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f03277fe640 1 -- 172.21.15.64:0/3127443538 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03180015f0 con 0x7f0328158520 2024-03-04T07:30:34.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032ec15640 1 -- 172.21.15.64:0/3127443538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0328158520 msgr2=0x7f032806d820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032ec15640 1 --2- 172.21.15.64:0/3127443538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0328158520 0x7f032806d820 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0320009dc0 tx=0x7f032002bde0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032ec15640 1 -- 172.21.15.64:0/3127443538 shutdown_connections 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032ec15640 1 --2- 172.21.15.64:0/3127443538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0328158520 0x7f032806d820 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032ec15640 1 -- 172.21.15.64:0/3127443538 >> 172.21.15.64:0/3127443538 conn(0x7f0328066d00 msgr2=0x7f03280670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032ec15640 1 -- 172.21.15.64:0/3127443538 shutdown_connections 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.530+0000 7f032ec15640 1 -- 172.21.15.64:0/3127443538 wait complete. 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 Processor -- start 2024-03-04T07:30:34.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 -- start start 2024-03-04T07:30:34.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f032806dd60 0x7f03281483d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03280aee10 0x7f0328156650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0328158520 0x7f03280ac970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0328158900 con 0x7f0328158520 2024-03-04T07:30:34.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0328071b70 con 0x7f032806dd60 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.532+0000 7f032ec15640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0328158a80 con 0x7f03280aee10 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0328158520 0x7f03280ac970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0328158520 0x7f03280ac970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42550/0 (socket says 172.21.15.64:42550) 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 -- 172.21.15.64:0/4102657297 learned_addr learned my addr 172.21.15.64:0/4102657297 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032c9b1640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f032806dd60 0x7f03281483d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f0327fff640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03280aee10 0x7f0328156650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 -- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03280aee10 msgr2=0x7f0328156650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03280aee10 0x7f0328156650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 -- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f032806dd60 msgr2=0x7f03281483d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f032806dd60 0x7f03281483d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 -- 172.21.15.64:0/4102657297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03281575c0 con 0x7f0328158520 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032c9b1640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f032806dd60 0x7f03281483d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032d1b2640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0328158520 0x7f03280ac970 secure :-1 s=READY pgs=907 cs=0 l=1 rev1=1 crypto rx=0x7f031000c490 tx=0x7f031000c950 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f03257fa640 1 -- 172.21.15.64:0/4102657297 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03100052d0 con 0x7f0328158520 2024-03-04T07:30:34.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f03257fa640 1 -- 172.21.15.64:0/4102657297 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0310005470 con 0x7f0328158520 2024-03-04T07:30:34.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f03257fa640 1 -- 172.21.15.64:0/4102657297 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0310012870 con 0x7f0328158520 2024-03-04T07:30:34.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03280ad880 con 0x7f0328158520 2024-03-04T07:30:34.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.533+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03280ade40 con 0x7f0328158520 2024-03-04T07:30:34.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.534+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0328067ff0 con 0x7f0328158520 2024-03-04T07:30:34.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.535+0000 7f03257fa640 1 -- 172.21.15.64:0/4102657297 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0310012a10 con 0x7f0328158520 2024-03-04T07:30:34.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.535+0000 7f03257fa640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f030806b4d0 0x7f030806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.535+0000 7f03257fa640 1 -- 172.21.15.64:0/4102657297 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(67..67 src has 1..67) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0310089f10 con 0x7f0328158520 2024-03-04T07:30:34.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.535+0000 7f032c9b1640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f030806b4d0 0x7f030806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.536+0000 7f032c9b1640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f030806b4d0 0x7f030806d990 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f0320036000 tx=0x7f0320030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.539+0000 7f03257fa640 1 -- 172.21.15.64:0/4102657297 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0310058d50 con 0x7f0328158520 2024-03-04T07:30:34.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.816+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f032806da80 con 0x7f0328158520 2024-03-04T07:30:34.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.817+0000 7f03257fa640 1 -- 172.21.15.64:0/4102657297 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f0310058650 con 0x7f0328158520 2024-03-04T07:30:34.817 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:34.817 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:34.817 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f030806b4d0 msgr2=0x7f030806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f030806b4d0 0x7f030806d990 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f0320036000 tx=0x7f0320030000 comp rx=0 tx=0).stop 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0328158520 msgr2=0x7f03280ac970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0328158520 0x7f03280ac970 secure :-1 s=READY pgs=907 cs=0 l=1 rev1=1 crypto rx=0x7f031000c490 tx=0x7f031000c950 comp rx=0 tx=0).stop 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 shutdown_connections 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f030806b4d0 0x7f030806d990 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0328158520 0x7f03280ac970 unknown :-1 s=CLOSED pgs=907 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03280aee10 0x7f0328156650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 --2- 172.21.15.64:0/4102657297 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f032806dd60 0x7f03281483d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 >> 172.21.15.64:0/4102657297 conn(0x7f0328066d00 msgr2=0x7f03280b44a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:34.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 shutdown_connections 2024-03-04T07:30:34.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.819+0000 7f032ec15640 1 -- 172.21.15.64:0/4102657297 wait complete. 2024-03-04T07:30:34.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:34.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.931+0000 7f6fb5dac640 1 Processor -- start 2024-03-04T07:30:34.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.931+0000 7f6fb5dac640 1 -- start start 2024-03-04T07:30:34.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.932+0000 7f6fb5dac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6fb006b5c0 0x7f6fb00b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.932+0000 7f6fb5dac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00b0e90 0x7f6fb006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.932+0000 7f6fb5dac640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6fb00b3340 con 0x7f6fb00ae9b0 2024-03-04T07:30:34.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.932+0000 7f6fb5dac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6fb00cfa00 con 0x7f6fb006b5c0 2024-03-04T07:30:34.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.932+0000 7f6fb5dac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb00cee90 con 0x7f6fb00b0e90 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.932+0000 7f6faffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6fb00ae9b0 0x7f6fb00d03a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50432/0 (socket says 172.21.15.64:50432) 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.932+0000 7f6faffff640 1 -- 172.21.15.64:0/1495490128 learned_addr learned my addr 172.21.15.64:0/1495490128 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/1495490128 >> v2:172.21.15.66:3300/0 conn(0x7f6fb006b5c0 0x7f6fb00b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6faeffd640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00b0e90 0x7f6fb006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 985280672 0 0) 0x7f6fb00b3340 con 0x7f6fb00ae9b0 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6f98003520 con 0x7f6fb00ae9b0 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3081503948 0 0) 0x7f6f98003520 con 0x7f6fb00ae9b0 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00b0e90 msgr2=0x7f6fb006b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00b0e90 0x7f6fb006b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 >> v2:172.21.15.66:3300/0 conn(0x7f6fb006b5c0 msgr2=0x7f6fb00b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 --2- 172.21.15.64:0/1495490128 >> v2:172.21.15.66:3300/0 conn(0x7f6fb006b5c0 0x7f6fb00b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fb00cfc50 con 0x7f6fb00ae9b0 2024-03-04T07:30:34.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f6fa80019f0 con 0x7f6fb00ae9b0 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 >> v1:172.21.15.64:6789/0 conn(0x7f6fb00ae9b0 legacy=0x7f6fb00d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6f98003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f98004800 0x7f6f98004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f980051c0 0x7f6f9800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb00cfa00 con 0x7f6f980039b0 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb00cee90 con 0x7f6f980051c0 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6f9800bf50 con 0x7f6f98004800 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6faeffd640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00b0e90 0x7f6fb006b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/1495490128 >> v2:172.21.15.66:3300/0 conn(0x7f6fb006b5c0 0x7f6fb00b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6f98003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6faffff640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f980051c0 0x7f6f9800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.933+0000 7f6faeffd640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f98004800 0x7f6f98004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6faf7fe640 1 -- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f98004800 msgr2=0x7f6f98004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f98004800 0x7f6f98004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6faf7fe640 1 -- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f980051c0 msgr2=0x7f6f9800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f980051c0 0x7f6f9800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6faf7fe640 1 -- 172.21.15.64:0/1495490128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f98003520 con 0x7f6f980039b0 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6f98003dd0 secure :-1 s=READY pgs=909 cs=0 l=1 rev1=1 crypto rx=0x7f6fa0009ca0 tx=0x7f6f98005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d82f265a9e44d3e6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fa0009840 con 0x7f6f980039b0 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.934+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fa0004020 con 0x7f6f980039b0 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6fae7fc640 1 -- 172.21.15.64:0/1495490128 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fa0004410 con 0x7f6f980039b0 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6fb5dac640 1 -- 172.21.15.64:0/1495490128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 msgr2=0x7f6f98003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6fb5dac640 1 --2- 172.21.15.64:0/1495490128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6f98003dd0 secure :-1 s=READY pgs=909 cs=0 l=1 rev1=1 crypto rx=0x7f6fa0009ca0 tx=0x7f6f98005830 comp rx=0 tx=0).stop 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6faffff640 1 -- 172.21.15.64:0/1495490128 reap_dead start 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6fb5dac640 1 -- 172.21.15.64:0/1495490128 shutdown_connections 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6fb5dac640 1 -- 172.21.15.64:0/1495490128 >> 172.21.15.64:0/1495490128 conn(0x7f6fb00b36d0 msgr2=0x7f6fb00b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6fb5dac640 1 -- 172.21.15.64:0/1495490128 shutdown_connections 2024-03-04T07:30:34.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.935+0000 7f6fb5dac640 1 -- 172.21.15.64:0/1495490128 wait complete. 2024-03-04T07:30:34.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.937+0000 7f6fb5dac640 1 Processor -- start 2024-03-04T07:30:34.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.937+0000 7f6fb5dac640 1 -- start start 2024-03-04T07:30:34.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6fb5dac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6fb01d1810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6fb5dac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb01e7e80 0x7f6fb01d48f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6fb5dac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00c8bd0 0x7f6fb00c9050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6fb5dac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb006c910 con 0x7f6f980039b0 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6fb5dac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb006c790 con 0x7f6fb01e7e80 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6fb5dac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb00d0ab0 con 0x7f6fb00c8bd0 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6faf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6fb01d1810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6faf7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6fb01d1810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42578/0 (socket says 172.21.15.64:42578) 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.938+0000 7f6faf7fe640 1 -- 172.21.15.64:0/2542754416 learned_addr learned my addr 172.21.15.64:0/2542754416 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faffff640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00c8bd0 0x7f6fb00c9050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faeffd640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb01e7e80 0x7f6fb01d48f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faf7fe640 1 -- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00c8bd0 msgr2=0x7f6fb00c9050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00c8bd0 0x7f6fb00c9050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faf7fe640 1 -- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb01e7e80 msgr2=0x7f6fb01d48f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:34.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb01e7e80 0x7f6fb01d48f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faf7fe640 1 -- 172.21.15.64:0/2542754416 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fb01e6de0 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6faf7fe640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6fb01d1810 secure :-1 s=READY pgs=910 cs=0 l=1 rev1=1 crypto rx=0x7f6fa0036040 tx=0x7f6fa0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6f97fff640 1 -- 172.21.15.64:0/2542754416 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fa0043020 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6f97fff640 1 -- 172.21.15.64:0/2542754416 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fa002c960 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6fb01e70c0 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6f97fff640 1 -- 172.21.15.64:0/2542754416 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fa0006e10 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.939+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6fb01e7680 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.940+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6fb0068700 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.941+0000 7f6f97fff640 1 -- 172.21.15.64:0/2542754416 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6fa0046020 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.941+0000 7f6f97fff640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9c06b500 0x7f6f9c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.941+0000 7f6f97fff640 1 -- 172.21.15.64:0/2542754416 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(67..67 src has 1..67) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6fa00ac040 con 0x7f6f980039b0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.941+0000 7f6faeffd640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9c06b500 0x7f6f9c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:34.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.942+0000 7f6faeffd640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9c06b500 0x7f6f9c06d9c0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f6fb01f7320 tx=0x7f6fa4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:34.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:34.946+0000 7f6f97fff640 1 -- 172.21.15.64:0/2542754416 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6fa004d030 con 0x7f6f980039b0 2024-03-04T07:30:35.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.240+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6fb00b24e0 con 0x7f6f980039b0 2024-03-04T07:30:35.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.240+0000 7f6f97fff640 1 -- 172.21.15.64:0/2542754416 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 36 v36) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f6fa007a830 con 0x7f6f980039b0 2024-03-04T07:30:35.240 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:35.240 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":36,"default_fscid":5,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33}],"filesystems":[{"mdsmap":{"epoch":36,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:04.437991+0000","modified":"2024-03-04T07:30:06.496433+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6546},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6546":{"gid":6546,"name":"a","rank":0,"incarnation":34,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6837/2304674412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2304674412},{"type":"v1","addr":"172.21.15.64:6837","nonce":2304674412}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[11],"metadata_pool":10,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6546,"qdb_cluster":[6546]},"id":5}]} 2024-03-04T07:30:35.240 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 36 2024-03-04T07:30:35.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9c06b500 msgr2=0x7f6f9c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9c06b500 0x7f6f9c06d9c0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f6fb01f7320 tx=0x7f6fa4009190 comp rx=0 tx=0).stop 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 msgr2=0x7f6fb01d1810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6fb01d1810 secure :-1 s=READY pgs=910 cs=0 l=1 rev1=1 crypto rx=0x7f6fa0036040 tx=0x7f6fa0004730 comp rx=0 tx=0).stop 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 shutdown_connections 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9c06b500 0x7f6f9c06d9c0 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb00c8bd0 0x7f6fb00c9050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb01e7e80 0x7f6fb01d48f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 --2- 172.21.15.64:0/2542754416 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f980039b0 0x7f6fb01d1810 unknown :-1 s=CLOSED pgs=910 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.242+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 >> 172.21.15.64:0/2542754416 conn(0x7f6fb00b36d0 msgr2=0x7f6fb00b2ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.243+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 shutdown_connections 2024-03-04T07:30:35.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.243+0000 7f6fb5dac640 1 -- 172.21.15.64:0/2542754416 wait complete. 2024-03-04T07:30:35.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:35.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.357+0000 7f2eee44e640 1 Processor -- start 2024-03-04T07:30:35.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.357+0000 7f2eee44e640 1 -- start start 2024-03-04T07:30:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2eee44e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2eee44e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2eee44e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2ee80cea30 con 0x7f2ee80b9a30 2024-03-04T07:30:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2eee44e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2ee80d2af0 con 0x7f2ee815df00 2024-03-04T07:30:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2eee44e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ee80cf3c0 con 0x7f2ee80b95a0 2024-03-04T07:30:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee7fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee7fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42594/0 (socket says 172.21.15.64:42594) 2024-03-04T07:30:35.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee7fff640 1 -- 172.21.15.64:0/2324090620 learned_addr learned my addr 172.21.15.64:0/2324090620 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:35.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee77fe640 1 --2- 172.21.15.64:0/2324090620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee7fff640 1 -- 172.21.15.64:0/2324090620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80b95a0 msgr2=0x7f2ee80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:35.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2324090620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee6ffd640 1 -- 172.21.15.64:0/2324090620 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3038616390 0 0) 0x7f2ee80cea30 con 0x7f2ee80b9a30 2024-03-04T07:30:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee7fff640 1 -- 172.21.15.64:0/2324090620 >> v1:172.21.15.64:6789/0 conn(0x7f2ee80b9a30 legacy=0x7f2ee80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee7fff640 1 -- 172.21.15.64:0/2324090620 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ee80cf0e0 con 0x7f2ee815df00 2024-03-04T07:30:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.358+0000 7f2ee77fe640 1 --2- 172.21.15.64:0/2324090620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.359+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2324090620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 0x7f2ee80d25b0 secure :-1 s=READY pgs=912 cs=0 l=1 rev1=1 crypto rx=0x7f2ed8009870 tx=0x7f2ed802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3cd5a0ce670ae8e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.359+0000 7f2ee6ffd640 1 -- 172.21.15.64:0/2324090620 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ed8034070 con 0x7f2ee815df00 2024-03-04T07:30:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.359+0000 7f2ee6ffd640 1 -- 172.21.15.64:0/2324090620 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2ed8030840 con 0x7f2ee815df00 2024-03-04T07:30:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.359+0000 7f2ee6ffd640 1 -- 172.21.15.64:0/2324090620 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ed8030c10 con 0x7f2ee815df00 2024-03-04T07:30:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.359+0000 7f2eee44e640 1 -- 172.21.15.64:0/2324090620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 msgr2=0x7f2ee80d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.359+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2324090620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 0x7f2ee80d25b0 secure :-1 s=READY pgs=912 cs=0 l=1 rev1=1 crypto rx=0x7f2ed8009870 tx=0x7f2ed802b980 comp rx=0 tx=0).stop 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.360+0000 7f2eee44e640 1 -- 172.21.15.64:0/2324090620 shutdown_connections 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.360+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2324090620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.360+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2324090620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=CLOSED pgs=912 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.360+0000 7f2eee44e640 1 -- 172.21.15.64:0/2324090620 >> 172.21.15.64:0/2324090620 conn(0x7f2ee8071960 msgr2=0x7f2ee8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.360+0000 7f2eee44e640 1 -- 172.21.15.64:0/2324090620 shutdown_connections 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.360+0000 7f2eee44e640 1 -- 172.21.15.64:0/2324090620 wait complete. 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.361+0000 7f2eee44e640 1 Processor -- start 2024-03-04T07:30:35.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.361+0000 7f2eee44e640 1 -- start start 2024-03-04T07:30:35.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eee44e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee80b95a0 0x7f2ee8158510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eee44e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee815df00 0x7f2ee80b4a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eee44e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b4f70 0x7f2ee81f72d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eee44e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ee80cec70 con 0x7f2ee80b4f70 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eee44e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ee80ce750 con 0x7f2ee80b95a0 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eee44e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ee80cedf0 con 0x7f2ee815df00 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b4f70 0x7f2ee81f72d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b4f70 0x7f2ee81f72d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42604/0 (socket says 172.21.15.64:42604) 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 -- 172.21.15.64:0/2581109134 learned_addr learned my addr 172.21.15.64:0/2581109134 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee80b95a0 0x7f2ee8158510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2ee77fe640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee815df00 0x7f2ee80b4a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 -- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee815df00 msgr2=0x7f2ee80b4a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee815df00 0x7f2ee80b4a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 -- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee80b95a0 msgr2=0x7f2ee8158510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee80b95a0 0x7f2ee8158510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 -- 172.21.15.64:0/2581109134 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ee8158a50 con 0x7f2ee80b4f70 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2eec9eb640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b4f70 0x7f2ee81f72d0 secure :-1 s=READY pgs=913 cs=0 l=1 rev1=1 crypto rx=0x7f2ee000c760 tx=0x7f2ee000cc20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2ee4ff9640 1 -- 172.21.15.64:0/2581109134 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ee0003da0 con 0x7f2ee80b4f70 2024-03-04T07:30:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.362+0000 7f2ee4ff9640 1 -- 172.21.15.64:0/2581109134 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2ee0005070 con 0x7f2ee80b4f70 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.363+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ee81f3cb0 con 0x7f2ee80b4f70 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.363+0000 7f2ee4ff9640 1 -- 172.21.15.64:0/2581109134 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ee0005460 con 0x7f2ee80b4f70 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.363+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ee81f4270 con 0x7f2ee80b4f70 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.363+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ee8067ff0 con 0x7f2ee80b4f70 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.364+0000 7f2ee4ff9640 1 -- 172.21.15.64:0/2581109134 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2ee0005600 con 0x7f2ee80b4f70 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.364+0000 7f2ee4ff9640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ed406b4d0 0x7f2ed406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.365+0000 7f2ee4ff9640 1 -- 172.21.15.64:0/2581109134 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(68..68 src has 1..68) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f2ee0089f40 con 0x7f2ee80b4f70 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.365+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ed406b4d0 0x7f2ed406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.365+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ed406b4d0 0x7f2ed406d990 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f2ed8036000 tx=0x7f2ed803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.369+0000 7f2ee4ff9640 1 -- 172.21.15.64:0/2581109134 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2ee00587c0 con 0x7f2ee80b4f70 2024-03-04T07:30:35.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.552+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2ee80ca870 con 0x7f2ee80b4f70 2024-03-04T07:30:35.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.554+0000 7f2ee4ff9640 1 -- 172.21.15.64:0/2581109134 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v68) v1 ==== 74+0+16717 (secure 0 0 0) 0x7f2ee0081e20 con 0x7f2ee80b4f70 2024-03-04T07:30:35.554 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:35.554 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":68,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:35.270434+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":10,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:02.706399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":11,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:03.395544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"11.4","mappings":[{"from":6,"to":0}]},{"pgid":"11.2e","mappings":[{"from":6,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:35.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.555+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ed406b4d0 msgr2=0x7f2ed406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.555+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ed406b4d0 0x7f2ed406d990 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f2ed8036000 tx=0x7f2ed803b040 comp rx=0 tx=0).stop 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.555+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b4f70 msgr2=0x7f2ee81f72d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.555+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b4f70 0x7f2ee81f72d0 secure :-1 s=READY pgs=913 cs=0 l=1 rev1=1 crypto rx=0x7f2ee000c760 tx=0x7f2ee000cc20 comp rx=0 tx=0).stop 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 shutdown_connections 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ed406b4d0 0x7f2ed406d990 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b4f70 0x7f2ee81f72d0 unknown :-1 s=CLOSED pgs=913 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee815df00 0x7f2ee80b4a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 --2- 172.21.15.64:0/2581109134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee80b95a0 0x7f2ee8158510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 >> 172.21.15.64:0/2581109134 conn(0x7f2ee8071960 msgr2=0x7f2ee80b8550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 shutdown_connections 2024-03-04T07:30:35.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.556+0000 7f2eee44e640 1 -- 172.21.15.64:0/2581109134 wait complete. 2024-03-04T07:30:35.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:30:35.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.673+0000 7f6fbbefa640 1 Processor -- start 2024-03-04T07:30:35.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.673+0000 7f6fbbefa640 1 -- start start 2024-03-04T07:30:35.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fbbefa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb40cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fbbefa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6fb40b4260 0x7f6fb40b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fbbefa640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6fb40b0dd0 con 0x7f6fb40d06c0 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fbbefa640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6fb40b0b80 con 0x7f6fb40b4260 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fbbefa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb40aeec0 con 0x7f6fb40d02e0 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fb9495640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6fb40b4260 0x7f6fb40b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fb9495640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6fb40b4260 0x7f6fb40b1500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42616/0 (socket says 172.21.15.64:42616) 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fba497640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6fb40d06c0 0x7f6fb40b1a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50462/0 (socket says 172.21.15.64:50462) 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.674+0000 7f6fb9495640 1 -- 172.21.15.64:0/171061423 learned_addr learned my addr 172.21.15.64:0/171061423 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.675+0000 7f6fb9c96640 1 --2- 172.21.15.64:0/171061423 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb40cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.675+0000 7f6fb9495640 1 -- 172.21.15.64:0/171061423 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 msgr2=0x7f6fb40cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.675+0000 7f6fb9495640 1 --2- 172.21.15.64:0/171061423 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb40cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.675+0000 7f6fb9495640 1 -- 172.21.15.64:0/171061423 >> v1:172.21.15.64:6789/0 conn(0x7f6fb40d06c0 legacy=0x7f6fb40b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.675+0000 7f6fb9495640 1 -- 172.21.15.64:0/171061423 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fb40b10b0 con 0x7f6fb40b4260 2024-03-04T07:30:35.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.675+0000 7f6fb9c96640 1 --2- 172.21.15.64:0/171061423 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb40cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.675+0000 7f6fb9495640 1 --2- 172.21.15.64:0/171061423 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40b1500 secure :-1 s=READY pgs=915 cs=0 l=1 rev1=1 crypto rx=0x7f6fa4009870 tx=0x7f6fa402ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=375b5d064103f376 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fb8c94640 1 -- 172.21.15.64:0/171061423 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fa402f840 con 0x7f6fb40b4260 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fb8c94640 1 -- 172.21.15.64:0/171061423 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fa402f9e0 con 0x7f6fb40b4260 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fb8c94640 1 -- 172.21.15.64:0/171061423 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fa402fdb0 con 0x7f6fb40b4260 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 -- 172.21.15.64:0/171061423 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 msgr2=0x7f6fb40b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/171061423 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40b1500 secure :-1 s=READY pgs=915 cs=0 l=1 rev1=1 crypto rx=0x7f6fa4009870 tx=0x7f6fa402ba20 comp rx=0 tx=0).stop 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 -- 172.21.15.64:0/171061423 shutdown_connections 2024-03-04T07:30:35.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/171061423 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40b1500 unknown :-1 s=CLOSED pgs=915 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/171061423 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb40cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 -- 172.21.15.64:0/171061423 >> 172.21.15.64:0/171061423 conn(0x7f6fb40b36d0 msgr2=0x7f6fb40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:35.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 -- 172.21.15.64:0/171061423 shutdown_connections 2024-03-04T07:30:35.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.676+0000 7f6fbbefa640 1 -- 172.21.15.64:0/171061423 wait complete. 2024-03-04T07:30:35.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.677+0000 7f6fbbefa640 1 Processor -- start 2024-03-04T07:30:35.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.677+0000 7f6fbbefa640 1 -- start start 2024-03-04T07:30:35.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fbbefa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40c72f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fbbefa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb41e6140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fbbefa640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb40d06c0 0x7f6fb41e6680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fbbefa640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb40b22d0 con 0x7f6fb40b4260 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fbbefa640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb40b2150 con 0x7f6fb40d06c0 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fbbefa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6fb40af350 con 0x7f6fb40d02e0 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fb9c96640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40c72f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fb9c96640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40c72f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42622/0 (socket says 172.21.15.64:42622) 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fb9c96640 1 -- 172.21.15.64:0/755524731 learned_addr learned my addr 172.21.15.64:0/755524731 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fba497640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb40d06c0 0x7f6fb41e6680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fb9c96640 1 -- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 msgr2=0x7f6fb41e6140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.678+0000 7f6fb9495640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb41e6140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fb9c96640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb41e6140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fb9c96640 1 -- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb40d06c0 msgr2=0x7f6fb41e6680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fb9c96640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb40d06c0 0x7f6fb41e6680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fb9c96640 1 -- 172.21.15.64:0/755524731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fb40c9750 con 0x7f6fb40b4260 2024-03-04T07:30:35.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fba497640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb40d06c0 0x7f6fb41e6680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fb9495640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb41e6140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fb9c96640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40c72f0 secure :-1 s=READY pgs=916 cs=0 l=1 rev1=1 crypto rx=0x7f6fb000e7e0 tx=0x7f6fb000eca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6faa7fc640 1 -- 172.21.15.64:0/755524731 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fb000f040 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6faa7fc640 1 -- 172.21.15.64:0/755524731 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6fb0004030 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6fb40c9a30 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6faa7fc640 1 -- 172.21.15.64:0/755524731 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fb0004400 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.679+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6fb40c7830 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.680+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6fb40743c0 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.680+0000 7f6faa7fc640 1 -- 172.21.15.64:0/755524731 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6fb000c6d0 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.681+0000 7f6faa7fc640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9806b4d0 0x7f6f9806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.681+0000 7f6faa7fc640 1 -- 172.21.15.64:0/755524731 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(68..68 src has 1..68) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6fb008a000 con 0x7f6fb40b4260 2024-03-04T07:30:35.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.681+0000 7f6fb9495640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9806b4d0 0x7f6f9806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:35.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.681+0000 7f6fb9495640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9806b4d0 0x7f6f9806d990 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f6fa4036000 tx=0x7f6fa4039040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:35.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.685+0000 7f6faa7fc640 1 -- 172.21.15.64:0/755524731 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6fb40743c0 con 0x7f6fb40b4260 2024-03-04T07:30:35.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.905+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f6fb4068700 con 0x7f6fb40b4260 2024-03-04T07:30:35.905 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:30:35.906+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:30:35.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.907+0000 7f6faa7fc640 1 -- 172.21.15.64:0/755524731 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v37) v1 ==== 161+0+0 (secure 0 0 0) 0x7f6fb0017020 con 0x7f6fb40b4260 2024-03-04T07:30:35.907 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:30:35.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.910+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9806b4d0 msgr2=0x7f6f9806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.910+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9806b4d0 0x7f6f9806d990 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f6fa4036000 tx=0x7f6fa4039040 comp rx=0 tx=0).stop 2024-03-04T07:30:35.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.910+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 msgr2=0x7f6fb40c72f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:35.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.910+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40c72f0 secure :-1 s=READY pgs=916 cs=0 l=1 rev1=1 crypto rx=0x7f6fb000e7e0 tx=0x7f6fb000eca0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 shutdown_connections 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6f9806b4d0 0x7f6f9806d990 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6fb40d06c0 0x7f6fb41e6680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6fb40d02e0 0x7f6fb41e6140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 --2- 172.21.15.64:0/755524731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6fb40b4260 0x7f6fb40c72f0 unknown :-1 s=CLOSED pgs=916 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 >> 172.21.15.64:0/755524731 conn(0x7f6fb40b36d0 msgr2=0x7f6fb40b1740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 shutdown_connections 2024-03-04T07:30:35.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:35.911+0000 7f6fbbefa640 1 -- 172.21.15.64:0/755524731 wait complete. 2024-03-04T07:30:35.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:30:35.951 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:30:36.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.027+0000 7efffdfd0640 1 Processor -- start 2024-03-04T07:30:36.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.027+0000 7efffdfd0640 1 -- start start 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efffdfd0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff8142170 0x7efff8142a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efffdfd0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efff80b4830 0x7efff8160280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efffdfd0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efff80b29a0 0x7efff80b2dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efffdfd0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efff81458e0 con 0x7efff80b4830 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efffdfd0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efff80b3310 con 0x7efff80b29a0 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efffdfd0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efff8070980 con 0x7efff8142170 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efff80b4830 0x7efff8160280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efff80b4830 0x7efff8160280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42642/0 (socket says 172.21.15.64:42642) 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 -- 172.21.15.64:0/790991708 learned_addr learned my addr 172.21.15.64:0/790991708 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff77fe640 1 --2- 172.21.15.64:0/790991708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff8142170 0x7efff8142a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:36.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff7fff640 1 --2- 172.21.15.64:0/790991708 >> v2:172.21.15.66:3300/0 conn(0x7efff80b29a0 0x7efff80b2dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 -- 172.21.15.64:0/790991708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff8142170 msgr2=0x7efff8142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 --2- 172.21.15.64:0/790991708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff8142170 0x7efff8142a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 -- 172.21.15.64:0/790991708 >> v2:172.21.15.66:3300/0 conn(0x7efff80b29a0 msgr2=0x7efff80b2dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 --2- 172.21.15.64:0/790991708 >> v2:172.21.15.66:3300/0 conn(0x7efff80b29a0 0x7efff80b2dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.028+0000 7efff6ffd640 1 -- 172.21.15.64:0/790991708 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efff8142f60 con 0x7efff80b4830 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.029+0000 7efff6ffd640 1 --2- 172.21.15.64:0/790991708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff8160280 secure :-1 s=READY pgs=919 cs=0 l=1 rev1=1 crypto rx=0x7efff80cf5f0 tx=0x7effec02b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d3e5f03dea6e2ce server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.029+0000 7efff67fc640 1 -- 172.21.15.64:0/790991708 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7effec035020 con 0x7efff80b4830 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.029+0000 7efff67fc640 1 -- 172.21.15.64:0/790991708 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7effec031640 con 0x7efff80b4830 2024-03-04T07:30:36.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.029+0000 7efff67fc640 1 -- 172.21.15.64:0/790991708 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7effec02d040 con 0x7efff80b4830 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 -- 172.21.15.64:0/790991708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 msgr2=0x7efff8160280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 --2- 172.21.15.64:0/790991708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff8160280 secure :-1 s=READY pgs=919 cs=0 l=1 rev1=1 crypto rx=0x7efff80cf5f0 tx=0x7effec02b8f0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 -- 172.21.15.64:0/790991708 shutdown_connections 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 --2- 172.21.15.64:0/790991708 >> v2:172.21.15.66:3300/0 conn(0x7efff80b29a0 0x7efff80b2dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 --2- 172.21.15.64:0/790991708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff8160280 unknown :-1 s=CLOSED pgs=919 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 --2- 172.21.15.64:0/790991708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff8142170 0x7efff8142a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 -- 172.21.15.64:0/790991708 >> 172.21.15.64:0/790991708 conn(0x7efff8066d00 msgr2=0x7efff8067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 -- 172.21.15.64:0/790991708 shutdown_connections 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.030+0000 7efffdfd0640 1 -- 172.21.15.64:0/790991708 wait complete. 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.031+0000 7efffdfd0640 1 Processor -- start 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.031+0000 7efffdfd0640 1 -- start start 2024-03-04T07:30:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efffdfd0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff80b29a0 0x7efff815a8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efffdfd0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff813c5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efffdfd0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efff8142170 0x7efff81f1940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efffdfd0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efff8070ed0 con 0x7efff80b4830 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efffdfd0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efff81607c0 con 0x7efff8142170 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efffdfd0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efff80711b0 con 0x7efff80b29a0 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff813c5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff813c5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42654/0 (socket says 172.21.15.64:42654) 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 -- 172.21.15.64:0/2383972453 learned_addr learned my addr 172.21.15.64:0/2383972453 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff7fff640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efff8142170 0x7efff81f1940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff77fe640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff80b29a0 0x7efff815a8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 -- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff80b29a0 msgr2=0x7efff815a8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff80b29a0 0x7efff815a8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 -- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efff8142170 msgr2=0x7efff81f1940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efff8142170 0x7efff81f1940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 -- 172.21.15.64:0/2383972453 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efff815b100 con 0x7efff80b4830 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.032+0000 7efff6ffd640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff813c5a0 secure :-1 s=READY pgs=920 cs=0 l=1 rev1=1 crypto rx=0x7effec000c00 tx=0x7effec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.033+0000 7effdffff640 1 -- 172.21.15.64:0/2383972453 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7effec040070 con 0x7efff80b4830 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.033+0000 7effdffff640 1 -- 172.21.15.64:0/2383972453 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7effec0399f0 con 0x7efff80b4830 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.033+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efff80af4a0 con 0x7efff80b4830 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.033+0000 7effdffff640 1 -- 172.21.15.64:0/2383972453 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7effec035040 con 0x7efff80b4830 2024-03-04T07:30:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.033+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efff80afa60 con 0x7efff80b4830 2024-03-04T07:30:36.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.034+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efff80733c0 con 0x7efff80b4830 2024-03-04T07:30:36.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.034+0000 7effdffff640 1 -- 172.21.15.64:0/2383972453 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7effec04b060 con 0x7efff80b4830 2024-03-04T07:30:36.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.034+0000 7effdffff640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7effe406b500 0x7effe406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:36.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.034+0000 7effdffff640 1 -- 172.21.15.64:0/2383972453 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(69..69 src has 1..69) v4 ==== 5853+0+0 (secure 0 0 0) 0x7effec0aba80 con 0x7efff80b4830 2024-03-04T07:30:36.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.035+0000 7efff77fe640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7effe406b500 0x7effe406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:36.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.035+0000 7efff77fe640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7effe406b500 0x7effe406d9c0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7effe8007920 tx=0x7effe8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:36.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.039+0000 7effdffff640 1 -- 172.21.15.64:0/2383972453 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7effec07a220 con 0x7efff80b4830 2024-03-04T07:30:36.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.264+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7efff80aedb0 con 0x7efff80b4830 2024-03-04T07:30:36.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.909+0000 7effdffff640 1 -- 172.21.15.64:0/2383972453 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v38) v1 ==== 104+0+0 (secure 0 0 0) 0x7effec07e2c0 con 0x7efff80b4830 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7effe406b500 msgr2=0x7effe406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7effe406b500 0x7effe406d9c0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7effe8007920 tx=0x7effe8008040 comp rx=0 tx=0).stop 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 msgr2=0x7efff813c5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff813c5a0 secure :-1 s=READY pgs=920 cs=0 l=1 rev1=1 crypto rx=0x7effec000c00 tx=0x7effec0046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 shutdown_connections 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7effe406b500 0x7effe406d9c0 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efff8142170 0x7efff81f1940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efff80b4830 0x7efff813c5a0 unknown :-1 s=CLOSED pgs=920 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 --2- 172.21.15.64:0/2383972453 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efff80b29a0 0x7efff815a8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 >> 172.21.15.64:0/2383972453 conn(0x7efff8066d00 msgr2=0x7efff80715d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 shutdown_connections 2024-03-04T07:30:36.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:36.912+0000 7efffdfd0640 1 -- 172.21.15.64:0/2383972453 wait complete. 2024-03-04T07:30:36.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:30:37.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.081+0000 7f72b3cfa640 1 Processor -- start 2024-03-04T07:30:37.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.081+0000 7f72b3cfa640 1 -- start start 2024-03-04T07:30:37.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b3cfa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac15e000 0x7f72ac0d1e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:37.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b3cfa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f72ac0cf1f0 0x7f72ac0b6eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:37.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b3cfa640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72ac0d2700 con 0x7f72ac0d37a0 2024-03-04T07:30:37.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b3cfa640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f72ac0d29e0 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b3cfa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72ac0d2cc0 con 0x7f72ac15e000 2024-03-04T07:30:37.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b2297640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f72ac0cf1f0 0x7f72ac0b6eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:37.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b1295640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f72ac0d37a0 0x7f72ac0ceae0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50470/0 (socket says 172.21.15.64:50470) 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b1295640 1 -- 172.21.15.64:0/134345178 learned_addr learned my addr 172.21.15.64:0/134345178 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b2297640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f72ac0cf1f0 0x7f72ac0b6eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42670/0 (socket says 172.21.15.64:42670) 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b1a96640 1 --2- 172.21.15.64:0/134345178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac15e000 0x7f72ac0d1e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b2297640 1 -- 172.21.15.64:0/134345178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac15e000 msgr2=0x7f72ac0d1e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b2297640 1 --2- 172.21.15.64:0/134345178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac15e000 0x7f72ac0d1e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b2297640 1 -- 172.21.15.64:0/134345178 >> v1:172.21.15.64:6789/0 conn(0x7f72ac0d37a0 legacy=0x7f72ac0ceae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b0a94640 1 -- 172.21.15.64:0/134345178 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2444280851 0 0) 0x7f72ac0d2700 con 0x7f72ac0d37a0 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.082+0000 7f72b2297640 1 -- 172.21.15.64:0/134345178 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72ac0d2fa0 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.083+0000 7f72b2297640 1 --2- 172.21.15.64:0/134345178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0b6eb0 secure :-1 s=READY pgs=922 cs=0 l=1 rev1=1 crypto rx=0x7f72a4009d90 tx=0x7f72a402bf30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9aa6066e294ad19e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.083+0000 7f72b0a94640 1 -- 172.21.15.64:0/134345178 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72a4039070 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.083+0000 7f72b0a94640 1 -- 172.21.15.64:0/134345178 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72a402cce0 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.083+0000 7f72b0a94640 1 -- 172.21.15.64:0/134345178 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72a4034660 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 -- 172.21.15.64:0/134345178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 msgr2=0x7f72ac0b6eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:37.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/134345178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0b6eb0 secure :-1 s=READY pgs=922 cs=0 l=1 rev1=1 crypto rx=0x7f72a4009d90 tx=0x7f72a402bf30 comp rx=0 tx=0).stop 2024-03-04T07:30:37.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 -- 172.21.15.64:0/134345178 shutdown_connections 2024-03-04T07:30:37.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/134345178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0b6eb0 unknown :-1 s=CLOSED pgs=922 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/134345178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac15e000 0x7f72ac0d1e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 -- 172.21.15.64:0/134345178 >> 172.21.15.64:0/134345178 conn(0x7f72ac066d00 msgr2=0x7f72ac067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:37.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 -- 172.21.15.64:0/134345178 shutdown_connections 2024-03-04T07:30:37.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.084+0000 7f72b3cfa640 1 -- 172.21.15.64:0/134345178 wait complete. 2024-03-04T07:30:37.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.085+0000 7f72b3cfa640 1 Processor -- start 2024-03-04T07:30:37.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.085+0000 7f72b3cfa640 1 -- start start 2024-03-04T07:30:37.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b3cfa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0c8ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:37.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b3cfa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac0d37a0 0x7f72ac152220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:37.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b3cfa640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72ac15e000 0x7f72ac15b6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:37.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b3cfa640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72ac0d2340 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b3cfa640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72ac15e3e0 con 0x7f72ac15e000 2024-03-04T07:30:37.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b3cfa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72ac0d24c0 con 0x7f72ac0d37a0 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b1a96640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0c8ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b1a96640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0c8ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42678/0 (socket says 172.21.15.64:42678) 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.086+0000 7f72b1a96640 1 -- 172.21.15.64:0/3715173519 learned_addr learned my addr 172.21.15.64:0/3715173519 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b1295640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac0d37a0 0x7f72ac152220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b2297640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72ac15e000 0x7f72ac15b6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b1a96640 1 -- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac0d37a0 msgr2=0x7f72ac152220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b1a96640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac0d37a0 0x7f72ac152220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b1a96640 1 -- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72ac15e000 msgr2=0x7f72ac15b6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b1a96640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72ac15e000 0x7f72ac15b6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b1a96640 1 -- 172.21.15.64:0/3715173519 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72ac151180 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b2297640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72ac15e000 0x7f72ac15b6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b1a96640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0c8ef0 secure :-1 s=READY pgs=923 cs=0 l=1 rev1=1 crypto rx=0x7f72a000e870 tx=0x7f72a000ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:37.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f729e7fc640 1 -- 172.21.15.64:0/3715173519 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72a0017070 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f729e7fc640 1 -- 172.21.15.64:0/3715173519 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72a000c940 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72ac151460 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f729e7fc640 1 -- 172.21.15.64:0/3715173519 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72a000cd30 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.087+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72ac151a20 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.088+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f72ac072a30 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.089+0000 7f729e7fc640 1 -- 172.21.15.64:0/3715173519 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f72a001a070 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.089+0000 7f729e7fc640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f729006b500 0x7f729006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:37.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.089+0000 7f729e7fc640 1 -- 172.21.15.64:0/3715173519 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(69..69 src has 1..69) v4 ==== 5853+0+0 (secure 0 0 0) 0x7f72a008a400 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.089+0000 7f72b1295640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f729006b500 0x7f729006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:37.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.090+0000 7f72b1295640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f729006b500 0x7f729006d9c0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f72ac0d48e0 tx=0x7f72a8007d50 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:37.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.094+0000 7f729e7fc640 1 -- 172.21.15.64:0/3715173519 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f72a0058ba0 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.277+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f72ac04f410 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.913+0000 7f729e7fc640 1 -- 172.21.15.64:0/3715173519 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v70) v1 ==== 181+0+0 (secure 0 0 0) 0x7f72a005cc40 con 0x7f72ac0cf1f0 2024-03-04T07:30:37.914 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:30:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.917+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f729006b500 msgr2=0x7f729006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.917+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f729006b500 0x7f729006d9c0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f72ac0d48e0 tx=0x7f72a8007d50 comp rx=0 tx=0).stop 2024-03-04T07:30:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.917+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 msgr2=0x7f72ac0c8ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.917+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0c8ef0 secure :-1 s=READY pgs=923 cs=0 l=1 rev1=1 crypto rx=0x7f72a000e870 tx=0x7f72a000ed30 comp rx=0 tx=0).stop 2024-03-04T07:30:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 shutdown_connections 2024-03-04T07:30:37.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f729006b500 0x7f729006d9c0 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72ac15e000 0x7f72ac15b6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72ac0d37a0 0x7f72ac152220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 --2- 172.21.15.64:0/3715173519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72ac0cf1f0 0x7f72ac0c8ef0 unknown :-1 s=CLOSED pgs=923 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 >> 172.21.15.64:0/3715173519 conn(0x7f72ac066d00 msgr2=0x7f72ac1561f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 shutdown_connections 2024-03-04T07:30:37.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:37.918+0000 7f72b3cfa640 1 -- 172.21.15.64:0/3715173519 wait complete. 2024-03-04T07:30:37.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:30:38.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.043+0000 7fa6c7b8a640 1 Processor -- start 2024-03-04T07:30:38.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.043+0000 7fa6c7b8a640 1 -- start start 2024-03-04T07:30:38.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c7b8a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa6c0146fd0 0x7fa6c006ef10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:38.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c7b8a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6c00b4840 0x7fa6c00b0ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:38.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c7b8a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa6c0161b50 con 0x7fa6c006f450 2024-03-04T07:30:38.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c7b8a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6c006e5f0 con 0x7fa6c00b4840 2024-03-04T07:30:38.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c7b8a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6c0161900 con 0x7fa6c0146fd0 2024-03-04T07:30:38.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c5125640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6c00b4840 0x7fa6c00b0ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c6127640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa6c006f450 0x7fa6c00ad8b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50478/0 (socket says 172.21.15.64:50478) 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c6127640 1 -- 172.21.15.64:0/2579493591 learned_addr learned my addr 172.21.15.64:0/2579493591 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c4924640 1 -- 172.21.15.64:0/2579493591 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 301069466 0 0) 0x7fa6c0161b50 con 0x7fa6c006f450 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c5926640 1 --2- 172.21.15.64:0/2579493591 >> v2:172.21.15.66:3300/0 conn(0x7fa6c0146fd0 0x7fa6c006ef10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c4924640 1 -- 172.21.15.64:0/2579493591 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa6ac003520 con 0x7fa6c006f450 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c5125640 1 -- 172.21.15.64:0/2579493591 >> v2:172.21.15.66:3300/0 conn(0x7fa6c0146fd0 msgr2=0x7fa6c006ef10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c5125640 1 --2- 172.21.15.64:0/2579493591 >> v2:172.21.15.66:3300/0 conn(0x7fa6c0146fd0 0x7fa6c006ef10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c5125640 1 -- 172.21.15.64:0/2579493591 >> v1:172.21.15.64:6789/0 conn(0x7fa6c006f450 legacy=0x7fa6c00ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:38.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.044+0000 7fa6c5125640 1 -- 172.21.15.64:0/2579493591 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6c0161e30 con 0x7fa6c00b4840 2024-03-04T07:30:38.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.045+0000 7fa6c5125640 1 --2- 172.21.15.64:0/2579493591 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c00b4840 0x7fa6c00b0ee0 secure :-1 s=READY pgs=925 cs=0 l=1 rev1=1 crypto rx=0x7fa6c00cfd00 tx=0x7fa6b002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c578ae68d6253ec0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:38.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.045+0000 7fa6c4924640 1 -- 172.21.15.64:0/2579493591 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6b00315a0 con 0x7fa6c00b4840 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.045+0000 7fa6c4924640 1 -- 172.21.15.64:0/2579493591 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa6b0037070 con 0x7fa6c00b4840 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.045+0000 7fa6c4924640 1 -- 172.21.15.64:0/2579493591 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6b0031940 con 0x7fa6c00b4840 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/2579493591 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c00b4840 msgr2=0x7fa6c00b0ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/2579493591 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c00b4840 0x7fa6c00b0ee0 secure :-1 s=READY pgs=925 cs=0 l=1 rev1=1 crypto rx=0x7fa6c00cfd00 tx=0x7fa6b002b840 comp rx=0 tx=0).stop 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/2579493591 shutdown_connections 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/2579493591 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c00b4840 0x7fa6c00b0ee0 unknown :-1 s=CLOSED pgs=925 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/2579493591 >> v2:172.21.15.66:3300/0 conn(0x7fa6c0146fd0 0x7fa6c006ef10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/2579493591 >> 172.21.15.64:0/2579493591 conn(0x7fa6c00afac0 msgr2=0x7fa6c0072d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/2579493591 shutdown_connections 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.046+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/2579493591 wait complete. 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.047+0000 7fa6c7b8a640 1 Processor -- start 2024-03-04T07:30:38.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.047+0000 7fa6c7b8a640 1 -- start start 2024-03-04T07:30:38.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6c006f450 0x7fa6c0142f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:38.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6c00b4840 0x7fa6c015d500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:38.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c0146fd0 0x7fa6c0145e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:38.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6c01473b0 con 0x7fa6c0146fd0 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6c01623f0 con 0x7fa6c00b4840 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6c0147530 con 0x7fa6c006f450 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c0146fd0 0x7fa6c0145e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c0146fd0 0x7fa6c0145e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42684/0 (socket says 172.21.15.64:42684) 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 -- 172.21.15.64:0/3470161183 learned_addr learned my addr 172.21.15.64:0/3470161183 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c5125640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6c00b4840 0x7fa6c015d500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c5926640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6c006f450 0x7fa6c0142f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 -- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6c006f450 msgr2=0x7fa6c0142f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6c006f450 0x7fa6c0142f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 -- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6c00b4840 msgr2=0x7fa6c015d500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6c00b4840 0x7fa6c015d500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 -- 172.21.15.64:0/3470161183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6c0144a90 con 0x7fa6c0146fd0 2024-03-04T07:30:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c5926640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6c006f450 0x7fa6c0142f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c6127640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c0146fd0 0x7fa6c0145e10 secure :-1 s=READY pgs=926 cs=0 l=1 rev1=1 crypto rx=0x7fa6b800cbb0 tx=0x7fa6b800a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6b67fc640 1 -- 172.21.15.64:0/3470161183 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6b8017070 con 0x7fa6c0146fd0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6b67fc640 1 -- 172.21.15.64:0/3470161183 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa6b800ad40 con 0x7fa6c0146fd0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa6c0144d70 con 0x7fa6c0146fd0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6b67fc640 1 -- 172.21.15.64:0/3470161183 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa6b8012600 con 0x7fa6c0146fd0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.048+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6c0145330 con 0x7fa6c0146fd0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.049+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa6c0073410 con 0x7fa6c0146fd0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.050+0000 7fa6b67fc640 1 -- 172.21.15.64:0/3470161183 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa6b801a070 con 0x7fa6c0146fd0 2024-03-04T07:30:38.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.050+0000 7fa6b67fc640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa6a406b4d0 0x7fa6a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:38.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.050+0000 7fa6b67fc640 1 -- 172.21.15.64:0/3470161183 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(70..70 src has 1..70) v4 ==== 5415+0+0 (secure 0 0 0) 0x7fa6b8089ea0 con 0x7fa6c0146fd0 2024-03-04T07:30:38.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.050+0000 7fa6c5926640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa6a406b4d0 0x7fa6a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:38.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.051+0000 7fa6c5926640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa6a406b4d0 0x7fa6a406d990 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7fa6bc00b820 tx=0x7fa6bc008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:38.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.055+0000 7fa6b67fc640 1 -- 172.21.15.64:0/3470161183 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa6b8058760 con 0x7fa6c0146fd0 2024-03-04T07:30:38.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.235+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fa6c0072e80 con 0x7fa6c0146fd0 2024-03-04T07:30:38.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.915+0000 7fa6b67fc640 1 -- 172.21.15.64:0/3470161183 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v71) v1 ==== 169+0+0 (secure 0 0 0) 0x7fa6b805c800 con 0x7fa6c0146fd0 2024-03-04T07:30:38.915 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:30:38.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.918+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa6a406b4d0 msgr2=0x7fa6a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:38.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.918+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa6a406b4d0 0x7fa6a406d990 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7fa6bc00b820 tx=0x7fa6bc008040 comp rx=0 tx=0).stop 2024-03-04T07:30:38.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.918+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c0146fd0 msgr2=0x7fa6c0145e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.918+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c0146fd0 0x7fa6c0145e10 secure :-1 s=READY pgs=926 cs=0 l=1 rev1=1 crypto rx=0x7fa6b800cbb0 tx=0x7fa6b800a430 comp rx=0 tx=0).stop 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.918+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 shutdown_connections 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.919+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa6a406b4d0 0x7fa6a406d990 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.919+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6c0146fd0 0x7fa6c0145e10 unknown :-1 s=CLOSED pgs=926 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.919+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6c00b4840 0x7fa6c015d500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.919+0000 7fa6c7b8a640 1 --2- 172.21.15.64:0/3470161183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6c006f450 0x7fa6c0142f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.919+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 >> 172.21.15.64:0/3470161183 conn(0x7fa6c00afac0 msgr2=0x7fa6c00722d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.919+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 shutdown_connections 2024-03-04T07:30:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:38.919+0000 7fa6c7b8a640 1 -- 172.21.15.64:0/3470161183 wait complete. 2024-03-04T07:30:38.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:30:39.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.042+0000 7fbe75d8a640 1 Processor -- start 2024-03-04T07:30:39.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.042+0000 7fbe75d8a640 1 -- start start 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.043+0000 7fbe75d8a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fbe700b5840 0x7fbe700b39b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.043+0000 7fbe75d8a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe7006f070 con 0x7fbe7015dbd0 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.043+0000 7fbe75d8a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbe70071940 con 0x7fbe7015dfb0 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.043+0000 7fbe75d8a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbe700b5db0 con 0x7fbe700b5840 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.043+0000 7fbe6f7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbe7015dbd0 0x7fbe7015ee70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50492/0 (socket says 172.21.15.64:50492) 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.043+0000 7fbe6f7fe640 1 -- 172.21.15.64:0/2266370534 learned_addr learned my addr 172.21.15.64:0/2266370534 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.043+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1201258538 0 0) 0x7fbe7006f070 con 0x7fbe7015dbd0 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbe58003520 con 0x7fbe7015dbd0 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6effd640 1 --2- 172.21.15.64:0/2266370534 >> v2:172.21.15.66:3300/0 conn(0x7fbe700b5840 0x7fbe700b39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 163185996 0 0) 0x7fbe58003520 con 0x7fbe7015dbd0 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 >> v2:172.21.15.66:3300/0 conn(0x7fbe700b5840 msgr2=0x7fbe700b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:39.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 --2- 172.21.15.64:0/2266370534 >> v2:172.21.15.66:3300/0 conn(0x7fbe700b5840 0x7fbe700b39b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 >> v1:172.21.15.66:6789/0 conn(0x7fbe7015dfb0 legacy=0x7fbe70066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe70071c00 con 0x7fbe7015dbd0 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fbe60001670 con 0x7fbe7015dbd0 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 >> v1:172.21.15.64:6789/0 conn(0x7fbe7015dbd0 legacy=0x7fbe7015ee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe580039b0 0x7fbe58003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe58004800 0x7fbe58004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe580051c0 0x7fbe5800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe700b5db0 con 0x7fbe580039b0 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe5800c060 con 0x7fbe580051c0 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe5800c260 con 0x7fbe58004800 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6f7fe640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe580039b0 0x7fbe58003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6effd640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe58004800 0x7fbe58004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6ffff640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe580051c0 0x7fbe5800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6f7fe640 1 -- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe58004800 msgr2=0x7fbe58004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6f7fe640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe58004800 0x7fbe58004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6f7fe640 1 -- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe580051c0 msgr2=0x7fbe5800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6f7fe640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe580051c0 0x7fbe5800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6f7fe640 1 -- 172.21.15.64:0/2266370534 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe58003520 con 0x7fbe580039b0 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6ffff640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe580051c0 0x7fbe5800b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.044+0000 7fbe6ffff640 1 -- 172.21.15.64:0/2266370534 reap_dead start 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.045+0000 7fbe6f7fe640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe580039b0 0x7fbe58003dd0 secure :-1 s=READY pgs=928 cs=0 l=1 rev1=1 crypto rx=0x7fbe6000c240 tx=0x7fbe58005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4b639754c13b4fda server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.045+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe60007d10 con 0x7fbe580039b0 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.045+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe60030070 con 0x7fbe580039b0 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.045+0000 7fbe6e7fc640 1 -- 172.21.15.64:0/2266370534 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe6002cdc0 con 0x7fbe580039b0 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.046+0000 7fbe75d8a640 1 -- 172.21.15.64:0/2266370534 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe580039b0 msgr2=0x7fbe58003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.046+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe580039b0 0x7fbe58003dd0 secure :-1 s=READY pgs=928 cs=0 l=1 rev1=1 crypto rx=0x7fbe6000c240 tx=0x7fbe58005830 comp rx=0 tx=0).stop 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.046+0000 7fbe75d8a640 1 -- 172.21.15.64:0/2266370534 shutdown_connections 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.046+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/2266370534 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe580039b0 0x7fbe58003dd0 unknown :-1 s=CLOSED pgs=928 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.046+0000 7fbe75d8a640 1 -- 172.21.15.64:0/2266370534 >> 172.21.15.64:0/2266370534 conn(0x7fbe700b26b0 msgr2=0x7fbe700b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.046+0000 7fbe75d8a640 1 -- 172.21.15.64:0/2266370534 shutdown_connections 2024-03-04T07:30:39.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.046+0000 7fbe75d8a640 1 -- 172.21.15.64:0/2266370534 wait complete. 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.047+0000 7fbe75d8a640 1 Processor -- start 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.047+0000 7fbe75d8a640 1 -- start start 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe75d8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe580039b0 0x7fbe70148e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe75d8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe700c8340 0x7fbe700c87c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe75d8a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe700ae610 0x7fbe700aea90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe75d8a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe700726e0 con 0x7fbe700c8340 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe75d8a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe700721c0 con 0x7fbe700ae610 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe75d8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbe70072860 con 0x7fbe580039b0 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe700c8340 0x7fbe700c87c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe700c8340 0x7fbe700c87c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42702/0 (socket says 172.21.15.64:42702) 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 -- 172.21.15.64:0/1617529842 learned_addr learned my addr 172.21.15.64:0/1617529842 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6ffff640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe700ae610 0x7fbe700aea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6f7fe640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe580039b0 0x7fbe70148e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 -- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe580039b0 msgr2=0x7fbe70148e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe580039b0 0x7fbe70148e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 -- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe700ae610 msgr2=0x7fbe700aea90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe700ae610 0x7fbe700aea90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 -- 172.21.15.64:0/1617529842 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe70149760 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe6effd640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe700c8340 0x7fbe700c87c0 secure :-1 s=READY pgs=929 cs=0 l=1 rev1=1 crypto rx=0x7fbe64011700 tx=0x7fbe64011bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.048+0000 7fbe57fff640 1 -- 172.21.15.64:0/1617529842 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe6400f9c0 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.049+0000 7fbe57fff640 1 -- 172.21.15.64:0/1617529842 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe6400fb60 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.049+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbe70149a40 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.049+0000 7fbe57fff640 1 -- 172.21.15.64:0/1617529842 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe64015460 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.049+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe700c5920 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.049+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbe70073380 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.050+0000 7fbe57fff640 1 -- 172.21.15.64:0/1617529842 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbe64015600 con 0x7fbe700c8340 2024-03-04T07:30:39.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.050+0000 7fbe57fff640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe5c06b500 0x7fbe5c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.050+0000 7fbe57fff640 1 -- 172.21.15.64:0/1617529842 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fbe6408a760 con 0x7fbe700c8340 2024-03-04T07:30:39.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.050+0000 7fbe6f7fe640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe5c06b500 0x7fbe5c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.051+0000 7fbe6f7fe640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe5c06b500 0x7fbe5c06d9c0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fbe60034000 tx=0x7fbe60039040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.055+0000 7fbe57fff640 1 -- 172.21.15.64:0/1617529842 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbe64019030 con 0x7fbe700c8340 2024-03-04T07:30:39.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.235+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7fbe7004f410 con 0x7fbe700c8340 2024-03-04T07:30:39.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.239+0000 7fbe57fff640 1 -- 172.21.15.64:0/1617529842 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v24) v1 ==== 104+0+0 (secure 0 0 0) 0x7fbe64059970 con 0x7fbe700c8340 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe5c06b500 msgr2=0x7fbe5c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe5c06b500 0x7fbe5c06d9c0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fbe60034000 tx=0x7fbe60039040 comp rx=0 tx=0).stop 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe700c8340 msgr2=0x7fbe700c87c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe700c8340 0x7fbe700c87c0 secure :-1 s=READY pgs=929 cs=0 l=1 rev1=1 crypto rx=0x7fbe64011700 tx=0x7fbe64011bc0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 shutdown_connections 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe5c06b500 0x7fbe5c06d9c0 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbe700ae610 0x7fbe700aea90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbe700c8340 0x7fbe700c87c0 unknown :-1 s=CLOSED pgs=929 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 --2- 172.21.15.64:0/1617529842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbe580039b0 0x7fbe70148e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 >> 172.21.15.64:0/1617529842 conn(0x7fbe700b26b0 msgr2=0x7fbe7015a800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:39.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 shutdown_connections 2024-03-04T07:30:39.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.241+0000 7fbe75d8a640 1 -- 172.21.15.64:0/1617529842 wait complete. 2024-03-04T07:30:39.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mds mds_bal_fragment_size_max 2024-03-04T07:30:39.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.358+0000 7facf6442640 1 Processor -- start 2024-03-04T07:30:39.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.358+0000 7facf6442640 1 -- start start 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.358+0000 7facf6442640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7facf00b5840 0x7facf00b39b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.358+0000 7facf6442640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf00b3ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.358+0000 7facf6442640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7facf006f070 con 0x7facf015dbd0 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.358+0000 7facf6442640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7facf0071940 con 0x7facf00b5840 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.358+0000 7facf6442640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facf00b5db0 con 0x7facf015dfb0 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7facef7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7facf00b5840 0x7facf00b39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7faceffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7facf015dbd0 0x7facf015ee70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50504/0 (socket says 172.21.15.64:50504) 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7facef7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7facf00b5840 0x7facf00b39b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42710/0 (socket says 172.21.15.64:42710) 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7faceffff640 1 -- 172.21.15.64:0/3004757729 learned_addr learned my addr 172.21.15.64:0/3004757729 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7facf49df640 1 --2- 172.21.15.64:0/3004757729 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf00b3ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7facef7fe640 1 -- 172.21.15.64:0/3004757729 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 msgr2=0x7facf00b3ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:39.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7facef7fe640 1 --2- 172.21.15.64:0/3004757729 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf00b3ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7facef7fe640 1 -- 172.21.15.64:0/3004757729 >> v1:172.21.15.64:6789/0 conn(0x7facf015dbd0 legacy=0x7facf015ee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7facef7fe640 1 -- 172.21.15.64:0/3004757729 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facf0071c00 con 0x7facf00b5840 2024-03-04T07:30:39.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.359+0000 7faceeffd640 1 -- 172.21.15.64:0/3004757729 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2994304065 0 0) 0x7facf006f070 con 0x7facf015dbd0 2024-03-04T07:30:39.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facef7fe640 1 --2- 172.21.15.64:0/3004757729 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf00b39b0 secure :-1 s=READY pgs=931 cs=0 l=1 rev1=1 crypto rx=0x7face40097a0 tx=0x7face402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=85365acfd230c2aa server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7faceeffd640 1 -- 172.21.15.64:0/3004757729 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4035020 con 0x7facf00b5840 2024-03-04T07:30:39.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7faceeffd640 1 -- 172.21.15.64:0/3004757729 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7face4031830 con 0x7facf00b5840 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7faceeffd640 1 -- 172.21.15.64:0/3004757729 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4031c00 con 0x7facf00b5840 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 -- 172.21.15.64:0/3004757729 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 msgr2=0x7facf00b39b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 --2- 172.21.15.64:0/3004757729 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf00b39b0 secure :-1 s=READY pgs=931 cs=0 l=1 rev1=1 crypto rx=0x7face40097a0 tx=0x7face402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 -- 172.21.15.64:0/3004757729 shutdown_connections 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 --2- 172.21.15.64:0/3004757729 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf00b3ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 --2- 172.21.15.64:0/3004757729 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf00b39b0 unknown :-1 s=CLOSED pgs=931 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 -- 172.21.15.64:0/3004757729 >> 172.21.15.64:0/3004757729 conn(0x7facf00b26b0 msgr2=0x7facf00b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 -- 172.21.15.64:0/3004757729 shutdown_connections 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.360+0000 7facf6442640 1 -- 172.21.15.64:0/3004757729 wait complete. 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 Processor -- start 2024-03-04T07:30:39.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 -- start start 2024-03-04T07:30:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf0157f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf015dbd0 0x7facf01f74c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf015fcd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf00726e0 con 0x7facf00b5840 2024-03-04T07:30:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf00721c0 con 0x7facf015dbd0 2024-03-04T07:30:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.362+0000 7facf6442640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facf0072860 con 0x7facf015dfb0 2024-03-04T07:30:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf0157f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf0157f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42722/0 (socket says 172.21.15.64:42722) 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 -- 172.21.15.64:0/1661919806 learned_addr learned my addr 172.21.15.64:0/1661919806 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facef7fe640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf015dbd0 0x7facf01f74c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facf49df640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf015fcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 -- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 msgr2=0x7facf015fcd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf015fcd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 -- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf015dbd0 msgr2=0x7facf01f74c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf015dbd0 0x7facf01f74c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 -- 172.21.15.64:0/1661919806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facf0153020 con 0x7facf00b5840 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facef7fe640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf015dbd0 0x7facf01f74c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facf49df640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf015fcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7faceffff640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf0157f40 secure :-1 s=READY pgs=932 cs=0 l=1 rev1=1 crypto rx=0x7face000c400 tx=0x7face000c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facecff9640 1 -- 172.21.15.64:0/1661919806 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face0017070 con 0x7facf00b5840 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facecff9640 1 -- 172.21.15.64:0/1661919806 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7face0005ce0 con 0x7facf00b5840 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7facf0155280 con 0x7facf00b5840 2024-03-04T07:30:39.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facecff9640 1 -- 172.21.15.64:0/1661919806 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face0007310 con 0x7facf00b5840 2024-03-04T07:30:39.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.363+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7facf01f7f10 con 0x7facf00b5840 2024-03-04T07:30:39.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.364+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7facf01536b0 con 0x7facf00b5840 2024-03-04T07:30:39.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.365+0000 7facecff9640 1 -- 172.21.15.64:0/1661919806 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7face001a070 con 0x7facf00b5840 2024-03-04T07:30:39.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.365+0000 7facecff9640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.365+0000 7facef7fe640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.365+0000 7facecff9640 1 -- 172.21.15.64:0/1661919806 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7face0089340 con 0x7facf00b5840 2024-03-04T07:30:39.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.366+0000 7facef7fe640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7face4036000 tx=0x7face4008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.370+0000 7facecff9640 1 -- 172.21.15.64:0/1661919806 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7facf0153020 con 0x7facf00b5840 2024-03-04T07:30:39.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.549+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"} v 0) v1 -- 0x7facf0073380 con 0x7facf00b5840 2024-03-04T07:30:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.553+0000 7facecff9640 1 -- 172.21.15.64:0/1661919806 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mds", "name": "mds_bal_fragment_size_max"}]=0 v25) v1 ==== 108+0+0 (secure 0 0 0) 0x7face00584c0 con 0x7facf00b5840 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 msgr2=0x7facdc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7face4036000 tx=0x7face4008030 comp rx=0 tx=0).stop 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 msgr2=0x7facf0157f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf0157f40 secure :-1 s=READY pgs=932 cs=0 l=1 rev1=1 crypto rx=0x7face000c400 tx=0x7face000c8c0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 shutdown_connections 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facf015dfb0 0x7facf015fcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf015dbd0 0x7facf01f74c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 --2- 172.21.15.64:0/1661919806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b5840 0x7facf0157f40 unknown :-1 s=CLOSED pgs=932 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 >> 172.21.15.64:0/1661919806 conn(0x7facf00b26b0 msgr2=0x7facf015a970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:39.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 shutdown_connections 2024-03-04T07:30:39.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.555+0000 7facf6442640 1 -- 172.21.15.64:0/1661919806 wait complete. 2024-03-04T07:30:39.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays' 2024-03-04T07:30:39.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.667+0000 7f0943355640 1 Processor -- start 2024-03-04T07:30:39.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.667+0000 7f0943355640 1 -- start start 2024-03-04T07:30:39.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f0943355640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f093c0b95a0 0x7f093c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f0943355640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f093c0b9a10 0x7f093c0b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f0943355640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f093c0cea30 con 0x7f093c15df00 2024-03-04T07:30:39.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f0943355640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f093c0cf3c0 con 0x7f093c0b9a10 2024-03-04T07:30:39.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f0943355640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f093c0cf0e0 con 0x7f093c0b95a0 2024-03-04T07:30:39.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f09410f1640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f093c15df00 0x7f093c0d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50506/0 (socket says 172.21.15.64:50506) 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f09410f1640 1 -- 172.21.15.64:0/131256367 learned_addr learned my addr 172.21.15.64:0/131256367 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.668+0000 7f09418f2640 1 --2- 172.21.15.64:0/131256367 >> v2:172.21.15.64:3300/0 conn(0x7f093c0b9a10 0x7f093c0b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f093bfff640 1 -- 172.21.15.64:0/131256367 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2875361296 0 0) 0x7f093c0cea30 con 0x7f093c15df00 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f093bfff640 1 -- 172.21.15.64:0/131256367 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0924003520 con 0x7f093c15df00 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f09408f0640 1 --2- 172.21.15.64:0/131256367 >> v2:172.21.15.66:3300/0 conn(0x7f093c0b95a0 0x7f093c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f09418f2640 1 -- 172.21.15.64:0/131256367 >> v2:172.21.15.66:3300/0 conn(0x7f093c0b95a0 msgr2=0x7f093c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f09418f2640 1 --2- 172.21.15.64:0/131256367 >> v2:172.21.15.66:3300/0 conn(0x7f093c0b95a0 0x7f093c0b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f09418f2640 1 -- 172.21.15.64:0/131256367 >> v1:172.21.15.64:6789/0 conn(0x7f093c15df00 legacy=0x7f093c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f09418f2640 1 -- 172.21.15.64:0/131256367 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f093c0ce470 con 0x7f093c0b9a10 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f09408f0640 1 --2- 172.21.15.64:0/131256367 >> v2:172.21.15.66:3300/0 conn(0x7f093c0b95a0 0x7f093c0b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.669+0000 7f09418f2640 1 --2- 172.21.15.64:0/131256367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c0b9a10 0x7f093c0b5c40 secure :-1 s=READY pgs=934 cs=0 l=1 rev1=1 crypto rx=0x7f093c0d1760 tx=0x7f093402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f90859de426aa766 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f093bfff640 1 -- 172.21.15.64:0/131256367 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0934035020 con 0x7f093c0b9a10 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f093bfff640 1 -- 172.21.15.64:0/131256367 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0934031830 con 0x7f093c0b9a10 2024-03-04T07:30:39.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f093bfff640 1 -- 172.21.15.64:0/131256367 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0934031c00 con 0x7f093c0b9a10 2024-03-04T07:30:39.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 -- 172.21.15.64:0/131256367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c0b9a10 msgr2=0x7f093c0b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 --2- 172.21.15.64:0/131256367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c0b9a10 0x7f093c0b5c40 secure :-1 s=READY pgs=934 cs=0 l=1 rev1=1 crypto rx=0x7f093c0d1760 tx=0x7f093402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 -- 172.21.15.64:0/131256367 shutdown_connections 2024-03-04T07:30:39.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 --2- 172.21.15.64:0/131256367 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c0b9a10 0x7f093c0b5c40 unknown :-1 s=CLOSED pgs=934 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 --2- 172.21.15.64:0/131256367 >> v2:172.21.15.66:3300/0 conn(0x7f093c0b95a0 0x7f093c0b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 -- 172.21.15.64:0/131256367 >> 172.21.15.64:0/131256367 conn(0x7f093c071960 msgr2=0x7f093c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:39.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 -- 172.21.15.64:0/131256367 shutdown_connections 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.670+0000 7f0943355640 1 -- 172.21.15.64:0/131256367 wait complete. 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.671+0000 7f0943355640 1 Processor -- start 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.671+0000 7f0943355640 1 -- start start 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f0943355640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f093c0b95a0 0x7f093c1f3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f0943355640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f093c0b9a10 0x7f093c1581f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f0943355640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c15df00 0x7f093c15ed80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f0943355640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f093c0cedf0 con 0x7f093c15df00 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f0943355640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f093c0cec70 con 0x7f093c0b95a0 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f0943355640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f093c0d3220 con 0x7f093c0b9a10 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c15df00 0x7f093c15ed80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c15df00 0x7f093c15ed80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42746/0 (socket says 172.21.15.64:42746) 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 -- 172.21.15.64:0/1246842952 learned_addr learned my addr 172.21.15.64:0/1246842952 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09410f1640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f093c0b95a0 0x7f093c1f3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09408f0640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f093c0b9a10 0x7f093c1581f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 -- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f093c0b9a10 msgr2=0x7f093c1581f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f093c0b9a10 0x7f093c1581f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 -- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f093c0b95a0 msgr2=0x7f093c1f3880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f093c0b95a0 0x7f093c1f3880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.672+0000 7f09418f2640 1 -- 172.21.15.64:0/1246842952 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f093c15f760 con 0x7f093c15df00 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.673+0000 7f09410f1640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f093c0b95a0 0x7f093c1f3880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.673+0000 7f09418f2640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c15df00 0x7f093c15ed80 secure :-1 s=READY pgs=935 cs=0 l=1 rev1=1 crypto rx=0x7f0934009860 tx=0x7f09340046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.673+0000 7f0939ffb640 1 -- 172.21.15.64:0/1246842952 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0934035040 con 0x7f093c15df00 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.673+0000 7f0939ffb640 1 -- 172.21.15.64:0/1246842952 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f093403f070 con 0x7f093c15df00 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.673+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f093c0cc100 con 0x7f093c15df00 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.673+0000 7f0939ffb640 1 -- 172.21.15.64:0/1246842952 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f093403baa0 con 0x7f093c15df00 2024-03-04T07:30:39.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.673+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f093c158730 con 0x7f093c15df00 2024-03-04T07:30:39.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.674+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f093c0b3120 con 0x7f093c15df00 2024-03-04T07:30:39.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.675+0000 7f0939ffb640 1 -- 172.21.15.64:0/1246842952 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0934038020 con 0x7f093c15df00 2024-03-04T07:30:39.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.675+0000 7f0939ffb640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f092806b4d0 0x7f092806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:39.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.675+0000 7f0939ffb640 1 -- 172.21.15.64:0/1246842952 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f09340abb50 con 0x7f093c15df00 2024-03-04T07:30:39.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.675+0000 7f09410f1640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f092806b4d0 0x7f092806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:39.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.675+0000 7f09410f1640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f092806b4d0 0x7f092806d990 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f092c005c00 tx=0x7f092c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:39.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.679+0000 7f0939ffb640 1 -- 172.21.15.64:0/1246842952 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f093c0b3120 con 0x7f093c15df00 2024-03-04T07:30:39.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:39.930+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]} v 0) v1 -- 0x7f093c067ff0 con 0x7f093c15df00 2024-03-04T07:30:40.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.925+0000 7f0939ffb640 1 -- 172.21.15.64:0/1246842952 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_dirfrag_limit_strays"]}]=0 v213) v1 ==== 140+0+0 (secure 0 0 0) 0x7f0934031800 con 0x7f093c15df00 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f092806b4d0 msgr2=0x7f092806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f092806b4d0 0x7f092806d990 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f092c005c00 tx=0x7f092c009040 comp rx=0 tx=0).stop 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c15df00 msgr2=0x7f093c15ed80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c15df00 0x7f093c15ed80 secure :-1 s=READY pgs=935 cs=0 l=1 rev1=1 crypto rx=0x7f0934009860 tx=0x7f09340046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 shutdown_connections 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f092806b4d0 0x7f092806d990 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f093c15df00 0x7f093c15ed80 unknown :-1 s=CLOSED pgs=935 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:40.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f093c0b9a10 0x7f093c1581f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:40.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 --2- 172.21.15.64:0/1246842952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f093c0b95a0 0x7f093c1f3880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:40.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 >> 172.21.15.64:0/1246842952 conn(0x7f093c071960 msgr2=0x7f093c071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:40.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 shutdown_connections 2024-03-04T07:30:40.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:40.928+0000 7f0943355640 1 -- 172.21.15.64:0/1246842952 wait complete. 2024-03-04T07:30:40.941 INFO:tasks.cephfs_test_runner:That unlinking fails when the stray directory fragment becomes too ... ok 2024-03-04T07:30:40.942 INFO:tasks.cephfs_test_runner:Starting test: test_fancy_layout (tasks.cephfs.test_strays.TestStrays) purge stray file with fancy layout 2024-03-04T07:30:40.942 INFO:tasks.cephfs_test_runner:test_fancy_layout (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:30:40.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:41.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.050+0000 7fd788481640 1 Processor -- start 2024-03-04T07:30:41.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.050+0000 7fd788481640 1 -- start start 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.051+0000 7fd788481640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd7800b95a0 0x7fd7800b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.051+0000 7fd788481640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd7800cea30 con 0x7fd7800b9a10 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.051+0000 7fd788481640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd7800cf3c0 con 0x7fd78015df00 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.051+0000 7fd788481640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd7800cf0e0 con 0x7fd7800b95a0 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.051+0000 7fd786a1e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd7800b9a10 0x7fd7800d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50508/0 (socket says 172.21.15.64:50508) 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.051+0000 7fd786a1e640 1 -- 172.21.15.64:0/1966079659 learned_addr learned my addr 172.21.15.64:0/1966079659 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd785a1c640 1 --2- 172.21.15.64:0/1966079659 >> v2:172.21.15.66:3300/0 conn(0x7fd7800b95a0 0x7fd7800b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1758386354 0 0) 0x7fd7800cea30 con 0x7fd7800b9a10 2024-03-04T07:30:41.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd768003520 con 0x7fd7800b9a10 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 704023095 0 0) 0x7fd768003520 con 0x7fd7800b9a10 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 >> v2:172.21.15.66:3300/0 conn(0x7fd7800b95a0 msgr2=0x7fd7800b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 --2- 172.21.15.64:0/1966079659 >> v2:172.21.15.66:3300/0 conn(0x7fd7800b95a0 0x7fd7800b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 >> v1:172.21.15.66:6789/0 conn(0x7fd78015df00 legacy=0x7fd7800d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7800ce470 con 0x7fd7800b9a10 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd7700019f0 con 0x7fd7800b9a10 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 >> v1:172.21.15.64:6789/0 conn(0x7fd7800b9a10 legacy=0x7fd7800d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7680039b0 0x7fd768003df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd768004800 0x7fd768004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd76800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7800cf0e0 con 0x7fd7680051c0 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd76800c090 con 0x7fd7680039b0 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd76800c290 con 0x7fd768004800 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd786a1e640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd76800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd78621d640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7680039b0 0x7fd768003df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd785a1c640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd768004800 0x7fd768004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd786a1e640 1 -- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd768004800 msgr2=0x7fd768004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd786a1e640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd768004800 0x7fd768004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd786a1e640 1 -- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7680039b0 msgr2=0x7fd768003df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd786a1e640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd7680039b0 0x7fd768003df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.052+0000 7fd786a1e640 1 -- 172.21.15.64:0/1966079659 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd768003520 con 0x7fd7680051c0 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.053+0000 7fd786a1e640 1 -- 172.21.15.64:0/1966079659 reap_dead start 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.053+0000 7fd786a1e640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd76800b9c0 secure :-1 s=READY pgs=937 cs=0 l=1 rev1=1 crypto rx=0x7fd77000a840 tx=0x7fd768005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b7e5be9eb8f740c9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.053+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd77000a460 con 0x7fd7680051c0 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.053+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd770034650 con 0x7fd7680051c0 2024-03-04T07:30:41.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.053+0000 7fd78521b640 1 -- 172.21.15.64:0/1966079659 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd770034a40 con 0x7fd7680051c0 2024-03-04T07:30:41.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.054+0000 7fd788481640 1 -- 172.21.15.64:0/1966079659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 msgr2=0x7fd76800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.054+0000 7fd788481640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd76800b9c0 secure :-1 s=READY pgs=937 cs=0 l=1 rev1=1 crypto rx=0x7fd77000a840 tx=0x7fd768005830 comp rx=0 tx=0).stop 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.054+0000 7fd788481640 1 -- 172.21.15.64:0/1966079659 shutdown_connections 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.054+0000 7fd788481640 1 --2- 172.21.15.64:0/1966079659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd76800b9c0 unknown :-1 s=CLOSED pgs=937 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.054+0000 7fd788481640 1 -- 172.21.15.64:0/1966079659 >> 172.21.15.64:0/1966079659 conn(0x7fd780071960 msgr2=0x7fd780071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.054+0000 7fd788481640 1 -- 172.21.15.64:0/1966079659 shutdown_connections 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.054+0000 7fd788481640 1 -- 172.21.15.64:0/1966079659 wait complete. 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.055+0000 7fd788481640 1 Processor -- start 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.055+0000 7fd788481640 1 -- start start 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd788481640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd780161b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd788481640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd780158180 0x7fd7801585e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd788481640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd78015f950 0x7fd78015fdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd788481640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7800cedf0 con 0x7fd7680051c0 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd788481640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd7800cec70 con 0x7fd780158180 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd788481640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd7800d25b0 con 0x7fd78015f950 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd780161b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd780161b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42756/0 (socket says 172.21.15.64:42756) 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 -- 172.21.15.64:0/4102893794 learned_addr learned my addr 172.21.15.64:0/4102893794 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 -- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd78015f950 msgr2=0x7fd78015fdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd785a1c640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd780158180 0x7fd7801585e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd786a1e640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd78015f950 0x7fd78015fdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd78015f950 0x7fd78015fdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 -- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd780158180 msgr2=0x7fd7801585e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd780158180 0x7fd7801585e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 -- 172.21.15.64:0/4102893794 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd780158b20 con 0x7fd7680051c0 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd785a1c640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd780158180 0x7fd7801585e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.056+0000 7fd78621d640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd780161b20 secure :-1 s=READY pgs=938 cs=0 l=1 rev1=1 crypto rx=0x7fd77800f5f0 tx=0x7fd77800fab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.057+0000 7fd77effd640 1 -- 172.21.15.64:0/4102893794 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7780052e0 con 0x7fd7680051c0 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.057+0000 7fd77effd640 1 -- 172.21.15.64:0/4102893794 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd778018070 con 0x7fd7680051c0 2024-03-04T07:30:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.057+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd7801606a0 con 0x7fd7680051c0 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.057+0000 7fd77effd640 1 -- 172.21.15.64:0/4102893794 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd778003810 con 0x7fd7680051c0 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.057+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd780160c60 con 0x7fd7680051c0 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.058+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd780067ff0 con 0x7fd7680051c0 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.058+0000 7fd77effd640 1 -- 172.21.15.64:0/4102893794 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd7780039b0 con 0x7fd7680051c0 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.058+0000 7fd77effd640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76c06b4d0 0x7fd76c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.059+0000 7fd77effd640 1 -- 172.21.15.64:0/4102893794 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fd778089d20 con 0x7fd7680051c0 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.059+0000 7fd785a1c640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76c06b4d0 0x7fd76c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.059+0000 7fd785a1c640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76c06b4d0 0x7fd76c06d990 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fd7740093d0 tx=0x7fd774008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.063+0000 7fd77effd640 1 -- 172.21.15.64:0/4102893794 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd778058ea0 con 0x7fd7680051c0 2024-03-04T07:30:41.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.248+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fd780162780 con 0x7fd7680051c0 2024-03-04T07:30:41.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.249+0000 7fd77effd640 1 -- 172.21.15.64:0/4102893794 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v71) v1 ==== 74+0+12973 (secure 0 0 0) 0x7fd7780587a0 con 0x7fd7680051c0 2024-03-04T07:30:41.249 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:41.249 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":71,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:38.914381+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/2304674412":"2024-03-05T07:30:35.906869+0000","172.21.15.64:6836/2304674412":"2024-03-05T07:30:35.906869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:41.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76c06b4d0 msgr2=0x7fd76c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76c06b4d0 0x7fd76c06d990 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fd7740093d0 tx=0x7fd774008040 comp rx=0 tx=0).stop 2024-03-04T07:30:41.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 msgr2=0x7fd780161b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd780161b20 secure :-1 s=READY pgs=938 cs=0 l=1 rev1=1 crypto rx=0x7fd77800f5f0 tx=0x7fd77800fab0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 shutdown_connections 2024-03-04T07:30:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76c06b4d0 0x7fd76c06d990 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd78015f950 0x7fd78015fdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd780158180 0x7fd7801585e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 --2- 172.21.15.64:0/4102893794 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd7680051c0 0x7fd780161b20 unknown :-1 s=CLOSED pgs=938 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 >> 172.21.15.64:0/4102893794 conn(0x7fd780071960 msgr2=0x7fd78015d0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 shutdown_connections 2024-03-04T07:30:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.251+0000 7fd788481640 1 -- 172.21.15.64:0/4102893794 wait complete. 2024-03-04T07:30:41.262 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:30:41.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:30:41.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.366+0000 7f46702df640 1 Processor -- start 2024-03-04T07:30:41.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.366+0000 7f46702df640 1 -- start start 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.367+0000 7f46702df640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f466815e2f0 0x7f46680b41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.367+0000 7f46702df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680cdac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.367+0000 7f46702df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f466815e6d0 con 0x7f466806b4e0 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.367+0000 7f46702df640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46680d0a10 con 0x7f466815e2f0 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.367+0000 7f46702df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46680ccd00 con 0x7f466806c050 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e07b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f466815e2f0 0x7f46680b41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466d87a640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f466806b4e0 0x7f466806b940 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50520/0 (socket says 172.21.15.64:50520) 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466d87a640 1 -- 172.21.15.64:0/1322605435 learned_addr learned my addr 172.21.15.64:0/1322605435 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e07b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f466815e2f0 0x7f46680b41f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42760/0 (socket says 172.21.15.64:42760) 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e87c640 1 --2- 172.21.15.64:0/1322605435 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680cdac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e07b640 1 -- 172.21.15.64:0/1322605435 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 msgr2=0x7f46680cdac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e07b640 1 --2- 172.21.15.64:0/1322605435 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680cdac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e07b640 1 -- 172.21.15.64:0/1322605435 >> v1:172.21.15.64:6789/0 conn(0x7f466806b4e0 legacy=0x7f466806b940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e07b640 1 -- 172.21.15.64:0/1322605435 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46680ccfe0 con 0x7f466815e2f0 2024-03-04T07:30:41.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466d079640 1 -- 172.21.15.64:0/1322605435 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2026086584 0 0) 0x7f466815e6d0 con 0x7f466806b4e0 2024-03-04T07:30:41.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.368+0000 7f466e87c640 1 --2- 172.21.15.64:0/1322605435 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680cdac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:41.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f466e07b640 1 --2- 172.21.15.64:0/1322605435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466815e2f0 0x7f46680b41f0 secure :-1 s=READY pgs=940 cs=0 l=1 rev1=1 crypto rx=0x7f46640097a0 tx=0x7f466402b7a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=482ccceec9d29912 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f466d079640 1 -- 172.21.15.64:0/1322605435 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4664038070 con 0x7f466815e2f0 2024-03-04T07:30:41.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f466d079640 1 -- 172.21.15.64:0/1322605435 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4664031700 con 0x7f466815e2f0 2024-03-04T07:30:41.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f466d079640 1 -- 172.21.15.64:0/1322605435 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4664031ad0 con 0x7f466815e2f0 2024-03-04T07:30:41.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 -- 172.21.15.64:0/1322605435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466815e2f0 msgr2=0x7f46680b41f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 --2- 172.21.15.64:0/1322605435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466815e2f0 0x7f46680b41f0 secure :-1 s=READY pgs=940 cs=0 l=1 rev1=1 crypto rx=0x7f46640097a0 tx=0x7f466402b7a0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 -- 172.21.15.64:0/1322605435 shutdown_connections 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 --2- 172.21.15.64:0/1322605435 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680cdac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 --2- 172.21.15.64:0/1322605435 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466815e2f0 0x7f46680b41f0 unknown :-1 s=CLOSED pgs=940 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 -- 172.21.15.64:0/1322605435 >> 172.21.15.64:0/1322605435 conn(0x7f46680b36d0 msgr2=0x7f46680b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 -- 172.21.15.64:0/1322605435 shutdown_connections 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.369+0000 7f46702df640 1 -- 172.21.15.64:0/1322605435 wait complete. 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 Processor -- start 2024-03-04T07:30:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 -- start start 2024-03-04T07:30:41.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466806b4e0 0x7f4668151000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680b6c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f466815e2f0 0x7f46680b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46680ce000 con 0x7f466806b4e0 2024-03-04T07:30:41.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46680cd5a0 con 0x7f466815e2f0 2024-03-04T07:30:41.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f46702df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46680ce180 con 0x7f466806c050 2024-03-04T07:30:41.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f466e07b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466806b4e0 0x7f4668151000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f466e07b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466806b4e0 0x7f4668151000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42766/0 (socket says 172.21.15.64:42766) 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f466e07b640 1 -- 172.21.15.64:0/3351877998 learned_addr learned my addr 172.21.15.64:0/3351877998 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f466e87c640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f466815e2f0 0x7f46680b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.371+0000 7f466d87a640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680b6c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f466e07b640 1 -- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 msgr2=0x7f46680b6c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f466e07b640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680b6c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f466e07b640 1 -- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f466815e2f0 msgr2=0x7f46680b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f466e07b640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f466815e2f0 0x7f46680b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f466e07b640 1 -- 172.21.15.64:0/3351877998 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46680b55f0 con 0x7f466806b4e0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f466e07b640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466806b4e0 0x7f4668151000 secure :-1 s=READY pgs=941 cs=0 l=1 rev1=1 crypto rx=0x7f46640097a0 tx=0x7f46640046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f465effd640 1 -- 172.21.15.64:0/3351877998 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4664038040 con 0x7f466806b4e0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f465effd640 1 -- 172.21.15.64:0/3351877998 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4664004030 con 0x7f466806b4e0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46680b58d0 con 0x7f466806b4e0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f465effd640 1 -- 172.21.15.64:0/3351877998 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4664004420 con 0x7f466806b4e0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.372+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46680b5e90 con 0x7f466806b4e0 2024-03-04T07:30:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.373+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4668074980 con 0x7f466806b4e0 2024-03-04T07:30:41.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.374+0000 7f465effd640 1 -- 172.21.15.64:0/3351877998 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f46640337a0 con 0x7f466806b4e0 2024-03-04T07:30:41.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.374+0000 7f465effd640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f464c06b4d0 0x7f464c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.374+0000 7f466d87a640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f464c06b4d0 0x7f464c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.374+0000 7f465effd640 1 -- 172.21.15.64:0/3351877998 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f46640ac110 con 0x7f466806b4e0 2024-03-04T07:30:41.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.375+0000 7f466d87a640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f464c06b4d0 0x7f464c06d990 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f46681f76f0 tx=0x7f4658007d50 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.378+0000 7f465effd640 1 -- 172.21.15.64:0/3351877998 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4668074980 con 0x7f466806b4e0 2024-03-04T07:30:41.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.557+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f4668067ff0 con 0x7f466806b4e0 2024-03-04T07:30:41.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.557+0000 7f465effd640 1 -- 172.21.15.64:0/3351877998 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v71) v1 ==== 93+0+10 (secure 0 0 0) 0x7f4668067ff0 con 0x7f466806b4e0 2024-03-04T07:30:41.557 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:30:41.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f464c06b4d0 msgr2=0x7f464c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f464c06b4d0 0x7f464c06d990 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f46681f76f0 tx=0x7f4658007d50 comp rx=0 tx=0).stop 2024-03-04T07:30:41.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466806b4e0 msgr2=0x7f4668151000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466806b4e0 0x7f4668151000 secure :-1 s=READY pgs=941 cs=0 l=1 rev1=1 crypto rx=0x7f46640097a0 tx=0x7f46640046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 shutdown_connections 2024-03-04T07:30:41.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f464c06b4d0 0x7f464c06d990 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f466815e2f0 0x7f46680b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f466806c050 0x7f46680b6c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 --2- 172.21.15.64:0/3351877998 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f466806b4e0 0x7f4668151000 unknown :-1 s=CLOSED pgs=941 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 >> 172.21.15.64:0/3351877998 conn(0x7f46680b36d0 msgr2=0x7f46680d1ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:41.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 shutdown_connections 2024-03-04T07:30:41.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.559+0000 7f46702df640 1 -- 172.21.15.64:0/3351877998 wait complete. 2024-03-04T07:30:41.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_fancy_layout' 2024-03-04T07:30:41.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.672+0000 7f46810c9640 1 Processor -- start 2024-03-04T07:30:41.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.672+0000 7f46810c9640 1 -- start start 2024-03-04T07:30:41.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f46810c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f467c15aa30 0x7f467c0cfac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f46810c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f467c0d0000 0x7f467c0ae850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f46810c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f467c15b160 con 0x7f467c0aed90 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f46810c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f467c0d0460 con 0x7f467c0d0000 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f46810c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f467c15b440 con 0x7f467c15aa30 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f467c0d0000 0x7f467c0ae850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467b59e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f467c0aed90 0x7f467c071a20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50528/0 (socket says 172.21.15.64:50528) 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467b59e640 1 -- 172.21.15.64:0/3044309673 learned_addr learned my addr 172.21.15.64:0/3044309673 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f467c0d0000 0x7f467c0ae850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42776/0 (socket says 172.21.15.64:42776) 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467ad9d640 1 --2- 172.21.15.64:0/3044309673 >> v2:172.21.15.66:3300/0 conn(0x7f467c15aa30 0x7f467c0cfac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467a59c640 1 -- 172.21.15.64:0/3044309673 >> v2:172.21.15.66:3300/0 conn(0x7f467c15aa30 msgr2=0x7f467c0cfac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467a59c640 1 --2- 172.21.15.64:0/3044309673 >> v2:172.21.15.66:3300/0 conn(0x7f467c15aa30 0x7f467c0cfac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467a59c640 1 -- 172.21.15.64:0/3044309673 >> v1:172.21.15.64:6789/0 conn(0x7f467c0aed90 legacy=0x7f467c071a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f467a59c640 1 -- 172.21.15.64:0/3044309673 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f467c15b720 con 0x7f467c0d0000 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.673+0000 7f4679d9b640 1 -- 172.21.15.64:0/3044309673 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 601459165 0 0) 0x7f467c15b160 con 0x7f467c0aed90 2024-03-04T07:30:41.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.674+0000 7f467a59c640 1 --2- 172.21.15.64:0/3044309673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c0ae850 secure :-1 s=READY pgs=943 cs=0 l=1 rev1=1 crypto rx=0x7f467c0b3470 tx=0x7f467002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2927c0d33d2c70c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.674+0000 7f4679d9b640 1 -- 172.21.15.64:0/3044309673 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4670035020 con 0x7f467c0d0000 2024-03-04T07:30:41.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.674+0000 7f4679d9b640 1 -- 172.21.15.64:0/3044309673 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4670031830 con 0x7f467c0d0000 2024-03-04T07:30:41.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.674+0000 7f4679d9b640 1 -- 172.21.15.64:0/3044309673 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4670031c00 con 0x7f467c0d0000 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 -- 172.21.15.64:0/3044309673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 msgr2=0x7f467c0ae850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 --2- 172.21.15.64:0/3044309673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c0ae850 secure :-1 s=READY pgs=943 cs=0 l=1 rev1=1 crypto rx=0x7f467c0b3470 tx=0x7f467002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 -- 172.21.15.64:0/3044309673 shutdown_connections 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 --2- 172.21.15.64:0/3044309673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c0ae850 unknown :-1 s=CLOSED pgs=943 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 --2- 172.21.15.64:0/3044309673 >> v2:172.21.15.66:3300/0 conn(0x7f467c15aa30 0x7f467c0cfac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 -- 172.21.15.64:0/3044309673 >> 172.21.15.64:0/3044309673 conn(0x7f467c066ee0 msgr2=0x7f467c0672f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 -- 172.21.15.64:0/3044309673 shutdown_connections 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.675+0000 7f46810c9640 1 -- 172.21.15.64:0/3044309673 wait complete. 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.676+0000 7f46810c9640 1 Processor -- start 2024-03-04T07:30:41.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.676+0000 7f46810c9640 1 -- start start 2024-03-04T07:30:41.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f46810c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f467c0aed90 0x7f467c143a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f46810c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c1478b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f46810c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f467c15aa30 0x7f467c147df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f46810c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f467c0af1d0 con 0x7f467c0d0000 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f46810c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f467c15bce0 con 0x7f467c15aa30 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f46810c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f467c15ae10 con 0x7f467c0aed90 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c1478b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c1478b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42792/0 (socket says 172.21.15.64:42792) 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 -- 172.21.15.64:0/2183910775 learned_addr learned my addr 172.21.15.64:0/2183910775 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467b59e640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f467c15aa30 0x7f467c147df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467ad9d640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f467c0aed90 0x7f467c143a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 -- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f467c0aed90 msgr2=0x7f467c143a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f467c0aed90 0x7f467c143a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 -- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f467c15aa30 msgr2=0x7f467c147df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f467c15aa30 0x7f467c147df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:41.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 -- 172.21.15.64:0/2183910775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f467c15df90 con 0x7f467c0d0000 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.677+0000 7f467a59c640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c1478b0 secure :-1 s=READY pgs=944 cs=0 l=1 rev1=1 crypto rx=0x7f467002bee0 tx=0x7f46700046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.678+0000 7f46637fe640 1 -- 172.21.15.64:0/2183910775 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4670041070 con 0x7f467c0d0000 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.678+0000 7f46637fe640 1 -- 172.21.15.64:0/2183910775 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f467003f070 con 0x7f467c0d0000 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.678+0000 7f46637fe640 1 -- 172.21.15.64:0/2183910775 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4670035040 con 0x7f467c0d0000 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.678+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f467c15d410 con 0x7f467c0d0000 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.678+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f467c15d9d0 con 0x7f467c0d0000 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.679+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f467c067ff0 con 0x7f467c0d0000 2024-03-04T07:30:41.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.679+0000 7f46637fe640 1 -- 172.21.15.64:0/2183910775 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4670038020 con 0x7f467c0d0000 2024-03-04T07:30:41.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.679+0000 7f46637fe640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f466806b4d0 0x7f466806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:41.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.680+0000 7f46637fe640 1 -- 172.21.15.64:0/2183910775 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f46700ab420 con 0x7f467c0d0000 2024-03-04T07:30:41.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.680+0000 7f467ad9d640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f466806b4d0 0x7f466806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:41.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.680+0000 7f467ad9d640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f466806b4d0 0x7f466806d990 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f4674009a30 tx=0x7f4674008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:41.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.684+0000 7f46637fe640 1 -- 172.21.15.64:0/2183910775 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4670031800 con 0x7f467c0d0000 2024-03-04T07:30:41.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:41.939+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]} v 0) v1 -- 0x7f467c159030 con 0x7f467c0d0000 2024-03-04T07:30:42.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.931+0000 7f46637fe640 1 -- 172.21.15.64:0/2183910775 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]}]=0 v215) v1 ==== 135+0+0 (secure 0 0 0) 0x7f467007a630 con 0x7f467c0d0000 2024-03-04T07:30:42.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f466806b4d0 msgr2=0x7f466806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:42.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f466806b4d0 0x7f466806d990 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f4674009a30 tx=0x7f4674008040 comp rx=0 tx=0).stop 2024-03-04T07:30:42.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 msgr2=0x7f467c1478b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c1478b0 secure :-1 s=READY pgs=944 cs=0 l=1 rev1=1 crypto rx=0x7f467002bee0 tx=0x7f46700046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 shutdown_connections 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f466806b4d0 0x7f466806d990 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f467c15aa30 0x7f467c147df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f467c0d0000 0x7f467c1478b0 unknown :-1 s=CLOSED pgs=944 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 --2- 172.21.15.64:0/2183910775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f467c0aed90 0x7f467c143a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 >> 172.21.15.64:0/2183910775 conn(0x7f467c066ee0 msgr2=0x7f467c071520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 shutdown_connections 2024-03-04T07:30:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:42.934+0000 7f46810c9640 1 -- 172.21.15.64:0/2183910775 wait complete. 2024-03-04T07:30:42.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.059+0000 7fd550f9a640 1 Processor -- start 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.059+0000 7fd550f9a640 1 -- start start 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd550f9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c1618e0 0x7fd54c06b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd550f9a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd54c0d0150 0x7fd54c0d05e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd550f9a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd54c15f730 con 0x7fd54c0ce4f0 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd550f9a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd54c0cfce0 con 0x7fd54c0d0150 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd550f9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd54c15ebb0 con 0x7fd54c1618e0 2024-03-04T07:30:43.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd54ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd54c0d0150 0x7fd54c0d05e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd549d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd54c0ce4f0 0x7fd54c0ce930 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50544/0 (socket says 172.21.15.64:50544) 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd549d9b640 1 -- 172.21.15.64:0/3630291590 learned_addr learned my addr 172.21.15.64:0/3630291590 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd54a59c640 1 --2- 172.21.15.64:0/3630291590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c1618e0 0x7fd54c06b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd54ad9d640 1 -- 172.21.15.64:0/3630291590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c1618e0 msgr2=0x7fd54c06b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd54ad9d640 1 --2- 172.21.15.64:0/3630291590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c1618e0 0x7fd54c06b130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd54ad9d640 1 -- 172.21.15.64:0/3630291590 >> v1:172.21.15.64:6789/0 conn(0x7fd54c0ce4f0 legacy=0x7fd54c0ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.060+0000 7fd54ad9d640 1 -- 172.21.15.64:0/3630291590 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd54c15ee90 con 0x7fd54c0d0150 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.061+0000 7fd54ad9d640 1 --2- 172.21.15.64:0/3630291590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0d0150 0x7fd54c0d05e0 secure :-1 s=READY pgs=946 cs=0 l=1 rev1=1 crypto rx=0x7fd53c009d90 tx=0x7fd53c02bf30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3dd95c779b26e56e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.061+0000 7fd54959a640 1 -- 172.21.15.64:0/3630291590 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd53c039070 con 0x7fd54c0d0150 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.061+0000 7fd54959a640 1 -- 172.21.15.64:0/3630291590 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd53c02cce0 con 0x7fd54c0d0150 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.061+0000 7fd54959a640 1 -- 172.21.15.64:0/3630291590 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd53c034660 con 0x7fd54c0d0150 2024-03-04T07:30:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 -- 172.21.15.64:0/3630291590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0d0150 msgr2=0x7fd54c0d05e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 --2- 172.21.15.64:0/3630291590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0d0150 0x7fd54c0d05e0 secure :-1 s=READY pgs=946 cs=0 l=1 rev1=1 crypto rx=0x7fd53c009d90 tx=0x7fd53c02bf30 comp rx=0 tx=0).stop 2024-03-04T07:30:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 -- 172.21.15.64:0/3630291590 shutdown_connections 2024-03-04T07:30:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 --2- 172.21.15.64:0/3630291590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0d0150 0x7fd54c0d05e0 unknown :-1 s=CLOSED pgs=946 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 --2- 172.21.15.64:0/3630291590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c1618e0 0x7fd54c06b130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 -- 172.21.15.64:0/3630291590 >> 172.21.15.64:0/3630291590 conn(0x7fd54c0b3660 msgr2=0x7fd54c0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 -- 172.21.15.64:0/3630291590 shutdown_connections 2024-03-04T07:30:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.062+0000 7fd550f9a640 1 -- 172.21.15.64:0/3630291590 wait complete. 2024-03-04T07:30:43.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.063+0000 7fd550f9a640 1 Processor -- start 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.063+0000 7fd550f9a640 1 -- start start 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd550f9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c0ce4f0 0x7fd54c0b5ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd550f9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd54c0d0150 0x7fd54c14d080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd550f9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0b5380 0x7fd54c0c65a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd550f9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd54c06b670 con 0x7fd54c0b5380 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd550f9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd54c15f450 con 0x7fd54c0d0150 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd550f9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd54c15e7d0 con 0x7fd54c0ce4f0 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0b5380 0x7fd54c0c65a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0b5380 0x7fd54c0c65a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42820/0 (socket says 172.21.15.64:42820) 2024-03-04T07:30:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 -- 172.21.15.64:0/503868838 learned_addr learned my addr 172.21.15.64:0/503868838 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54a59c640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c0ce4f0 0x7fd54c0b5ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd549d9b640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd54c0d0150 0x7fd54c14d080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 -- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c0ce4f0 msgr2=0x7fd54c0b5ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c0ce4f0 0x7fd54c0b5ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 -- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd54c0d0150 msgr2=0x7fd54c14d080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd54c0d0150 0x7fd54c14d080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.064+0000 7fd54ad9d640 1 -- 172.21.15.64:0/503868838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd54c0c6ae0 con 0x7fd54c0b5380 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.065+0000 7fd54ad9d640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0b5380 0x7fd54c0c65a0 secure :-1 s=READY pgs=947 cs=0 l=1 rev1=1 crypto rx=0x7fd53c009d90 tx=0x7fd53c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.065+0000 7fd542ffd640 1 -- 172.21.15.64:0/503868838 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd53c039050 con 0x7fd54c0b5380 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.065+0000 7fd542ffd640 1 -- 172.21.15.64:0/503868838 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd53c045070 con 0x7fd54c0b5380 2024-03-04T07:30:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.065+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd54c0c6d80 con 0x7fd54c0b5380 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.065+0000 7fd542ffd640 1 -- 172.21.15.64:0/503868838 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd53c03e830 con 0x7fd54c0b5380 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.065+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd54c0b4b80 con 0x7fd54c0b5380 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.066+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd54c09fc50 con 0x7fd54c0b5380 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.066+0000 7fd542ffd640 1 -- 172.21.15.64:0/503868838 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd53c03e9d0 con 0x7fd54c0b5380 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.067+0000 7fd542ffd640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd52c06b4d0 0x7fd52c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.067+0000 7fd542ffd640 1 -- 172.21.15.64:0/503868838 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fd53c0acd60 con 0x7fd54c0b5380 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.067+0000 7fd54a59c640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd52c06b4d0 0x7fd52c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.067+0000 7fd54a59c640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd52c06b4d0 0x7fd52c06d990 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fd544006fd0 tx=0x7fd544008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:43.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.071+0000 7fd542ffd640 1 -- 172.21.15.64:0/503868838 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd53c07c010 con 0x7fd54c0b5380 2024-03-04T07:30:43.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.256+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7fd54c072cb0 con 0x7fd54c0b5380 2024-03-04T07:30:43.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.261+0000 7fd542ffd640 1 -- 172.21.15.64:0/503868838 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v26)=0 v26) v1 ==== 122+0+0 (secure 0 0 0) 0x7fd53c07b910 con 0x7fd54c0b5380 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.263+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd52c06b4d0 msgr2=0x7fd52c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.263+0000 7fd550f9a640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd52c06b4d0 0x7fd52c06d990 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fd544006fd0 tx=0x7fd544008040 comp rx=0 tx=0).stop 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0b5380 msgr2=0x7fd54c0c65a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0b5380 0x7fd54c0c65a0 secure :-1 s=READY pgs=947 cs=0 l=1 rev1=1 crypto rx=0x7fd53c009d90 tx=0x7fd53c004a40 comp rx=0 tx=0).stop 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 shutdown_connections 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd52c06b4d0 0x7fd52c06d990 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd54c0b5380 0x7fd54c0c65a0 unknown :-1 s=CLOSED pgs=947 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd54c0d0150 0x7fd54c14d080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 --2- 172.21.15.64:0/503868838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd54c0ce4f0 0x7fd54c0b5ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 >> 172.21.15.64:0/503868838 conn(0x7fd54c0b3660 msgr2=0x7fd54c0cc180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 shutdown_connections 2024-03-04T07:30:43.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.264+0000 7fd550f9a640 1 -- 172.21.15.64:0/503868838 wait complete. 2024-03-04T07:30:43.274 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:30:43.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:30:43.325 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:30:43.377 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:30:43.377 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:43.377 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:43.392 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:30:43.392 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:30:43.392 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:43.410 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:30:43.410 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:43.410 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:43.447 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:30:43.447 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:30:43.447 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:43.467 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:30:43.468 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:30:43.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.589+0000 7fe40273a640 1 Processor -- start 2024-03-04T07:30:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.589+0000 7fe40273a640 1 -- start start 2024-03-04T07:30:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe40273a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3fc157f60 0x7fe3fc15b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe40273a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe3fc06c7b0 con 0x7fe3fc157b80 2024-03-04T07:30:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe40273a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe3fc06ca90 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe40273a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe3fc06cd70 con 0x7fe3fc157f60 2024-03-04T07:30:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe3fb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3fc157f60 0x7fe3fc15b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe3fbfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe3fc157b80 0x7fe3fc15b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47998/0 (socket says 172.21.15.64:47998) 2024-03-04T07:30:43.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe3fbfff640 1 -- 172.21.15.64:0/2391421518 learned_addr learned my addr 172.21.15.64:0/2391421518 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:43.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe3fb7fe640 1 -- 172.21.15.64:0/2391421518 >> v1:172.21.15.66:6789/0 conn(0x7fe3fc0cbab0 legacy=0x7fe3fc070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:43.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe3fb7fe640 1 -- 172.21.15.64:0/2391421518 >> v1:172.21.15.64:6789/0 conn(0x7fe3fc157b80 legacy=0x7fe3fc15b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe3faffd640 1 -- 172.21.15.64:0/2391421518 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2258688444 0 0) 0x7fe3fc06c7b0 con 0x7fe3fc157b80 2024-03-04T07:30:43.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.590+0000 7fe3fb7fe640 1 -- 172.21.15.64:0/2391421518 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3fc06d050 con 0x7fe3fc157f60 2024-03-04T07:30:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.591+0000 7fe3fb7fe640 1 --2- 172.21.15.64:0/2391421518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc157f60 0x7fe3fc15b260 secure :-1 s=READY pgs=949 cs=0 l=1 rev1=1 crypto rx=0x7fe3f000a940 tx=0x7fe3f002cd90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7b7023e590ca97b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.591+0000 7fe3faffd640 1 -- 172.21.15.64:0/2391421518 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe3f003a070 con 0x7fe3fc157f60 2024-03-04T07:30:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.591+0000 7fe3faffd640 1 -- 172.21.15.64:0/2391421518 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe3f002db40 con 0x7fe3fc157f60 2024-03-04T07:30:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.591+0000 7fe3faffd640 1 -- 172.21.15.64:0/2391421518 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe3f000b040 con 0x7fe3fc157f60 2024-03-04T07:30:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.592+0000 7fe40273a640 1 -- 172.21.15.64:0/2391421518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc157f60 msgr2=0x7fe3fc15b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.592+0000 7fe40273a640 1 --2- 172.21.15.64:0/2391421518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc157f60 0x7fe3fc15b260 secure :-1 s=READY pgs=949 cs=0 l=1 rev1=1 crypto rx=0x7fe3f000a940 tx=0x7fe3f002cd90 comp rx=0 tx=0).stop 2024-03-04T07:30:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.592+0000 7fe40273a640 1 -- 172.21.15.64:0/2391421518 shutdown_connections 2024-03-04T07:30:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.592+0000 7fe40273a640 1 --2- 172.21.15.64:0/2391421518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc157f60 0x7fe3fc15b260 unknown :-1 s=CLOSED pgs=949 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.592+0000 7fe40273a640 1 -- 172.21.15.64:0/2391421518 >> 172.21.15.64:0/2391421518 conn(0x7fe3fc066d00 msgr2=0x7fe3fc0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.592+0000 7fe40273a640 1 -- 172.21.15.64:0/2391421518 shutdown_connections 2024-03-04T07:30:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.592+0000 7fe40273a640 1 -- 172.21.15.64:0/2391421518 wait complete. 2024-03-04T07:30:43.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.593+0000 7fe40273a640 1 Processor -- start 2024-03-04T07:30:43.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.593+0000 7fe40273a640 1 -- start start 2024-03-04T07:30:43.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe40273a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc0cbab0 0x7fe3fc147ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe40273a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3fc157b80 0x7fe3fc0ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe40273a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3fc157f60 0x7fe3fc0addf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe40273a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3fc071250 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe40273a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3fc0cc260 con 0x7fe3fc157b80 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe40273a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe3fc0713d0 con 0x7fe3fc157f60 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc0cbab0 0x7fe3fc147ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc0cbab0 0x7fe3fc147ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42094/0 (socket says 172.21.15.64:42094) 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 -- 172.21.15.64:0/169422548 learned_addr learned my addr 172.21.15.64:0/169422548 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fb7fe640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3fc157b80 0x7fe3fc0ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 -- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3fc157f60 msgr2=0x7fe3fc0addf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe400cd7640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3fc157f60 0x7fe3fc0addf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3fc157f60 0x7fe3fc0addf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 -- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3fc157b80 msgr2=0x7fe3fc0ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3fc157b80 0x7fe3fc0ad8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 -- 172.21.15.64:0/169422548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3fc0b4470 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fb7fe640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3fc157b80 0x7fe3fc0ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:43.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe400cd7640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3fc157f60 0x7fe3fc0addf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:43.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.594+0000 7fe3fbfff640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc0cbab0 0x7fe3fc147ef0 secure :-1 s=READY pgs=950 cs=0 l=1 rev1=1 crypto rx=0x7fe3ec00c4d0 tx=0x7fe3ec00c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:43.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.595+0000 7fe3f8ff9640 1 -- 172.21.15.64:0/169422548 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe3ec018070 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.595+0000 7fe3f8ff9640 1 -- 172.21.15.64:0/169422548 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe3ec007500 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.595+0000 7fe3f8ff9640 1 -- 172.21.15.64:0/169422548 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe3ec004a20 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.595+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe3fc1f4020 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.595+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe3fc1f45e0 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.596+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe3fc067ff0 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.596+0000 7fe3f8ff9640 1 -- 172.21.15.64:0/169422548 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe3ec01b070 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.596+0000 7fe3f8ff9640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe3e806b4d0 0x7fe3e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:43.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.596+0000 7fe3f8ff9640 1 -- 172.21.15.64:0/169422548 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7fe3ec089240 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.596+0000 7fe3fb7fe640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe3e806b4d0 0x7fe3e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:43.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.597+0000 7fe3fb7fe640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe3e806b4d0 0x7fe3e806d990 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7fe3f0037000 tx=0x7fe3f0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:43.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.601+0000 7fe3f8ff9640 1 -- 172.21.15.64:0/169422548 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe3ec0583c0 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.819+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe3fc0d07a0 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.819+0000 7fe3f8ff9640 1 -- 172.21.15.64:0/169422548 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 38 v38) v1 ==== 94+0+3699 (secure 0 0 0) 0x7fe3ec05e0c0 con 0x7fe3fc0cbab0 2024-03-04T07:30:43.820 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:43.820 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":38,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6073,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1032284423","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1032284423},{"type":"v1","addr":"172.21.15.64:6835","nonce":1032284423}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6082,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2271055017","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2271055017},{"type":"v1","addr":"172.21.15.66:6833","nonce":2271055017}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6085,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2756587149","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":2756587149},{"type":"v1","addr":"172.21.15.66:6835","nonce":2756587149}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":33},{"gid":6867,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/3614624913","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":3614624913},{"type":"v1","addr":"172.21.15.64:6837","nonce":3614624913}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":38}],"filesystems":[]} 2024-03-04T07:30:43.820 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 38 2024-03-04T07:30:43.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe3e806b4d0 msgr2=0x7fe3e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe3e806b4d0 0x7fe3e806d990 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7fe3f0037000 tx=0x7fe3f0008040 comp rx=0 tx=0).stop 2024-03-04T07:30:43.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc0cbab0 msgr2=0x7fe3fc147ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:43.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc0cbab0 0x7fe3fc147ef0 secure :-1 s=READY pgs=950 cs=0 l=1 rev1=1 crypto rx=0x7fe3ec00c4d0 tx=0x7fe3ec00c990 comp rx=0 tx=0).stop 2024-03-04T07:30:43.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 shutdown_connections 2024-03-04T07:30:43.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe3e806b4d0 0x7fe3e806d990 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3fc157f60 0x7fe3fc0addf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3fc157b80 0x7fe3fc0ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 --2- 172.21.15.64:0/169422548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3fc0cbab0 0x7fe3fc147ef0 unknown :-1 s=CLOSED pgs=950 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:43.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 >> 172.21.15.64:0/169422548 conn(0x7fe3fc066d00 msgr2=0x7fe3fc1597b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:43.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 shutdown_connections 2024-03-04T07:30:43.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:43.821+0000 7fe40273a640 1 -- 172.21.15.64:0/169422548 wait complete. 2024-03-04T07:30:43.832 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:30:43.832 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:30:43.832 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:30:43.833 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:30:43.833 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:30:43.833 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:30:43.833 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:30:43.833 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:30:43.833 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:30:43.833 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:30:43.834 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:30:43.834 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:30:43.834 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:30:43.834 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:30:43.834 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:30:43.834 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:30:43.842 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:30:43.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:30:43.843 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:30:43.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:30:43.850 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:30:43.850 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:30:43.850 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:30:43.851 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:30:43.885 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:30:43.885 DEBUG:teuthology.parallel:result is None 2024-03-04T07:30:43.885 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:30:43.886 DEBUG:teuthology.parallel:result is None 2024-03-04T07:30:43.893 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:30:43.893 DEBUG:teuthology.parallel:result is None 2024-03-04T07:30:43.894 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:30:43.894 DEBUG:teuthology.parallel:result is None 2024-03-04T07:30:43.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:30:43.962 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:30:43.962 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:30:43.985 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:30:43.985 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:30:44.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.001+0000 7f67cd26d640 1 Processor -- start 2024-03-04T07:30:44.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.001+0000 7f67cd26d640 1 -- start start 2024-03-04T07:30:44.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.001+0000 7f67cd26d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 0x7f67c80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.001+0000 7f67cd26d640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67c80cea30 con 0x7f67c80b9a30 2024-03-04T07:30:44.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.001+0000 7f67cd26d640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67c80d2af0 con 0x7f67c80b95a0 2024-03-04T07:30:44.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67cd26d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67c80cf3c0 con 0x7f67c815df00 2024-03-04T07:30:44.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f67c80b9a30 0x7f67c80d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48006/0 (socket says 172.21.15.64:48006) 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c77fe640 1 -- 172.21.15.64:0/3155894713 learned_addr learned my addr 172.21.15.64:0/3155894713 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c6ffd640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 0x7f67c80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3191080985 0 0) 0x7f67c80cea30 con 0x7f67c80b9a30 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f67b0003520 con 0x7f67c80b9a30 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3494793860 0 0) 0x7f67b0003520 con 0x7f67c80b9a30 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 msgr2=0x7f67c80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 0x7f67c80d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 >> v1:172.21.15.66:6789/0 conn(0x7f67c80b95a0 legacy=0x7f67c80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67c80cf0e0 con 0x7f67c80b9a30 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f67c00019f0 con 0x7f67c80b9a30 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 >> v1:172.21.15.64:6789/0 conn(0x7f67c80b9a30 legacy=0x7f67c80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b00039e0 0x7f67b0003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67b0004390 0x7f67b0004860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 0x7f67b00050c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67c80cf3c0 con 0x7f67b0004390 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67b000b9c0 con 0x7f67b00039e0 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67b000bbc0 con 0x7f67c815df00 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c67fc640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67b0004390 0x7f67b0004860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c77fe640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 0x7f67b00050c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.002+0000 7f67c6ffd640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b00039e0 0x7f67b0003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.003+0000 7f67c67fc640 1 -- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 msgr2=0x7f67b00050c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.003+0000 7f67c67fc640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c815df00 0x7f67b00050c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.003+0000 7f67c67fc640 1 -- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b00039e0 msgr2=0x7f67b0003dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.003+0000 7f67c67fc640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b00039e0 0x7f67b0003dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.003+0000 7f67c67fc640 1 -- 172.21.15.64:0/3155894713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67b0003520 con 0x7f67b0004390 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.003+0000 7f67c6ffd640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b00039e0 0x7f67b0003dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.003+0000 7f67c67fc640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67b0004390 0x7f67b0004860 secure :-1 s=READY pgs=957 cs=0 l=1 rev1=1 crypto rx=0x7f67bc001860 tx=0x7f67b0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e766b7e873d8106 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67bc038070 con 0x7f67b0004390 2024-03-04T07:30:44.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f67bc033410 con 0x7f67b0004390 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67c5ffb640 1 -- 172.21.15.64:0/3155894713 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67bc001430 con 0x7f67b0004390 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67cd26d640 1 -- 172.21.15.64:0/3155894713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67b0004390 msgr2=0x7f67b0004860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67cd26d640 1 --2- 172.21.15.64:0/3155894713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67b0004390 0x7f67b0004860 secure :-1 s=READY pgs=957 cs=0 l=1 rev1=1 crypto rx=0x7f67bc001860 tx=0x7f67b0005830 comp rx=0 tx=0).stop 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67c77fe640 1 -- 172.21.15.64:0/3155894713 reap_dead start 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67cd26d640 1 -- 172.21.15.64:0/3155894713 shutdown_connections 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67cd26d640 1 -- 172.21.15.64:0/3155894713 >> 172.21.15.64:0/3155894713 conn(0x7f67c8071960 msgr2=0x7f67c8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67cd26d640 1 -- 172.21.15.64:0/3155894713 shutdown_connections 2024-03-04T07:30:44.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.004+0000 7f67cd26d640 1 -- 172.21.15.64:0/3155894713 wait complete. 2024-03-04T07:30:44.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.005+0000 7f67cd26d640 1 Processor -- start 2024-03-04T07:30:44.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.005+0000 7f67cd26d640 1 -- start start 2024-03-04T07:30:44.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67cd26d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b0004390 0x7f67c815b320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67cd26d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c80c9170 0x7f67c80ca1a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67cd26d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c81f4a40 0x7f67c80cc8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67cd26d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67c80cec70 con 0x7f67c81f4a40 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67cd26d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67c80ce750 con 0x7f67b0004390 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67cd26d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67c80cedf0 con 0x7f67c80c9170 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67c77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c81f4a40 0x7f67c80cc8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67c77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c81f4a40 0x7f67c80cc8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42144/0 (socket says 172.21.15.64:42144) 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67c77fe640 1 -- 172.21.15.64:0/4139889381 learned_addr learned my addr 172.21.15.64:0/4139889381 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67c67fc640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c80c9170 0x7f67c80ca1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.006+0000 7f67c6ffd640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b0004390 0x7f67c815b320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67c77fe640 1 -- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c80c9170 msgr2=0x7f67c80ca1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67c77fe640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c80c9170 0x7f67c80ca1a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67c77fe640 1 -- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b0004390 msgr2=0x7f67c815b320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:44.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67c77fe640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b0004390 0x7f67c815b320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67c77fe640 1 -- 172.21.15.64:0/4139889381 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67c81f4ed0 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67c77fe640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c81f4a40 0x7f67c80cc8f0 secure :-1 s=READY pgs=958 cs=0 l=1 rev1=1 crypto rx=0x7f67c80d11b0 tx=0x7f67c0007b10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67af7fe640 1 -- 172.21.15.64:0/4139889381 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67c0017020 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67af7fe640 1 -- 172.21.15.64:0/4139889381 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f67c000ab80 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f67c815bc20 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67af7fe640 1 -- 172.21.15.64:0/4139889381 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67b8009690 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.007+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f67c81f3f60 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.008+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f67c81f3d50 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.008+0000 7f67af7fe640 1 -- 172.21.15.64:0/4139889381 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f67c001a020 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.009+0000 7f67af7fe640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67b406b500 0x7f67b406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.009+0000 7f67af7fe640 1 -- 172.21.15.64:0/4139889381 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(71..71 src has 1..71) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f67c008a350 con 0x7f67c81f4a40 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.009+0000 7f67c6ffd640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67b406b500 0x7f67b406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:44.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.009+0000 7f67c6ffd640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67b406b500 0x7f67b406d9c0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f67c80c9b10 tx=0x7f67b8009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:44.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.013+0000 7f67af7fe640 1 -- 172.21.15.64:0/4139889381 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f67c0021020 con 0x7f67c81f4a40 2024-03-04T07:30:44.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.208+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f67c8072240 con 0x7f67c81f4a40 2024-03-04T07:30:44.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.953+0000 7f67af7fe640 1 -- 172.21.15.64:0/4139889381 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v73) v1 ==== 97+0+0 (secure 0 0 0) 0x7f67c0059560 con 0x7f67c81f4a40 2024-03-04T07:30:44.953 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:30:44.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67b406b500 msgr2=0x7f67b406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:44.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67b406b500 0x7f67b406d9c0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f67c80c9b10 tx=0x7f67b8009190 comp rx=0 tx=0).stop 2024-03-04T07:30:44.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c81f4a40 msgr2=0x7f67c80cc8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:44.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c81f4a40 0x7f67c80cc8f0 secure :-1 s=READY pgs=958 cs=0 l=1 rev1=1 crypto rx=0x7f67c80d11b0 tx=0x7f67c0007b10 comp rx=0 tx=0).stop 2024-03-04T07:30:44.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 shutdown_connections 2024-03-04T07:30:44.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67b406b500 0x7f67b406d9c0 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c81f4a40 0x7f67c80cc8f0 unknown :-1 s=CLOSED pgs=958 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c80c9170 0x7f67c80ca1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 --2- 172.21.15.64:0/4139889381 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67b0004390 0x7f67c815b320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:44.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 >> 172.21.15.64:0/4139889381 conn(0x7f67c8071960 msgr2=0x7f67c80b5780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:44.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 shutdown_connections 2024-03-04T07:30:44.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:44.956+0000 7f67cd26d640 1 -- 172.21.15.64:0/4139889381 wait complete. 2024-03-04T07:30:44.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:30:45.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.079+0000 7fe884f5e640 1 Processor -- start 2024-03-04T07:30:45.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.079+0000 7fe884f5e640 1 -- start start 2024-03-04T07:30:45.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe884f5e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe88006df30 0x7fe8800ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe884f5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8800aec40 0x7fe8800b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe884f5e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe880071240 con 0x7fe880158520 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe884f5e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe8800af0f0 con 0x7fe88006df30 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe884f5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe880071b70 con 0x7fe8800aec40 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe87ed9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8800aec40 0x7fe8800b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe877fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe88006df30 0x7fe8800ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe877fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe88006df30 0x7fe8800ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:51088/0 (socket says 172.21.15.64:51088) 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe87e59c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fe880158520 0x7fe88006d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:33950/0 (socket says 172.21.15.64:33950) 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe87ed9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8800aec40 0x7fe8800b3640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:42078/0 (socket says 172.21.15.64:42078) 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe877fff640 1 -- 172.21.15.64:0/830593263 learned_addr learned my addr 172.21.15.64:0/830593263 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe877fff640 1 -- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8800aec40 msgr2=0x7fe8800b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe877fff640 1 --2- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8800aec40 0x7fe8800b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe877fff640 1 -- 172.21.15.64:0/830593263 >> v1:172.21.15.66:6789/0 conn(0x7fe880158520 legacy=0x7fe88006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe877fff640 1 -- 172.21.15.64:0/830593263 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe880071520 con 0x7fe88006df30 2024-03-04T07:30:45.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.080+0000 7fe87ed9d640 1 --2- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8800aec40 0x7fe8800b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:45.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.081+0000 7fe877fff640 1 --2- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe88006df30 0x7fe8800ae700 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7fe868009870 tx=0x7fe86802b980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d9cf630595a1dfff server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.081+0000 7fe87dd9b640 1 -- 172.21.15.64:0/830593263 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe868034070 con 0x7fe88006df30 2024-03-04T07:30:45.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe87dd9b640 1 -- 172.21.15.64:0/830593263 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe868030840 con 0x7fe88006df30 2024-03-04T07:30:45.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe87dd9b640 1 -- 172.21.15.64:0/830593263 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe868030c10 con 0x7fe88006df30 2024-03-04T07:30:45.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 -- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe88006df30 msgr2=0x7fe8800ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 --2- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe88006df30 0x7fe8800ae700 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7fe868009870 tx=0x7fe86802b980 comp rx=0 tx=0).stop 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 -- 172.21.15.64:0/830593263 shutdown_connections 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 --2- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe8800aec40 0x7fe8800b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 --2- 172.21.15.64:0/830593263 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe88006df30 0x7fe8800ae700 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 -- 172.21.15.64:0/830593263 >> 172.21.15.64:0/830593263 conn(0x7fe880066d00 msgr2=0x7fe8800670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 -- 172.21.15.64:0/830593263 shutdown_connections 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.082+0000 7fe884f5e640 1 -- 172.21.15.64:0/830593263 wait complete. 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.083+0000 7fe884f5e640 1 Processor -- start 2024-03-04T07:30:45.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.083+0000 7fe884f5e640 1 -- start start 2024-03-04T07:30:45.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe884f5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe88006df30 0x7fe880151d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe884f5e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe8800aec40 0x7fe88014f830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe884f5e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe880158520 0x7fe8800cf240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe884f5e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe880158900 con 0x7fe880158520 2024-03-04T07:30:45.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe884f5e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe880072000 con 0x7fe8800aec40 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe884f5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe880158a80 con 0x7fe88006df30 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe87ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe880158520 0x7fe8800cf240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe87ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe880158520 0x7fe8800cf240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42146/0 (socket says 172.21.15.64:42146) 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe87ed9d640 1 -- 172.21.15.64:0/3320193877 learned_addr learned my addr 172.21.15.64:0/3320193877 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe87e59c640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe88006df30 0x7fe880151d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.084+0000 7fe877fff640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe8800aec40 0x7fe88014f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe87ed9d640 1 -- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe88006df30 msgr2=0x7fe880151d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe87ed9d640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe88006df30 0x7fe880151d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe87ed9d640 1 -- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe8800aec40 msgr2=0x7fe88014f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe87ed9d640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe8800aec40 0x7fe88014f830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe87ed9d640 1 -- 172.21.15.64:0/3320193877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe880155640 con 0x7fe880158520 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe87ed9d640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe880158520 0x7fe8800cf240 secure :-1 s=READY pgs=959 cs=0 l=1 rev1=1 crypto rx=0x7fe878009f00 tx=0x7fe87800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe876ffd640 1 -- 172.21.15.64:0/3320193877 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe878018030 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe876ffd640 1 -- 172.21.15.64:0/3320193877 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe87800ce50 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe880155920 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe876ffd640 1 -- 172.21.15.64:0/3320193877 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe878003bb0 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.085+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe880155ee0 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.086+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe880074980 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.087+0000 7fe876ffd640 1 -- 172.21.15.64:0/3320193877 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe87801b070 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.087+0000 7fe876ffd640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe86006b4d0 0x7fe86006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.087+0000 7fe876ffd640 1 -- 172.21.15.64:0/3320193877 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fe87808a800 con 0x7fe880158520 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.087+0000 7fe87e59c640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe86006b4d0 0x7fe86006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.088+0000 7fe87e59c640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe86006b4d0 0x7fe86006d990 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7fe870007fd0 tx=0x7fe870007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.091+0000 7fe876ffd640 1 -- 172.21.15.64:0/3320193877 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe880074980 con 0x7fe880158520 2024-03-04T07:30:45.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.346+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7fe880067ff0 con 0x7fe880158520 2024-03-04T07:30:45.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.346+0000 7fe876ffd640 1 -- 172.21.15.64:0/3320193877 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v18) v1 ==== 80+0+5304 (secure 0 0 0) 0x7fe878010040 con 0x7fe880158520 2024-03-04T07:30:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:45.346 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:30:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:30:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:30:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:30:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow r", 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:30:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:30:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:30:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:30:45.359 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:30:45.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe86006b4d0 msgr2=0x7fe86006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe86006b4d0 0x7fe86006d990 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7fe870007fd0 tx=0x7fe870007e80 comp rx=0 tx=0).stop 2024-03-04T07:30:45.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe880158520 msgr2=0x7fe8800cf240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe880158520 0x7fe8800cf240 secure :-1 s=READY pgs=959 cs=0 l=1 rev1=1 crypto rx=0x7fe878009f00 tx=0x7fe87800c490 comp rx=0 tx=0).stop 2024-03-04T07:30:45.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 shutdown_connections 2024-03-04T07:30:45.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe86006b4d0 0x7fe86006d990 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe880158520 0x7fe8800cf240 unknown :-1 s=CLOSED pgs=959 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe8800aec40 0x7fe88014f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 --2- 172.21.15.64:0/3320193877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe88006df30 0x7fe880151d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 >> 172.21.15.64:0/3320193877 conn(0x7fe880066d00 msgr2=0x7fe8800b2290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:45.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 shutdown_connections 2024-03-04T07:30:45.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.348+0000 7fe884f5e640 1 -- 172.21.15.64:0/3320193877 wait complete. 2024-03-04T07:30:45.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:45.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.465+0000 7f1b194fb640 1 Processor -- start 2024-03-04T07:30:45.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.465+0000 7f1b194fb640 1 -- start start 2024-03-04T07:30:45.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b194fb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1b140d18c0 0x7f1b14071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b194fb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b140b6f20 con 0x7f1b14071a80 2024-03-04T07:30:45.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b194fb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1b140d3030 con 0x7f1b1415df70 2024-03-04T07:30:45.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b194fb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1b14071eb0 con 0x7f1b140d18c0 2024-03-04T07:30:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b127fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1b140d18c0 0x7f1b14071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b137fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1b14071a80 0x7f1b140b2700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48008/0 (socket says 172.21.15.64:48008) 2024-03-04T07:30:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b137fe640 1 -- 172.21.15.64:0/116777163 learned_addr learned my addr 172.21.15.64:0/116777163 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b127fc640 1 -- 172.21.15.64:0/116777163 >> v1:172.21.15.66:6789/0 conn(0x7f1b1415df70 legacy=0x7f1b140d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b127fc640 1 -- 172.21.15.64:0/116777163 >> v1:172.21.15.64:6789/0 conn(0x7f1b14071a80 legacy=0x7f1b140b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b127fc640 1 -- 172.21.15.64:0/116777163 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b140d3310 con 0x7f1b140d18c0 2024-03-04T07:30:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.466+0000 7f1b11ffb640 1 -- 172.21.15.64:0/116777163 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2990626982 0 0) 0x7f1b140b6f20 con 0x7f1b14071a80 2024-03-04T07:30:45.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.467+0000 7f1b127fc640 1 --2- 172.21.15.64:0/116777163 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b140d18c0 0x7f1b14071540 secure :-1 s=READY pgs=961 cs=0 l=1 rev1=1 crypto rx=0x7f1afc0097a0 tx=0x7f1afc02b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dc2330832402c2cf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.467+0000 7f1b11ffb640 1 -- 172.21.15.64:0/116777163 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1afc034070 con 0x7f1b140d18c0 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.467+0000 7f1b11ffb640 1 -- 172.21.15.64:0/116777163 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1afc030610 con 0x7f1b140d18c0 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.467+0000 7f1b11ffb640 1 -- 172.21.15.64:0/116777163 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1afc0309e0 con 0x7f1b140d18c0 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.468+0000 7f1b194fb640 1 -- 172.21.15.64:0/116777163 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b140d18c0 msgr2=0x7f1b14071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.468+0000 7f1b194fb640 1 --2- 172.21.15.64:0/116777163 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b140d18c0 0x7f1b14071540 secure :-1 s=READY pgs=961 cs=0 l=1 rev1=1 crypto rx=0x7f1afc0097a0 tx=0x7f1afc02b960 comp rx=0 tx=0).stop 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.468+0000 7f1b194fb640 1 -- 172.21.15.64:0/116777163 shutdown_connections 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.468+0000 7f1b194fb640 1 --2- 172.21.15.64:0/116777163 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b140d18c0 0x7f1b14071540 unknown :-1 s=CLOSED pgs=961 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.468+0000 7f1b194fb640 1 -- 172.21.15.64:0/116777163 >> 172.21.15.64:0/116777163 conn(0x7f1b140d3cf0 msgr2=0x7f1b140d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.468+0000 7f1b194fb640 1 -- 172.21.15.64:0/116777163 shutdown_connections 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.468+0000 7f1b194fb640 1 -- 172.21.15.64:0/116777163 wait complete. 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.469+0000 7f1b194fb640 1 Processor -- start 2024-03-04T07:30:45.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.469+0000 7f1b194fb640 1 -- start start 2024-03-04T07:30:45.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b194fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b14071a80 0x7f1b140cadc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b194fb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b140d18c0 0x7f1b141f6c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b194fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b1415df70 0x7f1b14160990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b194fb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b1415e350 con 0x7f1b1415df70 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b194fb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b140d38d0 con 0x7f1b140d18c0 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b194fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1b1415e4d0 con 0x7f1b14071a80 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b1415df70 0x7f1b14160990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b1415df70 0x7f1b14160990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42156/0 (socket says 172.21.15.64:42156) 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 -- 172.21.15.64:0/2851293850 learned_addr learned my addr 172.21.15.64:0/2851293850 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b127fc640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b140d18c0 0x7f1b141f6c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b12ffd640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b14071a80 0x7f1b140cadc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 -- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b14071a80 msgr2=0x7f1b140cadc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b14071a80 0x7f1b140cadc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 -- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b140d18c0 msgr2=0x7f1b141f6c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b140d18c0 0x7f1b141f6c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 -- 172.21.15.64:0/2851293850 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b140cb610 con 0x7f1b1415df70 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.470+0000 7f1b137fe640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b1415df70 0x7f1b14160990 secure :-1 s=READY pgs=962 cs=0 l=1 rev1=1 crypto rx=0x7f1b0c00c480 tx=0x7f1b0c00c940 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.471+0000 7f1b0b7fe640 1 -- 172.21.15.64:0/2851293850 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b0c018070 con 0x7f1b1415df70 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.471+0000 7f1b0b7fe640 1 -- 172.21.15.64:0/2851293850 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1b0c007500 con 0x7f1b1415df70 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.471+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b140cb8f0 con 0x7f1b1415df70 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.471+0000 7f1b0b7fe640 1 -- 172.21.15.64:0/2851293850 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b0c004a20 con 0x7f1b1415df70 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.471+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b140cbeb0 con 0x7f1b1415df70 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.472+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1b14067ff0 con 0x7f1b1415df70 2024-03-04T07:30:45.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.472+0000 7f1b0b7fe640 1 -- 172.21.15.64:0/2851293850 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1b0c01b070 con 0x7f1b1415df70 2024-03-04T07:30:45.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.472+0000 7f1b0b7fe640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1af406b500 0x7f1af406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.473+0000 7f1b12ffd640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1af406b500 0x7f1af406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.473+0000 7f1b0b7fe640 1 -- 172.21.15.64:0/2851293850 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f1b0c089130 con 0x7f1b1415df70 2024-03-04T07:30:45.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.473+0000 7f1b12ffd640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1af406b500 0x7f1af406d9c0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f1b04009f20 tx=0x7f1b040094d0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.477+0000 7f1b0b7fe640 1 -- 172.21.15.64:0/2851293850 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1b0c058390 con 0x7f1b1415df70 2024-03-04T07:30:45.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.660+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1b140b9140 con 0x7f1b1415df70 2024-03-04T07:30:45.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.661+0000 7f1b0b7fe640 1 -- 172.21.15.64:0/2851293850 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v75) v1 ==== 74+0+12844 (secure 0 0 0) 0x7f1b0c057c90 con 0x7f1b1415df70 2024-03-04T07:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":75,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:44.968385+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1af406b500 msgr2=0x7f1af406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1af406b500 0x7f1af406d9c0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f1b04009f20 tx=0x7f1b040094d0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b1415df70 msgr2=0x7f1b14160990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b1415df70 0x7f1b14160990 secure :-1 s=READY pgs=962 cs=0 l=1 rev1=1 crypto rx=0x7f1b0c00c480 tx=0x7f1b0c00c940 comp rx=0 tx=0).stop 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 shutdown_connections 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1af406b500 0x7f1af406d9c0 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1b1415df70 0x7f1b14160990 unknown :-1 s=CLOSED pgs=962 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1b140d18c0 0x7f1b141f6c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 --2- 172.21.15.64:0/2851293850 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1b14071a80 0x7f1b140cadc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 >> 172.21.15.64:0/2851293850 conn(0x7f1b140d3cf0 msgr2=0x7f1b140b5f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 shutdown_connections 2024-03-04T07:30:45.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.663+0000 7f1b194fb640 1 -- 172.21.15.64:0/2851293850 wait complete. 2024-03-04T07:30:45.674 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:30:45.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:30:45.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.778+0000 7f1c66bb5640 1 Processor -- start 2024-03-04T07:30:45.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.778+0000 7f1c66bb5640 1 -- start start 2024-03-04T07:30:45.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c66bb5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c600b2910 0x7f1c600b0440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c66bb5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1c6015faf0 con 0x7f1c600b2d90 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c66bb5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1c600d0560 con 0x7f1c6015f710 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c66bb5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1c600affb0 con 0x7f1c600b2910 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c5ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c600b2910 0x7f1c600b0440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c5ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c600b2910 0x7f1c600b0440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42164/0 (socket says 172.21.15.64:42164) 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c65152640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1c600b2d90 0x7f1c600b0980 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48018/0 (socket says 172.21.15.64:48018) 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.779+0000 7f1c5ffff640 1 -- 172.21.15.64:0/788249573 learned_addr learned my addr 172.21.15.64:0/788249573 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.780+0000 7f1c5ffff640 1 -- 172.21.15.64:0/788249573 >> v1:172.21.15.66:6789/0 conn(0x7f1c6015f710 legacy=0x7f1c600cc5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.780+0000 7f1c5ffff640 1 -- 172.21.15.64:0/788249573 >> v1:172.21.15.64:6789/0 conn(0x7f1c600b2d90 legacy=0x7f1c600b0980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.780+0000 7f1c5ffff640 1 -- 172.21.15.64:0/788249573 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c600d0840 con 0x7f1c600b2910 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.780+0000 7f1c5ffff640 1 --2- 172.21.15.64:0/788249573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c600b0440 secure :-1 s=READY pgs=964 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c009870 tx=0x7f1c4c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1b158988599af864 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.780+0000 7f1c5f7fe640 1 -- 172.21.15.64:0/788249573 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c4c035020 con 0x7f1c600b2910 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c5f7fe640 1 -- 172.21.15.64:0/788249573 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1c4c031870 con 0x7f1c600b2910 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c5f7fe640 1 -- 172.21.15.64:0/788249573 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c4c031c40 con 0x7f1c600b2910 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c66bb5640 1 -- 172.21.15.64:0/788249573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 msgr2=0x7f1c600b0440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/788249573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c600b0440 secure :-1 s=READY pgs=964 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c009870 tx=0x7f1c4c02b910 comp rx=0 tx=0).stop 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c66bb5640 1 -- 172.21.15.64:0/788249573 shutdown_connections 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/788249573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c600b0440 unknown :-1 s=CLOSED pgs=964 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c66bb5640 1 -- 172.21.15.64:0/788249573 >> 172.21.15.64:0/788249573 conn(0x7f1c600b3770 msgr2=0x7f1c600b3b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c66bb5640 1 -- 172.21.15.64:0/788249573 shutdown_connections 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.781+0000 7f1c66bb5640 1 -- 172.21.15.64:0/788249573 wait complete. 2024-03-04T07:30:45.782 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.782+0000 7f1c66bb5640 1 Processor -- start 2024-03-04T07:30:45.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.782+0000 7f1c66bb5640 1 -- start start 2024-03-04T07:30:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c66bb5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c60159380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c66bb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c600b2d90 0x7f1c6006c220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c66bb5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c6015f710 0x7f1c6006c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c66bb5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c600d01c0 con 0x7f1c600b2910 2024-03-04T07:30:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c66bb5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c600af9f0 con 0x7f1c6015f710 2024-03-04T07:30:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c66bb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1c600d0340 con 0x7f1c600b2d90 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c60159380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c60159380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42168/0 (socket says 172.21.15.64:42168) 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 -- 172.21.15.64:0/2143806851 learned_addr learned my addr 172.21.15.64:0/2143806851 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c5ffff640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c600b2d90 0x7f1c6006c220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c65152640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c6015f710 0x7f1c6006c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 -- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c600b2d90 msgr2=0x7f1c6006c220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c600b2d90 0x7f1c6006c220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 -- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c6015f710 msgr2=0x7f1c6006c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c6015f710 0x7f1c6006c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.783+0000 7f1c64951640 1 -- 172.21.15.64:0/2143806851 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c600c4770 con 0x7f1c600b2910 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.784+0000 7f1c64951640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c60159380 secure :-1 s=READY pgs=965 cs=0 l=1 rev1=1 crypto rx=0x7f1c5400b410 tx=0x7f1c5400b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.784+0000 7f1c5d7fa640 1 -- 172.21.15.64:0/2143806851 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c54015020 con 0x7f1c600b2910 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.784+0000 7f1c5d7fa640 1 -- 172.21.15.64:0/2143806851 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1c54011830 con 0x7f1c600b2910 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.784+0000 7f1c5d7fa640 1 -- 172.21.15.64:0/2143806851 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c54011c20 con 0x7f1c600b2910 2024-03-04T07:30:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.784+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1c6006cd90 con 0x7f1c600b2910 2024-03-04T07:30:45.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.784+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c601f47c0 con 0x7f1c600b2910 2024-03-04T07:30:45.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.785+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1c6009fd60 con 0x7f1c600b2910 2024-03-04T07:30:45.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.785+0000 7f1c5d7fa640 1 -- 172.21.15.64:0/2143806851 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1c54018020 con 0x7f1c600b2910 2024-03-04T07:30:45.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.786+0000 7f1c5d7fa640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c4406b4d0 0x7f1c4406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:45.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.786+0000 7f1c5d7fa640 1 -- 172.21.15.64:0/2143806851 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f1c54089d90 con 0x7f1c600b2910 2024-03-04T07:30:45.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.786+0000 7f1c5ffff640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c4406b4d0 0x7f1c4406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:45.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.786+0000 7f1c5ffff640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c4406b4d0 0x7f1c4406d990 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c036000 tx=0x7f1c4c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:45.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.790+0000 7f1c5d7fa640 1 -- 172.21.15.64:0/2143806851 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1c5401f020 con 0x7f1c600b2910 2024-03-04T07:30:45.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.972+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f1c6004f410 con 0x7f1c600b2910 2024-03-04T07:30:45.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.973+0000 7f1c5d7fa640 1 -- 172.21.15.64:0/2143806851 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v75) v1 ==== 93+0+10 (secure 0 0 0) 0x7f1c540172c0 con 0x7f1c600b2910 2024-03-04T07:30:45.973 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:30:45.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.975+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c4406b4d0 msgr2=0x7f1c4406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.975+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c4406b4d0 0x7f1c4406d990 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c036000 tx=0x7f1c4c03c040 comp rx=0 tx=0).stop 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.975+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 msgr2=0x7f1c60159380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.975+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c60159380 secure :-1 s=READY pgs=965 cs=0 l=1 rev1=1 crypto rx=0x7f1c5400b410 tx=0x7f1c5400b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.975+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 shutdown_connections 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.975+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c4406b4d0 0x7f1c4406d990 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.975+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c6015f710 0x7f1c6006c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.976+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c600b2d90 0x7f1c6006c220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.976+0000 7f1c66bb5640 1 --2- 172.21.15.64:0/2143806851 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c600b2910 0x7f1c60159380 unknown :-1 s=CLOSED pgs=965 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.976+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 >> 172.21.15.64:0/2143806851 conn(0x7f1c600b3770 msgr2=0x7f1c600b1840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.976+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 shutdown_connections 2024-03-04T07:30:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:45.976+0000 7f1c66bb5640 1 -- 172.21.15.64:0/2143806851 wait complete. 2024-03-04T07:30:45.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:30:46.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.088+0000 7fd38fa39640 1 Processor -- start 2024-03-04T07:30:46.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38fa39640 1 -- start start 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38fa39640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 0x7fd3880b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38fa39640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd3880d3400 0x7fd388071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38fa39640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd388071e60 con 0x7fd3880b2b50 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38fa39640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd3880d2f70 con 0x7fd3880d3400 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38fa39640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd3880d4be0 con 0x7fd388071540 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38dfd6640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd3880b2b50 0x7fd3880d1870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48030/0 (socket says 172.21.15.64:48030) 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.089+0000 7fd38dfd6640 1 -- 172.21.15.64:0/3300595010 learned_addr learned my addr 172.21.15.64:0/3300595010 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38d7d5640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 0x7fd3880b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3719519025 0 0) 0x7fd388071e60 con 0x7fd3880b2b50 2024-03-04T07:30:46.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd374003520 con 0x7fd3880b2b50 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3300595010 >> v2:172.21.15.66:3300/0 conn(0x7fd3880d3400 0x7fd388071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1743330706 0 0) 0x7fd374003520 con 0x7fd3880b2b50 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 msgr2=0x7fd3880b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 0x7fd3880b30a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 >> v2:172.21.15.66:3300/0 conn(0x7fd3880d3400 msgr2=0x7fd388071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 --2- 172.21.15.64:0/3300595010 >> v2:172.21.15.66:3300/0 conn(0x7fd3880d3400 0x7fd388071920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3880ce610 con 0x7fd3880b2b50 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd3800019f0 con 0x7fd3880b2b50 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 >> v1:172.21.15.64:6789/0 conn(0x7fd3880b2b50 legacy=0x7fd3880d1870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 0x7fd3740039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd374003f30 0x7fd374004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374004d40 0x7fd3740051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3880d2f70 con 0x7fd374003f30 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3880d4be0 con 0x7fd374004d40 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd37400b9c0 con 0x7fd388071540 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd374003f30 0x7fd374004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38d7d5640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 0x7fd3740039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38dfd6640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374004d40 0x7fd3740051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38cfd4640 1 -- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 msgr2=0x7fd3740039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd388071540 0x7fd3740039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.090+0000 7fd38cfd4640 1 -- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374004d40 msgr2=0x7fd3740051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.091+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374004d40 0x7fd3740051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.091+0000 7fd38cfd4640 1 -- 172.21.15.64:0/3300595010 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd374003520 con 0x7fd374003f30 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.091+0000 7fd38dfd6640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374004d40 0x7fd3740051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.091+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd374003f30 0x7fd374004800 secure :-1 s=READY pgs=967 cs=0 l=1 rev1=1 crypto rx=0x7fd378009a90 tx=0x7fd374005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4445f60d89235508 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.091+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd378039070 con 0x7fd374003f30 2024-03-04T07:30:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.091+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd37802c6b0 con 0x7fd374003f30 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.091+0000 7fd37ffff640 1 -- 172.21.15.64:0/3300595010 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd378009690 con 0x7fd374003f30 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.092+0000 7fd38fa39640 1 -- 172.21.15.64:0/3300595010 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd374003f30 msgr2=0x7fd374004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.092+0000 7fd38fa39640 1 --2- 172.21.15.64:0/3300595010 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd374003f30 0x7fd374004800 secure :-1 s=READY pgs=967 cs=0 l=1 rev1=1 crypto rx=0x7fd378009a90 tx=0x7fd374005830 comp rx=0 tx=0).stop 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.092+0000 7fd38dfd6640 1 -- 172.21.15.64:0/3300595010 reap_dead start 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.092+0000 7fd38fa39640 1 -- 172.21.15.64:0/3300595010 shutdown_connections 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.092+0000 7fd38fa39640 1 -- 172.21.15.64:0/3300595010 >> 172.21.15.64:0/3300595010 conn(0x7fd3880d3ca0 msgr2=0x7fd3880d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.092+0000 7fd38fa39640 1 -- 172.21.15.64:0/3300595010 shutdown_connections 2024-03-04T07:30:46.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.092+0000 7fd38fa39640 1 -- 172.21.15.64:0/3300595010 wait complete. 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.093+0000 7fd38fa39640 1 Processor -- start 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.093+0000 7fd38fa39640 1 -- start start 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38fa39640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374003f30 0x7fd3881521b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38fa39640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3880b4550 0x7fd3880b5500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38fa39640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3880b49d0 0x7fd3880b5a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38fa39640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3880ce8f0 con 0x7fd3880b4550 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38fa39640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3880d2c00 con 0x7fd374003f30 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38fa39640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd38815e370 con 0x7fd3880b49d0 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3880b4550 0x7fd3880b5500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3880b4550 0x7fd3880b5500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42190/0 (socket says 172.21.15.64:42190) 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 -- 172.21.15.64:0/3601612819 learned_addr learned my addr 172.21.15.64:0/3601612819 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:46.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38dfd6640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3880b49d0 0x7fd3880b5a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38d7d5640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374003f30 0x7fd3881521b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 -- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3880b49d0 msgr2=0x7fd3880b5a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3880b49d0 0x7fd3880b5a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 -- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374003f30 msgr2=0x7fd3881521b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374003f30 0x7fd3881521b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38cfd4640 1 -- 172.21.15.64:0/3601612819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3881526f0 con 0x7fd3880b4550 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.094+0000 7fd38d7d5640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374003f30 0x7fd3881521b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.095+0000 7fd38cfd4640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3880b4550 0x7fd3880b5500 secure :-1 s=READY pgs=968 cs=0 l=1 rev1=1 crypto rx=0x7fd378009bb0 tx=0x7fd378004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.095+0000 7fd37dffb640 1 -- 172.21.15.64:0/3601612819 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd378042030 con 0x7fd3880b4550 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.095+0000 7fd37dffb640 1 -- 172.21.15.64:0/3601612819 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd37803a040 con 0x7fd3880b4550 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.095+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3880b34b0 con 0x7fd3880b4550 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.095+0000 7fd37dffb640 1 -- 172.21.15.64:0/3601612819 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd378039070 con 0x7fd3880b4550 2024-03-04T07:30:46.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.095+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3880b3a70 con 0x7fd3880b4550 2024-03-04T07:30:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.096+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd388068700 con 0x7fd3880b4550 2024-03-04T07:30:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.096+0000 7fd37dffb640 1 -- 172.21.15.64:0/3601612819 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd378045070 con 0x7fd3880b4550 2024-03-04T07:30:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.097+0000 7fd37dffb640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd36c06b4d0 0x7fd36c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.097+0000 7fd37dffb640 1 -- 172.21.15.64:0/3601612819 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fd3780aac30 con 0x7fd3880b4550 2024-03-04T07:30:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.097+0000 7fd38d7d5640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd36c06b4d0 0x7fd36c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.097+0000 7fd38d7d5640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd36c06b4d0 0x7fd36c06d990 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fd3840099e0 tx=0x7fd384008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:46.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.101+0000 7fd37dffb640 1 -- 172.21.15.64:0/3601612819 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd3881526f0 con 0x7fd3880b4550 2024-03-04T07:30:46.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.289+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7fd388071ac0 con 0x7fd3880b4550 2024-03-04T07:30:46.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.290+0000 7fd37dffb640 1 -- 172.21.15.64:0/3601612819 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v75) v1 ==== 84+0+65 (secure 0 0 0) 0x7fd378079e90 con 0x7fd3880b4550 2024-03-04T07:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:30:46.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd36c06b4d0 msgr2=0x7fd36c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd36c06b4d0 0x7fd36c06d990 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fd3840099e0 tx=0x7fd384008040 comp rx=0 tx=0).stop 2024-03-04T07:30:46.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3880b4550 msgr2=0x7fd3880b5500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3880b4550 0x7fd3880b5500 secure :-1 s=READY pgs=968 cs=0 l=1 rev1=1 crypto rx=0x7fd378009bb0 tx=0x7fd378004730 comp rx=0 tx=0).stop 2024-03-04T07:30:46.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 shutdown_connections 2024-03-04T07:30:46.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd36c06b4d0 0x7fd36c06d990 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd3880b49d0 0x7fd3880b5a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd3880b4550 0x7fd3880b5500 unknown :-1 s=CLOSED pgs=968 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 --2- 172.21.15.64:0/3601612819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd374003f30 0x7fd3881521b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 >> 172.21.15.64:0/3601612819 conn(0x7fd3880d3ca0 msgr2=0x7fd38815f030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:46.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 shutdown_connections 2024-03-04T07:30:46.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.292+0000 7fd38fa39640 1 -- 172.21.15.64:0/3601612819 wait complete. 2024-03-04T07:30:46.302 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:30:46.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:30:46.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.408+0000 7fcc90956640 1 Processor -- start 2024-03-04T07:30:46.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.409+0000 7fcc90956640 1 -- start start 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.409+0000 7fcc90956640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcc8c0ceea0 0x7fcc8c0d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.409+0000 7fcc90956640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc8c0d3740 con 0x7fcc8c15dd90 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.409+0000 7fcc90956640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc8c0d3120 con 0x7fcc8c0b6d10 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.409+0000 7fcc90956640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc8c0d3400 con 0x7fcc8c0ceea0 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.409+0000 7fcc8a59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcc8c15dd90 0x7fcc8c0d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48038/0 (socket says 172.21.15.64:48038) 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.409+0000 7fcc8a59c640 1 -- 172.21.15.64:0/425541576 learned_addr learned my addr 172.21.15.64:0/425541576 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2439417786 0 0) 0x7fcc8c0d3740 con 0x7fcc8c15dd90 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc83fff640 1 --2- 172.21.15.64:0/425541576 >> v2:172.21.15.66:3300/0 conn(0x7fcc8c0ceea0 0x7fcc8c0d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcc78003520 con 0x7fcc8c15dd90 2024-03-04T07:30:46.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 284991374 0 0) 0x7fcc78003520 con 0x7fcc8c15dd90 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 >> v2:172.21.15.66:3300/0 conn(0x7fcc8c0ceea0 msgr2=0x7fcc8c0d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 --2- 172.21.15.64:0/425541576 >> v2:172.21.15.66:3300/0 conn(0x7fcc8c0ceea0 0x7fcc8c0d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 >> v1:172.21.15.66:6789/0 conn(0x7fcc8c0b6d10 legacy=0x7fcc8c0d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc8c0d39e0 con 0x7fcc8c15dd90 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fcc7c001670 con 0x7fcc8c15dd90 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 >> v1:172.21.15.64:6789/0 conn(0x7fcc8c15dd90 legacy=0x7fcc8c0d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc780039b0 0x7fcc78003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc78004800 0x7fcc78004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc780051c0 0x7fcc7800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc8c0d3400 con 0x7fcc78004800 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc7800c070 con 0x7fcc780039b0 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc7800c270 con 0x7fcc780051c0 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc83fff640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc78004800 0x7fcc78004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc8a59c640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc780039b0 0x7fcc78003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc8ad9d640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc780051c0 0x7fcc7800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc83fff640 1 -- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc780051c0 msgr2=0x7fcc7800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc83fff640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc780051c0 0x7fcc7800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc83fff640 1 -- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc780039b0 msgr2=0x7fcc78003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc83fff640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc780039b0 0x7fcc78003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc83fff640 1 -- 172.21.15.64:0/425541576 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc78003520 con 0x7fcc78004800 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.410+0000 7fcc8ad9d640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc780051c0 0x7fcc7800b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.411+0000 7fcc8ad9d640 1 -- 172.21.15.64:0/425541576 reap_dead start 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.411+0000 7fcc83fff640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc78004800 0x7fcc78004c80 secure :-1 s=READY pgs=970 cs=0 l=1 rev1=1 crypto rx=0x7fcc74009ac0 tx=0x7fcc78005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ab10599650946404 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.411+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc74039070 con 0x7fcc78004800 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.411+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc7402c6b0 con 0x7fcc78004800 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.411+0000 7fcc89d9b640 1 -- 172.21.15.64:0/425541576 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc74009690 con 0x7fcc78004800 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.412+0000 7fcc90956640 1 -- 172.21.15.64:0/425541576 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc78004800 msgr2=0x7fcc78004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.412+0000 7fcc90956640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc78004800 0x7fcc78004c80 secure :-1 s=READY pgs=970 cs=0 l=1 rev1=1 crypto rx=0x7fcc74009ac0 tx=0x7fcc78005830 comp rx=0 tx=0).stop 2024-03-04T07:30:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.412+0000 7fcc90956640 1 -- 172.21.15.64:0/425541576 shutdown_connections 2024-03-04T07:30:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.412+0000 7fcc90956640 1 --2- 172.21.15.64:0/425541576 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc78004800 0x7fcc78004c80 unknown :-1 s=CLOSED pgs=970 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.412+0000 7fcc90956640 1 -- 172.21.15.64:0/425541576 >> 172.21.15.64:0/425541576 conn(0x7fcc8c071960 msgr2=0x7fcc8c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.412+0000 7fcc90956640 1 -- 172.21.15.64:0/425541576 shutdown_connections 2024-03-04T07:30:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.412+0000 7fcc90956640 1 -- 172.21.15.64:0/425541576 wait complete. 2024-03-04T07:30:46.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.413+0000 7fcc90956640 1 Processor -- start 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.413+0000 7fcc90956640 1 -- start start 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc90956640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc78004800 0x7fcc8c146c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc90956640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc8c1f4120 0x7fcc8c1f45a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc90956640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc8c0ccea0 0x7fcc8c0cd330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc90956640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc8c0b71a0 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc90956640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc8c15e170 con 0x7fcc78004800 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc90956640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc8c0b7320 con 0x7fcc8c1f4120 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc8c0ccea0 0x7fcc8c0cd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc8c0ccea0 0x7fcc8c0cd330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42212/0 (socket says 172.21.15.64:42212) 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 -- 172.21.15.64:0/1120703602 learned_addr learned my addr 172.21.15.64:0/1120703602 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc83fff640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc8c1f4120 0x7fcc8c1f45a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8a59c640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc78004800 0x7fcc8c146c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 -- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc8c1f4120 msgr2=0x7fcc8c1f45a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc8c1f4120 0x7fcc8c1f45a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 -- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc78004800 msgr2=0x7fcc8c146c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc78004800 0x7fcc8c146c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.414+0000 7fcc8ad9d640 1 -- 172.21.15.64:0/1120703602 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc8c1f4ae0 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.415+0000 7fcc8ad9d640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc8c0ccea0 0x7fcc8c0cd330 secure :-1 s=READY pgs=971 cs=0 l=1 rev1=1 crypto rx=0x7fcc8c072480 tx=0x7fcc8400eb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.415+0000 7fcc82ffd640 1 -- 172.21.15.64:0/1120703602 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc84003970 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.415+0000 7fcc82ffd640 1 -- 172.21.15.64:0/1120703602 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc84018070 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.415+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc8c0cdbd0 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.415+0000 7fcc82ffd640 1 -- 172.21.15.64:0/1120703602 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc84003d50 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.415+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc8c0ce190 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.416+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc8c162830 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.416+0000 7fcc82ffd640 1 -- 172.21.15.64:0/1120703602 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcc840050d0 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.416+0000 7fcc82ffd640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc6c06b4d0 0x7fcc6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.417+0000 7fcc82ffd640 1 -- 172.21.15.64:0/1120703602 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fcc84089840 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.417+0000 7fcc8a59c640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc6c06b4d0 0x7fcc6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:46.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.417+0000 7fcc8a59c640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc6c06b4d0 0x7fcc6c06d990 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fcc8c14a630 tx=0x7fcc7c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:46.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.421+0000 7fcc82ffd640 1 -- 172.21.15.64:0/1120703602 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcc8c162830 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.601+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7fcc8c072a30 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.973+0000 7fcc82ffd640 1 -- 172.21.15.64:0/1120703602 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v76) v1 ==== 138+0+0 (secure 0 0 0) 0x7fcc84022020 con 0x7fcc8c0ccea0 2024-03-04T07:30:46.973 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:30:46.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc6c06b4d0 msgr2=0x7fcc6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc6c06b4d0 0x7fcc6c06d990 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fcc8c14a630 tx=0x7fcc7c009040 comp rx=0 tx=0).stop 2024-03-04T07:30:46.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc8c0ccea0 msgr2=0x7fcc8c0cd330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:46.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc8c0ccea0 0x7fcc8c0cd330 secure :-1 s=READY pgs=971 cs=0 l=1 rev1=1 crypto rx=0x7fcc8c072480 tx=0x7fcc8400eb90 comp rx=0 tx=0).stop 2024-03-04T07:30:46.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 shutdown_connections 2024-03-04T07:30:46.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc6c06b4d0 0x7fcc6c06d990 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc8c0ccea0 0x7fcc8c0cd330 unknown :-1 s=CLOSED pgs=971 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc8c1f4120 0x7fcc8c1f45a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 --2- 172.21.15.64:0/1120703602 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc78004800 0x7fcc8c146c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:46.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 >> 172.21.15.64:0/1120703602 conn(0x7fcc8c071960 msgr2=0x7fcc8c0d4030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:46.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 shutdown_connections 2024-03-04T07:30:46.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:46.976+0000 7fcc90956640 1 -- 172.21.15.64:0/1120703602 wait complete. 2024-03-04T07:30:46.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:30:47.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.102+0000 7f669ff9a640 1 Processor -- start 2024-03-04T07:30:47.102 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.102+0000 7f669ff9a640 1 -- start start 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669ff9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 0x7f6698071160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669ff9a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66980b2e30 con 0x7f669815ae50 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669ff9a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f669806dfe0 con 0x7f669815b690 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669ff9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66980b30b0 con 0x7f669815b230 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669dd36640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f669815ae50 0x7f66980708e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48044/0 (socket says 172.21.15.64:48044) 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669dd36640 1 -- 172.21.15.64:0/389235307 learned_addr learned my addr 172.21.15.64:0/389235307 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2260048317 0 0) 0x7f6690001110 con 0x7f669815ae50 2024-03-04T07:30:47.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6680003520 con 0x7f669815ae50 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.103+0000 7f669d535640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 0x7f6698071160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3202934198 0 0) 0x7f6680003520 con 0x7f669815ae50 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 msgr2=0x7f6698071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 0x7f6698071160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 >> v1:172.21.15.66:6789/0 conn(0x7f669815b690 legacy=0x7f6698150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66980b3390 con 0x7f669815ae50 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f6690001840 con 0x7f669815ae50 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 >> v1:172.21.15.64:6789/0 conn(0x7f669815ae50 legacy=0x7f66980708e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66800039e0 0x7f6680003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 0x7f6680004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6680004d40 0x7f66800051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66980b30b0 con 0x7f6680004d40 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f668000cc30 con 0x7f66800039e0 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f668000ce30 con 0x7f669815b230 2024-03-04T07:30:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669e537640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6680004d40 0x7f66800051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669dd36640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66800039e0 0x7f6680003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669d535640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 0x7f6680004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669e537640 1 -- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 msgr2=0x7f6680004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669e537640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f669815b230 0x7f6680004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669e537640 1 -- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66800039e0 msgr2=0x7f6680003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669e537640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66800039e0 0x7f6680003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.104+0000 7f669e537640 1 -- 172.21.15.64:0/389235307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6680003520 con 0x7f6680004d40 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.105+0000 7f669e537640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6680004d40 0x7f66800051c0 secure :-1 s=READY pgs=973 cs=0 l=1 rev1=1 crypto rx=0x7f669815a980 tx=0x7f6680005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a06a2f46b50a1b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.105+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6688001a70 con 0x7f6680004d40 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.105+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6688003b70 con 0x7f6680004d40 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.105+0000 7f669cd34640 1 -- 172.21.15.64:0/389235307 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66880050d0 con 0x7f6680004d40 2024-03-04T07:30:47.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.106+0000 7f669ff9a640 1 -- 172.21.15.64:0/389235307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6680004d40 msgr2=0x7f66800051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:47.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.106+0000 7f669ff9a640 1 --2- 172.21.15.64:0/389235307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6680004d40 0x7f66800051c0 secure :-1 s=READY pgs=973 cs=0 l=1 rev1=1 crypto rx=0x7f669815a980 tx=0x7f6680005830 comp rx=0 tx=0).stop 2024-03-04T07:30:47.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.106+0000 7f669e537640 1 -- 172.21.15.64:0/389235307 reap_dead start 2024-03-04T07:30:47.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.106+0000 7f669ff9a640 1 -- 172.21.15.64:0/389235307 shutdown_connections 2024-03-04T07:30:47.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.106+0000 7f669ff9a640 1 -- 172.21.15.64:0/389235307 >> 172.21.15.64:0/389235307 conn(0x7f6698066d00 msgr2=0x7f6698067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:47.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.106+0000 7f669ff9a640 1 -- 172.21.15.64:0/389235307 shutdown_connections 2024-03-04T07:30:47.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.106+0000 7f669ff9a640 1 -- 172.21.15.64:0/389235307 wait complete. 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.107+0000 7f669ff9a640 1 Processor -- start 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.107+0000 7f669ff9a640 1 -- start start 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669ff9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66981e6e60 0x7f6698142b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669ff9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6698145590 0x7f6698144240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669ff9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6698144780 0x7f66980cded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669ff9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66980b4150 con 0x7f66981e6e60 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669ff9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66980b3950 con 0x7f6698145590 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669ff9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66980cc3f0 con 0x7f6698144780 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66981e6e60 0x7f6698142b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66981e6e60 0x7f6698142b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42220/0 (socket says 172.21.15.64:42220) 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 -- 172.21.15.64:0/4245453121 learned_addr learned my addr 172.21.15.64:0/4245453121 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669d535640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6698145590 0x7f6698144240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669e537640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6698144780 0x7f66980cded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 -- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6698144780 msgr2=0x7f66980cded0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6698144780 0x7f66980cded0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 -- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6698145590 msgr2=0x7f6698144240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6698145590 0x7f6698144240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 -- 172.21.15.64:0/4245453121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66981431a0 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669dd36640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66981e6e60 0x7f6698142b50 secure :-1 s=READY pgs=974 cs=0 l=1 rev1=1 crypto rx=0x7f6690007290 tx=0x7f6690007750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f66967fc640 1 -- 172.21.15.64:0/4245453121 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f669000f940 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f66967fc640 1 -- 172.21.15.64:0/4245453121 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6690018070 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6698143480 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.108+0000 7f66967fc640 1 -- 172.21.15.64:0/4245453121 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f669000fd20 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.109+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6698143a40 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.109+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f66980733c0 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.110+0000 7f66967fc640 1 -- 172.21.15.64:0/4245453121 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f669001d460 con 0x7f66981e6e60 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.110+0000 7f66967fc640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f668406b4d0 0x7f668406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.111+0000 7f669d535640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f668406b4d0 0x7f668406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.111+0000 7f66967fc640 1 -- 172.21.15.64:0/4245453121 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(76..76 src has 1..76) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f669008a240 con 0x7f66981e6e60 2024-03-04T07:30:47.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.111+0000 7f669d535640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f668406b4d0 0x7f668406d990 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f66981e3e60 tx=0x7f668c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:47.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.115+0000 7f66967fc640 1 -- 172.21.15.64:0/4245453121 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f66980b2e30 con 0x7f66981e6e60 2024-03-04T07:30:47.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.298+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f66980ceff0 con 0x7f66981e6e60 2024-03-04T07:30:47.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:30:47.725 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:30:47.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.978+0000 7f66967fc640 1 -- 172.21.15.64:0/4245453121 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v77) v1 ==== 170+0+0 (secure 0 0 0) 0x7f6690059210 con 0x7f66981e6e60 2024-03-04T07:30:47.978 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f668406b4d0 msgr2=0x7f668406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f668406b4d0 0x7f668406d990 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f66981e3e60 tx=0x7f668c008040 comp rx=0 tx=0).stop 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66981e6e60 msgr2=0x7f6698142b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66981e6e60 0x7f6698142b50 secure :-1 s=READY pgs=974 cs=0 l=1 rev1=1 crypto rx=0x7f6690007290 tx=0x7f6690007750 comp rx=0 tx=0).stop 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 shutdown_connections 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f668406b4d0 0x7f668406d990 unknown :-1 s=CLOSED pgs=454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6698144780 0x7f66980cded0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6698145590 0x7f6698144240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 --2- 172.21.15.64:0/4245453121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66981e6e60 0x7f6698142b50 unknown :-1 s=CLOSED pgs=974 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:47.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 >> 172.21.15.64:0/4245453121 conn(0x7f6698066d00 msgr2=0x7f669815c480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:47.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 shutdown_connections 2024-03-04T07:30:47.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:47.981+0000 7f669ff9a640 1 -- 172.21.15.64:0/4245453121 wait complete. 2024-03-04T07:30:48.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:30:48.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.111+0000 7f99cd06b640 1 Processor -- start 2024-03-04T07:30:48.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.111+0000 7f99cd06b640 1 -- start start 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99cd06b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f99c8072d50 0x7f99c806dba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99cd06b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c80d00e0 0x7f99c80d0500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99cd06b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99c815d660 0x7f99c815daf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99cd06b640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f99c806d6f0 con 0x7f99c815d660 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99cd06b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f99c80cfdd0 con 0x7f99c8072d50 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99cd06b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99c80d0a40 con 0x7f99c80d00e0 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99c815d660 0x7f99c815daf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99c815d660 0x7f99c815daf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42226/0 (socket says 172.21.15.64:42226) 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 -- 172.21.15.64:0/686733980 learned_addr learned my addr 172.21.15.64:0/686733980 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99bffff640 1 --2- 172.21.15.64:0/686733980 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c80d00e0 0x7f99c80d0500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c6d9d640 1 --2- 172.21.15.64:0/686733980 >> v2:172.21.15.66:3300/0 conn(0x7f99c8072d50 0x7f99c806dba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 -- 172.21.15.64:0/686733980 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c80d00e0 msgr2=0x7f99c80d0500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 --2- 172.21.15.64:0/686733980 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c80d00e0 0x7f99c80d0500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 -- 172.21.15.64:0/686733980 >> v2:172.21.15.66:3300/0 conn(0x7f99c8072d50 msgr2=0x7f99c806dba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 --2- 172.21.15.64:0/686733980 >> v2:172.21.15.66:3300/0 conn(0x7f99c8072d50 0x7f99c806dba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c759e640 1 -- 172.21.15.64:0/686733980 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99c8071130 con 0x7f99c815d660 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.112+0000 7f99c6d9d640 1 --2- 172.21.15.64:0/686733980 >> v2:172.21.15.66:3300/0 conn(0x7f99c8072d50 0x7f99c806dba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.113+0000 7f99c759e640 1 --2- 172.21.15.64:0/686733980 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c815d660 0x7f99c815daf0 secure :-1 s=READY pgs=975 cs=0 l=1 rev1=1 crypto rx=0x7f99c8074b00 tx=0x7f99c002bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=49e9835cdbb9e24c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.113+0000 7f99c659c640 1 -- 172.21.15.64:0/686733980 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99c002cb00 con 0x7f99c815d660 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.113+0000 7f99c659c640 1 -- 172.21.15.64:0/686733980 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99c002cca0 con 0x7f99c815d660 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.113+0000 7f99c659c640 1 -- 172.21.15.64:0/686733980 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99c0034610 con 0x7f99c815d660 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 -- 172.21.15.64:0/686733980 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c815d660 msgr2=0x7f99c815daf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 --2- 172.21.15.64:0/686733980 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c815d660 0x7f99c815daf0 secure :-1 s=READY pgs=975 cs=0 l=1 rev1=1 crypto rx=0x7f99c8074b00 tx=0x7f99c002bf60 comp rx=0 tx=0).stop 2024-03-04T07:30:48.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 -- 172.21.15.64:0/686733980 shutdown_connections 2024-03-04T07:30:48.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 --2- 172.21.15.64:0/686733980 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c815d660 0x7f99c815daf0 unknown :-1 s=CLOSED pgs=975 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 --2- 172.21.15.64:0/686733980 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c80d00e0 0x7f99c80d0500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 --2- 172.21.15.64:0/686733980 >> v2:172.21.15.66:3300/0 conn(0x7f99c8072d50 0x7f99c806dba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 -- 172.21.15.64:0/686733980 >> 172.21.15.64:0/686733980 conn(0x7f99c806af20 msgr2=0x7f99c806b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:48.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 -- 172.21.15.64:0/686733980 shutdown_connections 2024-03-04T07:30:48.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.114+0000 7f99cd06b640 1 -- 172.21.15.64:0/686733980 wait complete. 2024-03-04T07:30:48.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.115+0000 7f99cd06b640 1 Processor -- start 2024-03-04T07:30:48.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.115+0000 7f99cd06b640 1 -- start start 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99c8072d50 0x7f99c814af20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c80d00e0 0x7f99c809b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c815d660 0x7f99c81e7b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99c80cf780 con 0x7f99c80d00e0 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99c80cee40 con 0x7f99c8072d50 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99c80cfa60 con 0x7f99c815d660 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c80d00e0 0x7f99c809b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c80d00e0 0x7f99c809b6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42240/0 (socket says 172.21.15.64:42240) 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 -- 172.21.15.64:0/4194811975 learned_addr learned my addr 172.21.15.64:0/4194811975 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 -- 172.21.15.64:0/4194811975 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c815d660 msgr2=0x7f99c81e7b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c815d660 0x7f99c81e7b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 -- 172.21.15.64:0/4194811975 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99c8072d50 msgr2=0x7f99c814af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99c8072d50 0x7f99c814af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 -- 172.21.15.64:0/4194811975 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99c81e85c0 con 0x7f99c80d00e0 2024-03-04T07:30:48.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bffff640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c80d00e0 0x7f99c809b6c0 secure :-1 s=READY pgs=976 cs=0 l=1 rev1=1 crypto rx=0x7f99b000d800 tx=0x7f99b000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bf7fe640 1 -- 172.21.15.64:0/4194811975 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99b000b6c0 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bf7fe640 1 -- 172.21.15.64:0/4194811975 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99b0019070 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99c809abe0 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99bf7fe640 1 -- 172.21.15.64:0/4194811975 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99b000ba80 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.116+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99c809b1a0 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.117+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f99c809e320 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.118+0000 7f99bf7fe640 1 -- 172.21.15.64:0/4194811975 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f99b000bc20 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.118+0000 7f99bf7fe640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99a806b500 0x7f99a806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.118+0000 7f99bf7fe640 1 -- 172.21.15.64:0/4194811975 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f99b008a600 con 0x7f99c80d00e0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.118+0000 7f99c6d9d640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99a806b500 0x7f99a806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.119+0000 7f99c6d9d640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99a806b500 0x7f99a806d9c0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f99b800a8b0 tx=0x7f99b8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:48.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.123+0000 7f99bf7fe640 1 -- 172.21.15.64:0/4194811975 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f99b001c360 con 0x7f99c80d00e0 2024-03-04T07:30:48.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.343+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f99c804f410 con 0x7f99c80d00e0 2024-03-04T07:30:48.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.980+0000 7f99bf7fe640 1 -- 172.21.15.64:0/4194811975 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 12 and data pool 13 v41) v1 ==== 174+0+0 (secure 0 0 0) 0x7f99b00594e0 con 0x7f99c80d00e0 2024-03-04T07:30:48.980 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 12 and data pool 13 2024-03-04T07:30:48.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.984+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99a806b500 msgr2=0x7f99a806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:48.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.984+0000 7f99cd06b640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99a806b500 0x7f99a806d9c0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f99b800a8b0 tx=0x7f99b8008040 comp rx=0 tx=0).stop 2024-03-04T07:30:48.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.984+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c80d00e0 msgr2=0x7f99c809b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:48.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.984+0000 7f99cd06b640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c80d00e0 0x7f99c809b6c0 secure :-1 s=READY pgs=976 cs=0 l=1 rev1=1 crypto rx=0x7f99b000d800 tx=0x7f99b000dcc0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.986+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 shutdown_connections 2024-03-04T07:30:48.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.986+0000 7f99cd06b640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99a806b500 0x7f99a806d9c0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.987+0000 7f99cd06b640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99c815d660 0x7f99c81e7b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.987+0000 7f99cd06b640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99c80d00e0 0x7f99c809b6c0 unknown :-1 s=CLOSED pgs=976 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.987+0000 7f99cd06b640 1 --2- 172.21.15.64:0/4194811975 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99c8072d50 0x7f99c814af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:48.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.987+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 >> 172.21.15.64:0/4194811975 conn(0x7f99c806af20 msgr2=0x7f99c806b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:48.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.990+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 shutdown_connections 2024-03-04T07:30:48.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:48.990+0000 7f99cd06b640 1 -- 172.21.15.64:0/4194811975 wait complete. 2024-03-04T07:30:49.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.126+0000 7fc07319a640 1 Processor -- start 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.126+0000 7fc07319a640 1 -- start start 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc07319a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 0x7fc06c0b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc07319a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc06c1456f0 0x7fc06c0b2290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc07319a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc06c0b3c70 con 0x7fc06c15bde0 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc07319a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc06c0b4840 con 0x7fc06c1456f0 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc07319a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc06c0b53e0 con 0x7fc06c1451a0 2024-03-04T07:30:49.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc070f36640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc06c15bde0 0x7fc06c0b4cd0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48060/0 (socket says 172.21.15.64:48060) 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc070f36640 1 -- 172.21.15.64:0/4017088038 learned_addr learned my addr 172.21.15.64:0/4017088038 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06bfff640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 0x7fc06c0b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc071737640 1 --2- 172.21.15.64:0/4017088038 >> v2:172.21.15.66:3300/0 conn(0x7fc06c1456f0 0x7fc06c0b2290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2126583376 0 0) 0x7fc06c0b3c70 con 0x7fc06c15bde0 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc05c003520 con 0x7fc06c15bde0 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1204011354 0 0) 0x7fc05c003520 con 0x7fc06c15bde0 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 msgr2=0x7fc06c0b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 0x7fc06c0b1d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 >> v2:172.21.15.66:3300/0 conn(0x7fc06c1456f0 msgr2=0x7fc06c0b2290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 --2- 172.21.15.64:0/4017088038 >> v2:172.21.15.66:3300/0 conn(0x7fc06c1456f0 0x7fc06c0b2290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc06c0afad0 con 0x7fc06c15bde0 2024-03-04T07:30:49.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fc060001670 con 0x7fc06c15bde0 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 >> v1:172.21.15.64:6789/0 conn(0x7fc06c15bde0 legacy=0x7fc06c0b4cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c0039b0 0x7fc05c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc05c004800 0x7fc05c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 0x7fc05c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc06c0b4840 con 0x7fc05c004800 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc06c0b53e0 con 0x7fc05c0039b0 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.127+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc05c00b9c0 con 0x7fc06c1451a0 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc06bfff640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc05c004800 0x7fc05c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc070f36640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c0039b0 0x7fc05c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc071737640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 0x7fc05c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc06bfff640 1 -- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 msgr2=0x7fc05c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc06bfff640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 0x7fc05c0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc06bfff640 1 -- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c0039b0 msgr2=0x7fc05c003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:49.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc06bfff640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c0039b0 0x7fc05c003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc06bfff640 1 -- 172.21.15.64:0/4017088038 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc05c003520 con 0x7fc05c004800 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.128+0000 7fc071737640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c1451a0 0x7fc05c0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc06bfff640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc05c004800 0x7fc05c004c80 secure :-1 s=READY pgs=978 cs=0 l=1 rev1=1 crypto rx=0x7fc058009ac0 tx=0x7fc05c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a41176b85cd7637 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc058039070 con 0x7fc05c004800 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc05802c6b0 con 0x7fc05c004800 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc06b7fe640 1 -- 172.21.15.64:0/4017088038 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc058009690 con 0x7fc05c004800 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc07319a640 1 -- 172.21.15.64:0/4017088038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc05c004800 msgr2=0x7fc05c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc07319a640 1 --2- 172.21.15.64:0/4017088038 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc05c004800 0x7fc05c004c80 secure :-1 s=READY pgs=978 cs=0 l=1 rev1=1 crypto rx=0x7fc058009ac0 tx=0x7fc05c005830 comp rx=0 tx=0).stop 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc071737640 1 -- 172.21.15.64:0/4017088038 reap_dead start 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc07319a640 1 -- 172.21.15.64:0/4017088038 shutdown_connections 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc07319a640 1 -- 172.21.15.64:0/4017088038 >> 172.21.15.64:0/4017088038 conn(0x7fc06c0b55e0 msgr2=0x7fc06c0b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc07319a640 1 -- 172.21.15.64:0/4017088038 shutdown_connections 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.129+0000 7fc07319a640 1 -- 172.21.15.64:0/4017088038 wait complete. 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.130+0000 7fc07319a640 1 Processor -- start 2024-03-04T07:30:49.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.131+0000 7fc07319a640 1 -- start start 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.131+0000 7fc07319a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c004800 0x7fc06c1f6be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.131+0000 7fc07319a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c158c60 0x7fc06c1f7120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.131+0000 7fc07319a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc06c06ea80 0x7fc06c1f7660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.131+0000 7fc07319a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc06c0b44d0 con 0x7fc06c06ea80 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.131+0000 7fc07319a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc06c0b41f0 con 0x7fc05c004800 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.131+0000 7fc07319a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc06c0b1d50 con 0x7fc06c158c60 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc06c06ea80 0x7fc06c1f7660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc06c06ea80 0x7fc06c1f7660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42266/0 (socket says 172.21.15.64:42266) 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 -- 172.21.15.64:0/1991029461 learned_addr learned my addr 172.21.15.64:0/1991029461 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc070f36640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c004800 0x7fc06c1f6be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc06bfff640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c158c60 0x7fc06c1f7120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 -- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c158c60 msgr2=0x7fc06c1f7120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c158c60 0x7fc06c1f7120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 -- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c004800 msgr2=0x7fc06c1f6be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c004800 0x7fc06c1f6be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 -- 172.21.15.64:0/1991029461 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc06c1f4550 con 0x7fc06c06ea80 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc071737640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc06c06ea80 0x7fc06c1f7660 secure :-1 s=READY pgs=979 cs=0 l=1 rev1=1 crypto rx=0x7fc06c0ae6b0 tx=0x7fc064010590 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc0697fa640 1 -- 172.21.15.64:0/1991029461 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc06401a030 con 0x7fc06c06ea80 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc0697fa640 1 -- 172.21.15.64:0/1991029461 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc064010e00 con 0x7fc06c06ea80 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc06c06dcc0 con 0x7fc06c06ea80 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc0697fa640 1 -- 172.21.15.64:0/1991029461 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc064003bd0 con 0x7fc06c06ea80 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.132+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc06c06e280 con 0x7fc06c06ea80 2024-03-04T07:30:49.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.133+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc06c067ff0 con 0x7fc06c06ea80 2024-03-04T07:30:49.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.134+0000 7fc0697fa640 1 -- 172.21.15.64:0/1991029461 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc06401d070 con 0x7fc06c06ea80 2024-03-04T07:30:49.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.134+0000 7fc0697fa640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc05006b4d0 0x7fc05006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.134+0000 7fc0697fa640 1 -- 172.21.15.64:0/1991029461 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(78..78 src has 1..78) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc06408a8c0 con 0x7fc06c06ea80 2024-03-04T07:30:49.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.134+0000 7fc070f36640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc05006b4d0 0x7fc05006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.135+0000 7fc070f36640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc05006b4d0 0x7fc05006d990 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fc060001eb0 tx=0x7fc060009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.139+0000 7fc0697fa640 1 -- 172.21.15.64:0/1991029461 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc0640597d0 con 0x7fc06c06ea80 2024-03-04T07:30:49.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.320+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc06c06e5f0 con 0x7fc06c06ea80 2024-03-04T07:30:49.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.321+0000 7fc0697fa640 1 -- 172.21.15.64:0/1991029461 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v78) v1 ==== 74+0+16625 (secure 0 0 0) 0x7fc0640590d0 con 0x7fc06c06ea80 2024-03-04T07:30:49.321 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:49.321 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":78,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:48.344532+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:46.621410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:47.317858+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:49.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc05006b4d0 msgr2=0x7fc05006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc05006b4d0 0x7fc05006d990 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fc060001eb0 tx=0x7fc060009040 comp rx=0 tx=0).stop 2024-03-04T07:30:49.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc06c06ea80 msgr2=0x7fc06c1f7660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc06c06ea80 0x7fc06c1f7660 secure :-1 s=READY pgs=979 cs=0 l=1 rev1=1 crypto rx=0x7fc06c0ae6b0 tx=0x7fc064010590 comp rx=0 tx=0).stop 2024-03-04T07:30:49.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 shutdown_connections 2024-03-04T07:30:49.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc05006b4d0 0x7fc05006d990 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc06c06ea80 0x7fc06c1f7660 unknown :-1 s=CLOSED pgs=979 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc06c158c60 0x7fc06c1f7120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 --2- 172.21.15.64:0/1991029461 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc05c004800 0x7fc06c1f6be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 >> 172.21.15.64:0/1991029461 conn(0x7fc06c0b55e0 msgr2=0x7fc06c1595b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:49.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 shutdown_connections 2024-03-04T07:30:49.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.323+0000 7fc07319a640 1 -- 172.21.15.64:0/1991029461 wait complete. 2024-03-04T07:30:49.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:49.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.436+0000 7fb38db56640 1 Processor -- start 2024-03-04T07:30:49.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.436+0000 7fb38db56640 1 -- start start 2024-03-04T07:30:49.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb38db56640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb38815e060 0x7fb3880ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb38db56640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb3880d0880 0x7fb3880ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb38db56640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb3880d0dd0 con 0x7fb38806fe20 2024-03-04T07:30:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb38db56640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb3880d10b0 con 0x7fb3880d0880 2024-03-04T07:30:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb38db56640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb3880cc0e0 con 0x7fb38815e060 2024-03-04T07:30:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb387fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb3880d0880 0x7fb3880ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb387fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb3880d0880 0x7fb3880ccbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42274/0 (socket says 172.21.15.64:42274) 2024-03-04T07:30:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb387fff640 1 -- 172.21.15.64:0/3134788581 learned_addr learned my addr 172.21.15.64:0/3134788581 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb3877fe640 1 --2- 172.21.15.64:0/3134788581 >> v2:172.21.15.66:3300/0 conn(0x7fb38815e060 0x7fb3880ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb387fff640 1 -- 172.21.15.64:0/3134788581 >> v2:172.21.15.66:3300/0 conn(0x7fb38815e060 msgr2=0x7fb3880ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb387fff640 1 --2- 172.21.15.64:0/3134788581 >> v2:172.21.15.66:3300/0 conn(0x7fb38815e060 0x7fb3880ce310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb387fff640 1 -- 172.21.15.64:0/3134788581 >> v1:172.21.15.66:6789/0 conn(0x7fb38806fe20 legacy=0x7fb3880ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:49.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.437+0000 7fb387fff640 1 -- 172.21.15.64:0/3134788581 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3880cc3c0 con 0x7fb3880d0880 2024-03-04T07:30:49.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.438+0000 7fb387fff640 1 --2- 172.21.15.64:0/3134788581 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3880d0880 0x7fb3880ccbc0 secure :-1 s=READY pgs=980 cs=0 l=1 rev1=1 crypto rx=0x7fb3880722f0 tx=0x7fb38002bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6381587873aaeaa5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.438+0000 7fb3867fc640 1 -- 172.21.15.64:0/3134788581 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb38002ca80 con 0x7fb3880d0880 2024-03-04T07:30:49.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.438+0000 7fb3867fc640 1 -- 172.21.15.64:0/3134788581 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb38002cc20 con 0x7fb3880d0880 2024-03-04T07:30:49.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.438+0000 7fb3867fc640 1 -- 172.21.15.64:0/3134788581 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3800343f0 con 0x7fb3880d0880 2024-03-04T07:30:49.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 -- 172.21.15.64:0/3134788581 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3880d0880 msgr2=0x7fb3880ccbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 --2- 172.21.15.64:0/3134788581 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3880d0880 0x7fb3880ccbc0 secure :-1 s=READY pgs=980 cs=0 l=1 rev1=1 crypto rx=0x7fb3880722f0 tx=0x7fb38002bee0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 -- 172.21.15.64:0/3134788581 shutdown_connections 2024-03-04T07:30:49.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 --2- 172.21.15.64:0/3134788581 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3880d0880 0x7fb3880ccbc0 unknown :-1 s=CLOSED pgs=980 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 --2- 172.21.15.64:0/3134788581 >> v2:172.21.15.66:3300/0 conn(0x7fb38815e060 0x7fb3880ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 -- 172.21.15.64:0/3134788581 >> 172.21.15.64:0/3134788581 conn(0x7fb3880d1a90 msgr2=0x7fb3880d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:49.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 -- 172.21.15.64:0/3134788581 shutdown_connections 2024-03-04T07:30:49.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.439+0000 7fb38db56640 1 -- 172.21.15.64:0/3134788581 wait complete. 2024-03-04T07:30:49.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.440+0000 7fb38db56640 1 Processor -- start 2024-03-04T07:30:49.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.440+0000 7fb38db56640 1 -- start start 2024-03-04T07:30:49.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb38db56640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb38806fe20 0x7fb3880b7430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb38db56640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3880d0880 0x7fb38815e5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb38db56640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb38815e060 0x7fb388156f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb38db56640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3880cef60 con 0x7fb38806fe20 2024-03-04T07:30:49.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb38db56640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb38815e440 con 0x7fb3880d0880 2024-03-04T07:30:49.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb38db56640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb3880cf0e0 con 0x7fb38815e060 2024-03-04T07:30:49.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb38806fe20 0x7fb3880b7430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb38806fe20 0x7fb3880b7430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42286/0 (socket says 172.21.15.64:42286) 2024-03-04T07:30:49.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 -- 172.21.15.64:0/1439095071 learned_addr learned my addr 172.21.15.64:0/1439095071 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:49.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb387fff640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb38815e060 0x7fb388156f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb386ffd640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3880d0880 0x7fb38815e5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 -- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb38815e060 msgr2=0x7fb388156f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb38815e060 0x7fb388156f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 -- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3880d0880 msgr2=0x7fb38815e5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3880d0880 0x7fb38815e5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 -- 172.21.15.64:0/1439095071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb38815ed10 con 0x7fb38806fe20 2024-03-04T07:30:49.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb3877fe640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb38806fe20 0x7fb3880b7430 secure :-1 s=READY pgs=981 cs=0 l=1 rev1=1 crypto rx=0x7fb37800c820 tx=0x7fb37800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb36ffff640 1 -- 172.21.15.64:0/1439095071 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb378015070 con 0x7fb38806fe20 2024-03-04T07:30:49.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb36ffff640 1 -- 172.21.15.64:0/1439095071 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb378004020 con 0x7fb38806fe20 2024-03-04T07:30:49.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb38815eff0 con 0x7fb38806fe20 2024-03-04T07:30:49.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.441+0000 7fb36ffff640 1 -- 172.21.15.64:0/1439095071 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3780043f0 con 0x7fb38806fe20 2024-03-04T07:30:49.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.442+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb38815f5b0 con 0x7fb38806fe20 2024-03-04T07:30:49.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.443+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb388067ff0 con 0x7fb38806fe20 2024-03-04T07:30:49.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.443+0000 7fb36ffff640 1 -- 172.21.15.64:0/1439095071 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb378007830 con 0x7fb38806fe20 2024-03-04T07:30:49.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.443+0000 7fb36ffff640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb37406b4d0 0x7fb37406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.444+0000 7fb36ffff640 1 -- 172.21.15.64:0/1439095071 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(78..78 src has 1..78) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb378089f90 con 0x7fb38806fe20 2024-03-04T07:30:49.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.444+0000 7fb386ffd640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb37406b4d0 0x7fb37406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.444+0000 7fb386ffd640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb37406b4d0 0x7fb37406d990 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fb388161750 tx=0x7fb37c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.448+0000 7fb36ffff640 1 -- 172.21.15.64:0/1439095071 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb378058ea0 con 0x7fb38806fe20 2024-03-04T07:30:49.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.629+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fb3880d2160 con 0x7fb38806fe20 2024-03-04T07:30:49.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.630+0000 7fb36ffff640 1 -- 172.21.15.64:0/1439095071 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v78) v1 ==== 74+0+16625 (secure 0 0 0) 0x7fb3780587a0 con 0x7fb38806fe20 2024-03-04T07:30:49.630 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:49.630 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":78,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:48.344532+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:46.621410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:47.317858+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb37406b4d0 msgr2=0x7fb37406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb37406b4d0 0x7fb37406d990 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fb388161750 tx=0x7fb37c009040 comp rx=0 tx=0).stop 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb38806fe20 msgr2=0x7fb3880b7430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb38806fe20 0x7fb3880b7430 secure :-1 s=READY pgs=981 cs=0 l=1 rev1=1 crypto rx=0x7fb37800c820 tx=0x7fb37800cce0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 shutdown_connections 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb37406b4d0 0x7fb37406d990 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb38815e060 0x7fb388156f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3880d0880 0x7fb38815e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 --2- 172.21.15.64:0/1439095071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb38806fe20 0x7fb3880b7430 unknown :-1 s=CLOSED pgs=981 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 >> 172.21.15.64:0/1439095071 conn(0x7fb3880d1a90 msgr2=0x7fb38806af20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:49.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 shutdown_connections 2024-03-04T07:30:49.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.632+0000 7fb38db56640 1 -- 172.21.15.64:0/1439095071 wait complete. 2024-03-04T07:30:49.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:30:49.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.747+0000 7fe6e4e9d640 1 Processor -- start 2024-03-04T07:30:49.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.747+0000 7fe6e4e9d640 1 -- start start 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6e4e9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe6e015dbd0 0x7fe6e015ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6e4e9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e00b39b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6e4e9d640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe6e015f3b0 con 0x7fe6e015dfb0 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6e4e9d640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe6e006f070 con 0x7fe6e015dbd0 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6e4e9d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe6e0071940 con 0x7fe6e00b5840 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6ddd9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e00b39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6de59c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe6e015dbd0 0x7fe6e015ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6ddd9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e00b39b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:42254/0 (socket says 172.21.15.64:42254) 2024-03-04T07:30:49.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.748+0000 7fe6ddd9b640 1 -- 172.21.15.64:0/2116572512 learned_addr learned my addr 172.21.15.64:0/2116572512 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:49.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.749+0000 7fe6de59c640 1 -- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 msgr2=0x7fe6e00b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:49.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.749+0000 7fe6de59c640 1 --2- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e00b39b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.749+0000 7fe6de59c640 1 -- 172.21.15.64:0/2116572512 >> v1:172.21.15.66:6789/0 conn(0x7fe6e015dfb0 legacy=0x7fe6e0161a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.749+0000 7fe6de59c640 1 -- 172.21.15.64:0/2116572512 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6e00b5db0 con 0x7fe6e015dbd0 2024-03-04T07:30:49.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.749+0000 7fe6ddd9b640 1 --2- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e00b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:49.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6de59c640 1 --2- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dbd0 0x7fe6e015ee70 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fe6d80097a0 tx=0x7fe6d802b8b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7da6430ae65fcb4f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6dd59a640 1 -- 172.21.15.64:0/2116572512 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6d8035020 con 0x7fe6e015dbd0 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6dd59a640 1 -- 172.21.15.64:0/2116572512 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6d8031810 con 0x7fe6e015dbd0 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6dd59a640 1 -- 172.21.15.64:0/2116572512 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6d8031be0 con 0x7fe6e015dbd0 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dbd0 msgr2=0x7fe6e015ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dbd0 0x7fe6e015ee70 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fe6d80097a0 tx=0x7fe6d802b8b0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2116572512 shutdown_connections 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e00b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2116572512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dbd0 0x7fe6e015ee70 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2116572512 >> 172.21.15.64:0/2116572512 conn(0x7fe6e00b26b0 msgr2=0x7fe6e00b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2116572512 shutdown_connections 2024-03-04T07:30:49.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.750+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2116572512 wait complete. 2024-03-04T07:30:49.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 Processor -- start 2024-03-04T07:30:49.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 -- start start 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e0148370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6e015dbd0 0x7fe6e00b19b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dfb0 0x7fe6e015c540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6e00721c0 con 0x7fe6e015dbd0 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6e0071ee0 con 0x7fe6e015dfb0 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6e4e9d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe6e00726e0 con 0x7fe6e00b5840 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6ddd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6e015dbd0 0x7fe6e00b19b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6ddd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6e015dbd0 0x7fe6e00b19b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42288/0 (socket says 172.21.15.64:42288) 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.752+0000 7fe6ddd9b640 1 -- 172.21.15.64:0/2840815087 learned_addr learned my addr 172.21.15.64:0/2840815087 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6de59c640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e0148370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6ded9d640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dfb0 0x7fe6e015c540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6ddd9b640 1 -- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 msgr2=0x7fe6e0148370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6ddd9b640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e0148370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6ddd9b640 1 -- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dfb0 msgr2=0x7fe6e015c540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6ddd9b640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dfb0 0x7fe6e015c540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6ddd9b640 1 -- 172.21.15.64:0/2840815087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6e01f46a0 con 0x7fe6e015dbd0 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6ddd9b640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6e015dbd0 0x7fe6e00b19b0 secure :-1 s=READY pgs=982 cs=0 l=1 rev1=1 crypto rx=0x7fe6c800e9d0 tx=0x7fe6c800ee90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6d6ffd640 1 -- 172.21.15.64:0/2840815087 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6c8017070 con 0x7fe6e015dbd0 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6d6ffd640 1 -- 172.21.15.64:0/2840815087 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe6c800ca60 con 0x7fe6e015dbd0 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe6e01f4c30 con 0x7fe6e015dbd0 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6d6ffd640 1 -- 172.21.15.64:0/2840815087 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6c800ce10 con 0x7fe6e015dbd0 2024-03-04T07:30:49.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.753+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe6e01488b0 con 0x7fe6e015dbd0 2024-03-04T07:30:49.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.754+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe6e0073380 con 0x7fe6e015dbd0 2024-03-04T07:30:49.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.755+0000 7fe6d6ffd640 1 -- 172.21.15.64:0/2840815087 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe6c801a070 con 0x7fe6e015dbd0 2024-03-04T07:30:49.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.755+0000 7fe6d6ffd640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6c006b4d0 0x7fe6c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:49.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.755+0000 7fe6d6ffd640 1 -- 172.21.15.64:0/2840815087 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(78..78 src has 1..78) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe6c808a190 con 0x7fe6e015dbd0 2024-03-04T07:30:49.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.755+0000 7fe6de59c640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6c006b4d0 0x7fe6c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:49.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.756+0000 7fe6de59c640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6c006b4d0 0x7fe6c006d990 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fe6d8036000 tx=0x7fe6d8008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:49.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.759+0000 7fe6d6ffd640 1 -- 172.21.15.64:0/2840815087 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe6c8058930 con 0x7fe6e015dbd0 2024-03-04T07:30:49.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:49.977+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7fe6e004f410 con 0x7fe6e015dbd0 2024-03-04T07:30:50.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.001+0000 7fe6d6ffd640 1 -- 172.21.15.64:0/2840815087 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v42) v1 ==== 118+0+0 (secure 0 0 0) 0x7fe6c805c9d0 con 0x7fe6e015dbd0 2024-03-04T07:30:50.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.003+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6c006b4d0 msgr2=0x7fe6c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:50.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.003+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6c006b4d0 0x7fe6c006d990 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fe6d8036000 tx=0x7fe6d8008030 comp rx=0 tx=0).stop 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.003+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6e015dbd0 msgr2=0x7fe6e00b19b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.003+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6e015dbd0 0x7fe6e00b19b0 secure :-1 s=READY pgs=982 cs=0 l=1 rev1=1 crypto rx=0x7fe6c800e9d0 tx=0x7fe6c800ee90 comp rx=0 tx=0).stop 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 shutdown_connections 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe6c006b4d0 0x7fe6c006d990 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe6e015dfb0 0x7fe6e015c540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe6e015dbd0 0x7fe6e00b19b0 unknown :-1 s=CLOSED pgs=982 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 --2- 172.21.15.64:0/2840815087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe6e00b5840 0x7fe6e0148370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 >> 172.21.15.64:0/2840815087 conn(0x7fe6e00b26b0 msgr2=0x7fe6e015e3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 shutdown_connections 2024-03-04T07:30:50.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.004+0000 7fe6e4e9d640 1 -- 172.21.15.64:0/2840815087 wait complete. 2024-03-04T07:30:50.017 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:30:50.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:30:50.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.125+0000 7f183c3e9640 1 Processor -- start 2024-03-04T07:30:50.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.125+0000 7f183c3e9640 1 -- start start 2024-03-04T07:30:50.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183c3e9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f18340cdcd0 0x7f18340b1550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:50.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183c3e9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f183406b2c0 0x7f183406b710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:50.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183c3e9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1834161a40 con 0x7f18340b2cd0 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183c3e9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f18340d11e0 con 0x7f183406b2c0 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183c3e9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f18340b2a20 con 0x7f18340cdcd0 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183a986640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f183406b2c0 0x7f183406b710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f1839984640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f18340b2cd0 0x7f18340b1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48066/0 (socket says 172.21.15.64:48066) 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f1839984640 1 -- 172.21.15.64:0/2915195351 learned_addr learned my addr 172.21.15.64:0/2915195351 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183a185640 1 --2- 172.21.15.64:0/2915195351 >> v2:172.21.15.66:3300/0 conn(0x7f18340cdcd0 0x7f18340b1550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f1839183640 1 -- 172.21.15.64:0/2915195351 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3395675238 0 0) 0x7f1834161a40 con 0x7f18340b2cd0 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f1839183640 1 -- 172.21.15.64:0/2915195351 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1820003520 con 0x7f18340b2cd0 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183a986640 1 -- 172.21.15.64:0/2915195351 >> v2:172.21.15.66:3300/0 conn(0x7f18340cdcd0 msgr2=0x7f18340b1550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183a986640 1 --2- 172.21.15.64:0/2915195351 >> v2:172.21.15.66:3300/0 conn(0x7f18340cdcd0 0x7f18340b1550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183a986640 1 -- 172.21.15.64:0/2915195351 >> v1:172.21.15.64:6789/0 conn(0x7f18340b2cd0 legacy=0x7f18340b1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:50.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.126+0000 7f183a986640 1 -- 172.21.15.64:0/2915195351 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18340d0940 con 0x7f183406b2c0 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.127+0000 7f183a986640 1 --2- 172.21.15.64:0/2915195351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f183406b2c0 0x7f183406b710 secure :-1 s=READY pgs=984 cs=0 l=1 rev1=1 crypto rx=0x7f1834071ca0 tx=0x7f182c02be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1764b384caf94d18 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.127+0000 7f1839183640 1 -- 172.21.15.64:0/2915195351 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f182c039070 con 0x7f183406b2c0 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.127+0000 7f1839183640 1 -- 172.21.15.64:0/2915195351 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f182c02cbf0 con 0x7f183406b2c0 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.127+0000 7f1839183640 1 -- 172.21.15.64:0/2915195351 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f182c034440 con 0x7f183406b2c0 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 -- 172.21.15.64:0/2915195351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f183406b2c0 msgr2=0x7f183406b710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 --2- 172.21.15.64:0/2915195351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f183406b2c0 0x7f183406b710 secure :-1 s=READY pgs=984 cs=0 l=1 rev1=1 crypto rx=0x7f1834071ca0 tx=0x7f182c02be40 comp rx=0 tx=0).stop 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 -- 172.21.15.64:0/2915195351 shutdown_connections 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 --2- 172.21.15.64:0/2915195351 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f183406b2c0 0x7f183406b710 unknown :-1 s=CLOSED pgs=984 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 --2- 172.21.15.64:0/2915195351 >> v2:172.21.15.66:3300/0 conn(0x7f18340cdcd0 0x7f18340b1550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 -- 172.21.15.64:0/2915195351 >> 172.21.15.64:0/2915195351 conn(0x7f18340b3700 msgr2=0x7f18340b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:50.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 -- 172.21.15.64:0/2915195351 shutdown_connections 2024-03-04T07:30:50.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.128+0000 7f183c3e9640 1 -- 172.21.15.64:0/2915195351 wait complete. 2024-03-04T07:30:50.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.130+0000 7f183c3e9640 1 Processor -- start 2024-03-04T07:30:50.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.130+0000 7f183c3e9640 1 -- start start 2024-03-04T07:30:50.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183c3e9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f183406b2c0 0x7f183414af30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183c3e9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18340b2cd0 0x7f18340cb380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183c3e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18340cdcd0 0x7f18340b5d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183c3e9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18340cc1c0 con 0x7f18340b2cd0 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183c3e9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18340d0f00 con 0x7f183406b2c0 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183c3e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18340cc340 con 0x7f18340cdcd0 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18340b2cd0 0x7f18340cb380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18340b2cd0 0x7f18340cb380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42304/0 (socket says 172.21.15.64:42304) 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 -- 172.21.15.64:0/4123568140 learned_addr learned my addr 172.21.15.64:0/4123568140 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 -- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18340cdcd0 msgr2=0x7f18340b5d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183a986640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18340cdcd0 0x7f18340b5d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18340cdcd0 0x7f18340b5d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 -- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f183406b2c0 msgr2=0x7f183414af30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f183a185640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f183406b2c0 0x7f183414af30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:50.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f183406b2c0 0x7f183414af30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.131+0000 7f1839984640 1 -- 172.21.15.64:0/4123568140 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18340b4c80 con 0x7f18340b2cd0 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.132+0000 7f1839984640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18340b2cd0 0x7f18340cb380 secure :-1 s=READY pgs=985 cs=0 l=1 rev1=1 crypto rx=0x7f182400c430 tx=0x7f182400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.132+0000 7f182affd640 1 -- 172.21.15.64:0/4123568140 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1824017070 con 0x7f18340b2cd0 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.132+0000 7f182affd640 1 -- 172.21.15.64:0/4123568140 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1824005d00 con 0x7f18340b2cd0 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.132+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18340b4f60 con 0x7f18340b2cd0 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.132+0000 7f182affd640 1 -- 172.21.15.64:0/4123568140 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1824007310 con 0x7f18340b2cd0 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.132+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18340b5800 con 0x7f18340b2cd0 2024-03-04T07:30:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.133+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f183409fcf0 con 0x7f18340b2cd0 2024-03-04T07:30:50.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.133+0000 7f182affd640 1 -- 172.21.15.64:0/4123568140 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f182401a070 con 0x7f18340b2cd0 2024-03-04T07:30:50.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.134+0000 7f182affd640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f181806b4d0 0x7f181806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:50.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.134+0000 7f183a185640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f181806b4d0 0x7f181806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:50.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.134+0000 7f182affd640 1 -- 172.21.15.64:0/4123568140 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1824089710 con 0x7f18340b2cd0 2024-03-04T07:30:50.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.134+0000 7f183a185640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f181806b4d0 0x7f181806d990 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f1830009970 tx=0x7f1830008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:50.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.138+0000 7f182affd640 1 -- 172.21.15.64:0/4123568140 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1824058500 con 0x7f18340b2cd0 2024-03-04T07:30:50.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:50.409+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f183404f410 con 0x7f18340b2cd0 2024-03-04T07:30:51.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.006+0000 7f182affd640 1 -- 172.21.15.64:0/4123568140 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v43) v1 ==== 122+0+0 (secure 0 0 0) 0x7f1824057e00 con 0x7f18340b2cd0 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f181806b4d0 msgr2=0x7f181806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f181806b4d0 0x7f181806d990 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f1830009970 tx=0x7f1830008040 comp rx=0 tx=0).stop 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18340b2cd0 msgr2=0x7f18340cb380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18340b2cd0 0x7f18340cb380 secure :-1 s=READY pgs=985 cs=0 l=1 rev1=1 crypto rx=0x7f182400c430 tx=0x7f182400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 shutdown_connections 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f181806b4d0 0x7f181806d990 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18340cdcd0 0x7f18340b5d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18340b2cd0 0x7f18340cb380 unknown :-1 s=CLOSED pgs=985 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 --2- 172.21.15.64:0/4123568140 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f183406b2c0 0x7f183414af30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 >> 172.21.15.64:0/4123568140 conn(0x7f18340b3700 msgr2=0x7f183406ebd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:51.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 shutdown_connections 2024-03-04T07:30:51.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.009+0000 7f183c3e9640 1 -- 172.21.15.64:0/4123568140 wait complete. 2024-03-04T07:30:51.023 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:30:51.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:51.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.126+0000 7f300460e640 1 Processor -- start 2024-03-04T07:30:51.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.126+0000 7f300460e640 1 -- start start 2024-03-04T07:30:51.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f300460e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ffc15df00 0x7f2ffc0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f300460e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b9a30 0x7f2ffc0d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f300460e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2ffc0cea30 con 0x7f2ffc0b95a0 2024-03-04T07:30:51.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f300460e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2ffc0d2af0 con 0x7f2ffc15df00 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f300460e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ffc0cf3c0 con 0x7f2ffc0b9a30 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f30023aa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ffc15df00 0x7f2ffc0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f3001ba9640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2ffc0b95a0 0x7f2ffc0b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48080/0 (socket says 172.21.15.64:48080) 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f30023aa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ffc15df00 0x7f2ffc0d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42318/0 (socket says 172.21.15.64:42318) 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f3001ba9640 1 -- 172.21.15.64:0/1784605557 learned_addr learned my addr 172.21.15.64:0/1784605557 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f30023aa640 1 -- 172.21.15.64:0/1784605557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b9a30 msgr2=0x7f2ffc0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f3002bab640 1 --2- 172.21.15.64:0/1784605557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b9a30 0x7f2ffc0d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f30023aa640 1 --2- 172.21.15.64:0/1784605557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b9a30 0x7f2ffc0d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f30013a8640 1 -- 172.21.15.64:0/1784605557 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2056627826 0 0) 0x7f2ffc0cea30 con 0x7f2ffc0b95a0 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f30023aa640 1 -- 172.21.15.64:0/1784605557 >> v1:172.21.15.64:6789/0 conn(0x7f2ffc0b95a0 legacy=0x7f2ffc0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.127+0000 7f30023aa640 1 -- 172.21.15.64:0/1784605557 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ffc0cf0e0 con 0x7f2ffc15df00 2024-03-04T07:30:51.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.128+0000 7f30023aa640 1 --2- 172.21.15.64:0/1784605557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc0d25b0 secure :-1 s=READY pgs=987 cs=0 l=1 rev1=1 crypto rx=0x7f2ff80097a0 tx=0x7f2ff802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6cde3bb1b0cacdb9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.128+0000 7f30013a8640 1 -- 172.21.15.64:0/1784605557 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ff8035020 con 0x7f2ffc15df00 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.128+0000 7f30013a8640 1 -- 172.21.15.64:0/1784605557 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2ff8031830 con 0x7f2ffc15df00 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.128+0000 7f30013a8640 1 -- 172.21.15.64:0/1784605557 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ff8031c00 con 0x7f2ffc15df00 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 -- 172.21.15.64:0/1784605557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 msgr2=0x7f2ffc0d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 --2- 172.21.15.64:0/1784605557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc0d25b0 secure :-1 s=READY pgs=987 cs=0 l=1 rev1=1 crypto rx=0x7f2ff80097a0 tx=0x7f2ff802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 -- 172.21.15.64:0/1784605557 shutdown_connections 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 --2- 172.21.15.64:0/1784605557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b9a30 0x7f2ffc0d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 --2- 172.21.15.64:0/1784605557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc0d25b0 unknown :-1 s=CLOSED pgs=987 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 -- 172.21.15.64:0/1784605557 >> 172.21.15.64:0/1784605557 conn(0x7f2ffc071960 msgr2=0x7f2ffc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 -- 172.21.15.64:0/1784605557 shutdown_connections 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.129+0000 7f300460e640 1 -- 172.21.15.64:0/1784605557 wait complete. 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.130+0000 7f300460e640 1 Processor -- start 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.130+0000 7f300460e640 1 -- start start 2024-03-04T07:30:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f300460e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b95a0 0x7f2ffc155480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f300460e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc14e1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f300460e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ffc0c77b0 0x7f2ffc160cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f300460e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ffc0cec70 con 0x7f2ffc15df00 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f300460e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ffc0ce750 con 0x7f2ffc0c77b0 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f300460e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ffc0cedf0 con 0x7f2ffc0b95a0 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc14e1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc14e1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42320/0 (socket says 172.21.15.64:42320) 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 -- 172.21.15.64:0/761067785 learned_addr learned my addr 172.21.15.64:0/761067785 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f30023aa640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b95a0 0x7f2ffc155480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3002bab640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ffc0c77b0 0x7f2ffc160cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 -- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b95a0 msgr2=0x7f2ffc155480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b95a0 0x7f2ffc155480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 -- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ffc0c77b0 msgr2=0x7f2ffc160cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ffc0c77b0 0x7f2ffc160cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 -- 172.21.15.64:0/761067785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ffc0c7c40 con 0x7f2ffc15df00 2024-03-04T07:30:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f3001ba9640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc14e1c0 secure :-1 s=READY pgs=988 cs=0 l=1 rev1=1 crypto rx=0x7f2fec00c430 tx=0x7f2fec00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f2ff2ffd640 1 -- 172.21.15.64:0/761067785 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2fec0074e0 con 0x7f2ffc15df00 2024-03-04T07:30:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.131+0000 7f2ff2ffd640 1 -- 172.21.15.64:0/761067785 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2fec0047e0 con 0x7f2ffc15df00 2024-03-04T07:30:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.132+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ffc155af0 con 0x7f2ffc15df00 2024-03-04T07:30:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.132+0000 7f2ff2ffd640 1 -- 172.21.15.64:0/761067785 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2fec004bd0 con 0x7f2ffc15df00 2024-03-04T07:30:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.132+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ffc1560b0 con 0x7f2ffc15df00 2024-03-04T07:30:51.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.133+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ffc067ff0 con 0x7f2ffc15df00 2024-03-04T07:30:51.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.133+0000 7f2ff2ffd640 1 -- 172.21.15.64:0/761067785 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2fec011070 con 0x7f2ffc15df00 2024-03-04T07:30:51.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.133+0000 7f2ff2ffd640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2fe006b500 0x7f2fe006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.134+0000 7f30023aa640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2fe006b500 0x7f2fe006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.134+0000 7f2ff2ffd640 1 -- 172.21.15.64:0/761067785 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2fec089d50 con 0x7f2ffc15df00 2024-03-04T07:30:51.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.134+0000 7f30023aa640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2fe006b500 0x7f2fe006d9c0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f2ff8036000 tx=0x7f2ff8008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.138+0000 7f2ff2ffd640 1 -- 172.21.15.64:0/761067785 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2fec08d030 con 0x7f2ffc15df00 2024-03-04T07:30:51.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.358+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2ffc1579f0 con 0x7f2ffc15df00 2024-03-04T07:30:51.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.359+0000 7f2ff2ffd640 1 -- 172.21.15.64:0/761067785 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 43 v43) v1 ==== 94+0+4973 (secure 0 0 0) 0x7f2fec00e280 con 0x7f2ffc15df00 2024-03-04T07:30:51.359 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:51.359 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[{"mdsmap":{"epoch":43,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:48.344908+0000","modified":"2024-03-04T07:30:50.410439+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6966},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6966":{"gid":6966,"name":"c","rank":0,"incarnation":41,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/1889124368","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1889124368},{"type":"v1","addr":"172.21.15.64:6836","nonce":1889124368}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6966,"qdb_cluster":[6966]},"id":6}]} 2024-03-04T07:30:51.359 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 43 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2fe006b500 msgr2=0x7f2fe006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2fe006b500 0x7f2fe006d9c0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f2ff8036000 tx=0x7f2ff8008030 comp rx=0 tx=0).stop 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 msgr2=0x7f2ffc14e1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc14e1c0 secure :-1 s=READY pgs=988 cs=0 l=1 rev1=1 crypto rx=0x7f2fec00c430 tx=0x7f2fec00c8f0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 shutdown_connections 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2fe006b500 0x7f2fe006d9c0 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ffc0c77b0 0x7f2ffc160cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ffc15df00 0x7f2ffc14e1c0 unknown :-1 s=CLOSED pgs=988 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 --2- 172.21.15.64:0/761067785 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ffc0b95a0 0x7f2ffc155480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 >> 172.21.15.64:0/761067785 conn(0x7f2ffc071960 msgr2=0x7f2ffc0b5800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 shutdown_connections 2024-03-04T07:30:51.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.361+0000 7f300460e640 1 -- 172.21.15.64:0/761067785 wait complete. 2024-03-04T07:30:51.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:51.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.475+0000 7f3ee59d7640 1 Processor -- start 2024-03-04T07:30:51.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.475+0000 7f3ee59d7640 1 -- start start 2024-03-04T07:30:51.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.475+0000 7f3ee59d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00b1020 0x7f3ee006af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.475+0000 7f3ee59d7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ee00cd890 0x7f3ee015d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.475+0000 7f3ee59d7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3ee00cc0e0 con 0x7f3ee006ed10 2024-03-04T07:30:51.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.475+0000 7f3ee59d7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3ee00d28b0 con 0x7f3ee00cd890 2024-03-04T07:30:51.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.475+0000 7f3ee59d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ee00cf320 con 0x7f3ee00b1020 2024-03-04T07:30:51.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3edf7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ee00cd890 0x7f3ee015d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3ed7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3ee006ed10 0x7f3ee00cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48096/0 (socket says 172.21.15.64:48096) 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3edf7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ee00cd890 0x7f3ee015d700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42326/0 (socket says 172.21.15.64:42326) 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3ed7fff640 1 -- 172.21.15.64:0/72654842 learned_addr learned my addr 172.21.15.64:0/72654842 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3edeffd640 1 --2- 172.21.15.64:0/72654842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00b1020 0x7f3ee006af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3edf7fe640 1 -- 172.21.15.64:0/72654842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00b1020 msgr2=0x7f3ee006af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3edf7fe640 1 --2- 172.21.15.64:0/72654842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00b1020 0x7f3ee006af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3edf7fe640 1 -- 172.21.15.64:0/72654842 >> v1:172.21.15.64:6789/0 conn(0x7f3ee006ed10 legacy=0x7f3ee00cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3edf7fe640 1 -- 172.21.15.64:0/72654842 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ee00cc3c0 con 0x7f3ee00cd890 2024-03-04T07:30:51.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.476+0000 7f3ede7fc640 1 -- 172.21.15.64:0/72654842 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2926839683 0 0) 0x7f3ee00cc0e0 con 0x7f3ee006ed10 2024-03-04T07:30:51.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.477+0000 7f3edf7fe640 1 --2- 172.21.15.64:0/72654842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee00cd890 0x7f3ee015d700 secure :-1 s=READY pgs=990 cs=0 l=1 rev1=1 crypto rx=0x7f3ed8009d90 tx=0x7f3ed802bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2ae8f2879b7220fb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.477+0000 7f3ede7fc640 1 -- 172.21.15.64:0/72654842 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ed8039070 con 0x7f3ee00cd890 2024-03-04T07:30:51.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.477+0000 7f3ede7fc640 1 -- 172.21.15.64:0/72654842 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ed802cc50 con 0x7f3ee00cd890 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.477+0000 7f3ede7fc640 1 -- 172.21.15.64:0/72654842 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ed8034440 con 0x7f3ee00cd890 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 -- 172.21.15.64:0/72654842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee00cd890 msgr2=0x7f3ee015d700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/72654842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee00cd890 0x7f3ee015d700 secure :-1 s=READY pgs=990 cs=0 l=1 rev1=1 crypto rx=0x7f3ed8009d90 tx=0x7f3ed802bea0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 -- 172.21.15.64:0/72654842 shutdown_connections 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/72654842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee00cd890 0x7f3ee015d700 unknown :-1 s=CLOSED pgs=990 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/72654842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00b1020 0x7f3ee006af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 -- 172.21.15.64:0/72654842 >> 172.21.15.64:0/72654842 conn(0x7f3ee00d1a90 msgr2=0x7f3ee00d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 -- 172.21.15.64:0/72654842 shutdown_connections 2024-03-04T07:30:51.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.478+0000 7f3ee59d7640 1 -- 172.21.15.64:0/72654842 wait complete. 2024-03-04T07:30:51.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.479+0000 7f3ee59d7640 1 Processor -- start 2024-03-04T07:30:51.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.479+0000 7f3ee59d7640 1 -- start start 2024-03-04T07:30:51.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3ee59d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee006ed10 0x7f3ee00b4d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3ee59d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ee00b1020 0x7f3ee015d6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3ee59d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00cd890 0x7f3ee015dbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3ee59d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ee00ccc60 con 0x7f3ee006ed10 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3ee59d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ee00cc980 con 0x7f3ee00b1020 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3ee59d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ee00cdcc0 con 0x7f3ee00cd890 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee006ed10 0x7f3ee00b4d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee006ed10 0x7f3ee00b4d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42340/0 (socket says 172.21.15.64:42340) 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 -- 172.21.15.64:0/3693596116 learned_addr learned my addr 172.21.15.64:0/3693596116 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edf7fe640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00cd890 0x7f3ee015dbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 -- 172.21.15.64:0/3693596116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00cd890 msgr2=0x7f3ee015dbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00cd890 0x7f3ee015dbf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 -- 172.21.15.64:0/3693596116 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ee00b1020 msgr2=0x7f3ee015d6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ee00b1020 0x7f3ee015d6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 -- 172.21.15.64:0/3693596116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ee00b2580 con 0x7f3ee006ed10 2024-03-04T07:30:51.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.480+0000 7f3edeffd640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee006ed10 0x7f3ee00b4d30 secure :-1 s=READY pgs=991 cs=0 l=1 rev1=1 crypto rx=0x7f3ed000e9b0 tx=0x7f3ed000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.481+0000 7f3edcff9640 1 -- 172.21.15.64:0/3693596116 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ed0017070 con 0x7f3ee006ed10 2024-03-04T07:30:51.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.481+0000 7f3edcff9640 1 -- 172.21.15.64:0/3693596116 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ed000c9a0 con 0x7f3ee006ed10 2024-03-04T07:30:51.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.481+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3ee00b2860 con 0x7f3ee006ed10 2024-03-04T07:30:51.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.481+0000 7f3edcff9640 1 -- 172.21.15.64:0/3693596116 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ed000cd90 con 0x7f3ee006ed10 2024-03-04T07:30:51.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.481+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3ee00b2e20 con 0x7f3ee006ed10 2024-03-04T07:30:51.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.482+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3ee0067ff0 con 0x7f3ee006ed10 2024-03-04T07:30:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.482+0000 7f3edcff9640 1 -- 172.21.15.64:0/3693596116 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3ed001a070 con 0x7f3ee006ed10 2024-03-04T07:30:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.483+0000 7f3edcff9640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ec006b4d0 0x7f3ec006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.483+0000 7f3edcff9640 1 -- 172.21.15.64:0/3693596116 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3ed008a500 con 0x7f3ee006ed10 2024-03-04T07:30:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.483+0000 7f3ed7fff640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ec006b4d0 0x7f3ec006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.483+0000 7f3ed7fff640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ec006b4d0 0x7f3ec006d990 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7f3ec8007fd0 tx=0x7f3ec8007920 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.487+0000 7f3edcff9640 1 -- 172.21.15.64:0/3693596116 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3ed0059410 con 0x7f3ee006ed10 2024-03-04T07:30:51.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.669+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f3ee006f2f0 con 0x7f3ee006ed10 2024-03-04T07:30:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.670+0000 7f3edcff9640 1 -- 172.21.15.64:0/3693596116 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v79) v1 ==== 74+0+16612 (secure 0 0 0) 0x7f3ed0058d10 con 0x7f3ee006ed10 2024-03-04T07:30:51.670 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:51.670 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":79,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:49.980908+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:46.621410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:47.317858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"79","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ec006b4d0 msgr2=0x7f3ec006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ec006b4d0 0x7f3ec006d990 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7f3ec8007fd0 tx=0x7f3ec8007920 comp rx=0 tx=0).stop 2024-03-04T07:30:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee006ed10 msgr2=0x7f3ee00b4d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee006ed10 0x7f3ee00b4d30 secure :-1 s=READY pgs=991 cs=0 l=1 rev1=1 crypto rx=0x7f3ed000e9b0 tx=0x7f3ed000ee70 comp rx=0 tx=0).stop 2024-03-04T07:30:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 shutdown_connections 2024-03-04T07:30:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ec006b4d0 0x7f3ec006d990 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ee00cd890 0x7f3ee015dbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ee00b1020 0x7f3ee015d6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 --2- 172.21.15.64:0/3693596116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ee006ed10 0x7f3ee00b4d30 unknown :-1 s=CLOSED pgs=991 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 >> 172.21.15.64:0/3693596116 conn(0x7f3ee00d1a90 msgr2=0x7f3ee006f9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 shutdown_connections 2024-03-04T07:30:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.672+0000 7f3ee59d7640 1 -- 172.21.15.64:0/3693596116 wait complete. 2024-03-04T07:30:51.684 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:30:51.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:30:51.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.786+0000 7fbfe9b7a640 1 Processor -- start 2024-03-04T07:30:51.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.786+0000 7fbfe9b7a640 1 -- start start 2024-03-04T07:30:51.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.787+0000 7fbfe9b7a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbfe40b0e90 0x7fbfe406b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.787+0000 7fbfe9b7a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbfe40cee90 con 0x7fbfe40ae9b0 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.787+0000 7fbfe9b7a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbfe40cfc50 con 0x7fbfe406b5c0 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.787+0000 7fbfe9b7a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbfe40cfa00 con 0x7fbfe40b0e90 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.787+0000 7fbfe2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbfe40b0e90 0x7fbfe406b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.787+0000 7fbfe3fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbfe40ae9b0 0x7fbfe40d03a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48104/0 (socket says 172.21.15.64:48104) 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.787+0000 7fbfe3fff640 1 -- 172.21.15.64:0/2767212761 learned_addr learned my addr 172.21.15.64:0/2767212761 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.788+0000 7fbfe2ffd640 1 -- 172.21.15.64:0/2767212761 >> v1:172.21.15.66:6789/0 conn(0x7fbfe406b5c0 legacy=0x7fbfe40b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.788+0000 7fbfe2ffd640 1 -- 172.21.15.64:0/2767212761 >> v1:172.21.15.64:6789/0 conn(0x7fbfe40ae9b0 legacy=0x7fbfe40d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.788+0000 7fbfe27fc640 1 -- 172.21.15.64:0/2767212761 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4193908762 0 0) 0x7fbfe40cee90 con 0x7fbfe40ae9b0 2024-03-04T07:30:51.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.788+0000 7fbfe2ffd640 1 -- 172.21.15.64:0/2767212761 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbfe40cff30 con 0x7fbfe40b0e90 2024-03-04T07:30:51.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.789+0000 7fbfe2ffd640 1 --2- 172.21.15.64:0/2767212761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe406b9a0 secure :-1 s=READY pgs=993 cs=0 l=1 rev1=1 crypto rx=0x7fbfd40097a0 tx=0x7fbfd402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6e883697a785584 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.789+0000 7fbfe27fc640 1 -- 172.21.15.64:0/2767212761 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfd4035020 con 0x7fbfe40b0e90 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.789+0000 7fbfe27fc640 1 -- 172.21.15.64:0/2767212761 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbfd4031830 con 0x7fbfe40b0e90 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.789+0000 7fbfe27fc640 1 -- 172.21.15.64:0/2767212761 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfd4031c00 con 0x7fbfe40b0e90 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.789+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/2767212761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 msgr2=0x7fbfe406b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.789+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/2767212761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe406b9a0 secure :-1 s=READY pgs=993 cs=0 l=1 rev1=1 crypto rx=0x7fbfd40097a0 tx=0x7fbfd402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.789+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/2767212761 shutdown_connections 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.790+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/2767212761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe406b9a0 unknown :-1 s=CLOSED pgs=993 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.790+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/2767212761 >> 172.21.15.64:0/2767212761 conn(0x7fbfe40b36d0 msgr2=0x7fbfe40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.790+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/2767212761 shutdown_connections 2024-03-04T07:30:51.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.790+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/2767212761 wait complete. 2024-03-04T07:30:51.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.791+0000 7fbfe9b7a640 1 Processor -- start 2024-03-04T07:30:51.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe9b7a640 1 -- start start 2024-03-04T07:30:51.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe9b7a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbfe406b5c0 0x7fbfe41580e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe9b7a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbfe40ae9b0 0x7fbfe40b6640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe9b7a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe40c6d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe9b7a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbfe406c910 con 0x7fbfe40b0e90 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe9b7a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbfe406c790 con 0x7fbfe406b5c0 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe9b7a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbfe40d0ab0 con 0x7fbfe40ae9b0 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe40c6d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe40c6d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42352/0 (socket says 172.21.15.64:42352) 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe3fff640 1 -- 172.21.15.64:0/1649111380 learned_addr learned my addr 172.21.15.64:0/1649111380 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.792+0000 7fbfe2ffd640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbfe40ae9b0 0x7fbfe40b6640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe37fe640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbfe406b5c0 0x7fbfe41580e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe3fff640 1 -- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbfe40ae9b0 msgr2=0x7fbfe40b6640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe3fff640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbfe40ae9b0 0x7fbfe40b6640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe3fff640 1 -- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbfe406b5c0 msgr2=0x7fbfe41580e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe3fff640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbfe406b5c0 0x7fbfe41580e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe3fff640 1 -- 172.21.15.64:0/1649111380 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbfe4158620 con 0x7fbfe40b0e90 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe3fff640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe40c6d40 secure :-1 s=READY pgs=994 cs=0 l=1 rev1=1 crypto rx=0x7fbfdc00c7a0 tx=0x7fbfdc00cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfcbfff640 1 -- 172.21.15.64:0/1649111380 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfdc018070 con 0x7fbfe40b0e90 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfcbfff640 1 -- 172.21.15.64:0/1649111380 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbfdc00a640 con 0x7fbfe40b0e90 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbfe40c5630 con 0x7fbfe40b0e90 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfcbfff640 1 -- 172.21.15.64:0/1649111380 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfdc00aab0 con 0x7fbfe40b0e90 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.793+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbfe40c5bf0 con 0x7fbfe40b0e90 2024-03-04T07:30:51.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.794+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbfe4068700 con 0x7fbfe40b0e90 2024-03-04T07:30:51.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.795+0000 7fbfcbfff640 1 -- 172.21.15.64:0/1649111380 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbfdc01b070 con 0x7fbfe40b0e90 2024-03-04T07:30:51.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.795+0000 7fbfcbfff640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbfd006b4d0 0x7fbfd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:51.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.795+0000 7fbfcbfff640 1 -- 172.21.15.64:0/1649111380 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbfdc0897f0 con 0x7fbfe40b0e90 2024-03-04T07:30:51.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.795+0000 7fbfe37fe640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbfd006b4d0 0x7fbfd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:51.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.796+0000 7fbfe37fe640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbfd006b4d0 0x7fbfd006d990 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fbfd8000f60 tx=0x7fbfd8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:51.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.800+0000 7fbfcbfff640 1 -- 172.21.15.64:0/1649111380 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbfe4158620 con 0x7fbfe40b0e90 2024-03-04T07:30:51.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.983+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fbfe406ba40 con 0x7fbfd006b4d0 2024-03-04T07:30:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.989+0000 7fbfcbfff640 1 -- 172.21.15.64:0/1649111380 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252073 (secure 0 0 0) 0x7fbfe406ba40 con 0x7fbfd006b4d0 2024-03-04T07:30:51.992 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:51.993 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbfd006b4d0 msgr2=0x7fbfd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbfd006b4d0 0x7fbfd006d990 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fbfd8000f60 tx=0x7fbfd8009040 comp rx=0 tx=0).stop 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 msgr2=0x7fbfe40c6d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe40c6d40 secure :-1 s=READY pgs=994 cs=0 l=1 rev1=1 crypto rx=0x7fbfdc00c7a0 tx=0x7fbfdc00cc60 comp rx=0 tx=0).stop 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 shutdown_connections 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbfd006b4d0 0x7fbfd006d990 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbfe40b0e90 0x7fbfe40c6d40 unknown :-1 s=CLOSED pgs=994 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbfe40ae9b0 0x7fbfe40b6640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 --2- 172.21.15.64:0/1649111380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbfe406b5c0 0x7fbfe41580e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 >> 172.21.15.64:0/1649111380 conn(0x7fbfe40b36d0 msgr2=0x7fbfe40cdf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 shutdown_connections 2024-03-04T07:30:51.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:51.993+0000 7fbfe9b7a640 1 -- 172.21.15.64:0/1649111380 wait complete. 2024-03-04T07:30:52.004 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-03-04T07:30:51.184364+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":258268,"kb_used_data":8784,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754716452,"statfs":{"total":773094113280,"available":772829646848,"internally_reserved":0,"allocated":8994816,"data_stored":4465356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":23,"apply_latency_ms":23,"commit_latency_ns":23000000,"apply_latency_ns":23000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.209535"},"pg_stats":[{"pgid":"13.3f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316081+0000","last_change":"2024-03-04T07:30:49.012242+0000","last_active":"2024-03-04T07:30:50.316081+0000","last_peered":"2024-03-04T07:30:50.316081+0000","last_clean":"2024-03-04T07:30:50.316081+0000","last_became_active":"2024-03-04T07:30:49.011919+0000","last_became_peered":"2024-03-04T07:30:49.011919+0000","last_unstale":"2024-03-04T07:30:50.316081+0000","last_undegraded":"2024-03-04T07:30:50.316081+0000","last_fullsized":"2024-03-04T07:30:50.316081+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:52:53.722382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984526+0000","last_change":"2024-03-04T07:30:49.004230+0000","last_active":"2024-03-04T07:30:49.984526+0000","last_peered":"2024-03-04T07:30:49.984526+0000","last_clean":"2024-03-04T07:30:49.984526+0000","last_became_active":"2024-03-04T07:30:49.003961+0000","last_became_peered":"2024-03-04T07:30:49.003961+0000","last_unstale":"2024-03-04T07:30:49.984526+0000","last_undegraded":"2024-03-04T07:30:49.984526+0000","last_fullsized":"2024-03-04T07:30:49.984526+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:11.912503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.3d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316632+0000","last_change":"2024-03-04T07:30:48.999604+0000","last_active":"2024-03-04T07:30:50.316632+0000","last_peered":"2024-03-04T07:30:50.316632+0000","last_clean":"2024-03-04T07:30:50.316632+0000","last_became_active":"2024-03-04T07:30:48.999141+0000","last_became_peered":"2024-03-04T07:30:48.999141+0000","last_unstale":"2024-03-04T07:30:50.316632+0000","last_undegraded":"2024-03-04T07:30:50.316632+0000","last_fullsized":"2024-03-04T07:30:50.316632+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:18:33.481737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.3c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315593+0000","last_change":"2024-03-04T07:30:49.000233+0000","last_active":"2024-03-04T07:30:50.315593+0000","last_peered":"2024-03-04T07:30:50.315593+0000","last_clean":"2024-03-04T07:30:50.315593+0000","last_became_active":"2024-03-04T07:30:48.999972+0000","last_became_peered":"2024-03-04T07:30:48.999972+0000","last_unstale":"2024-03-04T07:30:50.315593+0000","last_undegraded":"2024-03-04T07:30:50.315593+0000","last_fullsized":"2024-03-04T07:30:50.315593+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:45:38.423092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.3b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988092+0000","last_change":"2024-03-04T07:30:49.015413+0000","last_active":"2024-03-04T07:30:49.988092+0000","last_peered":"2024-03-04T07:30:49.988092+0000","last_clean":"2024-03-04T07:30:49.988092+0000","last_became_active":"2024-03-04T07:30:49.015065+0000","last_became_peered":"2024-03-04T07:30:49.015065+0000","last_unstale":"2024-03-04T07:30:49.988092+0000","last_undegraded":"2024-03-04T07:30:49.988092+0000","last_fullsized":"2024-03-04T07:30:49.988092+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:02.000572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.989044+0000","last_change":"2024-03-04T07:30:49.006999+0000","last_active":"2024-03-04T07:30:49.989044+0000","last_peered":"2024-03-04T07:30:49.989044+0000","last_clean":"2024-03-04T07:30:49.989044+0000","last_became_active":"2024-03-04T07:30:49.006684+0000","last_became_peered":"2024-03-04T07:30:49.006684+0000","last_unstale":"2024-03-04T07:30:49.989044+0000","last_undegraded":"2024-03-04T07:30:49.989044+0000","last_fullsized":"2024-03-04T07:30:49.989044+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:53:34.835694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.39","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316013+0000","last_change":"2024-03-04T07:30:48.995479+0000","last_active":"2024-03-04T07:30:50.316013+0000","last_peered":"2024-03-04T07:30:50.316013+0000","last_clean":"2024-03-04T07:30:50.316013+0000","last_became_active":"2024-03-04T07:30:48.995215+0000","last_became_peered":"2024-03-04T07:30:48.995215+0000","last_unstale":"2024-03-04T07:30:50.316013+0000","last_undegraded":"2024-03-04T07:30:50.316013+0000","last_fullsized":"2024-03-04T07:30:50.316013+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:04:55.380489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.38","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.987883+0000","last_change":"2024-03-04T07:30:49.006280+0000","last_active":"2024-03-04T07:30:49.987883+0000","last_peered":"2024-03-04T07:30:49.987883+0000","last_clean":"2024-03-04T07:30:49.987883+0000","last_became_active":"2024-03-04T07:30:49.005363+0000","last_became_peered":"2024-03-04T07:30:49.005363+0000","last_unstale":"2024-03-04T07:30:49.987883+0000","last_undegraded":"2024-03-04T07:30:49.987883+0000","last_fullsized":"2024-03-04T07:30:49.987883+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:44:35.866927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.37","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988399+0000","last_change":"2024-03-04T07:30:49.009418+0000","last_active":"2024-03-04T07:30:49.988399+0000","last_peered":"2024-03-04T07:30:49.988399+0000","last_clean":"2024-03-04T07:30:49.988399+0000","last_became_active":"2024-03-04T07:30:49.009066+0000","last_became_peered":"2024-03-04T07:30:49.009066+0000","last_unstale":"2024-03-04T07:30:49.988399+0000","last_undegraded":"2024-03-04T07:30:49.988399+0000","last_fullsized":"2024-03-04T07:30:49.988399+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:46:39.225709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.36","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316107+0000","last_change":"2024-03-04T07:30:48.997816+0000","last_active":"2024-03-04T07:30:50.316107+0000","last_peered":"2024-03-04T07:30:50.316107+0000","last_clean":"2024-03-04T07:30:50.316107+0000","last_became_active":"2024-03-04T07:30:48.997506+0000","last_became_peered":"2024-03-04T07:30:48.997506+0000","last_unstale":"2024-03-04T07:30:50.316107+0000","last_undegraded":"2024-03-04T07:30:50.316107+0000","last_fullsized":"2024-03-04T07:30:50.316107+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:40.772647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.35","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984460+0000","last_change":"2024-03-04T07:30:49.008007+0000","last_active":"2024-03-04T07:30:49.984460+0000","last_peered":"2024-03-04T07:30:49.984460+0000","last_clean":"2024-03-04T07:30:49.984460+0000","last_became_active":"2024-03-04T07:30:49.007700+0000","last_became_peered":"2024-03-04T07:30:49.007700+0000","last_unstale":"2024-03-04T07:30:49.984460+0000","last_undegraded":"2024-03-04T07:30:49.984460+0000","last_fullsized":"2024-03-04T07:30:49.984460+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:32.645561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.34","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315724+0000","last_change":"2024-03-04T07:30:49.005039+0000","last_active":"2024-03-04T07:30:50.315724+0000","last_peered":"2024-03-04T07:30:50.315724+0000","last_clean":"2024-03-04T07:30:50.315724+0000","last_became_active":"2024-03-04T07:30:49.004747+0000","last_became_peered":"2024-03-04T07:30:49.004747+0000","last_unstale":"2024-03-04T07:30:50.315724+0000","last_undegraded":"2024-03-04T07:30:50.315724+0000","last_fullsized":"2024-03-04T07:30:50.315724+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:12.423638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.33","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316029+0000","last_change":"2024-03-04T07:30:49.007907+0000","last_active":"2024-03-04T07:30:50.316029+0000","last_peered":"2024-03-04T07:30:50.316029+0000","last_clean":"2024-03-04T07:30:50.316029+0000","last_became_active":"2024-03-04T07:30:49.007475+0000","last_became_peered":"2024-03-04T07:30:49.007475+0000","last_unstale":"2024-03-04T07:30:50.316029+0000","last_undegraded":"2024-03-04T07:30:50.316029+0000","last_fullsized":"2024-03-04T07:30:50.316029+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:52:53.722382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.32","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316426+0000","last_change":"2024-03-04T07:30:49.007138+0000","last_active":"2024-03-04T07:30:50.316426+0000","last_peered":"2024-03-04T07:30:50.316426+0000","last_clean":"2024-03-04T07:30:50.316426+0000","last_became_active":"2024-03-04T07:30:49.006528+0000","last_became_peered":"2024-03-04T07:30:49.006528+0000","last_unstale":"2024-03-04T07:30:50.316426+0000","last_undegraded":"2024-03-04T07:30:50.316426+0000","last_fullsized":"2024-03-04T07:30:50.316426+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:00:48.332069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.31","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315900+0000","last_change":"2024-03-04T07:30:49.005145+0000","last_active":"2024-03-04T07:30:50.315900+0000","last_peered":"2024-03-04T07:30:50.315900+0000","last_clean":"2024-03-04T07:30:50.315900+0000","last_became_active":"2024-03-04T07:30:49.004867+0000","last_became_peered":"2024-03-04T07:30:49.004867+0000","last_unstale":"2024-03-04T07:30:50.315900+0000","last_undegraded":"2024-03-04T07:30:50.315900+0000","last_fullsized":"2024-03-04T07:30:50.315900+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:18:26.851147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.30","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.987858+0000","last_change":"2024-03-04T07:30:49.011262+0000","last_active":"2024-03-04T07:30:49.987858+0000","last_peered":"2024-03-04T07:30:49.987858+0000","last_clean":"2024-03-04T07:30:49.987858+0000","last_became_active":"2024-03-04T07:30:49.010939+0000","last_became_peered":"2024-03-04T07:30:49.010939+0000","last_unstale":"2024-03-04T07:30:49.987858+0000","last_undegraded":"2024-03-04T07:30:49.987858+0000","last_fullsized":"2024-03-04T07:30:49.987858+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:01:53.739843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988477+0000","last_change":"2024-03-04T07:30:49.007921+0000","last_active":"2024-03-04T07:30:49.988477+0000","last_peered":"2024-03-04T07:30:49.988477+0000","last_clean":"2024-03-04T07:30:49.988477+0000","last_became_active":"2024-03-04T07:30:49.007626+0000","last_became_peered":"2024-03-04T07:30:49.007626+0000","last_unstale":"2024-03-04T07:30:49.988477+0000","last_undegraded":"2024-03-04T07:30:49.988477+0000","last_fullsized":"2024-03-04T07:30:49.988477+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:12:42.814855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315719+0000","last_change":"2024-03-04T07:30:48.997786+0000","last_active":"2024-03-04T07:30:50.315719+0000","last_peered":"2024-03-04T07:30:50.315719+0000","last_clean":"2024-03-04T07:30:50.315719+0000","last_became_active":"2024-03-04T07:30:48.997337+0000","last_became_peered":"2024-03-04T07:30:48.997337+0000","last_unstale":"2024-03-04T07:30:50.315719+0000","last_undegraded":"2024-03-04T07:30:50.315719+0000","last_fullsized":"2024-03-04T07:30:50.315719+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:15.795433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316295+0000","last_change":"2024-03-04T07:30:48.989857+0000","last_active":"2024-03-04T07:30:50.316295+0000","last_peered":"2024-03-04T07:30:50.316295+0000","last_clean":"2024-03-04T07:30:50.316295+0000","last_became_active":"2024-03-04T07:30:48.989566+0000","last_became_peered":"2024-03-04T07:30:48.989566+0000","last_unstale":"2024-03-04T07:30:50.316295+0000","last_undegraded":"2024-03-04T07:30:50.316295+0000","last_fullsized":"2024-03-04T07:30:50.316295+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:02.000572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.2c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315796+0000","last_change":"2024-03-04T07:30:48.992725+0000","last_active":"2024-03-04T07:30:50.315796+0000","last_peered":"2024-03-04T07:30:50.315796+0000","last_clean":"2024-03-04T07:30:50.315796+0000","last_became_active":"2024-03-04T07:30:48.992440+0000","last_became_peered":"2024-03-04T07:30:48.992440+0000","last_unstale":"2024-03-04T07:30:50.315796+0000","last_undegraded":"2024-03-04T07:30:50.315796+0000","last_fullsized":"2024-03-04T07:30:50.315796+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:34:35.192750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315377+0000","last_change":"2024-03-04T07:30:48.992658+0000","last_active":"2024-03-04T07:30:50.315377+0000","last_peered":"2024-03-04T07:30:50.315377+0000","last_clean":"2024-03-04T07:30:50.315377+0000","last_became_active":"2024-03-04T07:30:48.992314+0000","last_became_peered":"2024-03-04T07:30:48.992314+0000","last_unstale":"2024-03-04T07:30:50.315377+0000","last_undegraded":"2024-03-04T07:30:50.315377+0000","last_fullsized":"2024-03-04T07:30:50.315377+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:39:38.583112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984188+0000","last_change":"2024-03-04T07:30:49.011212+0000","last_active":"2024-03-04T07:30:49.984188+0000","last_peered":"2024-03-04T07:30:49.984188+0000","last_clean":"2024-03-04T07:30:49.984188+0000","last_became_active":"2024-03-04T07:30:49.010957+0000","last_became_peered":"2024-03-04T07:30:49.010957+0000","last_unstale":"2024-03-04T07:30:49.984188+0000","last_undegraded":"2024-03-04T07:30:49.984188+0000","last_fullsized":"2024-03-04T07:30:49.984188+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:27:37.989524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.29","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315467+0000","last_change":"2024-03-04T07:30:49.008950+0000","last_active":"2024-03-04T07:30:50.315467+0000","last_peered":"2024-03-04T07:30:50.315467+0000","last_clean":"2024-03-04T07:30:50.315467+0000","last_became_active":"2024-03-04T07:30:49.008557+0000","last_became_peered":"2024-03-04T07:30:49.008557+0000","last_unstale":"2024-03-04T07:30:50.315467+0000","last_undegraded":"2024-03-04T07:30:50.315467+0000","last_fullsized":"2024-03-04T07:30:50.315467+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:24.510630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.28","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315436+0000","last_change":"2024-03-04T07:30:48.989590+0000","last_active":"2024-03-04T07:30:50.315436+0000","last_peered":"2024-03-04T07:30:50.315436+0000","last_clean":"2024-03-04T07:30:50.315436+0000","last_became_active":"2024-03-04T07:30:48.989279+0000","last_became_peered":"2024-03-04T07:30:48.989279+0000","last_unstale":"2024-03-04T07:30:50.315436+0000","last_undegraded":"2024-03-04T07:30:50.315436+0000","last_fullsized":"2024-03-04T07:30:50.315436+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:09:23.017980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.27","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984504+0000","last_change":"2024-03-04T07:30:49.001293+0000","last_active":"2024-03-04T07:30:49.984504+0000","last_peered":"2024-03-04T07:30:49.984504+0000","last_clean":"2024-03-04T07:30:49.984504+0000","last_became_active":"2024-03-04T07:30:49.000948+0000","last_became_peered":"2024-03-04T07:30:49.000948+0000","last_unstale":"2024-03-04T07:30:49.984504+0000","last_undegraded":"2024-03-04T07:30:49.984504+0000","last_fullsized":"2024-03-04T07:30:49.984504+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:47.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.26","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316325+0000","last_change":"2024-03-04T07:30:49.005947+0000","last_active":"2024-03-04T07:30:50.316325+0000","last_peered":"2024-03-04T07:30:50.316325+0000","last_clean":"2024-03-04T07:30:50.316325+0000","last_became_active":"2024-03-04T07:30:49.005565+0000","last_became_peered":"2024-03-04T07:30:49.005565+0000","last_unstale":"2024-03-04T07:30:50.316325+0000","last_undegraded":"2024-03-04T07:30:50.316325+0000","last_fullsized":"2024-03-04T07:30:50.316325+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:04:00.152366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.25","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988205+0000","last_change":"2024-03-04T07:30:49.007009+0000","last_active":"2024-03-04T07:30:49.988205+0000","last_peered":"2024-03-04T07:30:49.988205+0000","last_clean":"2024-03-04T07:30:49.988205+0000","last_became_active":"2024-03-04T07:30:49.006678+0000","last_became_peered":"2024-03-04T07:30:49.006678+0000","last_unstale":"2024-03-04T07:30:49.988205+0000","last_undegraded":"2024-03-04T07:30:49.988205+0000","last_fullsized":"2024-03-04T07:30:49.988205+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:24.510630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.24","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315563+0000","last_change":"2024-03-04T07:30:49.012546+0000","last_active":"2024-03-04T07:30:50.315563+0000","last_peered":"2024-03-04T07:30:50.315563+0000","last_clean":"2024-03-04T07:30:50.315563+0000","last_became_active":"2024-03-04T07:30:49.012205+0000","last_became_peered":"2024-03-04T07:30:49.012205+0000","last_unstale":"2024-03-04T07:30:50.315563+0000","last_undegraded":"2024-03-04T07:30:50.315563+0000","last_fullsized":"2024-03-04T07:30:50.315563+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:05.868581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.23","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984214+0000","last_change":"2024-03-04T07:30:48.999135+0000","last_active":"2024-03-04T07:30:49.984214+0000","last_peered":"2024-03-04T07:30:49.984214+0000","last_clean":"2024-03-04T07:30:49.984214+0000","last_became_active":"2024-03-04T07:30:48.998850+0000","last_became_peered":"2024-03-04T07:30:48.998850+0000","last_unstale":"2024-03-04T07:30:49.984214+0000","last_undegraded":"2024-03-04T07:30:49.984214+0000","last_fullsized":"2024-03-04T07:30:49.984214+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:41:54.404669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.22","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984258+0000","last_change":"2024-03-04T07:30:49.005880+0000","last_active":"2024-03-04T07:30:49.984258+0000","last_peered":"2024-03-04T07:30:49.984258+0000","last_clean":"2024-03-04T07:30:49.984258+0000","last_became_active":"2024-03-04T07:30:49.005535+0000","last_became_peered":"2024-03-04T07:30:49.005535+0000","last_unstale":"2024-03-04T07:30:49.984258+0000","last_undegraded":"2024-03-04T07:30:49.984258+0000","last_fullsized":"2024-03-04T07:30:49.984258+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:25:03.653270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.21","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.987893+0000","last_change":"2024-03-04T07:30:49.013378+0000","last_active":"2024-03-04T07:30:49.987893+0000","last_peered":"2024-03-04T07:30:49.987893+0000","last_clean":"2024-03-04T07:30:49.987893+0000","last_became_active":"2024-03-04T07:30:49.013049+0000","last_became_peered":"2024-03-04T07:30:49.013049+0000","last_unstale":"2024-03-04T07:30:49.987893+0000","last_undegraded":"2024-03-04T07:30:49.987893+0000","last_fullsized":"2024-03-04T07:30:49.987893+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:26.008072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.20","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315430+0000","last_change":"2024-03-04T07:30:49.004437+0000","last_active":"2024-03-04T07:30:50.315430+0000","last_peered":"2024-03-04T07:30:50.315430+0000","last_clean":"2024-03-04T07:30:50.315430+0000","last_became_active":"2024-03-04T07:30:49.003525+0000","last_became_peered":"2024-03-04T07:30:49.003525+0000","last_unstale":"2024-03-04T07:30:50.315430+0000","last_undegraded":"2024-03-04T07:30:50.315430+0000","last_fullsized":"2024-03-04T07:30:50.315430+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:26.008072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988517+0000","last_change":"2024-03-04T07:30:49.016311+0000","last_active":"2024-03-04T07:30:49.988517+0000","last_peered":"2024-03-04T07:30:49.988517+0000","last_clean":"2024-03-04T07:30:49.988517+0000","last_became_active":"2024-03-04T07:30:49.015977+0000","last_became_peered":"2024-03-04T07:30:49.015977+0000","last_unstale":"2024-03-04T07:30:49.988517+0000","last_undegraded":"2024-03-04T07:30:49.988517+0000","last_fullsized":"2024-03-04T07:30:49.988517+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:52:53.722382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1f","version":"78'4","reported_seq":18,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316403+0000","last_change":"2024-03-04T07:30:47.995691+0000","last_active":"2024-03-04T07:30:50.316403+0000","last_peered":"2024-03-04T07:30:50.316403+0000","last_clean":"2024-03-04T07:30:50.316403+0000","last_became_active":"2024-03-04T07:30:47.995375+0000","last_became_peered":"2024-03-04T07:30:47.995375+0000","last_unstale":"2024-03-04T07:30:50.316403+0000","last_undegraded":"2024-03-04T07:30:50.316403+0000","last_fullsized":"2024-03-04T07:30:50.316403+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:10.910681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316278+0000","last_change":"2024-03-04T07:30:49.009335+0000","last_active":"2024-03-04T07:30:50.316278+0000","last_peered":"2024-03-04T07:30:50.316278+0000","last_clean":"2024-03-04T07:30:50.316278+0000","last_became_active":"2024-03-04T07:30:49.008718+0000","last_became_peered":"2024-03-04T07:30:49.008718+0000","last_unstale":"2024-03-04T07:30:50.316278+0000","last_undegraded":"2024-03-04T07:30:50.316278+0000","last_fullsized":"2024-03-04T07:30:50.316278+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:45.473804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1e","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988761+0000","last_change":"2024-03-04T07:30:47.998228+0000","last_active":"2024-03-04T07:30:49.988761+0000","last_peered":"2024-03-04T07:30:49.988761+0000","last_clean":"2024-03-04T07:30:49.988761+0000","last_became_active":"2024-03-04T07:30:47.997896+0000","last_became_peered":"2024-03-04T07:30:47.997896+0000","last_unstale":"2024-03-04T07:30:49.988761+0000","last_undegraded":"2024-03-04T07:30:49.988761+0000","last_fullsized":"2024-03-04T07:30:49.988761+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:06.687014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315874+0000","last_change":"2024-03-04T07:30:49.011844+0000","last_active":"2024-03-04T07:30:50.315874+0000","last_peered":"2024-03-04T07:30:50.315874+0000","last_clean":"2024-03-04T07:30:50.315874+0000","last_became_active":"2024-03-04T07:30:49.011531+0000","last_became_peered":"2024-03-04T07:30:49.011531+0000","last_unstale":"2024-03-04T07:30:50.315874+0000","last_undegraded":"2024-03-04T07:30:50.315874+0000","last_fullsized":"2024-03-04T07:30:50.315874+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:12:42.814855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1d","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316393+0000","last_change":"2024-03-04T07:30:47.994139+0000","last_active":"2024-03-04T07:30:50.316393+0000","last_peered":"2024-03-04T07:30:50.316393+0000","last_clean":"2024-03-04T07:30:50.316393+0000","last_became_active":"2024-03-04T07:30:47.993853+0000","last_became_peered":"2024-03-04T07:30:47.993853+0000","last_unstale":"2024-03-04T07:30:50.316393+0000","last_undegraded":"2024-03-04T07:30:50.316393+0000","last_fullsized":"2024-03-04T07:30:50.316393+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:05:43.306245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316304+0000","last_change":"2024-03-04T07:30:49.005155+0000","last_active":"2024-03-04T07:30:50.316304+0000","last_peered":"2024-03-04T07:30:50.316304+0000","last_clean":"2024-03-04T07:30:50.316304+0000","last_became_active":"2024-03-04T07:30:49.004851+0000","last_became_peered":"2024-03-04T07:30:49.004851+0000","last_unstale":"2024-03-04T07:30:50.316304+0000","last_undegraded":"2024-03-04T07:30:50.316304+0000","last_fullsized":"2024-03-04T07:30:50.316304+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:45.473804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1c","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316472+0000","last_change":"2024-03-04T07:30:47.994819+0000","last_active":"2024-03-04T07:30:50.316472+0000","last_peered":"2024-03-04T07:30:50.316472+0000","last_clean":"2024-03-04T07:30:50.316472+0000","last_became_active":"2024-03-04T07:30:47.994566+0000","last_became_peered":"2024-03-04T07:30:47.994566+0000","last_unstale":"2024-03-04T07:30:50.316472+0000","last_undegraded":"2024-03-04T07:30:50.316472+0000","last_fullsized":"2024-03-04T07:30:50.316472+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:44:34.865105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984335+0000","last_change":"2024-03-04T07:30:49.005803+0000","last_active":"2024-03-04T07:30:49.984335+0000","last_peered":"2024-03-04T07:30:49.984335+0000","last_clean":"2024-03-04T07:30:49.984335+0000","last_became_active":"2024-03-04T07:30:49.005159+0000","last_became_peered":"2024-03-04T07:30:49.005159+0000","last_unstale":"2024-03-04T07:30:49.984335+0000","last_undegraded":"2024-03-04T07:30:49.984335+0000","last_fullsized":"2024-03-04T07:30:49.984335+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:21.304097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.1b","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988672+0000","last_change":"2024-03-04T07:30:47.996445+0000","last_active":"2024-03-04T07:30:49.988672+0000","last_peered":"2024-03-04T07:30:49.988672+0000","last_clean":"2024-03-04T07:30:49.988672+0000","last_became_active":"2024-03-04T07:30:47.996085+0000","last_became_peered":"2024-03-04T07:30:47.996085+0000","last_unstale":"2024-03-04T07:30:49.988672+0000","last_undegraded":"2024-03-04T07:30:49.988672+0000","last_fullsized":"2024-03-04T07:30:49.988672+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:10:05.632096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988487+0000","last_change":"2024-03-04T07:30:49.012711+0000","last_active":"2024-03-04T07:30:49.988487+0000","last_peered":"2024-03-04T07:30:49.988487+0000","last_clean":"2024-03-04T07:30:49.988487+0000","last_became_active":"2024-03-04T07:30:49.012405+0000","last_became_peered":"2024-03-04T07:30:49.012405+0000","last_unstale":"2024-03-04T07:30:49.988487+0000","last_undegraded":"2024-03-04T07:30:49.988487+0000","last_fullsized":"2024-03-04T07:30:49.988487+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:15:55.458985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1a","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988037+0000","last_change":"2024-03-04T07:30:47.998690+0000","last_active":"2024-03-04T07:30:49.988037+0000","last_peered":"2024-03-04T07:30:49.988037+0000","last_clean":"2024-03-04T07:30:49.988037+0000","last_became_active":"2024-03-04T07:30:47.998339+0000","last_became_peered":"2024-03-04T07:30:47.998339+0000","last_unstale":"2024-03-04T07:30:49.988037+0000","last_undegraded":"2024-03-04T07:30:49.988037+0000","last_fullsized":"2024-03-04T07:30:49.988037+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:31.643739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.18","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315586+0000","last_change":"2024-03-04T07:30:49.009085+0000","last_active":"2024-03-04T07:30:50.315586+0000","last_peered":"2024-03-04T07:30:50.315586+0000","last_clean":"2024-03-04T07:30:50.315586+0000","last_became_active":"2024-03-04T07:30:49.008620+0000","last_became_peered":"2024-03-04T07:30:49.008620+0000","last_unstale":"2024-03-04T07:30:50.315586+0000","last_undegraded":"2024-03-04T07:30:50.315586+0000","last_fullsized":"2024-03-04T07:30:50.315586+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:12:42.814855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315763+0000","last_change":"2024-03-04T07:30:48.000338+0000","last_active":"2024-03-04T07:30:50.315763+0000","last_peered":"2024-03-04T07:30:50.315763+0000","last_clean":"2024-03-04T07:30:50.315763+0000","last_became_active":"2024-03-04T07:30:48.000044+0000","last_became_peered":"2024-03-04T07:30:48.000044+0000","last_unstale":"2024-03-04T07:30:50.315763+0000","last_undegraded":"2024-03-04T07:30:50.315763+0000","last_fullsized":"2024-03-04T07:30:50.315763+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:14:34.507741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.19","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316034+0000","last_change":"2024-03-04T07:30:48.997115+0000","last_active":"2024-03-04T07:30:50.316034+0000","last_peered":"2024-03-04T07:30:50.316034+0000","last_clean":"2024-03-04T07:30:50.316034+0000","last_became_active":"2024-03-04T07:30:48.996827+0000","last_became_peered":"2024-03-04T07:30:48.996827+0000","last_unstale":"2024-03-04T07:30:50.316034+0000","last_undegraded":"2024-03-04T07:30:50.316034+0000","last_fullsized":"2024-03-04T07:30:50.316034+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:57.419534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984050+0000","last_change":"2024-03-04T07:30:47.998585+0000","last_active":"2024-03-04T07:30:49.984050+0000","last_peered":"2024-03-04T07:30:49.984050+0000","last_clean":"2024-03-04T07:30:49.984050+0000","last_became_active":"2024-03-04T07:30:47.998267+0000","last_became_peered":"2024-03-04T07:30:47.998267+0000","last_unstale":"2024-03-04T07:30:49.984050+0000","last_undegraded":"2024-03-04T07:30:49.984050+0000","last_fullsized":"2024-03-04T07:30:49.984050+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:53:21.514299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.16","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316879+0000","last_change":"2024-03-04T07:30:49.001460+0000","last_active":"2024-03-04T07:30:50.316879+0000","last_peered":"2024-03-04T07:30:50.316879+0000","last_clean":"2024-03-04T07:30:50.316879+0000","last_became_active":"2024-03-04T07:30:49.001202+0000","last_became_peered":"2024-03-04T07:30:49.001202+0000","last_unstale":"2024-03-04T07:30:50.316879+0000","last_undegraded":"2024-03-04T07:30:50.316879+0000","last_fullsized":"2024-03-04T07:30:50.316879+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:42:00.836176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316399+0000","last_change":"2024-03-04T07:30:47.994776+0000","last_active":"2024-03-04T07:30:50.316399+0000","last_peered":"2024-03-04T07:30:50.316399+0000","last_clean":"2024-03-04T07:30:50.316399+0000","last_became_active":"2024-03-04T07:30:47.994204+0000","last_became_peered":"2024-03-04T07:30:47.994204+0000","last_unstale":"2024-03-04T07:30:50.316399+0000","last_undegraded":"2024-03-04T07:30:50.316399+0000","last_fullsized":"2024-03-04T07:30:50.316399+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:06.687014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988572+0000","last_change":"2024-03-04T07:30:49.006750+0000","last_active":"2024-03-04T07:30:49.988572+0000","last_peered":"2024-03-04T07:30:49.988572+0000","last_clean":"2024-03-04T07:30:49.988572+0000","last_became_active":"2024-03-04T07:30:49.006394+0000","last_became_peered":"2024-03-04T07:30:49.006394+0000","last_unstale":"2024-03-04T07:30:49.988572+0000","last_undegraded":"2024-03-04T07:30:49.988572+0000","last_fullsized":"2024-03-04T07:30:49.988572+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:16.705233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316188+0000","last_change":"2024-03-04T07:30:47.996670+0000","last_active":"2024-03-04T07:30:50.316188+0000","last_peered":"2024-03-04T07:30:50.316188+0000","last_clean":"2024-03-04T07:30:50.316188+0000","last_became_active":"2024-03-04T07:30:47.996290+0000","last_became_peered":"2024-03-04T07:30:47.996290+0000","last_unstale":"2024-03-04T07:30:50.316188+0000","last_undegraded":"2024-03-04T07:30:50.316188+0000","last_fullsized":"2024-03-04T07:30:50.316188+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:31.643739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.14","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316371+0000","last_change":"2024-03-04T07:30:49.000791+0000","last_active":"2024-03-04T07:30:50.316371+0000","last_peered":"2024-03-04T07:30:50.316371+0000","last_clean":"2024-03-04T07:30:50.316371+0000","last_became_active":"2024-03-04T07:30:49.000511+0000","last_became_peered":"2024-03-04T07:30:49.000511+0000","last_unstale":"2024-03-04T07:30:50.316371+0000","last_undegraded":"2024-03-04T07:30:50.316371+0000","last_fullsized":"2024-03-04T07:30:50.316371+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:05.868581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.15","version":"78'3","reported_seq":17,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316289+0000","last_change":"2024-03-04T07:30:47.994763+0000","last_active":"2024-03-04T07:30:50.316289+0000","last_peered":"2024-03-04T07:30:50.316289+0000","last_clean":"2024-03-04T07:30:50.316289+0000","last_became_active":"2024-03-04T07:30:47.994432+0000","last_became_peered":"2024-03-04T07:30:47.994432+0000","last_unstale":"2024-03-04T07:30:50.316289+0000","last_undegraded":"2024-03-04T07:30:50.316289+0000","last_fullsized":"2024-03-04T07:30:50.316289+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:20.302275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.15","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988190+0000","last_change":"2024-03-04T07:30:49.013600+0000","last_active":"2024-03-04T07:30:49.988190+0000","last_peered":"2024-03-04T07:30:49.988190+0000","last_clean":"2024-03-04T07:30:49.988190+0000","last_became_active":"2024-03-04T07:30:49.012745+0000","last_became_peered":"2024-03-04T07:30:49.012745+0000","last_unstale":"2024-03-04T07:30:49.988190+0000","last_undegraded":"2024-03-04T07:30:49.988190+0000","last_fullsized":"2024-03-04T07:30:49.988190+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:05.868581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"78'6","reported_seq":20,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.989056+0000","last_change":"2024-03-04T07:30:47.993284+0000","last_active":"2024-03-04T07:30:49.989056+0000","last_peered":"2024-03-04T07:30:49.989056+0000","last_clean":"2024-03-04T07:30:49.989056+0000","last_became_active":"2024-03-04T07:30:47.992982+0000","last_became_peered":"2024-03-04T07:30:47.992982+0000","last_unstale":"2024-03-04T07:30:49.989056+0000","last_undegraded":"2024-03-04T07:30:49.989056+0000","last_fullsized":"2024-03-04T07:30:49.989056+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:20.302275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.12","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315417+0000","last_change":"2024-03-04T07:30:49.007860+0000","last_active":"2024-03-04T07:30:50.315417+0000","last_peered":"2024-03-04T07:30:50.315417+0000","last_clean":"2024-03-04T07:30:50.315417+0000","last_became_active":"2024-03-04T07:30:49.007550+0000","last_became_peered":"2024-03-04T07:30:49.007550+0000","last_unstale":"2024-03-04T07:30:50.315417+0000","last_undegraded":"2024-03-04T07:30:50.315417+0000","last_fullsized":"2024-03-04T07:30:50.315417+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:26.008072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.13","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984318+0000","last_change":"2024-03-04T07:30:47.990436+0000","last_active":"2024-03-04T07:30:49.984318+0000","last_peered":"2024-03-04T07:30:49.984318+0000","last_clean":"2024-03-04T07:30:49.984318+0000","last_became_active":"2024-03-04T07:30:47.990145+0000","last_became_peered":"2024-03-04T07:30:47.990145+0000","last_unstale":"2024-03-04T07:30:49.984318+0000","last_undegraded":"2024-03-04T07:30:49.984318+0000","last_fullsized":"2024-03-04T07:30:49.984318+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:52.371659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.13","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316077+0000","last_change":"2024-03-04T07:30:49.000488+0000","last_active":"2024-03-04T07:30:50.316077+0000","last_peered":"2024-03-04T07:30:50.316077+0000","last_clean":"2024-03-04T07:30:50.316077+0000","last_became_active":"2024-03-04T07:30:49.000189+0000","last_became_peered":"2024-03-04T07:30:49.000189+0000","last_unstale":"2024-03-04T07:30:50.316077+0000","last_undegraded":"2024-03-04T07:30:50.316077+0000","last_fullsized":"2024-03-04T07:30:50.316077+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:40.772647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315640+0000","last_change":"2024-03-04T07:30:47.998081+0000","last_active":"2024-03-04T07:30:50.315640+0000","last_peered":"2024-03-04T07:30:50.315640+0000","last_clean":"2024-03-04T07:30:50.315640+0000","last_became_active":"2024-03-04T07:30:47.997756+0000","last_became_peered":"2024-03-04T07:30:47.997756+0000","last_unstale":"2024-03-04T07:30:50.315640+0000","last_undegraded":"2024-03-04T07:30:50.315640+0000","last_fullsized":"2024-03-04T07:30:50.315640+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:39.770825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.10","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984022+0000","last_change":"2024-03-04T07:30:49.006346+0000","last_active":"2024-03-04T07:30:49.984022+0000","last_peered":"2024-03-04T07:30:49.984022+0000","last_clean":"2024-03-04T07:30:49.984022+0000","last_became_active":"2024-03-04T07:30:49.006076+0000","last_became_peered":"2024-03-04T07:30:49.006076+0000","last_unstale":"2024-03-04T07:30:49.984022+0000","last_undegraded":"2024-03-04T07:30:49.984022+0000","last_fullsized":"2024-03-04T07:30:49.984022+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:35:45.932518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988477+0000","last_change":"2024-03-04T07:30:47.995953+0000","last_active":"2024-03-04T07:30:49.988477+0000","last_peered":"2024-03-04T07:30:49.988477+0000","last_clean":"2024-03-04T07:30:49.988477+0000","last_became_active":"2024-03-04T07:30:47.995518+0000","last_became_peered":"2024-03-04T07:30:47.995518+0000","last_unstale":"2024-03-04T07:30:49.988477+0000","last_undegraded":"2024-03-04T07:30:49.988477+0000","last_fullsized":"2024-03-04T07:30:49.988477+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:31.643739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.11","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984017+0000","last_change":"2024-03-04T07:30:48.999352+0000","last_active":"2024-03-04T07:30:49.984017+0000","last_peered":"2024-03-04T07:30:49.984017+0000","last_clean":"2024-03-04T07:30:49.984017+0000","last_became_active":"2024-03-04T07:30:48.999066+0000","last_became_peered":"2024-03-04T07:30:48.999066+0000","last_unstale":"2024-03-04T07:30:49.984017+0000","last_undegraded":"2024-03-04T07:30:49.984017+0000","last_fullsized":"2024-03-04T07:30:49.984017+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:29.273470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.10","version":"78'8","reported_seq":22,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315908+0000","last_change":"2024-03-04T07:30:47.999029+0000","last_active":"2024-03-04T07:30:50.315908+0000","last_peered":"2024-03-04T07:30:50.315908+0000","last_clean":"2024-03-04T07:30:50.315908+0000","last_became_active":"2024-03-04T07:30:47.998725+0000","last_became_peered":"2024-03-04T07:30:47.998725+0000","last_unstale":"2024-03-04T07:30:50.315908+0000","last_undegraded":"2024-03-04T07:30:50.315908+0000","last_fullsized":"2024-03-04T07:30:50.315908+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:09:01.359067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316482+0000","last_change":"2024-03-04T07:30:49.001803+0000","last_active":"2024-03-04T07:30:50.316482+0000","last_peered":"2024-03-04T07:30:50.316482+0000","last_clean":"2024-03-04T07:30:50.316482+0000","last_became_active":"2024-03-04T07:30:49.001497+0000","last_became_peered":"2024-03-04T07:30:49.001497+0000","last_unstale":"2024-03-04T07:30:50.316482+0000","last_undegraded":"2024-03-04T07:30:50.316482+0000","last_fullsized":"2024-03-04T07:30:50.316482+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:25:27.140576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.f","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316519+0000","last_change":"2024-03-04T07:30:47.988386+0000","last_active":"2024-03-04T07:30:50.316519+0000","last_peered":"2024-03-04T07:30:50.316519+0000","last_clean":"2024-03-04T07:30:50.316519+0000","last_became_active":"2024-03-04T07:30:47.988102+0000","last_became_peered":"2024-03-04T07:30:47.988102+0000","last_unstale":"2024-03-04T07:30:50.316519+0000","last_undegraded":"2024-03-04T07:30:50.316519+0000","last_fullsized":"2024-03-04T07:30:50.316519+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:10:05.632096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316328+0000","last_change":"2024-03-04T07:30:49.006698+0000","last_active":"2024-03-04T07:30:50.316328+0000","last_peered":"2024-03-04T07:30:50.316328+0000","last_clean":"2024-03-04T07:30:50.316328+0000","last_became_active":"2024-03-04T07:30:49.005131+0000","last_became_peered":"2024-03-04T07:30:49.005131+0000","last_unstale":"2024-03-04T07:30:50.316328+0000","last_undegraded":"2024-03-04T07:30:50.316328+0000","last_fullsized":"2024-03-04T07:30:50.316328+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:04:00.152366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316493+0000","last_change":"2024-03-04T07:30:48.000133+0000","last_active":"2024-03-04T07:30:50.316493+0000","last_peered":"2024-03-04T07:30:50.316493+0000","last_clean":"2024-03-04T07:30:50.316493+0000","last_became_active":"2024-03-04T07:30:47.999801+0000","last_became_peered":"2024-03-04T07:30:47.999801+0000","last_unstale":"2024-03-04T07:30:50.316493+0000","last_undegraded":"2024-03-04T07:30:50.316493+0000","last_fullsized":"2024-03-04T07:30:50.316493+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:43:08.764831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988644+0000","last_change":"2024-03-04T07:30:49.013867+0000","last_active":"2024-03-04T07:30:49.988644+0000","last_peered":"2024-03-04T07:30:49.988644+0000","last_clean":"2024-03-04T07:30:49.988644+0000","last_became_active":"2024-03-04T07:30:49.013536+0000","last_became_peered":"2024-03-04T07:30:49.013536+0000","last_unstale":"2024-03-04T07:30:49.988644+0000","last_undegraded":"2024-03-04T07:30:49.988644+0000","last_fullsized":"2024-03-04T07:30:49.988644+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:40.772647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316564+0000","last_change":"2024-03-04T07:30:47.999182+0000","last_active":"2024-03-04T07:30:50.316564+0000","last_peered":"2024-03-04T07:30:50.316564+0000","last_clean":"2024-03-04T07:30:50.316564+0000","last_became_active":"2024-03-04T07:30:47.998877+0000","last_became_peered":"2024-03-04T07:30:47.998877+0000","last_unstale":"2024-03-04T07:30:50.316564+0000","last_undegraded":"2024-03-04T07:30:50.316564+0000","last_fullsized":"2024-03-04T07:30:50.316564+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:57:58.248895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316436+0000","last_change":"2024-03-04T07:30:49.006916+0000","last_active":"2024-03-04T07:30:50.316436+0000","last_peered":"2024-03-04T07:30:50.316436+0000","last_clean":"2024-03-04T07:30:50.316436+0000","last_became_active":"2024-03-04T07:30:49.006195+0000","last_became_peered":"2024-03-04T07:30:49.006195+0000","last_unstale":"2024-03-04T07:30:50.316436+0000","last_undegraded":"2024-03-04T07:30:50.316436+0000","last_fullsized":"2024-03-04T07:30:50.316436+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:53.373481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316337+0000","last_change":"2024-03-04T07:30:47.996148+0000","last_active":"2024-03-04T07:30:50.316337+0000","last_peered":"2024-03-04T07:30:50.316337+0000","last_clean":"2024-03-04T07:30:50.316337+0000","last_became_active":"2024-03-04T07:30:47.995884+0000","last_became_peered":"2024-03-04T07:30:47.995884+0000","last_unstale":"2024-03-04T07:30:50.316337+0000","last_undegraded":"2024-03-04T07:30:50.316337+0000","last_fullsized":"2024-03-04T07:30:50.316337+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:03:59.150544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988206+0000","last_change":"2024-03-04T07:30:48.998811+0000","last_active":"2024-03-04T07:30:49.988206+0000","last_peered":"2024-03-04T07:30:49.988206+0000","last_clean":"2024-03-04T07:30:49.988206+0000","last_became_active":"2024-03-04T07:30:48.998405+0000","last_became_peered":"2024-03-04T07:30:48.998405+0000","last_unstale":"2024-03-04T07:30:49.988206+0000","last_undegraded":"2024-03-04T07:30:49.988206+0000","last_fullsized":"2024-03-04T07:30:49.988206+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:57:59.250717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316491+0000","last_change":"2024-03-04T07:30:47.991449+0000","last_active":"2024-03-04T07:30:50.316491+0000","last_peered":"2024-03-04T07:30:50.316491+0000","last_clean":"2024-03-04T07:30:50.316491+0000","last_became_active":"2024-03-04T07:30:47.991177+0000","last_became_peered":"2024-03-04T07:30:47.991177+0000","last_unstale":"2024-03-04T07:30:50.316491+0000","last_undegraded":"2024-03-04T07:30:50.316491+0000","last_fullsized":"2024-03-04T07:30:50.316491+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:19:56.702799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315571+0000","last_change":"2024-03-04T07:30:48.998228+0000","last_active":"2024-03-04T07:30:50.315571+0000","last_peered":"2024-03-04T07:30:50.315571+0000","last_clean":"2024-03-04T07:30:50.315571+0000","last_became_active":"2024-03-04T07:30:48.997947+0000","last_became_peered":"2024-03-04T07:30:48.997947+0000","last_unstale":"2024-03-04T07:30:50.315571+0000","last_undegraded":"2024-03-04T07:30:50.315571+0000","last_fullsized":"2024-03-04T07:30:50.315571+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:06.923347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316097+0000","last_change":"2024-03-04T07:30:47.996096+0000","last_active":"2024-03-04T07:30:50.316097+0000","last_peered":"2024-03-04T07:30:50.316097+0000","last_clean":"2024-03-04T07:30:50.316097+0000","last_became_active":"2024-03-04T07:30:47.995823+0000","last_became_peered":"2024-03-04T07:30:47.995823+0000","last_unstale":"2024-03-04T07:30:50.316097+0000","last_undegraded":"2024-03-04T07:30:50.316097+0000","last_fullsized":"2024-03-04T07:30:50.316097+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:01:19.526458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.9","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988364+0000","last_change":"2024-03-04T07:30:49.010132+0000","last_active":"2024-03-04T07:30:49.988364+0000","last_peered":"2024-03-04T07:30:49.988364+0000","last_clean":"2024-03-04T07:30:49.988364+0000","last_became_active":"2024-03-04T07:30:49.009703+0000","last_became_peered":"2024-03-04T07:30:49.009703+0000","last_unstale":"2024-03-04T07:30:49.988364+0000","last_undegraded":"2024-03-04T07:30:49.988364+0000","last_fullsized":"2024-03-04T07:30:49.988364+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:05:44.308067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315086+0000","last_change":"2024-03-04T07:30:47.993917+0000","last_active":"2024-03-04T07:30:50.315086+0000","last_peered":"2024-03-04T07:30:50.315086+0000","last_clean":"2024-03-04T07:30:50.315086+0000","last_became_active":"2024-03-04T07:30:47.993631+0000","last_became_peered":"2024-03-04T07:30:47.993631+0000","last_unstale":"2024-03-04T07:30:50.315086+0000","last_undegraded":"2024-03-04T07:30:50.315086+0000","last_fullsized":"2024-03-04T07:30:50.315086+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:44.471982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988824+0000","last_change":"2024-03-04T07:30:49.009786+0000","last_active":"2024-03-04T07:30:49.988824+0000","last_peered":"2024-03-04T07:30:49.988824+0000","last_clean":"2024-03-04T07:30:49.988824+0000","last_became_active":"2024-03-04T07:30:49.009358+0000","last_became_peered":"2024-03-04T07:30:49.009358+0000","last_unstale":"2024-03-04T07:30:49.988824+0000","last_undegraded":"2024-03-04T07:30:49.988824+0000","last_fullsized":"2024-03-04T07:30:49.988824+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:45.473804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984490+0000","last_change":"2024-03-04T07:30:47.992251+0000","last_active":"2024-03-04T07:30:49.984490+0000","last_peered":"2024-03-04T07:30:49.984490+0000","last_clean":"2024-03-04T07:30:49.984490+0000","last_became_active":"2024-03-04T07:30:47.991991+0000","last_became_peered":"2024-03-04T07:30:47.991991+0000","last_unstale":"2024-03-04T07:30:49.984490+0000","last_undegraded":"2024-03-04T07:30:49.984490+0000","last_fullsized":"2024-03-04T07:30:49.984490+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:50.331686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.7","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316455+0000","last_change":"2024-03-04T07:30:49.003511+0000","last_active":"2024-03-04T07:30:50.316455+0000","last_peered":"2024-03-04T07:30:50.316455+0000","last_clean":"2024-03-04T07:30:50.316455+0000","last_became_active":"2024-03-04T07:30:49.003163+0000","last_became_peered":"2024-03-04T07:30:49.003163+0000","last_unstale":"2024-03-04T07:30:50.316455+0000","last_undegraded":"2024-03-04T07:30:50.316455+0000","last_fullsized":"2024-03-04T07:30:50.316455+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:11:56.451211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.6","version":"79'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.987490+0000","last_change":"2024-03-04T07:30:47.991297+0000","last_active":"2024-03-04T07:30:50.987490+0000","last_peered":"2024-03-04T07:30:50.987490+0000","last_clean":"2024-03-04T07:30:50.987490+0000","last_became_active":"2024-03-04T07:30:47.990997+0000","last_became_peered":"2024-03-04T07:30:47.990997+0000","last_unstale":"2024-03-04T07:30:50.987490+0000","last_undegraded":"2024-03-04T07:30:50.987490+0000","last_fullsized":"2024-03-04T07:30:50.987490+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:04.427400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.4","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315899+0000","last_change":"2024-03-04T07:30:49.006306+0000","last_active":"2024-03-04T07:30:50.315899+0000","last_peered":"2024-03-04T07:30:50.315899+0000","last_clean":"2024-03-04T07:30:50.315899+0000","last_became_active":"2024-03-04T07:30:49.005136+0000","last_became_peered":"2024-03-04T07:30:49.005136+0000","last_unstale":"2024-03-04T07:30:50.315899+0000","last_undegraded":"2024-03-04T07:30:50.315899+0000","last_fullsized":"2024-03-04T07:30:50.315899+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:15:55.458985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315684+0000","last_change":"2024-03-04T07:30:47.996074+0000","last_active":"2024-03-04T07:30:50.315684+0000","last_peered":"2024-03-04T07:30:50.315684+0000","last_clean":"2024-03-04T07:30:50.315684+0000","last_became_active":"2024-03-04T07:30:47.995749+0000","last_became_peered":"2024-03-04T07:30:47.995749+0000","last_unstale":"2024-03-04T07:30:50.315684+0000","last_undegraded":"2024-03-04T07:30:50.315684+0000","last_fullsized":"2024-03-04T07:30:50.315684+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:43:28.639676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988415+0000","last_change":"2024-03-04T07:30:49.013080+0000","last_active":"2024-03-04T07:30:49.988415+0000","last_peered":"2024-03-04T07:30:49.988415+0000","last_clean":"2024-03-04T07:30:49.988415+0000","last_became_active":"2024-03-04T07:30:49.012334+0000","last_became_peered":"2024-03-04T07:30:49.012334+0000","last_unstale":"2024-03-04T07:30:49.988415+0000","last_undegraded":"2024-03-04T07:30:49.988415+0000","last_fullsized":"2024-03-04T07:30:49.988415+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:12.423638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316018+0000","last_change":"2024-03-04T07:30:47.993482+0000","last_active":"2024-03-04T07:30:50.316018+0000","last_peered":"2024-03-04T07:30:50.316018+0000","last_clean":"2024-03-04T07:30:50.316018+0000","last_became_active":"2024-03-04T07:30:47.993165+0000","last_became_peered":"2024-03-04T07:30:47.993165+0000","last_unstale":"2024-03-04T07:30:50.316018+0000","last_undegraded":"2024-03-04T07:30:50.316018+0000","last_fullsized":"2024-03-04T07:30:50.316018+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:46.683475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.989120+0000","last_change":"2024-03-04T07:30:48.998687+0000","last_active":"2024-03-04T07:30:49.989120+0000","last_peered":"2024-03-04T07:30:49.989120+0000","last_clean":"2024-03-04T07:30:49.989120+0000","last_became_active":"2024-03-04T07:30:48.998380+0000","last_became_peered":"2024-03-04T07:30:48.998380+0000","last_unstale":"2024-03-04T07:30:49.989120+0000","last_undegraded":"2024-03-04T07:30:49.989120+0000","last_fullsized":"2024-03-04T07:30:49.989120+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:01:35.965763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316045+0000","last_change":"2024-03-04T07:30:47.994305+0000","last_active":"2024-03-04T07:30:50.316045+0000","last_peered":"2024-03-04T07:30:50.316045+0000","last_clean":"2024-03-04T07:30:50.316045+0000","last_became_active":"2024-03-04T07:30:47.994057+0000","last_became_peered":"2024-03-04T07:30:47.994057+0000","last_unstale":"2024-03-04T07:30:50.316045+0000","last_undegraded":"2024-03-04T07:30:50.316045+0000","last_fullsized":"2024-03-04T07:30:50.316045+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:13:52.850140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.3","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315441+0000","last_change":"2024-03-04T07:30:49.008570+0000","last_active":"2024-03-04T07:30:50.315441+0000","last_peered":"2024-03-04T07:30:50.315441+0000","last_clean":"2024-03-04T07:30:50.315441+0000","last_became_active":"2024-03-04T07:30:49.008294+0000","last_became_peered":"2024-03-04T07:30:49.008294+0000","last_unstale":"2024-03-04T07:30:50.315441+0000","last_undegraded":"2024-03-04T07:30:50.315441+0000","last_fullsized":"2024-03-04T07:30:50.315441+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:24.510630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988153+0000","last_change":"2024-03-04T07:30:47.999342+0000","last_active":"2024-03-04T07:30:49.988153+0000","last_peered":"2024-03-04T07:30:49.988153+0000","last_clean":"2024-03-04T07:30:49.988153+0000","last_became_active":"2024-03-04T07:30:47.997726+0000","last_became_peered":"2024-03-04T07:30:47.997726+0000","last_unstale":"2024-03-04T07:30:49.988153+0000","last_undegraded":"2024-03-04T07:30:49.988153+0000","last_fullsized":"2024-03-04T07:30:49.988153+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:46.683475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.0","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315802+0000","last_change":"2024-03-04T07:30:49.002119+0000","last_active":"2024-03-04T07:30:50.315802+0000","last_peered":"2024-03-04T07:30:50.315802+0000","last_clean":"2024-03-04T07:30:50.315802+0000","last_became_active":"2024-03-04T07:30:49.001807+0000","last_became_peered":"2024-03-04T07:30:49.001807+0000","last_unstale":"2024-03-04T07:30:50.315802+0000","last_undegraded":"2024-03-04T07:30:50.315802+0000","last_fullsized":"2024-03-04T07:30:50.315802+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:15:55.458985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315637+0000","last_change":"2024-03-04T07:30:47.992253+0000","last_active":"2024-03-04T07:30:50.315637+0000","last_peered":"2024-03-04T07:30:50.315637+0000","last_clean":"2024-03-04T07:30:50.315637+0000","last_became_active":"2024-03-04T07:30:47.991944+0000","last_became_peered":"2024-03-04T07:30:47.991944+0000","last_unstale":"2024-03-04T07:30:50.315637+0000","last_undegraded":"2024-03-04T07:30:50.315637+0000","last_fullsized":"2024-03-04T07:30:50.315637+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:25:02.651448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315482+0000","last_change":"2024-03-04T07:30:48.996455+0000","last_active":"2024-03-04T07:30:50.315482+0000","last_peered":"2024-03-04T07:30:50.315482+0000","last_clean":"2024-03-04T07:30:50.315482+0000","last_became_active":"2024-03-04T07:30:48.996124+0000","last_became_peered":"2024-03-04T07:30:48.996124+0000","last_unstale":"2024-03-04T07:30:50.315482+0000","last_undegraded":"2024-03-04T07:30:50.315482+0000","last_fullsized":"2024-03-04T07:30:50.315482+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:02.000572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988042+0000","last_change":"2024-03-04T07:30:47.999754+0000","last_active":"2024-03-04T07:30:49.988042+0000","last_peered":"2024-03-04T07:30:49.988042+0000","last_clean":"2024-03-04T07:30:49.988042+0000","last_became_active":"2024-03-04T07:30:47.999328+0000","last_became_peered":"2024-03-04T07:30:47.999328+0000","last_unstale":"2024-03-04T07:30:49.988042+0000","last_undegraded":"2024-03-04T07:30:49.988042+0000","last_fullsized":"2024-03-04T07:30:49.988042+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:10.910681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984276+0000","last_change":"2024-03-04T07:30:49.000154+0000","last_active":"2024-03-04T07:30:49.984276+0000","last_peered":"2024-03-04T07:30:49.984276+0000","last_clean":"2024-03-04T07:30:49.984276+0000","last_became_active":"2024-03-04T07:30:48.999228+0000","last_became_peered":"2024-03-04T07:30:48.999228+0000","last_unstale":"2024-03-04T07:30:49.984276+0000","last_undegraded":"2024-03-04T07:30:49.984276+0000","last_fullsized":"2024-03-04T07:30:49.984276+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:43:29.641498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316411+0000","last_change":"2024-03-04T07:30:47.995321+0000","last_active":"2024-03-04T07:30:50.316411+0000","last_peered":"2024-03-04T07:30:50.316411+0000","last_clean":"2024-03-04T07:30:50.316411+0000","last_became_active":"2024-03-04T07:30:47.994967+0000","last_became_peered":"2024-03-04T07:30:47.994967+0000","last_unstale":"2024-03-04T07:30:50.316411+0000","last_undegraded":"2024-03-04T07:30:50.316411+0000","last_fullsized":"2024-03-04T07:30:50.316411+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:46:38.223887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":277,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315652+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:30:50.315652+0000","last_peered":"2024-03-04T07:30:50.315652+0000","last_clean":"2024-03-04T07:30:50.315652+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:30:50.315652+0000","last_undegraded":"2024-03-04T07:30:50.315652+0000","last_fullsized":"2024-03-04T07:30:50.315652+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574903,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32704,"kb_used_data":1520,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339136,"statfs":{"total":96636764160,"available":96603275264,"internally_reserved":0,"allocated":1556480,"data_stored":995688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574902,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32132,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339708,"statfs":{"total":96636764160,"available":96603860992,"internally_reserved":0,"allocated":970752,"data_stored":405944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574902,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32152,"kb_used_data":964,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339688,"statfs":{"total":96636764160,"available":96603840512,"internally_reserved":0,"allocated":987136,"data_stored":413184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574903,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32132,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339708,"statfs":{"total":96636764160,"available":96603860992,"internally_reserved":0,"allocated":970752,"data_stored":411550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574902,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32148,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339692,"statfs":{"total":96636764160,"available":96603844608,"internally_reserved":0,"allocated":983040,"data_stored":413162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574902,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32140,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339700,"statfs":{"total":96636764160,"available":96603852800,"internally_reserved":0,"allocated":974848,"data_stored":412132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574902,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32148,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339692,"statfs":{"total":96636764160,"available":96603844608,"internally_reserved":0,"allocated":983040,"data_stored":411744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574902,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32712,"kb_used_data":1532,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339128,"statfs":{"total":96636764160,"available":96603267072,"internally_reserved":0,"allocated":1568768,"data_stored":1001952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:30:52.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:30:52.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.145+0000 7f892e9b5640 1 Processor -- start 2024-03-04T07:30:52.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.145+0000 7f892e9b5640 1 -- start start 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.146+0000 7f892e9b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8928071540 0x7f89280b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.146+0000 7f892e9b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f89280d3400 0x7f8928071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.146+0000 7f892e9b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8928071e60 con 0x7f89280b2b50 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.146+0000 7f892e9b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f89280d2f70 con 0x7f8928071540 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.146+0000 7f892e9b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f89280d4be0 con 0x7f89280d3400 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f8927fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8928071540 0x7f89280b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f892cf52640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f89280b2b50 0x7f89280d1870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48114/0 (socket says 172.21.15.64:48114) 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f892cf52640 1 -- 172.21.15.64:0/957915138 learned_addr learned my addr 172.21.15.64:0/957915138 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f89277fe640 1 --2- 172.21.15.64:0/957915138 >> v2:172.21.15.66:3300/0 conn(0x7f89280d3400 0x7f8928071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f8927fff640 1 -- 172.21.15.64:0/957915138 >> v2:172.21.15.66:3300/0 conn(0x7f89280d3400 msgr2=0x7f8928071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:52.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f8927fff640 1 --2- 172.21.15.64:0/957915138 >> v2:172.21.15.66:3300/0 conn(0x7f89280d3400 0x7f8928071920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f8927fff640 1 -- 172.21.15.64:0/957915138 >> v1:172.21.15.64:6789/0 conn(0x7f89280b2b50 legacy=0x7f89280d1870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f8926ffd640 1 -- 172.21.15.64:0/957915138 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2345590708 0 0) 0x7f8928071e60 con 0x7f89280b2b50 2024-03-04T07:30:52.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.147+0000 7f8927fff640 1 -- 172.21.15.64:0/957915138 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89280ce610 con 0x7f8928071540 2024-03-04T07:30:52.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.148+0000 7f8927fff640 1 --2- 172.21.15.64:0/957915138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f89280b30a0 secure :-1 s=READY pgs=996 cs=0 l=1 rev1=1 crypto rx=0x7f89280b6860 tx=0x7f891802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=62ee9ca4b323a62 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.148+0000 7f8926ffd640 1 -- 172.21.15.64:0/957915138 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8918035020 con 0x7f8928071540 2024-03-04T07:30:52.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.148+0000 7f8926ffd640 1 -- 172.21.15.64:0/957915138 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8918031830 con 0x7f8928071540 2024-03-04T07:30:52.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.148+0000 7f8926ffd640 1 -- 172.21.15.64:0/957915138 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8918031c00 con 0x7f8928071540 2024-03-04T07:30:52.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 -- 172.21.15.64:0/957915138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 msgr2=0x7f89280b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 --2- 172.21.15.64:0/957915138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f89280b30a0 secure :-1 s=READY pgs=996 cs=0 l=1 rev1=1 crypto rx=0x7f89280b6860 tx=0x7f891802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 -- 172.21.15.64:0/957915138 shutdown_connections 2024-03-04T07:30:52.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 --2- 172.21.15.64:0/957915138 >> v2:172.21.15.66:3300/0 conn(0x7f89280d3400 0x7f8928071920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 --2- 172.21.15.64:0/957915138 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f89280b30a0 unknown :-1 s=CLOSED pgs=996 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 -- 172.21.15.64:0/957915138 >> 172.21.15.64:0/957915138 conn(0x7f89280d3ca0 msgr2=0x7f89280d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:52.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 -- 172.21.15.64:0/957915138 shutdown_connections 2024-03-04T07:30:52.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.149+0000 7f892e9b5640 1 -- 172.21.15.64:0/957915138 wait complete. 2024-03-04T07:30:52.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.150+0000 7f892e9b5640 1 Processor -- start 2024-03-04T07:30:52.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.150+0000 7f892e9b5640 1 -- start start 2024-03-04T07:30:52.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892e9b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f8928160990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892e9b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89280b2b50 0x7f89280ad410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892e9b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89280d3400 0x7f89280ad950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892e9b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89280ce8f0 con 0x7f8928071540 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892e9b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89280d2c00 con 0x7f89280b2b50 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892e9b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f892815e370 con 0x7f89280d3400 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f8928160990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f8928160990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42370/0 (socket says 172.21.15.64:42370) 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 -- 172.21.15.64:0/321929034 learned_addr learned my addr 172.21.15.64:0/321929034 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892cf52640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89280d3400 0x7f89280ad950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f89277fe640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89280b2b50 0x7f89280ad410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 -- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89280d3400 msgr2=0x7f89280ad950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89280d3400 0x7f89280ad950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 -- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89280b2b50 msgr2=0x7f89280ad410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89280b2b50 0x7f89280ad410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f8927fff640 1 -- 172.21.15.64:0/321929034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89281f6d90 con 0x7f8928071540 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f892cf52640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89280d3400 0x7f89280ad950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.151+0000 7f89277fe640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89280b2b50 0x7f89280ad410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.152+0000 7f8927fff640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f8928160990 secure :-1 s=READY pgs=997 cs=0 l=1 rev1=1 crypto rx=0x7f8918009890 tx=0x7f89180046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.152+0000 7f8924ff9640 1 -- 172.21.15.64:0/321929034 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8918035070 con 0x7f8928071540 2024-03-04T07:30:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.152+0000 7f8924ff9640 1 -- 172.21.15.64:0/321929034 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8918004250 con 0x7f8928071540 2024-03-04T07:30:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.152+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f89281f7050 con 0x7f8928071540 2024-03-04T07:30:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.152+0000 7f8924ff9640 1 -- 172.21.15.64:0/321929034 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f891803a6f0 con 0x7f8928071540 2024-03-04T07:30:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.152+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89281f7610 con 0x7f8928071540 2024-03-04T07:30:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.153+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8928068700 con 0x7f8928071540 2024-03-04T07:30:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.153+0000 7f8924ff9640 1 -- 172.21.15.64:0/321929034 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8918038020 con 0x7f8928071540 2024-03-04T07:30:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.154+0000 7f8924ff9640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891406b4d0 0x7f891406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.154+0000 7f8924ff9640 1 -- 172.21.15.64:0/321929034 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f89180abf60 con 0x7f8928071540 2024-03-04T07:30:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.154+0000 7f89277fe640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891406b4d0 0x7f891406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.154+0000 7f89277fe640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891406b4d0 0x7f891406d990 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f891c006fd0 tx=0x7f891c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.158+0000 7f8924ff9640 1 -- 172.21.15.64:0/321929034 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8918040070 con 0x7f8928071540 2024-03-04T07:30:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.336+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f89280ade90 con 0x7f891406b4d0 2024-03-04T07:30:52.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.342+0000 7f8924ff9640 1 -- 172.21.15.64:0/321929034 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252073 (secure 0 0 0) 0x7f89280ade90 con 0x7f891406b4d0 2024-03-04T07:30:52.344 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:52.345 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:30:52.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891406b4d0 msgr2=0x7f891406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891406b4d0 0x7f891406d990 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f891c006fd0 tx=0x7f891c008040 comp rx=0 tx=0).stop 2024-03-04T07:30:52.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 msgr2=0x7f8928160990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f8928160990 secure :-1 s=READY pgs=997 cs=0 l=1 rev1=1 crypto rx=0x7f8918009890 tx=0x7f89180046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 shutdown_connections 2024-03-04T07:30:52.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891406b4d0 0x7f891406d990 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89280d3400 0x7f89280ad950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89280b2b50 0x7f89280ad410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 --2- 172.21.15.64:0/321929034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8928071540 0x7f8928160990 unknown :-1 s=CLOSED pgs=997 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 >> 172.21.15.64:0/321929034 conn(0x7f89280d3ca0 msgr2=0x7f89280d2010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:52.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 shutdown_connections 2024-03-04T07:30:52.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.345+0000 7f892e9b5640 1 -- 172.21.15.64:0/321929034 wait complete. 2024-03-04T07:30:52.356 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-03-04T07:30:51.184364+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":258268,"kb_used_data":8784,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754716452,"statfs":{"total":773094113280,"available":772829646848,"internally_reserved":0,"allocated":8994816,"data_stored":4465356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":23,"apply_latency_ms":23,"commit_latency_ns":23000000,"apply_latency_ns":23000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.209535"},"pg_stats":[{"pgid":"13.3f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316081+0000","last_change":"2024-03-04T07:30:49.012242+0000","last_active":"2024-03-04T07:30:50.316081+0000","last_peered":"2024-03-04T07:30:50.316081+0000","last_clean":"2024-03-04T07:30:50.316081+0000","last_became_active":"2024-03-04T07:30:49.011919+0000","last_became_peered":"2024-03-04T07:30:49.011919+0000","last_unstale":"2024-03-04T07:30:50.316081+0000","last_undegraded":"2024-03-04T07:30:50.316081+0000","last_fullsized":"2024-03-04T07:30:50.316081+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:52:53.722382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984526+0000","last_change":"2024-03-04T07:30:49.004230+0000","last_active":"2024-03-04T07:30:49.984526+0000","last_peered":"2024-03-04T07:30:49.984526+0000","last_clean":"2024-03-04T07:30:49.984526+0000","last_became_active":"2024-03-04T07:30:49.003961+0000","last_became_peered":"2024-03-04T07:30:49.003961+0000","last_unstale":"2024-03-04T07:30:49.984526+0000","last_undegraded":"2024-03-04T07:30:49.984526+0000","last_fullsized":"2024-03-04T07:30:49.984526+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:11.912503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.3d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316632+0000","last_change":"2024-03-04T07:30:48.999604+0000","last_active":"2024-03-04T07:30:50.316632+0000","last_peered":"2024-03-04T07:30:50.316632+0000","last_clean":"2024-03-04T07:30:50.316632+0000","last_became_active":"2024-03-04T07:30:48.999141+0000","last_became_peered":"2024-03-04T07:30:48.999141+0000","last_unstale":"2024-03-04T07:30:50.316632+0000","last_undegraded":"2024-03-04T07:30:50.316632+0000","last_fullsized":"2024-03-04T07:30:50.316632+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:18:33.481737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.3c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315593+0000","last_change":"2024-03-04T07:30:49.000233+0000","last_active":"2024-03-04T07:30:50.315593+0000","last_peered":"2024-03-04T07:30:50.315593+0000","last_clean":"2024-03-04T07:30:50.315593+0000","last_became_active":"2024-03-04T07:30:48.999972+0000","last_became_peered":"2024-03-04T07:30:48.999972+0000","last_unstale":"2024-03-04T07:30:50.315593+0000","last_undegraded":"2024-03-04T07:30:50.315593+0000","last_fullsized":"2024-03-04T07:30:50.315593+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:45:38.423092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.3b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988092+0000","last_change":"2024-03-04T07:30:49.015413+0000","last_active":"2024-03-04T07:30:49.988092+0000","last_peered":"2024-03-04T07:30:49.988092+0000","last_clean":"2024-03-04T07:30:49.988092+0000","last_became_active":"2024-03-04T07:30:49.015065+0000","last_became_peered":"2024-03-04T07:30:49.015065+0000","last_unstale":"2024-03-04T07:30:49.988092+0000","last_undegraded":"2024-03-04T07:30:49.988092+0000","last_fullsized":"2024-03-04T07:30:49.988092+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:02.000572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.989044+0000","last_change":"2024-03-04T07:30:49.006999+0000","last_active":"2024-03-04T07:30:49.989044+0000","last_peered":"2024-03-04T07:30:49.989044+0000","last_clean":"2024-03-04T07:30:49.989044+0000","last_became_active":"2024-03-04T07:30:49.006684+0000","last_became_peered":"2024-03-04T07:30:49.006684+0000","last_unstale":"2024-03-04T07:30:49.989044+0000","last_undegraded":"2024-03-04T07:30:49.989044+0000","last_fullsized":"2024-03-04T07:30:49.989044+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:53:34.835694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.39","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316013+0000","last_change":"2024-03-04T07:30:48.995479+0000","last_active":"2024-03-04T07:30:50.316013+0000","last_peered":"2024-03-04T07:30:50.316013+0000","last_clean":"2024-03-04T07:30:50.316013+0000","last_became_active":"2024-03-04T07:30:48.995215+0000","last_became_peered":"2024-03-04T07:30:48.995215+0000","last_unstale":"2024-03-04T07:30:50.316013+0000","last_undegraded":"2024-03-04T07:30:50.316013+0000","last_fullsized":"2024-03-04T07:30:50.316013+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:04:55.380489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.38","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.987883+0000","last_change":"2024-03-04T07:30:49.006280+0000","last_active":"2024-03-04T07:30:49.987883+0000","last_peered":"2024-03-04T07:30:49.987883+0000","last_clean":"2024-03-04T07:30:49.987883+0000","last_became_active":"2024-03-04T07:30:49.005363+0000","last_became_peered":"2024-03-04T07:30:49.005363+0000","last_unstale":"2024-03-04T07:30:49.987883+0000","last_undegraded":"2024-03-04T07:30:49.987883+0000","last_fullsized":"2024-03-04T07:30:49.987883+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:44:35.866927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.37","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988399+0000","last_change":"2024-03-04T07:30:49.009418+0000","last_active":"2024-03-04T07:30:49.988399+0000","last_peered":"2024-03-04T07:30:49.988399+0000","last_clean":"2024-03-04T07:30:49.988399+0000","last_became_active":"2024-03-04T07:30:49.009066+0000","last_became_peered":"2024-03-04T07:30:49.009066+0000","last_unstale":"2024-03-04T07:30:49.988399+0000","last_undegraded":"2024-03-04T07:30:49.988399+0000","last_fullsized":"2024-03-04T07:30:49.988399+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:46:39.225709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.36","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316107+0000","last_change":"2024-03-04T07:30:48.997816+0000","last_active":"2024-03-04T07:30:50.316107+0000","last_peered":"2024-03-04T07:30:50.316107+0000","last_clean":"2024-03-04T07:30:50.316107+0000","last_became_active":"2024-03-04T07:30:48.997506+0000","last_became_peered":"2024-03-04T07:30:48.997506+0000","last_unstale":"2024-03-04T07:30:50.316107+0000","last_undegraded":"2024-03-04T07:30:50.316107+0000","last_fullsized":"2024-03-04T07:30:50.316107+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:40.772647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.35","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984460+0000","last_change":"2024-03-04T07:30:49.008007+0000","last_active":"2024-03-04T07:30:49.984460+0000","last_peered":"2024-03-04T07:30:49.984460+0000","last_clean":"2024-03-04T07:30:49.984460+0000","last_became_active":"2024-03-04T07:30:49.007700+0000","last_became_peered":"2024-03-04T07:30:49.007700+0000","last_unstale":"2024-03-04T07:30:49.984460+0000","last_undegraded":"2024-03-04T07:30:49.984460+0000","last_fullsized":"2024-03-04T07:30:49.984460+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:32.645561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.34","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315724+0000","last_change":"2024-03-04T07:30:49.005039+0000","last_active":"2024-03-04T07:30:50.315724+0000","last_peered":"2024-03-04T07:30:50.315724+0000","last_clean":"2024-03-04T07:30:50.315724+0000","last_became_active":"2024-03-04T07:30:49.004747+0000","last_became_peered":"2024-03-04T07:30:49.004747+0000","last_unstale":"2024-03-04T07:30:50.315724+0000","last_undegraded":"2024-03-04T07:30:50.315724+0000","last_fullsized":"2024-03-04T07:30:50.315724+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:12.423638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.33","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316029+0000","last_change":"2024-03-04T07:30:49.007907+0000","last_active":"2024-03-04T07:30:50.316029+0000","last_peered":"2024-03-04T07:30:50.316029+0000","last_clean":"2024-03-04T07:30:50.316029+0000","last_became_active":"2024-03-04T07:30:49.007475+0000","last_became_peered":"2024-03-04T07:30:49.007475+0000","last_unstale":"2024-03-04T07:30:50.316029+0000","last_undegraded":"2024-03-04T07:30:50.316029+0000","last_fullsized":"2024-03-04T07:30:50.316029+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:52:53.722382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.32","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316426+0000","last_change":"2024-03-04T07:30:49.007138+0000","last_active":"2024-03-04T07:30:50.316426+0000","last_peered":"2024-03-04T07:30:50.316426+0000","last_clean":"2024-03-04T07:30:50.316426+0000","last_became_active":"2024-03-04T07:30:49.006528+0000","last_became_peered":"2024-03-04T07:30:49.006528+0000","last_unstale":"2024-03-04T07:30:50.316426+0000","last_undegraded":"2024-03-04T07:30:50.316426+0000","last_fullsized":"2024-03-04T07:30:50.316426+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:00:48.332069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.31","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315900+0000","last_change":"2024-03-04T07:30:49.005145+0000","last_active":"2024-03-04T07:30:50.315900+0000","last_peered":"2024-03-04T07:30:50.315900+0000","last_clean":"2024-03-04T07:30:50.315900+0000","last_became_active":"2024-03-04T07:30:49.004867+0000","last_became_peered":"2024-03-04T07:30:49.004867+0000","last_unstale":"2024-03-04T07:30:50.315900+0000","last_undegraded":"2024-03-04T07:30:50.315900+0000","last_fullsized":"2024-03-04T07:30:50.315900+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:18:26.851147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.30","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.987858+0000","last_change":"2024-03-04T07:30:49.011262+0000","last_active":"2024-03-04T07:30:49.987858+0000","last_peered":"2024-03-04T07:30:49.987858+0000","last_clean":"2024-03-04T07:30:49.987858+0000","last_became_active":"2024-03-04T07:30:49.010939+0000","last_became_peered":"2024-03-04T07:30:49.010939+0000","last_unstale":"2024-03-04T07:30:49.987858+0000","last_undegraded":"2024-03-04T07:30:49.987858+0000","last_fullsized":"2024-03-04T07:30:49.987858+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:01:53.739843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988477+0000","last_change":"2024-03-04T07:30:49.007921+0000","last_active":"2024-03-04T07:30:49.988477+0000","last_peered":"2024-03-04T07:30:49.988477+0000","last_clean":"2024-03-04T07:30:49.988477+0000","last_became_active":"2024-03-04T07:30:49.007626+0000","last_became_peered":"2024-03-04T07:30:49.007626+0000","last_unstale":"2024-03-04T07:30:49.988477+0000","last_undegraded":"2024-03-04T07:30:49.988477+0000","last_fullsized":"2024-03-04T07:30:49.988477+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:12:42.814855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.2e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315719+0000","last_change":"2024-03-04T07:30:48.997786+0000","last_active":"2024-03-04T07:30:50.315719+0000","last_peered":"2024-03-04T07:30:50.315719+0000","last_clean":"2024-03-04T07:30:50.315719+0000","last_became_active":"2024-03-04T07:30:48.997337+0000","last_became_peered":"2024-03-04T07:30:48.997337+0000","last_unstale":"2024-03-04T07:30:50.315719+0000","last_undegraded":"2024-03-04T07:30:50.315719+0000","last_fullsized":"2024-03-04T07:30:50.315719+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:10:15.795433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316295+0000","last_change":"2024-03-04T07:30:48.989857+0000","last_active":"2024-03-04T07:30:50.316295+0000","last_peered":"2024-03-04T07:30:50.316295+0000","last_clean":"2024-03-04T07:30:50.316295+0000","last_became_active":"2024-03-04T07:30:48.989566+0000","last_became_peered":"2024-03-04T07:30:48.989566+0000","last_unstale":"2024-03-04T07:30:50.316295+0000","last_undegraded":"2024-03-04T07:30:50.316295+0000","last_fullsized":"2024-03-04T07:30:50.316295+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:02.000572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.2c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315796+0000","last_change":"2024-03-04T07:30:48.992725+0000","last_active":"2024-03-04T07:30:50.315796+0000","last_peered":"2024-03-04T07:30:50.315796+0000","last_clean":"2024-03-04T07:30:50.315796+0000","last_became_active":"2024-03-04T07:30:48.992440+0000","last_became_peered":"2024-03-04T07:30:48.992440+0000","last_unstale":"2024-03-04T07:30:50.315796+0000","last_undegraded":"2024-03-04T07:30:50.315796+0000","last_fullsized":"2024-03-04T07:30:50.315796+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:34:35.192750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315377+0000","last_change":"2024-03-04T07:30:48.992658+0000","last_active":"2024-03-04T07:30:50.315377+0000","last_peered":"2024-03-04T07:30:50.315377+0000","last_clean":"2024-03-04T07:30:50.315377+0000","last_became_active":"2024-03-04T07:30:48.992314+0000","last_became_peered":"2024-03-04T07:30:48.992314+0000","last_unstale":"2024-03-04T07:30:50.315377+0000","last_undegraded":"2024-03-04T07:30:50.315377+0000","last_fullsized":"2024-03-04T07:30:50.315377+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:39:38.583112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.2a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984188+0000","last_change":"2024-03-04T07:30:49.011212+0000","last_active":"2024-03-04T07:30:49.984188+0000","last_peered":"2024-03-04T07:30:49.984188+0000","last_clean":"2024-03-04T07:30:49.984188+0000","last_became_active":"2024-03-04T07:30:49.010957+0000","last_became_peered":"2024-03-04T07:30:49.010957+0000","last_unstale":"2024-03-04T07:30:49.984188+0000","last_undegraded":"2024-03-04T07:30:49.984188+0000","last_fullsized":"2024-03-04T07:30:49.984188+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:27:37.989524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.29","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315467+0000","last_change":"2024-03-04T07:30:49.008950+0000","last_active":"2024-03-04T07:30:50.315467+0000","last_peered":"2024-03-04T07:30:50.315467+0000","last_clean":"2024-03-04T07:30:50.315467+0000","last_became_active":"2024-03-04T07:30:49.008557+0000","last_became_peered":"2024-03-04T07:30:49.008557+0000","last_unstale":"2024-03-04T07:30:50.315467+0000","last_undegraded":"2024-03-04T07:30:50.315467+0000","last_fullsized":"2024-03-04T07:30:50.315467+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:24.510630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.28","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315436+0000","last_change":"2024-03-04T07:30:48.989590+0000","last_active":"2024-03-04T07:30:50.315436+0000","last_peered":"2024-03-04T07:30:50.315436+0000","last_clean":"2024-03-04T07:30:50.315436+0000","last_became_active":"2024-03-04T07:30:48.989279+0000","last_became_peered":"2024-03-04T07:30:48.989279+0000","last_unstale":"2024-03-04T07:30:50.315436+0000","last_undegraded":"2024-03-04T07:30:50.315436+0000","last_fullsized":"2024-03-04T07:30:50.315436+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:09:23.017980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.27","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984504+0000","last_change":"2024-03-04T07:30:49.001293+0000","last_active":"2024-03-04T07:30:49.984504+0000","last_peered":"2024-03-04T07:30:49.984504+0000","last_clean":"2024-03-04T07:30:49.984504+0000","last_became_active":"2024-03-04T07:30:49.000948+0000","last_became_peered":"2024-03-04T07:30:49.000948+0000","last_unstale":"2024-03-04T07:30:49.984504+0000","last_undegraded":"2024-03-04T07:30:49.984504+0000","last_fullsized":"2024-03-04T07:30:49.984504+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:47.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.26","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316325+0000","last_change":"2024-03-04T07:30:49.005947+0000","last_active":"2024-03-04T07:30:50.316325+0000","last_peered":"2024-03-04T07:30:50.316325+0000","last_clean":"2024-03-04T07:30:50.316325+0000","last_became_active":"2024-03-04T07:30:49.005565+0000","last_became_peered":"2024-03-04T07:30:49.005565+0000","last_unstale":"2024-03-04T07:30:50.316325+0000","last_undegraded":"2024-03-04T07:30:50.316325+0000","last_fullsized":"2024-03-04T07:30:50.316325+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:04:00.152366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.25","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988205+0000","last_change":"2024-03-04T07:30:49.007009+0000","last_active":"2024-03-04T07:30:49.988205+0000","last_peered":"2024-03-04T07:30:49.988205+0000","last_clean":"2024-03-04T07:30:49.988205+0000","last_became_active":"2024-03-04T07:30:49.006678+0000","last_became_peered":"2024-03-04T07:30:49.006678+0000","last_unstale":"2024-03-04T07:30:49.988205+0000","last_undegraded":"2024-03-04T07:30:49.988205+0000","last_fullsized":"2024-03-04T07:30:49.988205+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:24.510630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.24","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315563+0000","last_change":"2024-03-04T07:30:49.012546+0000","last_active":"2024-03-04T07:30:50.315563+0000","last_peered":"2024-03-04T07:30:50.315563+0000","last_clean":"2024-03-04T07:30:50.315563+0000","last_became_active":"2024-03-04T07:30:49.012205+0000","last_became_peered":"2024-03-04T07:30:49.012205+0000","last_unstale":"2024-03-04T07:30:50.315563+0000","last_undegraded":"2024-03-04T07:30:50.315563+0000","last_fullsized":"2024-03-04T07:30:50.315563+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:05.868581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.23","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984214+0000","last_change":"2024-03-04T07:30:48.999135+0000","last_active":"2024-03-04T07:30:49.984214+0000","last_peered":"2024-03-04T07:30:49.984214+0000","last_clean":"2024-03-04T07:30:49.984214+0000","last_became_active":"2024-03-04T07:30:48.998850+0000","last_became_peered":"2024-03-04T07:30:48.998850+0000","last_unstale":"2024-03-04T07:30:49.984214+0000","last_undegraded":"2024-03-04T07:30:49.984214+0000","last_fullsized":"2024-03-04T07:30:49.984214+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:41:54.404669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.22","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984258+0000","last_change":"2024-03-04T07:30:49.005880+0000","last_active":"2024-03-04T07:30:49.984258+0000","last_peered":"2024-03-04T07:30:49.984258+0000","last_clean":"2024-03-04T07:30:49.984258+0000","last_became_active":"2024-03-04T07:30:49.005535+0000","last_became_peered":"2024-03-04T07:30:49.005535+0000","last_unstale":"2024-03-04T07:30:49.984258+0000","last_undegraded":"2024-03-04T07:30:49.984258+0000","last_fullsized":"2024-03-04T07:30:49.984258+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:25:03.653270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.21","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.987893+0000","last_change":"2024-03-04T07:30:49.013378+0000","last_active":"2024-03-04T07:30:49.987893+0000","last_peered":"2024-03-04T07:30:49.987893+0000","last_clean":"2024-03-04T07:30:49.987893+0000","last_became_active":"2024-03-04T07:30:49.013049+0000","last_became_peered":"2024-03-04T07:30:49.013049+0000","last_unstale":"2024-03-04T07:30:49.987893+0000","last_undegraded":"2024-03-04T07:30:49.987893+0000","last_fullsized":"2024-03-04T07:30:49.987893+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:26.008072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.20","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315430+0000","last_change":"2024-03-04T07:30:49.004437+0000","last_active":"2024-03-04T07:30:50.315430+0000","last_peered":"2024-03-04T07:30:50.315430+0000","last_clean":"2024-03-04T07:30:50.315430+0000","last_became_active":"2024-03-04T07:30:49.003525+0000","last_became_peered":"2024-03-04T07:30:49.003525+0000","last_unstale":"2024-03-04T07:30:50.315430+0000","last_undegraded":"2024-03-04T07:30:50.315430+0000","last_fullsized":"2024-03-04T07:30:50.315430+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:26.008072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988517+0000","last_change":"2024-03-04T07:30:49.016311+0000","last_active":"2024-03-04T07:30:49.988517+0000","last_peered":"2024-03-04T07:30:49.988517+0000","last_clean":"2024-03-04T07:30:49.988517+0000","last_became_active":"2024-03-04T07:30:49.015977+0000","last_became_peered":"2024-03-04T07:30:49.015977+0000","last_unstale":"2024-03-04T07:30:49.988517+0000","last_undegraded":"2024-03-04T07:30:49.988517+0000","last_fullsized":"2024-03-04T07:30:49.988517+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:52:53.722382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1f","version":"78'4","reported_seq":18,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316403+0000","last_change":"2024-03-04T07:30:47.995691+0000","last_active":"2024-03-04T07:30:50.316403+0000","last_peered":"2024-03-04T07:30:50.316403+0000","last_clean":"2024-03-04T07:30:50.316403+0000","last_became_active":"2024-03-04T07:30:47.995375+0000","last_became_peered":"2024-03-04T07:30:47.995375+0000","last_unstale":"2024-03-04T07:30:50.316403+0000","last_undegraded":"2024-03-04T07:30:50.316403+0000","last_fullsized":"2024-03-04T07:30:50.316403+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:10.910681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316278+0000","last_change":"2024-03-04T07:30:49.009335+0000","last_active":"2024-03-04T07:30:50.316278+0000","last_peered":"2024-03-04T07:30:50.316278+0000","last_clean":"2024-03-04T07:30:50.316278+0000","last_became_active":"2024-03-04T07:30:49.008718+0000","last_became_peered":"2024-03-04T07:30:49.008718+0000","last_unstale":"2024-03-04T07:30:50.316278+0000","last_undegraded":"2024-03-04T07:30:50.316278+0000","last_fullsized":"2024-03-04T07:30:50.316278+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:45.473804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1e","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988761+0000","last_change":"2024-03-04T07:30:47.998228+0000","last_active":"2024-03-04T07:30:49.988761+0000","last_peered":"2024-03-04T07:30:49.988761+0000","last_clean":"2024-03-04T07:30:49.988761+0000","last_became_active":"2024-03-04T07:30:47.997896+0000","last_became_peered":"2024-03-04T07:30:47.997896+0000","last_unstale":"2024-03-04T07:30:49.988761+0000","last_undegraded":"2024-03-04T07:30:49.988761+0000","last_fullsized":"2024-03-04T07:30:49.988761+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:06.687014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315874+0000","last_change":"2024-03-04T07:30:49.011844+0000","last_active":"2024-03-04T07:30:50.315874+0000","last_peered":"2024-03-04T07:30:50.315874+0000","last_clean":"2024-03-04T07:30:50.315874+0000","last_became_active":"2024-03-04T07:30:49.011531+0000","last_became_peered":"2024-03-04T07:30:49.011531+0000","last_unstale":"2024-03-04T07:30:50.315874+0000","last_undegraded":"2024-03-04T07:30:50.315874+0000","last_fullsized":"2024-03-04T07:30:50.315874+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:12:42.814855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.1d","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316393+0000","last_change":"2024-03-04T07:30:47.994139+0000","last_active":"2024-03-04T07:30:50.316393+0000","last_peered":"2024-03-04T07:30:50.316393+0000","last_clean":"2024-03-04T07:30:50.316393+0000","last_became_active":"2024-03-04T07:30:47.993853+0000","last_became_peered":"2024-03-04T07:30:47.993853+0000","last_unstale":"2024-03-04T07:30:50.316393+0000","last_undegraded":"2024-03-04T07:30:50.316393+0000","last_fullsized":"2024-03-04T07:30:50.316393+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:05:43.306245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316304+0000","last_change":"2024-03-04T07:30:49.005155+0000","last_active":"2024-03-04T07:30:50.316304+0000","last_peered":"2024-03-04T07:30:50.316304+0000","last_clean":"2024-03-04T07:30:50.316304+0000","last_became_active":"2024-03-04T07:30:49.004851+0000","last_became_peered":"2024-03-04T07:30:49.004851+0000","last_unstale":"2024-03-04T07:30:50.316304+0000","last_undegraded":"2024-03-04T07:30:50.316304+0000","last_fullsized":"2024-03-04T07:30:50.316304+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:45.473804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1c","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316472+0000","last_change":"2024-03-04T07:30:47.994819+0000","last_active":"2024-03-04T07:30:50.316472+0000","last_peered":"2024-03-04T07:30:50.316472+0000","last_clean":"2024-03-04T07:30:50.316472+0000","last_became_active":"2024-03-04T07:30:47.994566+0000","last_became_peered":"2024-03-04T07:30:47.994566+0000","last_unstale":"2024-03-04T07:30:50.316472+0000","last_undegraded":"2024-03-04T07:30:50.316472+0000","last_fullsized":"2024-03-04T07:30:50.316472+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:44:34.865105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.1a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984335+0000","last_change":"2024-03-04T07:30:49.005803+0000","last_active":"2024-03-04T07:30:49.984335+0000","last_peered":"2024-03-04T07:30:49.984335+0000","last_clean":"2024-03-04T07:30:49.984335+0000","last_became_active":"2024-03-04T07:30:49.005159+0000","last_became_peered":"2024-03-04T07:30:49.005159+0000","last_unstale":"2024-03-04T07:30:49.984335+0000","last_undegraded":"2024-03-04T07:30:49.984335+0000","last_fullsized":"2024-03-04T07:30:49.984335+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:21.304097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.1b","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988672+0000","last_change":"2024-03-04T07:30:47.996445+0000","last_active":"2024-03-04T07:30:49.988672+0000","last_peered":"2024-03-04T07:30:49.988672+0000","last_clean":"2024-03-04T07:30:49.988672+0000","last_became_active":"2024-03-04T07:30:47.996085+0000","last_became_peered":"2024-03-04T07:30:47.996085+0000","last_unstale":"2024-03-04T07:30:49.988672+0000","last_undegraded":"2024-03-04T07:30:49.988672+0000","last_fullsized":"2024-03-04T07:30:49.988672+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:10:05.632096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988487+0000","last_change":"2024-03-04T07:30:49.012711+0000","last_active":"2024-03-04T07:30:49.988487+0000","last_peered":"2024-03-04T07:30:49.988487+0000","last_clean":"2024-03-04T07:30:49.988487+0000","last_became_active":"2024-03-04T07:30:49.012405+0000","last_became_peered":"2024-03-04T07:30:49.012405+0000","last_unstale":"2024-03-04T07:30:49.988487+0000","last_undegraded":"2024-03-04T07:30:49.988487+0000","last_fullsized":"2024-03-04T07:30:49.988487+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:15:55.458985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1a","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988037+0000","last_change":"2024-03-04T07:30:47.998690+0000","last_active":"2024-03-04T07:30:49.988037+0000","last_peered":"2024-03-04T07:30:49.988037+0000","last_clean":"2024-03-04T07:30:49.988037+0000","last_became_active":"2024-03-04T07:30:47.998339+0000","last_became_peered":"2024-03-04T07:30:47.998339+0000","last_unstale":"2024-03-04T07:30:49.988037+0000","last_undegraded":"2024-03-04T07:30:49.988037+0000","last_fullsized":"2024-03-04T07:30:49.988037+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:31.643739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.18","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315586+0000","last_change":"2024-03-04T07:30:49.009085+0000","last_active":"2024-03-04T07:30:50.315586+0000","last_peered":"2024-03-04T07:30:50.315586+0000","last_clean":"2024-03-04T07:30:50.315586+0000","last_became_active":"2024-03-04T07:30:49.008620+0000","last_became_peered":"2024-03-04T07:30:49.008620+0000","last_unstale":"2024-03-04T07:30:50.315586+0000","last_undegraded":"2024-03-04T07:30:50.315586+0000","last_fullsized":"2024-03-04T07:30:50.315586+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:12:42.814855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315763+0000","last_change":"2024-03-04T07:30:48.000338+0000","last_active":"2024-03-04T07:30:50.315763+0000","last_peered":"2024-03-04T07:30:50.315763+0000","last_clean":"2024-03-04T07:30:50.315763+0000","last_became_active":"2024-03-04T07:30:48.000044+0000","last_became_peered":"2024-03-04T07:30:48.000044+0000","last_unstale":"2024-03-04T07:30:50.315763+0000","last_undegraded":"2024-03-04T07:30:50.315763+0000","last_fullsized":"2024-03-04T07:30:50.315763+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:14:34.507741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.19","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316034+0000","last_change":"2024-03-04T07:30:48.997115+0000","last_active":"2024-03-04T07:30:50.316034+0000","last_peered":"2024-03-04T07:30:50.316034+0000","last_clean":"2024-03-04T07:30:50.316034+0000","last_became_active":"2024-03-04T07:30:48.996827+0000","last_became_peered":"2024-03-04T07:30:48.996827+0000","last_unstale":"2024-03-04T07:30:50.316034+0000","last_undegraded":"2024-03-04T07:30:50.316034+0000","last_fullsized":"2024-03-04T07:30:50.316034+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:41:57.419534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984050+0000","last_change":"2024-03-04T07:30:47.998585+0000","last_active":"2024-03-04T07:30:49.984050+0000","last_peered":"2024-03-04T07:30:49.984050+0000","last_clean":"2024-03-04T07:30:49.984050+0000","last_became_active":"2024-03-04T07:30:47.998267+0000","last_became_peered":"2024-03-04T07:30:47.998267+0000","last_unstale":"2024-03-04T07:30:49.984050+0000","last_undegraded":"2024-03-04T07:30:49.984050+0000","last_fullsized":"2024-03-04T07:30:49.984050+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:53:21.514299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.16","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316879+0000","last_change":"2024-03-04T07:30:49.001460+0000","last_active":"2024-03-04T07:30:50.316879+0000","last_peered":"2024-03-04T07:30:50.316879+0000","last_clean":"2024-03-04T07:30:50.316879+0000","last_became_active":"2024-03-04T07:30:49.001202+0000","last_became_peered":"2024-03-04T07:30:49.001202+0000","last_unstale":"2024-03-04T07:30:50.316879+0000","last_undegraded":"2024-03-04T07:30:50.316879+0000","last_fullsized":"2024-03-04T07:30:50.316879+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:42:00.836176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316399+0000","last_change":"2024-03-04T07:30:47.994776+0000","last_active":"2024-03-04T07:30:50.316399+0000","last_peered":"2024-03-04T07:30:50.316399+0000","last_clean":"2024-03-04T07:30:50.316399+0000","last_became_active":"2024-03-04T07:30:47.994204+0000","last_became_peered":"2024-03-04T07:30:47.994204+0000","last_unstale":"2024-03-04T07:30:50.316399+0000","last_undegraded":"2024-03-04T07:30:50.316399+0000","last_fullsized":"2024-03-04T07:30:50.316399+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:35:06.687014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988572+0000","last_change":"2024-03-04T07:30:49.006750+0000","last_active":"2024-03-04T07:30:49.988572+0000","last_peered":"2024-03-04T07:30:49.988572+0000","last_clean":"2024-03-04T07:30:49.988572+0000","last_became_active":"2024-03-04T07:30:49.006394+0000","last_became_peered":"2024-03-04T07:30:49.006394+0000","last_unstale":"2024-03-04T07:30:49.988572+0000","last_undegraded":"2024-03-04T07:30:49.988572+0000","last_fullsized":"2024-03-04T07:30:49.988572+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:16.705233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316188+0000","last_change":"2024-03-04T07:30:47.996670+0000","last_active":"2024-03-04T07:30:50.316188+0000","last_peered":"2024-03-04T07:30:50.316188+0000","last_clean":"2024-03-04T07:30:50.316188+0000","last_became_active":"2024-03-04T07:30:47.996290+0000","last_became_peered":"2024-03-04T07:30:47.996290+0000","last_unstale":"2024-03-04T07:30:50.316188+0000","last_undegraded":"2024-03-04T07:30:50.316188+0000","last_fullsized":"2024-03-04T07:30:50.316188+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:31.643739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.14","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316371+0000","last_change":"2024-03-04T07:30:49.000791+0000","last_active":"2024-03-04T07:30:50.316371+0000","last_peered":"2024-03-04T07:30:50.316371+0000","last_clean":"2024-03-04T07:30:50.316371+0000","last_became_active":"2024-03-04T07:30:49.000511+0000","last_became_peered":"2024-03-04T07:30:49.000511+0000","last_unstale":"2024-03-04T07:30:50.316371+0000","last_undegraded":"2024-03-04T07:30:50.316371+0000","last_fullsized":"2024-03-04T07:30:50.316371+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:05.868581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.15","version":"78'3","reported_seq":17,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316289+0000","last_change":"2024-03-04T07:30:47.994763+0000","last_active":"2024-03-04T07:30:50.316289+0000","last_peered":"2024-03-04T07:30:50.316289+0000","last_clean":"2024-03-04T07:30:50.316289+0000","last_became_active":"2024-03-04T07:30:47.994432+0000","last_became_peered":"2024-03-04T07:30:47.994432+0000","last_unstale":"2024-03-04T07:30:50.316289+0000","last_undegraded":"2024-03-04T07:30:50.316289+0000","last_fullsized":"2024-03-04T07:30:50.316289+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:20.302275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.15","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988190+0000","last_change":"2024-03-04T07:30:49.013600+0000","last_active":"2024-03-04T07:30:49.988190+0000","last_peered":"2024-03-04T07:30:49.988190+0000","last_clean":"2024-03-04T07:30:49.988190+0000","last_became_active":"2024-03-04T07:30:49.012745+0000","last_became_peered":"2024-03-04T07:30:49.012745+0000","last_unstale":"2024-03-04T07:30:49.988190+0000","last_undegraded":"2024-03-04T07:30:49.988190+0000","last_fullsized":"2024-03-04T07:30:49.988190+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:05.868581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"78'6","reported_seq":20,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.989056+0000","last_change":"2024-03-04T07:30:47.993284+0000","last_active":"2024-03-04T07:30:49.989056+0000","last_peered":"2024-03-04T07:30:49.989056+0000","last_clean":"2024-03-04T07:30:49.989056+0000","last_became_active":"2024-03-04T07:30:47.992982+0000","last_became_peered":"2024-03-04T07:30:47.992982+0000","last_unstale":"2024-03-04T07:30:49.989056+0000","last_undegraded":"2024-03-04T07:30:49.989056+0000","last_fullsized":"2024-03-04T07:30:49.989056+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:20.302275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.12","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315417+0000","last_change":"2024-03-04T07:30:49.007860+0000","last_active":"2024-03-04T07:30:50.315417+0000","last_peered":"2024-03-04T07:30:50.315417+0000","last_clean":"2024-03-04T07:30:50.315417+0000","last_became_active":"2024-03-04T07:30:49.007550+0000","last_became_peered":"2024-03-04T07:30:49.007550+0000","last_unstale":"2024-03-04T07:30:50.315417+0000","last_undegraded":"2024-03-04T07:30:50.315417+0000","last_fullsized":"2024-03-04T07:30:50.315417+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:26.008072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.13","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984318+0000","last_change":"2024-03-04T07:30:47.990436+0000","last_active":"2024-03-04T07:30:49.984318+0000","last_peered":"2024-03-04T07:30:49.984318+0000","last_clean":"2024-03-04T07:30:49.984318+0000","last_became_active":"2024-03-04T07:30:47.990145+0000","last_became_peered":"2024-03-04T07:30:47.990145+0000","last_unstale":"2024-03-04T07:30:49.984318+0000","last_undegraded":"2024-03-04T07:30:49.984318+0000","last_fullsized":"2024-03-04T07:30:49.984318+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:52.371659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.13","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316077+0000","last_change":"2024-03-04T07:30:49.000488+0000","last_active":"2024-03-04T07:30:50.316077+0000","last_peered":"2024-03-04T07:30:50.316077+0000","last_clean":"2024-03-04T07:30:50.316077+0000","last_became_active":"2024-03-04T07:30:49.000189+0000","last_became_peered":"2024-03-04T07:30:49.000189+0000","last_unstale":"2024-03-04T07:30:50.316077+0000","last_undegraded":"2024-03-04T07:30:50.316077+0000","last_fullsized":"2024-03-04T07:30:50.316077+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:40.772647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315640+0000","last_change":"2024-03-04T07:30:47.998081+0000","last_active":"2024-03-04T07:30:50.315640+0000","last_peered":"2024-03-04T07:30:50.315640+0000","last_clean":"2024-03-04T07:30:50.315640+0000","last_became_active":"2024-03-04T07:30:47.997756+0000","last_became_peered":"2024-03-04T07:30:47.997756+0000","last_unstale":"2024-03-04T07:30:50.315640+0000","last_undegraded":"2024-03-04T07:30:50.315640+0000","last_fullsized":"2024-03-04T07:30:50.315640+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:39.770825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.10","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984022+0000","last_change":"2024-03-04T07:30:49.006346+0000","last_active":"2024-03-04T07:30:49.984022+0000","last_peered":"2024-03-04T07:30:49.984022+0000","last_clean":"2024-03-04T07:30:49.984022+0000","last_became_active":"2024-03-04T07:30:49.006076+0000","last_became_peered":"2024-03-04T07:30:49.006076+0000","last_unstale":"2024-03-04T07:30:49.984022+0000","last_undegraded":"2024-03-04T07:30:49.984022+0000","last_fullsized":"2024-03-04T07:30:49.984022+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:35:45.932518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988477+0000","last_change":"2024-03-04T07:30:47.995953+0000","last_active":"2024-03-04T07:30:49.988477+0000","last_peered":"2024-03-04T07:30:49.988477+0000","last_clean":"2024-03-04T07:30:49.988477+0000","last_became_active":"2024-03-04T07:30:47.995518+0000","last_became_peered":"2024-03-04T07:30:47.995518+0000","last_unstale":"2024-03-04T07:30:49.988477+0000","last_undegraded":"2024-03-04T07:30:49.988477+0000","last_fullsized":"2024-03-04T07:30:49.988477+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:31.643739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.11","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984017+0000","last_change":"2024-03-04T07:30:48.999352+0000","last_active":"2024-03-04T07:30:49.984017+0000","last_peered":"2024-03-04T07:30:49.984017+0000","last_clean":"2024-03-04T07:30:49.984017+0000","last_became_active":"2024-03-04T07:30:48.999066+0000","last_became_peered":"2024-03-04T07:30:48.999066+0000","last_unstale":"2024-03-04T07:30:49.984017+0000","last_undegraded":"2024-03-04T07:30:49.984017+0000","last_fullsized":"2024-03-04T07:30:49.984017+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:29.273470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.10","version":"78'8","reported_seq":22,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315908+0000","last_change":"2024-03-04T07:30:47.999029+0000","last_active":"2024-03-04T07:30:50.315908+0000","last_peered":"2024-03-04T07:30:50.315908+0000","last_clean":"2024-03-04T07:30:50.315908+0000","last_became_active":"2024-03-04T07:30:47.998725+0000","last_became_peered":"2024-03-04T07:30:47.998725+0000","last_unstale":"2024-03-04T07:30:50.315908+0000","last_undegraded":"2024-03-04T07:30:50.315908+0000","last_fullsized":"2024-03-04T07:30:50.315908+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:09:01.359067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316482+0000","last_change":"2024-03-04T07:30:49.001803+0000","last_active":"2024-03-04T07:30:50.316482+0000","last_peered":"2024-03-04T07:30:50.316482+0000","last_clean":"2024-03-04T07:30:50.316482+0000","last_became_active":"2024-03-04T07:30:49.001497+0000","last_became_peered":"2024-03-04T07:30:49.001497+0000","last_unstale":"2024-03-04T07:30:50.316482+0000","last_undegraded":"2024-03-04T07:30:50.316482+0000","last_fullsized":"2024-03-04T07:30:50.316482+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:25:27.140576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.f","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316519+0000","last_change":"2024-03-04T07:30:47.988386+0000","last_active":"2024-03-04T07:30:50.316519+0000","last_peered":"2024-03-04T07:30:50.316519+0000","last_clean":"2024-03-04T07:30:50.316519+0000","last_became_active":"2024-03-04T07:30:47.988102+0000","last_became_peered":"2024-03-04T07:30:47.988102+0000","last_unstale":"2024-03-04T07:30:50.316519+0000","last_undegraded":"2024-03-04T07:30:50.316519+0000","last_fullsized":"2024-03-04T07:30:50.316519+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:10:05.632096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316328+0000","last_change":"2024-03-04T07:30:49.006698+0000","last_active":"2024-03-04T07:30:50.316328+0000","last_peered":"2024-03-04T07:30:50.316328+0000","last_clean":"2024-03-04T07:30:50.316328+0000","last_became_active":"2024-03-04T07:30:49.005131+0000","last_became_peered":"2024-03-04T07:30:49.005131+0000","last_unstale":"2024-03-04T07:30:50.316328+0000","last_undegraded":"2024-03-04T07:30:50.316328+0000","last_fullsized":"2024-03-04T07:30:50.316328+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:04:00.152366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316493+0000","last_change":"2024-03-04T07:30:48.000133+0000","last_active":"2024-03-04T07:30:50.316493+0000","last_peered":"2024-03-04T07:30:50.316493+0000","last_clean":"2024-03-04T07:30:50.316493+0000","last_became_active":"2024-03-04T07:30:47.999801+0000","last_became_peered":"2024-03-04T07:30:47.999801+0000","last_unstale":"2024-03-04T07:30:50.316493+0000","last_undegraded":"2024-03-04T07:30:50.316493+0000","last_fullsized":"2024-03-04T07:30:50.316493+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:43:08.764831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.d","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988644+0000","last_change":"2024-03-04T07:30:49.013867+0000","last_active":"2024-03-04T07:30:49.988644+0000","last_peered":"2024-03-04T07:30:49.988644+0000","last_clean":"2024-03-04T07:30:49.988644+0000","last_became_active":"2024-03-04T07:30:49.013536+0000","last_became_peered":"2024-03-04T07:30:49.013536+0000","last_unstale":"2024-03-04T07:30:49.988644+0000","last_undegraded":"2024-03-04T07:30:49.988644+0000","last_fullsized":"2024-03-04T07:30:49.988644+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:17:40.772647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316564+0000","last_change":"2024-03-04T07:30:47.999182+0000","last_active":"2024-03-04T07:30:50.316564+0000","last_peered":"2024-03-04T07:30:50.316564+0000","last_clean":"2024-03-04T07:30:50.316564+0000","last_became_active":"2024-03-04T07:30:47.998877+0000","last_became_peered":"2024-03-04T07:30:47.998877+0000","last_unstale":"2024-03-04T07:30:50.316564+0000","last_undegraded":"2024-03-04T07:30:50.316564+0000","last_fullsized":"2024-03-04T07:30:50.316564+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:57:58.248895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316436+0000","last_change":"2024-03-04T07:30:49.006916+0000","last_active":"2024-03-04T07:30:50.316436+0000","last_peered":"2024-03-04T07:30:50.316436+0000","last_clean":"2024-03-04T07:30:50.316436+0000","last_became_active":"2024-03-04T07:30:49.006195+0000","last_became_peered":"2024-03-04T07:30:49.006195+0000","last_unstale":"2024-03-04T07:30:50.316436+0000","last_undegraded":"2024-03-04T07:30:50.316436+0000","last_fullsized":"2024-03-04T07:30:50.316436+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:14:53.373481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316337+0000","last_change":"2024-03-04T07:30:47.996148+0000","last_active":"2024-03-04T07:30:50.316337+0000","last_peered":"2024-03-04T07:30:50.316337+0000","last_clean":"2024-03-04T07:30:50.316337+0000","last_became_active":"2024-03-04T07:30:47.995884+0000","last_became_peered":"2024-03-04T07:30:47.995884+0000","last_unstale":"2024-03-04T07:30:50.316337+0000","last_undegraded":"2024-03-04T07:30:50.316337+0000","last_fullsized":"2024-03-04T07:30:50.316337+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:03:59.150544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988206+0000","last_change":"2024-03-04T07:30:48.998811+0000","last_active":"2024-03-04T07:30:49.988206+0000","last_peered":"2024-03-04T07:30:49.988206+0000","last_clean":"2024-03-04T07:30:49.988206+0000","last_became_active":"2024-03-04T07:30:48.998405+0000","last_became_peered":"2024-03-04T07:30:48.998405+0000","last_unstale":"2024-03-04T07:30:49.988206+0000","last_undegraded":"2024-03-04T07:30:49.988206+0000","last_fullsized":"2024-03-04T07:30:49.988206+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:57:59.250717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316491+0000","last_change":"2024-03-04T07:30:47.991449+0000","last_active":"2024-03-04T07:30:50.316491+0000","last_peered":"2024-03-04T07:30:50.316491+0000","last_clean":"2024-03-04T07:30:50.316491+0000","last_became_active":"2024-03-04T07:30:47.991177+0000","last_became_peered":"2024-03-04T07:30:47.991177+0000","last_unstale":"2024-03-04T07:30:50.316491+0000","last_undegraded":"2024-03-04T07:30:50.316491+0000","last_fullsized":"2024-03-04T07:30:50.316491+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:19:56.702799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.8","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315571+0000","last_change":"2024-03-04T07:30:48.998228+0000","last_active":"2024-03-04T07:30:50.315571+0000","last_peered":"2024-03-04T07:30:50.315571+0000","last_clean":"2024-03-04T07:30:50.315571+0000","last_became_active":"2024-03-04T07:30:48.997947+0000","last_became_peered":"2024-03-04T07:30:48.997947+0000","last_unstale":"2024-03-04T07:30:50.315571+0000","last_undegraded":"2024-03-04T07:30:50.315571+0000","last_fullsized":"2024-03-04T07:30:50.315571+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:06.923347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316097+0000","last_change":"2024-03-04T07:30:47.996096+0000","last_active":"2024-03-04T07:30:50.316097+0000","last_peered":"2024-03-04T07:30:50.316097+0000","last_clean":"2024-03-04T07:30:50.316097+0000","last_became_active":"2024-03-04T07:30:47.995823+0000","last_became_peered":"2024-03-04T07:30:47.995823+0000","last_unstale":"2024-03-04T07:30:50.316097+0000","last_undegraded":"2024-03-04T07:30:50.316097+0000","last_fullsized":"2024-03-04T07:30:50.316097+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:01:19.526458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.9","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988364+0000","last_change":"2024-03-04T07:30:49.010132+0000","last_active":"2024-03-04T07:30:49.988364+0000","last_peered":"2024-03-04T07:30:49.988364+0000","last_clean":"2024-03-04T07:30:49.988364+0000","last_became_active":"2024-03-04T07:30:49.009703+0000","last_became_peered":"2024-03-04T07:30:49.009703+0000","last_unstale":"2024-03-04T07:30:49.988364+0000","last_undegraded":"2024-03-04T07:30:49.988364+0000","last_fullsized":"2024-03-04T07:30:49.988364+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:05:44.308067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315086+0000","last_change":"2024-03-04T07:30:47.993917+0000","last_active":"2024-03-04T07:30:50.315086+0000","last_peered":"2024-03-04T07:30:50.315086+0000","last_clean":"2024-03-04T07:30:50.315086+0000","last_became_active":"2024-03-04T07:30:47.993631+0000","last_became_peered":"2024-03-04T07:30:47.993631+0000","last_unstale":"2024-03-04T07:30:50.315086+0000","last_undegraded":"2024-03-04T07:30:50.315086+0000","last_fullsized":"2024-03-04T07:30:50.315086+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:44.471982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988824+0000","last_change":"2024-03-04T07:30:49.009786+0000","last_active":"2024-03-04T07:30:49.988824+0000","last_peered":"2024-03-04T07:30:49.988824+0000","last_clean":"2024-03-04T07:30:49.988824+0000","last_became_active":"2024-03-04T07:30:49.009358+0000","last_became_peered":"2024-03-04T07:30:49.009358+0000","last_unstale":"2024-03-04T07:30:49.988824+0000","last_undegraded":"2024-03-04T07:30:49.988824+0000","last_fullsized":"2024-03-04T07:30:49.988824+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:12:45.473804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"78'2","reported_seq":16,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984490+0000","last_change":"2024-03-04T07:30:47.992251+0000","last_active":"2024-03-04T07:30:49.984490+0000","last_peered":"2024-03-04T07:30:49.984490+0000","last_clean":"2024-03-04T07:30:49.984490+0000","last_became_active":"2024-03-04T07:30:47.991991+0000","last_became_peered":"2024-03-04T07:30:47.991991+0000","last_unstale":"2024-03-04T07:30:49.984490+0000","last_undegraded":"2024-03-04T07:30:49.984490+0000","last_fullsized":"2024-03-04T07:30:49.984490+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:32:50.331686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.7","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316455+0000","last_change":"2024-03-04T07:30:49.003511+0000","last_active":"2024-03-04T07:30:50.316455+0000","last_peered":"2024-03-04T07:30:50.316455+0000","last_clean":"2024-03-04T07:30:50.316455+0000","last_became_active":"2024-03-04T07:30:49.003163+0000","last_became_peered":"2024-03-04T07:30:49.003163+0000","last_unstale":"2024-03-04T07:30:50.316455+0000","last_undegraded":"2024-03-04T07:30:50.316455+0000","last_fullsized":"2024-03-04T07:30:50.316455+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:11:56.451211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.6","version":"79'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.987490+0000","last_change":"2024-03-04T07:30:47.991297+0000","last_active":"2024-03-04T07:30:50.987490+0000","last_peered":"2024-03-04T07:30:50.987490+0000","last_clean":"2024-03-04T07:30:50.987490+0000","last_became_active":"2024-03-04T07:30:47.990997+0000","last_became_peered":"2024-03-04T07:30:47.990997+0000","last_unstale":"2024-03-04T07:30:50.987490+0000","last_undegraded":"2024-03-04T07:30:50.987490+0000","last_fullsized":"2024-03-04T07:30:50.987490+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:29:04.427400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.4","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315899+0000","last_change":"2024-03-04T07:30:49.006306+0000","last_active":"2024-03-04T07:30:50.315899+0000","last_peered":"2024-03-04T07:30:50.315899+0000","last_clean":"2024-03-04T07:30:50.315899+0000","last_became_active":"2024-03-04T07:30:49.005136+0000","last_became_peered":"2024-03-04T07:30:49.005136+0000","last_unstale":"2024-03-04T07:30:50.315899+0000","last_undegraded":"2024-03-04T07:30:50.315899+0000","last_fullsized":"2024-03-04T07:30:50.315899+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:15:55.458985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315684+0000","last_change":"2024-03-04T07:30:47.996074+0000","last_active":"2024-03-04T07:30:50.315684+0000","last_peered":"2024-03-04T07:30:50.315684+0000","last_clean":"2024-03-04T07:30:50.315684+0000","last_became_active":"2024-03-04T07:30:47.995749+0000","last_became_peered":"2024-03-04T07:30:47.995749+0000","last_unstale":"2024-03-04T07:30:50.315684+0000","last_undegraded":"2024-03-04T07:30:50.315684+0000","last_fullsized":"2024-03-04T07:30:50.315684+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:43:28.639676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988415+0000","last_change":"2024-03-04T07:30:49.013080+0000","last_active":"2024-03-04T07:30:49.988415+0000","last_peered":"2024-03-04T07:30:49.988415+0000","last_clean":"2024-03-04T07:30:49.988415+0000","last_became_active":"2024-03-04T07:30:49.012334+0000","last_became_peered":"2024-03-04T07:30:49.012334+0000","last_unstale":"2024-03-04T07:30:49.988415+0000","last_undegraded":"2024-03-04T07:30:49.988415+0000","last_fullsized":"2024-03-04T07:30:49.988415+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:12.423638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316018+0000","last_change":"2024-03-04T07:30:47.993482+0000","last_active":"2024-03-04T07:30:50.316018+0000","last_peered":"2024-03-04T07:30:50.316018+0000","last_clean":"2024-03-04T07:30:50.316018+0000","last_became_active":"2024-03-04T07:30:47.993165+0000","last_became_peered":"2024-03-04T07:30:47.993165+0000","last_unstale":"2024-03-04T07:30:50.316018+0000","last_undegraded":"2024-03-04T07:30:50.316018+0000","last_fullsized":"2024-03-04T07:30:50.316018+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:46.683475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.989120+0000","last_change":"2024-03-04T07:30:48.998687+0000","last_active":"2024-03-04T07:30:49.989120+0000","last_peered":"2024-03-04T07:30:49.989120+0000","last_clean":"2024-03-04T07:30:49.989120+0000","last_became_active":"2024-03-04T07:30:48.998380+0000","last_became_peered":"2024-03-04T07:30:48.998380+0000","last_unstale":"2024-03-04T07:30:49.989120+0000","last_undegraded":"2024-03-04T07:30:49.989120+0000","last_fullsized":"2024-03-04T07:30:49.989120+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:01:35.965763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316045+0000","last_change":"2024-03-04T07:30:47.994305+0000","last_active":"2024-03-04T07:30:50.316045+0000","last_peered":"2024-03-04T07:30:50.316045+0000","last_clean":"2024-03-04T07:30:50.316045+0000","last_became_active":"2024-03-04T07:30:47.994057+0000","last_became_peered":"2024-03-04T07:30:47.994057+0000","last_unstale":"2024-03-04T07:30:50.316045+0000","last_undegraded":"2024-03-04T07:30:50.316045+0000","last_fullsized":"2024-03-04T07:30:50.316045+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:13:52.850140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.3","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315441+0000","last_change":"2024-03-04T07:30:49.008570+0000","last_active":"2024-03-04T07:30:50.315441+0000","last_peered":"2024-03-04T07:30:50.315441+0000","last_clean":"2024-03-04T07:30:50.315441+0000","last_became_active":"2024-03-04T07:30:49.008294+0000","last_became_peered":"2024-03-04T07:30:49.008294+0000","last_unstale":"2024-03-04T07:30:50.315441+0000","last_undegraded":"2024-03-04T07:30:50.315441+0000","last_fullsized":"2024-03-04T07:30:50.315441+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:24.510630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988153+0000","last_change":"2024-03-04T07:30:47.999342+0000","last_active":"2024-03-04T07:30:49.988153+0000","last_peered":"2024-03-04T07:30:49.988153+0000","last_clean":"2024-03-04T07:30:49.988153+0000","last_became_active":"2024-03-04T07:30:47.997726+0000","last_became_peered":"2024-03-04T07:30:47.997726+0000","last_unstale":"2024-03-04T07:30:49.988153+0000","last_undegraded":"2024-03-04T07:30:49.988153+0000","last_fullsized":"2024-03-04T07:30:49.988153+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:46.683475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.0","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315802+0000","last_change":"2024-03-04T07:30:49.002119+0000","last_active":"2024-03-04T07:30:50.315802+0000","last_peered":"2024-03-04T07:30:50.315802+0000","last_clean":"2024-03-04T07:30:50.315802+0000","last_became_active":"2024-03-04T07:30:49.001807+0000","last_became_peered":"2024-03-04T07:30:49.001807+0000","last_unstale":"2024-03-04T07:30:50.315802+0000","last_undegraded":"2024-03-04T07:30:50.315802+0000","last_fullsized":"2024-03-04T07:30:50.315802+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:15:55.458985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315637+0000","last_change":"2024-03-04T07:30:47.992253+0000","last_active":"2024-03-04T07:30:50.315637+0000","last_peered":"2024-03-04T07:30:50.315637+0000","last_clean":"2024-03-04T07:30:50.315637+0000","last_became_active":"2024-03-04T07:30:47.991944+0000","last_became_peered":"2024-03-04T07:30:47.991944+0000","last_unstale":"2024-03-04T07:30:50.315637+0000","last_undegraded":"2024-03-04T07:30:50.315637+0000","last_fullsized":"2024-03-04T07:30:50.315637+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:25:02.651448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315482+0000","last_change":"2024-03-04T07:30:48.996455+0000","last_active":"2024-03-04T07:30:50.315482+0000","last_peered":"2024-03-04T07:30:50.315482+0000","last_clean":"2024-03-04T07:30:50.315482+0000","last_became_active":"2024-03-04T07:30:48.996124+0000","last_became_peered":"2024-03-04T07:30:48.996124+0000","last_unstale":"2024-03-04T07:30:50.315482+0000","last_undegraded":"2024-03-04T07:30:50.315482+0000","last_fullsized":"2024-03-04T07:30:50.315482+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:02.000572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":14,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.988042+0000","last_change":"2024-03-04T07:30:47.999754+0000","last_active":"2024-03-04T07:30:49.988042+0000","last_peered":"2024-03-04T07:30:49.988042+0000","last_clean":"2024-03-04T07:30:49.988042+0000","last_became_active":"2024-03-04T07:30:47.999328+0000","last_became_peered":"2024-03-04T07:30:47.999328+0000","last_unstale":"2024-03-04T07:30:49.988042+0000","last_undegraded":"2024-03-04T07:30:49.988042+0000","last_fullsized":"2024-03-04T07:30:49.988042+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:10.910681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.c","version":"0'0","reported_seq":12,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:49.984276+0000","last_change":"2024-03-04T07:30:49.000154+0000","last_active":"2024-03-04T07:30:49.984276+0000","last_peered":"2024-03-04T07:30:49.984276+0000","last_clean":"2024-03-04T07:30:49.984276+0000","last_became_active":"2024-03-04T07:30:48.999228+0000","last_became_peered":"2024-03-04T07:30:48.999228+0000","last_unstale":"2024-03-04T07:30:49.984276+0000","last_undegraded":"2024-03-04T07:30:49.984276+0000","last_fullsized":"2024-03-04T07:30:49.984276+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":77,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:47.974706+0000","last_clean_scrub_stamp":"2024-03-04T07:30:47.974706+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:43:29.641498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"78'1","reported_seq":15,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.316411+0000","last_change":"2024-03-04T07:30:47.995321+0000","last_active":"2024-03-04T07:30:50.316411+0000","last_peered":"2024-03-04T07:30:50.316411+0000","last_clean":"2024-03-04T07:30:50.316411+0000","last_became_active":"2024-03-04T07:30:47.994967+0000","last_became_peered":"2024-03-04T07:30:47.994967+0000","last_unstale":"2024-03-04T07:30:50.316411+0000","last_undegraded":"2024-03-04T07:30:50.316411+0000","last_fullsized":"2024-03-04T07:30:50.316411+0000","mapping_epoch":76,"log_start":"0'0","ondisk_log_start":"0'0","created":76,"last_epoch_clean":77,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:30:46.972884+0000","last_clean_scrub_stamp":"2024-03-04T07:30:46.972884+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:46:38.223887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":277,"reported_epoch":79,"state":"active+clean","last_fresh":"2024-03-04T07:30:50.315652+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:30:50.315652+0000","last_peered":"2024-03-04T07:30:50.315652+0000","last_clean":"2024-03-04T07:30:50.315652+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:30:50.315652+0000","last_undegraded":"2024-03-04T07:30:50.315652+0000","last_fullsized":"2024-03-04T07:30:50.315652+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574903,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32704,"kb_used_data":1520,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339136,"statfs":{"total":96636764160,"available":96603275264,"internally_reserved":0,"allocated":1556480,"data_stored":995688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574902,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32132,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339708,"statfs":{"total":96636764160,"available":96603860992,"internally_reserved":0,"allocated":970752,"data_stored":405944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574902,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32152,"kb_used_data":964,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339688,"statfs":{"total":96636764160,"available":96603840512,"internally_reserved":0,"allocated":987136,"data_stored":413184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574903,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32132,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339708,"statfs":{"total":96636764160,"available":96603860992,"internally_reserved":0,"allocated":970752,"data_stored":411550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574902,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32148,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339692,"statfs":{"total":96636764160,"available":96603844608,"internally_reserved":0,"allocated":983040,"data_stored":413162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574902,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32140,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339700,"statfs":{"total":96636764160,"available":96603852800,"internally_reserved":0,"allocated":974848,"data_stored":412132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574902,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32148,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339692,"statfs":{"total":96636764160,"available":96603844608,"internally_reserved":0,"allocated":983040,"data_stored":411744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574902,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32712,"kb_used_data":1532,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339128,"statfs":{"total":96636764160,"available":96603267072,"internally_reserved":0,"allocated":1568768,"data_stored":1001952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:30:52.360 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:30:52.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:30:52.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.505+0000 7f18e4639640 1 Processor -- start 2024-03-04T07:30:52.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.505+0000 7f18e4639640 1 -- start start 2024-03-04T07:30:52.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e4639640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f18dc0ceea0 0x7f18dc0d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e4639640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e4639640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f18dc0d3740 con 0x7f18dc15dd90 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e4639640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f18dc0d3400 con 0x7f18dc0ceea0 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e4639640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18dc0d3120 con 0x7f18dc0b6d10 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e1bd4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f18dc0ceea0 0x7f18dc0d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e1bd4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f18dc0ceea0 0x7f18dc0d29d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42384/0 (socket says 172.21.15.64:42384) 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.506+0000 7f18e1bd4640 1 -- 172.21.15.64:0/3395110466 learned_addr learned my addr 172.21.15.64:0/3395110466 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.507+0000 7f18e2bd6640 1 --2- 172.21.15.64:0/3395110466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.507+0000 7f18e1bd4640 1 -- 172.21.15.64:0/3395110466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 msgr2=0x7f18dc0b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.507+0000 7f18e1bd4640 1 --2- 172.21.15.64:0/3395110466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.507+0000 7f18e1bd4640 1 -- 172.21.15.64:0/3395110466 >> v1:172.21.15.66:6789/0 conn(0x7f18dc15dd90 legacy=0x7f18dc0d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:52.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.507+0000 7f18e1bd4640 1 -- 172.21.15.64:0/3395110466 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18dc0d39e0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e1bd4640 1 --2- 172.21.15.64:0/3395110466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc0d29d0 secure :-1 s=READY pgs=998 cs=0 l=1 rev1=1 crypto rx=0x7f18d8009870 tx=0x7f18d802ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8bc0080ca9a31401 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e13d3640 1 -- 172.21.15.64:0/3395110466 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d802f840 con 0x7f18dc0ceea0 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e13d3640 1 -- 172.21.15.64:0/3395110466 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f18d802f9e0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e13d3640 1 -- 172.21.15.64:0/3395110466 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d802fdb0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 -- 172.21.15.64:0/3395110466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 msgr2=0x7f18dc0d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 --2- 172.21.15.64:0/3395110466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc0d29d0 secure :-1 s=READY pgs=998 cs=0 l=1 rev1=1 crypto rx=0x7f18d8009870 tx=0x7f18d802ba20 comp rx=0 tx=0).stop 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 -- 172.21.15.64:0/3395110466 shutdown_connections 2024-03-04T07:30:52.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 --2- 172.21.15.64:0/3395110466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 --2- 172.21.15.64:0/3395110466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc0d29d0 unknown :-1 s=CLOSED pgs=998 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 -- 172.21.15.64:0/3395110466 >> 172.21.15.64:0/3395110466 conn(0x7f18dc071960 msgr2=0x7f18dc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:52.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 -- 172.21.15.64:0/3395110466 shutdown_connections 2024-03-04T07:30:52.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.508+0000 7f18e4639640 1 -- 172.21.15.64:0/3395110466 wait complete. 2024-03-04T07:30:52.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.509+0000 7f18e4639640 1 Processor -- start 2024-03-04T07:30:52.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.509+0000 7f18e4639640 1 -- start start 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e4639640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0cd540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e4639640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc161940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e4639640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18dc15dd90 0x7f18dc0c9bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e4639640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18dc0d48f0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e4639640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18dc15e170 con 0x7f18dc15dd90 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e4639640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18dc0d4a70 con 0x7f18dc0b6d10 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e1bd4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc161940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e1bd4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc161940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42398/0 (socket says 172.21.15.64:42398) 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e1bd4640 1 -- 172.21.15.64:0/1084085883 learned_addr learned my addr 172.21.15.64:0/1084085883 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e23d5640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0cd540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.510+0000 7f18e2bd6640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18dc15dd90 0x7f18dc0c9bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e1bd4640 1 -- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 msgr2=0x7f18dc0cd540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e1bd4640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0cd540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e1bd4640 1 -- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18dc15dd90 msgr2=0x7f18dc0c9bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e1bd4640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18dc15dd90 0x7f18dc0c9bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e1bd4640 1 -- 172.21.15.64:0/1084085883 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18dc15e6f0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e1bd4640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc161940 secure :-1 s=READY pgs=999 cs=0 l=1 rev1=1 crypto rx=0x7f18d8000c00 tx=0x7f18d8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18ceffd640 1 -- 172.21.15.64:0/1084085883 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d8037bc0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18ceffd640 1 -- 172.21.15.64:0/1084085883 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f18d8046070 con 0x7f18dc0ceea0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18dc15e9d0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18ceffd640 1 -- 172.21.15.64:0/1084085883 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18d8041400 con 0x7f18dc0ceea0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.511+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18dc15ef90 con 0x7f18dc0ceea0 2024-03-04T07:30:52.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.512+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f18dc072a30 con 0x7f18dc0ceea0 2024-03-04T07:30:52.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.512+0000 7f18ceffd640 1 -- 172.21.15.64:0/1084085883 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f18d8040070 con 0x7f18dc0ceea0 2024-03-04T07:30:52.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.513+0000 7f18ceffd640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f18c006b500 0x7f18c006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.513+0000 7f18ceffd640 1 -- 172.21.15.64:0/1084085883 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f18d80ac750 con 0x7f18dc0ceea0 2024-03-04T07:30:52.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.513+0000 7f18e23d5640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f18c006b500 0x7f18c006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.513+0000 7f18e23d5640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f18c006b500 0x7f18c006d9c0 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7f18d0000f60 tx=0x7f18d0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.517+0000 7f18ceffd640 1 -- 172.21.15.64:0/1084085883 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f18d8033030 con 0x7f18dc0ceea0 2024-03-04T07:30:52.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.772+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f18dc04f410 con 0x7f18dc0ceea0 2024-03-04T07:30:52.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.774+0000 7f18ceffd640 1 -- 172.21.15.64:0/1084085883 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v19) v1 ==== 190+0+147 (secure 0 0 0) 0x7f18d807aef0 con 0x7f18dc0ceea0 2024-03-04T07:30:52.774 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:30:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:30:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:30:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:30:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:30:52.775 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f18c006b500 msgr2=0x7f18c006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f18c006b500 0x7f18c006d9c0 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7f18d0000f60 tx=0x7f18d0009040 comp rx=0 tx=0).stop 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 msgr2=0x7f18dc161940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc161940 secure :-1 s=READY pgs=999 cs=0 l=1 rev1=1 crypto rx=0x7f18d8000c00 tx=0x7f18d8004730 comp rx=0 tx=0).stop 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 shutdown_connections 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f18c006b500 0x7f18c006d9c0 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f18dc15dd90 0x7f18dc0c9bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f18dc0ceea0 0x7f18dc161940 unknown :-1 s=CLOSED pgs=999 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 --2- 172.21.15.64:0/1084085883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f18dc0b6d10 0x7f18dc0cd540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.776+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 >> 172.21.15.64:0/1084085883 conn(0x7f18dc071960 msgr2=0x7f18dc0b3e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.777+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 shutdown_connections 2024-03-04T07:30:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.777+0000 7f18e4639640 1 -- 172.21.15.64:0/1084085883 wait complete. 2024-03-04T07:30:52.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:52.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.888+0000 7f717753a640 1 Processor -- start 2024-03-04T07:30:52.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.889+0000 7f717753a640 1 -- start start 2024-03-04T07:30:52.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f717753a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f71700ae9b0 0x7f71700d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f717753a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71700cee90 con 0x7f71700b0e90 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f717753a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71700cfc50 con 0x7f717006b5c0 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f717753a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f71700cfa00 con 0x7f71700ae9b0 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f7174ad5640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f71700b0e90 0x7f717006b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:48130/0 (socket says 172.21.15.64:48130) 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f7174ad5640 1 -- 172.21.15.64:0/1979124814 learned_addr learned my addr 172.21.15.64:0/1979124814 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f7175ad7640 1 --2- 172.21.15.64:0/1979124814 >> v2:172.21.15.66:3300/0 conn(0x7f71700ae9b0 0x7f71700d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 430016225 0 0) 0x7f71700cee90 con 0x7f71700b0e90 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f715c003520 con 0x7f71700b0e90 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2403625891 0 0) 0x7f71700cfc50 con 0x7f717006b5c0 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.890+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f71700cee90 con 0x7f717006b5c0 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3497761604 0 0) 0x7f715c003520 con 0x7f71700b0e90 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 >> v2:172.21.15.66:3300/0 conn(0x7f71700ae9b0 msgr2=0x7f71700d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 --2- 172.21.15.64:0/1979124814 >> v2:172.21.15.66:3300/0 conn(0x7f71700ae9b0 0x7f71700d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 >> v1:172.21.15.66:6789/0 conn(0x7f717006b5c0 legacy=0x7f71700b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71700cff30 con 0x7f71700b0e90 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f7160001670 con 0x7f71700b0e90 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7175ad7640 1 --2- 172.21.15.64:0/1979124814 >> v2:172.21.15.66:3300/0 conn(0x7f71700ae9b0 0x7f71700d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 >> v1:172.21.15.64:6789/0 conn(0x7f71700b0e90 legacy=0x7f717006b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f715c003b40 0x7f715c003f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f715c004800 0x7f715c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c0051c0 0x7f715c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71700cfa00 con 0x7f715c004800 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f715c00c0f0 con 0x7f715c003b40 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f715c00c2f0 con 0x7f715c0051c0 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7174ad5640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f715c004800 0x7f715c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f71752d6640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f715c003b40 0x7f715c003f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7175ad7640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c0051c0 0x7f715c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7174ad5640 1 -- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c0051c0 msgr2=0x7f715c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7174ad5640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c0051c0 0x7f715c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7174ad5640 1 -- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f715c003b40 msgr2=0x7f715c003f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7174ad5640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f715c003b40 0x7f715c003f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7174ad5640 1 -- 172.21.15.64:0/1979124814 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f715c003520 con 0x7f715c004800 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.891+0000 7f7175ad7640 1 -- 172.21.15.64:0/1979124814 reap_dead start 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.892+0000 7f7174ad5640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f715c004800 0x7f715c004c80 secure :-1 s=READY pgs=1001 cs=0 l=1 rev1=1 crypto rx=0x7f716000c190 tx=0x7f715c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=92c3a9500c354336 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.892+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7160007ce0 con 0x7f715c004800 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.892+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7160030070 con 0x7f715c004800 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.892+0000 7f7167fff640 1 -- 172.21.15.64:0/1979124814 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7160005d90 con 0x7f715c004800 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.893+0000 7f717753a640 1 -- 172.21.15.64:0/1979124814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f715c004800 msgr2=0x7f715c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.893+0000 7f717753a640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f715c004800 0x7f715c004c80 secure :-1 s=READY pgs=1001 cs=0 l=1 rev1=1 crypto rx=0x7f716000c190 tx=0x7f715c005830 comp rx=0 tx=0).stop 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.893+0000 7f717753a640 1 -- 172.21.15.64:0/1979124814 shutdown_connections 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.893+0000 7f717753a640 1 --2- 172.21.15.64:0/1979124814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f715c004800 0x7f715c004c80 unknown :-1 s=CLOSED pgs=1001 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.893+0000 7f717753a640 1 -- 172.21.15.64:0/1979124814 >> 172.21.15.64:0/1979124814 conn(0x7f71700b36d0 msgr2=0x7f71700b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.893+0000 7f717753a640 1 -- 172.21.15.64:0/1979124814 shutdown_connections 2024-03-04T07:30:52.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.893+0000 7f717753a640 1 -- 172.21.15.64:0/1979124814 wait complete. 2024-03-04T07:30:52.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.894+0000 7f717753a640 1 Processor -- start 2024-03-04T07:30:52.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.894+0000 7f717753a640 1 -- start start 2024-03-04T07:30:52.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f717753a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c004800 0x7f71701f81e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f717753a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71701e8150 0x7f71701e47f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f717753a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71701db640 0x7f71701dbac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f717753a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71700aef40 con 0x7f71701db640 2024-03-04T07:30:52.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f717753a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71700aedc0 con 0x7f71701e8150 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f717753a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f71700b2e00 con 0x7f715c004800 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71701db640 0x7f71701dbac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71701db640 0x7f71701dbac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42426/0 (socket says 172.21.15.64:42426) 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 -- 172.21.15.64:0/4227597725 learned_addr learned my addr 172.21.15.64:0/4227597725 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 -- 172.21.15.64:0/4227597725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c004800 msgr2=0x7f71701f81e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c004800 0x7f71701f81e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 -- 172.21.15.64:0/4227597725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71701e8150 msgr2=0x7f71701e47f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71701e8150 0x7f71701e47f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 -- 172.21.15.64:0/4227597725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71701e8710 con 0x7f71701db640 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7175ad7640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71701db640 0x7f71701dbac0 secure :-1 s=READY pgs=1002 cs=0 l=1 rev1=1 crypto rx=0x7f7170071c60 tx=0x7f716800f580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7165ffb640 1 -- 172.21.15.64:0/4227597725 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f716800fe20 con 0x7f71701db640 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7165ffb640 1 -- 172.21.15.64:0/4227597725 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7168003990 con 0x7f71701db640 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f71701e89f0 con 0x7f71701db640 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.895+0000 7f7165ffb640 1 -- 172.21.15.64:0/4227597725 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7168003d80 con 0x7f71701db640 2024-03-04T07:30:52.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.896+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f71701e8fb0 con 0x7f71701db640 2024-03-04T07:30:52.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.896+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7170068700 con 0x7f71701db640 2024-03-04T07:30:52.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.897+0000 7f7165ffb640 1 -- 172.21.15.64:0/4227597725 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f716801f020 con 0x7f71701db640 2024-03-04T07:30:52.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.897+0000 7f7165ffb640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f715406b4d0 0x7f715406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:52.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.897+0000 7f7165ffb640 1 -- 172.21.15.64:0/4227597725 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f716808a590 con 0x7f71701db640 2024-03-04T07:30:52.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.898+0000 7f71752d6640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f715406b4d0 0x7f715406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:52.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.898+0000 7f71752d6640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f715406b4d0 0x7f715406d990 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f716c001830 tx=0x7f716c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:52.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:52.902+0000 7f7165ffb640 1 -- 172.21.15.64:0/4227597725 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7168010040 con 0x7f71701db640 2024-03-04T07:30:53.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.122+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f71700c8460 con 0x7f71701db640 2024-03-04T07:30:53.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.122+0000 7f7165ffb640 1 -- 172.21.15.64:0/4227597725 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 43 v43) v1 ==== 94+0+4973 (secure 0 0 0) 0x7f71680594a0 con 0x7f71701db640 2024-03-04T07:30:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:53.122 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[{"mdsmap":{"epoch":43,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:48.344908+0000","modified":"2024-03-04T07:30:50.410439+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6966},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6966":{"gid":6966,"name":"c","rank":0,"incarnation":41,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/1889124368","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1889124368},{"type":"v1","addr":"172.21.15.64:6836","nonce":1889124368}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6966,"qdb_cluster":[6966]},"id":6}]} 2024-03-04T07:30:53.122 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 43 2024-03-04T07:30:53.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f715406b4d0 msgr2=0x7f715406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f715406b4d0 0x7f715406d990 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f716c001830 tx=0x7f716c009040 comp rx=0 tx=0).stop 2024-03-04T07:30:53.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71701db640 msgr2=0x7f71701dbac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71701db640 0x7f71701dbac0 secure :-1 s=READY pgs=1002 cs=0 l=1 rev1=1 crypto rx=0x7f7170071c60 tx=0x7f716800f580 comp rx=0 tx=0).stop 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 shutdown_connections 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f715406b4d0 0x7f715406d990 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71701db640 0x7f71701dbac0 unknown :-1 s=CLOSED pgs=1002 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71701e8150 0x7f71701e47f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 --2- 172.21.15.64:0/4227597725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f715c004800 0x7f71701f81e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 >> 172.21.15.64:0/4227597725 conn(0x7f71700b36d0 msgr2=0x7f71700af730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 shutdown_connections 2024-03-04T07:30:53.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.124+0000 7f717753a640 1 -- 172.21.15.64:0/4227597725 wait complete. 2024-03-04T07:30:53.538 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 43, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:30:48.344908+0000', 'modified': '2024-03-04T07:30:50.410439+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 6966}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6966': {'gid': 6966, 'name': 'c', 'rank': 0, 'incarnation': 41, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.64:6836/1889124368', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6835', 'nonce': 1889124368}, {'type': 'v1', 'addr': '172.21.15.64:6836', 'nonce': 1889124368}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [13], 'metadata_pool': 12, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 6966, 'qdb_cluster': [6966]} 2024-03-04T07:30:53.538 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:30:53.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-04T07:30:53.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.641+0000 7fb646193640 1 Processor -- start 2024-03-04T07:30:53.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.642+0000 7fb646193640 1 -- start start 2024-03-04T07:30:53.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.642+0000 7fb646193640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb64015df00 0x7fb6400d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.642+0000 7fb646193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b9a30 0x7fb6400d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.642+0000 7fb646193640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6400cea30 con 0x7fb6400b95a0 2024-03-04T07:30:53.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.642+0000 7fb646193640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb6400d2af0 con 0x7fb64015df00 2024-03-04T07:30:53.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.642+0000 7fb646193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6400cf3c0 con 0x7fb6400b9a30 2024-03-04T07:30:53.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b9a30 0x7fb6400d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63f7fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb64015df00 0x7fb6400d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63effd640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fb6400b95a0 0x7fb6400b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43638/0 (socket says 172.21.15.64:43638) 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b9a30 0x7fb6400d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:55184/0 (socket says 172.21.15.64:55184) 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63effd640 1 -- 172.21.15.64:0/449916345 learned_addr learned my addr 172.21.15.64:0/449916345 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63e7fc640 1 -- 172.21.15.64:0/449916345 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2395127266 0 0) 0x7fb6400cea30 con 0x7fb6400b95a0 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63e7fc640 1 -- 172.21.15.64:0/449916345 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb628003520 con 0x7fb6400b95a0 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63ffff640 1 -- 172.21.15.64:0/449916345 >> v2:172.21.15.66:3300/0 conn(0x7fb64015df00 msgr2=0x7fb6400d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63ffff640 1 --2- 172.21.15.64:0/449916345 >> v2:172.21.15.66:3300/0 conn(0x7fb64015df00 0x7fb6400d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63ffff640 1 -- 172.21.15.64:0/449916345 >> v1:172.21.15.66:6789/0 conn(0x7fb6400b95a0 legacy=0x7fb6400b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63ffff640 1 -- 172.21.15.64:0/449916345 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6400cf0e0 con 0x7fb6400b9a30 2024-03-04T07:30:53.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.643+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/449916345 >> v2:172.21.15.66:3300/0 conn(0x7fb64015df00 0x7fb6400d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.644+0000 7fb63ffff640 1 --2- 172.21.15.64:0/449916345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b9a30 0x7fb6400d3220 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fb638009d90 tx=0x7fb63802bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=912a1d6503398ea4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.644+0000 7fb63e7fc640 1 -- 172.21.15.64:0/449916345 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb638039070 con 0x7fb6400b9a30 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.644+0000 7fb63e7fc640 1 -- 172.21.15.64:0/449916345 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb63802cc50 con 0x7fb6400b9a30 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.644+0000 7fb63e7fc640 1 -- 172.21.15.64:0/449916345 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb638034440 con 0x7fb6400b9a30 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 -- 172.21.15.64:0/449916345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b9a30 msgr2=0x7fb6400d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 --2- 172.21.15.64:0/449916345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b9a30 0x7fb6400d3220 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fb638009d90 tx=0x7fb63802bea0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 -- 172.21.15.64:0/449916345 shutdown_connections 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 --2- 172.21.15.64:0/449916345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b9a30 0x7fb6400d3220 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 --2- 172.21.15.64:0/449916345 >> v2:172.21.15.66:3300/0 conn(0x7fb64015df00 0x7fb6400d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 -- 172.21.15.64:0/449916345 >> 172.21.15.64:0/449916345 conn(0x7fb640071960 msgr2=0x7fb640071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 -- 172.21.15.64:0/449916345 shutdown_connections 2024-03-04T07:30:53.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.645+0000 7fb646193640 1 -- 172.21.15.64:0/449916345 wait complete. 2024-03-04T07:30:53.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.646+0000 7fb646193640 1 Processor -- start 2024-03-04T07:30:53.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.646+0000 7fb646193640 1 -- start start 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb646193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b95a0 0x7fb640151810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb646193640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb64015df00 0x7fb6400c9d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb646193640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb640151d50 0x7fb6400ca260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb646193640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6400cec70 con 0x7fb640151d50 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb646193640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6400ce750 con 0x7fb64015df00 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb646193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6400cedf0 con 0x7fb6400b95a0 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb640151d50 0x7fb6400ca260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb640151d50 0x7fb6400ca260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48396/0 (socket says 172.21.15.64:48396) 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 -- 172.21.15.64:0/576563941 learned_addr learned my addr 172.21.15.64:0/576563941 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b95a0 0x7fb640151810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63effd640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb64015df00 0x7fb6400c9d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 -- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b95a0 msgr2=0x7fb640151810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b95a0 0x7fb640151810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 -- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb64015df00 msgr2=0x7fb6400c9d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb64015df00 0x7fb6400c9d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63ffff640 1 -- 172.21.15.64:0/576563941 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6400b5ee0 con 0x7fb640151d50 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.647+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b95a0 0x7fb640151810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.648+0000 7fb63ffff640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb640151d50 0x7fb6400ca260 secure :-1 s=READY pgs=1003 cs=0 l=1 rev1=1 crypto rx=0x7fb638009d90 tx=0x7fb638004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.648+0000 7fb627fff640 1 -- 172.21.15.64:0/576563941 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb638042030 con 0x7fb640151d50 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.648+0000 7fb627fff640 1 -- 172.21.15.64:0/576563941 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb63803a040 con 0x7fb640151d50 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.648+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6400cb930 con 0x7fb640151d50 2024-03-04T07:30:53.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.648+0000 7fb627fff640 1 -- 172.21.15.64:0/576563941 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb638039070 con 0x7fb640151d50 2024-03-04T07:30:53.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.648+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6400b5c00 con 0x7fb640151d50 2024-03-04T07:30:53.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.650+0000 7fb627fff640 1 -- 172.21.15.64:0/576563941 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb638045070 con 0x7fb640151d50 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.650+0000 7fb627fff640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb62c06b4d0 0x7fb62c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.650+0000 7fb627fff640 1 -- 172.21.15.64:0/576563941 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb6380abf30 con 0x7fb640151d50 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.650+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb62c06b4d0 0x7fb62c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.651+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb62c06b4d0 0x7fb62c06d990 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fb630009ba0 tx=0x7fb630008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.651+0000 7fb648d77740 1 Processor -- start 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.651+0000 7fb648d77740 1 -- start start 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13625870 0x561c13625c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb648d77740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c136298c0 0x561c1362b360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c13655a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb648d77740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x561c1358f6b0 con 0x561c136298c0 2024-03-04T07:30:53.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x561c13538a60 con 0x561c13625870 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x561c1358b750 con 0x561c13655600 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c136298c0 0x561c1362b360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c136298c0 0x561c1362b360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48398/0 (socket says 172.21.15.64:48398) 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 -- 172.21.15.64:0/862852781 learned_addr learned my addr 172.21.15.64:0/862852781 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13625870 0x561c13625c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63ffff640 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c13655a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 -- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 msgr2=0x561c13655a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c13655a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 -- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13625870 msgr2=0x561c13625c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13625870 0x561c13625c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 -- 172.21.15.64:0/862852781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561c13556f30 con 0x561c136298c0 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13625870 0x561c13625c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.652+0000 7fb63effd640 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c136298c0 0x561c1362b360 secure :-1 s=READY pgs=1004 cs=0 l=1 rev1=1 crypto rx=0x7fb634001560 tx=0x7fb634000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2fb7dfac629ca19c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb6257fa640 1 -- 172.21.15.64:0/862852781 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb634019070 con 0x561c136298c0 2024-03-04T07:30:53.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb6257fa640 1 -- 172.21.15.64:0/862852781 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb634005ac0 con 0x561c136298c0 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb6257fa640 1 -- 172.21.15.64:0/862852781 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6340070d0 con 0x561c136298c0 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 -- 172.21.15.64:0/862852781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c136298c0 msgr2=0x561c1362b360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c136298c0 0x561c1362b360 secure :-1 s=READY pgs=1004 cs=0 l=1 rev1=1 crypto rx=0x7fb634001560 tx=0x7fb634000bc0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 -- 172.21.15.64:0/862852781 shutdown_connections 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c13655a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c136298c0 0x561c1362b360 unknown :-1 s=CLOSED pgs=1004 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 --2- 172.21.15.64:0/862852781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13625870 0x561c13625c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 -- 172.21.15.64:0/862852781 >> 172.21.15.64:0/862852781 conn(0x561c136278f0 msgr2=0x561c136291b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 -- 172.21.15.64:0/862852781 shutdown_connections 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.653+0000 7fb648d77740 1 -- 172.21.15.64:0/862852781 wait complete. 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.654+0000 7fb648d77740 1 -- start start 2024-03-04T07:30:53.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.654+0000 7fb648d77740 1 Processor -- start 2024-03-04T07:30:53.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c1364fd60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13650420 0x561c1366e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 0x561c13658bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x561c13629d50 con 0x561c13650420 2024-03-04T07:30:53.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x561c1354bea0 con 0x561c1366bc80 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x561c1362a030 con 0x561c13655600 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 10 client.0 _ll_register_callbacks cb 0x561c1354b870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13650420 0x561c1366e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13650420 0x561c1366e2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48410/0 (socket says 172.21.15.64:48410) 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 -- 172.21.15.64:0/934732833 learned_addr learned my addr 172.21.15.64:0/934732833 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c1364fd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 -- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 msgr2=0x561c1364fd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c1364fd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63ffff640 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 0x561c13658bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 -- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 msgr2=0x561c13658bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 0x561c13658bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 -- 172.21.15.64:0/934732833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x561c1354c270 con 0x561c13650420 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb63effd640 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13650420 0x561c1366e2f0 secure :-1 s=READY pgs=1005 cs=0 l=1 rev1=1 crypto rx=0x7fb634001eb0 tx=0x7fb634000ed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dea445a16b213a0c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb61e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb61e7fc640 1 -- 172.21.15.64:0/934732833 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb634019070 con 0x561c13650420 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb61e7fc640 1 -- 172.21.15.64:0/934732833 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb6340074e0 con 0x561c13650420 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.655+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb634005ac0 con 0x561c13650420 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb61e7fc640 1 -- 172.21.15.64:0/934732833 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=43) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb63402c050 con 0x561c13650420 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 10 client.7113 fetch_fsmap learned FSMap version 43 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=43,monmap=2+}) v3 -- 0x561c132a42d0 con 0x561c13650420 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb61e7fc640 1 -- 172.21.15.64:0/934732833 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb634005ac0 con 0x561c13650420 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb61e7fc640 1 -- 172.21.15.64:0/934732833 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 43) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fb634014480 con 0x561c13650420 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 10 client.7113 fetch_fsmap finished waiting for FSMap version 43 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 10 client.7113 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 20 client.7113 populate_metadata read hostname 'smithi064' 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136571c0 0x561c136575a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 4 client.7113 mds_command: new command op to 6966 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:30:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 --> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x561c1359cbf0 con 0x561c136571c0 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136571c0 0x561c136575a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb61e7fc640 10 client.7113 ms_handle_connect on v2:172.21.15.64:6835/1889124368 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.656+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136571c0 0x561c136575a0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9b8b2163284ffc5b server_cookie=e7dbdb5286cc5d50 in_seq=0 out_seq=0 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb61e7fc640 1 -- 172.21.15.64:0/934732833 <== mds.0 v2:172.21.15.64:6835/1889124368 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x561c1359cbf0 con 0x561c136571c0 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb61e7fc640 10 client.7113 handle_command_reply: tid=0 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 client.7113 shutdown 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13650420 msgr2=0x561c1366e2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13650420 0x561c1366e2f0 secure :-1 s=READY pgs=1005 cs=0 l=1 rev1=1 crypto rx=0x7fb634001eb0 tx=0x7fb634000ed0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 shutdown_connections 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136571c0 0x561c136575a0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 0x561c13658bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13650420 0x561c1366e2f0 unknown :-1 s=CLOSED pgs=1005 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 --2- 172.21.15.64:0/934732833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13655600 0x561c1364fd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb61e7fc640 0 client.7113 ms_handle_reset on v2:172.21.15.64:6835/1889124368 2024-03-04T07:30:53.663 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 >> 172.21.15.64:0/934732833 conn(0x561c13625870 msgr2=0x561c13624510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 shutdown_connections 2024-03-04T07:30:53.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 1 -- 172.21.15.64:0/934732833 wait complete. 2024-03-04T07:30:53.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.657+0000 7fb648d77740 20 client.7113 trim_cache size 0 max 16384 2024-03-04T07:30:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.674+0000 7fb648d77740 1 Processor -- start 2024-03-04T07:30:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.674+0000 7fb648d77740 1 -- start start 2024-03-04T07:30:53.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13650420 0x561c13658370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb648d77740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c13624510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c1366bc80 0x561c13624a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb648d77740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6401ff270 con 0x561c13655600 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6400cc8c0 con 0x561c13650420 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6400ccec0 con 0x561c1366bc80 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c13624510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c13624510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48412/0 (socket says 172.21.15.64:48412) 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63effd640 1 -- 172.21.15.64:0/1480981752 learned_addr learned my addr 172.21.15.64:0/1480981752 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63effd640 1 -- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c1366bc80 msgr2=0x561c13624a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63ffff640 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c1366bc80 0x561c13624a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13650420 0x561c13658370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63effd640 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c1366bc80 0x561c13624a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63effd640 1 -- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13650420 msgr2=0x561c13658370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.675+0000 7fb63effd640 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13650420 0x561c13658370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.676+0000 7fb63effd640 1 -- 172.21.15.64:0/1480981752 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561c1336e6b0 con 0x561c13655600 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.676+0000 7fb63ffff640 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c1366bc80 0x561c13624a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.676+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13650420 0x561c13658370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.676+0000 7fb63effd640 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c13624510 secure :-1 s=READY pgs=1006 cs=0 l=1 rev1=1 crypto rx=0x7fb63400e2c0 tx=0x7fb63401e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c4d0e8e9cb9c105 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.676+0000 7fb624ff9640 1 -- 172.21.15.64:0/1480981752 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb634014480 con 0x561c13655600 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.676+0000 7fb624ff9640 1 -- 172.21.15.64:0/1480981752 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb634024070 con 0x561c13655600 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.676+0000 7fb624ff9640 1 -- 172.21.15.64:0/1480981752 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6340074e0 con 0x561c13655600 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 -- 172.21.15.64:0/1480981752 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 msgr2=0x561c13624510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c13624510 secure :-1 s=READY pgs=1006 cs=0 l=1 rev1=1 crypto rx=0x7fb63400e2c0 tx=0x7fb63401e000 comp rx=0 tx=0).stop 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 -- 172.21.15.64:0/1480981752 shutdown_connections 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c1366bc80 0x561c13624a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c13624510 unknown :-1 s=CLOSED pgs=1006 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 --2- 172.21.15.64:0/1480981752 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c13650420 0x561c13658370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 -- 172.21.15.64:0/1480981752 >> 172.21.15.64:0/1480981752 conn(0x561c13625870 msgr2=0x561c1366fb80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 -- 172.21.15.64:0/1480981752 shutdown_connections 2024-03-04T07:30:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 -- 172.21.15.64:0/1480981752 wait complete. 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.677+0000 7fb648d77740 1 -- start start 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.678+0000 7fb648d77740 1 Processor -- start 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13650420 0x561c13654f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb648d77740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c1366c9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb648d77740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 0x561c13642f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb648d77740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x561c13650bc0 con 0x561c13655600 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x561c136707d0 con 0x561c1366bc80 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb648d77740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x561c13671360 con 0x561c13650420 2024-03-04T07:30:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb648d77740 10 client.0 _ll_register_callbacks cb 0x7fb634001400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c1366c9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c1366c9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48420/0 (socket says 172.21.15.64:48420) 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 -- 172.21.15.64:0/2853729985 learned_addr learned my addr 172.21.15.64:0/2853729985 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 -- 172.21.15.64:0/2853729985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13650420 msgr2=0x561c13654f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13650420 0x561c13654f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 -- 172.21.15.64:0/2853729985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 msgr2=0x561c13642f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 0x561c13642f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 -- 172.21.15.64:0/2853729985 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x561c1366cf30 con 0x561c13655600 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb63effd640 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c1366c9f0 secure :-1 s=READY pgs=1007 cs=0 l=1 rev1=1 crypto rx=0x7fb634000f60 tx=0x7fb634009040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ca28f240e3d0a9a5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.679+0000 7fb61e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb61e7fc640 1 -- 172.21.15.64:0/2853729985 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6340074e0 con 0x561c13655600 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb61e7fc640 1 -- 172.21.15.64:0/2853729985 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb634007680 con 0x561c13655600 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb634024070 con 0x561c13655600 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb61e7fc640 1 -- 172.21.15.64:0/2853729985 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=43) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb634040050 con 0x561c13655600 2024-03-04T07:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 10 client.7119 fetch_fsmap learned FSMap version 43 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=43,monmap=2+}) v3 -- 0x561c13653330 con 0x561c13655600 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb61e7fc640 1 -- 172.21.15.64:0/2853729985 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb634024070 con 0x561c13655600 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb61e7fc640 1 -- 172.21.15.64:0/2853729985 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 43) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fb634043020 con 0x561c13655600 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 10 client.7119 fetch_fsmap finished waiting for FSMap version 43 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 10 client.7119 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 20 client.7119 populate_metadata read hostname 'smithi064' 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136e2d00 0x561c136e30e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 4 client.7119 mds_command: new command op to 6966 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 --> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] -- command(tid 0: {"prefix": "status"}) v1 -- 0x561c13628050 con 0x561c136e2d00 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136e2d00 0x561c136e30e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb61e7fc640 10 client.7119 ms_handle_connect on v2:172.21.15.64:6835/1889124368 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.680+0000 7fb63f7fe640 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136e2d00 0x561c136e30e0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8989104ce4f875e1 server_cookie=7bd5a783a07cbf6d in_seq=0 out_seq=0 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb61e7fc640 1 -- 172.21.15.64:0/2853729985 <== mds.0 v2:172.21.15.64:6835/1889124368 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x561c13628050 con 0x561c136e2d00 2024-03-04T07:30:53.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb61e7fc640 10 client.7119 handle_command_reply: tid=0 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 client.7119 shutdown 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 msgr2=0x561c1366c9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c1366c9f0 secure :-1 s=READY pgs=1007 cs=0 l=1 rev1=1 crypto rx=0x7fb634000f60 tx=0x7fb634009040 comp rx=0 tx=0).stop 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 shutdown_connections 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:6835/1889124368,v1:172.21.15.64:6836/1889124368] conn(0x561c136e2d00 0x561c136e30e0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561c1366bc80 0x561c13642f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561c13655600 0x561c1366c9f0 unknown :-1 s=CLOSED pgs=1007 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb61e7fc640 0 client.7119 ms_handle_reset on v2:172.21.15.64:6835/1889124368 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 --2- 172.21.15.64:0/2853729985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561c13650420 0x561c13654f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 >> 172.21.15.64:0/2853729985 conn(0x561c13625870 msgr2=0x561c1366fb80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 shutdown_connections 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 1 -- 172.21.15.64:0/2853729985 wait complete. 2024-03-04T07:30:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb648d77740 20 client.7119 trim_cache size 0 max 16384 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 6966, 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 4.7002090760000002, 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 43, 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 79, 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 79, 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 9.7167679570000001 2024-03-04T07:30:53.686 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:30:53.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb62c06b4d0 msgr2=0x7fb62c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb646193640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb62c06b4d0 0x7fb62c06d990 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fb630009ba0 tx=0x7fb630008040 comp rx=0 tx=0).stop 2024-03-04T07:30:53.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb640151d50 msgr2=0x7fb6400ca260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.681+0000 7fb646193640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb640151d50 0x7fb6400ca260 secure :-1 s=READY pgs=1003 cs=0 l=1 rev1=1 crypto rx=0x7fb638009d90 tx=0x7fb638004a40 comp rx=0 tx=0).stop 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 shutdown_connections 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb62c06b4d0 0x7fb62c06d990 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb640151d50 0x7fb6400ca260 unknown :-1 s=CLOSED pgs=1003 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb64015df00 0x7fb6400c9d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 --2- 172.21.15.64:0/576563941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6400b95a0 0x7fb640151810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 >> 172.21.15.64:0/576563941 conn(0x7fb640071960 msgr2=0x7fb6400b6e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 shutdown_connections 2024-03-04T07:30:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.682+0000 7fb646193640 1 -- 172.21.15.64:0/576563941 wait complete. 2024-03-04T07:30:53.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:30:53.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.811+0000 7fab7a49d640 1 Processor -- start 2024-03-04T07:30:53.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.811+0000 7fab7a49d640 1 -- start start 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.811+0000 7fab7a49d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab740d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.811+0000 7fab7a49d640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fab740cea30 con 0x7fab740b95a0 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.811+0000 7fab7a49d640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fab740cf3c0 con 0x7fab7415df00 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.811+0000 7fab7a49d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab740cf0e0 con 0x7fab740b9a10 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.811+0000 7fab737fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fab740b95a0 0x7fab740b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35582/0 (socket says 172.21.15.64:35582) 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab737fe640 1 -- 172.21.15.64:0/1943197807 learned_addr learned my addr 172.21.15.64:0/1943197807 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab78a3a640 1 --2- 172.21.15.64:0/1943197807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab740d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab72ffd640 1 -- 172.21.15.64:0/1943197807 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2344376936 0 0) 0x7fab740cea30 con 0x7fab740b95a0 2024-03-04T07:30:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab72ffd640 1 -- 172.21.15.64:0/1943197807 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fab60003520 con 0x7fab740b95a0 2024-03-04T07:30:53.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab72ffd640 1 -- 172.21.15.64:0/1943197807 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1294333835 0 0) 0x7fab740cf3c0 con 0x7fab7415df00 2024-03-04T07:30:53.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab72ffd640 1 -- 172.21.15.64:0/1943197807 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fab740cea30 con 0x7fab7415df00 2024-03-04T07:30:53.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab78a3a640 1 -- 172.21.15.64:0/1943197807 >> v1:172.21.15.66:6789/0 conn(0x7fab7415df00 legacy=0x7fab740d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab78a3a640 1 -- 172.21.15.64:0/1943197807 >> v1:172.21.15.64:6789/0 conn(0x7fab740b95a0 legacy=0x7fab740b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.812+0000 7fab78a3a640 1 -- 172.21.15.64:0/1943197807 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab740ce470 con 0x7fab740b9a10 2024-03-04T07:30:53.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.813+0000 7fab78a3a640 1 --2- 172.21.15.64:0/1943197807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab740d3220 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fab6c009dc0 tx=0x7fab6c02bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c544ea91c9f79d65 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.813+0000 7fab72ffd640 1 -- 172.21.15.64:0/1943197807 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab6c039070 con 0x7fab740b9a10 2024-03-04T07:30:53.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.813+0000 7fab72ffd640 1 -- 172.21.15.64:0/1943197807 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab6c02ccd0 con 0x7fab740b9a10 2024-03-04T07:30:53.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.813+0000 7fab72ffd640 1 -- 172.21.15.64:0/1943197807 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab6c034640 con 0x7fab740b9a10 2024-03-04T07:30:53.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.814+0000 7fab7a49d640 1 -- 172.21.15.64:0/1943197807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 msgr2=0x7fab740d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.814+0000 7fab7a49d640 1 --2- 172.21.15.64:0/1943197807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab740d3220 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fab6c009dc0 tx=0x7fab6c02bf20 comp rx=0 tx=0).stop 2024-03-04T07:30:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.814+0000 7fab7a49d640 1 -- 172.21.15.64:0/1943197807 shutdown_connections 2024-03-04T07:30:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.814+0000 7fab7a49d640 1 --2- 172.21.15.64:0/1943197807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab740d3220 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.814+0000 7fab7a49d640 1 -- 172.21.15.64:0/1943197807 >> 172.21.15.64:0/1943197807 conn(0x7fab74071960 msgr2=0x7fab74071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.814+0000 7fab7a49d640 1 -- 172.21.15.64:0/1943197807 shutdown_connections 2024-03-04T07:30:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.814+0000 7fab7a49d640 1 -- 172.21.15.64:0/1943197807 wait complete. 2024-03-04T07:30:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.815+0000 7fab7a49d640 1 Processor -- start 2024-03-04T07:30:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.816+0000 7fab7a49d640 1 -- start start 2024-03-04T07:30:53.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab7a49d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab740b95a0 0x7fab7415f9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab7a49d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab741553c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab7a49d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab7415df00 0x7fab740b7dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab7a49d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab740cedf0 con 0x7fab7415df00 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab7a49d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab740cec70 con 0x7fab740b95a0 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab7a49d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab740b5e10 con 0x7fab740b9a10 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab7415df00 0x7fab740b7dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab7415df00 0x7fab740b7dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48430/0 (socket says 172.21.15.64:48430) 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 -- 172.21.15.64:0/976678795 learned_addr learned my addr 172.21.15.64:0/976678795 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab737fe640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab741553c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab73fff640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab740b95a0 0x7fab7415f9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 -- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 msgr2=0x7fab741553c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab741553c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 -- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab740b95a0 msgr2=0x7fab7415f9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab740b95a0 0x7fab7415f9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.817+0000 7fab78a3a640 1 -- 172.21.15.64:0/976678795 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab741f3c00 con 0x7fab7415df00 2024-03-04T07:30:53.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.818+0000 7fab78a3a640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab7415df00 0x7fab740b7dc0 secure :-1 s=READY pgs=1009 cs=0 l=1 rev1=1 crypto rx=0x7fab6c009ee0 tx=0x7fab6c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.818+0000 7fab70ff9640 1 -- 172.21.15.64:0/976678795 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab6c042020 con 0x7fab7415df00 2024-03-04T07:30:53.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.818+0000 7fab70ff9640 1 -- 172.21.15.64:0/976678795 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab6c03a040 con 0x7fab7415df00 2024-03-04T07:30:53.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.818+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab741f3ee0 con 0x7fab7415df00 2024-03-04T07:30:53.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.818+0000 7fab70ff9640 1 -- 172.21.15.64:0/976678795 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab6c039070 con 0x7fab7415df00 2024-03-04T07:30:53.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.818+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab741f44a0 con 0x7fab7415df00 2024-03-04T07:30:53.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.819+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab74067ff0 con 0x7fab7415df00 2024-03-04T07:30:53.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.820+0000 7fab70ff9640 1 -- 172.21.15.64:0/976678795 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fab6c045020 con 0x7fab7415df00 2024-03-04T07:30:53.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.820+0000 7fab70ff9640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5406b4d0 0x7fab5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:53.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.820+0000 7fab70ff9640 1 -- 172.21.15.64:0/976678795 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fab6c0ac330 con 0x7fab7415df00 2024-03-04T07:30:53.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.820+0000 7fab73fff640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5406b4d0 0x7fab5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:53.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.821+0000 7fab73fff640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5406b4d0 0x7fab5406d990 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fab5c007fd0 tx=0x7fab5c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:53.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:53.825+0000 7fab70ff9640 1 -- 172.21.15.64:0/976678795 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fab6c04c030 con 0x7fab7415df00 2024-03-04T07:30:54.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.045+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fab74071de0 con 0x7fab7415df00 2024-03-04T07:30:54.045 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.046+0000 7fab70ff9640 1 -- 172.21.15.64:0/976678795 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v43) v1 ==== 53+0+73 (secure 0 0 0) 0x7fab6c07aae0 con 0x7fab7415df00 2024-03-04T07:30:54.046 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5406b4d0 msgr2=0x7fab5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5406b4d0 0x7fab5406d990 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fab5c007fd0 tx=0x7fab5c009040 comp rx=0 tx=0).stop 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab7415df00 msgr2=0x7fab740b7dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab7415df00 0x7fab740b7dc0 secure :-1 s=READY pgs=1009 cs=0 l=1 rev1=1 crypto rx=0x7fab6c009ee0 tx=0x7fab6c004a40 comp rx=0 tx=0).stop 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 shutdown_connections 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5406b4d0 0x7fab5406d990 unknown :-1 s=CLOSED pgs=467 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab7415df00 0x7fab740b7dc0 unknown :-1 s=CLOSED pgs=1009 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab740b9a10 0x7fab741553c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:54.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 --2- 172.21.15.64:0/976678795 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab740b95a0 0x7fab7415f9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:54.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 >> 172.21.15.64:0/976678795 conn(0x7fab74071960 msgr2=0x7fab740d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:54.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 shutdown_connections 2024-03-04T07:30:54.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54.048+0000 7fab7a49d640 1 -- 172.21.15.64:0/976678795 wait complete. 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:30:54.059 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:30:54.059 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.060 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:30:54.074 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:30:54.074 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:54.075 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42458sec preferred_lft 42458sec 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:30:54.076 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:54.077 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:30:54.077 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:54.077 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:30:54.077 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:54.077 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:30:54.077 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:30:54.077 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.077 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:30:54.140 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:30:54.142 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.142 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:30:54.198 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:30:54.199 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:30:54.199 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:30:54.199 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:30:54.199 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:30:54.199 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.199 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:30:54.199 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:30:54.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:30:54.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:30:54.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:30:54.200 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:30:54.293 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.293 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:30:54.308 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:30:54.308 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:30:54.309 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:30:54.309 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:30:54.375 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:30:54.375 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:30:54.375 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:54.429 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:30:54.430 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:30:54.430 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:54.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:30:54.551 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:30:54.606 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:30:54.607 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:54.608 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:54.609 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.609 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:30:54.663 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:30:54.663 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:30:54.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:30:54.733 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:30:54.789 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:30:54.789 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:30:54.789 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:30:54.791 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:54.792 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:30:54.792 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:54.792 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:30:54.792 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.792 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:30:54.798 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:30:54.798+0000 7f6b8c244480 -1 init, newargv = 0x55a75653fb50 newargc=15 2024-03-04T07:30:54.798 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[65956]: starting ceph client 2024-03-04T07:30:54.802 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[65956]: starting fuse 2024-03-04T07:30:54.813 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:30:54.813 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:30:54.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:30:54.813 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:30:54.813 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:30:54.813 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:30:54.814 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:30:54.815 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:30:54.934 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.65956.asok 2024-03-04T07:30:54.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:54 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:30:54.944 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.65956.asok 2024-03-04T07:30:54.944 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:54.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.65956.asok status 2024-03-04T07:30:54.956 INFO:tasks.ceph.mds.c.smithi064.stderr:2024-03-04T07:30:54.957+0000 7f2671423640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:30:55.053 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:30:55.053 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "65956", 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 7131, 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:30:55.054 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 7131 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 379718771 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:30:55.055 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 379718771 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.7131 192.168.0.1:0/379718771", 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/379718771", 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 43, 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 79, 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:30:55.056 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:30:55.062 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:55.062 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:30:55.062 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:55.077 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:30:55.077 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:55.077 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:55.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:55.144 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:55.145 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec touch fancy_layout_file) 2024-03-04T07:30:55.205 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:55.205 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec setfattr -n ceph.file.layout -v 'stripe_unit=1048576 stripe_count=4 object_size=8388608' fancy_layout_file) 2024-03-04T07:30:55.271 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:55.271 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:55.327 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:55.327 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec dd if=/dev/urandom of=fancy_layout_file bs=1M conv=fdatasync count=35 seek=0) 2024-03-04T07:30:55.693 INFO:teuthology.orchestra.run.smithi064.stderr:35+0 records in 2024-03-04T07:30:55.693 INFO:teuthology.orchestra.run.smithi064.stderr:35+0 records out 2024-03-04T07:30:55.693 INFO:teuthology.orchestra.run.smithi064.stderr:36700160 bytes (37 MB, 35 MiB) copied, 0.308636 s, 119 MB/s 2024-03-04T07:30:55.694 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:55.694 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f fancy_layout_file) 2024-03-04T07:30:55.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:55.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.858+0000 7faa4ff13640 1 Processor -- start 2024-03-04T07:30:55.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.858+0000 7faa4ff13640 1 -- start start 2024-03-04T07:30:55.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4ff13640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7faa4815df00 0x7faa480d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4ff13640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faa480b95a0 0x7faa480b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4ff13640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7faa480cea30 con 0x7faa480b9a30 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4ff13640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faa480d2af0 con 0x7faa480b95a0 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4ff13640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7faa480cf3c0 con 0x7faa4815df00 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4d4ae640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faa480b95a0 0x7faa480b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4d4ae640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7faa480b95a0 0x7faa480b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48436/0 (socket says 172.21.15.64:48436) 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4d4ae640 1 -- 172.21.15.64:0/2602658290 learned_addr learned my addr 172.21.15.64:0/2602658290 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4dcaf640 1 --2- 172.21.15.64:0/2602658290 >> v2:172.21.15.66:3300/0 conn(0x7faa4815df00 0x7faa480d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4d4ae640 1 -- 172.21.15.64:0/2602658290 >> v2:172.21.15.66:3300/0 conn(0x7faa4815df00 msgr2=0x7faa480d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4d4ae640 1 --2- 172.21.15.64:0/2602658290 >> v2:172.21.15.66:3300/0 conn(0x7faa4815df00 0x7faa480d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4d4ae640 1 -- 172.21.15.64:0/2602658290 >> v1:172.21.15.66:6789/0 conn(0x7faa480b9a30 legacy=0x7faa480d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.859+0000 7faa4d4ae640 1 -- 172.21.15.64:0/2602658290 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa480cf0e0 con 0x7faa480b95a0 2024-03-04T07:30:55.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4d4ae640 1 --2- 172.21.15.64:0/2602658290 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa480b5700 secure :-1 s=READY pgs=1012 cs=0 l=1 rev1=1 crypto rx=0x7faa480d11b0 tx=0x7faa3802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84185897246133b0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:55.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ccad640 1 -- 172.21.15.64:0/2602658290 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa38035020 con 0x7faa480b95a0 2024-03-04T07:30:55.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ccad640 1 -- 172.21.15.64:0/2602658290 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faa38031870 con 0x7faa480b95a0 2024-03-04T07:30:55.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ccad640 1 -- 172.21.15.64:0/2602658290 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa38031c40 con 0x7faa480b95a0 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ff13640 1 -- 172.21.15.64:0/2602658290 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 msgr2=0x7faa480b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ff13640 1 --2- 172.21.15.64:0/2602658290 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa480b5700 secure :-1 s=READY pgs=1012 cs=0 l=1 rev1=1 crypto rx=0x7faa480d11b0 tx=0x7faa3802b910 comp rx=0 tx=0).stop 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ff13640 1 -- 172.21.15.64:0/2602658290 shutdown_connections 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ff13640 1 --2- 172.21.15.64:0/2602658290 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa480b5700 unknown :-1 s=CLOSED pgs=1012 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ff13640 1 --2- 172.21.15.64:0/2602658290 >> v2:172.21.15.66:3300/0 conn(0x7faa4815df00 0x7faa480d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.860+0000 7faa4ff13640 1 -- 172.21.15.64:0/2602658290 >> 172.21.15.64:0/2602658290 conn(0x7faa48071960 msgr2=0x7faa48071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.861+0000 7faa4ff13640 1 -- 172.21.15.64:0/2602658290 shutdown_connections 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.861+0000 7faa4ff13640 1 -- 172.21.15.64:0/2602658290 wait complete. 2024-03-04T07:30:55.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.862+0000 7faa4ff13640 1 Processor -- start 2024-03-04T07:30:55.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.862+0000 7faa4ff13640 1 -- start start 2024-03-04T07:30:55.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa48160520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:55.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faa4815df00 0x7faa480cd470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:55.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faa480c8bf0 0x7faa480c9070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:55.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa480cec70 con 0x7faa480b95a0 2024-03-04T07:30:55.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa480ce750 con 0x7faa4815df00 2024-03-04T07:30:55.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7faa480cedf0 con 0x7faa480c8bf0 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa48160520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa48160520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48448/0 (socket says 172.21.15.64:48448) 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 -- 172.21.15.64:0/3719175626 learned_addr learned my addr 172.21.15.64:0/3719175626 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4e4b0640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faa480c8bf0 0x7faa480c9070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4d4ae640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faa4815df00 0x7faa480cd470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 -- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faa480c8bf0 msgr2=0x7faa480c9070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faa480c8bf0 0x7faa480c9070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 -- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faa4815df00 msgr2=0x7faa480cd470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faa4815df00 0x7faa480cd470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 -- 172.21.15.64:0/3719175626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa480c7b50 con 0x7faa480b95a0 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4d4ae640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faa4815df00 0x7faa480cd470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4dcaf640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa48160520 secure :-1 s=READY pgs=1013 cs=0 l=1 rev1=1 crypto rx=0x7faa4400c7f0 tx=0x7faa4400ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa3e7fc640 1 -- 172.21.15.64:0/3719175626 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa44015070 con 0x7faa480b95a0 2024-03-04T07:30:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa3e7fc640 1 -- 172.21.15.64:0/3719175626 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7faa44004220 con 0x7faa480b95a0 2024-03-04T07:30:55.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faa480c7e30 con 0x7faa480b95a0 2024-03-04T07:30:55.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa3e7fc640 1 -- 172.21.15.64:0/3719175626 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa44007620 con 0x7faa480b95a0 2024-03-04T07:30:55.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.864+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faa480c83f0 con 0x7faa480b95a0 2024-03-04T07:30:55.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.865+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faa48067ff0 con 0x7faa480b95a0 2024-03-04T07:30:55.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.866+0000 7faa3e7fc640 1 -- 172.21.15.64:0/3719175626 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7faa440079d0 con 0x7faa480b95a0 2024-03-04T07:30:55.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.866+0000 7faa3e7fc640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faa2c06b4d0 0x7faa2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:55.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.867+0000 7faa4d4ae640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faa2c06b4d0 0x7faa2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:55.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.867+0000 7faa3e7fc640 1 -- 172.21.15.64:0/3719175626 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7faa4408a100 con 0x7faa480b95a0 2024-03-04T07:30:55.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.867+0000 7faa4d4ae640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faa2c06b4d0 0x7faa2c06d990 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7faa38036000 tx=0x7faa3803c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:55.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:55.871+0000 7faa3e7fc640 1 -- 172.21.15.64:0/3719175626 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7faa44058930 con 0x7faa480b95a0 2024-03-04T07:30:56.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.093+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7faa4815d400 con 0x7faa480b95a0 2024-03-04T07:30:56.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.094+0000 7faa3e7fc640 1 -- 172.21.15.64:0/3719175626 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 43 v43) v1 ==== 94+0+4973 (secure 0 0 0) 0x7faa4405c9d0 con 0x7faa480b95a0 2024-03-04T07:30:56.094 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:56.094 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[{"mdsmap":{"epoch":43,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:48.344908+0000","modified":"2024-03-04T07:30:50.410439+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6966},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6966":{"gid":6966,"name":"c","rank":0,"incarnation":41,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/1889124368","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1889124368},{"type":"v1","addr":"172.21.15.64:6836","nonce":1889124368}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6966,"qdb_cluster":[6966]},"id":6}]} 2024-03-04T07:30:56.094 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 43 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faa2c06b4d0 msgr2=0x7faa2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faa2c06b4d0 0x7faa2c06d990 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7faa38036000 tx=0x7faa3803c040 comp rx=0 tx=0).stop 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 msgr2=0x7faa48160520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa48160520 secure :-1 s=READY pgs=1013 cs=0 l=1 rev1=1 crypto rx=0x7faa4400c7f0 tx=0x7faa4400ccb0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 shutdown_connections 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7faa2c06b4d0 0x7faa2c06d990 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7faa480c8bf0 0x7faa480c9070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7faa4815df00 0x7faa480cd470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 --2- 172.21.15.64:0/3719175626 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7faa480b95a0 0x7faa48160520 unknown :-1 s=CLOSED pgs=1013 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 >> 172.21.15.64:0/3719175626 conn(0x7faa48071960 msgr2=0x7faa481618b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:56.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 shutdown_connections 2024-03-04T07:30:56.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.096+0000 7faa4ff13640 1 -- 172.21.15.64:0/3719175626 wait complete. 2024-03-04T07:30:56.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-04T07:30:56.237 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:30:56.237 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:30:56.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph df --format=json-pretty 2024-03-04T07:30:56.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.338+0000 7f9f509ee640 1 Processor -- start 2024-03-04T07:30:56.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.338+0000 7f9f509ee640 1 -- start start 2024-03-04T07:30:56.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f509ee640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9f4c15df00 0x7f9f4c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f509ee640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9f4c0b95a0 0x7f9f4c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f509ee640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9f4c0cea30 con 0x7f9f4c0b9a30 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f509ee640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f4c0d2af0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f509ee640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9f4c0cf3c0 con 0x7f9f4c15df00 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f43fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9f4c0b95a0 0x7f9f4c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f43fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9f4c0b95a0 0x7f9f4c0b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48458/0 (socket says 172.21.15.64:48458) 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f43fff640 1 -- 172.21.15.64:0/4153308466 learned_addr learned my addr 172.21.15.64:0/4153308466 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f4a59c640 1 --2- 172.21.15.64:0/4153308466 >> v2:172.21.15.66:3300/0 conn(0x7f9f4c15df00 0x7f9f4c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f43fff640 1 -- 172.21.15.64:0/4153308466 >> v2:172.21.15.66:3300/0 conn(0x7f9f4c15df00 msgr2=0x7f9f4c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f43fff640 1 --2- 172.21.15.64:0/4153308466 >> v2:172.21.15.66:3300/0 conn(0x7f9f4c15df00 0x7f9f4c0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f43fff640 1 -- 172.21.15.64:0/4153308466 >> v1:172.21.15.66:6789/0 conn(0x7f9f4c0b9a30 legacy=0x7f9f4c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.339+0000 7f9f43fff640 1 -- 172.21.15.64:0/4153308466 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f4c0cf0e0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.340+0000 7f9f43fff640 1 --2- 172.21.15.64:0/4153308466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c0b5700 secure :-1 s=READY pgs=1014 cs=0 l=1 rev1=1 crypto rx=0x7f9f4c0d11b0 tx=0x7f9f3402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d520e0e7ee9b262b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:56.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.340+0000 7f9f49d9b640 1 -- 172.21.15.64:0/4153308466 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f34034070 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.340+0000 7f9f49d9b640 1 -- 172.21.15.64:0/4153308466 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f34030860 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.340+0000 7f9f49d9b640 1 -- 172.21.15.64:0/4153308466 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f34030c30 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 -- 172.21.15.64:0/4153308466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 msgr2=0x7f9f4c0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 --2- 172.21.15.64:0/4153308466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c0b5700 secure :-1 s=READY pgs=1014 cs=0 l=1 rev1=1 crypto rx=0x7f9f4c0d11b0 tx=0x7f9f3402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 -- 172.21.15.64:0/4153308466 shutdown_connections 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 --2- 172.21.15.64:0/4153308466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c0b5700 unknown :-1 s=CLOSED pgs=1014 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 --2- 172.21.15.64:0/4153308466 >> v2:172.21.15.66:3300/0 conn(0x7f9f4c15df00 0x7f9f4c0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 -- 172.21.15.64:0/4153308466 >> 172.21.15.64:0/4153308466 conn(0x7f9f4c071960 msgr2=0x7f9f4c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 -- 172.21.15.64:0/4153308466 shutdown_connections 2024-03-04T07:30:56.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.341+0000 7f9f509ee640 1 -- 172.21.15.64:0/4153308466 wait complete. 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.342+0000 7f9f509ee640 1 Processor -- start 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.342+0000 7f9f509ee640 1 -- start start 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f509ee640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c1553e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f509ee640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f4c15df00 0x7f9f4c0cb640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f509ee640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f4c0cc960 0x7f9f4c1f3930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f509ee640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f4c0cec70 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f509ee640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f4c0ce750 con 0x7f9f4c15df00 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f509ee640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9f4c0cedf0 con 0x7f9f4c0cc960 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c1553e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c1553e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48464/0 (socket says 172.21.15.64:48464) 2024-03-04T07:30:56.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 -- 172.21.15.64:0/725961036 learned_addr learned my addr 172.21.15.64:0/725961036 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f43fff640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f4c15df00 0x7f9f4c0cb640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 -- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f4c0cc960 msgr2=0x7f9f4c1f3930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4ad9d640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f4c0cc960 0x7f9f4c1f3930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f4c0cc960 0x7f9f4c1f3930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 -- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f4c15df00 msgr2=0x7f9f4c0cb640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f4c15df00 0x7f9f4c0cb640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4a59c640 1 -- 172.21.15.64:0/725961036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f4c0cae80 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f43fff640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f4c15df00 0x7f9f4c0cb640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.343+0000 7f9f4ad9d640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f4c0cc960 0x7f9f4c1f3930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.344+0000 7f9f4a59c640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c1553e0 secure :-1 s=READY pgs=1015 cs=0 l=1 rev1=1 crypto rx=0x7f9f3c00d5f0 tx=0x7f9f3c00dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.344+0000 7f9f42ffd640 1 -- 172.21.15.64:0/725961036 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f3c0125f0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.344+0000 7f9f42ffd640 1 -- 172.21.15.64:0/725961036 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9f3c019070 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.344+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f4c1f48a0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.344+0000 7f9f42ffd640 1 -- 172.21.15.64:0/725961036 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f3c0129d0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.344+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f4c1f45c0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.345+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f4c067ff0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.345+0000 7f9f42ffd640 1 -- 172.21.15.64:0/725961036 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9f3c012b70 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.346+0000 7f9f42ffd640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f2c06b4d0 0x7f9f2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.346+0000 7f9f42ffd640 1 -- 172.21.15.64:0/725961036 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9f3c08a580 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.346+0000 7f9f43fff640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f2c06b4d0 0x7f9f2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.346+0000 7f9f43fff640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f2c06b4d0 0x7f9f2c06d990 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f9f34036000 tx=0x7f9f3403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:56.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.350+0000 7f9f42ffd640 1 -- 172.21.15.64:0/725961036 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9f3c059490 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.622+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "df", "format": "json-pretty"} v 0) v1 -- 0x7f9f4c0b7cf0 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.623+0000 7f9f42ffd640 1 -- 172.21.15.64:0/725961036 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "df", "format": "json-pretty"}]=0 v0) v1 ==== 75+0+1653 (secure 0 0 0) 0x7f9f3c058d90 con 0x7f9f4c0b95a0 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "total_bytes": 773094113280, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "total_avail_bytes": 772829646848, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_bytes": 264466432, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_bytes": 264466432, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_ratio": 0.00034208828583359718, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "num_osds": 8, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "num_per_pool_osds": 8, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "num_per_pool_omap_osds": 8 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "stats_by_class": { 2024-03-04T07:30:56.623 INFO:teuthology.orchestra.run.smithi064.stdout: "ssd": { 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "total_bytes": 773094113280, 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "total_avail_bytes": 772829646848, 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_bytes": 264466432, 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_bytes": 264466432, 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "total_used_raw_ratio": 0.00034208828583359718 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "pools": [ 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "name": ".mgr", 2024-03-04T07:30:56.624 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 1, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 590368, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 2, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 1160, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 1187840, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 1.6179301383090205e-06, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367085715456 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "cephfs_metadata", 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 12, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 2410, 2024-03-04T07:30:56.625 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 22, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 64, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 65536, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 8.9265249414438586e-08, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367085715456 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "cephfs_data", 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 13, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "stats": { 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "stored": 0, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "objects": 0, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "kb_used": 0, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "bytes_used": 0, 2024-03-04T07:30:56.626 INFO:teuthology.orchestra.run.smithi064.stdout: "percent_used": 0, 2024-03-04T07:30:56.627 INFO:teuthology.orchestra.run.smithi064.stdout: "max_avail": 367085715456 2024-03-04T07:30:56.627 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:56.627 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:30:56.627 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:30:56.627 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f2c06b4d0 msgr2=0x7f9f2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f2c06b4d0 0x7f9f2c06d990 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f9f34036000 tx=0x7f9f3403b040 comp rx=0 tx=0).stop 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 msgr2=0x7f9f4c1553e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c1553e0 secure :-1 s=READY pgs=1015 cs=0 l=1 rev1=1 crypto rx=0x7f9f3c00d5f0 tx=0x7f9f3c00dab0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 shutdown_connections 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9f2c06b4d0 0x7f9f2c06d990 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9f4c0cc960 0x7f9f4c1f3930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9f4c15df00 0x7f9f4c0cb640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 --2- 172.21.15.64:0/725961036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9f4c0b95a0 0x7f9f4c1553e0 unknown :-1 s=CLOSED pgs=1015 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.624+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 >> 172.21.15.64:0/725961036 conn(0x7f9f4c071960 msgr2=0x7f9f4c0d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.625+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 shutdown_connections 2024-03-04T07:30:56.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.625+0000 7f9f509ee640 1 -- 172.21.15.64:0/725961036 wait complete. 2024-03-04T07:30:56.635 DEBUG:tasks.ceph_test_case:wait_until_true: success in 0s and 0 retries 2024-03-04T07:30:56.635 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:30:56.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:30:56.679 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:30:56.727 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:30:56.727 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:56.727 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:56.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:56.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.897+0000 7f049adb5640 1 Processor -- start 2024-03-04T07:30:56.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.897+0000 7f049adb5640 1 -- start start 2024-03-04T07:30:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f049adb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 0x7f049406d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f049adb5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f04940aee10 0x7f04940b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f049adb5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0494071240 con 0x7f049406dd60 2024-03-04T07:30:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f049adb5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f049406e1c0 con 0x7f04940aee10 2024-03-04T07:30:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f049adb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0494071b70 con 0x7f0494158520 2024-03-04T07:30:56.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f0493fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f049406dd60 0x7f04940ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35592/0 (socket says 172.21.15.64:35592) 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f0493fff640 1 -- 172.21.15.64:0/199421649 learned_addr learned my addr 172.21.15.64:0/199421649 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 801034265 0 0) 0x7f0494071240 con 0x7f049406dd60 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0484003520 con 0x7f049406dd60 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f0498b51640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 0x7f049406d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f0499352640 1 --2- 172.21.15.64:0/199421649 >> v2:172.21.15.66:3300/0 conn(0x7f04940aee10 0x7f04940b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 999075953 0 0) 0x7f0484003520 con 0x7f049406dd60 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 msgr2=0x7f049406d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 0x7f049406d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 >> v2:172.21.15.66:3300/0 conn(0x7f04940aee10 msgr2=0x7f04940b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 --2- 172.21.15.64:0/199421649 >> v2:172.21.15.66:3300/0 conn(0x7f04940aee10 0x7f04940b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0494071520 con 0x7f049406dd60 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f0480001670 con 0x7f049406dd60 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f0498b51640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 0x7f049406d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 >> v1:172.21.15.64:6789/0 conn(0x7f049406dd60 legacy=0x7f04940ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 0x7f04840039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.898+0000 7f04937fe640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0484003fa0 0x7f0484004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f04937fe640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484004d40 0x7f04840051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0494071b70 con 0x7f0484003fa0 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f048400ba20 con 0x7f0484004d40 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f048400bc20 con 0x7f0494158520 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0499352640 1 --2- 172.21.15.64:0/199421649 >> v2:172.21.15.66:3300/0 conn(0x7f04940aee10 0x7f04940b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0493fff640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0484003fa0 0x7f0484004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0498b51640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 0x7f04840039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0499352640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484004d40 0x7f04840051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0493fff640 1 -- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 msgr2=0x7f04840039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0493fff640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494158520 0x7f04840039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0493fff640 1 -- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484004d40 msgr2=0x7f04840051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0493fff640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484004d40 0x7f04840051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0493fff640 1 -- 172.21.15.64:0/199421649 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0484003520 con 0x7f0484003fa0 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.899+0000 7f0499352640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484004d40 0x7f04840051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f0493fff640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0484003fa0 0x7f0484004800 secure :-1 s=READY pgs=1017 cs=0 l=1 rev1=1 crypto rx=0x7f048000c240 tx=0x7f0484005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4d06b6dab3840044 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0480007d30 con 0x7f0484003fa0 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0480030070 con 0x7f0484003fa0 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f04937fe640 1 -- 172.21.15.64:0/199421649 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f048002cda0 con 0x7f0484003fa0 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f049adb5640 1 -- 172.21.15.64:0/199421649 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0484003fa0 msgr2=0x7f0484004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f049adb5640 1 --2- 172.21.15.64:0/199421649 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0484003fa0 0x7f0484004800 secure :-1 s=READY pgs=1017 cs=0 l=1 rev1=1 crypto rx=0x7f048000c240 tx=0x7f0484005830 comp rx=0 tx=0).stop 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f0499352640 1 -- 172.21.15.64:0/199421649 reap_dead start 2024-03-04T07:30:56.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f049adb5640 1 -- 172.21.15.64:0/199421649 shutdown_connections 2024-03-04T07:30:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f049adb5640 1 -- 172.21.15.64:0/199421649 >> 172.21.15.64:0/199421649 conn(0x7f0494066d00 msgr2=0x7f04940670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f049adb5640 1 -- 172.21.15.64:0/199421649 shutdown_connections 2024-03-04T07:30:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.900+0000 7f049adb5640 1 -- 172.21.15.64:0/199421649 wait complete. 2024-03-04T07:30:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 Processor -- start 2024-03-04T07:30:56.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 -- start start 2024-03-04T07:30:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484003fa0 0x7f04941542f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494155970 0x7f049415d550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04941482b0 0x7f0494156be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0494158900 con 0x7f04941482b0 2024-03-04T07:30:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0494072000 con 0x7f0484003fa0 2024-03-04T07:30:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.902+0000 7f049adb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0494158a80 con 0x7f0494155970 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04941482b0 0x7f0494156be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04941482b0 0x7f0494156be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48478/0 (socket says 172.21.15.64:48478) 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 -- 172.21.15.64:0/3413864432 learned_addr learned my addr 172.21.15.64:0/3413864432 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0498b51640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484003fa0 0x7f04941542f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0493fff640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494155970 0x7f049415d550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 -- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494155970 msgr2=0x7f049415d550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494155970 0x7f049415d550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 -- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484003fa0 msgr2=0x7f04941542f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484003fa0 0x7f04941542f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 -- 172.21.15.64:0/3413864432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04941548d0 con 0x7f04941482b0 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f0499352640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04941482b0 0x7f0494156be0 secure :-1 s=READY pgs=1018 cs=0 l=1 rev1=1 crypto rx=0x7f048c009fd0 tx=0x7f048c007700 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f04917fa640 1 -- 172.21.15.64:0/3413864432 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f048c014410 con 0x7f04941482b0 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f04917fa640 1 -- 172.21.15.64:0/3413864432 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f048c019070 con 0x7f04941482b0 2024-03-04T07:30:56.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0494154bb0 con 0x7f04941482b0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f04917fa640 1 -- 172.21.15.64:0/3413864432 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f048c0147f0 con 0x7f04941482b0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.903+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0494155170 con 0x7f04941482b0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.904+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0494067ff0 con 0x7f04941482b0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.905+0000 7f04917fa640 1 -- 172.21.15.64:0/3413864432 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f048c014990 con 0x7f04941482b0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.906+0000 7f04917fa640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f047806b4d0 0x7f047806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.906+0000 7f04917fa640 1 -- 172.21.15.64:0/3413864432 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f048c08a770 con 0x7f04941482b0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.906+0000 7f0498b51640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f047806b4d0 0x7f047806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.906+0000 7f0498b51640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f047806b4d0 0x7f047806d990 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f0488006fa0 tx=0x7f0488008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:56.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:56.910+0000 7f04917fa640 1 -- 172.21.15.64:0/3413864432 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f048c058fa0 con 0x7f04941482b0 2024-03-04T07:30:57.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.139+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f049406cbe0 con 0x7f04941482b0 2024-03-04T07:30:57.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.140+0000 7f04917fa640 1 -- 172.21.15.64:0/3413864432 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v79) v1 ==== 74+0+16612 (secure 0 0 0) 0x7f048c05d040 con 0x7f04941482b0 2024-03-04T07:30:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:57.140 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":79,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:49.980908+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:46.621410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:47.317858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"79","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:57.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f047806b4d0 msgr2=0x7f047806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f047806b4d0 0x7f047806d990 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f0488006fa0 tx=0x7f0488008040 comp rx=0 tx=0).stop 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04941482b0 msgr2=0x7f0494156be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04941482b0 0x7f0494156be0 secure :-1 s=READY pgs=1018 cs=0 l=1 rev1=1 crypto rx=0x7f048c009fd0 tx=0x7f048c007700 comp rx=0 tx=0).stop 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 shutdown_connections 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f047806b4d0 0x7f047806d990 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04941482b0 0x7f0494156be0 unknown :-1 s=CLOSED pgs=1018 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0494155970 0x7f049415d550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 --2- 172.21.15.64:0/3413864432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0484003fa0 0x7f04941542f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 >> 172.21.15.64:0/3413864432 conn(0x7f0494066d00 msgr2=0x7f049406d3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 shutdown_connections 2024-03-04T07:30:57.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.143+0000 7f049adb5640 1 -- 172.21.15.64:0/3413864432 wait complete. 2024-03-04T07:30:57.154 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 12, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:30:46.621410+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '78', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2, 'score_stable': 2, 'optimal_score': 1, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 13, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:30:47.317858+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '79', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:30:57.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:30:57.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.260+0000 7f6267466640 1 Processor -- start 2024-03-04T07:30:57.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.260+0000 7f6267466640 1 -- start start 2024-03-04T07:30:57.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6267466640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6260071540 0x7f6260071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6267466640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62600b2d30 0x7f62600ce770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6267466640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6260071e60 con 0x7f62600cecb0 2024-03-04T07:30:57.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6267466640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62600d3230 con 0x7f62600b2d30 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6267466640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62600d4960 con 0x7f6260071540 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6264a01640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62600b2d30 0x7f62600ce770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6264a01640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62600b2d30 0x7f62600ce770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48488/0 (socket says 172.21.15.64:48488) 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6264a01640 1 -- 172.21.15.64:0/2721828746 learned_addr learned my addr 172.21.15.64:0/2721828746 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6265202640 1 --2- 172.21.15.64:0/2721828746 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6260071540 0x7f6260071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6264a01640 1 -- 172.21.15.64:0/2721828746 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6260071540 msgr2=0x7f6260071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6264a01640 1 --2- 172.21.15.64:0/2721828746 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6260071540 0x7f6260071920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6264a01640 1 -- 172.21.15.64:0/2721828746 >> v1:172.21.15.66:6789/0 conn(0x7f62600cecb0 legacy=0x7f62600b72e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.261+0000 7f6264a01640 1 -- 172.21.15.64:0/2721828746 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62600d36c0 con 0x7f62600b2d30 2024-03-04T07:30:57.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.262+0000 7f6264a01640 1 --2- 172.21.15.64:0/2721828746 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600b2d30 0x7f62600ce770 secure :-1 s=READY pgs=1019 cs=0 l=1 rev1=1 crypto rx=0x7f6250009870 tx=0x7f625002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=11bb47ea75f01d20 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.262+0000 7f6257fff640 1 -- 172.21.15.64:0/2721828746 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6250034070 con 0x7f62600b2d30 2024-03-04T07:30:57.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.262+0000 7f6257fff640 1 -- 172.21.15.64:0/2721828746 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6250030840 con 0x7f62600b2d30 2024-03-04T07:30:57.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.262+0000 7f6257fff640 1 -- 172.21.15.64:0/2721828746 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6250030c10 con 0x7f62600b2d30 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 -- 172.21.15.64:0/2721828746 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600b2d30 msgr2=0x7f62600ce770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 --2- 172.21.15.64:0/2721828746 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600b2d30 0x7f62600ce770 secure :-1 s=READY pgs=1019 cs=0 l=1 rev1=1 crypto rx=0x7f6250009870 tx=0x7f625002b980 comp rx=0 tx=0).stop 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 -- 172.21.15.64:0/2721828746 shutdown_connections 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 --2- 172.21.15.64:0/2721828746 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600b2d30 0x7f62600ce770 unknown :-1 s=CLOSED pgs=1019 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 --2- 172.21.15.64:0/2721828746 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6260071540 0x7f6260071920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 -- 172.21.15.64:0/2721828746 >> 172.21.15.64:0/2721828746 conn(0x7f62600d3d60 msgr2=0x7f62600d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 -- 172.21.15.64:0/2721828746 shutdown_connections 2024-03-04T07:30:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.263+0000 7f6267466640 1 -- 172.21.15.64:0/2721828746 wait complete. 2024-03-04T07:30:57.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.264+0000 7f6267466640 1 Processor -- start 2024-03-04T07:30:57.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.264+0000 7f6267466640 1 -- start start 2024-03-04T07:30:57.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6267466640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6260071540 0x7f62601f3b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6267466640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62600b2d30 0x7f62600c92d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6267466640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600cecb0 0x7f62600c9810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6267466640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62600d2be0 con 0x7f62600cecb0 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6267466640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62600d2900 con 0x7f6260071540 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6267466640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62600d2ec0 con 0x7f62600b2d30 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6265a03640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600cecb0 0x7f62600c9810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6265a03640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600cecb0 0x7f62600c9810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48500/0 (socket says 172.21.15.64:48500) 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6265a03640 1 -- 172.21.15.64:0/674776501 learned_addr learned my addr 172.21.15.64:0/674776501 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6265202640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6260071540 0x7f62601f3b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.265+0000 7f6264a01640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62600b2d30 0x7f62600c92d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6265a03640 1 -- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62600b2d30 msgr2=0x7f62600c92d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6265a03640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62600b2d30 0x7f62600c92d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6265a03640 1 -- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6260071540 msgr2=0x7f62601f3b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6265a03640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6260071540 0x7f62601f3b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6265a03640 1 -- 172.21.15.64:0/674776501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62600ab390 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6265a03640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600cecb0 0x7f62600c9810 secure :-1 s=READY pgs=1020 cs=0 l=1 rev1=1 crypto rx=0x7f625800c7a0 tx=0x7f625800cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6255ffb640 1 -- 172.21.15.64:0/674776501 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6258003b70 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6255ffb640 1 -- 172.21.15.64:0/674776501 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6258003d30 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f626014bdf0 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6255ffb640 1 -- 172.21.15.64:0/674776501 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f625800aa90 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.266+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f626014c3b0 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.267+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f62600b2980 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.268+0000 7f6255ffb640 1 -- 172.21.15.64:0/674776501 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f625800ac30 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.268+0000 7f6255ffb640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f624406b4d0 0x7f624406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.268+0000 7f6255ffb640 1 -- 172.21.15.64:0/674776501 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f625808a000 con 0x7f62600cecb0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.268+0000 7f6265202640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f624406b4d0 0x7f624406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.269+0000 7f6265202640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f624406b4d0 0x7f624406d990 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f625c0079a0 tx=0x7f625c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.273+0000 7f6255ffb640 1 -- 172.21.15.64:0/674776501 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6258058830 con 0x7f62600cecb0 2024-03-04T07:30:57.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.507+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f626004f410 con 0x7f62600cecb0 2024-03-04T07:30:57.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.507+0000 7f6255ffb640 1 -- 172.21.15.64:0/674776501 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v79) v1 ==== 93+0+10 (secure 0 0 0) 0x7f625805c8d0 con 0x7f62600cecb0 2024-03-04T07:30:57.507 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f624406b4d0 msgr2=0x7f624406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f624406b4d0 0x7f624406d990 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f625c0079a0 tx=0x7f625c008040 comp rx=0 tx=0).stop 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600cecb0 msgr2=0x7f62600c9810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600cecb0 0x7f62600c9810 secure :-1 s=READY pgs=1020 cs=0 l=1 rev1=1 crypto rx=0x7f625800c7a0 tx=0x7f625800cc60 comp rx=0 tx=0).stop 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 shutdown_connections 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f624406b4d0 0x7f624406d990 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62600cecb0 0x7f62600c9810 unknown :-1 s=CLOSED pgs=1020 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62600b2d30 0x7f62600c92d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 --2- 172.21.15.64:0/674776501 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6260071540 0x7f62601f3b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 >> 172.21.15.64:0/674776501 conn(0x7f62600d3d60 msgr2=0x7f626015b900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:57.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 shutdown_connections 2024-03-04T07:30:57.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.509+0000 7f6267466640 1 -- 172.21.15.64:0/674776501 wait complete. 2024-03-04T07:30:57.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:30:57.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.625+0000 7f1f8e888640 1 Processor -- start 2024-03-04T07:30:57.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.625+0000 7f1f8e888640 1 -- start start 2024-03-04T07:30:57.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f8e888640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1f880ceea0 0x7f1f880d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f8e888640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1f880d3740 con 0x7f1f880b6d10 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f8e888640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1f880d3120 con 0x7f1f8815dd90 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f8e888640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1f880d3400 con 0x7f1f880ceea0 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f8ce25640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1f880b6d10 0x7f1f880d0c30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35598/0 (socket says 172.21.15.64:35598) 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f8ce25640 1 -- 172.21.15.64:0/1074806902 learned_addr learned my addr 172.21.15.64:0/1074806902 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f877fe640 1 --2- 172.21.15.64:0/1074806902 >> v2:172.21.15.66:3300/0 conn(0x7f1f880ceea0 0x7f1f880d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3666771305 0 0) 0x7f1f880d3740 con 0x7f1f880b6d10 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.626+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1f70003520 con 0x7f1f880b6d10 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1162639201 0 0) 0x7f1f70003520 con 0x7f1f880b6d10 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 >> v2:172.21.15.66:3300/0 conn(0x7f1f880ceea0 msgr2=0x7f1f880d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 --2- 172.21.15.64:0/1074806902 >> v2:172.21.15.66:3300/0 conn(0x7f1f880ceea0 0x7f1f880d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 >> v1:172.21.15.66:6789/0 conn(0x7f1f8815dd90 legacy=0x7f1f880d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f880d39e0 con 0x7f1f880b6d10 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f1f800019f0 con 0x7f1f880b6d10 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 >> v1:172.21.15.64:6789/0 conn(0x7f1f880b6d10 legacy=0x7f1f880d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f700039b0 0x7f1f70003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f700043b0 0x7f1f70004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f7000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f880d3400 con 0x7f1f70005080 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f7000bf30 con 0x7f1f700043b0 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f7000c130 con 0x7f1f700039b0 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f8ce25640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f7000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f877fe640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f700043b0 0x7f1f70004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f87fff640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f700039b0 0x7f1f70003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f8ce25640 1 -- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f700039b0 msgr2=0x7f1f70003d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f8ce25640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f700039b0 0x7f1f70003d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f8ce25640 1 -- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f700043b0 msgr2=0x7f1f70004880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f8ce25640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f700043b0 0x7f1f70004880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.627+0000 7f1f8ce25640 1 -- 172.21.15.64:0/1074806902 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f70003520 con 0x7f1f70005080 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.628+0000 7f1f8ce25640 1 -- 172.21.15.64:0/1074806902 reap_dead start 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.628+0000 7f1f8ce25640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f7000b9c0 secure :-1 s=READY pgs=1022 cs=0 l=1 rev1=1 crypto rx=0x7f1f8000a860 tx=0x7f1f70005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f919d159379b2206 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.628+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f8000a480 con 0x7f1f70005080 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.628+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f80034630 con 0x7f1f70005080 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.628+0000 7f1f86ffd640 1 -- 172.21.15.64:0/1074806902 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f80034a00 con 0x7f1f70005080 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.629+0000 7f1f8e888640 1 -- 172.21.15.64:0/1074806902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 msgr2=0x7f1f7000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.629+0000 7f1f8e888640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f7000b9c0 secure :-1 s=READY pgs=1022 cs=0 l=1 rev1=1 crypto rx=0x7f1f8000a860 tx=0x7f1f70005830 comp rx=0 tx=0).stop 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.629+0000 7f1f8e888640 1 -- 172.21.15.64:0/1074806902 shutdown_connections 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.629+0000 7f1f8e888640 1 --2- 172.21.15.64:0/1074806902 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f7000b9c0 unknown :-1 s=CLOSED pgs=1022 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.629+0000 7f1f8e888640 1 -- 172.21.15.64:0/1074806902 >> 172.21.15.64:0/1074806902 conn(0x7f1f88071960 msgr2=0x7f1f88071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.629+0000 7f1f8e888640 1 -- 172.21.15.64:0/1074806902 shutdown_connections 2024-03-04T07:30:57.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.629+0000 7f1f8e888640 1 -- 172.21.15.64:0/1074806902 wait complete. 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.630+0000 7f1f8e888640 1 Processor -- start 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.630+0000 7f1f8e888640 1 -- start start 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f881505c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f8815ecd0 0x7f1f88160b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f88161040 0x7f1f880b3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f880b71a0 con 0x7f1f70005080 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f8815e170 con 0x7f1f88161040 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1f880b7320 con 0x7f1f8815ecd0 2024-03-04T07:30:57.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f881505c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f881505c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48526/0 (socket says 172.21.15.64:48526) 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 -- 172.21.15.64:0/2039086340 learned_addr learned my addr 172.21.15.64:0/2039086340 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f877fe640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f8815ecd0 0x7f1f88160b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8ce25640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f88161040 0x7f1f880b3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 -- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f8815ecd0 msgr2=0x7f1f88160b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f8815ecd0 0x7f1f88160b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 -- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f88161040 msgr2=0x7f1f880b3800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f88161040 0x7f1f880b3800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 -- 172.21.15.64:0/2039086340 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f8815f170 con 0x7f1f70005080 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f877fe640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f8815ecd0 0x7f1f88160b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f87fff640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f881505c0 secure :-1 s=READY pgs=1023 cs=0 l=1 rev1=1 crypto rx=0x7f1f7800f7f0 tx=0x7f1f7800fcb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f84ff9640 1 -- 172.21.15.64:0/2039086340 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f7800fea0 con 0x7f1f70005080 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f84ff9640 1 -- 172.21.15.64:0/2039086340 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f78018070 con 0x7f1f70005080 2024-03-04T07:30:57.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1f8815f450 con 0x7f1f70005080 2024-03-04T07:30:57.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f84ff9640 1 -- 172.21.15.64:0/2039086340 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f7800d730 con 0x7f1f70005080 2024-03-04T07:30:57.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.632+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1f8815fa10 con 0x7f1f70005080 2024-03-04T07:30:57.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.633+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1f88072a30 con 0x7f1f70005080 2024-03-04T07:30:57.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.634+0000 7f1f84ff9640 1 -- 172.21.15.64:0/2039086340 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1f7800d8d0 con 0x7f1f70005080 2024-03-04T07:30:57.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.635+0000 7f1f84ff9640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f7406b4d0 0x7f1f7406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.635+0000 7f1f84ff9640 1 -- 172.21.15.64:0/2039086340 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1f7808a140 con 0x7f1f70005080 2024-03-04T07:30:57.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.635+0000 7f1f877fe640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f7406b4d0 0x7f1f7406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.635+0000 7f1f877fe640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f7406b4d0 0x7f1f7406d990 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f1f7c009b40 tx=0x7f1f7c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.639+0000 7f1f84ff9640 1 -- 172.21.15.64:0/2039086340 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1f7801a310 con 0x7f1f70005080 2024-03-04T07:30:57.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.823+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f1f8804f410 con 0x7f1f70005080 2024-03-04T07:30:57.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.823+0000 7f1f84ff9640 1 -- 172.21.15.64:0/2039086340 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v79) v1 ==== 104+0+11 (secure 0 0 0) 0x7f1f78058fc0 con 0x7f1f70005080 2024-03-04T07:30:57.823 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:30:57.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f7406b4d0 msgr2=0x7f1f7406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f7406b4d0 0x7f1f7406d990 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f1f7c009b40 tx=0x7f1f7c008040 comp rx=0 tx=0).stop 2024-03-04T07:30:57.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 msgr2=0x7f1f881505c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f881505c0 secure :-1 s=READY pgs=1023 cs=0 l=1 rev1=1 crypto rx=0x7f1f7800f7f0 tx=0x7f1f7800fcb0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 shutdown_connections 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f7406b4d0 0x7f1f7406d990 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1f88161040 0x7f1f880b3800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1f8815ecd0 0x7f1f88160b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 --2- 172.21.15.64:0/2039086340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1f70005080 0x7f1f881505c0 unknown :-1 s=CLOSED pgs=1023 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 >> 172.21.15.64:0/2039086340 conn(0x7f1f88071960 msgr2=0x7f1f880b9690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 shutdown_connections 2024-03-04T07:30:57.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.825+0000 7f1f8e888640 1 -- 172.21.15.64:0/2039086340 wait complete. 2024-03-04T07:30:57.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:30:57.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.941+0000 7f9b713e9640 1 Processor -- start 2024-03-04T07:30:57.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.941+0000 7f9b713e9640 1 -- start start 2024-03-04T07:30:57.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b713e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c1445b0 0x7f9b6c0b43c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b713e9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b6c06f1e0 con 0x7f9b6c06ec40 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b713e9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9b6c06f4c0 con 0x7f9b6c072130 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b713e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b6c144990 con 0x7f9b6c1445b0 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b63fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9b6c06ec40 0x7f9b6c071a20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35602/0 (socket says 172.21.15.64:35602) 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b63fff640 1 -- 172.21.15.64:0/1694365032 learned_addr learned my addr 172.21.15.64:0/1694365032 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6affd640 1 --2- 172.21.15.64:0/1694365032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c1445b0 0x7f9b6c0b43c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6a7fc640 1 -- 172.21.15.64:0/1694365032 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2125775795 0 0) 0x7f9b6c06f1e0 con 0x7f9b6c06ec40 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6a7fc640 1 -- 172.21.15.64:0/1694365032 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9b58003520 con 0x7f9b6c06ec40 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6a7fc640 1 -- 172.21.15.64:0/1694365032 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2001844160 0 0) 0x7f9b6c06f4c0 con 0x7f9b6c072130 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6a7fc640 1 -- 172.21.15.64:0/1694365032 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9b6c06f1e0 con 0x7f9b6c072130 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6affd640 1 -- 172.21.15.64:0/1694365032 >> v1:172.21.15.66:6789/0 conn(0x7f9b6c072130 legacy=0x7f9b6c0ce570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6affd640 1 -- 172.21.15.64:0/1694365032 >> v1:172.21.15.64:6789/0 conn(0x7f9b6c06ec40 legacy=0x7f9b6c071a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.942+0000 7f9b6affd640 1 -- 172.21.15.64:0/1694365032 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b6c06f7a0 con 0x7f9b6c1445b0 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.943+0000 7f9b6affd640 1 --2- 172.21.15.64:0/1694365032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c1445b0 0x7f9b6c0b43c0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f9b5c0097d0 tx=0x7f9b5c02b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=31f5c9bd3748be2d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.943+0000 7f9b6a7fc640 1 -- 172.21.15.64:0/1694365032 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b5c035020 con 0x7f9b6c1445b0 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.943+0000 7f9b6a7fc640 1 -- 172.21.15.64:0/1694365032 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b5c031890 con 0x7f9b6c1445b0 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.943+0000 7f9b6a7fc640 1 -- 172.21.15.64:0/1694365032 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b5c031c60 con 0x7f9b6c1445b0 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.944+0000 7f9b713e9640 1 -- 172.21.15.64:0/1694365032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c1445b0 msgr2=0x7f9b6c0b43c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.944+0000 7f9b713e9640 1 --2- 172.21.15.64:0/1694365032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c1445b0 0x7f9b6c0b43c0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f9b5c0097d0 tx=0x7f9b5c02b930 comp rx=0 tx=0).stop 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.944+0000 7f9b713e9640 1 -- 172.21.15.64:0/1694365032 shutdown_connections 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.944+0000 7f9b713e9640 1 --2- 172.21.15.64:0/1694365032 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c1445b0 0x7f9b6c0b43c0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.944+0000 7f9b713e9640 1 -- 172.21.15.64:0/1694365032 >> 172.21.15.64:0/1694365032 conn(0x7f9b6c0d09c0 msgr2=0x7f9b6c0d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.944+0000 7f9b713e9640 1 -- 172.21.15.64:0/1694365032 shutdown_connections 2024-03-04T07:30:57.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.944+0000 7f9b713e9640 1 -- 172.21.15.64:0/1694365032 wait complete. 2024-03-04T07:30:57.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.945+0000 7f9b713e9640 1 Processor -- start 2024-03-04T07:30:57.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.945+0000 7f9b713e9640 1 -- start start 2024-03-04T07:30:57.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b713e9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b6c06ec40 0x7f9b6c13a570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b713e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c072130 0x7f9b6c152850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b713e9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b6c1445b0 0x7f9b6c137400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b713e9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b6c0cdc70 con 0x7f9b6c1445b0 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b713e9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b6c0b4900 con 0x7f9b6c06ec40 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b713e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9b6c0cddf0 con 0x7f9b6c072130 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b6c1445b0 0x7f9b6c137400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b6c1445b0 0x7f9b6c137400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48528/0 (socket says 172.21.15.64:48528) 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 -- 172.21.15.64:0/830487444 learned_addr learned my addr 172.21.15.64:0/830487444 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b63fff640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c072130 0x7f9b6c152850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6affd640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b6c06ec40 0x7f9b6c13a570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 -- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c072130 msgr2=0x7f9b6c152850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c072130 0x7f9b6c152850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 -- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b6c06ec40 msgr2=0x7f9b6c13a570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b6c06ec40 0x7f9b6c13a570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b6b7fe640 1 -- 172.21.15.64:0/830487444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b6c1ef6e0 con 0x7f9b6c1445b0 2024-03-04T07:30:57.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.946+0000 7f9b63fff640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c072130 0x7f9b6c152850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.947+0000 7f9b6b7fe640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b6c1445b0 0x7f9b6c137400 secure :-1 s=READY pgs=1025 cs=0 l=1 rev1=1 crypto rx=0x7f9b6400c7d0 tx=0x7f9b6400cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.947+0000 7f9b637fe640 1 -- 172.21.15.64:0/830487444 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b64003b70 con 0x7f9b6c1445b0 2024-03-04T07:30:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.947+0000 7f9b637fe640 1 -- 172.21.15.64:0/830487444 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9b64003d10 con 0x7f9b6c1445b0 2024-03-04T07:30:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.947+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b6c1ef9c0 con 0x7f9b6c1445b0 2024-03-04T07:30:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.947+0000 7f9b637fe640 1 -- 172.21.15.64:0/830487444 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b6400aa90 con 0x7f9b6c1445b0 2024-03-04T07:30:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.947+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b6c1eff80 con 0x7f9b6c1445b0 2024-03-04T07:30:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.948+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b6c071c90 con 0x7f9b6c1445b0 2024-03-04T07:30:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.948+0000 7f9b637fe640 1 -- 172.21.15.64:0/830487444 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9b6400ac30 con 0x7f9b6c1445b0 2024-03-04T07:30:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.949+0000 7f9b637fe640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b4c06b4d0 0x7f9b4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.949+0000 7f9b637fe640 1 -- 172.21.15.64:0/830487444 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f9b6408a1a0 con 0x7f9b6c1445b0 2024-03-04T07:30:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.949+0000 7f9b6affd640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b4c06b4d0 0x7f9b4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.949+0000 7f9b6affd640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b4c06b4d0 0x7f9b4c06d990 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f9b5c036000 tx=0x7f9b5c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:57.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:57.957+0000 7f9b637fe640 1 -- 172.21.15.64:0/830487444 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9b6c071c90 con 0x7f9b6c1445b0 2024-03-04T07:30:58.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.143+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f9b6c0733d0 con 0x7f9b6c1445b0 2024-03-04T07:30:58.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.144+0000 7f9b637fe640 1 -- 172.21.15.64:0/830487444 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v79) v1 ==== 100+0+11 (secure 0 0 0) 0x7f9b64059020 con 0x7f9b6c1445b0 2024-03-04T07:30:58.144 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:30:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b4c06b4d0 msgr2=0x7f9b4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b4c06b4d0 0x7f9b4c06d990 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f9b5c036000 tx=0x7f9b5c03c040 comp rx=0 tx=0).stop 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b6c1445b0 msgr2=0x7f9b6c137400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b6c1445b0 0x7f9b6c137400 secure :-1 s=READY pgs=1025 cs=0 l=1 rev1=1 crypto rx=0x7f9b6400c7d0 tx=0x7f9b6400cc90 comp rx=0 tx=0).stop 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 shutdown_connections 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9b4c06b4d0 0x7f9b4c06d990 unknown :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9b6c1445b0 0x7f9b6c137400 unknown :-1 s=CLOSED pgs=1025 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9b6c072130 0x7f9b6c152850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 --2- 172.21.15.64:0/830487444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9b6c06ec40 0x7f9b6c13a570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.145+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 >> 172.21.15.64:0/830487444 conn(0x7f9b6c0d09c0 msgr2=0x7f9b6c0b1990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:58.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.146+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 shutdown_connections 2024-03-04T07:30:58.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.146+0000 7f9b713e9640 1 -- 172.21.15.64:0/830487444 wait complete. 2024-03-04T07:30:58.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:30:58.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.259+0000 7fc80afbf640 1 Processor -- start 2024-03-04T07:30:58.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.259+0000 7fc80afbf640 1 -- start start 2024-03-04T07:30:58.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.259+0000 7fc80afbf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc80415df00 0x7fc8040d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc80afbf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc8040b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc80afbf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc8040cea30 con 0x7fc8040b9a30 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc80afbf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc8040d2af0 con 0x7fc80415df00 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc80afbf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc8040cf3c0 con 0x7fc8040b95a0 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc80955c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc8040b9a30 0x7fc8040d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35618/0 (socket says 172.21.15.64:35618) 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc80955c640 1 -- 172.21.15.64:0/258745204 learned_addr learned my addr 172.21.15.64:0/258745204 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc808d5b640 1 --2- 172.21.15.64:0/258745204 >> v2:172.21.15.66:3300/0 conn(0x7fc80415df00 0x7fc8040d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc803fff640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc8040b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 28203592 0 0) 0x7fc8040cea30 con 0x7fc8040b9a30 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc7f0003520 con 0x7fc8040b9a30 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1143997105 0 0) 0x7fc7f0003520 con 0x7fc8040b9a30 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 msgr2=0x7fc8040b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc8040b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 >> v2:172.21.15.66:3300/0 conn(0x7fc80415df00 msgr2=0x7fc8040d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 --2- 172.21.15.64:0/258745204 >> v2:172.21.15.66:3300/0 conn(0x7fc80415df00 0x7fc8040d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc8040cf0e0 con 0x7fc8040b9a30 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fc7f4001670 con 0x7fc8040b9a30 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 >> v1:172.21.15.64:6789/0 conn(0x7fc8040b9a30 legacy=0x7fc8040d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc7f00039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7f0003f30 0x7fc7f0004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7f0004d40 0x7fc7f00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8040d2af0 con 0x7fc7f0004d40 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8040cf3c0 con 0x7fc7f0003f30 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.260+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc7f000b9c0 con 0x7fc8040b95a0 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc808d5b640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc7f00039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc80955c640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7f0004d40 0x7fc7f00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc803fff640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7f0003f30 0x7fc7f0004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc808d5b640 1 -- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7f0003f30 msgr2=0x7fc7f0004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc808d5b640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc7f0003f30 0x7fc7f0004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc808d5b640 1 -- 172.21.15.64:0/258745204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7f0004d40 msgr2=0x7fc7f00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc808d5b640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc7f0004d40 0x7fc7f00051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.261+0000 7fc808d5b640 1 -- 172.21.15.64:0/258745204 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7f0003520 con 0x7fc8040b95a0 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc808d5b640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc7f00039b0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fc7fc009fc0 tx=0x7fc7f0005830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d423a0638d62b15a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7fc039070 con 0x7fc8040b95a0 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc7fc02cc60 con 0x7fc8040b95a0 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc8037fe640 1 -- 172.21.15.64:0/258745204 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7fc009c10 con 0x7fc8040b95a0 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc80afbf640 1 -- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 msgr2=0x7fc7f00039b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc80afbf640 1 --2- 172.21.15.64:0/258745204 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc7f00039b0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fc7fc009fc0 tx=0x7fc7f0005830 comp rx=0 tx=0).stop 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc80955c640 1 -- 172.21.15.64:0/258745204 reap_dead start 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc80afbf640 1 -- 172.21.15.64:0/258745204 shutdown_connections 2024-03-04T07:30:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc80afbf640 1 -- 172.21.15.64:0/258745204 >> 172.21.15.64:0/258745204 conn(0x7fc804071960 msgr2=0x7fc804071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:58.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc80afbf640 1 -- 172.21.15.64:0/258745204 shutdown_connections 2024-03-04T07:30:58.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.262+0000 7fc80afbf640 1 -- 172.21.15.64:0/258745204 wait complete. 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.264+0000 7fc80afbf640 1 Processor -- start 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.264+0000 7fc80afbf640 1 -- start start 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.264+0000 7fc80afbf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc8041f8240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.264+0000 7fc80afbf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8040cca20 0x7fc8040ccea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.264+0000 7fc80afbf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc804157ab0 0x7fc804157f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.264+0000 7fc80afbf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8040cec70 con 0x7fc8040cca20 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.264+0000 7fc80afbf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8040ce750 con 0x7fc804157ab0 2024-03-04T07:30:58.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc80afbf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc8040cedf0 con 0x7fc8040b95a0 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8040cca20 0x7fc8040ccea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8040cca20 0x7fc8040ccea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48552/0 (socket says 172.21.15.64:48552) 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 -- 172.21.15.64:0/3080332726 learned_addr learned my addr 172.21.15.64:0/3080332726 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc808d5b640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc8041f8240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc80955c640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc804157ab0 0x7fc804157f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 -- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 msgr2=0x7fc8041f8240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc8041f8240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 -- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc804157ab0 msgr2=0x7fc804157f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc804157ab0 0x7fc804157f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 -- 172.21.15.64:0/3080332726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc8041587e0 con 0x7fc8040cca20 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc80955c640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc804157ab0 0x7fc804157f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc803fff640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8040cca20 0x7fc8040ccea0 secure :-1 s=READY pgs=1027 cs=0 l=1 rev1=1 crypto rx=0x7fc7ec011990 tx=0x7fc7ec011e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc8017fa640 1 -- 172.21.15.64:0/3080332726 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7ec00fa70 con 0x7fc8040cca20 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc8017fa640 1 -- 172.21.15.64:0/3080332726 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc7ec00fc10 con 0x7fc8040cca20 2024-03-04T07:30:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc804158ac0 con 0x7fc8040cca20 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc8017fa640 1 -- 172.21.15.64:0/3080332726 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7ec014440 con 0x7fc8040cca20 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.265+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc804152b50 con 0x7fc8040cca20 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.266+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc8040d28e0 con 0x7fc8040cca20 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.267+0000 7fc8017fa640 1 -- 172.21.15.64:0/3080332726 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc7ec012040 con 0x7fc8040cca20 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.267+0000 7fc8017fa640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7e406b500 0x7fc7e406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.267+0000 7fc8017fa640 1 -- 172.21.15.64:0/3080332726 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc7ec08a760 con 0x7fc8040cca20 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.267+0000 7fc808d5b640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7e406b500 0x7fc7e406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.268+0000 7fc808d5b640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7e406b500 0x7fc7e406d9c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7fc7fc036000 tx=0x7fc7fc02e000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:58.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.272+0000 7fc8017fa640 1 -- 172.21.15.64:0/3080332726 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc8040d28e0 con 0x7fc8040cca20 2024-03-04T07:30:58.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.466+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fc804067ff0 con 0x7fc8040cca20 2024-03-04T07:30:58.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.467+0000 7fc8017fa640 1 -- 172.21.15.64:0/3080332726 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v79) v1 ==== 80+0+0 (secure 0 0 0) 0x7fc7ec019030 con 0x7fc8040cca20 2024-03-04T07:30:58.467 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7e406b500 msgr2=0x7fc7e406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7e406b500 0x7fc7e406d9c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7fc7fc036000 tx=0x7fc7fc02e000 comp rx=0 tx=0).stop 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8040cca20 msgr2=0x7fc8040ccea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8040cca20 0x7fc8040ccea0 secure :-1 s=READY pgs=1027 cs=0 l=1 rev1=1 crypto rx=0x7fc7ec011990 tx=0x7fc7ec011e50 comp rx=0 tx=0).stop 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 shutdown_connections 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc7e406b500 0x7fc7e406d9c0 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc804157ab0 0x7fc804157f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc8040cca20 0x7fc8040ccea0 unknown :-1 s=CLOSED pgs=1027 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 --2- 172.21.15.64:0/3080332726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc8040b95a0 0x7fc8041f8240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 >> 172.21.15.64:0/3080332726 conn(0x7fc804071960 msgr2=0x7fc8040d3770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 shutdown_connections 2024-03-04T07:30:58.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.469+0000 7fc80afbf640 1 -- 172.21.15.64:0/3080332726 wait complete. 2024-03-04T07:30:58.479 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:30:58.479 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:58.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:30:58.505 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[65956]: fuse finished with error 0 and tester_r 0 2024-03-04T07:30:58.507 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:30:58.507 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:58.507 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:30:58.583 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:30:58.583 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:58.603 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:30:58.603 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:30:58.603 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:58.603 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:30:58.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:30:58.662 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:30:58.662 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:30:58.662 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:58.677 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:30:58.677 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:30:58.677 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:58.678 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:30:58.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:30:58.732 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:30:58.732 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:30:58.732 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:30:58.751 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:30:58.751 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:30:58.751 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:30:58.751 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:30:58.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:30:58.809 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:30:58.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:30:58.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.910+0000 7f14b06b5640 1 Processor -- start 2024-03-04T07:30:58.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.910+0000 7f14b06b5640 1 -- start start 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14b06b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14a815df00 0x7f14a80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14b06b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f14a80b95a0 0x7f14a80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14b06b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b9a30 0x7f14a80b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14b06b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14a80cea30 con 0x7f14a815df00 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14b06b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14a80d2af0 con 0x7f14a80b95a0 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14b06b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14a80cf3c0 con 0x7f14a80b9a30 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14ae451640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14a815df00 0x7f14a80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14ae451640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14a815df00 0x7f14a80d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48558/0 (socket says 172.21.15.64:48558) 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.911+0000 7f14ae451640 1 -- 172.21.15.64:0/3382204087 learned_addr learned my addr 172.21.15.64:0/3382204087 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14aec52640 1 --2- 172.21.15.64:0/3382204087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b9a30 0x7f14a80b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14adc50640 1 --2- 172.21.15.64:0/3382204087 >> v2:172.21.15.66:3300/0 conn(0x7f14a80b95a0 0x7f14a80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14ae451640 1 -- 172.21.15.64:0/3382204087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b9a30 msgr2=0x7f14a80b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14ae451640 1 --2- 172.21.15.64:0/3382204087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b9a30 0x7f14a80b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14ae451640 1 -- 172.21.15.64:0/3382204087 >> v2:172.21.15.66:3300/0 conn(0x7f14a80b95a0 msgr2=0x7f14a80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14ae451640 1 --2- 172.21.15.64:0/3382204087 >> v2:172.21.15.66:3300/0 conn(0x7f14a80b95a0 0x7f14a80b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14ae451640 1 -- 172.21.15.64:0/3382204087 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14a80cf0e0 con 0x7f14a815df00 2024-03-04T07:30:58.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14aec52640 1 --2- 172.21.15.64:0/3382204087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b9a30 0x7f14a80b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.912+0000 7f14ae451640 1 --2- 172.21.15.64:0/3382204087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80d25b0 secure :-1 s=READY pgs=1028 cs=0 l=1 rev1=1 crypto rx=0x7f14a80d1760 tx=0x7f14a402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dafe0a1ccc9f3481 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14ad44f640 1 -- 172.21.15.64:0/3382204087 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14a4035020 con 0x7f14a815df00 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14ad44f640 1 -- 172.21.15.64:0/3382204087 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f14a4031870 con 0x7f14a815df00 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14ad44f640 1 -- 172.21.15.64:0/3382204087 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f14a4031c40 con 0x7f14a815df00 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 -- 172.21.15.64:0/3382204087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 msgr2=0x7f14a80d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 --2- 172.21.15.64:0/3382204087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80d25b0 secure :-1 s=READY pgs=1028 cs=0 l=1 rev1=1 crypto rx=0x7f14a80d1760 tx=0x7f14a402b910 comp rx=0 tx=0).stop 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 -- 172.21.15.64:0/3382204087 shutdown_connections 2024-03-04T07:30:58.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 --2- 172.21.15.64:0/3382204087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b9a30 0x7f14a80b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 --2- 172.21.15.64:0/3382204087 >> v2:172.21.15.66:3300/0 conn(0x7f14a80b95a0 0x7f14a80b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 --2- 172.21.15.64:0/3382204087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80d25b0 unknown :-1 s=CLOSED pgs=1028 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 -- 172.21.15.64:0/3382204087 >> 172.21.15.64:0/3382204087 conn(0x7f14a8071960 msgr2=0x7f14a8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 -- 172.21.15.64:0/3382204087 shutdown_connections 2024-03-04T07:30:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.913+0000 7f14b06b5640 1 -- 172.21.15.64:0/3382204087 wait complete. 2024-03-04T07:30:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.914+0000 7f14b06b5640 1 Processor -- start 2024-03-04T07:30:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.914+0000 7f14b06b5640 1 -- start start 2024-03-04T07:30:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14b06b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14a80b95a0 0x7f14a8158510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14b06b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80b49a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14b06b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b4ee0 0x7f14a8160620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14b06b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14a80cec70 con 0x7f14a815df00 2024-03-04T07:30:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14b06b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14a80ce750 con 0x7f14a80b95a0 2024-03-04T07:30:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14b06b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14a80cedf0 con 0x7f14a80b4ee0 2024-03-04T07:30:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14adc50640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80b49a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14adc50640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80b49a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48560/0 (socket says 172.21.15.64:48560) 2024-03-04T07:30:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14adc50640 1 -- 172.21.15.64:0/1285343308 learned_addr learned my addr 172.21.15.64:0/1285343308 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:58.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14ae451640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14a80b95a0 0x7f14a8158510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.915+0000 7f14aec52640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b4ee0 0x7f14a8160620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14adc50640 1 -- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b4ee0 msgr2=0x7f14a8160620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14adc50640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b4ee0 0x7f14a8160620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14adc50640 1 -- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14a80b95a0 msgr2=0x7f14a8158510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14adc50640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14a80b95a0 0x7f14a8158510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:58.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14adc50640 1 -- 172.21.15.64:0/1285343308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14a80cafa0 con 0x7f14a815df00 2024-03-04T07:30:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14adc50640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80b49a0 secure :-1 s=READY pgs=1029 cs=0 l=1 rev1=1 crypto rx=0x7f149800e9e0 tx=0x7f149800eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f149effd640 1 -- 172.21.15.64:0/1285343308 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1498017070 con 0x7f14a815df00 2024-03-04T07:30:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f149effd640 1 -- 172.21.15.64:0/1285343308 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f149800c9d0 con 0x7f14a815df00 2024-03-04T07:30:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f14a80cb280 con 0x7f14a815df00 2024-03-04T07:30:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f149effd640 1 -- 172.21.15.64:0/1285343308 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f149800cdc0 con 0x7f14a815df00 2024-03-04T07:30:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.916+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14a80cb840 con 0x7f14a815df00 2024-03-04T07:30:58.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.917+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f14a80cad30 con 0x7f14a815df00 2024-03-04T07:30:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.918+0000 7f149effd640 1 -- 172.21.15.64:0/1285343308 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f149801a070 con 0x7f14a815df00 2024-03-04T07:30:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.918+0000 7f149effd640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f148c06b4d0 0x7f148c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.918+0000 7f149effd640 1 -- 172.21.15.64:0/1285343308 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f149808a820 con 0x7f14a815df00 2024-03-04T07:30:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.918+0000 7f14ae451640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f148c06b4d0 0x7f148c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.919+0000 7f14ae451640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f148c06b4d0 0x7f148c06d990 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f14a4036000 tx=0x7f14a403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:58.922+0000 7f149effd640 1 -- 172.21.15.64:0/1285343308 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f14980591c0 con 0x7f14a815df00 2024-03-04T07:30:59.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.139+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f14a80b52f0 con 0x7f14a815df00 2024-03-04T07:30:59.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.140+0000 7f149effd640 1 -- 172.21.15.64:0/1285343308 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 43 v43) v1 ==== 94+0+4973 (secure 0 0 0) 0x7f14a80b52f0 con 0x7f14a815df00 2024-03-04T07:30:59.141 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:59.141 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[{"mdsmap":{"epoch":43,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:48.344908+0000","modified":"2024-03-04T07:30:50.410439+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6966},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6966":{"gid":6966,"name":"c","rank":0,"incarnation":41,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/1889124368","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1889124368},{"type":"v1","addr":"172.21.15.64:6836","nonce":1889124368}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6966,"qdb_cluster":[6966]},"id":6}]} 2024-03-04T07:30:59.141 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 43 2024-03-04T07:30:59.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.141+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f148c06b4d0 msgr2=0x7f148c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.141+0000 7f14b06b5640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f148c06b4d0 0x7f148c06d990 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f14a4036000 tx=0x7f14a403c040 comp rx=0 tx=0).stop 2024-03-04T07:30:59.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 msgr2=0x7f14a80b49a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80b49a0 secure :-1 s=READY pgs=1029 cs=0 l=1 rev1=1 crypto rx=0x7f149800e9e0 tx=0x7f149800eea0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 shutdown_connections 2024-03-04T07:30:59.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f148c06b4d0 0x7f148c06d990 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14a80b4ee0 0x7f14a8160620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14a815df00 0x7f14a80b49a0 unknown :-1 s=CLOSED pgs=1029 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 --2- 172.21.15.64:0/1285343308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14a80b95a0 0x7f14a8158510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 >> 172.21.15.64:0/1285343308 conn(0x7f14a8071960 msgr2=0x7f14a80b78e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:59.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 shutdown_connections 2024-03-04T07:30:59.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.142+0000 7f14b06b5640 1 -- 172.21.15.64:0/1285343308 wait complete. 2024-03-04T07:30:59.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:30:59.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.258+0000 7f4290c15640 1 Processor -- start 2024-03-04T07:30:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.258+0000 7f4290c15640 1 -- start start 2024-03-04T07:30:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f4290c15640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f428c15df00 0x7f428c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f4290c15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0b9a30 0x7f428c0d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f4290c15640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f428c0cea30 con 0x7f428c0b95a0 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f4290c15640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f428c0d2af0 con 0x7f428c15df00 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f4290c15640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f428c0cf3c0 con 0x7f428c0b9a30 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f4281d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f428c0b95a0 0x7f428c0b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35626/0 (socket says 172.21.15.64:35626) 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f4281d9b640 1 -- 172.21.15.64:0/3202503522 learned_addr learned my addr 172.21.15.64:0/3202503522 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f428a59c640 1 --2- 172.21.15.64:0/3202503522 >> v2:172.21.15.66:3300/0 conn(0x7f428c15df00 0x7f428c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.259+0000 7f428ad9d640 1 --2- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0b9a30 0x7f428c0d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f4289d9b640 1 -- 172.21.15.64:0/3202503522 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 915939163 0 0) 0x7f428c0cea30 con 0x7f428c0b95a0 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f4289d9b640 1 -- 172.21.15.64:0/3202503522 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4278003520 con 0x7f428c0b95a0 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f428a59c640 1 -- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0b9a30 msgr2=0x7f428c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f428a59c640 1 --2- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0b9a30 0x7f428c0d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f428a59c640 1 -- 172.21.15.64:0/3202503522 >> v1:172.21.15.64:6789/0 conn(0x7f428c0b95a0 legacy=0x7f428c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f4289d9b640 1 -- 172.21.15.64:0/3202503522 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 986673728 0 0) 0x7f4278003520 con 0x7f428c0b95a0 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f428a59c640 1 -- 172.21.15.64:0/3202503522 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f428c0cf0e0 con 0x7f428c15df00 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.260+0000 7f428ad9d640 1 --2- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0b9a30 0x7f428c0d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:30:59.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f428a59c640 1 --2- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 0x7f428c0d25b0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f427c0097a0 tx=0x7f427c02b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7ae13ee4c84d653 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4289d9b640 1 -- 172.21.15.64:0/3202503522 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f427c037070 con 0x7f428c15df00 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4289d9b640 1 -- 172.21.15.64:0/3202503522 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f427c030510 con 0x7f428c15df00 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4289d9b640 1 -- 172.21.15.64:0/3202503522 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4274001670 con 0x7f428c15df00 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 -- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 msgr2=0x7f428c0d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 --2- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 0x7f428c0d25b0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f427c0097a0 tx=0x7f427c02b790 comp rx=0 tx=0).stop 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 -- 172.21.15.64:0/3202503522 shutdown_connections 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 --2- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0b9a30 0x7f428c0d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 --2- 172.21.15.64:0/3202503522 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 0x7f428c0d25b0 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 -- 172.21.15.64:0/3202503522 >> 172.21.15.64:0/3202503522 conn(0x7f428c071960 msgr2=0x7f428c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 -- 172.21.15.64:0/3202503522 shutdown_connections 2024-03-04T07:30:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.261+0000 7f4290c15640 1 -- 172.21.15.64:0/3202503522 wait complete. 2024-03-04T07:30:59.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.263+0000 7f4290c15640 1 Processor -- start 2024-03-04T07:30:59.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.263+0000 7f4290c15640 1 -- start start 2024-03-04T07:30:59.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f428c0b95a0 0x7f428c161b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 0x7f428c15ecf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0c8bc0 0x7f428c150c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f428c0cec70 con 0x7f428c0b95a0 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f428c0ce750 con 0x7f428c15df00 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f428c0cedf0 con 0x7f428c0c8bc0 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f428c0b95a0 0x7f428c161b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f428c0b95a0 0x7f428c161b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48570/0 (socket says 172.21.15.64:48570) 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 -- 172.21.15.64:0/487851457 learned_addr learned my addr 172.21.15.64:0/487851457 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428ad9d640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0c8bc0 0x7f428c150c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4281d9b640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 0x7f428c15ecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 -- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0c8bc0 msgr2=0x7f428c150c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0c8bc0 0x7f428c150c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 -- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 msgr2=0x7f428c15ecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 0x7f428c15ecf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 -- 172.21.15.64:0/487851457 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f428c0c7b20 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f428a59c640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f428c0b95a0 0x7f428c161b20 secure :-1 s=READY pgs=1031 cs=0 l=1 rev1=1 crypto rx=0x7f427c02b7d0 tx=0x7f427c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f42837fe640 1 -- 172.21.15.64:0/487851457 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f427c041020 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f42837fe640 1 -- 172.21.15.64:0/487851457 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f427c03c870 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f428c0c7e00 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f42837fe640 1 -- 172.21.15.64:0/487851457 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f427c037040 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.264+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f428c0c83c0 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.265+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f428c067ff0 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.266+0000 7f42837fe640 1 -- 172.21.15.64:0/487851457 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f427c044020 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.266+0000 7f42837fe640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f426c06b4d0 0x7f426c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.266+0000 7f42837fe640 1 -- 172.21.15.64:0/487851457 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f427c0abae0 con 0x7f428c0b95a0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.266+0000 7f4281d9b640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f426c06b4d0 0x7f426c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.267+0000 7f4281d9b640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f426c06b4d0 0x7f426c06d990 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f4274001320 tx=0x7f42740012b0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.274+0000 7f42837fe640 1 -- 172.21.15.64:0/487851457 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f427c04b030 con 0x7f428c0b95a0 2024-03-04T07:30:59.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.513+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f428c0721e0 con 0x7f428c0b95a0 2024-03-04T07:30:59.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.515+0000 7f42837fe640 1 -- 172.21.15.64:0/487851457 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v79) v1 ==== 74+0+16612 (secure 0 0 0) 0x7f427c07a8d0 con 0x7f428c0b95a0 2024-03-04T07:30:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:30:59.516 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":79,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:49.980908+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:46.621410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:47.317858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"79","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:30:59.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f426c06b4d0 msgr2=0x7f426c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f426c06b4d0 0x7f426c06d990 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f4274001320 tx=0x7f42740012b0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f428c0b95a0 msgr2=0x7f428c161b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f428c0b95a0 0x7f428c161b20 secure :-1 s=READY pgs=1031 cs=0 l=1 rev1=1 crypto rx=0x7f427c02b7d0 tx=0x7f427c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 shutdown_connections 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f426c06b4d0 0x7f426c06d990 unknown :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f428c0c8bc0 0x7f428c150c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f428c15df00 0x7f428c15ecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 --2- 172.21.15.64:0/487851457 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f428c0b95a0 0x7f428c161b20 unknown :-1 s=CLOSED pgs=1031 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 >> 172.21.15.64:0/487851457 conn(0x7f428c071960 msgr2=0x7f428c0b82a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 shutdown_connections 2024-03-04T07:30:59.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.517+0000 7f4290c15640 1 -- 172.21.15.64:0/487851457 wait complete. 2024-03-04T07:30:59.529 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 12, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:30:46.621410+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '78', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2, 'score_stable': 2, 'optimal_score': 1, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 13, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:30:47.317858+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '79', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.3799999952316284, 'score_stable': 1.3799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.3799999952316284, 'raw_score_stable': 1.3799999952316284, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:30:59.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:30:59.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.636+0000 7ff437039640 1 Processor -- start 2024-03-04T07:30:59.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.636+0000 7ff437039640 1 -- start start 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff437039640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff4300cdcd0 0x7ff4300b1550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff437039640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff430161a40 con 0x7ff4300b2cd0 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff437039640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4300b2a20 con 0x7ff43006b2c0 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff437039640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff4300d11e0 con 0x7ff4300cdcd0 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff434dd5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff4300cdcd0 0x7ff4300b1550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff434dd5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff4300cdcd0 0x7ff4300b1550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48584/0 (socket says 172.21.15.64:48584) 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff42ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff4300b2cd0 0x7ff4300b1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35642/0 (socket says 172.21.15.64:35642) 2024-03-04T07:30:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff434dd5640 1 -- 172.21.15.64:0/2012986063 learned_addr learned my addr 172.21.15.64:0/2012986063 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff434dd5640 1 -- 172.21.15.64:0/2012986063 >> v1:172.21.15.66:6789/0 conn(0x7ff43006b2c0 legacy=0x7ff43006b710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff434dd5640 1 -- 172.21.15.64:0/2012986063 >> v1:172.21.15.64:6789/0 conn(0x7ff4300b2cd0 legacy=0x7ff4300b1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.637+0000 7ff434dd5640 1 -- 172.21.15.64:0/2012986063 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4300d0940 con 0x7ff4300cdcd0 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.638+0000 7ff434dd5640 1 --2- 172.21.15.64:0/2012986063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300b1550 secure :-1 s=READY pgs=1033 cs=0 l=1 rev1=1 crypto rx=0x7ff424009870 tx=0x7ff42402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c80a81c033721f2e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.638+0000 7ff42f7fe640 1 -- 172.21.15.64:0/2012986063 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff424034070 con 0x7ff4300cdcd0 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.638+0000 7ff42f7fe640 1 -- 172.21.15.64:0/2012986063 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff424030860 con 0x7ff4300cdcd0 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.638+0000 7ff42f7fe640 1 -- 172.21.15.64:0/2012986063 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff424030c30 con 0x7ff4300cdcd0 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.639+0000 7ff437039640 1 -- 172.21.15.64:0/2012986063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 msgr2=0x7ff4300b1550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.639+0000 7ff437039640 1 --2- 172.21.15.64:0/2012986063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300b1550 secure :-1 s=READY pgs=1033 cs=0 l=1 rev1=1 crypto rx=0x7ff424009870 tx=0x7ff42402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.639+0000 7ff437039640 1 -- 172.21.15.64:0/2012986063 shutdown_connections 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.639+0000 7ff437039640 1 --2- 172.21.15.64:0/2012986063 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300b1550 unknown :-1 s=CLOSED pgs=1033 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.639+0000 7ff437039640 1 -- 172.21.15.64:0/2012986063 >> 172.21.15.64:0/2012986063 conn(0x7ff4300b3700 msgr2=0x7ff4300b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.639+0000 7ff437039640 1 -- 172.21.15.64:0/2012986063 shutdown_connections 2024-03-04T07:30:59.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.639+0000 7ff437039640 1 -- 172.21.15.64:0/2012986063 wait complete. 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.640+0000 7ff437039640 1 Processor -- start 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.640+0000 7ff437039640 1 -- start start 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff437039640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff43006b2c0 0x7ff4301608f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff437039640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4300b2cd0 0x7ff4300c9610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff437039640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300c7390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff437039640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4300cc1c0 con 0x7ff4300cdcd0 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff437039640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4300d0f00 con 0x7ff4300b2cd0 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff437039640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4300cc340 con 0x7ff43006b2c0 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300c7390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300c7390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48598/0 (socket says 172.21.15.64:48598) 2024-03-04T07:30:59.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 -- 172.21.15.64:0/945774326 learned_addr learned my addr 172.21.15.64:0/945774326 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff434dd5640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff43006b2c0 0x7ff4301608f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff42ffff640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4300b2cd0 0x7ff4300c9610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 -- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff43006b2c0 msgr2=0x7ff4301608f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff43006b2c0 0x7ff4301608f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 -- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4300b2cd0 msgr2=0x7ff4300c9610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4300b2cd0 0x7ff4300c9610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 -- 172.21.15.64:0/945774326 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4300ca0f0 con 0x7ff4300cdcd0 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff42ffff640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4300b2cd0 0x7ff4300c9610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff434dd5640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff43006b2c0 0x7ff4301608f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.641+0000 7ff4355d6640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300c7390 secure :-1 s=READY pgs=1034 cs=0 l=1 rev1=1 crypto rx=0x7ff42800c7a0 tx=0x7ff42800cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.642+0000 7ff42d7fa640 1 -- 172.21.15.64:0/945774326 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff428003b70 con 0x7ff4300cdcd0 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.642+0000 7ff42d7fa640 1 -- 172.21.15.64:0/945774326 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff428003d30 con 0x7ff4300cdcd0 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.642+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4300ca3d0 con 0x7ff4300cdcd0 2024-03-04T07:30:59.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.642+0000 7ff42d7fa640 1 -- 172.21.15.64:0/945774326 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff42800aa90 con 0x7ff4300cdcd0 2024-03-04T07:30:59.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.642+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4301f4500 con 0x7ff4300cdcd0 2024-03-04T07:30:59.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.643+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4300ca5a0 con 0x7ff4300cdcd0 2024-03-04T07:30:59.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.643+0000 7ff42d7fa640 1 -- 172.21.15.64:0/945774326 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff42800ac30 con 0x7ff4300cdcd0 2024-03-04T07:30:59.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.643+0000 7ff42d7fa640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff41406b4d0 0x7ff41406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.644+0000 7ff42d7fa640 1 -- 172.21.15.64:0/945774326 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff42808a140 con 0x7ff4300cdcd0 2024-03-04T07:30:59.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.644+0000 7ff434dd5640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff41406b4d0 0x7ff41406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.644+0000 7ff434dd5640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff41406b4d0 0x7ff41406d990 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7ff424036000 tx=0x7ff42403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.648+0000 7ff42d7fa640 1 -- 172.21.15.64:0/945774326 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff428058fc0 con 0x7ff4300cdcd0 2024-03-04T07:30:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.828+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7ff4301f4760 con 0x7ff4300cdcd0 2024-03-04T07:30:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.829+0000 7ff42d7fa640 1 -- 172.21.15.64:0/945774326 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v79) v1 ==== 93+0+10 (secure 0 0 0) 0x7ff4280588c0 con 0x7ff4300cdcd0 2024-03-04T07:30:59.829 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:30:59.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.830+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff41406b4d0 msgr2=0x7ff41406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.830+0000 7ff437039640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff41406b4d0 0x7ff41406d990 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7ff424036000 tx=0x7ff42403b040 comp rx=0 tx=0).stop 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.830+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 msgr2=0x7ff4300c7390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.830+0000 7ff437039640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300c7390 secure :-1 s=READY pgs=1034 cs=0 l=1 rev1=1 crypto rx=0x7ff42800c7a0 tx=0x7ff42800cc60 comp rx=0 tx=0).stop 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 shutdown_connections 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff41406b4d0 0x7ff41406d990 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4300cdcd0 0x7ff4300c7390 unknown :-1 s=CLOSED pgs=1034 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4300b2cd0 0x7ff4300c9610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 --2- 172.21.15.64:0/945774326 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff43006b2c0 0x7ff4301608f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 >> 172.21.15.64:0/945774326 conn(0x7ff4300b3700 msgr2=0x7ff4300b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 shutdown_connections 2024-03-04T07:30:59.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.831+0000 7ff437039640 1 -- 172.21.15.64:0/945774326 wait complete. 2024-03-04T07:30:59.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:30:59.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.946+0000 7f6cff193640 1 Processor -- start 2024-03-04T07:30:59.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.946+0000 7f6cff193640 1 -- start start 2024-03-04T07:30:59.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.946+0000 7f6cff193640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6cf815dd90 0x7f6cf80ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.946+0000 7f6cff193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf80d29d0 0x7f6cf80d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.946+0000 7f6cff193640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6cf80d3740 con 0x7f6cf80d4720 2024-03-04T07:30:59.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.946+0000 7f6cff193640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6cf80d3400 con 0x7f6cf815dd90 2024-03-04T07:30:59.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.946+0000 7f6cff193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6cf80d3120 con 0x7f6cf80d29d0 2024-03-04T07:30:59.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cfd730640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6cf80d4720 0x7f6cf80b6d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35656/0 (socket says 172.21.15.64:35656) 2024-03-04T07:30:59.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cfd730640 1 -- 172.21.15.64:0/497963703 learned_addr learned my addr 172.21.15.64:0/497963703 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cfcf2f640 1 --2- 172.21.15.64:0/497963703 >> v2:172.21.15.66:3300/0 conn(0x7f6cf815dd90 0x7f6cf80ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cf7fff640 1 --2- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf80d29d0 0x7f6cf80d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cf77fe640 1 -- 172.21.15.64:0/497963703 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1473689667 0 0) 0x7f6cf80d3740 con 0x7f6cf80d4720 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cf77fe640 1 -- 172.21.15.64:0/497963703 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6ce8003520 con 0x7f6cf80d4720 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cfcf2f640 1 -- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf80d29d0 msgr2=0x7f6cf80d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cfcf2f640 1 --2- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf80d29d0 0x7f6cf80d41e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cfcf2f640 1 -- 172.21.15.64:0/497963703 >> v1:172.21.15.64:6789/0 conn(0x7f6cf80d4720 legacy=0x7f6cf80b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.947+0000 7f6cfcf2f640 1 -- 172.21.15.64:0/497963703 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cf80d39e0 con 0x7f6cf815dd90 2024-03-04T07:30:59.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.948+0000 7f6cfcf2f640 1 --2- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf815dd90 0x7f6cf80ceea0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f6cec0097a0 tx=0x7f6cec02b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6a63d598d72abe9a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.948+0000 7f6cf77fe640 1 -- 172.21.15.64:0/497963703 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cec034070 con 0x7f6cf815dd90 2024-03-04T07:30:59.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.948+0000 7f6cf77fe640 1 -- 172.21.15.64:0/497963703 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cec030610 con 0x7f6cf815dd90 2024-03-04T07:30:59.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.948+0000 7f6cf77fe640 1 -- 172.21.15.64:0/497963703 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cec0309e0 con 0x7f6cf815dd90 2024-03-04T07:30:59.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 -- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf815dd90 msgr2=0x7f6cf80ceea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 --2- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf815dd90 0x7f6cf80ceea0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f6cec0097a0 tx=0x7f6cec02b960 comp rx=0 tx=0).stop 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 -- 172.21.15.64:0/497963703 shutdown_connections 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 --2- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf80d29d0 0x7f6cf80d41e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 --2- 172.21.15.64:0/497963703 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf815dd90 0x7f6cf80ceea0 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 -- 172.21.15.64:0/497963703 >> 172.21.15.64:0/497963703 conn(0x7f6cf8071960 msgr2=0x7f6cf8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 -- 172.21.15.64:0/497963703 shutdown_connections 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.949+0000 7f6cff193640 1 -- 172.21.15.64:0/497963703 wait complete. 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.951+0000 7f6cff193640 1 Processor -- start 2024-03-04T07:30:59.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.951+0000 7f6cff193640 1 -- start start 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cff193640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6cf80d29d0 0x7f6cf80b1c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cff193640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf80d4720 0x7f6cf815e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cff193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf815dd90 0x7f6cf815e830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cff193640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cf80d2df0 con 0x7f6cf80d29d0 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cff193640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cf815e170 con 0x7f6cf80d4720 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cff193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6cf80b7420 con 0x7f6cf815dd90 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6cf80d29d0 0x7f6cf80b1c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6cf80d29d0 0x7f6cf80b1c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48604/0 (socket says 172.21.15.64:48604) 2024-03-04T07:30:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 -- 172.21.15.64:0/131332104 learned_addr learned my addr 172.21.15.64:0/131332104 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfd730640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf815dd90 0x7f6cf815e830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cf7fff640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf80d4720 0x7f6cf815e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 -- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf815dd90 msgr2=0x7f6cf815e830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf815dd90 0x7f6cf815e830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 -- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf80d4720 msgr2=0x7f6cf815e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf80d4720 0x7f6cf815e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.952+0000 7f6cfcf2f640 1 -- 172.21.15.64:0/131332104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cf80b21b0 con 0x7f6cf80d29d0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.953+0000 7f6cfcf2f640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6cf80d29d0 0x7f6cf80b1c70 secure :-1 s=READY pgs=1036 cs=0 l=1 rev1=1 crypto rx=0x7f6cec02bf70 tx=0x7f6cec0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.953+0000 7f6cf57fa640 1 -- 172.21.15.64:0/131332104 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cec034040 con 0x7f6cf80d29d0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.953+0000 7f6cf57fa640 1 -- 172.21.15.64:0/131332104 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cec03aaa0 con 0x7f6cf80d29d0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.953+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6cf80cb7a0 con 0x7f6cf80d29d0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.953+0000 7f6cf57fa640 1 -- 172.21.15.64:0/131332104 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cec006e10 con 0x7f6cf80d29d0 2024-03-04T07:30:59.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.953+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6cf80cbd60 con 0x7f6cf80d29d0 2024-03-04T07:30:59.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.954+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6cf80b5540 con 0x7f6cf80d29d0 2024-03-04T07:30:59.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.954+0000 7f6cf57fa640 1 -- 172.21.15.64:0/131332104 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6cec04b060 con 0x7f6cf80d29d0 2024-03-04T07:30:59.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.954+0000 7f6cf57fa640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cdc06b4d0 0x7f6cdc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:30:59.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.955+0000 7f6cf57fa640 1 -- 172.21.15.64:0/131332104 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6cec0abfb0 con 0x7f6cf80d29d0 2024-03-04T07:30:59.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.955+0000 7f6cf7fff640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cdc06b4d0 0x7f6cdc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:30:59.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.955+0000 7f6cf7fff640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cdc06b4d0 0x7f6cdc06d990 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f6cf80b9570 tx=0x7f6ce4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:30:59.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:30:59.963+0000 7f6cf57fa640 1 -- 172.21.15.64:0/131332104 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6cf80b5540 con 0x7f6cf80d29d0 2024-03-04T07:31:00.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.144+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f6cf804f410 con 0x7f6cf80d29d0 2024-03-04T07:31:00.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.144+0000 7f6cf57fa640 1 -- 172.21.15.64:0/131332104 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v79) v1 ==== 104+0+11 (secure 0 0 0) 0x7f6cec03f070 con 0x7f6cf80d29d0 2024-03-04T07:31:00.145 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cdc06b4d0 msgr2=0x7f6cdc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cdc06b4d0 0x7f6cdc06d990 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f6cf80b9570 tx=0x7f6ce4008040 comp rx=0 tx=0).stop 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6cf80d29d0 msgr2=0x7f6cf80b1c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6cf80d29d0 0x7f6cf80b1c70 secure :-1 s=READY pgs=1036 cs=0 l=1 rev1=1 crypto rx=0x7f6cec02bf70 tx=0x7f6cec0046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 shutdown_connections 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cdc06b4d0 0x7f6cdc06d990 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6cf815dd90 0x7f6cf815e830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6cf80d4720 0x7f6cf815e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 --2- 172.21.15.64:0/131332104 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6cf80d29d0 0x7f6cf80b1c70 unknown :-1 s=CLOSED pgs=1036 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 >> 172.21.15.64:0/131332104 conn(0x7f6cf8071960 msgr2=0x7f6cf80b44f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 shutdown_connections 2024-03-04T07:31:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.146+0000 7f6cff193640 1 -- 172.21.15.64:0/131332104 wait complete. 2024-03-04T07:31:00.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:31:00.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.262+0000 7f387a1d6640 1 Processor -- start 2024-03-04T07:31:00.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.262+0000 7f387a1d6640 1 -- start start 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f387a1d6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740cdcd0 0x7f38740b1550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f387a1d6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b2cd0 0x7f38740b1a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f387a1d6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f38740b1fd0 0x7f38740ce3f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f387a1d6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f38740d11e0 con 0x7f38740cdcd0 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f387a1d6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3874161a40 con 0x7f38740b1fd0 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f387a1d6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38740b2a20 con 0x7f38740b2cd0 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740cdcd0 0x7f38740b1550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f38740cdcd0 0x7f38740b1550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48616/0 (socket says 172.21.15.64:48616) 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 -- 172.21.15.64:0/1641360062 learned_addr learned my addr 172.21.15.64:0/1641360062 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f3873fff640 1 --2- 172.21.15.64:0/1641360062 >> v2:172.21.15.66:3300/0 conn(0x7f38740b1fd0 0x7f38740ce3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f3872ffd640 1 --2- 172.21.15.64:0/1641360062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b2cd0 0x7f38740b1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 -- 172.21.15.64:0/1641360062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b2cd0 msgr2=0x7f38740b1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 --2- 172.21.15.64:0/1641360062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b2cd0 0x7f38740b1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 -- 172.21.15.64:0/1641360062 >> v2:172.21.15.66:3300/0 conn(0x7f38740b1fd0 msgr2=0x7f38740ce3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 --2- 172.21.15.64:0/1641360062 >> v2:172.21.15.66:3300/0 conn(0x7f38740b1fd0 0x7f38740ce3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f38737fe640 1 -- 172.21.15.64:0/1641360062 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38740d0940 con 0x7f38740cdcd0 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.263+0000 7f3873fff640 1 --2- 172.21.15.64:0/1641360062 >> v2:172.21.15.66:3300/0 conn(0x7f38740b1fd0 0x7f38740ce3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.264+0000 7f38737fe640 1 --2- 172.21.15.64:0/1641360062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740b1550 secure :-1 s=READY pgs=1037 cs=0 l=1 rev1=1 crypto rx=0x7f387406cf10 tx=0x7f386402be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=965018faeff20f48 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.264+0000 7f38727fc640 1 -- 172.21.15.64:0/1641360062 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386402c9e0 con 0x7f38740cdcd0 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.264+0000 7f38727fc640 1 -- 172.21.15.64:0/1641360062 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f386402cb80 con 0x7f38740cdcd0 2024-03-04T07:31:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.264+0000 7f38727fc640 1 -- 172.21.15.64:0/1641360062 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38640343f0 con 0x7f38740cdcd0 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 -- 172.21.15.64:0/1641360062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 msgr2=0x7f38740b1550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 --2- 172.21.15.64:0/1641360062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740b1550 secure :-1 s=READY pgs=1037 cs=0 l=1 rev1=1 crypto rx=0x7f387406cf10 tx=0x7f386402be40 comp rx=0 tx=0).stop 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 -- 172.21.15.64:0/1641360062 shutdown_connections 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 --2- 172.21.15.64:0/1641360062 >> v2:172.21.15.66:3300/0 conn(0x7f38740b1fd0 0x7f38740ce3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 --2- 172.21.15.64:0/1641360062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b2cd0 0x7f38740b1a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 --2- 172.21.15.64:0/1641360062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740b1550 unknown :-1 s=CLOSED pgs=1037 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 -- 172.21.15.64:0/1641360062 >> 172.21.15.64:0/1641360062 conn(0x7f38740b3700 msgr2=0x7f38740b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 -- 172.21.15.64:0/1641360062 shutdown_connections 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.265+0000 7f387a1d6640 1 -- 172.21.15.64:0/1641360062 wait complete. 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.266+0000 7f387a1d6640 1 Processor -- start 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.266+0000 7f387a1d6640 1 -- start start 2024-03-04T07:31:00.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f387a1d6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b1fd0 0x7f38740cb910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f387a1d6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b2cd0 0x7f38740b5120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f387a1d6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740af9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f387a1d6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38740cc1c0 con 0x7f38740cdcd0 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f387a1d6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f38740d0f00 con 0x7f38740b2cd0 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f387a1d6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f38740cc340 con 0x7f38740b1fd0 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740af9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740af9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48618/0 (socket says 172.21.15.64:48618) 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 -- 172.21.15.64:0/2520955046 learned_addr learned my addr 172.21.15.64:0/2520955046 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f38737fe640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b1fd0 0x7f38740cb910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3872ffd640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b2cd0 0x7f38740b5120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 -- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b1fd0 msgr2=0x7f38740cb910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b1fd0 0x7f38740cb910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 -- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b2cd0 msgr2=0x7f38740b5120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b2cd0 0x7f38740b5120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 -- 172.21.15.64:0/2520955046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38740b6200 con 0x7f38740cdcd0 2024-03-04T07:31:00.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.267+0000 7f3873fff640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740af9c0 secure :-1 s=READY pgs=1038 cs=0 l=1 rev1=1 crypto rx=0x7f386c00e5e0 tx=0x7f386c00eaa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.268+0000 7f385bfff640 1 -- 172.21.15.64:0/2520955046 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c00f040 con 0x7f38740cdcd0 2024-03-04T07:31:00.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.268+0000 7f385bfff640 1 -- 172.21.15.64:0/2520955046 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f386c004230 con 0x7f38740cdcd0 2024-03-04T07:31:00.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.268+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f38740b64e0 con 0x7f38740cdcd0 2024-03-04T07:31:00.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.268+0000 7f385bfff640 1 -- 172.21.15.64:0/2520955046 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c003610 con 0x7f38740cdcd0 2024-03-04T07:31:00.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.268+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f38740b4c00 con 0x7f38740cdcd0 2024-03-04T07:31:00.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.269+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f387409fcf0 con 0x7f38740cdcd0 2024-03-04T07:31:00.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.269+0000 7f385bfff640 1 -- 172.21.15.64:0/2520955046 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f386c0037b0 con 0x7f38740cdcd0 2024-03-04T07:31:00.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.269+0000 7f385bfff640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.269+0000 7f385bfff640 1 -- 172.21.15.64:0/2520955046 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f386c089d60 con 0x7f38740cdcd0 2024-03-04T07:31:00.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.269+0000 7f38737fe640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.270+0000 7f38737fe640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f3864035000 tx=0x7f3864030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.274+0000 7f385bfff640 1 -- 172.21.15.64:0/2520955046 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f386c017020 con 0x7f38740cdcd0 2024-03-04T07:31:00.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.453+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f38740d1690 con 0x7f38740cdcd0 2024-03-04T07:31:00.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.453+0000 7f385bfff640 1 -- 172.21.15.64:0/2520955046 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v79) v1 ==== 100+0+11 (secure 0 0 0) 0x7f386c058620 con 0x7f38740cdcd0 2024-03-04T07:31:00.454 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:31:00.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 msgr2=0x7f386006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f3864035000 tx=0x7f3864030000 comp rx=0 tx=0).stop 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 msgr2=0x7f38740af9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740af9c0 secure :-1 s=READY pgs=1038 cs=0 l=1 rev1=1 crypto rx=0x7f386c00e5e0 tx=0x7f386c00eaa0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 shutdown_connections 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f386006b4d0 0x7f386006d990 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f38740cdcd0 0x7f38740af9c0 unknown :-1 s=CLOSED pgs=1038 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f38740b2cd0 0x7f38740b5120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 --2- 172.21.15.64:0/2520955046 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f38740b1fd0 0x7f38740cb910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 >> 172.21.15.64:0/2520955046 conn(0x7f38740b3700 msgr2=0x7f38740b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 shutdown_connections 2024-03-04T07:31:00.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.455+0000 7f387a1d6640 1 -- 172.21.15.64:0/2520955046 wait complete. 2024-03-04T07:31:00.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:00.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.571+0000 7f26e78fa640 1 Processor -- start 2024-03-04T07:31:00.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.571+0000 7f26e78fa640 1 -- start start 2024-03-04T07:31:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e78fa640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f26e015df00 0x7f26e00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e78fa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b95a0 0x7f26e00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e78fa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f26e00b9a30 0x7f26e00b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e78fa640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f26e00cea30 con 0x7f26e00b9a30 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e78fa640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f26e00d2af0 con 0x7f26e015df00 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e78fa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f26e00cf3c0 con 0x7f26e00b95a0 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e5e97640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f26e00b9a30 0x7f26e00b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e5e97640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f26e00b9a30 0x7f26e00b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48628/0 (socket says 172.21.15.64:48628) 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.572+0000 7f26e5e97640 1 -- 172.21.15.64:0/2124497690 learned_addr learned my addr 172.21.15.64:0/2124497690 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5696640 1 --2- 172.21.15.64:0/2124497690 >> v2:172.21.15.66:3300/0 conn(0x7f26e015df00 0x7f26e00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e4e95640 1 --2- 172.21.15.64:0/2124497690 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b95a0 0x7f26e00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5e97640 1 -- 172.21.15.64:0/2124497690 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b95a0 msgr2=0x7f26e00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5e97640 1 --2- 172.21.15.64:0/2124497690 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b95a0 0x7f26e00b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5e97640 1 -- 172.21.15.64:0/2124497690 >> v2:172.21.15.66:3300/0 conn(0x7f26e015df00 msgr2=0x7f26e00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:00.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5e97640 1 --2- 172.21.15.64:0/2124497690 >> v2:172.21.15.66:3300/0 conn(0x7f26e015df00 0x7f26e00d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5e97640 1 -- 172.21.15.64:0/2124497690 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f26e00cf0e0 con 0x7f26e00b9a30 2024-03-04T07:31:00.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5696640 1 --2- 172.21.15.64:0/2124497690 >> v2:172.21.15.66:3300/0 conn(0x7f26e015df00 0x7f26e00d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:00.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e4e95640 1 --2- 172.21.15.64:0/2124497690 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b95a0 0x7f26e00b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26e5e97640 1 --2- 172.21.15.64:0/2124497690 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b9a30 0x7f26e00b5c40 secure :-1 s=READY pgs=1039 cs=0 l=1 rev1=1 crypto rx=0x7f26e00d11b0 tx=0x7f26d802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d55bba4c9cdc6af4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.573+0000 7f26d3fff640 1 -- 172.21.15.64:0/2124497690 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f26d802ca70 con 0x7f26e00b9a30 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26d3fff640 1 -- 172.21.15.64:0/2124497690 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f26d802cc10 con 0x7f26e00b9a30 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26d3fff640 1 -- 172.21.15.64:0/2124497690 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f26d80343f0 con 0x7f26e00b9a30 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 -- 172.21.15.64:0/2124497690 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b9a30 msgr2=0x7f26e00b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 --2- 172.21.15.64:0/2124497690 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b9a30 0x7f26e00b5c40 secure :-1 s=READY pgs=1039 cs=0 l=1 rev1=1 crypto rx=0x7f26e00d11b0 tx=0x7f26d802bed0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 -- 172.21.15.64:0/2124497690 shutdown_connections 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 --2- 172.21.15.64:0/2124497690 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b9a30 0x7f26e00b5c40 unknown :-1 s=CLOSED pgs=1039 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 --2- 172.21.15.64:0/2124497690 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b95a0 0x7f26e00b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 --2- 172.21.15.64:0/2124497690 >> v2:172.21.15.66:3300/0 conn(0x7f26e015df00 0x7f26e00d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 -- 172.21.15.64:0/2124497690 >> 172.21.15.64:0/2124497690 conn(0x7f26e0071960 msgr2=0x7f26e0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:00.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 -- 172.21.15.64:0/2124497690 shutdown_connections 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.574+0000 7f26e78fa640 1 -- 172.21.15.64:0/2124497690 wait complete. 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.575+0000 7f26e78fa640 1 Processor -- start 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.575+0000 7f26e78fa640 1 -- start start 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e78fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b95a0 0x7f26e00b49e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e78fa640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26e015df00 0x7f26e00b4f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e78fa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b2df0 0x7f26e00b32a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e78fa640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f26e00cec70 con 0x7f26e00b95a0 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e78fa640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f26e00ce750 con 0x7f26e015df00 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e78fa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f26e00cedf0 con 0x7f26e00b2df0 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e5696640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b95a0 0x7f26e00b49e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e5696640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b95a0 0x7f26e00b49e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48638/0 (socket says 172.21.15.64:48638) 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e5696640 1 -- 172.21.15.64:0/46003155 learned_addr learned my addr 172.21.15.64:0/46003155 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e5696640 1 -- 172.21.15.64:0/46003155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b2df0 msgr2=0x7f26e00b32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e5696640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b2df0 0x7f26e00b32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e4e95640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26e015df00 0x7f26e00b4f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e5696640 1 -- 172.21.15.64:0/46003155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26e015df00 msgr2=0x7f26e00b4f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.576+0000 7f26e5696640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26e015df00 0x7f26e00b4f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.577+0000 7f26e5696640 1 -- 172.21.15.64:0/46003155 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f26e015f5d0 con 0x7f26e00b95a0 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.577+0000 7f26e5696640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b95a0 0x7f26e00b49e0 secure :-1 s=READY pgs=1040 cs=0 l=1 rev1=1 crypto rx=0x7f26dc00c7f0 tx=0x7f26dc00cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.577+0000 7f26d1ffb640 1 -- 172.21.15.64:0/46003155 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f26dc015070 con 0x7f26e00b95a0 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.577+0000 7f26d1ffb640 1 -- 172.21.15.64:0/46003155 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f26dc007a30 con 0x7f26e00b95a0 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.577+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f26e00b3960 con 0x7f26e00b95a0 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.577+0000 7f26d1ffb640 1 -- 172.21.15.64:0/46003155 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f26dc007e00 con 0x7f26e00b95a0 2024-03-04T07:31:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.577+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f26e00b3f20 con 0x7f26e00b95a0 2024-03-04T07:31:00.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.578+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f26e0067ff0 con 0x7f26e00b95a0 2024-03-04T07:31:00.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.578+0000 7f26d1ffb640 1 -- 172.21.15.64:0/46003155 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f26dc018070 con 0x7f26e00b95a0 2024-03-04T07:31:00.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.579+0000 7f26d1ffb640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f26c406b4d0 0x7f26c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.579+0000 7f26d1ffb640 1 -- 172.21.15.64:0/46003155 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f26dc08a100 con 0x7f26e00b95a0 2024-03-04T07:31:00.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.579+0000 7f26e4e95640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f26c406b4d0 0x7f26c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.579+0000 7f26e4e95640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f26c406b4d0 0x7f26c406d990 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f26e015b4d0 tx=0x7f26d4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.583+0000 7f26d1ffb640 1 -- 172.21.15.64:0/46003155 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f26dc059010 con 0x7f26e00b95a0 2024-03-04T07:31:00.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.802+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f26e00d3810 con 0x7f26e00b95a0 2024-03-04T07:31:00.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.802+0000 7f26d1ffb640 1 -- 172.21.15.64:0/46003155 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 43 v43) v1 ==== 94+0+4973 (secure 0 0 0) 0x7f26dc058910 con 0x7f26e00b95a0 2024-03-04T07:31:00.803 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:00.803 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[{"mdsmap":{"epoch":43,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:48.344908+0000","modified":"2024-03-04T07:30:50.410439+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6966},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6966":{"gid":6966,"name":"c","rank":0,"incarnation":41,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/1889124368","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1889124368},{"type":"v1","addr":"172.21.15.64:6836","nonce":1889124368}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6966,"qdb_cluster":[6966]},"id":6}]} 2024-03-04T07:31:00.804 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 43 2024-03-04T07:31:00.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f26c406b4d0 msgr2=0x7f26c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f26c406b4d0 0x7f26c406d990 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f26e015b4d0 tx=0x7f26d4008040 comp rx=0 tx=0).stop 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b95a0 msgr2=0x7f26e00b49e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b95a0 0x7f26e00b49e0 secure :-1 s=READY pgs=1040 cs=0 l=1 rev1=1 crypto rx=0x7f26dc00c7f0 tx=0x7f26dc00cbf0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 shutdown_connections 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f26c406b4d0 0x7f26c406d990 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26e00b2df0 0x7f26e00b32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26e015df00 0x7f26e00b4f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 --2- 172.21.15.64:0/46003155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26e00b95a0 0x7f26e00b49e0 unknown :-1 s=CLOSED pgs=1040 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 >> 172.21.15.64:0/46003155 conn(0x7f26e0071960 msgr2=0x7f26e00d3d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 shutdown_connections 2024-03-04T07:31:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.804+0000 7f26e78fa640 1 -- 172.21.15.64:0/46003155 wait complete. 2024-03-04T07:31:00.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:00.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.957+0000 7f2857471640 1 Processor -- start 2024-03-04T07:31:00.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.957+0000 7f2857471640 1 -- start start 2024-03-04T07:31:00.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2857471640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f285015df00 0x7f28500d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2857471640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2857471640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28500cea30 con 0x7f28500b9a30 2024-03-04T07:31:00.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2857471640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f28500d2af0 con 0x7f285015df00 2024-03-04T07:31:00.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2857471640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f28500cf3c0 con 0x7f28500b95a0 2024-03-04T07:31:00.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2855a0e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f28500b9a30 0x7f28500d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35658/0 (socket says 172.21.15.64:35658) 2024-03-04T07:31:00.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2855a0e640 1 -- 172.21.15.64:0/452347696 learned_addr learned my addr 172.21.15.64:0/452347696 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2854a0c640 1 --2- 172.21.15.64:0/452347696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f285520d640 1 --2- 172.21.15.64:0/452347696 >> v2:172.21.15.66:3300/0 conn(0x7f285015df00 0x7f28500d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2847fff640 1 -- 172.21.15.64:0/452347696 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1213284748 0 0) 0x7f28500cea30 con 0x7f28500b9a30 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2847fff640 1 -- 172.21.15.64:0/452347696 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f283c003520 con 0x7f28500b9a30 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2854a0c640 1 -- 172.21.15.64:0/452347696 >> v2:172.21.15.66:3300/0 conn(0x7f285015df00 msgr2=0x7f28500d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2854a0c640 1 --2- 172.21.15.64:0/452347696 >> v2:172.21.15.66:3300/0 conn(0x7f285015df00 0x7f28500d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2854a0c640 1 -- 172.21.15.64:0/452347696 >> v1:172.21.15.64:6789/0 conn(0x7f28500b9a30 legacy=0x7f28500d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.958+0000 7f2854a0c640 1 -- 172.21.15.64:0/452347696 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28500cf0e0 con 0x7f28500b95a0 2024-03-04T07:31:00.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.959+0000 7f2854a0c640 1 --2- 172.21.15.64:0/452347696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500b5700 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f28400097a0 tx=0x7f284002b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9b104974a84e856c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2847fff640 1 -- 172.21.15.64:0/452347696 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2840035020 con 0x7f28500b95a0 2024-03-04T07:31:00.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2847fff640 1 -- 172.21.15.64:0/452347696 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2840031830 con 0x7f28500b95a0 2024-03-04T07:31:00.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2847fff640 1 -- 172.21.15.64:0/452347696 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2840031c00 con 0x7f28500b95a0 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 -- 172.21.15.64:0/452347696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 msgr2=0x7f28500b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 --2- 172.21.15.64:0/452347696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500b5700 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f28400097a0 tx=0x7f284002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 -- 172.21.15.64:0/452347696 shutdown_connections 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 --2- 172.21.15.64:0/452347696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500b5700 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 --2- 172.21.15.64:0/452347696 >> v2:172.21.15.66:3300/0 conn(0x7f285015df00 0x7f28500d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 -- 172.21.15.64:0/452347696 >> 172.21.15.64:0/452347696 conn(0x7f2850071960 msgr2=0x7f2850071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 -- 172.21.15.64:0/452347696 shutdown_connections 2024-03-04T07:31:00.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.960+0000 7f2857471640 1 -- 172.21.15.64:0/452347696 wait complete. 2024-03-04T07:31:00.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.961+0000 7f2857471640 1 Processor -- start 2024-03-04T07:31:00.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2857471640 1 -- start start 2024-03-04T07:31:00.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2857471640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500c8ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2857471640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f285015df00 0x7f28500b3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2857471640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f285015b670 0x7f28500b4810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2857471640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28500cec70 con 0x7f285015df00 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2857471640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28500ce750 con 0x7f285015b670 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2857471640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f28500cedf0 con 0x7f28500b95a0 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2854a0c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f285015df00 0x7f28500b3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2854a0c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f285015df00 0x7f28500b3960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48640/0 (socket says 172.21.15.64:48640) 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f2854a0c640 1 -- 172.21.15.64:0/2676153570 learned_addr learned my addr 172.21.15.64:0/2676153570 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.962+0000 7f285520d640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500c8ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2855a0e640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f285015b670 0x7f28500b4810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2854a0c640 1 -- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 msgr2=0x7f28500c8ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2854a0c640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500c8ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2854a0c640 1 -- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f285015b670 msgr2=0x7f28500b4810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2854a0c640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f285015b670 0x7f28500b4810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2854a0c640 1 -- 172.21.15.64:0/2676153570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28500b3ea0 con 0x7f285015df00 2024-03-04T07:31:00.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f285520d640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500c8ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:00.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2854a0c640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f285015df00 0x7f28500b3960 secure :-1 s=READY pgs=1042 cs=0 l=1 rev1=1 crypto rx=0x7f284002bee0 tx=0x7f28400046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2845ffb640 1 -- 172.21.15.64:0/2676153570 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2840035040 con 0x7f285015df00 2024-03-04T07:31:00.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2845ffb640 1 -- 172.21.15.64:0/2676153570 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f284003b920 con 0x7f285015df00 2024-03-04T07:31:00.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28500c7c00 con 0x7f285015df00 2024-03-04T07:31:00.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2845ffb640 1 -- 172.21.15.64:0/2676153570 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f284003bd10 con 0x7f285015df00 2024-03-04T07:31:00.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.963+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28500c81c0 con 0x7f285015df00 2024-03-04T07:31:00.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.964+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2850067ff0 con 0x7f285015df00 2024-03-04T07:31:00.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.965+0000 7f2845ffb640 1 -- 172.21.15.64:0/2676153570 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2840038020 con 0x7f285015df00 2024-03-04T07:31:00.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.965+0000 7f2845ffb640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f283406b500 0x7f283406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:00.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.965+0000 7f2845ffb640 1 -- 172.21.15.64:0/2676153570 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f28400ac3a0 con 0x7f285015df00 2024-03-04T07:31:00.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.965+0000 7f285520d640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f283406b500 0x7f283406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:00.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.966+0000 7f285520d640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f283406b500 0x7f283406d9c0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f284c009af0 tx=0x7f284c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:00.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:00.969+0000 7f2845ffb640 1 -- 172.21.15.64:0/2676153570 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f284007b100 con 0x7f285015df00 2024-03-04T07:31:01.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.151+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f28500ca900 con 0x7f285015df00 2024-03-04T07:31:01.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.152+0000 7f2845ffb640 1 -- 172.21.15.64:0/2676153570 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v79) v1 ==== 74+0+16612 (secure 0 0 0) 0x7f284007aa00 con 0x7f285015df00 2024-03-04T07:31:01.153 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:01.153 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":79,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:49.980908+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:46.621410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:47.317858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"79","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f283406b500 msgr2=0x7f283406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f283406b500 0x7f283406d9c0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f284c009af0 tx=0x7f284c008040 comp rx=0 tx=0).stop 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f285015df00 msgr2=0x7f28500b3960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f285015df00 0x7f28500b3960 secure :-1 s=READY pgs=1042 cs=0 l=1 rev1=1 crypto rx=0x7f284002bee0 tx=0x7f28400046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 shutdown_connections 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f283406b500 0x7f283406d9c0 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f285015b670 0x7f28500b4810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f285015df00 0x7f28500b3960 unknown :-1 s=CLOSED pgs=1042 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 --2- 172.21.15.64:0/2676153570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28500b95a0 0x7f28500c8ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 >> 172.21.15.64:0/2676153570 conn(0x7f2850071960 msgr2=0x7f28500b80e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:01.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 shutdown_connections 2024-03-04T07:31:01.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.154+0000 7f2857471640 1 -- 172.21.15.64:0/2676153570 wait complete. 2024-03-04T07:31:01.169 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:31:01.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:01.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.271+0000 7f09ea1b6640 1 Processor -- start 2024-03-04T07:31:01.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.271+0000 7f09ea1b6640 1 -- start start 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09ea1b6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f09e415df00 0x7f09e40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09ea1b6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f09e40b95a0 0x7f09e40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09ea1b6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f09e40cea30 con 0x7f09e40b9a30 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09ea1b6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f09e40d2af0 con 0x7f09e40b95a0 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09ea1b6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f09e40cf3c0 con 0x7f09e415df00 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f09e40b95a0 0x7f09e40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e2ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f09e40b95a0 0x7f09e40b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48650/0 (socket says 172.21.15.64:48650) 2024-03-04T07:31:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e2ffd640 1 -- 172.21.15.64:0/3649330866 learned_addr learned my addr 172.21.15.64:0/3649330866 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e37fe640 1 --2- 172.21.15.64:0/3649330866 >> v2:172.21.15.66:3300/0 conn(0x7f09e415df00 0x7f09e40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e2ffd640 1 -- 172.21.15.64:0/3649330866 >> v2:172.21.15.66:3300/0 conn(0x7f09e415df00 msgr2=0x7f09e40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e2ffd640 1 --2- 172.21.15.64:0/3649330866 >> v2:172.21.15.66:3300/0 conn(0x7f09e415df00 0x7f09e40d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e2ffd640 1 -- 172.21.15.64:0/3649330866 >> v1:172.21.15.66:6789/0 conn(0x7f09e40b9a30 legacy=0x7f09e40d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e2ffd640 1 -- 172.21.15.64:0/3649330866 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09e40cf0e0 con 0x7f09e40b95a0 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.273+0000 7f09e37fe640 1 --2- 172.21.15.64:0/3649330866 >> v2:172.21.15.66:3300/0 conn(0x7f09e415df00 0x7f09e40d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.274+0000 7f09e2ffd640 1 --2- 172.21.15.64:0/3649330866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e40b95a0 0x7f09e40b5700 secure :-1 s=READY pgs=1043 cs=0 l=1 rev1=1 crypto rx=0x7f09e40d11b0 tx=0x7f09d802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3660c4b93ca48533 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.274+0000 7f09e27fc640 1 -- 172.21.15.64:0/3649330866 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d8035020 con 0x7f09e40b95a0 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.274+0000 7f09e27fc640 1 -- 172.21.15.64:0/3649330866 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f09d8031870 con 0x7f09e40b95a0 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.274+0000 7f09e27fc640 1 -- 172.21.15.64:0/3649330866 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09d8031c40 con 0x7f09e40b95a0 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 -- 172.21.15.64:0/3649330866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e40b95a0 msgr2=0x7f09e40b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/3649330866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e40b95a0 0x7f09e40b5700 secure :-1 s=READY pgs=1043 cs=0 l=1 rev1=1 crypto rx=0x7f09e40d11b0 tx=0x7f09d802b910 comp rx=0 tx=0).stop 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 -- 172.21.15.64:0/3649330866 shutdown_connections 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/3649330866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e40b95a0 0x7f09e40b5700 unknown :-1 s=CLOSED pgs=1043 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/3649330866 >> v2:172.21.15.66:3300/0 conn(0x7f09e415df00 0x7f09e40d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 -- 172.21.15.64:0/3649330866 >> 172.21.15.64:0/3649330866 conn(0x7f09e4071960 msgr2=0x7f09e4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 -- 172.21.15.64:0/3649330866 shutdown_connections 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.275+0000 7f09ea1b6640 1 -- 172.21.15.64:0/3649330866 wait complete. 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.276+0000 7f09ea1b6640 1 Processor -- start 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.276+0000 7f09ea1b6640 1 -- start start 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09ea1b6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40b95a0 0x7f09e4160d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09ea1b6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e415df00 0x7f09e415f620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09ea1b6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e415b290 0x7f09e40c8b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09ea1b6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09e40cec70 con 0x7f09e415b290 2024-03-04T07:31:01.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09ea1b6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09e40ce750 con 0x7f09e40b95a0 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09ea1b6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f09e40cedf0 con 0x7f09e415df00 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e415b290 0x7f09e40c8b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e415b290 0x7f09e40c8b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48658/0 (socket says 172.21.15.64:48658) 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 -- 172.21.15.64:0/1049155466 learned_addr learned my addr 172.21.15.64:0/1049155466 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e37fe640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40b95a0 0x7f09e4160d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e2ffd640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e415df00 0x7f09e415f620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 -- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e415df00 msgr2=0x7f09e415f620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e415df00 0x7f09e415f620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 -- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40b95a0 msgr2=0x7f09e4160d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40b95a0 0x7f09e4160d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 -- 172.21.15.64:0/1049155466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09e415fce0 con 0x7f09e415b290 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e2ffd640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e415df00 0x7f09e415f620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09e3fff640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e415b290 0x7f09e40c8b20 secure :-1 s=READY pgs=1044 cs=0 l=1 rev1=1 crypto rx=0x7f09dc00c410 tx=0x7f09dc00c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.277+0000 7f09cbfff640 1 -- 172.21.15.64:0/1049155466 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09dc011790 con 0x7f09e415b290 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.278+0000 7f09cbfff640 1 -- 172.21.15.64:0/1049155466 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f09dc017070 con 0x7f09e415b290 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.278+0000 7f09cbfff640 1 -- 172.21.15.64:0/1049155466 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f09dc011bb0 con 0x7f09e415b290 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.278+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f09e415ffc0 con 0x7f09e415b290 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.278+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f09e4160580 con 0x7f09e415b290 2024-03-04T07:31:01.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.279+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f09e4067ff0 con 0x7f09e415b290 2024-03-04T07:31:01.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.279+0000 7f09cbfff640 1 -- 172.21.15.64:0/1049155466 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f09dc01c460 con 0x7f09e415b290 2024-03-04T07:31:01.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.279+0000 7f09cbfff640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09d006b500 0x7f09d006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.280+0000 7f09e37fe640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09d006b500 0x7f09d006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.280+0000 7f09cbfff640 1 -- 172.21.15.64:0/1049155466 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f09dc089db0 con 0x7f09e415b290 2024-03-04T07:31:01.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.280+0000 7f09e37fe640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09d006b500 0x7f09d006d9c0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f09d4007510 tx=0x7f09d40074a0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:01.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.284+0000 7f09cbfff640 1 -- 172.21.15.64:0/1049155466 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f09dc021020 con 0x7f09e415b290 2024-03-04T07:31:01.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.580+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f09e40c7560 con 0x7f09e415b290 2024-03-04T07:31:01.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.581+0000 7f09cbfff640 1 -- 172.21.15.64:0/1049155466 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 43 v43) v1 ==== 94+0+4973 (secure 0 0 0) 0x7f09dc058c30 con 0x7f09e415b290 2024-03-04T07:31:01.582 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:01.582 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[{"mdsmap":{"epoch":43,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:48.344908+0000","modified":"2024-03-04T07:30:50.410439+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6966},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6966":{"gid":6966,"name":"c","rank":0,"incarnation":41,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/1889124368","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1889124368},{"type":"v1","addr":"172.21.15.64:6836","nonce":1889124368}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6966,"qdb_cluster":[6966]},"id":6}]} 2024-03-04T07:31:01.582 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 43 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09d006b500 msgr2=0x7f09d006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09d006b500 0x7f09d006d9c0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f09d4007510 tx=0x7f09d40074a0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e415b290 msgr2=0x7f09e40c8b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e415b290 0x7f09e40c8b20 secure :-1 s=READY pgs=1044 cs=0 l=1 rev1=1 crypto rx=0x7f09dc00c410 tx=0x7f09dc00c8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 shutdown_connections 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09d006b500 0x7f09d006d9c0 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f09e415b290 0x7f09e40c8b20 unknown :-1 s=CLOSED pgs=1044 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09e415df00 0x7f09e415f620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 --2- 172.21.15.64:0/1049155466 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f09e40b95a0 0x7f09e4160d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 >> 172.21.15.64:0/1049155466 conn(0x7f09e4071960 msgr2=0x7f09e40b5b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 shutdown_connections 2024-03-04T07:31:01.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.584+0000 7f09ea1b6640 1 -- 172.21.15.64:0/1049155466 wait complete. 2024-03-04T07:31:01.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:01.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efd04bbc640 1 Processor -- start 2024-03-04T07:31:01.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efd04bbc640 1 -- start start 2024-03-04T07:31:01.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efd04bbc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efd0015df00 0x7efd000d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efd04bbc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efd000b9a30 0x7efd000d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efd04bbc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efd000cea30 con 0x7efd000b95a0 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efd04bbc640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efd000d2af0 con 0x7efd000b9a30 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efd04bbc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efd000cf3c0 con 0x7efd0015df00 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.697+0000 7efcfed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efd000b9a30 0x7efd000d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efd000b9a30 0x7efd000d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48660/0 (socket says 172.21.15.64:48660) 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfed9d640 1 -- 172.21.15.64:0/519575525 learned_addr learned my addr 172.21.15.64:0/519575525 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfe59c640 1 --2- 172.21.15.64:0/519575525 >> v2:172.21.15.66:3300/0 conn(0x7efd0015df00 0x7efd000d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfed9d640 1 -- 172.21.15.64:0/519575525 >> v2:172.21.15.66:3300/0 conn(0x7efd0015df00 msgr2=0x7efd000d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfed9d640 1 --2- 172.21.15.64:0/519575525 >> v2:172.21.15.66:3300/0 conn(0x7efd0015df00 0x7efd000d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfdd9b640 1 -- 172.21.15.64:0/519575525 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3310719860 0 0) 0x7efd000cea30 con 0x7efd000b95a0 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfed9d640 1 -- 172.21.15.64:0/519575525 >> v1:172.21.15.64:6789/0 conn(0x7efd000b95a0 legacy=0x7efd000b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.698+0000 7efcfed9d640 1 -- 172.21.15.64:0/519575525 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efd000cf0e0 con 0x7efd000b9a30 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efcfed9d640 1 --2- 172.21.15.64:0/519575525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd000b9a30 0x7efd000d3220 secure :-1 s=READY pgs=1046 cs=0 l=1 rev1=1 crypto rx=0x7efd000d11b0 tx=0x7efcf802bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=abd3b9a083338756 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efcfdd9b640 1 -- 172.21.15.64:0/519575525 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efcf802cb20 con 0x7efd000b9a30 2024-03-04T07:31:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efcfdd9b640 1 -- 172.21.15.64:0/519575525 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efcf802ccc0 con 0x7efd000b9a30 2024-03-04T07:31:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efcfdd9b640 1 -- 172.21.15.64:0/519575525 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efcf8034610 con 0x7efd000b9a30 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 -- 172.21.15.64:0/519575525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd000b9a30 msgr2=0x7efd000d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 --2- 172.21.15.64:0/519575525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd000b9a30 0x7efd000d3220 secure :-1 s=READY pgs=1046 cs=0 l=1 rev1=1 crypto rx=0x7efd000d11b0 tx=0x7efcf802bf80 comp rx=0 tx=0).stop 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 -- 172.21.15.64:0/519575525 shutdown_connections 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 --2- 172.21.15.64:0/519575525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd000b9a30 0x7efd000d3220 unknown :-1 s=CLOSED pgs=1046 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 --2- 172.21.15.64:0/519575525 >> v2:172.21.15.66:3300/0 conn(0x7efd0015df00 0x7efd000d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 -- 172.21.15.64:0/519575525 >> 172.21.15.64:0/519575525 conn(0x7efd00071960 msgr2=0x7efd00071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 -- 172.21.15.64:0/519575525 shutdown_connections 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.699+0000 7efd04bbc640 1 -- 172.21.15.64:0/519575525 wait complete. 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.701+0000 7efd04bbc640 1 Processor -- start 2024-03-04T07:31:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.701+0000 7efd04bbc640 1 -- start start 2024-03-04T07:31:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efd04bbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efd000b95a0 0x7efd000cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efd04bbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efd0015df00 0x7efd000b49d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efd04bbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd001555f0 0x7efd000b4f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efd04bbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efd000cec70 con 0x7efd001555f0 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efd04bbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efd000ce750 con 0x7efd000b95a0 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efd04bbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efd000cedf0 con 0x7efd0015df00 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd001555f0 0x7efd000b4f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd001555f0 0x7efd000b4f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48666/0 (socket says 172.21.15.64:48666) 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 -- 172.21.15.64:0/369003502 learned_addr learned my addr 172.21.15.64:0/369003502 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfe59c640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efd000b95a0 0x7efd000cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcf7fff640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efd0015df00 0x7efd000b49d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 -- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efd0015df00 msgr2=0x7efd000b49d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efd0015df00 0x7efd000b49d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 -- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efd000b95a0 msgr2=0x7efd000cad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efd000b95a0 0x7efd000cad50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfed9d640 1 -- 172.21.15.64:0/369003502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efd0015e2e0 con 0x7efd001555f0 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcfe59c640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efd000b95a0 0x7efd000cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:01.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.702+0000 7efcf7fff640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efd0015df00 0x7efd000b49d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.703+0000 7efcfed9d640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd001555f0 0x7efd000b4f10 secure :-1 s=READY pgs=1047 cs=0 l=1 rev1=1 crypto rx=0x7efcf8009f40 tx=0x7efcf8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.703+0000 7efcf6ffd640 1 -- 172.21.15.64:0/369003502 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efcf8043030 con 0x7efd001555f0 2024-03-04T07:31:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.703+0000 7efcf6ffd640 1 -- 172.21.15.64:0/369003502 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efcf803e450 con 0x7efd001555f0 2024-03-04T07:31:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.703+0000 7efcf6ffd640 1 -- 172.21.15.64:0/369003502 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efcf803e620 con 0x7efd001555f0 2024-03-04T07:31:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.703+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efd000cb810 con 0x7efd001555f0 2024-03-04T07:31:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.703+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efd000cbdd0 con 0x7efd001555f0 2024-03-04T07:31:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.704+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efd00067ff0 con 0x7efd001555f0 2024-03-04T07:31:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.704+0000 7efcf6ffd640 1 -- 172.21.15.64:0/369003502 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efcf8046070 con 0x7efd001555f0 2024-03-04T07:31:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.705+0000 7efcf6ffd640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efce006b4d0 0x7efce006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.705+0000 7efcf6ffd640 1 -- 172.21.15.64:0/369003502 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7efcf80ac950 con 0x7efd001555f0 2024-03-04T07:31:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.705+0000 7efcfe59c640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efce006b4d0 0x7efce006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.705+0000 7efcfe59c640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efce006b4d0 0x7efce006d990 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7efcf000b820 tx=0x7efcf0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:01.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.713+0000 7efcf6ffd640 1 -- 172.21.15.64:0/369003502 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efcf804d070 con 0x7efd001555f0 2024-03-04T07:31:01.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.938+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7efd0006b780 con 0x7efd001555f0 2024-03-04T07:31:01.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.938+0000 7efcf6ffd640 1 -- 172.21.15.64:0/369003502 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 43 v43) v1 ==== 94+0+4973 (secure 0 0 0) 0x7efcf807b7c0 con 0x7efd001555f0 2024-03-04T07:31:01.939 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:01.939 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":43,"default_fscid":6,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[{"mdsmap":{"epoch":43,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:30:48.344908+0000","modified":"2024-03-04T07:30:50.410439+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":6966},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6966":{"gid":6966,"name":"c","rank":0,"incarnation":41,"state":"up:active","state_seq":3,"addr":"172.21.15.64:6836/1889124368","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1889124368},{"type":"v1","addr":"172.21.15.64:6836","nonce":1889124368}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[13],"metadata_pool":12,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":6966,"qdb_cluster":[6966]},"id":6}]} 2024-03-04T07:31:01.939 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 43 2024-03-04T07:31:01.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efce006b4d0 msgr2=0x7efce006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efce006b4d0 0x7efce006d990 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7efcf000b820 tx=0x7efcf0008040 comp rx=0 tx=0).stop 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd001555f0 msgr2=0x7efd000b4f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd001555f0 0x7efd000b4f10 secure :-1 s=READY pgs=1047 cs=0 l=1 rev1=1 crypto rx=0x7efcf8009f40 tx=0x7efcf8004a40 comp rx=0 tx=0).stop 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 shutdown_connections 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efce006b4d0 0x7efce006d990 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efd001555f0 0x7efd000b4f10 unknown :-1 s=CLOSED pgs=1047 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efd0015df00 0x7efd000b49d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.940+0000 7efd04bbc640 1 --2- 172.21.15.64:0/369003502 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efd000b95a0 0x7efd000cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.941+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 >> 172.21.15.64:0/369003502 conn(0x7efd00071960 msgr2=0x7efd000b81f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.941+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 shutdown_connections 2024-03-04T07:31:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:01.941+0000 7efd04bbc640 1 -- 172.21.15.64:0/369003502 wait complete. 2024-03-04T07:31:01.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:02.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.059+0000 7fa0819c9640 1 Processor -- start 2024-03-04T07:31:02.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.059+0000 7fa0819c9640 1 -- start start 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa0819c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa07c0cead0 0x7fa07c0b13a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa0819c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa07c0cf610 con 0x7fa07c15bdc0 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa0819c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa07c0cf8f0 con 0x7fa07c0d06b0 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa0819c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa07c0cfbd0 con 0x7fa07c0cead0 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa07b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa07c0cead0 0x7fa07c0b13a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa07b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa07c0cead0 0x7fa07c0b13a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48678/0 (socket says 172.21.15.64:48678) 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa07b7fe640 1 -- 172.21.15.64:0/3143806732 learned_addr learned my addr 172.21.15.64:0/3143806732 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:02.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa07b7fe640 1 -- 172.21.15.64:0/3143806732 >> v1:172.21.15.66:6789/0 conn(0x7fa07c0d06b0 legacy=0x7fa07c0ce3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:02.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa07b7fe640 1 -- 172.21.15.64:0/3143806732 >> v1:172.21.15.64:6789/0 conn(0x7fa07c15bdc0 legacy=0x7fa07c0cd210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.060+0000 7fa07b7fe640 1 -- 172.21.15.64:0/3143806732 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa07c0cfeb0 con 0x7fa07c0cead0 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.061+0000 7fa07b7fe640 1 --2- 172.21.15.64:0/3143806732 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b13a0 secure :-1 s=READY pgs=1049 cs=0 l=1 rev1=1 crypto rx=0x7fa074009e50 tx=0x7fa07402c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5edf10e692227fb0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.061+0000 7fa079ffb640 1 -- 172.21.15.64:0/3143806732 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa07402cbb0 con 0x7fa07c0cead0 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.061+0000 7fa079ffb640 1 -- 172.21.15.64:0/3143806732 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa07402cd50 con 0x7fa07c0cead0 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.061+0000 7fa079ffb640 1 -- 172.21.15.64:0/3143806732 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa074034610 con 0x7fa07c0cead0 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.062+0000 7fa0819c9640 1 -- 172.21.15.64:0/3143806732 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0cead0 msgr2=0x7fa07c0b13a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.062+0000 7fa0819c9640 1 --2- 172.21.15.64:0/3143806732 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b13a0 secure :-1 s=READY pgs=1049 cs=0 l=1 rev1=1 crypto rx=0x7fa074009e50 tx=0x7fa07402c010 comp rx=0 tx=0).stop 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.062+0000 7fa0819c9640 1 -- 172.21.15.64:0/3143806732 shutdown_connections 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.062+0000 7fa0819c9640 1 --2- 172.21.15.64:0/3143806732 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b13a0 unknown :-1 s=CLOSED pgs=1049 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.062+0000 7fa0819c9640 1 -- 172.21.15.64:0/3143806732 >> 172.21.15.64:0/3143806732 conn(0x7fa07c0d1a40 msgr2=0x7fa07c0d1e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:02.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.062+0000 7fa0819c9640 1 -- 172.21.15.64:0/3143806732 shutdown_connections 2024-03-04T07:31:02.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.062+0000 7fa0819c9640 1 -- 172.21.15.64:0/3143806732 wait complete. 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.063+0000 7fa0819c9640 1 Processor -- start 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.063+0000 7fa0819c9640 1 -- start start 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa0819c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b6fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa0819c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0d06b0 0x7fa07c157e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa0819c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa07c15bdc0 0x7fa07c159ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa0819c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa07c0d2760 con 0x7fa07c0d06b0 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa0819c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa07c0d0b10 con 0x7fa07c0cead0 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa0819c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa07c15c1a0 con 0x7fa07c15bdc0 2024-03-04T07:31:02.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0d06b0 0x7fa07c157e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0d06b0 0x7fa07c157e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48690/0 (socket says 172.21.15.64:48690) 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07a7fc640 1 -- 172.21.15.64:0/1207077629 learned_addr learned my addr 172.21.15.64:0/1207077629 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07affd640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b6fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07b7fe640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa07c15bdc0 0x7fa07c159ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07a7fc640 1 -- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa07c15bdc0 msgr2=0x7fa07c159ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07a7fc640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa07c15bdc0 0x7fa07c159ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07a7fc640 1 -- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa07c0cead0 msgr2=0x7fa07c0b6fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.064+0000 7fa07a7fc640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b6fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa07a7fc640 1 -- 172.21.15.64:0/1207077629 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa07c0ca7a0 con 0x7fa07c0d06b0 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa07affd640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b6fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa07a7fc640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0d06b0 0x7fa07c157e70 secure :-1 s=READY pgs=1050 cs=0 l=1 rev1=1 crypto rx=0x7fa06c00c410 tx=0x7fa06c00c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa063fff640 1 -- 172.21.15.64:0/1207077629 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa06c017070 con 0x7fa07c0d06b0 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa063fff640 1 -- 172.21.15.64:0/1207077629 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa06c0035f0 con 0x7fa07c0d06b0 2024-03-04T07:31:02.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa07c0caa80 con 0x7fa07c0d06b0 2024-03-04T07:31:02.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa063fff640 1 -- 172.21.15.64:0/1207077629 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa06c003a20 con 0x7fa07c0d06b0 2024-03-04T07:31:02.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.065+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa07c0cb040 con 0x7fa07c0d06b0 2024-03-04T07:31:02.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.066+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa07c068700 con 0x7fa07c0d06b0 2024-03-04T07:31:02.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.066+0000 7fa063fff640 1 -- 172.21.15.64:0/1207077629 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa06c01a070 con 0x7fa07c0d06b0 2024-03-04T07:31:02.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.067+0000 7fa063fff640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa06806b4d0 0x7fa06806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.067+0000 7fa063fff640 1 -- 172.21.15.64:0/1207077629 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa06c0896f0 con 0x7fa07c0d06b0 2024-03-04T07:31:02.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.067+0000 7fa07affd640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa06806b4d0 0x7fa06806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.067+0000 7fa07affd640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa06806b4d0 0x7fa06806d990 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7fa070001660 tx=0x7fa070009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.075+0000 7fa063fff640 1 -- 172.21.15.64:0/1207077629 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa06c058570 con 0x7fa07c0d06b0 2024-03-04T07:31:02.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.330+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa07c0b79c0 con 0x7fa07c0d06b0 2024-03-04T07:31:02.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.332+0000 7fa063fff640 1 -- 172.21.15.64:0/1207077629 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v79) v1 ==== 74+0+16612 (secure 0 0 0) 0x7fa06c057e70 con 0x7fa07c0d06b0 2024-03-04T07:31:02.333 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:02.333 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":79,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:30:49.980908+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":12,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:30:46.621410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":13,"pool_name":"cephfs_data","create_time":"2024-03-04T07:30:47.317858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"79","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:02.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa06806b4d0 msgr2=0x7fa06806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa06806b4d0 0x7fa06806d990 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7fa070001660 tx=0x7fa070009040 comp rx=0 tx=0).stop 2024-03-04T07:31:02.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0d06b0 msgr2=0x7fa07c157e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0d06b0 0x7fa07c157e70 secure :-1 s=READY pgs=1050 cs=0 l=1 rev1=1 crypto rx=0x7fa06c00c410 tx=0x7fa06c00c8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 shutdown_connections 2024-03-04T07:31:02.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa06806b4d0 0x7fa06806d990 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa07c15bdc0 0x7fa07c159ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa07c0d06b0 0x7fa07c157e70 unknown :-1 s=CLOSED pgs=1050 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 --2- 172.21.15.64:0/1207077629 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa07c0cead0 0x7fa07c0b6fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 >> 172.21.15.64:0/1207077629 conn(0x7fa07c0d1a40 msgr2=0x7fa07c0c45c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:02.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 shutdown_connections 2024-03-04T07:31:02.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.334+0000 7fa0819c9640 1 -- 172.21.15.64:0/1207077629 wait complete. 2024-03-04T07:31:02.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:31:02.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.448+0000 7facf666f640 1 Processor -- start 2024-03-04T07:31:02.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.448+0000 7facf666f640 1 -- start start 2024-03-04T07:31:02.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.448+0000 7facf666f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7facf0071c50 0x7facf00d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.448+0000 7facf666f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7facf00ce970 con 0x7facf015e000 2024-03-04T07:31:02.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.448+0000 7facf666f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7facf00cef30 con 0x7facf00b2c50 2024-03-04T07:31:02.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.448+0000 7facf666f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7facf00cece0 con 0x7facf0071c50 2024-03-04T07:31:02.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7facf015e000 0x7facf0071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35680/0 (socket says 172.21.15.64:35680) 2024-03-04T07:31:02.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceffff640 1 -- 172.21.15.64:0/2673612205 learned_addr learned my addr 172.21.15.64:0/2673612205 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7facef7fe640 1 --2- 172.21.15.64:0/2673612205 >> v2:172.21.15.66:3300/0 conn(0x7facf0071c50 0x7facf00d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 906442567 0 0) 0x7facf00ce970 con 0x7facf015e000 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7facd8003520 con 0x7facf015e000 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1738367416 0 0) 0x7facd8003520 con 0x7facf015e000 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 >> v2:172.21.15.66:3300/0 conn(0x7facf0071c50 msgr2=0x7facf00d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 --2- 172.21.15.64:0/2673612205 >> v2:172.21.15.66:3300/0 conn(0x7facf0071c50 0x7facf00d25d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 >> v1:172.21.15.66:6789/0 conn(0x7facf00b2c50 legacy=0x7facf00d2b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facf00cf210 con 0x7facf015e000 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7face0001670 con 0x7facf015e000 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 >> v1:172.21.15.64:6789/0 conn(0x7facf015e000 legacy=0x7facf0071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd80039b0 0x7facd8003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd8004800 0x7facd8004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80051c0 0x7facd800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf00cece0 con 0x7facd8004800 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facd800c160 con 0x7facd80039b0 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facd800c360 con 0x7facd80051c0 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.449+0000 7facef7fe640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd8004800 0x7facd8004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7faceffff640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd80039b0 0x7facd8003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7facf4c0c640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80051c0 0x7facd800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7facef7fe640 1 -- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80051c0 msgr2=0x7facd800b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7facef7fe640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80051c0 0x7facd800b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7facef7fe640 1 -- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd80039b0 msgr2=0x7facd8003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7facef7fe640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd80039b0 0x7facd8003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7facef7fe640 1 -- 172.21.15.64:0/2673612205 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facd8003520 con 0x7facd8004800 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.450+0000 7facf4c0c640 1 -- 172.21.15.64:0/2673612205 reap_dead start 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facef7fe640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd8004800 0x7facd8004c80 secure :-1 s=READY pgs=1052 cs=0 l=1 rev1=1 crypto rx=0x7face4009ac0 tx=0x7facd8005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e7ef5aee06bbd2d7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4039070 con 0x7facd8004800 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7face402c6b0 con 0x7facd8004800 2024-03-04T07:31:02.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7faceeffd640 1 -- 172.21.15.64:0/2673612205 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4009690 con 0x7facd8004800 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facf666f640 1 -- 172.21.15.64:0/2673612205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd8004800 msgr2=0x7facd8004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facf666f640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd8004800 0x7facd8004c80 secure :-1 s=READY pgs=1052 cs=0 l=1 rev1=1 crypto rx=0x7face4009ac0 tx=0x7facd8005830 comp rx=0 tx=0).stop 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facf666f640 1 -- 172.21.15.64:0/2673612205 shutdown_connections 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facf666f640 1 --2- 172.21.15.64:0/2673612205 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd8004800 0x7facd8004c80 unknown :-1 s=CLOSED pgs=1052 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facf666f640 1 -- 172.21.15.64:0/2673612205 >> 172.21.15.64:0/2673612205 conn(0x7facf00d3c80 msgr2=0x7facf00d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facf666f640 1 -- 172.21.15.64:0/2673612205 shutdown_connections 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.451+0000 7facf666f640 1 -- 172.21.15.64:0/2673612205 wait complete. 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.452+0000 7facf666f640 1 Processor -- start 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facf666f640 1 -- start start 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facf666f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8004800 0x7facf0153180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facf666f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b8420 0x7facf00b88a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facf666f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf00ab550 0x7facf00ab9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facf666f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf00b3320 con 0x7facf00b8420 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facf666f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facf00b31a0 con 0x7facf00ab550 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facf666f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facf015e3e0 con 0x7facd8004800 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b8420 0x7facf00b88a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b8420 0x7facf00b88a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48708/0 (socket says 172.21.15.64:48708) 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.453+0000 7facef7fe640 1 -- 172.21.15.64:0/89412359 learned_addr learned my addr 172.21.15.64:0/89412359 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facf4c0c640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf00ab550 0x7facf00ab9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7faceffff640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8004800 0x7facf0153180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facef7fe640 1 -- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8004800 msgr2=0x7facf0153180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facef7fe640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8004800 0x7facf0153180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facef7fe640 1 -- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf00ab550 msgr2=0x7facf00ab9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facef7fe640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf00ab550 0x7facf00ab9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facef7fe640 1 -- 172.21.15.64:0/89412359 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facf00ac290 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facef7fe640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b8420 0x7facf00b88a0 secure :-1 s=READY pgs=1053 cs=0 l=1 rev1=1 crypto rx=0x7face4000c00 tx=0x7face4004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facecff9640 1 -- 172.21.15.64:0/89412359 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4042020 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facecff9640 1 -- 172.21.15.64:0/89412359 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7face403a040 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7facf00ac570 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facecff9640 1 -- 172.21.15.64:0/89412359 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7face4039070 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.454+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7facf00acb30 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.455+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7facf00b8de0 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.456+0000 7facecff9640 1 -- 172.21.15.64:0/89412359 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7face4045020 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.456+0000 7facecff9640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.456+0000 7facecff9640 1 -- 172.21.15.64:0/89412359 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 5709+0+0 (secure 0 0 0) 0x7face40ab670 con 0x7facf00b8420 2024-03-04T07:31:02.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.456+0000 7faceffff640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.457+0000 7faceffff640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7facf01f74f0 tx=0x7face0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.461+0000 7facecff9640 1 -- 172.21.15.64:0/89412359 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7face404c070 con 0x7facf00b8420 2024-03-04T07:31:02.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.693+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7facf0067ff0 con 0x7facf00b8420 2024-03-04T07:31:02.694 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:31:02.694+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:31:02.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.695+0000 7facecff9640 1 -- 172.21.15.64:0/89412359 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v44) v1 ==== 161+0+0 (secure 0 0 0) 0x7face407a4f0 con 0x7facf00b8420 2024-03-04T07:31:02.696 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:31:02.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.698+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 msgr2=0x7facdc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.698+0000 7facf666f640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7facf01f74f0 tx=0x7face0009040 comp rx=0 tx=0).stop 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.698+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b8420 msgr2=0x7facf00b88a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.698+0000 7facf666f640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b8420 0x7facf00b88a0 secure :-1 s=READY pgs=1053 cs=0 l=1 rev1=1 crypto rx=0x7face4000c00 tx=0x7face4004730 comp rx=0 tx=0).stop 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 shutdown_connections 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facdc06b4d0 0x7facdc06d990 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facf00ab550 0x7facf00ab9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facf00b8420 0x7facf00b88a0 unknown :-1 s=CLOSED pgs=1053 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 --2- 172.21.15.64:0/89412359 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8004800 0x7facf0153180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 >> 172.21.15.64:0/89412359 conn(0x7facf00d3c80 msgr2=0x7facf015f790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:02.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 shutdown_connections 2024-03-04T07:31:02.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.699+0000 7facf666f640 1 -- 172.21.15.64:0/89412359 wait complete. 2024-03-04T07:31:02.703 INFO:tasks.ceph.mds.c.smithi064.stderr: -2507> 2024-03-04T07:30:54.957+0000 7f2671423640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:31:02.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:31:02.729 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:31:02.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.816+0000 7fd740b16640 1 Processor -- start 2024-03-04T07:31:02.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.816+0000 7fd740b16640 1 -- start start 2024-03-04T07:31:02.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd740b16640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd73c15dbd0 0x7fd73c15ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd740b16640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd73c0b40c0 0x7fd73c066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd740b16640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd73c15f3b0 con 0x7fd73c0b5840 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd740b16640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd73c06f070 con 0x7fd73c15dbd0 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd740b16640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd73c071940 con 0x7fd73c0b40c0 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd73a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd73c15dbd0 0x7fd73c15ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd73a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd73c15dbd0 0x7fd73c15ee70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48714/0 (socket says 172.21.15.64:48714) 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.817+0000 7fd73a59c640 1 -- 172.21.15.64:0/1543158166 learned_addr learned my addr 172.21.15.64:0/1543158166 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73ad9d640 1 --2- 172.21.15.64:0/1543158166 >> v2:172.21.15.66:3300/0 conn(0x7fd73c0b40c0 0x7fd73c066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73a59c640 1 -- 172.21.15.64:0/1543158166 >> v2:172.21.15.66:3300/0 conn(0x7fd73c0b40c0 msgr2=0x7fd73c066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73a59c640 1 --2- 172.21.15.64:0/1543158166 >> v2:172.21.15.66:3300/0 conn(0x7fd73c0b40c0 0x7fd73c066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73a59c640 1 -- 172.21.15.64:0/1543158166 >> v1:172.21.15.66:6789/0 conn(0x7fd73c0b5840 legacy=0x7fd73c0b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73a59c640 1 -- 172.21.15.64:0/1543158166 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd73c0b5db0 con 0x7fd73c15dbd0 2024-03-04T07:31:02.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73ad9d640 1 --2- 172.21.15.64:0/1543158166 >> v2:172.21.15.66:3300/0 conn(0x7fd73c0b40c0 0x7fd73c066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:02.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73a59c640 1 --2- 172.21.15.64:0/1543158166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c15dbd0 0x7fd73c15ee70 secure :-1 s=READY pgs=1055 cs=0 l=1 rev1=1 crypto rx=0x7fd73c0d04a0 tx=0x7fd73402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4ea14683c1b503c3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.818+0000 7fd73959a640 1 -- 172.21.15.64:0/1543158166 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd734035020 con 0x7fd73c15dbd0 2024-03-04T07:31:02.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd73959a640 1 -- 172.21.15.64:0/1543158166 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd734031870 con 0x7fd73c15dbd0 2024-03-04T07:31:02.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd73959a640 1 -- 172.21.15.64:0/1543158166 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd734031c40 con 0x7fd73c15dbd0 2024-03-04T07:31:02.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 -- 172.21.15.64:0/1543158166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c15dbd0 msgr2=0x7fd73c15ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 --2- 172.21.15.64:0/1543158166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c15dbd0 0x7fd73c15ee70 secure :-1 s=READY pgs=1055 cs=0 l=1 rev1=1 crypto rx=0x7fd73c0d04a0 tx=0x7fd73402b910 comp rx=0 tx=0).stop 2024-03-04T07:31:02.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 -- 172.21.15.64:0/1543158166 shutdown_connections 2024-03-04T07:31:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 --2- 172.21.15.64:0/1543158166 >> v2:172.21.15.66:3300/0 conn(0x7fd73c0b40c0 0x7fd73c066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 --2- 172.21.15.64:0/1543158166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c15dbd0 0x7fd73c15ee70 unknown :-1 s=CLOSED pgs=1055 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 -- 172.21.15.64:0/1543158166 >> 172.21.15.64:0/1543158166 conn(0x7fd73c0b26b0 msgr2=0x7fd73c0b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 -- 172.21.15.64:0/1543158166 shutdown_connections 2024-03-04T07:31:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.819+0000 7fd740b16640 1 -- 172.21.15.64:0/1543158166 wait complete. 2024-03-04T07:31:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.820+0000 7fd740b16640 1 Processor -- start 2024-03-04T07:31:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.820+0000 7fd740b16640 1 -- start start 2024-03-04T07:31:02.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd740b16640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd73c0b40c0 0x7fd73c1f7850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd740b16640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c0b5840 0x7fd73c14afc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd740b16640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd73c15dbd0 0x7fd73c14b500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd740b16640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd73c0721c0 con 0x7fd73c0b5840 2024-03-04T07:31:02.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd740b16640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd73c071ee0 con 0x7fd73c0b40c0 2024-03-04T07:31:02.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd740b16640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd73c0726e0 con 0x7fd73c15dbd0 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd739d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c0b5840 0x7fd73c14afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd739d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c0b5840 0x7fd73c14afc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:48716/0 (socket says 172.21.15.64:48716) 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd739d9b640 1 -- 172.21.15.64:0/1470370548 learned_addr learned my addr 172.21.15.64:0/1470370548 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd73ad9d640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd73c15dbd0 0x7fd73c14b500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.821+0000 7fd73a59c640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd73c0b40c0 0x7fd73c1f7850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd739d9b640 1 -- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd73c15dbd0 msgr2=0x7fd73c14b500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd739d9b640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd73c15dbd0 0x7fd73c14b500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd739d9b640 1 -- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd73c0b40c0 msgr2=0x7fd73c1f7850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd739d9b640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd73c0b40c0 0x7fd73c1f7850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd739d9b640 1 -- 172.21.15.64:0/1470370548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd73c1f4070 con 0x7fd73c0b5840 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd73a59c640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd73c0b40c0 0x7fd73c1f7850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd739d9b640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c0b5840 0x7fd73c14afc0 secure :-1 s=READY pgs=1056 cs=0 l=1 rev1=1 crypto rx=0x7fd72400c430 tx=0x7fd72400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd732ffd640 1 -- 172.21.15.64:0/1470370548 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd724011790 con 0x7fd73c0b5840 2024-03-04T07:31:02.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd732ffd640 1 -- 172.21.15.64:0/1470370548 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd724017070 con 0x7fd73c0b5840 2024-03-04T07:31:02.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd73c155930 con 0x7fd73c0b5840 2024-03-04T07:31:02.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd732ffd640 1 -- 172.21.15.64:0/1470370548 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd724011bb0 con 0x7fd73c0b5840 2024-03-04T07:31:02.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.822+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd73c0b1610 con 0x7fd73c0b5840 2024-03-04T07:31:02.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.823+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd73c072a70 con 0x7fd73c0b5840 2024-03-04T07:31:02.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.824+0000 7fd732ffd640 1 -- 172.21.15.64:0/1470370548 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd72401c460 con 0x7fd73c0b5840 2024-03-04T07:31:02.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.824+0000 7fd732ffd640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd71c06b4d0 0x7fd71c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:02.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.824+0000 7fd73a59c640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd71c06b4d0 0x7fd71c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:02.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.824+0000 7fd732ffd640 1 -- 172.21.15.64:0/1470370548 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 5795+0+0 (secure 0 0 0) 0x7fd72408a0b0 con 0x7fd73c0b5840 2024-03-04T07:31:02.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.825+0000 7fd73a59c640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd71c06b4d0 0x7fd71c06d990 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7fd734036000 tx=0x7fd734008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:02.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:02.829+0000 7fd732ffd640 1 -- 172.21.15.64:0/1470370548 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd73c072a70 con 0x7fd73c0b5840 2024-03-04T07:31:03.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.049+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7fd73c15b9b0 con 0x7fd73c0b5840 2024-03-04T07:31:03.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.697+0000 7fd732ffd640 1 -- 172.21.15.64:0/1470370548 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v45) v1 ==== 104+0+0 (secure 0 0 0) 0x7fd724058f60 con 0x7fd73c0b5840 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd71c06b4d0 msgr2=0x7fd71c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd71c06b4d0 0x7fd71c06d990 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7fd734036000 tx=0x7fd734008030 comp rx=0 tx=0).stop 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c0b5840 msgr2=0x7fd73c14afc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c0b5840 0x7fd73c14afc0 secure :-1 s=READY pgs=1056 cs=0 l=1 rev1=1 crypto rx=0x7fd72400c430 tx=0x7fd72400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 shutdown_connections 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd71c06b4d0 0x7fd71c06d990 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd73c15dbd0 0x7fd73c14b500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd73c0b5840 0x7fd73c14afc0 unknown :-1 s=CLOSED pgs=1056 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 --2- 172.21.15.64:0/1470370548 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd73c0b40c0 0x7fd73c1f7850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 >> 172.21.15.64:0/1470370548 conn(0x7fd73c0b26b0 msgr2=0x7fd73c15eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:03.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 shutdown_connections 2024-03-04T07:31:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.700+0000 7fd740b16640 1 -- 172.21.15.64:0/1470370548 wait complete. 2024-03-04T07:31:03.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:31:03.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.866+0000 7fa2a500f640 1 Processor -- start 2024-03-04T07:31:03.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.866+0000 7fa2a500f640 1 -- start start 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa2a500f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa2a00ceea0 0x7fa2a00d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa2a500f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 0x7fa2a00b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa2a500f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa2a00d3740 con 0x7fa2a015dd90 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa2a500f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa2a00d3400 con 0x7fa2a00ceea0 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa2a500f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa2a00d3120 con 0x7fa2a00b6d10 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa29ed9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa2a015dd90 0x7fa2a00d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55328/0 (socket says 172.21.15.64:55328) 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa29ed9d640 1 -- 172.21.15.64:0/52440891 learned_addr learned my addr 172.21.15.64:0/52440891 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3859585719 0 0) 0x7fa2a00d3740 con 0x7fa2a015dd90 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.867+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa28c003520 con 0x7fa2a015dd90 2024-03-04T07:31:03.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29f59e640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 0x7fa2a00b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29e59c640 1 --2- 172.21.15.64:0/52440891 >> v2:172.21.15.66:3300/0 conn(0x7fa2a00ceea0 0x7fa2a00d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2230916160 0 0) 0x7fa28c003520 con 0x7fa2a015dd90 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 msgr2=0x7fa2a00b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 0x7fa2a00b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 >> v2:172.21.15.66:3300/0 conn(0x7fa2a00ceea0 msgr2=0x7fa2a00d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 --2- 172.21.15.64:0/52440891 >> v2:172.21.15.66:3300/0 conn(0x7fa2a00ceea0 0x7fa2a00d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2a00d39e0 con 0x7fa2a015dd90 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fa298001690 con 0x7fa2a015dd90 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 >> v1:172.21.15.64:6789/0 conn(0x7fa2a015dd90 legacy=0x7fa2a00d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa28c0039b0 0x7fa28c003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa28c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 0x7fa28c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2a00d3400 con 0x7fa28c004800 2024-03-04T07:31:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2a00d3120 con 0x7fa28c0039b0 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa28c00b9f0 con 0x7fa2a00b6d10 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29e59c640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa28c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29ed9d640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa28c0039b0 0x7fa28c003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29f59e640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 0x7fa28c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29e59c640 1 -- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 msgr2=0x7fa28c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29e59c640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 0x7fa28c0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29e59c640 1 -- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa28c0039b0 msgr2=0x7fa28c003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29e59c640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa28c0039b0 0x7fa28c003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29e59c640 1 -- 172.21.15.64:0/52440891 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa28c003520 con 0x7fa28c004800 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.868+0000 7fa29f59e640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a00b6d10 0x7fa28c0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.869+0000 7fa29e59c640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa28c004c80 secure :-1 s=READY pgs=1058 cs=0 l=1 rev1=1 crypto rx=0x7fa288009ac0 tx=0x7fa28c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98612064bfa9740b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.869+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa288039070 con 0x7fa28c004800 2024-03-04T07:31:03.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.869+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa28802c710 con 0x7fa28c004800 2024-03-04T07:31:03.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.869+0000 7fa29dd9b640 1 -- 172.21.15.64:0/52440891 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa288009690 con 0x7fa28c004800 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.870+0000 7fa2a500f640 1 -- 172.21.15.64:0/52440891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 msgr2=0x7fa28c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.870+0000 7fa2a500f640 1 --2- 172.21.15.64:0/52440891 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa28c004c80 secure :-1 s=READY pgs=1058 cs=0 l=1 rev1=1 crypto rx=0x7fa288009ac0 tx=0x7fa28c005830 comp rx=0 tx=0).stop 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.870+0000 7fa29f59e640 1 -- 172.21.15.64:0/52440891 reap_dead start 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.870+0000 7fa2a500f640 1 -- 172.21.15.64:0/52440891 shutdown_connections 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.870+0000 7fa2a500f640 1 -- 172.21.15.64:0/52440891 >> 172.21.15.64:0/52440891 conn(0x7fa2a0071960 msgr2=0x7fa2a0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.870+0000 7fa2a500f640 1 -- 172.21.15.64:0/52440891 shutdown_connections 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.870+0000 7fa2a500f640 1 -- 172.21.15.64:0/52440891 wait complete. 2024-03-04T07:31:03.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.871+0000 7fa2a500f640 1 Processor -- start 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.871+0000 7fa2a500f640 1 -- start start 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa2a500f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa2a015f840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa2a500f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a015fd80 0x7fa2a0152d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa2a500f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2a01532d0 0x7fa2a0153730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa2a500f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2a00d48f0 con 0x7fa28c004800 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa2a500f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2a015e170 con 0x7fa2a01532d0 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa2a500f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa2a00d4a70 con 0x7fa2a015fd80 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa2a015f840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa2a015f840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54680/0 (socket says 172.21.15.64:54680) 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 -- 172.21.15.64:0/1732198302 learned_addr learned my addr 172.21.15.64:0/1732198302 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29f59e640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2a01532d0 0x7fa2a0153730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29e59c640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a015fd80 0x7fa2a0152d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 -- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a015fd80 msgr2=0x7fa2a0152d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:03.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a015fd80 0x7fa2a0152d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 -- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2a01532d0 msgr2=0x7fa2a0153730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2a01532d0 0x7fa2a0153730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29ed9d640 1 -- 172.21.15.64:0/1732198302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2a01605c0 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.872+0000 7fa29e59c640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a015fd80 0x7fa2a0152d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.873+0000 7fa29ed9d640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa2a015f840 secure :-1 s=READY pgs=1059 cs=0 l=1 rev1=1 crypto rx=0x7fa298007340 tx=0x7fa298007800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.873+0000 7fa2977fe640 1 -- 172.21.15.64:0/1732198302 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa29800f960 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.873+0000 7fa2977fe640 1 -- 172.21.15.64:0/1732198302 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa298018070 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.873+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa2a01608a0 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.873+0000 7fa2977fe640 1 -- 172.21.15.64:0/1732198302 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa29800fd40 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.873+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa2a0160e60 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.874+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa2a0072a30 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.874+0000 7fa2977fe640 1 -- 172.21.15.64:0/1732198302 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa29801d460 con 0x7fa28c004800 2024-03-04T07:31:03.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.875+0000 7fa2977fe640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa28006b500 0x7fa28006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:03.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.875+0000 7fa2977fe640 1 -- 172.21.15.64:0/1732198302 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 5795+0+0 (secure 0 0 0) 0x7fa29808a3c0 con 0x7fa28c004800 2024-03-04T07:31:03.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.875+0000 7fa29e59c640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa28006b500 0x7fa28006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:03.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.875+0000 7fa29e59c640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa28006b500 0x7fa28006d9c0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7fa288036000 tx=0x7fa288030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:03.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:03.879+0000 7fa2977fe640 1 -- 172.21.15.64:0/1732198302 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa298059150 con 0x7fa28c004800 2024-03-04T07:31:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:04.062+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fa2a004f410 con 0x7fa28c004800 2024-03-04T07:31:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.064+0000 7fa2977fe640 1 -- 172.21.15.64:0/1732198302 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v81) v1 ==== 181+0+0 (secure 0 0 0) 0x7fa298058a50 con 0x7fa28c004800 2024-03-04T07:31:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:31:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa28006b500 msgr2=0x7fa28006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa28006b500 0x7fa28006d9c0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7fa288036000 tx=0x7fa288030000 comp rx=0 tx=0).stop 2024-03-04T07:31:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 msgr2=0x7fa2a015f840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa2a015f840 secure :-1 s=READY pgs=1059 cs=0 l=1 rev1=1 crypto rx=0x7fa298007340 tx=0x7fa298007800 comp rx=0 tx=0).stop 2024-03-04T07:31:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 shutdown_connections 2024-03-04T07:31:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa28006b500 0x7fa28006d9c0 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2a01532d0 0x7fa2a0153730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2a015fd80 0x7fa2a0152d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 --2- 172.21.15.64:0/1732198302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa28c004800 0x7fa2a015f840 unknown :-1 s=CLOSED pgs=1059 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 >> 172.21.15.64:0/1732198302 conn(0x7fa2a0071960 msgr2=0x7fa2a00d0c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.066+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 shutdown_connections 2024-03-04T07:31:05.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.067+0000 7fa2a500f640 1 -- 172.21.15.64:0/1732198302 wait complete. 2024-03-04T07:31:05.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:31:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.193+0000 7f5610cc9640 1 Processor -- start 2024-03-04T07:31:05.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.193+0000 7f5610cc9640 1 -- start start 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.194+0000 7f5610cc9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f560c15dbd0 0x7f560c15ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.194+0000 7f5610cc9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f560c0b5840 0x7f560c0b39b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.194+0000 7f5610cc9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f560c15f3b0 con 0x7f560c15dfb0 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.194+0000 7f5610cc9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f560c06f070 con 0x7f560c15dbd0 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.194+0000 7f5610cc9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f560c071940 con 0x7f560c0b5840 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f560c15dbd0 0x7f560c15ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560ad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f560c15dfb0 0x7f560c161a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55330/0 (socket says 172.21.15.64:55330) 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560ad9d640 1 -- 172.21.15.64:0/4234428956 learned_addr learned my addr 172.21.15.64:0/4234428956 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560a59c640 1 -- 172.21.15.64:0/4234428956 >> v2:172.21.15.66:3300/0 conn(0x7f560c0b5840 msgr2=0x7f560c0b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560a59c640 1 --2- 172.21.15.64:0/4234428956 >> v2:172.21.15.66:3300/0 conn(0x7f560c0b5840 0x7f560c0b39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560a59c640 1 -- 172.21.15.64:0/4234428956 >> v1:172.21.15.64:6789/0 conn(0x7f560c15dfb0 legacy=0x7f560c161a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560a59c640 1 -- 172.21.15.64:0/4234428956 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f560c0b5db0 con 0x7f560c15dbd0 2024-03-04T07:31:05.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.195+0000 7f560959a640 1 -- 172.21.15.64:0/4234428956 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 867247940 0 0) 0x7f560c15f3b0 con 0x7f560c15dfb0 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f560a59c640 1 --2- 172.21.15.64:0/4234428956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dbd0 0x7f560c15ee70 secure :-1 s=READY pgs=1061 cs=0 l=1 rev1=1 crypto rx=0x7f560c073350 tx=0x7f560402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71e420aa28aa2c35 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f560959a640 1 -- 172.21.15.64:0/4234428956 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5604039070 con 0x7f560c15dbd0 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f560959a640 1 -- 172.21.15.64:0/4234428956 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f560402cc80 con 0x7f560c15dbd0 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f560959a640 1 -- 172.21.15.64:0/4234428956 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5604034660 con 0x7f560c15dbd0 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 -- 172.21.15.64:0/4234428956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dbd0 msgr2=0x7f560c15ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 --2- 172.21.15.64:0/4234428956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dbd0 0x7f560c15ee70 secure :-1 s=READY pgs=1061 cs=0 l=1 rev1=1 crypto rx=0x7f560c073350 tx=0x7f560402bed0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 -- 172.21.15.64:0/4234428956 shutdown_connections 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 --2- 172.21.15.64:0/4234428956 >> v2:172.21.15.66:3300/0 conn(0x7f560c0b5840 0x7f560c0b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 --2- 172.21.15.64:0/4234428956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dbd0 0x7f560c15ee70 unknown :-1 s=CLOSED pgs=1061 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 -- 172.21.15.64:0/4234428956 >> 172.21.15.64:0/4234428956 conn(0x7f560c0b26b0 msgr2=0x7f560c0b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 -- 172.21.15.64:0/4234428956 shutdown_connections 2024-03-04T07:31:05.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.196+0000 7f5610cc9640 1 -- 172.21.15.64:0/4234428956 wait complete. 2024-03-04T07:31:05.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 Processor -- start 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 -- start start 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f560c0b5840 0x7f560c0cfc80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f560c15dbd0 0x7f560c157020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dfb0 0x7f560c157560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f560c0721c0 con 0x7f560c15dfb0 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f560c071ee0 con 0x7f560c0b5840 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.198+0000 7f5610cc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f560c0726e0 con 0x7f560c15dbd0 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dfb0 0x7f560c157560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dfb0 0x7f560c157560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54694/0 (socket says 172.21.15.64:54694) 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 -- 172.21.15.64:0/1118136810 learned_addr learned my addr 172.21.15.64:0/1118136810 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560a59c640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f560c0b5840 0x7f560c0cfc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f5609d9b640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f560c15dbd0 0x7f560c157020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:05.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 -- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f560c15dbd0 msgr2=0x7f560c157020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f560c15dbd0 0x7f560c157020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 -- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f560c0b5840 msgr2=0x7f560c0cfc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f560c0b5840 0x7f560c0cfc80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 -- 172.21.15.64:0/1118136810 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f560c146cd0 con 0x7f560c15dfb0 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f5609d9b640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f560c15dbd0 0x7f560c157020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f560ad9d640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dfb0 0x7f560c157560 secure :-1 s=READY pgs=1062 cs=0 l=1 rev1=1 crypto rx=0x7f55fc00c450 tx=0x7f55fc00c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f5602ffd640 1 -- 172.21.15.64:0/1118136810 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55fc018070 con 0x7f560c15dfb0 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f5602ffd640 1 -- 172.21.15.64:0/1118136810 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f55fc007500 con 0x7f560c15dfb0 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f560c146fb0 con 0x7f560c15dfb0 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f5602ffd640 1 -- 172.21.15.64:0/1118136810 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55fc004a20 con 0x7f560c15dfb0 2024-03-04T07:31:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.199+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f560c0d01c0 con 0x7f560c15dfb0 2024-03-04T07:31:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.200+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f560c073380 con 0x7f560c15dfb0 2024-03-04T07:31:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.201+0000 7f5602ffd640 1 -- 172.21.15.64:0/1118136810 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f55fc01b070 con 0x7f560c15dfb0 2024-03-04T07:31:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.201+0000 7f5602ffd640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f55ec06b4d0 0x7f55ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.201+0000 7f5602ffd640 1 -- 172.21.15.64:0/1118136810 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 5357+0+0 (secure 0 0 0) 0x7f55fc089310 con 0x7f560c15dfb0 2024-03-04T07:31:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.201+0000 7f560a59c640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f55ec06b4d0 0x7f55ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.202+0000 7f560a59c640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f55ec06b4d0 0x7f55ec06d990 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f5604036000 tx=0x7f5604030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.206+0000 7f5602ffd640 1 -- 172.21.15.64:0/1118136810 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f55fc057c10 con 0x7f560c15dfb0 2024-03-04T07:31:05.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:05.391+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f560c04f410 con 0x7f560c15dfb0 2024-03-04T07:31:06.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.071+0000 7f5602ffd640 1 -- 172.21.15.64:0/1118136810 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v82) v1 ==== 169+0+0 (secure 0 0 0) 0x7f55fc05bcb0 con 0x7f560c15dfb0 2024-03-04T07:31:06.072 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:31:06.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f55ec06b4d0 msgr2=0x7f55ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f55ec06b4d0 0x7f55ec06d990 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f5604036000 tx=0x7f5604030000 comp rx=0 tx=0).stop 2024-03-04T07:31:06.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dfb0 msgr2=0x7f560c157560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dfb0 0x7f560c157560 secure :-1 s=READY pgs=1062 cs=0 l=1 rev1=1 crypto rx=0x7f55fc00c450 tx=0x7f55fc00c910 comp rx=0 tx=0).stop 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 shutdown_connections 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f55ec06b4d0 0x7f55ec06d990 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f560c15dfb0 0x7f560c157560 unknown :-1 s=CLOSED pgs=1062 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f560c15dbd0 0x7f560c157020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 --2- 172.21.15.64:0/1118136810 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f560c0b5840 0x7f560c0cfc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.073+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 >> 172.21.15.64:0/1118136810 conn(0x7f560c0b26b0 msgr2=0x7f560c15cea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.074+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 shutdown_connections 2024-03-04T07:31:06.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.074+0000 7f5610cc9640 1 -- 172.21.15.64:0/1118136810 wait complete. 2024-03-04T07:31:06.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:31:06.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.197+0000 7f97a893a640 1 Processor -- start 2024-03-04T07:31:06.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.197+0000 7f97a893a640 1 -- start start 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a893a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f97a4144640 0x7f97a406e400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a893a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f97a40ce900 0x7f97a40ced30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a893a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f97a415bc40 con 0x7f97a406e940 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a893a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97a406df70 con 0x7f97a40ce900 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a893a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f97a40cf270 con 0x7f97a4144640 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a2d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f97a40ce900 0x7f97a40ced30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a2d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f97a40ce900 0x7f97a40ced30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54700/0 (socket says 172.21.15.64:54700) 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a2d9d640 1 -- 172.21.15.64:0/2334702927 learned_addr learned my addr 172.21.15.64:0/2334702927 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a259c640 1 --2- 172.21.15.64:0/2334702927 >> v2:172.21.15.66:3300/0 conn(0x7f97a4144640 0x7f97a406e400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a2d9d640 1 -- 172.21.15.64:0/2334702927 >> v2:172.21.15.66:3300/0 conn(0x7f97a4144640 msgr2=0x7f97a406e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a2d9d640 1 --2- 172.21.15.64:0/2334702927 >> v2:172.21.15.66:3300/0 conn(0x7f97a4144640 0x7f97a406e400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a2d9d640 1 -- 172.21.15.64:0/2334702927 >> v1:172.21.15.66:6789/0 conn(0x7f97a406e940 legacy=0x7f97a415b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:06.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.198+0000 7f97a2d9d640 1 -- 172.21.15.64:0/2334702927 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97a415bf20 con 0x7f97a40ce900 2024-03-04T07:31:06.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.199+0000 7f97a2d9d640 1 --2- 172.21.15.64:0/2334702927 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a40ce900 0x7f97a40ced30 secure :-1 s=READY pgs=1063 cs=0 l=1 rev1=1 crypto rx=0x7f97a406d360 tx=0x7f979c02bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c8cdf4846fabda44 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:06.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.199+0000 7f97a1d9b640 1 -- 172.21.15.64:0/2334702927 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c02ca80 con 0x7f97a40ce900 2024-03-04T07:31:06.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.199+0000 7f97a1d9b640 1 -- 172.21.15.64:0/2334702927 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f979c02cc20 con 0x7f97a40ce900 2024-03-04T07:31:06.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.199+0000 7f97a1d9b640 1 -- 172.21.15.64:0/2334702927 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c0343f0 con 0x7f97a40ce900 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 -- 172.21.15.64:0/2334702927 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a40ce900 msgr2=0x7f97a40ced30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 --2- 172.21.15.64:0/2334702927 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a40ce900 0x7f97a40ced30 secure :-1 s=READY pgs=1063 cs=0 l=1 rev1=1 crypto rx=0x7f97a406d360 tx=0x7f979c02bee0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 -- 172.21.15.64:0/2334702927 shutdown_connections 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 --2- 172.21.15.64:0/2334702927 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a40ce900 0x7f97a40ced30 unknown :-1 s=CLOSED pgs=1063 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 --2- 172.21.15.64:0/2334702927 >> v2:172.21.15.66:3300/0 conn(0x7f97a4144640 0x7f97a406e400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 -- 172.21.15.64:0/2334702927 >> 172.21.15.64:0/2334702927 conn(0x7f97a4066d00 msgr2=0x7f97a4067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 -- 172.21.15.64:0/2334702927 shutdown_connections 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.200+0000 7f97a893a640 1 -- 172.21.15.64:0/2334702927 wait complete. 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.201+0000 7f97a893a640 1 Processor -- start 2024-03-04T07:31:06.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.201+0000 7f97a893a640 1 -- start start 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a893a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a406e940 0x7f97a415a060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a893a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a40ce900 0x7f97a4137c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a893a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a4144640 0x7f97a413bd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a893a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97a4070c20 con 0x7f97a4144640 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a893a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97a4070aa0 con 0x7f97a40ce900 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a893a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f97a415b840 con 0x7f97a406e940 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a4144640 0x7f97a413bd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a4144640 0x7f97a413bd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54716/0 (socket says 172.21.15.64:54716) 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 -- 172.21.15.64:0/158620875 learned_addr learned my addr 172.21.15.64:0/158620875 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a259c640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a406e940 0x7f97a415a060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f979bfff640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a40ce900 0x7f97a4137c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 -- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a406e940 msgr2=0x7f97a415a060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a406e940 0x7f97a415a060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 -- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a40ce900 msgr2=0x7f97a4137c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a40ce900 0x7f97a4137c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 -- 172.21.15.64:0/158620875 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97a41385d0 con 0x7f97a4144640 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.202+0000 7f97a2d9d640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a4144640 0x7f97a413bd50 secure :-1 s=READY pgs=1064 cs=0 l=1 rev1=1 crypto rx=0x7f979c036040 tx=0x7f979c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.203+0000 7f979affd640 1 -- 172.21.15.64:0/158620875 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c043020 con 0x7f97a4144640 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.203+0000 7f979affd640 1 -- 172.21.15.64:0/158620875 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f979c03e450 con 0x7f97a4144640 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.203+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97a413a440 con 0x7f97a4144640 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.203+0000 7f979affd640 1 -- 172.21.15.64:0/158620875 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f979c03e860 con 0x7f97a4144640 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.203+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f97a415a5a0 con 0x7f97a4144640 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.204+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f97a40733c0 con 0x7f97a4144640 2024-03-04T07:31:06.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.204+0000 7f979affd640 1 -- 172.21.15.64:0/158620875 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f979c046020 con 0x7f97a4144640 2024-03-04T07:31:06.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.205+0000 7f979affd640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f978406b4d0 0x7f978406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.205+0000 7f97a259c640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f978406b4d0 0x7f978406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.205+0000 7f979affd640 1 -- 172.21.15.64:0/158620875 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f979c0ac6f0 con 0x7f97a4144640 2024-03-04T07:31:06.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.205+0000 7f97a259c640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f978406b4d0 0x7f978406d990 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f97a40ce480 tx=0x7f9794008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:06.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.209+0000 7f979affd640 1 -- 172.21.15.64:0/158620875 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f979c04d030 con 0x7f97a4144640 2024-03-04T07:31:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.390+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f97a4140d40 con 0x7f97a4144640 2024-03-04T07:31:06.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.394+0000 7f979affd640 1 -- 172.21.15.64:0/158620875 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v27) v1 ==== 104+0+0 (secure 0 0 0) 0x7f979c07b220 con 0x7f97a4144640 2024-03-04T07:31:06.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.395+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f978406b4d0 msgr2=0x7f978406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.395+0000 7f97a893a640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f978406b4d0 0x7f978406d990 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f97a40ce480 tx=0x7f9794008040 comp rx=0 tx=0).stop 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.395+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a4144640 msgr2=0x7f97a413bd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.395+0000 7f97a893a640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a4144640 0x7f97a413bd50 secure :-1 s=READY pgs=1064 cs=0 l=1 rev1=1 crypto rx=0x7f979c036040 tx=0x7f979c004a40 comp rx=0 tx=0).stop 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 shutdown_connections 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f978406b4d0 0x7f978406d990 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f97a4144640 0x7f97a413bd50 unknown :-1 s=CLOSED pgs=1064 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f97a40ce900 0x7f97a4137c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 --2- 172.21.15.64:0/158620875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f97a406e940 0x7f97a415a060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 >> 172.21.15.64:0/158620875 conn(0x7f97a4066d00 msgr2=0x7f97a41438a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 shutdown_connections 2024-03-04T07:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.396+0000 7f97a893a640 1 -- 172.21.15.64:0/158620875 wait complete. 2024-03-04T07:31:06.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_fancy_layout' 2024-03-04T07:31:06.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.510+0000 7f1a12d93640 1 Processor -- start 2024-03-04T07:31:06.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.510+0000 7f1a12d93640 1 -- start start 2024-03-04T07:31:06.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.510+0000 7f1a12d93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 0x7f1a0c0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.510+0000 7f1a12d93640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1a0c0d0880 0x7f1a0c0ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.510+0000 7f1a12d93640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1a0c0d0dd0 con 0x7f1a0c06fe20 2024-03-04T07:31:06.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.510+0000 7f1a12d93640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1a0c0d10b0 con 0x7f1a0c0d0880 2024-03-04T07:31:06.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.510+0000 7f1a12d93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a0c0cc0e0 con 0x7f1a0c15e060 2024-03-04T07:31:06.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1a0c06fe20 0x7f1a0c0ce850 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55346/0 (socket says 172.21.15.64:55346) 2024-03-04T07:31:06.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0bfff640 1 -- 172.21.15.64:0/3820937201 learned_addr learned my addr 172.21.15.64:0/3820937201 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a10b2f640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 0x7f1a0c0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3776180494 0 0) 0x7f1a0c0d0dd0 con 0x7f1a0c06fe20 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a11330640 1 --2- 172.21.15.64:0/3820937201 >> v2:172.21.15.66:3300/0 conn(0x7f1a0c0d0880 0x7f1a0c0ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f19fc003520 con 0x7f1a0c06fe20 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2984942697 0 0) 0x7f19fc003520 con 0x7f1a0c06fe20 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 msgr2=0x7f1a0c0ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 0x7f1a0c0ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 >> v2:172.21.15.66:3300/0 conn(0x7f1a0c0d0880 msgr2=0x7f1a0c0ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 --2- 172.21.15.64:0/3820937201 >> v2:172.21.15.66:3300/0 conn(0x7f1a0c0d0880 0x7f1a0c0ccbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a0c0cc3c0 con 0x7f1a0c06fe20 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f19f8001670 con 0x7f1a0c06fe20 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 >> v1:172.21.15.64:6789/0 conn(0x7f1a0c06fe20 legacy=0x7f1a0c0ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 0x7f19fc0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19fc003f30 0x7f19fc004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19fc004d40 0x7f19fc0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a0c0d10b0 con 0x7f19fc004d40 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a0c0cc0e0 con 0x7f19fc003f30 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f19fc00b9c0 con 0x7f1a0c15e060 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a11330640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19fc004d40 0x7f19fc0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.511+0000 7f1a10b2f640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 0x7f19fc0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a0bfff640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19fc003f30 0x7f19fc004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a11330640 1 -- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 msgr2=0x7f19fc0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a11330640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a0c15e060 0x7f19fc0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a11330640 1 -- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19fc003f30 msgr2=0x7f19fc004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a11330640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19fc003f30 0x7f19fc004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a11330640 1 -- 172.21.15.64:0/3820937201 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19fc003520 con 0x7f19fc004d40 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a0bfff640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19fc003f30 0x7f19fc004800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:06.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.512+0000 7f1a11330640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19fc004d40 0x7f19fc0051c0 secure :-1 s=READY pgs=1066 cs=0 l=1 rev1=1 crypto rx=0x7f1a04009fc0 tx=0x7f19fc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=850af96912cf275d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a04039070 con 0x7f19fc004d40 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a0402cc40 con 0x7f19fc004d40 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a0b7fe640 1 -- 172.21.15.64:0/3820937201 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a04009c10 con 0x7f19fc004d40 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a12d93640 1 -- 172.21.15.64:0/3820937201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19fc004d40 msgr2=0x7f19fc0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a12d93640 1 --2- 172.21.15.64:0/3820937201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19fc004d40 0x7f19fc0051c0 secure :-1 s=READY pgs=1066 cs=0 l=1 rev1=1 crypto rx=0x7f1a04009fc0 tx=0x7f19fc005830 comp rx=0 tx=0).stop 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a11330640 1 -- 172.21.15.64:0/3820937201 reap_dead start 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a12d93640 1 -- 172.21.15.64:0/3820937201 shutdown_connections 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a12d93640 1 -- 172.21.15.64:0/3820937201 >> 172.21.15.64:0/3820937201 conn(0x7f1a0c0d1a90 msgr2=0x7f1a0c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a12d93640 1 -- 172.21.15.64:0/3820937201 shutdown_connections 2024-03-04T07:31:06.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.513+0000 7f1a12d93640 1 -- 172.21.15.64:0/3820937201 wait complete. 2024-03-04T07:31:06.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.514+0000 7f1a12d93640 1 Processor -- start 2024-03-04T07:31:06.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.514+0000 7f1a12d93640 1 -- start start 2024-03-04T07:31:06.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a12d93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19fc004d40 0x7f1a0c157d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a12d93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a0c15a9b0 0x7f1a0c0c8b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a12d93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a0c15b3c0 0x7f1a0c15b850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a12d93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a0c0cef60 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a12d93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a0c15e440 con 0x7f1a0c15a9b0 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a12d93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a0c0cf0e0 con 0x7f19fc004d40 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a11330640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a0c15b3c0 0x7f1a0c15b850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a11330640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a0c15b3c0 0x7f1a0c15b850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54734/0 (socket says 172.21.15.64:54734) 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a11330640 1 -- 172.21.15.64:0/3873525481 learned_addr learned my addr 172.21.15.64:0/3873525481 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a0bfff640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a0c15a9b0 0x7f1a0c0c8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a10b2f640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19fc004d40 0x7f1a0c157d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a11330640 1 -- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19fc004d40 msgr2=0x7f1a0c157d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a11330640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19fc004d40 0x7f1a0c157d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a11330640 1 -- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a0c15a9b0 msgr2=0x7f1a0c0c8b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.515+0000 7f1a11330640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a0c15a9b0 0x7f1a0c0c8b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.516+0000 7f1a11330640 1 -- 172.21.15.64:0/3873525481 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a0c0cb0b0 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.516+0000 7f1a11330640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a0c15b3c0 0x7f1a0c15b850 secure :-1 s=READY pgs=1067 cs=0 l=1 rev1=1 crypto rx=0x7f1a0400e230 tx=0x7f1a04009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.516+0000 7f1a097fa640 1 -- 172.21.15.64:0/3873525481 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a04039030 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.516+0000 7f1a097fa640 1 -- 172.21.15.64:0/3873525481 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a0403a040 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.516+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1a0c0ae8e0 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.516+0000 7f1a097fa640 1 -- 172.21.15.64:0/3873525481 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a04003990 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.516+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a0c0cb630 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.517+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1a0c067ff0 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.517+0000 7f1a097fa640 1 -- 172.21.15.64:0/3873525481 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1a04045030 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.518+0000 7f1a097fa640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19f006b4d0 0x7f19f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.518+0000 7f1a097fa640 1 -- 172.21.15.64:0/3873525481 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f1a040abd40 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.518+0000 7f1a10b2f640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19f006b4d0 0x7f19f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.518+0000 7f1a10b2f640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19f006b4d0 0x7f19f006d990 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f1a00009b00 tx=0x7f1a00008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:06.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.522+0000 7f1a097fa640 1 -- 172.21.15.64:0/3873525481 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1a0404c020 con 0x7f1a0c15b3c0 2024-03-04T07:31:06.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:06.776+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]} v 0) v1 -- 0x7f1a0c0cbdc0 con 0x7f1a0c15b3c0 2024-03-04T07:31:07.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.072+0000 7f1a097fa640 1 -- 172.21.15.64:0/3873525481 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_fancy_layout"]}]=0 v239) v1 ==== 132+0+0 (secure 0 0 0) 0x7f1a0407aec0 con 0x7f1a0c15b3c0 2024-03-04T07:31:07.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.075+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19f006b4d0 msgr2=0x7f19f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.075+0000 7f1a12d93640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19f006b4d0 0x7f19f006d990 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f1a00009b00 tx=0x7f1a00008040 comp rx=0 tx=0).stop 2024-03-04T07:31:07.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.075+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a0c15b3c0 msgr2=0x7f1a0c15b850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.075+0000 7f1a12d93640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a0c15b3c0 0x7f1a0c15b850 secure :-1 s=READY pgs=1067 cs=0 l=1 rev1=1 crypto rx=0x7f1a0400e230 tx=0x7f1a04009750 comp rx=0 tx=0).stop 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 shutdown_connections 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f19f006b4d0 0x7f19f006d990 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a0c15b3c0 0x7f1a0c15b850 unknown :-1 s=CLOSED pgs=1067 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a0c15a9b0 0x7f1a0c0c8b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 --2- 172.21.15.64:0/3873525481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19fc004d40 0x7f1a0c157d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 >> 172.21.15.64:0/3873525481 conn(0x7f1a0c0d1a90 msgr2=0x7f1a0c06baf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 shutdown_connections 2024-03-04T07:31:07.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.076+0000 7f1a12d93640 1 -- 172.21.15.64:0/3873525481 wait complete. 2024-03-04T07:31:07.088 INFO:tasks.cephfs_test_runner:purge stray file with fancy layout ... ok 2024-03-04T07:31:07.088 INFO:tasks.cephfs_test_runner:Starting test: test_files_throttle (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:31:07.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:07.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.189+0000 7f4350e2a640 1 Processor -- start 2024-03-04T07:31:07.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.189+0000 7f4350e2a640 1 -- start start 2024-03-04T07:31:07.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f4350e2a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f434c15df00 0x7f434c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f4350e2a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f434c0b9a30 0x7f434c0d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f4350e2a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f434c0cea30 con 0x7f434c0b95a0 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f4350e2a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f434c0d2af0 con 0x7f434c0b9a30 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f4350e2a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f434c0cf3c0 con 0x7f434c15df00 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f434ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f434c0b9a30 0x7f434c0d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f434ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f434c0b9a30 0x7f434c0d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54738/0 (socket says 172.21.15.64:54738) 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.190+0000 7f434ad9d640 1 -- 172.21.15.64:0/332861172 learned_addr learned my addr 172.21.15.64:0/332861172 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.191+0000 7f434ad9d640 1 -- 172.21.15.64:0/332861172 >> v2:172.21.15.66:3300/0 conn(0x7f434c15df00 msgr2=0x7f434c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.191+0000 7f434ad9d640 1 --2- 172.21.15.64:0/332861172 >> v2:172.21.15.66:3300/0 conn(0x7f434c15df00 0x7f434c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.191+0000 7f434ad9d640 1 -- 172.21.15.64:0/332861172 >> v1:172.21.15.66:6789/0 conn(0x7f434c0b95a0 legacy=0x7f434c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:07.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.191+0000 7f434ad9d640 1 -- 172.21.15.64:0/332861172 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f434c0cf0e0 con 0x7f434c0b9a30 2024-03-04T07:31:07.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.191+0000 7f434ad9d640 1 --2- 172.21.15.64:0/332861172 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c0b9a30 0x7f434c0d3220 secure :-1 s=READY pgs=1068 cs=0 l=1 rev1=1 crypto rx=0x7f434c0d11b0 tx=0x7f434402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=46243d8551f625fd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f434959a640 1 -- 172.21.15.64:0/332861172 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4344035020 con 0x7f434c0b9a30 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f434959a640 1 -- 172.21.15.64:0/332861172 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4344031870 con 0x7f434c0b9a30 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f434959a640 1 -- 172.21.15.64:0/332861172 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4344031c40 con 0x7f434c0b9a30 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 -- 172.21.15.64:0/332861172 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c0b9a30 msgr2=0x7f434c0d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 --2- 172.21.15.64:0/332861172 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c0b9a30 0x7f434c0d3220 secure :-1 s=READY pgs=1068 cs=0 l=1 rev1=1 crypto rx=0x7f434c0d11b0 tx=0x7f434402b910 comp rx=0 tx=0).stop 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 -- 172.21.15.64:0/332861172 shutdown_connections 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 --2- 172.21.15.64:0/332861172 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c0b9a30 0x7f434c0d3220 unknown :-1 s=CLOSED pgs=1068 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 --2- 172.21.15.64:0/332861172 >> v2:172.21.15.66:3300/0 conn(0x7f434c15df00 0x7f434c0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 -- 172.21.15.64:0/332861172 >> 172.21.15.64:0/332861172 conn(0x7f434c071960 msgr2=0x7f434c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 -- 172.21.15.64:0/332861172 shutdown_connections 2024-03-04T07:31:07.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.192+0000 7f4350e2a640 1 -- 172.21.15.64:0/332861172 wait complete. 2024-03-04T07:31:07.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.193+0000 7f4350e2a640 1 Processor -- start 2024-03-04T07:31:07.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4350e2a640 1 -- start start 2024-03-04T07:31:07.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4350e2a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434c0b95a0 0x7f434c152170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4350e2a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c15df00 0x7f434c15a240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4350e2a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434c0b4270 0x7f434c0b4a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4350e2a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f434c0cec70 con 0x7f434c15df00 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4350e2a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f434c0ce750 con 0x7f434c0b4270 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4350e2a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f434c0cedf0 con 0x7f434c0b95a0 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4349d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c15df00 0x7f434c15a240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4349d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c15df00 0x7f434c15a240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54748/0 (socket says 172.21.15.64:54748) 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.194+0000 7f4349d9b640 1 -- 172.21.15.64:0/2871722006 learned_addr learned my addr 172.21.15.64:0/2871722006 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4349d9b640 1 -- 172.21.15.64:0/2871722006 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434c0b95a0 msgr2=0x7f434c152170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f434ad9d640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434c0b4270 0x7f434c0b4a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4349d9b640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434c0b95a0 0x7f434c152170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4349d9b640 1 -- 172.21.15.64:0/2871722006 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434c0b4270 msgr2=0x7f434c0b4a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4349d9b640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434c0b4270 0x7f434c0b4a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4349d9b640 1 -- 172.21.15.64:0/2871722006 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f434c15aae0 con 0x7f434c15df00 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4349d9b640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c15df00 0x7f434c15a240 secure :-1 s=READY pgs=1069 cs=0 l=1 rev1=1 crypto rx=0x7f433400c430 tx=0x7f433400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4342ffd640 1 -- 172.21.15.64:0/2871722006 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4334011790 con 0x7f434c15df00 2024-03-04T07:31:07.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4342ffd640 1 -- 172.21.15.64:0/2871722006 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4334017070 con 0x7f434c15df00 2024-03-04T07:31:07.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f434c15adc0 con 0x7f434c15df00 2024-03-04T07:31:07.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4342ffd640 1 -- 172.21.15.64:0/2871722006 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4334011b70 con 0x7f434c15df00 2024-03-04T07:31:07.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.195+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f434c1526b0 con 0x7f434c15df00 2024-03-04T07:31:07.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.196+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f434c067ff0 con 0x7f434c15df00 2024-03-04T07:31:07.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.196+0000 7f4342ffd640 1 -- 172.21.15.64:0/2871722006 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4334011d10 con 0x7f434c15df00 2024-03-04T07:31:07.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.197+0000 7f4342ffd640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432c06b4d0 0x7f432c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.197+0000 7f434a59c640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432c06b4d0 0x7f432c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.197+0000 7f4342ffd640 1 -- 172.21.15.64:0/2871722006 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f4334089d60 con 0x7f434c15df00 2024-03-04T07:31:07.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.197+0000 7f434a59c640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432c06b4d0 0x7f432c06d990 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f433c00ab90 tx=0x7f433c009250 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.201+0000 7f4342ffd640 1 -- 172.21.15.64:0/2871722006 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4334058f70 con 0x7f434c15df00 2024-03-04T07:31:07.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.386+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f434c0b7c50 con 0x7f434c15df00 2024-03-04T07:31:07.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.387+0000 7f4342ffd640 1 -- 172.21.15.64:0/2871722006 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v82) v1 ==== 74+0+12973 (secure 0 0 0) 0x7f4334058870 con 0x7f434c15df00 2024-03-04T07:31:07.390 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:07.390 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":82,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:06.066425+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6836/1889124368":"2024-03-05T07:31:02.695894+0000","172.21.15.64:6835/1889124368":"2024-03-05T07:31:02.695894+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432c06b4d0 msgr2=0x7f432c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432c06b4d0 0x7f432c06d990 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f433c00ab90 tx=0x7f433c009250 comp rx=0 tx=0).stop 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c15df00 msgr2=0x7f434c15a240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c15df00 0x7f434c15a240 secure :-1 s=READY pgs=1069 cs=0 l=1 rev1=1 crypto rx=0x7f433400c430 tx=0x7f433400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 shutdown_connections 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432c06b4d0 0x7f432c06d990 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434c0b4270 0x7f434c0b4a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434c15df00 0x7f434c15a240 unknown :-1 s=CLOSED pgs=1069 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 --2- 172.21.15.64:0/2871722006 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434c0b95a0 0x7f434c152170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 >> 172.21.15.64:0/2871722006 conn(0x7f434c071960 msgr2=0x7f434c0b8680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:07.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 shutdown_connections 2024-03-04T07:31:07.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.389+0000 7f4350e2a640 1 -- 172.21.15.64:0/2871722006 wait complete. 2024-03-04T07:31:07.401 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:31:07.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:31:07.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.505+0000 7f62a0b3a640 1 Processor -- start 2024-03-04T07:31:07.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.505+0000 7f62a0b3a640 1 -- start start 2024-03-04T07:31:07.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f62a0b3a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f629c0b41f0 0x7f629c06bbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f62a0b3a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c06c130 0x7f629c0cdac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f62a0b3a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f629c15e6d0 con 0x7f629c15e2f0 2024-03-04T07:31:07.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f62a0b3a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f629c0d0a10 con 0x7f629c0b41f0 2024-03-04T07:31:07.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f62a0b3a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f629c0ccd00 con 0x7f629c06c130 2024-03-04T07:31:07.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f6299d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f629c0b41f0 0x7f629c06bbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f629a59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f629c15e2f0 0x7f629c06b4e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55350/0 (socket says 172.21.15.64:55350) 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f629a59c640 1 -- 172.21.15.64:0/3294772380 learned_addr learned my addr 172.21.15.64:0/3294772380 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f629ad9d640 1 --2- 172.21.15.64:0/3294772380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c06c130 0x7f629c0cdac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f6299d9b640 1 -- 172.21.15.64:0/3294772380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c06c130 msgr2=0x7f629c0cdac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f6299d9b640 1 --2- 172.21.15.64:0/3294772380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c06c130 0x7f629c0cdac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f6299d9b640 1 -- 172.21.15.64:0/3294772380 >> v1:172.21.15.64:6789/0 conn(0x7f629c15e2f0 legacy=0x7f629c06b4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f629959a640 1 -- 172.21.15.64:0/3294772380 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2748392239 0 0) 0x7f629c15e6d0 con 0x7f629c15e2f0 2024-03-04T07:31:07.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.506+0000 7f6299d9b640 1 -- 172.21.15.64:0/3294772380 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f629c0ccfe0 con 0x7f629c0b41f0 2024-03-04T07:31:07.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.507+0000 7f6299d9b640 1 --2- 172.21.15.64:0/3294772380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c0b41f0 0x7f629c06bbf0 secure :-1 s=READY pgs=1071 cs=0 l=1 rev1=1 crypto rx=0x7f62900097a0 tx=0x7f629002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b9bc1d00733a6f5a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.507+0000 7f629959a640 1 -- 172.21.15.64:0/3294772380 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62900315a0 con 0x7f629c0b41f0 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.507+0000 7f629959a640 1 -- 172.21.15.64:0/3294772380 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6290037070 con 0x7f629c0b41f0 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.507+0000 7f629959a640 1 -- 172.21.15.64:0/3294772380 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6290031960 con 0x7f629c0b41f0 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 -- 172.21.15.64:0/3294772380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c0b41f0 msgr2=0x7f629c06bbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/3294772380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c0b41f0 0x7f629c06bbf0 secure :-1 s=READY pgs=1071 cs=0 l=1 rev1=1 crypto rx=0x7f62900097a0 tx=0x7f629002b840 comp rx=0 tx=0).stop 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 -- 172.21.15.64:0/3294772380 shutdown_connections 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/3294772380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c06c130 0x7f629c0cdac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/3294772380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c0b41f0 0x7f629c06bbf0 unknown :-1 s=CLOSED pgs=1071 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 -- 172.21.15.64:0/3294772380 >> 172.21.15.64:0/3294772380 conn(0x7f629c0b36d0 msgr2=0x7f629c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 -- 172.21.15.64:0/3294772380 shutdown_connections 2024-03-04T07:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.508+0000 7f62a0b3a640 1 -- 172.21.15.64:0/3294772380 wait complete. 2024-03-04T07:31:07.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.509+0000 7f62a0b3a640 1 Processor -- start 2024-03-04T07:31:07.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.509+0000 7f62a0b3a640 1 -- start start 2024-03-04T07:31:07.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f62a0b3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c06c130 0x7f629c156630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f62a0b3a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c0b41f0 0x7f629c0c5630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f62a0b3a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f629c15e2f0 0x7f629c0c7af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f62a0b3a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f629c0ce000 con 0x7f629c06c130 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f62a0b3a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f629c0cd5a0 con 0x7f629c15e2f0 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f62a0b3a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f629c0ce180 con 0x7f629c0b41f0 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c06c130 0x7f629c156630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c06c130 0x7f629c156630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54774/0 (socket says 172.21.15.64:54774) 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 -- 172.21.15.64:0/1512588425 learned_addr learned my addr 172.21.15.64:0/1512588425 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 -- 172.21.15.64:0/1512588425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c0b41f0 msgr2=0x7f629c0c5630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c0b41f0 0x7f629c0c5630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 -- 172.21.15.64:0/1512588425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f629c15e2f0 msgr2=0x7f629c0c7af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f629c15e2f0 0x7f629c0c7af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 -- 172.21.15.64:0/1512588425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f629c0ca6d0 con 0x7f629c06c130 2024-03-04T07:31:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.510+0000 7f629a59c640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c06c130 0x7f629c156630 secure :-1 s=READY pgs=1072 cs=0 l=1 rev1=1 crypto rx=0x7f629400c430 tx=0x7f629400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.511+0000 7f628affd640 1 -- 172.21.15.64:0/1512588425 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62940074e0 con 0x7f629c06c130 2024-03-04T07:31:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.511+0000 7f628affd640 1 -- 172.21.15.64:0/1512588425 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62940047e0 con 0x7f629c06c130 2024-03-04T07:31:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.511+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f629c0ca9b0 con 0x7f629c06c130 2024-03-04T07:31:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.511+0000 7f628affd640 1 -- 172.21.15.64:0/1512588425 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6294004bd0 con 0x7f629c06c130 2024-03-04T07:31:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.511+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f629c0caf70 con 0x7f629c06c130 2024-03-04T07:31:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.512+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f629c06af20 con 0x7f629c06c130 2024-03-04T07:31:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.512+0000 7f628affd640 1 -- 172.21.15.64:0/1512588425 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6294011070 con 0x7f629c06c130 2024-03-04T07:31:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.512+0000 7f628affd640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f627c06b4d0 0x7f627c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.513+0000 7f628affd640 1 -- 172.21.15.64:0/1512588425 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f6294089b30 con 0x7f629c06c130 2024-03-04T07:31:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.513+0000 7f6299d9b640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f627c06b4d0 0x7f627c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.513+0000 7f6299d9b640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f627c06b4d0 0x7f627c06d990 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7f6290036000 tx=0x7f6290030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.517+0000 7f628affd640 1 -- 172.21.15.64:0/1512588425 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f629c06af20 con 0x7f629c06c130 2024-03-04T07:31:07.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.695+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f629c04f410 con 0x7f629c06c130 2024-03-04T07:31:07.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.695+0000 7f628affd640 1 -- 172.21.15.64:0/1512588425 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v82) v1 ==== 93+0+10 (secure 0 0 0) 0x7f6294058d40 con 0x7f629c06c130 2024-03-04T07:31:07.696 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:31:07.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f627c06b4d0 msgr2=0x7f627c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f627c06b4d0 0x7f627c06d990 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7f6290036000 tx=0x7f6290030000 comp rx=0 tx=0).stop 2024-03-04T07:31:07.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c06c130 msgr2=0x7f629c156630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c06c130 0x7f629c156630 secure :-1 s=READY pgs=1072 cs=0 l=1 rev1=1 crypto rx=0x7f629400c430 tx=0x7f629400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 shutdown_connections 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f627c06b4d0 0x7f627c06d990 unknown :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f629c15e2f0 0x7f629c0c7af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f629c0b41f0 0x7f629c0c5630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 --2- 172.21.15.64:0/1512588425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f629c06c130 0x7f629c156630 unknown :-1 s=CLOSED pgs=1072 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 >> 172.21.15.64:0/1512588425 conn(0x7f629c0b36d0 msgr2=0x7f629c0b3ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 shutdown_connections 2024-03-04T07:31:07.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.697+0000 7f62a0b3a640 1 -- 172.21.15.64:0/1512588425 wait complete. 2024-03-04T07:31:07.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_files_throttle' 2024-03-04T07:31:07.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.811+0000 7f4a9c332640 1 Processor -- start 2024-03-04T07:31:07.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.811+0000 7f4a9c332640 1 -- start start 2024-03-04T07:31:07.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.811+0000 7f4a9c332640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4a9406c240 0x7f4a9406c6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.811+0000 7f4a9c332640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4a940d10f0 con 0x7f4a9415df50 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.811+0000 7f4a9c332640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4a940cc150 con 0x7f4a9406cc10 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.811+0000 7f4a9c332640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4a940cca80 con 0x7f4a9406c240 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.812+0000 7f4a998cd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4a9406c240 0x7f4a9406c6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.812+0000 7f4a9a0ce640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4a9415df50 0x7f4a940d08a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55354/0 (socket says 172.21.15.64:55354) 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.812+0000 7f4a9a0ce640 1 -- 172.21.15.64:0/3122991256 learned_addr learned my addr 172.21.15.64:0/3122991256 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.812+0000 7f4a998cd640 1 -- 172.21.15.64:0/3122991256 >> v1:172.21.15.66:6789/0 conn(0x7f4a9406cc10 legacy=0x7f4a9406ecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.812+0000 7f4a998cd640 1 -- 172.21.15.64:0/3122991256 >> v1:172.21.15.64:6789/0 conn(0x7f4a9415df50 legacy=0x7f4a940d08a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.812+0000 7f4a990cc640 1 -- 172.21.15.64:0/3122991256 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2909135510 0 0) 0x7f4a940d10f0 con 0x7f4a9415df50 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.812+0000 7f4a998cd640 1 -- 172.21.15.64:0/3122991256 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4a940cc430 con 0x7f4a9406c240 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.813+0000 7f4a998cd640 1 --2- 172.21.15.64:0/3122991256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9406c240 0x7f4a9406c6d0 secure :-1 s=READY pgs=1074 cs=0 l=1 rev1=1 crypto rx=0x7f4a880097a0 tx=0x7f4a8802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d5f14ae05113f256 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.813+0000 7f4a990cc640 1 -- 172.21.15.64:0/3122991256 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a88035020 con 0x7f4a9406c240 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.813+0000 7f4a990cc640 1 -- 172.21.15.64:0/3122991256 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4a88031830 con 0x7f4a9406c240 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.813+0000 7f4a990cc640 1 -- 172.21.15.64:0/3122991256 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a88031c00 con 0x7f4a9406c240 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.813+0000 7f4a9c332640 1 -- 172.21.15.64:0/3122991256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9406c240 msgr2=0x7f4a9406c6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.813+0000 7f4a9c332640 1 --2- 172.21.15.64:0/3122991256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9406c240 0x7f4a9406c6d0 secure :-1 s=READY pgs=1074 cs=0 l=1 rev1=1 crypto rx=0x7f4a880097a0 tx=0x7f4a8802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.814+0000 7f4a9c332640 1 -- 172.21.15.64:0/3122991256 shutdown_connections 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.814+0000 7f4a9c332640 1 --2- 172.21.15.64:0/3122991256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9406c240 0x7f4a9406c6d0 unknown :-1 s=CLOSED pgs=1074 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.814+0000 7f4a9c332640 1 -- 172.21.15.64:0/3122991256 >> 172.21.15.64:0/3122991256 conn(0x7f4a940d1a60 msgr2=0x7f4a940d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.814+0000 7f4a9c332640 1 -- 172.21.15.64:0/3122991256 shutdown_connections 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.814+0000 7f4a9c332640 1 -- 172.21.15.64:0/3122991256 wait complete. 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 Processor -- start 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 -- start start 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a9406c240 0x7f4a94157420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a9406cc10 0x7f4a9414df30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9415df50 0x7f4a9415f9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a9415e330 con 0x7f4a9415df50 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a940cdf20 con 0x7f4a9406cc10 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.815+0000 7f4a9c332640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4a9406af20 con 0x7f4a9406c240 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9415df50 0x7f4a9415f9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9415df50 0x7f4a9415f9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54798/0 (socket says 172.21.15.64:54798) 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 -- 172.21.15.64:0/2008617766 learned_addr learned my addr 172.21.15.64:0/2008617766 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a0ce640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a9406c240 0x7f4a94157420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a998cd640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a9406cc10 0x7f4a9414df30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 -- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a9406c240 msgr2=0x7f4a94157420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a9406c240 0x7f4a94157420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 -- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a9406cc10 msgr2=0x7f4a9414df30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a9406cc10 0x7f4a9414df30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 -- 172.21.15.64:0/2008617766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4a94160140 con 0x7f4a9415df50 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9a8cf640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9415df50 0x7f4a9415f9d0 secure :-1 s=READY pgs=1075 cs=0 l=1 rev1=1 crypto rx=0x7f4a8400c6e0 tx=0x7f4a8400cba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a92ffd640 1 -- 172.21.15.64:0/2008617766 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a84003b70 con 0x7f4a9415df50 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a92ffd640 1 -- 172.21.15.64:0/2008617766 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4a84003d30 con 0x7f4a9415df50 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a92ffd640 1 -- 172.21.15.64:0/2008617766 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a84005310 con 0x7f4a9415df50 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4a94160420 con 0x7f4a9415df50 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.816+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4a941609e0 con 0x7f4a9415df50 2024-03-04T07:31:07.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.817+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4a94067ff0 con 0x7f4a9415df50 2024-03-04T07:31:07.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.818+0000 7f4a92ffd640 1 -- 172.21.15.64:0/2008617766 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4a840054b0 con 0x7f4a9415df50 2024-03-04T07:31:07.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.818+0000 7f4a92ffd640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a8006b4d0 0x7f4a8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:07.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.818+0000 7f4a92ffd640 1 -- 172.21.15.64:0/2008617766 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f4a84089bf0 con 0x7f4a9415df50 2024-03-04T07:31:07.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.818+0000 7f4a9a0ce640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a8006b4d0 0x7f4a8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:07.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.818+0000 7f4a9a0ce640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a8006b4d0 0x7f4a8006d990 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7f4a8c000f60 tx=0x7f4a8c007e40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:07.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:07.822+0000 7f4a92ffd640 1 -- 172.21.15.64:0/2008617766 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4a84058720 con 0x7f4a9415df50 2024-03-04T07:31:08.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:08.076+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]} v 0) v1 -- 0x7f4a940b0ba0 con 0x7f4a9415df50 2024-03-04T07:31:09.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.078+0000 7f4a92ffd640 1 -- 172.21.15.64:0/2008617766 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]}]=0 v241) v1 ==== 137+0+0 (secure 0 0 0) 0x7f4a8405c7c0 con 0x7f4a9415df50 2024-03-04T07:31:09.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a8006b4d0 msgr2=0x7f4a8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a8006b4d0 0x7f4a8006d990 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7f4a8c000f60 tx=0x7f4a8c007e40 comp rx=0 tx=0).stop 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9415df50 msgr2=0x7f4a9415f9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9415df50 0x7f4a9415f9d0 secure :-1 s=READY pgs=1075 cs=0 l=1 rev1=1 crypto rx=0x7f4a8400c6e0 tx=0x7f4a8400cba0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 shutdown_connections 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4a8006b4d0 0x7f4a8006d990 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4a9415df50 0x7f4a9415f9d0 unknown :-1 s=CLOSED pgs=1075 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4a9406cc10 0x7f4a9414df30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 --2- 172.21.15.64:0/2008617766 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4a9406c240 0x7f4a94157420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.082+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 >> 172.21.15.64:0/2008617766 conn(0x7f4a940d1a60 msgr2=0x7f4a940cfd50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.083+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 shutdown_connections 2024-03-04T07:31:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.083+0000 7f4a9c332640 1 -- 172.21.15.64:0/2008617766 wait complete. 2024-03-04T07:31:09.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:31:09.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.207+0000 7f1230264640 1 Processor -- start 2024-03-04T07:31:09.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.207+0000 7f1230264640 1 -- start start 2024-03-04T07:31:09.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f1230264640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f122815b130 0x7f122815b5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f1230264640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f12280ce900 0x7f12280ced30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f1230264640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f122815bc40 con 0x7f1228144640 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f1230264640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f122806df70 con 0x7f12280ce900 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f1230264640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f12280cf270 con 0x7f122815b130 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f122e000640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1228144640 0x7f122806e400 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55360/0 (socket says 172.21.15.64:55360) 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f122e000640 1 -- 172.21.15.64:0/705034471 learned_addr learned my addr 172.21.15.64:0/705034471 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f122e801640 1 --2- 172.21.15.64:0/705034471 >> v2:172.21.15.64:3300/0 conn(0x7f12280ce900 0x7f12280ced30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.208+0000 7f122d7ff640 1 --2- 172.21.15.64:0/705034471 >> v2:172.21.15.66:3300/0 conn(0x7f122815b130 0x7f122815b5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.209+0000 7f122cffe640 1 -- 172.21.15.64:0/705034471 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 981612869 0 0) 0x7f122815bc40 con 0x7f1228144640 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.209+0000 7f122cffe640 1 -- 172.21.15.64:0/705034471 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1214003520 con 0x7f1228144640 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.209+0000 7f122e801640 1 -- 172.21.15.64:0/705034471 >> v2:172.21.15.66:3300/0 conn(0x7f122815b130 msgr2=0x7f122815b5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.209+0000 7f122e801640 1 --2- 172.21.15.64:0/705034471 >> v2:172.21.15.66:3300/0 conn(0x7f122815b130 0x7f122815b5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.209+0000 7f122e801640 1 -- 172.21.15.64:0/705034471 >> v1:172.21.15.64:6789/0 conn(0x7f1228144640 legacy=0x7f122806e400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.209+0000 7f122e801640 1 -- 172.21.15.64:0/705034471 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f122815bf20 con 0x7f12280ce900 2024-03-04T07:31:09.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.209+0000 7f122e801640 1 --2- 172.21.15.64:0/705034471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12280ce900 0x7f12280ced30 secure :-1 s=READY pgs=1077 cs=0 l=1 rev1=1 crypto rx=0x7f12280706a0 tx=0x7f122002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=19560f60dcbf70ae server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:09.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f122cffe640 1 -- 172.21.15.64:0/705034471 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1220039070 con 0x7f12280ce900 2024-03-04T07:31:09.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f122cffe640 1 -- 172.21.15.64:0/705034471 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f122002cc80 con 0x7f12280ce900 2024-03-04T07:31:09.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f122cffe640 1 -- 172.21.15.64:0/705034471 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1220034660 con 0x7f12280ce900 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 -- 172.21.15.64:0/705034471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12280ce900 msgr2=0x7f12280ced30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 --2- 172.21.15.64:0/705034471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12280ce900 0x7f12280ced30 secure :-1 s=READY pgs=1077 cs=0 l=1 rev1=1 crypto rx=0x7f12280706a0 tx=0x7f122002bed0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 -- 172.21.15.64:0/705034471 shutdown_connections 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 --2- 172.21.15.64:0/705034471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12280ce900 0x7f12280ced30 unknown :-1 s=CLOSED pgs=1077 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 --2- 172.21.15.64:0/705034471 >> v2:172.21.15.66:3300/0 conn(0x7f122815b130 0x7f122815b5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 -- 172.21.15.64:0/705034471 >> 172.21.15.64:0/705034471 conn(0x7f1228066d00 msgr2=0x7f1228067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 -- 172.21.15.64:0/705034471 shutdown_connections 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.210+0000 7f1230264640 1 -- 172.21.15.64:0/705034471 wait complete. 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 Processor -- start 2024-03-04T07:31:09.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 -- start start 2024-03-04T07:31:09.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12280ce900 0x7f122813c860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1228144640 0x7f122815a440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f122815b130 0x7f12281f3ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1228070aa0 con 0x7f1228144640 2024-03-04T07:31:09.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f122806eb10 con 0x7f122815b130 2024-03-04T07:31:09.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f1230264640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1228070c20 con 0x7f12280ce900 2024-03-04T07:31:09.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f122d7ff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1228144640 0x7f122815a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f122d7ff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1228144640 0x7f122815a440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54812/0 (socket says 172.21.15.64:54812) 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.212+0000 7f122d7ff640 1 -- 172.21.15.64:0/2281715538 learned_addr learned my addr 172.21.15.64:0/2281715538 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122e000640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12280ce900 0x7f122813c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122e801640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f122815b130 0x7f12281f3ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122d7ff640 1 -- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12280ce900 msgr2=0x7f122813c860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122d7ff640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12280ce900 0x7f122813c860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122d7ff640 1 -- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f122815b130 msgr2=0x7f12281f3ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122d7ff640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f122815b130 0x7f12281f3ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122d7ff640 1 -- 172.21.15.64:0/2281715538 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f122813c620 con 0x7f1228144640 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f122d7ff640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1228144640 0x7f122815a440 secure :-1 s=READY pgs=1078 cs=0 l=1 rev1=1 crypto rx=0x7f121800e8a0 tx=0x7f121800ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f121e7fc640 1 -- 172.21.15.64:0/2281715538 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1218017070 con 0x7f1228144640 2024-03-04T07:31:09.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f121e7fc640 1 -- 172.21.15.64:0/2281715538 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f121800c940 con 0x7f1228144640 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f121e7fc640 1 -- 172.21.15.64:0/2281715538 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f121800cd30 con 0x7f1228144640 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f122813cda0 con 0x7f1228144640 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.213+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1228141dc0 con 0x7f1228144640 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.214+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f12280733c0 con 0x7f1228144640 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.215+0000 7f121e7fc640 1 -- 172.21.15.64:0/2281715538 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f121801a070 con 0x7f1228144640 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.215+0000 7f121e7fc640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f120c06b4d0 0x7f120c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.215+0000 7f121e7fc640 1 -- 172.21.15.64:0/2281715538 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f1218089ce0 con 0x7f1228144640 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.215+0000 7f122e000640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f120c06b4d0 0x7f120c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.215+0000 7f122e000640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f120c06b4d0 0x7f120c06d990 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f1224005880 tx=0x7f1224009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:09.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.219+0000 7f121e7fc640 1 -- 172.21.15.64:0/2281715538 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1218058810 con 0x7f1228144640 2024-03-04T07:31:09.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.400+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f1228143ad0 con 0x7f1228144640 2024-03-04T07:31:09.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.403+0000 7f121e7fc640 1 -- 172.21.15.64:0/2281715538 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v28)=0 v28) v1 ==== 122+0+0 (secure 0 0 0) 0x7f121805c8b0 con 0x7f1228144640 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f120c06b4d0 msgr2=0x7f120c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f120c06b4d0 0x7f120c06d990 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f1224005880 tx=0x7f1224009040 comp rx=0 tx=0).stop 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1228144640 msgr2=0x7f122815a440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1228144640 0x7f122815a440 secure :-1 s=READY pgs=1078 cs=0 l=1 rev1=1 crypto rx=0x7f121800e8a0 tx=0x7f121800ed60 comp rx=0 tx=0).stop 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 shutdown_connections 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f120c06b4d0 0x7f120c06d990 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f122815b130 0x7f12281f3ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1228144640 0x7f122815a440 unknown :-1 s=CLOSED pgs=1078 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 --2- 172.21.15.64:0/2281715538 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12280ce900 0x7f122813c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 >> 172.21.15.64:0/2281715538 conn(0x7f1228066d00 msgr2=0x7f12280670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:09.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 shutdown_connections 2024-03-04T07:31:09.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.405+0000 7f1230264640 1 -- 172.21.15.64:0/2281715538 wait complete. 2024-03-04T07:31:09.417 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:31:09.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:31:09.464 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:31:09.513 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:31:09.513 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:31:09.513 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:09.528 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:31:09.529 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:31:09.529 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:09.547 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:31:09.547 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:31:09.547 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:09.583 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:31:09.583 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:31:09.583 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:09.606 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:31:09.606 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:31:09.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:09.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.724+0000 7f62ffb13640 1 Processor -- start 2024-03-04T07:31:09.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.724+0000 7f62ffb13640 1 -- start start 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62ffb13640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62f806e700 0x7f62f815ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62ffb13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f8072660 0x7f62f806d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62ffb13640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f62f80b4160 con 0x7f62f80b55d0 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62ffb13640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62f80b5b40 con 0x7f62f806e700 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62ffb13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62f80b3ba0 con 0x7f62f8072660 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd8af640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62f806e700 0x7f62f815ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd8af640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62f806e700 0x7f62f815ef80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54820/0 (socket says 172.21.15.64:54820) 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd8af640 1 -- 172.21.15.64:0/3894788490 learned_addr learned my addr 172.21.15.64:0/3894788490 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd0ae640 1 --2- 172.21.15.64:0/3894788490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f8072660 0x7f62f806d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd8af640 1 -- 172.21.15.64:0/3894788490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f8072660 msgr2=0x7f62f806d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:09.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd8af640 1 --2- 172.21.15.64:0/3894788490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f8072660 0x7f62f806d9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd8af640 1 -- 172.21.15.64:0/3894788490 >> v1:172.21.15.66:6789/0 conn(0x7f62f80b55d0 legacy=0x7f62f8071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:09.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.725+0000 7f62fd8af640 1 -- 172.21.15.64:0/3894788490 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62f80b3e80 con 0x7f62f806e700 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.726+0000 7f62fd8af640 1 --2- 172.21.15.64:0/3894788490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f806e700 0x7f62f815ef80 secure :-1 s=READY pgs=1079 cs=0 l=1 rev1=1 crypto rx=0x7f62f4009870 tx=0x7f62f402b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=457aa7bc594afe35 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.726+0000 7f62fc8ad640 1 -- 172.21.15.64:0/3894788490 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62f4035020 con 0x7f62f806e700 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.726+0000 7f62fc8ad640 1 -- 172.21.15.64:0/3894788490 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62f4031640 con 0x7f62f806e700 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.726+0000 7f62fc8ad640 1 -- 172.21.15.64:0/3894788490 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62f402d040 con 0x7f62f806e700 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 -- 172.21.15.64:0/3894788490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f806e700 msgr2=0x7f62f815ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 --2- 172.21.15.64:0/3894788490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f806e700 0x7f62f815ef80 secure :-1 s=READY pgs=1079 cs=0 l=1 rev1=1 crypto rx=0x7f62f4009870 tx=0x7f62f402b8f0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 -- 172.21.15.64:0/3894788490 shutdown_connections 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 --2- 172.21.15.64:0/3894788490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f8072660 0x7f62f806d9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 --2- 172.21.15.64:0/3894788490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f806e700 0x7f62f815ef80 unknown :-1 s=CLOSED pgs=1079 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 -- 172.21.15.64:0/3894788490 >> 172.21.15.64:0/3894788490 conn(0x7f62f80b2720 msgr2=0x7f62f80b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:09.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 -- 172.21.15.64:0/3894788490 shutdown_connections 2024-03-04T07:31:09.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.727+0000 7f62ffb13640 1 -- 172.21.15.64:0/3894788490 wait complete. 2024-03-04T07:31:09.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.728+0000 7f62ffb13640 1 Processor -- start 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.728+0000 7f62ffb13640 1 -- start start 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.728+0000 7f62ffb13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f806e700 0x7f62f8149e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.728+0000 7f62ffb13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f8072660 0x7f62f80cd420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.728+0000 7f62ffb13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62f80b55d0 0x7f62f8161860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ffb13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62f806ec60 con 0x7f62f8072660 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ffb13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62f806eae0 con 0x7f62f80b55d0 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ffb13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62f8072090 con 0x7f62f806e700 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f8072660 0x7f62f80cd420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f8072660 0x7f62f80cd420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54836/0 (socket says 172.21.15.64:54836) 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 -- 172.21.15.64:0/1671325956 learned_addr learned my addr 172.21.15.64:0/1671325956 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 -- 172.21.15.64:0/1671325956 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f806e700 msgr2=0x7f62f8149e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f806e700 0x7f62f8149e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fe0b0640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62f80b55d0 0x7f62f8161860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 -- 172.21.15.64:0/1671325956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62f80b55d0 msgr2=0x7f62f8161860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62f80b55d0 0x7f62f8161860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 -- 172.21.15.64:0/1671325956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62f80cdee0 con 0x7f62f8072660 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62fd0ae640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f8072660 0x7f62f80cd420 secure :-1 s=READY pgs=1080 cs=0 l=1 rev1=1 crypto rx=0x7f62e800c820 tx=0x7f62e800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ee7fc640 1 -- 172.21.15.64:0/1671325956 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62e8015070 con 0x7f62f8072660 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ee7fc640 1 -- 172.21.15.64:0/1671325956 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f62e8004220 con 0x7f62f8072660 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f62f80ce1c0 con 0x7f62f8072660 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ee7fc640 1 -- 172.21.15.64:0/1671325956 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62e800ce10 con 0x7f62f8072660 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.729+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62f80ce780 con 0x7f62f8072660 2024-03-04T07:31:09.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.730+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f62f8067ff0 con 0x7f62f8072660 2024-03-04T07:31:09.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.731+0000 7f62ee7fc640 1 -- 172.21.15.64:0/1671325956 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f62e8007830 con 0x7f62f8072660 2024-03-04T07:31:09.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.731+0000 7f62ee7fc640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62dc06b4d0 0x7f62dc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:09.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.731+0000 7f62ee7fc640 1 -- 172.21.15.64:0/1671325956 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f62e8089da0 con 0x7f62f8072660 2024-03-04T07:31:09.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.731+0000 7f62fd8af640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62dc06b4d0 0x7f62dc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:09.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.731+0000 7f62fd8af640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62dc06b4d0 0x7f62dc06d990 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f62f4036000 tx=0x7f62f4030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:09.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.735+0000 7f62ee7fc640 1 -- 172.21.15.64:0/1671325956 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f62e8058f20 con 0x7f62f8072660 2024-03-04T07:31:09.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.956+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f62f815be20 con 0x7f62f8072660 2024-03-04T07:31:09.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.956+0000 7f62ee7fc640 1 -- 172.21.15.64:0/1671325956 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 45 v45) v1 ==== 94+0+3693 (secure 0 0 0) 0x7f62e8058820 con 0x7f62f8072660 2024-03-04T07:31:09.957 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:09.957 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":45,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":5801,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/30336518","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":30336518},{"type":"v1","addr":"172.21.15.64:6837","nonce":30336518}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":5963,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/1233694531","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":1233694531},{"type":"v1","addr":"172.21.15.64:6836","nonce":1233694531}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":45},{"gid":6379,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1761903530","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1761903530},{"type":"v1","addr":"172.21.15.66:6834","nonce":1761903530}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40},{"gid":6382,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3731045051","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3731045051},{"type":"v1","addr":"172.21.15.66:6835","nonce":3731045051}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":40}],"filesystems":[]} 2024-03-04T07:31:09.958 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 45 2024-03-04T07:31:09.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62dc06b4d0 msgr2=0x7f62dc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62dc06b4d0 0x7f62dc06d990 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f62f4036000 tx=0x7f62f4030000 comp rx=0 tx=0).stop 2024-03-04T07:31:09.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f8072660 msgr2=0x7f62f80cd420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:09.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f8072660 0x7f62f80cd420 secure :-1 s=READY pgs=1080 cs=0 l=1 rev1=1 crypto rx=0x7f62e800c820 tx=0x7f62e800cce0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 shutdown_connections 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f62dc06b4d0 0x7f62dc06d990 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f62f80b55d0 0x7f62f8161860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62f8072660 0x7f62f80cd420 unknown :-1 s=CLOSED pgs=1080 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 --2- 172.21.15.64:0/1671325956 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62f806e700 0x7f62f8149e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 >> 172.21.15.64:0/1671325956 conn(0x7f62f80b2720 msgr2=0x7f62f80b2b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 shutdown_connections 2024-03-04T07:31:09.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:09.958+0000 7f62ffb13640 1 -- 172.21.15.64:0/1671325956 wait complete. 2024-03-04T07:31:09.970 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:31:09.970 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:31:09.970 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:31:09.970 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:09.971 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:31:09.971 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:31:09.971 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:31:09.971 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:09.971 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:31:09.971 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:31:09.971 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:31:09.971 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:09.972 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:31:09.972 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:31:09.972 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:31:09.972 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:09.980 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:31:09.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:31:09.981 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:31:09.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:31:09.984 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:31:09.984 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:31:09.985 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:31:09.985 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:31:10.022 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:31:10.022 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:10.022 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:31:10.023 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:10.027 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:31:10.027 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:10.027 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:31:10.028 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:10.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:31:10.098 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:31:10.098 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:31:10.119 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:31:10.119 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:31:10.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.134+0000 7f89291e2640 1 Processor -- start 2024-03-04T07:31:10.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.134+0000 7f89291e2640 1 -- start start 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f89291e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 0x7f89240d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f89291e2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f89240cea30 con 0x7f89240b9a30 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f89291e2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f89240d2af0 con 0x7f89240b95a0 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f89291e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89240cf3c0 con 0x7f892415df00 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f892359e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f89240b9a30 0x7f89240d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55388/0 (socket says 172.21.15.64:55388) 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f892359e640 1 -- 172.21.15.64:0/238747107 learned_addr learned my addr 172.21.15.64:0/238747107 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 640992136 0 0) 0x7f89240cea30 con 0x7f89240b9a30 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f890c003520 con 0x7f89240b9a30 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8922d9d640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 0x7f89240d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1649641121 0 0) 0x7f890c003520 con 0x7f89240b9a30 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 msgr2=0x7f89240d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 0x7f89240d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 >> v1:172.21.15.66:6789/0 conn(0x7f89240b95a0 legacy=0x7f89240b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:10.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89240cf0e0 con 0x7f89240b9a30 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f891c0019f0 con 0x7f89240b9a30 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8922d9d640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 0x7f89240d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 >> v1:172.21.15.64:6789/0 conn(0x7f89240b9a30 legacy=0x7f89240d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 0x7f890c0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c003fd0 0x7f890c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f890c004d40 0x7f890c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.135+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89240cf3c0 con 0x7f890c004d40 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f890c014330 con 0x7f890c003fd0 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f890c014530 con 0x7f892415df00 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f892359e640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f890c004d40 0x7f890c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f8922d9d640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 0x7f890c0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f892259c640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c003fd0 0x7f890c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f892359e640 1 -- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 msgr2=0x7f890c0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:10.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f892359e640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f892415df00 0x7f890c0039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:10.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f892359e640 1 -- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c003fd0 msgr2=0x7f890c004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:10.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f892359e640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c003fd0 0x7f890c004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:10.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.136+0000 7f892359e640 1 -- 172.21.15.64:0/238747107 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f890c003520 con 0x7f890c004d40 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f892359e640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f890c004d40 0x7f890c0051c0 secure :-1 s=READY pgs=1089 cs=0 l=1 rev1=1 crypto rx=0x7f89240d1780 tx=0x7f890c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d13b1bd43a56ee63 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f891c00a4a0 con 0x7f890c004d40 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f891c034650 con 0x7f890c004d40 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f8921d9b640 1 -- 172.21.15.64:0/238747107 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f891c034a80 con 0x7f890c004d40 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f89291e2640 1 -- 172.21.15.64:0/238747107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f890c004d40 msgr2=0x7f890c0051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f89291e2640 1 --2- 172.21.15.64:0/238747107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f890c004d40 0x7f890c0051c0 secure :-1 s=READY pgs=1089 cs=0 l=1 rev1=1 crypto rx=0x7f89240d1780 tx=0x7f890c005830 comp rx=0 tx=0).stop 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f892359e640 1 -- 172.21.15.64:0/238747107 reap_dead start 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f89291e2640 1 -- 172.21.15.64:0/238747107 shutdown_connections 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f89291e2640 1 -- 172.21.15.64:0/238747107 >> 172.21.15.64:0/238747107 conn(0x7f8924071960 msgr2=0x7f8924071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f89291e2640 1 -- 172.21.15.64:0/238747107 shutdown_connections 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.137+0000 7f89291e2640 1 -- 172.21.15.64:0/238747107 wait complete. 2024-03-04T07:31:10.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 Processor -- start 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 -- start start 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c004d40 0x7f892414b000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89240b8740 0x7f89240b8ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89240cce10 0x7f89240cd290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89240cec70 con 0x7f89240b8740 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89240ce750 con 0x7f890c004d40 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.139+0000 7f89291e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89240cedf0 con 0x7f89240cce10 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89240b8740 0x7f89240b8ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89240b8740 0x7f89240b8ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54870/0 (socket says 172.21.15.64:54870) 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 -- 172.21.15.64:0/374167128 learned_addr learned my addr 172.21.15.64:0/374167128 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892359e640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89240cce10 0x7f89240cd290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f8922d9d640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c004d40 0x7f892414b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 -- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89240cce10 msgr2=0x7f89240cd290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89240cce10 0x7f89240cd290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 -- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c004d40 msgr2=0x7f892414b000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c004d40 0x7f892414b000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 -- 172.21.15.64:0/374167128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89240cdb90 con 0x7f89240b8740 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f892259c640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89240b8740 0x7f89240b8ba0 secure :-1 s=READY pgs=1090 cs=0 l=1 rev1=1 crypto rx=0x7f891800f880 tx=0x7f891800fd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f890b7fe640 1 -- 172.21.15.64:0/374167128 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8918001d20 con 0x7f89240b8740 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f890b7fe640 1 -- 172.21.15.64:0/374167128 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8918018070 con 0x7f89240b8740 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.140+0000 7f890b7fe640 1 -- 172.21.15.64:0/374167128 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f891800d730 con 0x7f89240b8740 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.141+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f89240c78e0 con 0x7f89240b8740 2024-03-04T07:31:10.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.141+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89240c7e60 con 0x7f89240b8740 2024-03-04T07:31:10.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.141+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8924067ff0 con 0x7f89240b8740 2024-03-04T07:31:10.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.142+0000 7f890b7fe640 1 -- 172.21.15.64:0/374167128 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f891800d8d0 con 0x7f89240b8740 2024-03-04T07:31:10.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.142+0000 7f890b7fe640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891006b4d0 0x7f891006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:10.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.142+0000 7f890b7fe640 1 -- 172.21.15.64:0/374167128 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 4943+0+0 (secure 0 0 0) 0x7f8918089d80 con 0x7f89240b8740 2024-03-04T07:31:10.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.142+0000 7f8922d9d640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891006b4d0 0x7f891006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:10.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.143+0000 7f8922d9d640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891006b4d0 0x7f891006d990 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f8914006fd0 tx=0x7f8914008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:10.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.147+0000 7f890b7fe640 1 -- 172.21.15.64:0/374167128 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8918058f00 con 0x7f89240b8740 2024-03-04T07:31:10.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:10.338+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f89240d3a80 con 0x7f89240b8740 2024-03-04T07:31:11.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.093+0000 7f890b7fe640 1 -- 172.21.15.64:0/374167128 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v84) v1 ==== 97+0+0 (secure 0 0 0) 0x7f8918058800 con 0x7f89240b8740 2024-03-04T07:31:11.094 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:31:11.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.096+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891006b4d0 msgr2=0x7f891006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.096+0000 7f89291e2640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891006b4d0 0x7f891006d990 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f8914006fd0 tx=0x7f8914008040 comp rx=0 tx=0).stop 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.096+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89240b8740 msgr2=0x7f89240b8ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.096+0000 7f89291e2640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89240b8740 0x7f89240b8ba0 secure :-1 s=READY pgs=1090 cs=0 l=1 rev1=1 crypto rx=0x7f891800f880 tx=0x7f891800fd40 comp rx=0 tx=0).stop 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 shutdown_connections 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f891006b4d0 0x7f891006d990 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89240cce10 0x7f89240cd290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89240b8740 0x7f89240b8ba0 unknown :-1 s=CLOSED pgs=1090 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 --2- 172.21.15.64:0/374167128 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f890c004d40 0x7f892414b000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 >> 172.21.15.64:0/374167128 conn(0x7f8924071960 msgr2=0x7f89240d3fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 shutdown_connections 2024-03-04T07:31:11.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.097+0000 7f89291e2640 1 -- 172.21.15.64:0/374167128 wait complete. 2024-03-04T07:31:11.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:31:11.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.219+0000 7f2aadf8a640 1 Processor -- start 2024-03-04T07:31:11.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.219+0000 7f2aadf8a640 1 -- start start 2024-03-04T07:31:11.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.220+0000 7f2aadf8a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2aa815df00 0x7f2aa80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.220+0000 7f2aadf8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa80b9a30 0x7f2aa80d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.220+0000 7f2aadf8a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2aa80cea30 con 0x7f2aa80b95a0 2024-03-04T07:31:11.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.220+0000 7f2aadf8a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2aa80d2af0 con 0x7f2aa815df00 2024-03-04T07:31:11.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.220+0000 7f2aadf8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2aa80cf3c0 con 0x7f2aa80b9a30 2024-03-04T07:31:11.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.220+0000 7f2aa6ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2aa80b95a0 0x7f2aa80b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55394/0 (socket says 172.21.15.64:55394) 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.220+0000 7f2aa6ffd640 1 -- 172.21.15.64:0/4276085735 learned_addr learned my addr 172.21.15.64:0/4276085735 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa77fe640 1 --2- 172.21.15.64:0/4276085735 >> v2:172.21.15.66:3300/0 conn(0x7f2aa815df00 0x7f2aa80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2029222429 0 0) 0x7f2aa80cea30 con 0x7f2aa80b95a0 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2a90003520 con 0x7f2aa80b95a0 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa7fff640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa80b9a30 0x7f2aa80d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4087481800 0 0) 0x7f2a90003520 con 0x7f2aa80b95a0 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa80b9a30 msgr2=0x7f2aa80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa80b9a30 0x7f2aa80d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 >> v2:172.21.15.66:3300/0 conn(0x7f2aa815df00 msgr2=0x7f2aa80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 --2- 172.21.15.64:0/4276085735 >> v2:172.21.15.66:3300/0 conn(0x7f2aa815df00 0x7f2aa80d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2aa80cf0e0 con 0x7f2aa80b95a0 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2a9c001670 con 0x7f2aa80b95a0 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 >> v1:172.21.15.64:6789/0 conn(0x7f2aa80b95a0 legacy=0x7f2aa80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a900039b0 0x7f2a90003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a90004800 0x7f2a90004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a900051c0 0x7f2a9000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2aa80d2af0 con 0x7f2a90004800 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2aa80cf3c0 con 0x7f2a900039b0 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a900045f0 con 0x7f2a900051c0 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa6ffd640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a90004800 0x7f2a90004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa77fe640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a900039b0 0x7f2a90003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa7fff640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a900051c0 0x7f2a9000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa6ffd640 1 -- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a900051c0 msgr2=0x7f2a9000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa6ffd640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a900051c0 0x7f2a9000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa6ffd640 1 -- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a900039b0 msgr2=0x7f2a90003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa6ffd640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a900039b0 0x7f2a90003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.221+0000 7f2aa6ffd640 1 -- 172.21.15.64:0/4276085735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a90003520 con 0x7f2a90004800 2024-03-04T07:31:11.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.222+0000 7f2aa6ffd640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a90004800 0x7f2a90004c80 secure :-1 s=READY pgs=1092 cs=0 l=1 rev1=1 crypto rx=0x7f2a9c00c190 tx=0x7f2a90005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cdf6f998eb20d4fc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.222+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a9c007c80 con 0x7f2a90004800 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.222+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a9c030070 con 0x7f2a90004800 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.222+0000 7f2aa67fc640 1 -- 172.21.15.64:0/4276085735 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a9c005d30 con 0x7f2a90004800 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.223+0000 7f2aadf8a640 1 -- 172.21.15.64:0/4276085735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a90004800 msgr2=0x7f2a90004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.223+0000 7f2aadf8a640 1 --2- 172.21.15.64:0/4276085735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a90004800 0x7f2a90004c80 secure :-1 s=READY pgs=1092 cs=0 l=1 rev1=1 crypto rx=0x7f2a9c00c190 tx=0x7f2a90005830 comp rx=0 tx=0).stop 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.223+0000 7f2aa7fff640 1 -- 172.21.15.64:0/4276085735 reap_dead start 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.223+0000 7f2aadf8a640 1 -- 172.21.15.64:0/4276085735 shutdown_connections 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.223+0000 7f2aadf8a640 1 -- 172.21.15.64:0/4276085735 >> 172.21.15.64:0/4276085735 conn(0x7f2aa8071960 msgr2=0x7f2aa8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.223+0000 7f2aadf8a640 1 -- 172.21.15.64:0/4276085735 shutdown_connections 2024-03-04T07:31:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.223+0000 7f2aadf8a640 1 -- 172.21.15.64:0/4276085735 wait complete. 2024-03-04T07:31:11.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.224+0000 7f2aadf8a640 1 Processor -- start 2024-03-04T07:31:11.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.224+0000 7f2aadf8a640 1 -- start start 2024-03-04T07:31:11.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aadf8a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a90004800 0x7f2aa8160dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aadf8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa81f39b0 0x7f2aa81f3e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aadf8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2aa81f4370 0x7f2aa81f6fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aadf8a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2aa80cec70 con 0x7f2aa81f4370 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aadf8a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2aa80ce750 con 0x7f2a90004800 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aadf8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2aa80cedf0 con 0x7f2aa81f39b0 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2aa81f4370 0x7f2aa81f6fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2aa81f4370 0x7f2aa81f6fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54876/0 (socket says 172.21.15.64:54876) 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 -- 172.21.15.64:0/513298448 learned_addr learned my addr 172.21.15.64:0/513298448 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa77fe640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a90004800 0x7f2aa8160dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa6ffd640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa81f39b0 0x7f2aa81f3e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 -- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa81f39b0 msgr2=0x7f2aa81f3e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa81f39b0 0x7f2aa81f3e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 -- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a90004800 msgr2=0x7f2aa8160dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a90004800 0x7f2aa8160dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa7fff640 1 -- 172.21.15.64:0/513298448 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2aa81f4800 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.225+0000 7f2aa6ffd640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa81f39b0 0x7f2aa81f3e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.226+0000 7f2aa7fff640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2aa81f4370 0x7f2aa81f6fa0 secure :-1 s=READY pgs=1093 cs=0 l=1 rev1=1 crypto rx=0x7f2aa80d0f40 tx=0x7f2aa0011920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.226+0000 7f2a8ffff640 1 -- 172.21.15.64:0/513298448 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2aa0003610 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.226+0000 7f2a8ffff640 1 -- 172.21.15.64:0/513298448 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2aa001b070 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.226+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2aa8158eb0 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.226+0000 7f2a8ffff640 1 -- 172.21.15.64:0/513298448 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2aa0003a30 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.226+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2aa80caf60 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.227+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2aa80d27d0 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.227+0000 7f2a8ffff640 1 -- 172.21.15.64:0/513298448 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2aa0003bd0 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.228+0000 7f2a8ffff640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a9406b4d0 0x7f2a9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.228+0000 7f2aa77fe640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a9406b4d0 0x7f2a9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.228+0000 7f2a8ffff640 1 -- 172.21.15.64:0/513298448 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f2aa0089e70 con 0x7f2aa81f4370 2024-03-04T07:31:11.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.228+0000 7f2aa77fe640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a9406b4d0 0x7f2a9406d990 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f2aa80cdd50 tx=0x7f2a9800b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.232+0000 7f2a8ffff640 1 -- 172.21.15.64:0/513298448 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2aa80d27d0 con 0x7f2aa81f4370 2024-03-04T07:31:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.481+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f2aa80cad50 con 0x7f2aa81f4370 2024-03-04T07:31:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.482+0000 7f2a8ffff640 1 -- 172.21.15.64:0/513298448 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v19) v1 ==== 80+0+5304 (secure 0 0 0) 0x7f2aa0025020 con 0x7f2aa81f4370 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.484 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:31:11.486 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:31:11.488 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:31:11.490 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow r", 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.491 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.492 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:31:11.494 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:31:11.495 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a9406b4d0 msgr2=0x7f2a9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a9406b4d0 0x7f2a9406d990 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f2aa80cdd50 tx=0x7f2a9800b040 comp rx=0 tx=0).stop 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2aa81f4370 msgr2=0x7f2aa81f6fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2aa81f4370 0x7f2aa81f6fa0 secure :-1 s=READY pgs=1093 cs=0 l=1 rev1=1 crypto rx=0x7f2aa80d0f40 tx=0x7f2aa0011920 comp rx=0 tx=0).stop 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 shutdown_connections 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a9406b4d0 0x7f2a9406d990 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2aa81f4370 0x7f2aa81f6fa0 unknown :-1 s=CLOSED pgs=1093 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2aa81f39b0 0x7f2aa81f3e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 --2- 172.21.15.64:0/513298448 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a90004800 0x7f2aa8160dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 >> 172.21.15.64:0/513298448 conn(0x7f2aa8071960 msgr2=0x7f2aa80d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:11.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 shutdown_connections 2024-03-04T07:31:11.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.484+0000 7f2aadf8a640 1 -- 172.21.15.64:0/513298448 wait complete. 2024-03-04T07:31:11.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:11.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d96ff6640 1 Processor -- start 2024-03-04T07:31:11.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d96ff6640 1 -- start start 2024-03-04T07:31:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d96ff6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2d9006fe20 0x7f2d900cea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d96ff6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d900d0dd0 con 0x7f2d9015e060 2024-03-04T07:31:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d96ff6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d900d10b0 con 0x7f2d900d0880 2024-03-04T07:31:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d96ff6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d900cc0e0 con 0x7f2d9006fe20 2024-03-04T07:31:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d94d92640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2d9015e060 0x7f2d900ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:55408/0 (socket says 172.21.15.64:55408) 2024-03-04T07:31:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.599+0000 7f2d94d92640 1 -- 172.21.15.64:0/1464153596 learned_addr learned my addr 172.21.15.64:0/1464153596 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1492092711 0 0) 0x7f2d900d0dd0 con 0x7f2d9015e060 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1464153596 >> v2:172.21.15.66:3300/0 conn(0x7f2d9006fe20 0x7f2d900cea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d7c003520 con 0x7f2d9015e060 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4152837123 0 0) 0x7f2d7c003520 con 0x7f2d9015e060 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 >> v2:172.21.15.66:3300/0 conn(0x7f2d9006fe20 msgr2=0x7f2d900cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 --2- 172.21.15.64:0/1464153596 >> v2:172.21.15.66:3300/0 conn(0x7f2d9006fe20 0x7f2d900cea20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 >> v1:172.21.15.66:6789/0 conn(0x7f2d900d0880 legacy=0x7f2d900ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d900cc3c0 con 0x7f2d9015e060 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2d880019f0 con 0x7f2d9015e060 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 >> v1:172.21.15.64:6789/0 conn(0x7f2d9015e060 legacy=0x7f2d900ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d7c003a00 0x7f2d7c003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d7c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d7c0051c0 0x7f2d7c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d900cc0e0 con 0x7f2d7c004800 2024-03-04T07:31:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d7c00c0c0 con 0x7f2d7c003a00 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d7c00c2c0 con 0x7f2d7c0051c0 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d7c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d94d92640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d7c003a00 0x7f2d7c003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d95593640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d7c0051c0 0x7f2d7c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8ffff640 1 -- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d7c0051c0 msgr2=0x7f2d7c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d7c0051c0 0x7f2d7c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8ffff640 1 -- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d7c003a00 msgr2=0x7f2d7c003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d7c003a00 0x7f2d7c003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d8ffff640 1 -- 172.21.15.64:0/1464153596 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d7c003520 con 0x7f2d7c004800 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d95593640 1 -- 172.21.15.64:0/1464153596 reap_dead start 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.600+0000 7f2d94d92640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d7c003a00 0x7f2d7c003e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.601+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d7c004c80 secure :-1 s=READY pgs=1095 cs=0 l=1 rev1=1 crypto rx=0x7f2d78009a90 tx=0x7f2d7c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8057e4ae936fb522 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.601+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d78039070 con 0x7f2d7c004800 2024-03-04T07:31:11.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.601+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d7802c680 con 0x7f2d7c004800 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.601+0000 7f2d8f7fe640 1 -- 172.21.15.64:0/1464153596 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d78009690 con 0x7f2d7c004800 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.602+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1464153596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 msgr2=0x7f2d7c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.602+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d7c004c80 secure :-1 s=READY pgs=1095 cs=0 l=1 rev1=1 crypto rx=0x7f2d78009a90 tx=0x7f2d7c005830 comp rx=0 tx=0).stop 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.602+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1464153596 shutdown_connections 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.602+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1464153596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d7c004c80 unknown :-1 s=CLOSED pgs=1095 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.602+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1464153596 >> 172.21.15.64:0/1464153596 conn(0x7f2d900d1a90 msgr2=0x7f2d900d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.602+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1464153596 shutdown_connections 2024-03-04T07:31:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.602+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1464153596 wait complete. 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.603+0000 7f2d96ff6640 1 Processor -- start 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.603+0000 7f2d96ff6640 1 -- start start 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d96ff6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d900b6220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d96ff6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d900c8120 0x7f2d900b48e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d96ff6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d90154370 0x7f2d900cd530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d96ff6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d900ce310 con 0x7f2d7c004800 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d96ff6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d9015e440 con 0x7f2d900c8120 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d96ff6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d900ce490 con 0x7f2d90154370 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d900b6220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d900b6220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54886/0 (socket says 172.21.15.64:54886) 2024-03-04T07:31:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 -- 172.21.15.64:0/1821385287 learned_addr learned my addr 172.21.15.64:0/1821385287 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d900c8120 0x7f2d900b48e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 -- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d90154370 msgr2=0x7f2d900cd530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d95593640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d90154370 0x7f2d900cd530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d90154370 0x7f2d900cd530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 -- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d900c8120 msgr2=0x7f2d900b48e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d900c8120 0x7f2d900b48e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d94d92640 1 -- 172.21.15.64:0/1821385287 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d900b6760 con 0x7f2d7c004800 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.604+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d900c8120 0x7f2d900b48e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.605+0000 7f2d94d92640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d900b6220 secure :-1 s=READY pgs=1096 cs=0 l=1 rev1=1 crypto rx=0x7f2d880079e0 tx=0x7f2d8800e530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.605+0000 7f2d8d7fa640 1 -- 172.21.15.64:0/1821385287 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d8800ee50 con 0x7f2d7c004800 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.605+0000 7f2d8d7fa640 1 -- 172.21.15.64:0/1821385287 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d88018070 con 0x7f2d7c004800 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.605+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d900b5180 con 0x7f2d7c004800 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.605+0000 7f2d8d7fa640 1 -- 172.21.15.64:0/1821385287 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d88013620 con 0x7f2d7c004800 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.605+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d900b5a20 con 0x7f2d7c004800 2024-03-04T07:31:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.606+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d900b5570 con 0x7f2d7c004800 2024-03-04T07:31:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.606+0000 7f2d8d7fa640 1 -- 172.21.15.64:0/1821385287 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d880137c0 con 0x7f2d7c004800 2024-03-04T07:31:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.607+0000 7f2d8d7fa640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7006b500 0x7f2d7006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.607+0000 7f2d8d7fa640 1 -- 172.21.15.64:0/1821385287 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f2d8808a420 con 0x7f2d7c004800 2024-03-04T07:31:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.607+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7006b500 0x7f2d7006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.607+0000 7f2d8ffff640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7006b500 0x7f2d7006d9c0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f2d78036000 tx=0x7f2d78030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.611+0000 7f2d8d7fa640 1 -- 172.21.15.64:0/1821385287 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d880595f0 con 0x7f2d7c004800 2024-03-04T07:31:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.795+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2d900cec60 con 0x7f2d7c004800 2024-03-04T07:31:11.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.796+0000 7f2d8d7fa640 1 -- 172.21.15.64:0/1821385287 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v86) v1 ==== 74+0+12844 (secure 0 0 0) 0x7f2d88058ef0 con 0x7f2d7c004800 2024-03-04T07:31:11.797 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:11.797 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":86,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:11.102546+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":13,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7006b500 msgr2=0x7f2d7006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7006b500 0x7f2d7006d9c0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f2d78036000 tx=0x7f2d78030000 comp rx=0 tx=0).stop 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 msgr2=0x7f2d900b6220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d900b6220 secure :-1 s=READY pgs=1096 cs=0 l=1 rev1=1 crypto rx=0x7f2d880079e0 tx=0x7f2d8800e530 comp rx=0 tx=0).stop 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 shutdown_connections 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7006b500 0x7f2d7006d9c0 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d90154370 0x7f2d900cd530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d900c8120 0x7f2d900b48e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 --2- 172.21.15.64:0/1821385287 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d7c004800 0x7f2d900b6220 unknown :-1 s=CLOSED pgs=1096 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 >> 172.21.15.64:0/1821385287 conn(0x7f2d900d1a90 msgr2=0x7f2d90068e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:11.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 shutdown_connections 2024-03-04T07:31:11.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.798+0000 7f2d96ff6640 1 -- 172.21.15.64:0/1821385287 wait complete. 2024-03-04T07:31:11.810 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:31:11.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:31:11.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.914+0000 7f777e81c640 1 Processor -- start 2024-03-04T07:31:11.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.914+0000 7f777e81c640 1 -- start start 2024-03-04T07:31:11.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f777e81c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f777815e2f0 0x7f77780b41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f777e81c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f777806c050 0x7f77780cdac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f777e81c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f777815e6d0 con 0x7f777806b4e0 2024-03-04T07:31:11.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f777e81c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f77780d0a10 con 0x7f777815e2f0 2024-03-04T07:31:11.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f777e81c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f77780ccd00 con 0x7f777806c050 2024-03-04T07:31:11.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f7777fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f777815e2f0 0x7f77780b41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f7777fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f777815e2f0 0x7f77780b41f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54902/0 (socket says 172.21.15.64:54902) 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f7777fff640 1 -- 172.21.15.64:0/1065492044 learned_addr learned my addr 172.21.15.64:0/1065492044 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f777cdb9640 1 --2- 172.21.15.64:0/1065492044 >> v2:172.21.15.66:3300/0 conn(0x7f777806c050 0x7f77780cdac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f7777fff640 1 -- 172.21.15.64:0/1065492044 >> v2:172.21.15.66:3300/0 conn(0x7f777806c050 msgr2=0x7f77780cdac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f7777fff640 1 --2- 172.21.15.64:0/1065492044 >> v2:172.21.15.66:3300/0 conn(0x7f777806c050 0x7f77780cdac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f7777fff640 1 -- 172.21.15.64:0/1065492044 >> v1:172.21.15.64:6789/0 conn(0x7f777806b4e0 legacy=0x7f777806b940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.915+0000 7f7777fff640 1 -- 172.21.15.64:0/1065492044 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77780ccfe0 con 0x7f777815e2f0 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.916+0000 7f7777fff640 1 --2- 172.21.15.64:0/1065492044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777815e2f0 0x7f77780b41f0 secure :-1 s=READY pgs=1098 cs=0 l=1 rev1=1 crypto rx=0x7f7778072430 tx=0x7f776c02b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d79acd7489e9cae server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.916+0000 7f7776ffd640 1 -- 172.21.15.64:0/1065492044 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f776c02e070 con 0x7f777815e2f0 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.916+0000 7f7776ffd640 1 -- 172.21.15.64:0/1065492044 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f776c037070 con 0x7f777815e2f0 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.916+0000 7f7776ffd640 1 -- 172.21.15.64:0/1065492044 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f776c031a00 con 0x7f777815e2f0 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.916+0000 7f777e81c640 1 -- 172.21.15.64:0/1065492044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777815e2f0 msgr2=0x7f77780b41f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.916+0000 7f777e81c640 1 --2- 172.21.15.64:0/1065492044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777815e2f0 0x7f77780b41f0 secure :-1 s=READY pgs=1098 cs=0 l=1 rev1=1 crypto rx=0x7f7778072430 tx=0x7f776c02b880 comp rx=0 tx=0).stop 2024-03-04T07:31:11.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.916+0000 7f777e81c640 1 -- 172.21.15.64:0/1065492044 shutdown_connections 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.917+0000 7f777e81c640 1 --2- 172.21.15.64:0/1065492044 >> v2:172.21.15.66:3300/0 conn(0x7f777806c050 0x7f77780cdac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.917+0000 7f777e81c640 1 --2- 172.21.15.64:0/1065492044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777815e2f0 0x7f77780b41f0 unknown :-1 s=CLOSED pgs=1098 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.917+0000 7f777e81c640 1 -- 172.21.15.64:0/1065492044 >> 172.21.15.64:0/1065492044 conn(0x7f77780b36d0 msgr2=0x7f77780b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.917+0000 7f777e81c640 1 -- 172.21.15.64:0/1065492044 shutdown_connections 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.917+0000 7f777e81c640 1 -- 172.21.15.64:0/1065492044 wait complete. 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.918+0000 7f777e81c640 1 Processor -- start 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.918+0000 7f777e81c640 1 -- start start 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777806b4e0 0x7f77780b7330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f777806c050 0x7f7778147dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f777815e2f0 0x7f77780b47c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77780ce000 con 0x7f777806b4e0 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77780cd5a0 con 0x7f777806c050 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f77780ce180 con 0x7f777815e2f0 2024-03-04T07:31:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777806b4e0 0x7f77780b7330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777806b4e0 0x7f77780b7330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54906/0 (socket says 172.21.15.64:54906) 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 -- 172.21.15.64:0/4109483261 learned_addr learned my addr 172.21.15.64:0/4109483261 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777cdb9640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f777815e2f0 0x7f77780b47c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f77777fe640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f777806c050 0x7f7778147dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 -- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f777815e2f0 msgr2=0x7f77780b47c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f777815e2f0 0x7f77780b47c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 -- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f777806c050 msgr2=0x7f7778147dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f777806c050 0x7f7778147dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 -- 172.21.15.64:0/4109483261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f777815e990 con 0x7f777806b4e0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f77777fe640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f777806c050 0x7f7778147dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7777fff640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777806b4e0 0x7f77780b7330 secure :-1 s=READY pgs=1099 cs=0 l=1 rev1=1 crypto rx=0x7f776c000c00 tx=0x7f776c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7774ff9640 1 -- 172.21.15.64:0/4109483261 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f776c032670 con 0x7f777806b4e0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7774ff9640 1 -- 172.21.15.64:0/4109483261 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f776c032850 con 0x7f777806b4e0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f77780cb3d0 con 0x7f777806b4e0 2024-03-04T07:31:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f7774ff9640 1 -- 172.21.15.64:0/4109483261 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f776c032c40 con 0x7f777806b4e0 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.919+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f77780cb6b0 con 0x7f777806b4e0 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.920+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f777809fd30 con 0x7f777806b4e0 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.921+0000 7f7774ff9640 1 -- 172.21.15.64:0/4109483261 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f776c0453f0 con 0x7f777806b4e0 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.921+0000 7f7774ff9640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f776406b4d0 0x7f776406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.921+0000 7f7774ff9640 1 -- 172.21.15.64:0/4109483261 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f776c0abc40 con 0x7f777806b4e0 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.921+0000 7f77777fe640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f776406b4d0 0x7f776406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.922+0000 7f77777fe640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f776406b4d0 0x7f776406d990 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f77781619a0 tx=0x7f7768009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:11.925+0000 7f7774ff9640 1 -- 172.21.15.64:0/4109483261 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f776c040070 con 0x7f777806b4e0 2024-03-04T07:31:12.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.108+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f777806ef10 con 0x7f777806b4e0 2024-03-04T07:31:12.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.109+0000 7f7774ff9640 1 -- 172.21.15.64:0/4109483261 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v86) v1 ==== 93+0+10 (secure 0 0 0) 0x7f776c07a740 con 0x7f777806b4e0 2024-03-04T07:31:12.110 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f776406b4d0 msgr2=0x7f776406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f776406b4d0 0x7f776406d990 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f77781619a0 tx=0x7f7768009040 comp rx=0 tx=0).stop 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777806b4e0 msgr2=0x7f77780b7330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777806b4e0 0x7f77780b7330 secure :-1 s=READY pgs=1099 cs=0 l=1 rev1=1 crypto rx=0x7f776c000c00 tx=0x7f776c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 shutdown_connections 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f776406b4d0 0x7f776406d990 unknown :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f777815e2f0 0x7f77780b47c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f777806c050 0x7f7778147dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 --2- 172.21.15.64:0/4109483261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f777806b4e0 0x7f77780b7330 unknown :-1 s=CLOSED pgs=1099 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 >> 172.21.15.64:0/4109483261 conn(0x7f77780b36d0 msgr2=0x7f777806b940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:12.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 shutdown_connections 2024-03-04T07:31:12.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.111+0000 7f777e81c640 1 -- 172.21.15.64:0/4109483261 wait complete. 2024-03-04T07:31:12.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:31:12.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.224+0000 7f89dcd02640 1 Processor -- start 2024-03-04T07:31:12.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.225+0000 7f89dcd02640 1 -- start start 2024-03-04T07:31:12.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.225+0000 7f89dcd02640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f89d815df70 0x7f89d80d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.225+0000 7f89dcd02640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f89d80d18c0 0x7f89d8071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.225+0000 7f89dcd02640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f89d80b6f20 con 0x7f89d8071a80 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.225+0000 7f89dcd02640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f89d8071eb0 con 0x7f89d80d18c0 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.225+0000 7f89dcd02640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f89d80d3030 con 0x7f89d815df70 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d5d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f89d80d18c0 0x7f89d8071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d5d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f89d80d18c0 0x7f89d8071540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54914/0 (socket says 172.21.15.64:54914) 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1891089763 learned_addr learned my addr 172.21.15.64:0/1891089763 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d659c640 1 --2- 172.21.15.64:0/1891089763 >> v2:172.21.15.66:3300/0 conn(0x7f89d815df70 0x7f89d80d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1891089763 >> v2:172.21.15.66:3300/0 conn(0x7f89d815df70 msgr2=0x7f89d80d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d5d9b640 1 --2- 172.21.15.64:0/1891089763 >> v2:172.21.15.66:3300/0 conn(0x7f89d815df70 0x7f89d80d27b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1891089763 >> v1:172.21.15.66:6789/0 conn(0x7f89d8071a80 legacy=0x7f89d80b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1891089763 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89d80d3310 con 0x7f89d80d18c0 2024-03-04T07:31:12.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.226+0000 7f89d659c640 1 --2- 172.21.15.64:0/1891089763 >> v2:172.21.15.66:3300/0 conn(0x7f89d815df70 0x7f89d80d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89d5d9b640 1 --2- 172.21.15.64:0/1891089763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8071540 secure :-1 s=READY pgs=1100 cs=0 l=1 rev1=1 crypto rx=0x7f89d804f3a0 tx=0x7f89cc02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bdd6656c15e2a6c8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89d559a640 1 -- 172.21.15.64:0/1891089763 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89cc034070 con 0x7f89d80d18c0 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89d559a640 1 -- 172.21.15.64:0/1891089763 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89cc030860 con 0x7f89d80d18c0 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89d559a640 1 -- 172.21.15.64:0/1891089763 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89cc030c30 con 0x7f89d80d18c0 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 -- 172.21.15.64:0/1891089763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 msgr2=0x7f89d8071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1891089763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8071540 secure :-1 s=READY pgs=1100 cs=0 l=1 rev1=1 crypto rx=0x7f89d804f3a0 tx=0x7f89cc02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 -- 172.21.15.64:0/1891089763 shutdown_connections 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1891089763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8071540 unknown :-1 s=CLOSED pgs=1100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1891089763 >> v2:172.21.15.66:3300/0 conn(0x7f89d815df70 0x7f89d80d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 -- 172.21.15.64:0/1891089763 >> 172.21.15.64:0/1891089763 conn(0x7f89d80d3cf0 msgr2=0x7f89d80d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 -- 172.21.15.64:0/1891089763 shutdown_connections 2024-03-04T07:31:12.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.227+0000 7f89dcd02640 1 -- 172.21.15.64:0/1891089763 wait complete. 2024-03-04T07:31:12.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 Processor -- start 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 -- start start 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89d8071a80 0x7f89d8157020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8158c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89d815df70 0x7f89d815fdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89d80d2cf0 con 0x7f89d80d18c0 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89d80d38d0 con 0x7f89d8071a80 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89dcd02640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89d815e350 con 0x7f89d815df70 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89d5d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8158c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89d5d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8158c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54916/0 (socket says 172.21.15.64:54916) 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.229+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1431103186 learned_addr learned my addr 172.21.15.64:0/1431103186 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d659c640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89d8071a80 0x7f89d8157020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d6d9d640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89d815df70 0x7f89d815fdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89d815df70 msgr2=0x7f89d815fdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d5d9b640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89d815df70 0x7f89d815fdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89d8071a80 msgr2=0x7f89d8157020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d5d9b640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89d8071a80 0x7f89d8157020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d5d9b640 1 -- 172.21.15.64:0/1431103186 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89d8157c10 con 0x7f89d80d18c0 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d6d9d640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89d815df70 0x7f89d815fdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89d5d9b640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8158c40 secure :-1 s=READY pgs=1101 cs=0 l=1 rev1=1 crypto rx=0x7f89cc000c00 tx=0x7f89cc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89c6ffd640 1 -- 172.21.15.64:0/1431103186 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89cc048070 con 0x7f89d80d18c0 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89c6ffd640 1 -- 172.21.15.64:0/1431103186 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89cc039d90 con 0x7f89d80d18c0 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89c6ffd640 1 -- 172.21.15.64:0/1431103186 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89cc034040 con 0x7f89d80d18c0 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f89d8157e80 con 0x7f89d80d18c0 2024-03-04T07:31:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.230+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89d8158440 con 0x7f89d80d18c0 2024-03-04T07:31:12.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.231+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f89d8067ff0 con 0x7f89d80d18c0 2024-03-04T07:31:12.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.232+0000 7f89c6ffd640 1 -- 172.21.15.64:0/1431103186 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f89cc04b070 con 0x7f89d80d18c0 2024-03-04T07:31:12.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.232+0000 7f89c6ffd640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89b806b4d0 0x7f89b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.232+0000 7f89c6ffd640 1 -- 172.21.15.64:0/1431103186 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f89cc0abf30 con 0x7f89d80d18c0 2024-03-04T07:31:12.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.232+0000 7f89d659c640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89b806b4d0 0x7f89b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.233+0000 7f89d659c640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89b806b4d0 0x7f89b806d990 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f89d000aac0 tx=0x7f89d0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:12.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.236+0000 7f89c6ffd640 1 -- 172.21.15.64:0/1431103186 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f89cc07ab40 con 0x7f89d80d18c0 2024-03-04T07:31:12.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.419+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f89d815d570 con 0x7f89d80d18c0 2024-03-04T07:31:12.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.419+0000 7f89c6ffd640 1 -- 172.21.15.64:0/1431103186 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v86) v1 ==== 84+0+65 (secure 0 0 0) 0x7f89cc07ebe0 con 0x7f89d80d18c0 2024-03-04T07:31:12.420 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:12.420 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:31:12.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:31:12.421 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:31:12.421 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:31:12.421 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:12.421 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:31:12.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89b806b4d0 msgr2=0x7f89b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89b806b4d0 0x7f89b806d990 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f89d000aac0 tx=0x7f89d0008040 comp rx=0 tx=0).stop 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 msgr2=0x7f89d8158c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8158c40 secure :-1 s=READY pgs=1101 cs=0 l=1 rev1=1 crypto rx=0x7f89cc000c00 tx=0x7f89cc004730 comp rx=0 tx=0).stop 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 shutdown_connections 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89b806b4d0 0x7f89b806d990 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89d815df70 0x7f89d815fdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89d80d18c0 0x7f89d8158c40 unknown :-1 s=CLOSED pgs=1101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 --2- 172.21.15.64:0/1431103186 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89d8071a80 0x7f89d8157020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 >> 172.21.15.64:0/1431103186 conn(0x7f89d80d3cf0 msgr2=0x7f89d8160e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 shutdown_connections 2024-03-04T07:31:12.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.421+0000 7f89dcd02640 1 -- 172.21.15.64:0/1431103186 wait complete. 2024-03-04T07:31:12.433 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:31:12.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:31:12.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.538+0000 7f2a5f6af640 1 Processor -- start 2024-03-04T07:31:12.539 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.538+0000 7f2a5f6af640 1 -- start start 2024-03-04T07:31:12.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5f6af640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a5815f5c0 0x7f2a580cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5f6af640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2a580cff40 0x7f2a580d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5f6af640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2a580b1470 0x7f2a580b1880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5f6af640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2a580b0af0 con 0x7f2a580cff40 2024-03-04T07:31:12.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5f6af640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2a580b33b0 con 0x7f2a580b1470 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5f6af640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a580b0d40 con 0x7f2a5815f5c0 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2a580cff40 0x7f2a580d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2a580cff40 0x7f2a580d03a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54926/0 (socket says 172.21.15.64:54926) 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 -- 172.21.15.64:0/391031266 learned_addr learned my addr 172.21.15.64:0/391031266 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 -- 172.21.15.64:0/391031266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a5815f5c0 msgr2=0x7f2a580cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 --2- 172.21.15.64:0/391031266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a5815f5c0 0x7f2a580cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 -- 172.21.15.64:0/391031266 >> v2:172.21.15.66:3300/0 conn(0x7f2a580b1470 msgr2=0x7f2a580b1880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 --2- 172.21.15.64:0/391031266 >> v2:172.21.15.66:3300/0 conn(0x7f2a580b1470 0x7f2a580b1880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.539+0000 7f2a5cc4a640 1 -- 172.21.15.64:0/391031266 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a580b1020 con 0x7f2a580cff40 2024-03-04T07:31:12.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.540+0000 7f2a5cc4a640 1 --2- 172.21.15.64:0/391031266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a580cff40 0x7f2a580d03a0 secure :-1 s=READY pgs=1102 cs=0 l=1 rev1=1 crypto rx=0x7f2a580b4770 tx=0x7f2a4c02b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8d06fe77583f113f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.540+0000 7f2a57fff640 1 -- 172.21.15.64:0/391031266 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a4c02e070 con 0x7f2a580cff40 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.540+0000 7f2a57fff640 1 -- 172.21.15.64:0/391031266 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a4c037070 con 0x7f2a580cff40 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.540+0000 7f2a57fff640 1 -- 172.21.15.64:0/391031266 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a4c031a00 con 0x7f2a580cff40 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 -- 172.21.15.64:0/391031266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a580cff40 msgr2=0x7f2a580d03a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/391031266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a580cff40 0x7f2a580d03a0 secure :-1 s=READY pgs=1102 cs=0 l=1 rev1=1 crypto rx=0x7f2a580b4770 tx=0x7f2a4c02b880 comp rx=0 tx=0).stop 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 -- 172.21.15.64:0/391031266 shutdown_connections 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/391031266 >> v2:172.21.15.66:3300/0 conn(0x7f2a580b1470 0x7f2a580b1880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/391031266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a580cff40 0x7f2a580d03a0 unknown :-1 s=CLOSED pgs=1102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/391031266 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a5815f5c0 0x7f2a580cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 -- 172.21.15.64:0/391031266 >> 172.21.15.64:0/391031266 conn(0x7f2a580b36d0 msgr2=0x7f2a580b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 -- 172.21.15.64:0/391031266 shutdown_connections 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.541+0000 7f2a5f6af640 1 -- 172.21.15.64:0/391031266 wait complete. 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.542+0000 7f2a5f6af640 1 Processor -- start 2024-03-04T07:31:12.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.542+0000 7f2a5f6af640 1 -- start start 2024-03-04T07:31:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5f6af640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a580b1470 0x7f2a58154bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5f6af640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a580cff40 0x7f2a580ae9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5f6af640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a5815f5c0 0x7f2a58156520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5f6af640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a5815f9a0 con 0x7f2a5815f5c0 2024-03-04T07:31:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5f6af640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a580ced80 con 0x7f2a580cff40 2024-03-04T07:31:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5f6af640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a580cf2a0 con 0x7f2a580b1470 2024-03-04T07:31:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a5815f5c0 0x7f2a58156520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a5815f5c0 0x7f2a58156520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54940/0 (socket says 172.21.15.64:54940) 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 -- 172.21.15.64:0/1868013937 learned_addr learned my addr 172.21.15.64:0/1868013937 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5d44b640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a580b1470 0x7f2a58154bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5cc4a640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a580cff40 0x7f2a580ae9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 -- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a580b1470 msgr2=0x7f2a58154bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a580b1470 0x7f2a58154bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 -- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a580cff40 msgr2=0x7f2a580ae9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a580cff40 0x7f2a580ae9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 -- 172.21.15.64:0/1868013937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a580af0b0 con 0x7f2a5815f5c0 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5d44b640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a580b1470 0x7f2a58154bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5cc4a640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a580cff40 0x7f2a580ae9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.543+0000 7f2a5dc4c640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a5815f5c0 0x7f2a58156520 secure :-1 s=READY pgs=1103 cs=0 l=1 rev1=1 crypto rx=0x7f2a4800b970 tx=0x7f2a4800be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.544+0000 7f2a55ffb640 1 -- 172.21.15.64:0/1868013937 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a48019070 con 0x7f2a5815f5c0 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.544+0000 7f2a55ffb640 1 -- 172.21.15.64:0/1868013937 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a4800cb60 con 0x7f2a5815f5c0 2024-03-04T07:31:12.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.544+0000 7f2a55ffb640 1 -- 172.21.15.64:0/1868013937 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a48014440 con 0x7f2a5815f5c0 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.544+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a580af390 con 0x7f2a5815f5c0 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.544+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a580af950 con 0x7f2a5815f5c0 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.545+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2a5809fd30 con 0x7f2a5815f5c0 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.545+0000 7f2a55ffb640 1 -- 172.21.15.64:0/1868013937 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2a4801c070 con 0x7f2a5815f5c0 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.545+0000 7f2a55ffb640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4406b520 0x7f2a4406d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.546+0000 7f2a55ffb640 1 -- 172.21.15.64:0/1868013937 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f2a48089b90 con 0x7f2a5815f5c0 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.546+0000 7f2a5d44b640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4406b520 0x7f2a4406d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:12.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.546+0000 7f2a5d44b640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4406b520 0x7f2a4406d9e0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f2a50009930 tx=0x7f2a50008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:12.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.550+0000 7f2a55ffb640 1 -- 172.21.15.64:0/1868013937 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2a48058e40 con 0x7f2a5815f5c0 2024-03-04T07:31:12.730 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:12.729+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f2a5804f410 con 0x7f2a5815f5c0 2024-03-04T07:31:13.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.104+0000 7f2a55ffb640 1 -- 172.21.15.64:0/1868013937 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v87) v1 ==== 138+0+0 (secure 0 0 0) 0x7f2a48058740 con 0x7f2a5815f5c0 2024-03-04T07:31:13.105 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:31:13.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4406b520 msgr2=0x7f2a4406d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4406b520 0x7f2a4406d9e0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f2a50009930 tx=0x7f2a50008040 comp rx=0 tx=0).stop 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a5815f5c0 msgr2=0x7f2a58156520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a5815f5c0 0x7f2a58156520 secure :-1 s=READY pgs=1103 cs=0 l=1 rev1=1 crypto rx=0x7f2a4800b970 tx=0x7f2a4800be30 comp rx=0 tx=0).stop 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 shutdown_connections 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4406b520 0x7f2a4406d9e0 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a5815f5c0 0x7f2a58156520 unknown :-1 s=CLOSED pgs=1103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a580cff40 0x7f2a580ae9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 --2- 172.21.15.64:0/1868013937 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a580b1470 0x7f2a58154bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 >> 172.21.15.64:0/1868013937 conn(0x7f2a580b36d0 msgr2=0x7f2a580cd750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 shutdown_connections 2024-03-04T07:31:13.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.107+0000 7f2a5f6af640 1 -- 172.21.15.64:0/1868013937 wait complete. 2024-03-04T07:31:13.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:31:13.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.231+0000 7fb3dbaaf640 1 Processor -- start 2024-03-04T07:31:13.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.232+0000 7fb3dbaaf640 1 -- start start 2024-03-04T07:31:13.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.232+0000 7fb3dbaaf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb3d40ceea0 0x7fb3d40d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.232+0000 7fb3dbaaf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb3d40b6d10 0x7fb3d40b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.232+0000 7fb3dbaaf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb3d40d3740 con 0x7fb3d415dd90 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.232+0000 7fb3dbaaf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb3d40d3400 con 0x7fb3d40b6d10 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.232+0000 7fb3dbaaf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb3d40d3120 con 0x7fb3d40ceea0 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.232+0000 7fb3da04c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb3d40b6d10 0x7fb3d40b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.233+0000 7fb3da04c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb3d40b6d10 0x7fb3d40b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54952/0 (socket says 172.21.15.64:54952) 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.233+0000 7fb3da04c640 1 -- 172.21.15.64:0/749816154 learned_addr learned my addr 172.21.15.64:0/749816154 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.233+0000 7fb3da04c640 1 -- 172.21.15.64:0/749816154 >> v2:172.21.15.66:3300/0 conn(0x7fb3d40ceea0 msgr2=0x7fb3d40d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.233+0000 7fb3da04c640 1 --2- 172.21.15.64:0/749816154 >> v2:172.21.15.66:3300/0 conn(0x7fb3d40ceea0 0x7fb3d40d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.233+0000 7fb3da04c640 1 -- 172.21.15.64:0/749816154 >> v1:172.21.15.66:6789/0 conn(0x7fb3d415dd90 legacy=0x7fb3d40d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.233+0000 7fb3da04c640 1 -- 172.21.15.64:0/749816154 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3d40d39e0 con 0x7fb3d40b6d10 2024-03-04T07:31:13.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.233+0000 7fb3da04c640 1 --2- 172.21.15.64:0/749816154 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d40b71a0 secure :-1 s=READY pgs=1104 cs=0 l=1 rev1=1 crypto rx=0x7fb3d40724d0 tx=0x7fb3cc02bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ec7d8ac29a755c7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3d8849640 1 -- 172.21.15.64:0/749816154 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3cc02cb20 con 0x7fb3d40b6d10 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3d8849640 1 -- 172.21.15.64:0/749816154 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb3cc02ccc0 con 0x7fb3d40b6d10 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3d8849640 1 -- 172.21.15.64:0/749816154 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3cc034610 con 0x7fb3d40b6d10 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/749816154 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 msgr2=0x7fb3d40b71a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/749816154 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d40b71a0 secure :-1 s=READY pgs=1104 cs=0 l=1 rev1=1 crypto rx=0x7fb3d40724d0 tx=0x7fb3cc02bf80 comp rx=0 tx=0).stop 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/749816154 shutdown_connections 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/749816154 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d40b71a0 unknown :-1 s=CLOSED pgs=1104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/749816154 >> v2:172.21.15.66:3300/0 conn(0x7fb3d40ceea0 0x7fb3d40d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/749816154 >> 172.21.15.64:0/749816154 conn(0x7fb3d4071960 msgr2=0x7fb3d4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/749816154 shutdown_connections 2024-03-04T07:31:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.234+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/749816154 wait complete. 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.235+0000 7fb3dbaaf640 1 Processor -- start 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.235+0000 7fb3dbaaf640 1 -- start start 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3dbaaf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d414a230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3dbaaf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3d40ceea0 0x7fb3d415e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3dbaaf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb3d415dd90 0x7fb3d40c8800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3dbaaf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3d40d48f0 con 0x7fb3d40b6d10 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3dbaaf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3d415e170 con 0x7fb3d40ceea0 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3dbaaf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb3d40d4a70 con 0x7fb3d415dd90 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3d984b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d414a230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3d984b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d414a230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54968/0 (socket says 172.21.15.64:54968) 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3d984b640 1 -- 172.21.15.64:0/3977534947 learned_addr learned my addr 172.21.15.64:0/3977534947 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3d904a640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3d40ceea0 0x7fb3d415e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3da04c640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb3d415dd90 0x7fb3d40c8800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3d984b640 1 -- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb3d415dd90 msgr2=0x7fb3d40c8800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3d984b640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb3d415dd90 0x7fb3d40c8800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.236+0000 7fb3d984b640 1 -- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3d40ceea0 msgr2=0x7fb3d415e6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3d984b640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3d40ceea0 0x7fb3d415e6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3d984b640 1 -- 172.21.15.64:0/3977534947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3d40cafb0 con 0x7fb3d40b6d10 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3d984b640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d414a230 secure :-1 s=READY pgs=1105 cs=0 l=1 rev1=1 crypto rx=0x7fb3d000b3e0 tx=0x7fb3d000b8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3ca7fc640 1 -- 172.21.15.64:0/3977534947 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3d0015020 con 0x7fb3d40b6d10 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3ca7fc640 1 -- 172.21.15.64:0/3977534947 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb3d0011850 con 0x7fb3d40b6d10 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb3d40cb290 con 0x7fb3d40b6d10 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3ca7fc640 1 -- 172.21.15.64:0/3977534947 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3d0011c40 con 0x7fb3d40b6d10 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.237+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb3d40cb850 con 0x7fb3d40b6d10 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.238+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb3d4072a30 con 0x7fb3d40b6d10 2024-03-04T07:31:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.238+0000 7fb3ca7fc640 1 -- 172.21.15.64:0/3977534947 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb3d0018020 con 0x7fb3d40b6d10 2024-03-04T07:31:13.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.239+0000 7fb3ca7fc640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb3b806b4d0 0x7fb3b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:13.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.239+0000 7fb3ca7fc640 1 -- 172.21.15.64:0/3977534947 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 5227+0+0 (secure 0 0 0) 0x7fb3d0089f30 con 0x7fb3d40b6d10 2024-03-04T07:31:13.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.239+0000 7fb3d904a640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb3b806b4d0 0x7fb3b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:13.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.239+0000 7fb3d904a640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb3b806b4d0 0x7fb3b806d990 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fb3d40cdde0 tx=0x7fb3c4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:13.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.243+0000 7fb3ca7fc640 1 -- 172.21.15.64:0/3977534947 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb3d001f020 con 0x7fb3d40b6d10 2024-03-04T07:31:13.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:13.423+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fb3d40c7570 con 0x7fb3d40b6d10 2024-03-04T07:31:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.110+0000 7fb3ca7fc640 1 -- 172.21.15.64:0/3977534947 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v88) v1 ==== 170+0+0 (secure 0 0 0) 0x7fb3d00172c0 con 0x7fb3d40b6d10 2024-03-04T07:31:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:31:14.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb3b806b4d0 msgr2=0x7fb3b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:14.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb3b806b4d0 0x7fb3b806d990 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fb3d40cdde0 tx=0x7fb3c4008040 comp rx=0 tx=0).stop 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 msgr2=0x7fb3d414a230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d414a230 secure :-1 s=READY pgs=1105 cs=0 l=1 rev1=1 crypto rx=0x7fb3d000b3e0 tx=0x7fb3d000b8a0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 shutdown_connections 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb3b806b4d0 0x7fb3b806d990 secure :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fb3d40cdde0 tx=0x7fb3c4008040 comp rx=0 tx=0).stop 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb3d415dd90 0x7fb3d40c8800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb3d40ceea0 0x7fb3d415e6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 --2- 172.21.15.64:0/3977534947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb3d40b6d10 0x7fb3d414a230 unknown :-1 s=CLOSED pgs=1105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.113+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 >> 172.21.15.64:0/3977534947 conn(0x7fb3d4071960 msgr2=0x7fb3d40b7320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.114+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 shutdown_connections 2024-03-04T07:31:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.114+0000 7fb3dbaaf640 1 -- 172.21.15.64:0/3977534947 wait complete. 2024-03-04T07:31:14.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:31:14.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.243+0000 7f0de32aa640 1 Processor -- start 2024-03-04T07:31:14.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.243+0000 7f0de32aa640 1 -- start start 2024-03-04T07:31:14.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de32aa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc141170 0x7f0ddc15f250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de32aa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0ddc162360 0x7f0ddc0d1d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de32aa640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0ddc141550 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de32aa640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0ddc0d1320 con 0x7f0ddc162360 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de32aa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc15edc0 con 0x7f0ddc141170 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0ddc162360 0x7f0ddc0d1d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0ddc162360 0x7f0ddc0d1d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38622/0 (socket says 172.21.15.64:38622) 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 -- 172.21.15.64:0/2023848615 learned_addr learned my addr 172.21.15.64:0/2023848615 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1046640 1 --2- 172.21.15.64:0/2023848615 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc141170 0x7f0ddc15f250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 -- 172.21.15.64:0/2023848615 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc141170 msgr2=0x7f0ddc15f250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 --2- 172.21.15.64:0/2023848615 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc141170 0x7f0ddc15f250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 -- 172.21.15.64:0/2023848615 >> v1:172.21.15.64:6789/0 conn(0x7f0ddc0d18f0 legacy=0x7f0ddc161c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:14.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 -- 172.21.15.64:0/2023848615 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ddc162860 con 0x7f0ddc162360 2024-03-04T07:31:14.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0de1847640 1 --2- 172.21.15.64:0/2023848615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc162360 0x7f0ddc0d1d70 secure :-1 s=READY pgs=1107 cs=0 l=1 rev1=1 crypto rx=0x7f0dd4009e50 tx=0x7f0dd402bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=27c983031e4b13fa server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:14.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0dd3fff640 1 -- 172.21.15.64:0/2023848615 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dd402cb00 con 0x7f0ddc162360 2024-03-04T07:31:14.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0dd3fff640 1 -- 172.21.15.64:0/2023848615 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0dd402cca0 con 0x7f0ddc162360 2024-03-04T07:31:14.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.245+0000 7f0dd3fff640 1 -- 172.21.15.64:0/2023848615 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dd4034610 con 0x7f0ddc162360 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 -- 172.21.15.64:0/2023848615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc162360 msgr2=0x7f0ddc0d1d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2023848615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc162360 0x7f0ddc0d1d70 secure :-1 s=READY pgs=1107 cs=0 l=1 rev1=1 crypto rx=0x7f0dd4009e50 tx=0x7f0dd402bf60 comp rx=0 tx=0).stop 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 -- 172.21.15.64:0/2023848615 shutdown_connections 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2023848615 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc162360 0x7f0ddc0d1d70 unknown :-1 s=CLOSED pgs=1107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2023848615 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc141170 0x7f0ddc15f250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 -- 172.21.15.64:0/2023848615 >> 172.21.15.64:0/2023848615 conn(0x7f0ddc06eb00 msgr2=0x7f0ddc06ef10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 -- 172.21.15.64:0/2023848615 shutdown_connections 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 -- 172.21.15.64:0/2023848615 wait complete. 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 Processor -- start 2024-03-04T07:31:14.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.247+0000 7f0de32aa640 1 -- start start 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de32aa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0d18f0 0x7f0ddc0cb540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de32aa640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc141170 0x7f0ddc1573e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de32aa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc162360 0x7f0ddc157920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de32aa640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc15ea50 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de32aa640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc15e770 con 0x7f0ddc141170 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de32aa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ddc0702d0 con 0x7f0ddc162360 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0d18f0 0x7f0ddc0cb540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0d18f0 0x7f0ddc0cb540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38628/0 (socket says 172.21.15.64:38628) 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 -- 172.21.15.64:0/2170244526 learned_addr learned my addr 172.21.15.64:0/2170244526 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 -- 172.21.15.64:0/2170244526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc162360 msgr2=0x7f0ddc157920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:14.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc162360 0x7f0ddc157920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 -- 172.21.15.64:0/2170244526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc141170 msgr2=0x7f0ddc1573e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc141170 0x7f0ddc1573e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 -- 172.21.15.64:0/2170244526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ddc0cdea0 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de1046640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0d18f0 0x7f0ddc0cb540 secure :-1 s=READY pgs=1108 cs=0 l=1 rev1=1 crypto rx=0x7f0dd800d800 tx=0x7f0dd800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0dd1ffb640 1 -- 172.21.15.64:0/2170244526 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dd800b6c0 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0dd1ffb640 1 -- 172.21.15.64:0/2170244526 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0dd8019070 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0ddc15b820 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.248+0000 7f0dd1ffb640 1 -- 172.21.15.64:0/2170244526 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0dd800ba80 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.249+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0ddc142aa0 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.249+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0ddc1425f0 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.250+0000 7f0dd1ffb640 1 -- 172.21.15.64:0/2170244526 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0dd800bc20 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.251+0000 7f0dd1ffb640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dc006b4d0 0x7f0dc006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:14.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.251+0000 7f0dd1ffb640 1 -- 172.21.15.64:0/2170244526 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f0dd808b0a0 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.251+0000 7f0de0845640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dc006b4d0 0x7f0dc006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:14.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.251+0000 7f0de0845640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dc006b4d0 0x7f0dc006d990 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7f0dcc000f60 tx=0x7f0dcc0079d0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:14.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.256+0000 7f0dd1ffb640 1 -- 172.21.15.64:0/2170244526 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0dd805a060 con 0x7f0ddc0d18f0 2024-03-04T07:31:14.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:14.474+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f0ddc0700c0 con 0x7f0ddc0d18f0 2024-03-04T07:31:15.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.115+0000 7f0dd1ffb640 1 -- 172.21.15.64:0/2170244526 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 14 and data pool 15 v48) v1 ==== 174+0+0 (secure 0 0 0) 0x7f0dd8059960 con 0x7f0ddc0d18f0 2024-03-04T07:31:15.118 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 14 and data pool 15 2024-03-04T07:31:15.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.119+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dc006b4d0 msgr2=0x7f0dc006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.119+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dc006b4d0 0x7f0dc006d990 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7f0dcc000f60 tx=0x7f0dcc0079d0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.119+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0d18f0 msgr2=0x7f0ddc0cb540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.119+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0d18f0 0x7f0ddc0cb540 secure :-1 s=READY pgs=1108 cs=0 l=1 rev1=1 crypto rx=0x7f0dd800d800 tx=0x7f0dd800dcc0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.121+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 shutdown_connections 2024-03-04T07:31:15.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.121+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dc006b4d0 0x7f0dc006d990 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.121+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ddc162360 0x7f0ddc157920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.121+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ddc141170 0x7f0ddc1573e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.121+0000 7f0de32aa640 1 --2- 172.21.15.64:0/2170244526 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ddc0d18f0 0x7f0ddc0cb540 unknown :-1 s=CLOSED pgs=1108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.121+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 >> 172.21.15.64:0/2170244526 conn(0x7f0ddc06eb00 msgr2=0x7f0ddc15e020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:15.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.126+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 shutdown_connections 2024-03-04T07:31:15.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.126+0000 7f0de32aa640 1 -- 172.21.15.64:0/2170244526 wait complete. 2024-03-04T07:31:15.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.248+0000 7fba6f7e2640 1 Processor -- start 2024-03-04T07:31:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.248+0000 7fba6f7e2640 1 -- start start 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6f7e2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba68158520 0x7fba6806d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6f7e2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fba680aee10 0x7fba680b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6f7e2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fba68071240 con 0x7fba6806dd60 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6f7e2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fba68071b70 con 0x7fba68158520 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6f7e2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fba68071520 con 0x7fba680aee10 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6d57e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba68158520 0x7fba6806d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6d57e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fba68158520 0x7fba6806d820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38640/0 (socket says 172.21.15.64:38640) 2024-03-04T07:31:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6d57e640 1 -- 172.21.15.64:0/3083637836 learned_addr learned my addr 172.21.15.64:0/3083637836 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6dd7f640 1 --2- 172.21.15.64:0/3083637836 >> v2:172.21.15.66:3300/0 conn(0x7fba680aee10 0x7fba680b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6d57e640 1 -- 172.21.15.64:0/3083637836 >> v2:172.21.15.66:3300/0 conn(0x7fba680aee10 msgr2=0x7fba680b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6d57e640 1 --2- 172.21.15.64:0/3083637836 >> v2:172.21.15.66:3300/0 conn(0x7fba680aee10 0x7fba680b3640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6d57e640 1 -- 172.21.15.64:0/3083637836 >> v1:172.21.15.66:6789/0 conn(0x7fba6806dd60 legacy=0x7fba680ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.249+0000 7fba6d57e640 1 -- 172.21.15.64:0/3083637836 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba68071800 con 0x7fba68158520 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.250+0000 7fba6d57e640 1 --2- 172.21.15.64:0/3083637836 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba68158520 0x7fba6806d820 secure :-1 s=READY pgs=1109 cs=0 l=1 rev1=1 crypto rx=0x7fba680b2d50 tx=0x7fba6402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a45e86cde27604c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.250+0000 7fba5ffff640 1 -- 172.21.15.64:0/3083637836 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba64035020 con 0x7fba68158520 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.250+0000 7fba5ffff640 1 -- 172.21.15.64:0/3083637836 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba64031870 con 0x7fba68158520 2024-03-04T07:31:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.250+0000 7fba5ffff640 1 -- 172.21.15.64:0/3083637836 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba64031c40 con 0x7fba68158520 2024-03-04T07:31:15.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 -- 172.21.15.64:0/3083637836 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba68158520 msgr2=0x7fba6806d820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/3083637836 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba68158520 0x7fba6806d820 secure :-1 s=READY pgs=1109 cs=0 l=1 rev1=1 crypto rx=0x7fba680b2d50 tx=0x7fba6402b910 comp rx=0 tx=0).stop 2024-03-04T07:31:15.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 -- 172.21.15.64:0/3083637836 shutdown_connections 2024-03-04T07:31:15.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/3083637836 >> v2:172.21.15.66:3300/0 conn(0x7fba680aee10 0x7fba680b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/3083637836 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba68158520 0x7fba6806d820 unknown :-1 s=CLOSED pgs=1109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 -- 172.21.15.64:0/3083637836 >> 172.21.15.64:0/3083637836 conn(0x7fba68066d00 msgr2=0x7fba680670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 -- 172.21.15.64:0/3083637836 shutdown_connections 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.251+0000 7fba6f7e2640 1 -- 172.21.15.64:0/3083637836 wait complete. 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.252+0000 7fba6f7e2640 1 Processor -- start 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.252+0000 7fba6f7e2640 1 -- start start 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6f7e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba6806dd60 0x7fba68155de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6f7e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba680aee10 0x7fba68140420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6f7e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba68158520 0x7fba6813f9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6f7e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba68158a80 con 0x7fba680aee10 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6f7e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba68158900 con 0x7fba6806dd60 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6f7e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fba680b3b80 con 0x7fba68158520 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba680aee10 0x7fba68140420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba680aee10 0x7fba68140420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38644/0 (socket says 172.21.15.64:38644) 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 -- 172.21.15.64:0/2870668237 learned_addr learned my addr 172.21.15.64:0/2870668237 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6dd7f640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba68158520 0x7fba6813f9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6d57e640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba6806dd60 0x7fba68155de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 -- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba68158520 msgr2=0x7fba6813f9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba68158520 0x7fba6813f9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 -- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba6806dd60 msgr2=0x7fba68155de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba6806dd60 0x7fba68155de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 -- 172.21.15.64:0/2870668237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba68151500 con 0x7fba680aee10 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6cd7d640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba680aee10 0x7fba68140420 secure :-1 s=READY pgs=1110 cs=0 l=1 rev1=1 crypto rx=0x7fba5800c430 tx=0x7fba5800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba5dffb640 1 -- 172.21.15.64:0/2870668237 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba58011790 con 0x7fba680aee10 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba5dffb640 1 -- 172.21.15.64:0/2870668237 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba58017070 con 0x7fba680aee10 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba5dffb640 1 -- 172.21.15.64:0/2870668237 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba58011bb0 con 0x7fba680aee10 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.253+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fba681517c0 con 0x7fba680aee10 2024-03-04T07:31:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.254+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fba68151d80 con 0x7fba680aee10 2024-03-04T07:31:15.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.255+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fba68151f80 con 0x7fba680aee10 2024-03-04T07:31:15.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.255+0000 7fba5dffb640 1 -- 172.21.15.64:0/2870668237 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fba5801c460 con 0x7fba680aee10 2024-03-04T07:31:15.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.255+0000 7fba5dffb640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba4c06b4d0 0x7fba4c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.256+0000 7fba5dffb640 1 -- 172.21.15.64:0/2870668237 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fba58089f50 con 0x7fba680aee10 2024-03-04T07:31:15.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.256+0000 7fba6d57e640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba4c06b4d0 0x7fba4c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.256+0000 7fba6d57e640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba4c06b4d0 0x7fba4c06d990 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fba64036000 tx=0x7fba6403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.260+0000 7fba5dffb640 1 -- 172.21.15.64:0/2870668237 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fba68151f80 con 0x7fba680aee10 2024-03-04T07:31:15.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.440+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fba680b31f0 con 0x7fba680aee10 2024-03-04T07:31:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.441+0000 7fba5dffb640 1 -- 172.21.15.64:0/2870668237 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) v1 ==== 74+0+16573 (secure 0 0 0) 0x7fba58058780 con 0x7fba680aee10 2024-03-04T07:31:15.441 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:15.441 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:14.475140+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:31:12.750651+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-04T07:31:13.444529+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:15.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba4c06b4d0 msgr2=0x7fba4c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba4c06b4d0 0x7fba4c06d990 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fba64036000 tx=0x7fba6403c040 comp rx=0 tx=0).stop 2024-03-04T07:31:15.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba680aee10 msgr2=0x7fba68140420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba680aee10 0x7fba68140420 secure :-1 s=READY pgs=1110 cs=0 l=1 rev1=1 crypto rx=0x7fba5800c430 tx=0x7fba5800c8f0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 shutdown_connections 2024-03-04T07:31:15.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba4c06b4d0 0x7fba4c06d990 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba68158520 0x7fba6813f9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba680aee10 0x7fba68140420 unknown :-1 s=CLOSED pgs=1110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 --2- 172.21.15.64:0/2870668237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba6806dd60 0x7fba68155de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 >> 172.21.15.64:0/2870668237 conn(0x7fba68066d00 msgr2=0x7fba681566f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:15.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 shutdown_connections 2024-03-04T07:31:15.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.443+0000 7fba6f7e2640 1 -- 172.21.15.64:0/2870668237 wait complete. 2024-03-04T07:31:15.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.556+0000 7fdf59671640 1 Processor -- start 2024-03-04T07:31:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.556+0000 7fdf59671640 1 -- start start 2024-03-04T07:31:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf59671640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fdf540aee10 0x7fdf5415cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf59671640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fdf5406d400 0x7fdf54070870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf59671640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdf540729b0 con 0x7fdf5415bac0 2024-03-04T07:31:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf59671640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdf5406cf70 con 0x7fdf540aee10 2024-03-04T07:31:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf59671640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdf5406c920 con 0x7fdf5406d400 2024-03-04T07:31:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf4bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fdf540aee10 0x7fdf5415cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf4bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fdf540aee10 0x7fdf5415cbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38650/0 (socket says 172.21.15.64:38650) 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf52ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fdf5415bac0 0x7fdf540ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45676/0 (socket says 172.21.15.64:45676) 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf4bfff640 1 -- 172.21.15.64:0/1248410556 learned_addr learned my addr 172.21.15.64:0/1248410556 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf537fe640 1 --2- 172.21.15.64:0/1248410556 >> v2:172.21.15.66:3300/0 conn(0x7fdf5406d400 0x7fdf54070870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf4bfff640 1 -- 172.21.15.64:0/1248410556 >> v2:172.21.15.66:3300/0 conn(0x7fdf5406d400 msgr2=0x7fdf54070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf4bfff640 1 --2- 172.21.15.64:0/1248410556 >> v2:172.21.15.66:3300/0 conn(0x7fdf5406d400 0x7fdf54070870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf4bfff640 1 -- 172.21.15.64:0/1248410556 >> v1:172.21.15.64:6789/0 conn(0x7fdf5415bac0 legacy=0x7fdf540ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.557+0000 7fdf4bfff640 1 -- 172.21.15.64:0/1248410556 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdf5406cc00 con 0x7fdf540aee10 2024-03-04T07:31:15.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.558+0000 7fdf4bfff640 1 --2- 172.21.15.64:0/1248410556 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf540aee10 0x7fdf5415cbe0 secure :-1 s=READY pgs=1112 cs=0 l=1 rev1=1 crypto rx=0x7fdf5406ea40 tx=0x7fdf3c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2c5c2f707eecf8d2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.558+0000 7fdf527fc640 1 -- 172.21.15.64:0/1248410556 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf3c034070 con 0x7fdf540aee10 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.558+0000 7fdf527fc640 1 -- 172.21.15.64:0/1248410556 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdf3c030860 con 0x7fdf540aee10 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.558+0000 7fdf527fc640 1 -- 172.21.15.64:0/1248410556 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf3c030c30 con 0x7fdf540aee10 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 -- 172.21.15.64:0/1248410556 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf540aee10 msgr2=0x7fdf5415cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 --2- 172.21.15.64:0/1248410556 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf540aee10 0x7fdf5415cbe0 secure :-1 s=READY pgs=1112 cs=0 l=1 rev1=1 crypto rx=0x7fdf5406ea40 tx=0x7fdf3c02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 -- 172.21.15.64:0/1248410556 shutdown_connections 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 --2- 172.21.15.64:0/1248410556 >> v2:172.21.15.66:3300/0 conn(0x7fdf5406d400 0x7fdf54070870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 --2- 172.21.15.64:0/1248410556 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf540aee10 0x7fdf5415cbe0 unknown :-1 s=CLOSED pgs=1112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 -- 172.21.15.64:0/1248410556 >> 172.21.15.64:0/1248410556 conn(0x7fdf54066d00 msgr2=0x7fdf540670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 -- 172.21.15.64:0/1248410556 shutdown_connections 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.559+0000 7fdf59671640 1 -- 172.21.15.64:0/1248410556 wait complete. 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.560+0000 7fdf59671640 1 Processor -- start 2024-03-04T07:31:15.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.560+0000 7fdf59671640 1 -- start start 2024-03-04T07:31:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf5406d400 0x7fdf54147450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdf540aee10 0x7fdf541453c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdf5415bac0 0x7fdf54144a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf5415c020 con 0x7fdf5406d400 2024-03-04T07:31:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf5415bea0 con 0x7fdf5415bac0 2024-03-04T07:31:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdf54072580 con 0x7fdf540aee10 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf5406d400 0x7fdf54147450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf5406d400 0x7fdf54147450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38656/0 (socket says 172.21.15.64:38656) 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 -- 172.21.15.64:0/2429329302 learned_addr learned my addr 172.21.15.64:0/2429329302 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf537fe640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdf5415bac0 0x7fdf54144a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf4bfff640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdf540aee10 0x7fdf541453c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 -- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdf540aee10 msgr2=0x7fdf541453c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdf540aee10 0x7fdf541453c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 -- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdf5415bac0 msgr2=0x7fdf54144a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdf5415bac0 0x7fdf54144a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 -- 172.21.15.64:0/2429329302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdf54147990 con 0x7fdf5406d400 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf537fe640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdf5415bac0 0x7fdf54144a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf52ffd640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf5406d400 0x7fdf54147450 secure :-1 s=READY pgs=1113 cs=0 l=1 rev1=1 crypto rx=0x7fdf4400c3c0 tx=0x7fdf4400c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf4b7fe640 1 -- 172.21.15.64:0/2429329302 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf440074e0 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf4b7fe640 1 -- 172.21.15.64:0/2429329302 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdf44007680 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdf54145c30 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf4b7fe640 1 -- 172.21.15.64:0/2429329302 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf44004a00 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.561+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdf54147ef0 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.562+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdf54067ff0 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.563+0000 7fdf4b7fe640 1 -- 172.21.15.64:0/2429329302 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdf4401a020 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.563+0000 7fdf4b7fe640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdf3406b4d0 0x7fdf3406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.563+0000 7fdf4b7fe640 1 -- 172.21.15.64:0/2429329302 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fdf44089970 con 0x7fdf5406d400 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.563+0000 7fdf4bfff640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdf3406b4d0 0x7fdf3406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.564+0000 7fdf4bfff640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdf3406b4d0 0x7fdf3406d990 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fdf3c036000 tx=0x7fdf3c03b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.567+0000 7fdf4b7fe640 1 -- 172.21.15.64:0/2429329302 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdf54147990 con 0x7fdf5406d400 2024-03-04T07:31:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.750+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fdf5415e6a0 con 0x7fdf5406d400 2024-03-04T07:31:15.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.752+0000 7fdf4b7fe640 1 -- 172.21.15.64:0/2429329302 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) v1 ==== 74+0+16573 (secure 0 0 0) 0x7fdf440587f0 con 0x7fdf5406d400 2024-03-04T07:31:15.752 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:15.752 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:14.475140+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:31:12.750651+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-04T07:31:13.444529+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdf3406b4d0 msgr2=0x7fdf3406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdf3406b4d0 0x7fdf3406d990 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fdf3c036000 tx=0x7fdf3c03b040 comp rx=0 tx=0).stop 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf5406d400 msgr2=0x7fdf54147450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf5406d400 0x7fdf54147450 secure :-1 s=READY pgs=1113 cs=0 l=1 rev1=1 crypto rx=0x7fdf4400c3c0 tx=0x7fdf4400c880 comp rx=0 tx=0).stop 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 shutdown_connections 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdf3406b4d0 0x7fdf3406d990 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdf5415bac0 0x7fdf54144a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdf540aee10 0x7fdf541453c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 --2- 172.21.15.64:0/2429329302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdf5406d400 0x7fdf54147450 unknown :-1 s=CLOSED pgs=1113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 >> 172.21.15.64:0/2429329302 conn(0x7fdf54066d00 msgr2=0x7fdf540cf410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 shutdown_connections 2024-03-04T07:31:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.754+0000 7fdf59671640 1 -- 172.21.15.64:0/2429329302 wait complete. 2024-03-04T07:31:15.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:31:15.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.870+0000 7fc040e47640 1 Processor -- start 2024-03-04T07:31:15.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc040e47640 1 -- start start 2024-03-04T07:31:15.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc040e47640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc03c15e2f0 0x7fc03c0b41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc040e47640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc03c06b4e0 0x7fc03c06b940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc040e47640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc03c15e6d0 con 0x7fc03c06be80 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc040e47640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc03c0ccd00 con 0x7fc03c06b4e0 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc040e47640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc03c0ccfe0 con 0x7fc03c15e2f0 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc033fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc03c06b4e0 0x7fc03c06b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc033fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc03c06b4e0 0x7fc03c06b940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38666/0 (socket says 172.21.15.64:38666) 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc03ad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc03c06be80 0x7fc03c0cfa00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45686/0 (socket says 172.21.15.64:45686) 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.871+0000 7fc033fff640 1 -- 172.21.15.64:0/1628977759 learned_addr learned my addr 172.21.15.64:0/1628977759 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.872+0000 7fc03a59c640 1 --2- 172.21.15.64:0/1628977759 >> v2:172.21.15.66:3300/0 conn(0x7fc03c15e2f0 0x7fc03c0b41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.872+0000 7fc033fff640 1 -- 172.21.15.64:0/1628977759 >> v2:172.21.15.66:3300/0 conn(0x7fc03c15e2f0 msgr2=0x7fc03c0b41f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.872+0000 7fc033fff640 1 --2- 172.21.15.64:0/1628977759 >> v2:172.21.15.66:3300/0 conn(0x7fc03c15e2f0 0x7fc03c0b41f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.872+0000 7fc033fff640 1 -- 172.21.15.64:0/1628977759 >> v1:172.21.15.64:6789/0 conn(0x7fc03c06be80 legacy=0x7fc03c0cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.872+0000 7fc033fff640 1 -- 172.21.15.64:0/1628977759 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc03c0cd2c0 con 0x7fc03c06b4e0 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.872+0000 7fc033fff640 1 --2- 172.21.15.64:0/1628977759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c06b940 secure :-1 s=READY pgs=1115 cs=0 l=1 rev1=1 crypto rx=0x7fc03c071dc0 tx=0x7fc02402b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5981da96d265d70f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc039d9b640 1 -- 172.21.15.64:0/1628977759 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc02402e070 con 0x7fc03c06b4e0 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc039d9b640 1 -- 172.21.15.64:0/1628977759 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc024037070 con 0x7fc03c06b4e0 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc039d9b640 1 -- 172.21.15.64:0/1628977759 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc024031a00 con 0x7fc03c06b4e0 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 -- 172.21.15.64:0/1628977759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 msgr2=0x7fc03c06b940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 --2- 172.21.15.64:0/1628977759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c06b940 secure :-1 s=READY pgs=1115 cs=0 l=1 rev1=1 crypto rx=0x7fc03c071dc0 tx=0x7fc02402b880 comp rx=0 tx=0).stop 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 -- 172.21.15.64:0/1628977759 shutdown_connections 2024-03-04T07:31:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 --2- 172.21.15.64:0/1628977759 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c06b940 unknown :-1 s=CLOSED pgs=1115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 --2- 172.21.15.64:0/1628977759 >> v2:172.21.15.66:3300/0 conn(0x7fc03c15e2f0 0x7fc03c0b41f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 -- 172.21.15.64:0/1628977759 >> 172.21.15.64:0/1628977759 conn(0x7fc03c0b36d0 msgr2=0x7fc03c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:15.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 -- 172.21.15.64:0/1628977759 shutdown_connections 2024-03-04T07:31:15.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.873+0000 7fc040e47640 1 -- 172.21.15.64:0/1628977759 wait complete. 2024-03-04T07:31:15.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.874+0000 7fc040e47640 1 Processor -- start 2024-03-04T07:31:15.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc040e47640 1 -- start start 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc040e47640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c15f540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc040e47640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc03c06be80 0x7fc03c0d22f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc040e47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc03c15e2f0 0x7fc03c0c5630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc040e47640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc03c06c450 con 0x7fc03c06b4e0 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc040e47640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc03c06c2d0 con 0x7fc03c06be80 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc040e47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc03c0cead0 con 0x7fc03c15e2f0 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc03a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c15f540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc03a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c15f540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38676/0 (socket says 172.21.15.64:38676) 2024-03-04T07:31:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.875+0000 7fc03a59c640 1 -- 172.21.15.64:0/4089037059 learned_addr learned my addr 172.21.15.64:0/4089037059 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc033fff640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc03c06be80 0x7fc03c0d22f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc03ad9d640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc03c15e2f0 0x7fc03c0c5630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc03a59c640 1 -- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc03c15e2f0 msgr2=0x7fc03c0c5630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc03a59c640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc03c15e2f0 0x7fc03c0c5630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc03a59c640 1 -- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc03c06be80 msgr2=0x7fc03c0d22f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc03a59c640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc03c06be80 0x7fc03c0d22f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc03a59c640 1 -- 172.21.15.64:0/4089037059 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc03c0b47c0 con 0x7fc03c06b4e0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc033fff640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc03c06be80 0x7fc03c0d22f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc03a59c640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c15f540 secure :-1 s=READY pgs=1116 cs=0 l=1 rev1=1 crypto rx=0x7fc02c00d5f0 tx=0x7fc02c00dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc032ffd640 1 -- 172.21.15.64:0/4089037059 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc02c0125f0 con 0x7fc03c06b4e0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc032ffd640 1 -- 172.21.15.64:0/4089037059 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc02c019070 con 0x7fc03c06b4e0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc03c15fc40 con 0x7fc03c06b4e0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc032ffd640 1 -- 172.21.15.64:0/4089037059 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc02c0129d0 con 0x7fc03c06b4e0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.876+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc03c0c6bb0 con 0x7fc03c06b4e0 2024-03-04T07:31:15.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.877+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc03c09fd30 con 0x7fc03c06b4e0 2024-03-04T07:31:15.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.877+0000 7fc032ffd640 1 -- 172.21.15.64:0/4089037059 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc02c012b70 con 0x7fc03c06b4e0 2024-03-04T07:31:15.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.878+0000 7fc032ffd640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc01c06b4d0 0x7fc01c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:15.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.878+0000 7fc032ffd640 1 -- 172.21.15.64:0/4089037059 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc02c08a600 con 0x7fc03c06b4e0 2024-03-04T07:31:15.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.878+0000 7fc033fff640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc01c06b4d0 0x7fc01c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:15.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.878+0000 7fc033fff640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc01c06b4d0 0x7fc01c06d990 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fc024036000 tx=0x7fc024008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:15.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:15.882+0000 7fc032ffd640 1 -- 172.21.15.64:0/4089037059 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc02c059490 con 0x7fc03c06b4e0 2024-03-04T07:31:16.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.101+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7fc03c068320 con 0x7fc03c06b4e0 2024-03-04T07:31:16.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.119+0000 7fc032ffd640 1 -- 172.21.15.64:0/4089037059 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v49) v1 ==== 118+0+0 (secure 0 0 0) 0x7fc02c058d90 con 0x7fc03c06b4e0 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc01c06b4d0 msgr2=0x7fc01c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc01c06b4d0 0x7fc01c06d990 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fc024036000 tx=0x7fc024008030 comp rx=0 tx=0).stop 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 msgr2=0x7fc03c15f540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c15f540 secure :-1 s=READY pgs=1116 cs=0 l=1 rev1=1 crypto rx=0x7fc02c00d5f0 tx=0x7fc02c00dab0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 shutdown_connections 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc01c06b4d0 0x7fc01c06d990 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc03c15e2f0 0x7fc03c0c5630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc03c06be80 0x7fc03c0d22f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 --2- 172.21.15.64:0/4089037059 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc03c06b4e0 0x7fc03c15f540 unknown :-1 s=CLOSED pgs=1116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 >> 172.21.15.64:0/4089037059 conn(0x7fc03c0b36d0 msgr2=0x7fc03c0b3ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:16.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 shutdown_connections 2024-03-04T07:31:16.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.121+0000 7fc040e47640 1 -- 172.21.15.64:0/4089037059 wait complete. 2024-03-04T07:31:16.133 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:31:16.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:31:16.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.239+0000 7f7855fbc640 1 Processor -- start 2024-03-04T07:31:16.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.239+0000 7f7855fbc640 1 -- start start 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.239+0000 7f7855fbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500ceea0 0x7f78500d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.239+0000 7f7855fbc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f78500d3740 con 0x7f78500b6d10 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.239+0000 7f7855fbc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f78500d3120 con 0x7f785015dd90 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.239+0000 7f7855fbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f78500d3400 con 0x7f78500ceea0 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f78500b6d10 0x7f78500d0c30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45696/0 (socket says 172.21.15.64:45696) 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784ffff640 1 -- 172.21.15.64:0/1362400823 learned_addr learned my addr 172.21.15.64:0/1362400823 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784effd640 1 --2- 172.21.15.64:0/1362400823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500ceea0 0x7f78500d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784e7fc640 1 -- 172.21.15.64:0/1362400823 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2684358388 0 0) 0x7f78500d3740 con 0x7f78500b6d10 2024-03-04T07:31:16.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784e7fc640 1 -- 172.21.15.64:0/1362400823 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7838003520 con 0x7f78500b6d10 2024-03-04T07:31:16.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784e7fc640 1 -- 172.21.15.64:0/1362400823 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3095896619 0 0) 0x7f78500d3120 con 0x7f785015dd90 2024-03-04T07:31:16.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784e7fc640 1 -- 172.21.15.64:0/1362400823 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f78500d3740 con 0x7f785015dd90 2024-03-04T07:31:16.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784effd640 1 -- 172.21.15.64:0/1362400823 >> v1:172.21.15.66:6789/0 conn(0x7f785015dd90 legacy=0x7f78500d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:16.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784effd640 1 -- 172.21.15.64:0/1362400823 >> v1:172.21.15.64:6789/0 conn(0x7f78500b6d10 legacy=0x7f78500d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:16.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.240+0000 7f784effd640 1 -- 172.21.15.64:0/1362400823 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78500d39e0 con 0x7f78500ceea0 2024-03-04T07:31:16.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.241+0000 7f784effd640 1 --2- 172.21.15.64:0/1362400823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500ceea0 0x7f78500d29d0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f7844009800 tx=0x7f784402b9c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c9eb53eb49fae289 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:16.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.241+0000 7f784e7fc640 1 -- 172.21.15.64:0/1362400823 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7844034070 con 0x7f78500ceea0 2024-03-04T07:31:16.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.241+0000 7f784e7fc640 1 -- 172.21.15.64:0/1362400823 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7844030880 con 0x7f78500ceea0 2024-03-04T07:31:16.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.241+0000 7f784e7fc640 1 -- 172.21.15.64:0/1362400823 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7844030c50 con 0x7f78500ceea0 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.242+0000 7f7855fbc640 1 -- 172.21.15.64:0/1362400823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500ceea0 msgr2=0x7f78500d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.242+0000 7f7855fbc640 1 --2- 172.21.15.64:0/1362400823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500ceea0 0x7f78500d29d0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f7844009800 tx=0x7f784402b9c0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.242+0000 7f7855fbc640 1 -- 172.21.15.64:0/1362400823 shutdown_connections 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.242+0000 7f7855fbc640 1 --2- 172.21.15.64:0/1362400823 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500ceea0 0x7f78500d29d0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.242+0000 7f7855fbc640 1 -- 172.21.15.64:0/1362400823 >> 172.21.15.64:0/1362400823 conn(0x7f7850071960 msgr2=0x7f7850071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.242+0000 7f7855fbc640 1 -- 172.21.15.64:0/1362400823 shutdown_connections 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.242+0000 7f7855fbc640 1 -- 172.21.15.64:0/1362400823 wait complete. 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.243+0000 7f7855fbc640 1 Processor -- start 2024-03-04T07:31:16.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.243+0000 7f7855fbc640 1 -- start start 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7855fbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500b6d10 0x7f7850149bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7855fbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78500ceea0 0x7f78500cad70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7855fbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f785015dd90 0x7f78500cb2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7855fbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78500d48f0 con 0x7f78500ceea0 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7855fbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f785015e170 con 0x7f785015dd90 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7855fbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f78500d4a70 con 0x7f78500b6d10 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78500ceea0 0x7f78500cad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:16.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78500ceea0 0x7f78500cad70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38690/0 (socket says 172.21.15.64:38690) 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 -- 172.21.15.64:0/2814240366 learned_addr learned my addr 172.21.15.64:0/2814240366 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784ffff640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f785015dd90 0x7f78500cb2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784f7fe640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500b6d10 0x7f7850149bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 -- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500b6d10 msgr2=0x7f7850149bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500b6d10 0x7f7850149bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 -- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f785015dd90 msgr2=0x7f78500cb2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f785015dd90 0x7f78500cb2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 -- 172.21.15.64:0/2814240366 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f785014a110 con 0x7f78500ceea0 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f784effd640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78500ceea0 0x7f78500cad70 secure :-1 s=READY pgs=1118 cs=0 l=1 rev1=1 crypto rx=0x7f7844009920 tx=0x7f78440046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7837fff640 1 -- 172.21.15.64:0/2814240366 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7844034040 con 0x7f78500ceea0 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7837fff640 1 -- 172.21.15.64:0/2814240366 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7844030680 con 0x7f78500ceea0 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7850147120 con 0x7f78500ceea0 2024-03-04T07:31:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.244+0000 7f7837fff640 1 -- 172.21.15.64:0/2814240366 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7844006e10 con 0x7f78500ceea0 2024-03-04T07:31:16.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.245+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78500cde50 con 0x7f78500ceea0 2024-03-04T07:31:16.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.245+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7850072a30 con 0x7f78500ceea0 2024-03-04T07:31:16.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.246+0000 7f7837fff640 1 -- 172.21.15.64:0/2814240366 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7844037070 con 0x7f78500ceea0 2024-03-04T07:31:16.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.246+0000 7f7837fff640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f783c06b4d0 0x7f783c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.246+0000 7f7837fff640 1 -- 172.21.15.64:0/2814240366 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f78440ac2d0 con 0x7f78500ceea0 2024-03-04T07:31:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.246+0000 7f784f7fe640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f783c06b4d0 0x7f783c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.247+0000 7f784f7fe640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f783c06b4d0 0x7f783c06d990 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f7840007fd0 tx=0x7f7840009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:16.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.251+0000 7f7837fff640 1 -- 172.21.15.64:0/2814240366 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7850147120 con 0x7f78500ceea0 2024-03-04T07:31:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:16.470+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f785004f410 con 0x7f78500ceea0 2024-03-04T07:31:17.099 INFO:tasks.ceph.mds.b.smithi066.stderr:2024-03-04T07:31:17.099+0000 7f9006882640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:31:17.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.120+0000 7f7837fff640 1 -- 172.21.15.64:0/2814240366 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v50) v1 ==== 122+0+0 (secure 0 0 0) 0x7f784407ab90 con 0x7f78500ceea0 2024-03-04T07:31:17.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f783c06b4d0 msgr2=0x7f783c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f783c06b4d0 0x7f783c06d990 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f7840007fd0 tx=0x7f7840009040 comp rx=0 tx=0).stop 2024-03-04T07:31:17.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78500ceea0 msgr2=0x7f78500cad70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78500ceea0 0x7f78500cad70 secure :-1 s=READY pgs=1118 cs=0 l=1 rev1=1 crypto rx=0x7f7844009920 tx=0x7f78440046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 shutdown_connections 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f783c06b4d0 0x7f783c06d990 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f785015dd90 0x7f78500cb2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78500ceea0 0x7f78500cad70 unknown :-1 s=CLOSED pgs=1118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 --2- 172.21.15.64:0/2814240366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78500b6d10 0x7f7850149bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.122+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 >> 172.21.15.64:0/2814240366 conn(0x7f7850071960 msgr2=0x7f7850071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.123+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 shutdown_connections 2024-03-04T07:31:17.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.123+0000 7f7855fbc640 1 -- 172.21.15.64:0/2814240366 wait complete. 2024-03-04T07:31:17.135 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:31:17.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:17.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.242+0000 7f802c6c9640 1 Processor -- start 2024-03-04T07:31:17.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.242+0000 7f802c6c9640 1 -- start start 2024-03-04T07:31:17.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802c6c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80240aec20 0x7f80240ce3f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802c6c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f802406d830 con 0x7f80240ae790 2024-03-04T07:31:17.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802c6c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f802415c030 con 0x7f802415bc50 2024-03-04T07:31:17.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802c6c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80240af0b0 con 0x7f80240aec20 2024-03-04T07:31:17.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f8029c64640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f80240ae790 0x7f80240cdce0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45702/0 (socket says 172.21.15.64:45702) 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f8029c64640 1 -- 172.21.15.64:0/2902297947 learned_addr learned my addr 172.21.15.64:0/2902297947 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802ac66640 1 --2- 172.21.15.64:0/2902297947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80240aec20 0x7f80240ce3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f8029463640 1 -- 172.21.15.64:0/2902297947 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3827995364 0 0) 0x7f802406d830 con 0x7f80240ae790 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f8029463640 1 -- 172.21.15.64:0/2902297947 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8010003520 con 0x7f80240ae790 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802ac66640 1 -- 172.21.15.64:0/2902297947 >> v1:172.21.15.66:6789/0 conn(0x7f802415bc50 legacy=0x7f802406d120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f8029463640 1 -- 172.21.15.64:0/2902297947 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1247431899 0 0) 0x7f802415c030 con 0x7f802415bc50 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802ac66640 1 -- 172.21.15.64:0/2902297947 >> v1:172.21.15.64:6789/0 conn(0x7f80240ae790 legacy=0x7f80240cdce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.243+0000 7f802ac66640 1 -- 172.21.15.64:0/2902297947 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8024070c50 con 0x7f80240aec20 2024-03-04T07:31:17.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.244+0000 7f802ac66640 1 --2- 172.21.15.64:0/2902297947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80240aec20 0x7f80240ce3f0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f801c009dc0 tx=0x7f801c02bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=286f4b24a3a012d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.244+0000 7f8029463640 1 -- 172.21.15.64:0/2902297947 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f801c039070 con 0x7f80240aec20 2024-03-04T07:31:17.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.244+0000 7f8029463640 1 -- 172.21.15.64:0/2902297947 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f801c02ccd0 con 0x7f80240aec20 2024-03-04T07:31:17.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.244+0000 7f8029463640 1 -- 172.21.15.64:0/2902297947 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f801c034640 con 0x7f80240aec20 2024-03-04T07:31:17.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.245+0000 7f802c6c9640 1 -- 172.21.15.64:0/2902297947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80240aec20 msgr2=0x7f80240ce3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.245+0000 7f802c6c9640 1 --2- 172.21.15.64:0/2902297947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80240aec20 0x7f80240ce3f0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f801c009dc0 tx=0x7f801c02bf20 comp rx=0 tx=0).stop 2024-03-04T07:31:17.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.245+0000 7f802c6c9640 1 -- 172.21.15.64:0/2902297947 shutdown_connections 2024-03-04T07:31:17.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.245+0000 7f802c6c9640 1 --2- 172.21.15.64:0/2902297947 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80240aec20 0x7f80240ce3f0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.245+0000 7f802c6c9640 1 -- 172.21.15.64:0/2902297947 >> 172.21.15.64:0/2902297947 conn(0x7f8024066d00 msgr2=0x7f80240670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:17.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.245+0000 7f802c6c9640 1 -- 172.21.15.64:0/2902297947 shutdown_connections 2024-03-04T07:31:17.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.245+0000 7f802c6c9640 1 -- 172.21.15.64:0/2902297947 wait complete. 2024-03-04T07:31:17.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.246+0000 7f802c6c9640 1 Processor -- start 2024-03-04T07:31:17.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.246+0000 7f802c6c9640 1 -- start start 2024-03-04T07:31:17.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802c6c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80240ae790 0x7f80241475c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802c6c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80240aec20 0x7f80240a9a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802c6c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802415bc50 0x7f80240a9fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802c6c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80240714f0 con 0x7f80240ae790 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802c6c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8024071210 con 0x7f80240aec20 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802c6c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80240717d0 con 0x7f802415bc50 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80240ae790 0x7f80241475c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80240ae790 0x7f80241475c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38692/0 (socket says 172.21.15.64:38692) 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 -- 172.21.15.64:0/1654515804 learned_addr learned my addr 172.21.15.64:0/1654515804 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f8029c64640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80240aec20 0x7f80240a9a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802ac66640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802415bc50 0x7f80240a9fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 -- 172.21.15.64:0/1654515804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802415bc50 msgr2=0x7f80240a9fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802415bc50 0x7f80240a9fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 -- 172.21.15.64:0/1654515804 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80240aec20 msgr2=0x7f80240a9a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80240aec20 0x7f80240a9a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.247+0000 7f802a465640 1 -- 172.21.15.64:0/1654515804 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8024150130 con 0x7f80240ae790 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.248+0000 7f802a465640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80240ae790 0x7f80241475c0 secure :-1 s=READY pgs=1120 cs=0 l=1 rev1=1 crypto rx=0x7f802000c410 tx=0x7f802000c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.248+0000 7f801affd640 1 -- 172.21.15.64:0/1654515804 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8020017070 con 0x7f80240ae790 2024-03-04T07:31:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.248+0000 7f801affd640 1 -- 172.21.15.64:0/1654515804 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f80200035f0 con 0x7f80240ae790 2024-03-04T07:31:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.248+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8024150410 con 0x7f80240ae790 2024-03-04T07:31:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.248+0000 7f801affd640 1 -- 172.21.15.64:0/1654515804 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80200039e0 con 0x7f80240ae790 2024-03-04T07:31:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.248+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80241509d0 con 0x7f80240ae790 2024-03-04T07:31:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.249+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8024067ff0 con 0x7f80240ae790 2024-03-04T07:31:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.249+0000 7f801affd640 1 -- 172.21.15.64:0/1654515804 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f802001a070 con 0x7f80240ae790 2024-03-04T07:31:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.250+0000 7f801affd640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.250+0000 7f801affd640 1 -- 172.21.15.64:0/1654515804 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f8020089640 con 0x7f80240ae790 2024-03-04T07:31:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.250+0000 7f8029c64640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.250+0000 7f8029c64640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f802415b560 tx=0x7f8014009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.254+0000 7f801affd640 1 -- 172.21.15.64:0/1654515804 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8020057df0 con 0x7f80240ae790 2024-03-04T07:31:17.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.473+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f802406de50 con 0x7f80240ae790 2024-03-04T07:31:17.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.474+0000 7f801affd640 1 -- 172.21.15.64:0/1654515804 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f802005be90 con 0x7f80240ae790 2024-03-04T07:31:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:17.474 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":50,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6619,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3500669041","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3500669041},{"type":"v1","addr":"172.21.15.64:6836","nonce":3500669041}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6634,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/362862056","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":362862056},{"type":"v1","addr":"172.21.15.66:6834","nonce":362862056}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":47},{"gid":7341,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/3848675976","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":3848675976},{"type":"v1","addr":"172.21.15.64:6837","nonce":3848675976}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":47}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:16.471582+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7353},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7353":{"gid":7353,"name":"b","rank":0,"incarnation":48,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/3397672669","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3397672669},{"type":"v1","addr":"172.21.15.66:6835","nonce":3397672669}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7353,"qdb_cluster":[7353]},"id":7}]} 2024-03-04T07:31:17.474 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 50 2024-03-04T07:31:17.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 msgr2=0x7f800806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f802415b560 tx=0x7f8014009040 comp rx=0 tx=0).stop 2024-03-04T07:31:17.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80240ae790 msgr2=0x7f80241475c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80240ae790 0x7f80241475c0 secure :-1 s=READY pgs=1120 cs=0 l=1 rev1=1 crypto rx=0x7f802000c410 tx=0x7f802000c8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 shutdown_connections 2024-03-04T07:31:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802415bc50 0x7f80240a9fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80240aec20 0x7f80240a9a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 --2- 172.21.15.64:0/1654515804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80240ae790 0x7f80241475c0 unknown :-1 s=CLOSED pgs=1120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 >> 172.21.15.64:0/1654515804 conn(0x7f8024066d00 msgr2=0x7f80240ce490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 shutdown_connections 2024-03-04T07:31:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.476+0000 7f802c6c9640 1 -- 172.21.15.64:0/1654515804 wait complete. 2024-03-04T07:31:17.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:17.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.589+0000 7f46f4a76640 1 Processor -- start 2024-03-04T07:31:17.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.589+0000 7f46f4a76640 1 -- start start 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46f4a76640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f46f0158520 0x7f46f006d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46f4a76640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f46f006dd60 0x7f46f00ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46f4a76640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f46f0071240 con 0x7f46f00aec40 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46f4a76640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46f0071b70 con 0x7f46f006dd60 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46f4a76640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46f00af0b0 con 0x7f46f0158520 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46edd9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f46f006dd60 0x7f46f00ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46edd9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f46f006dd60 0x7f46f00ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38708/0 (socket says 172.21.15.64:38708) 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46edd9b640 1 -- 172.21.15.64:0/947914703 learned_addr learned my addr 172.21.15.64:0/947914703 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46ee59c640 1 --2- 172.21.15.64:0/947914703 >> v2:172.21.15.66:3300/0 conn(0x7f46f0158520 0x7f46f006d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46edd9b640 1 -- 172.21.15.64:0/947914703 >> v2:172.21.15.66:3300/0 conn(0x7f46f0158520 msgr2=0x7f46f006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:17.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46edd9b640 1 --2- 172.21.15.64:0/947914703 >> v2:172.21.15.66:3300/0 conn(0x7f46f0158520 0x7f46f006d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46edd9b640 1 -- 172.21.15.64:0/947914703 >> v1:172.21.15.64:6789/0 conn(0x7f46f00aec40 legacy=0x7f46f015e670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.590+0000 7f46edd9b640 1 -- 172.21.15.64:0/947914703 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46f0071520 con 0x7f46f006dd60 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46edd9b640 1 --2- 172.21.15.64:0/947914703 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f006dd60 0x7f46f00ae700 secure :-1 s=READY pgs=1122 cs=0 l=1 rev1=1 crypto rx=0x7f46f006e900 tx=0x7f46e402e170 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f43b9045140ecf9a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46ed59a640 1 -- 172.21.15.64:0/947914703 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46e403b070 con 0x7f46f006dd60 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46ed59a640 1 -- 172.21.15.64:0/947914703 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46e4036440 con 0x7f46f006dd60 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46ed59a640 1 -- 172.21.15.64:0/947914703 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46e4036810 con 0x7f46f006dd60 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46f4a76640 1 -- 172.21.15.64:0/947914703 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f006dd60 msgr2=0x7f46f00ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46f4a76640 1 --2- 172.21.15.64:0/947914703 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f006dd60 0x7f46f00ae700 secure :-1 s=READY pgs=1122 cs=0 l=1 rev1=1 crypto rx=0x7f46f006e900 tx=0x7f46e402e170 comp rx=0 tx=0).stop 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46f4a76640 1 -- 172.21.15.64:0/947914703 shutdown_connections 2024-03-04T07:31:17.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46f4a76640 1 --2- 172.21.15.64:0/947914703 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f006dd60 0x7f46f00ae700 unknown :-1 s=CLOSED pgs=1122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46f4a76640 1 --2- 172.21.15.64:0/947914703 >> v2:172.21.15.66:3300/0 conn(0x7f46f0158520 0x7f46f006d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46f4a76640 1 -- 172.21.15.64:0/947914703 >> 172.21.15.64:0/947914703 conn(0x7f46f0066d00 msgr2=0x7f46f00670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:17.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.591+0000 7f46f4a76640 1 -- 172.21.15.64:0/947914703 shutdown_connections 2024-03-04T07:31:17.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.592+0000 7f46f4a76640 1 -- 172.21.15.64:0/947914703 wait complete. 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 Processor -- start 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 -- start start 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f46f006dd60 0x7f46f00ccb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f00aec40 0x7f46f014a360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46f0158520 0x7f46f0157160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46f0072000 con 0x7f46f00aec40 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46f00b4650 con 0x7f46f006dd60 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.593+0000 7f46f4a76640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46f0158900 con 0x7f46f0158520 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f00aec40 0x7f46f014a360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f00aec40 0x7f46f014a360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38720/0 (socket says 172.21.15.64:38720) 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 -- 172.21.15.64:0/514632430 learned_addr learned my addr 172.21.15.64:0/514632430 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:17.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46ee59c640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f46f006dd60 0x7f46f00ccb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46eed9d640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46f0158520 0x7f46f0157160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 -- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46f0158520 msgr2=0x7f46f0157160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46f0158520 0x7f46f0157160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 -- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f46f006dd60 msgr2=0x7f46f00ccb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f46f006dd60 0x7f46f00ccb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 -- 172.21.15.64:0/514632430 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46f0155080 con 0x7f46f00aec40 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46edd9b640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f00aec40 0x7f46f014a360 secure :-1 s=READY pgs=1123 cs=0 l=1 rev1=1 crypto rx=0x7f46e402e1b0 tx=0x7f46e40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46deffd640 1 -- 172.21.15.64:0/514632430 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46e403b040 con 0x7f46f00aec40 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46deffd640 1 -- 172.21.15.64:0/514632430 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f46e4046070 con 0x7f46f00aec40 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46f00cd240 con 0x7f46f00aec40 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46deffd640 1 -- 172.21.15.64:0/514632430 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46e4034880 con 0x7f46f00aec40 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.594+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46f00cd800 con 0x7f46f00aec40 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.595+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f46f0067ff0 con 0x7f46f00aec40 2024-03-04T07:31:17.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.596+0000 7f46deffd640 1 -- 172.21.15.64:0/514632430 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f46e4034a20 con 0x7f46f00aec40 2024-03-04T07:31:17.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.596+0000 7f46deffd640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f46d006b4d0 0x7f46d006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.596+0000 7f46deffd640 1 -- 172.21.15.64:0/514632430 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f46e40aeb00 con 0x7f46f00aec40 2024-03-04T07:31:17.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.596+0000 7f46ee59c640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f46d006b4d0 0x7f46d006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.596+0000 7f46ee59c640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f46d006b4d0 0x7f46d006d990 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f46f0074b60 tx=0x7f46e0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.600+0000 7f46deffd640 1 -- 172.21.15.64:0/514632430 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f46e407d3c0 con 0x7f46f00aec40 2024-03-04T07:31:17.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.785+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f46f00b31f0 con 0x7f46f00aec40 2024-03-04T07:31:17.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.786+0000 7f46deffd640 1 -- 172.21.15.64:0/514632430 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v90) v1 ==== 74+0+16560 (secure 0 0 0) 0x7f46e4081460 con 0x7f46f00aec40 2024-03-04T07:31:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:17.786 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:16.116491+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:31:12.750651+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-04T07:31:13.444529+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"90","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:17.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f46d006b4d0 msgr2=0x7f46d006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f46d006b4d0 0x7f46d006d990 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f46f0074b60 tx=0x7f46e0009190 comp rx=0 tx=0).stop 2024-03-04T07:31:17.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f00aec40 msgr2=0x7f46f014a360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f00aec40 0x7f46f014a360 secure :-1 s=READY pgs=1123 cs=0 l=1 rev1=1 crypto rx=0x7f46e402e1b0 tx=0x7f46e40046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 shutdown_connections 2024-03-04T07:31:17.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f46d006b4d0 0x7f46d006d990 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46f0158520 0x7f46f0157160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f46f00aec40 0x7f46f014a360 unknown :-1 s=CLOSED pgs=1123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 --2- 172.21.15.64:0/514632430 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f46f006dd60 0x7f46f00ccb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 >> 172.21.15.64:0/514632430 conn(0x7f46f0066d00 msgr2=0x7f46f00670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:17.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 shutdown_connections 2024-03-04T07:31:17.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.788+0000 7f46f4a76640 1 -- 172.21.15.64:0/514632430 wait complete. 2024-03-04T07:31:17.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:31:17.797 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:31:17.799 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:31:17.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:31:17.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.941+0000 7f41c39bc640 1 Processor -- start 2024-03-04T07:31:17.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.941+0000 7f41c39bc640 1 -- start start 2024-03-04T07:31:17.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c39bc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f41bc15df00 0x7f41bc0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c39bc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f41bc0b95a0 0x7f41bc0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c39bc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f41bc0cea30 con 0x7f41bc0b9a30 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c39bc640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f41bc0d2af0 con 0x7f41bc15df00 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c39bc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f41bc0cf3c0 con 0x7f41bc0b95a0 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c1758640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f41bc15df00 0x7f41bc0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c1758640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f41bc15df00 0x7f41bc0d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38730/0 (socket says 172.21.15.64:38730) 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c1758640 1 -- 172.21.15.64:0/437940952 learned_addr learned my addr 172.21.15.64:0/437940952 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.942+0000 7f41c0f57640 1 --2- 172.21.15.64:0/437940952 >> v2:172.21.15.66:3300/0 conn(0x7f41bc0b95a0 0x7f41bc0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.943+0000 7f41c1758640 1 -- 172.21.15.64:0/437940952 >> v2:172.21.15.66:3300/0 conn(0x7f41bc0b95a0 msgr2=0x7f41bc0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.943+0000 7f41c1758640 1 --2- 172.21.15.64:0/437940952 >> v2:172.21.15.66:3300/0 conn(0x7f41bc0b95a0 0x7f41bc0b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.943+0000 7f41c1758640 1 -- 172.21.15.64:0/437940952 >> v1:172.21.15.64:6789/0 conn(0x7f41bc0b9a30 legacy=0x7f41bc0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.943+0000 7f41c1758640 1 -- 172.21.15.64:0/437940952 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f41bc0cf0e0 con 0x7f41bc15df00 2024-03-04T07:31:17.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.943+0000 7f41c1758640 1 --2- 172.21.15.64:0/437940952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0d25b0 secure :-1 s=READY pgs=1125 cs=0 l=1 rev1=1 crypto rx=0x7f41bc0d1760 tx=0x7f41b402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ff9a990eff06c3a8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41bbfff640 1 -- 172.21.15.64:0/437940952 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41b4034070 con 0x7f41bc15df00 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41bbfff640 1 -- 172.21.15.64:0/437940952 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f41b4030860 con 0x7f41bc15df00 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41bbfff640 1 -- 172.21.15.64:0/437940952 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41b4030c30 con 0x7f41bc15df00 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 -- 172.21.15.64:0/437940952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 msgr2=0x7f41bc0d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 --2- 172.21.15.64:0/437940952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0d25b0 secure :-1 s=READY pgs=1125 cs=0 l=1 rev1=1 crypto rx=0x7f41bc0d1760 tx=0x7f41b402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 -- 172.21.15.64:0/437940952 shutdown_connections 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 --2- 172.21.15.64:0/437940952 >> v2:172.21.15.66:3300/0 conn(0x7f41bc0b95a0 0x7f41bc0b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 --2- 172.21.15.64:0/437940952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0d25b0 unknown :-1 s=CLOSED pgs=1125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 -- 172.21.15.64:0/437940952 >> 172.21.15.64:0/437940952 conn(0x7f41bc071960 msgr2=0x7f41bc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 -- 172.21.15.64:0/437940952 shutdown_connections 2024-03-04T07:31:17.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.944+0000 7f41c39bc640 1 -- 172.21.15.64:0/437940952 wait complete. 2024-03-04T07:31:17.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.945+0000 7f41c39bc640 1 Processor -- start 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.945+0000 7f41c39bc640 1 -- start start 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c39bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41bc0b95a0 0x7f41bc0cd6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c39bc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0c8d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c39bc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41bc0cb760 0x7f41bc0b4a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c39bc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41bc0cec70 con 0x7f41bc15df00 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c39bc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41bc0ce750 con 0x7f41bc0cb760 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c39bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f41bc0cedf0 con 0x7f41bc0b95a0 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c0f57640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0c8d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c0f57640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0c8d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38738/0 (socket says 172.21.15.64:38738) 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c0f57640 1 -- 172.21.15.64:0/3494275735 learned_addr learned my addr 172.21.15.64:0/3494275735 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c1758640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41bc0b95a0 0x7f41bc0cd6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.946+0000 7f41c1f59640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41bc0cb760 0x7f41bc0b4a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c0f57640 1 -- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41bc0b95a0 msgr2=0x7f41bc0cd6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c0f57640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41bc0b95a0 0x7f41bc0cd6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c0f57640 1 -- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41bc0cb760 msgr2=0x7f41bc0b4a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c0f57640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41bc0cb760 0x7f41bc0b4a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c0f57640 1 -- 172.21.15.64:0/3494275735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f41bc0c9260 con 0x7f41bc15df00 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c1f59640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41bc0cb760 0x7f41bc0b4a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c0f57640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0c8d20 secure :-1 s=READY pgs=1126 cs=0 l=1 rev1=1 crypto rx=0x7f41b000e9e0 tx=0x7f41b000eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41b9ffb640 1 -- 172.21.15.64:0/3494275735 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41b0017070 con 0x7f41bc15df00 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41b9ffb640 1 -- 172.21.15.64:0/3494275735 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f41b000c9d0 con 0x7f41bc15df00 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f41bc0ccf30 con 0x7f41bc15df00 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41b9ffb640 1 -- 172.21.15.64:0/3494275735 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41b000cdc0 con 0x7f41bc15df00 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.947+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f41bc0c7f60 con 0x7f41bc15df00 2024-03-04T07:31:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.948+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f41bc067ff0 con 0x7f41bc15df00 2024-03-04T07:31:17.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.949+0000 7f41b9ffb640 1 -- 172.21.15.64:0/3494275735 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f41b001a070 con 0x7f41bc15df00 2024-03-04T07:31:17.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.949+0000 7f41b9ffb640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f41a806b4d0 0x7f41a806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:17.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.949+0000 7f41c1758640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f41a806b4d0 0x7f41a806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:17.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.949+0000 7f41b9ffb640 1 -- 172.21.15.64:0/3494275735 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f41b008a100 con 0x7f41bc15df00 2024-03-04T07:31:17.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.950+0000 7f41c1758640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f41a806b4d0 0x7f41a806d990 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f41b4036000 tx=0x7f41b403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:17.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:17.953+0000 7f41b9ffb640 1 -- 172.21.15.64:0/3494275735 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f41bc0c9260 con 0x7f41bc15df00 2024-03-04T07:31:18.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.133+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f41bc0d30e0 con 0x7f41a806b4d0 2024-03-04T07:31:18.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.139+0000 7f41b9ffb640 1 -- 172.21.15.64:0/3494275735 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252082 (secure 0 0 0) 0x7f41bc0d30e0 con 0x7f41a806b4d0 2024-03-04T07:31:18.141 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:18.142 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:31:18.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f41a806b4d0 msgr2=0x7f41a806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f41a806b4d0 0x7f41a806d990 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f41b4036000 tx=0x7f41b403b040 comp rx=0 tx=0).stop 2024-03-04T07:31:18.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 msgr2=0x7f41bc0c8d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0c8d20 secure :-1 s=READY pgs=1126 cs=0 l=1 rev1=1 crypto rx=0x7f41b000e9e0 tx=0x7f41b000eea0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 shutdown_connections 2024-03-04T07:31:18.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f41a806b4d0 0x7f41a806d990 unknown :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f41bc0cb760 0x7f41bc0b4a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f41bc15df00 0x7f41bc0c8d20 unknown :-1 s=CLOSED pgs=1126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.141+0000 7f41c39bc640 1 --2- 172.21.15.64:0/3494275735 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f41bc0b95a0 0x7f41bc0cd6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.142+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 >> 172.21.15.64:0/3494275735 conn(0x7f41bc071960 msgr2=0x7f41bc15bb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:18.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.142+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 shutdown_connections 2024-03-04T07:31:18.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.142+0000 7f41c39bc640 1 -- 172.21.15.64:0/3494275735 wait complete. 2024-03-04T07:31:18.152 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2024-03-04T07:31:17.188752+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":97,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":259284,"kb_used_data":9832,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754715436,"statfs":{"total":773094113280,"available":772828606464,"internally_reserved":0,"allocated":10067968,"data_stored":4933156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":27,"apply_latency_ms":27,"commit_latency_ns":27000000,"apply_latency_ns":27000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.081572"},"pg_stats":[{"pgid":"15.3f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177145+0000","last_change":"2024-03-04T07:31:15.133828+0000","last_active":"2024-03-04T07:31:16.177145+0000","last_peered":"2024-03-04T07:31:16.177145+0000","last_clean":"2024-03-04T07:31:16.177145+0000","last_became_active":"2024-03-04T07:31:15.130220+0000","last_became_peered":"2024-03-04T07:31:15.130220+0000","last_unstale":"2024-03-04T07:31:16.177145+0000","last_undegraded":"2024-03-04T07:31:16.177145+0000","last_fullsized":"2024-03-04T07:31:16.177145+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:42:23.551532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124793+0000","last_change":"2024-03-04T07:31:15.121443+0000","last_active":"2024-03-04T07:31:16.124793+0000","last_peered":"2024-03-04T07:31:16.124793+0000","last_clean":"2024-03-04T07:31:16.124793+0000","last_became_active":"2024-03-04T07:31:15.120975+0000","last_became_peered":"2024-03-04T07:31:15.120975+0000","last_unstale":"2024-03-04T07:31:16.124793+0000","last_undegraded":"2024-03-04T07:31:16.124793+0000","last_fullsized":"2024-03-04T07:31:16.124793+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:55:55.499344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177195+0000","last_change":"2024-03-04T07:31:15.128769+0000","last_active":"2024-03-04T07:31:16.177195+0000","last_peered":"2024-03-04T07:31:16.177195+0000","last_clean":"2024-03-04T07:31:16.177195+0000","last_became_active":"2024-03-04T07:31:15.128341+0000","last_became_peered":"2024-03-04T07:31:15.128341+0000","last_unstale":"2024-03-04T07:31:16.177195+0000","last_undegraded":"2024-03-04T07:31:16.177195+0000","last_fullsized":"2024-03-04T07:31:16.177195+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:42:26.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122338+0000","last_change":"2024-03-04T07:31:15.130417+0000","last_active":"2024-03-04T07:31:16.122338+0000","last_peered":"2024-03-04T07:31:16.122338+0000","last_clean":"2024-03-04T07:31:16.122338+0000","last_became_active":"2024-03-04T07:31:15.130092+0000","last_became_peered":"2024-03-04T07:31:15.130092+0000","last_unstale":"2024-03-04T07:31:16.122338+0000","last_undegraded":"2024-03-04T07:31:16.122338+0000","last_fullsized":"2024-03-04T07:31:16.122338+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:44:46.054516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.3b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177556+0000","last_change":"2024-03-04T07:31:15.131579+0000","last_active":"2024-03-04T07:31:16.177556+0000","last_peered":"2024-03-04T07:31:16.177556+0000","last_clean":"2024-03-04T07:31:16.177556+0000","last_became_active":"2024-03-04T07:31:15.131065+0000","last_became_peered":"2024-03-04T07:31:15.131065+0000","last_unstale":"2024-03-04T07:31:16.177556+0000","last_undegraded":"2024-03-04T07:31:16.177556+0000","last_fullsized":"2024-03-04T07:31:16.177556+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:46.335494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.3a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178089+0000","last_change":"2024-03-04T07:31:15.134359+0000","last_active":"2024-03-04T07:31:16.178089+0000","last_peered":"2024-03-04T07:31:16.178089+0000","last_clean":"2024-03-04T07:31:16.178089+0000","last_became_active":"2024-03-04T07:31:15.134065+0000","last_became_peered":"2024-03-04T07:31:15.134065+0000","last_unstale":"2024-03-04T07:31:16.178089+0000","last_undegraded":"2024-03-04T07:31:16.178089+0000","last_fullsized":"2024-03-04T07:31:16.178089+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:44:46.054516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.39","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177391+0000","last_change":"2024-03-04T07:31:15.133727+0000","last_active":"2024-03-04T07:31:16.177391+0000","last_peered":"2024-03-04T07:31:16.177391+0000","last_clean":"2024-03-04T07:31:16.177391+0000","last_became_active":"2024-03-04T07:31:15.133396+0000","last_became_peered":"2024-03-04T07:31:15.133396+0000","last_unstale":"2024-03-04T07:31:16.177391+0000","last_undegraded":"2024-03-04T07:31:16.177391+0000","last_fullsized":"2024-03-04T07:31:16.177391+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:46.335494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.38","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124401+0000","last_change":"2024-03-04T07:31:15.134334+0000","last_active":"2024-03-04T07:31:16.124401+0000","last_peered":"2024-03-04T07:31:16.124401+0000","last_clean":"2024-03-04T07:31:16.124401+0000","last_became_active":"2024-03-04T07:31:15.133991+0000","last_became_peered":"2024-03-04T07:31:15.133991+0000","last_unstale":"2024-03-04T07:31:16.124401+0000","last_undegraded":"2024-03-04T07:31:16.124401+0000","last_fullsized":"2024-03-04T07:31:16.124401+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.37","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124177+0000","last_change":"2024-03-04T07:31:15.126237+0000","last_active":"2024-03-04T07:31:16.124177+0000","last_peered":"2024-03-04T07:31:16.124177+0000","last_clean":"2024-03-04T07:31:16.124177+0000","last_became_active":"2024-03-04T07:31:15.125262+0000","last_became_peered":"2024-03-04T07:31:15.125262+0000","last_unstale":"2024-03-04T07:31:16.124177+0000","last_undegraded":"2024-03-04T07:31:16.124177+0000","last_fullsized":"2024-03-04T07:31:16.124177+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.36","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177737+0000","last_change":"2024-03-04T07:31:15.126272+0000","last_active":"2024-03-04T07:31:16.177737+0000","last_peered":"2024-03-04T07:31:16.177737+0000","last_clean":"2024-03-04T07:31:16.177737+0000","last_became_active":"2024-03-04T07:31:15.126005+0000","last_became_peered":"2024-03-04T07:31:15.126005+0000","last_unstale":"2024-03-04T07:31:16.177737+0000","last_undegraded":"2024-03-04T07:31:16.177737+0000","last_fullsized":"2024-03-04T07:31:16.177737+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.35","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177558+0000","last_change":"2024-03-04T07:31:15.138694+0000","last_active":"2024-03-04T07:31:16.177558+0000","last_peered":"2024-03-04T07:31:16.177558+0000","last_clean":"2024-03-04T07:31:16.177558+0000","last_became_active":"2024-03-04T07:31:15.138372+0000","last_became_peered":"2024-03-04T07:31:15.138372+0000","last_unstale":"2024-03-04T07:31:16.177558+0000","last_undegraded":"2024-03-04T07:31:16.177558+0000","last_fullsized":"2024-03-04T07:31:16.177558+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.34","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123704+0000","last_change":"2024-03-04T07:31:15.131929+0000","last_active":"2024-03-04T07:31:16.123704+0000","last_peered":"2024-03-04T07:31:16.123704+0000","last_clean":"2024-03-04T07:31:16.123704+0000","last_became_active":"2024-03-04T07:31:15.130469+0000","last_became_peered":"2024-03-04T07:31:15.130469+0000","last_unstale":"2024-03-04T07:31:16.123704+0000","last_undegraded":"2024-03-04T07:31:16.123704+0000","last_fullsized":"2024-03-04T07:31:16.123704+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:44:38.851234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.33","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122156+0000","last_change":"2024-03-04T07:31:15.135865+0000","last_active":"2024-03-04T07:31:16.122156+0000","last_peered":"2024-03-04T07:31:16.122156+0000","last_clean":"2024-03-04T07:31:16.122156+0000","last_became_active":"2024-03-04T07:31:15.135589+0000","last_became_peered":"2024-03-04T07:31:15.135589+0000","last_unstale":"2024-03-04T07:31:16.122156+0000","last_undegraded":"2024-03-04T07:31:16.122156+0000","last_fullsized":"2024-03-04T07:31:16.122156+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.32","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126596+0000","last_change":"2024-03-04T07:31:15.125996+0000","last_active":"2024-03-04T07:31:16.126596+0000","last_peered":"2024-03-04T07:31:16.126596+0000","last_clean":"2024-03-04T07:31:16.126596+0000","last_became_active":"2024-03-04T07:31:15.125691+0000","last_became_peered":"2024-03-04T07:31:15.125691+0000","last_unstale":"2024-03-04T07:31:16.126596+0000","last_undegraded":"2024-03-04T07:31:16.126596+0000","last_fullsized":"2024-03-04T07:31:16.126596+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:05.357706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.31","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121812+0000","last_change":"2024-03-04T07:31:15.122342+0000","last_active":"2024-03-04T07:31:16.121812+0000","last_peered":"2024-03-04T07:31:16.121812+0000","last_clean":"2024-03-04T07:31:16.121812+0000","last_became_active":"2024-03-04T07:31:15.122071+0000","last_became_peered":"2024-03-04T07:31:15.122071+0000","last_unstale":"2024-03-04T07:31:16.121812+0000","last_undegraded":"2024-03-04T07:31:16.121812+0000","last_fullsized":"2024-03-04T07:31:16.121812+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.30","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178027+0000","last_change":"2024-03-04T07:31:15.134097+0000","last_active":"2024-03-04T07:31:16.178027+0000","last_peered":"2024-03-04T07:31:16.178027+0000","last_clean":"2024-03-04T07:31:16.178027+0000","last_became_active":"2024-03-04T07:31:15.133790+0000","last_became_peered":"2024-03-04T07:31:15.133790+0000","last_unstale":"2024-03-04T07:31:16.178027+0000","last_undegraded":"2024-03-04T07:31:16.178027+0000","last_fullsized":"2024-03-04T07:31:16.178027+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.2f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124643+0000","last_change":"2024-03-04T07:31:15.125051+0000","last_active":"2024-03-04T07:31:16.124643+0000","last_peered":"2024-03-04T07:31:16.124643+0000","last_clean":"2024-03-04T07:31:16.124643+0000","last_became_active":"2024-03-04T07:31:15.124661+0000","last_became_peered":"2024-03-04T07:31:15.124661+0000","last_unstale":"2024-03-04T07:31:16.124643+0000","last_undegraded":"2024-03-04T07:31:16.124643+0000","last_fullsized":"2024-03-04T07:31:16.124643+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:13:06.579050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.2e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122337+0000","last_change":"2024-03-04T07:31:15.130315+0000","last_active":"2024-03-04T07:31:16.122337+0000","last_peered":"2024-03-04T07:31:16.122337+0000","last_clean":"2024-03-04T07:31:16.122337+0000","last_became_active":"2024-03-04T07:31:15.130060+0000","last_became_peered":"2024-03-04T07:31:15.130060+0000","last_unstale":"2024-03-04T07:31:16.122337+0000","last_undegraded":"2024-03-04T07:31:16.122337+0000","last_fullsized":"2024-03-04T07:31:16.122337+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:13:06.579050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.2d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126712+0000","last_change":"2024-03-04T07:31:15.125645+0000","last_active":"2024-03-04T07:31:16.126712+0000","last_peered":"2024-03-04T07:31:16.126712+0000","last_clean":"2024-03-04T07:31:16.126712+0000","last_became_active":"2024-03-04T07:31:15.124161+0000","last_became_peered":"2024-03-04T07:31:15.124161+0000","last_unstale":"2024-03-04T07:31:16.126712+0000","last_undegraded":"2024-03-04T07:31:16.126712+0000","last_fullsized":"2024-03-04T07:31:16.126712+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:42.837231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123902+0000","last_change":"2024-03-04T07:31:15.136716+0000","last_active":"2024-03-04T07:31:16.123902+0000","last_peered":"2024-03-04T07:31:16.123902+0000","last_clean":"2024-03-04T07:31:16.123902+0000","last_became_active":"2024-03-04T07:31:15.136413+0000","last_became_peered":"2024-03-04T07:31:15.136413+0000","last_unstale":"2024-03-04T07:31:16.123902+0000","last_undegraded":"2024-03-04T07:31:16.123902+0000","last_fullsized":"2024-03-04T07:31:16.123902+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:38:12.157284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.2b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126767+0000","last_change":"2024-03-04T07:31:15.125035+0000","last_active":"2024-03-04T07:31:16.126767+0000","last_peered":"2024-03-04T07:31:16.126767+0000","last_clean":"2024-03-04T07:31:16.126767+0000","last_became_active":"2024-03-04T07:31:15.124758+0000","last_became_peered":"2024-03-04T07:31:15.124758+0000","last_unstale":"2024-03-04T07:31:16.126767+0000","last_undegraded":"2024-03-04T07:31:16.126767+0000","last_fullsized":"2024-03-04T07:31:16.126767+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:02:02.097760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124330+0000","last_change":"2024-03-04T07:31:15.120756+0000","last_active":"2024-03-04T07:31:16.124330+0000","last_peered":"2024-03-04T07:31:16.124330+0000","last_clean":"2024-03-04T07:31:16.124330+0000","last_became_active":"2024-03-04T07:31:15.120468+0000","last_became_peered":"2024-03-04T07:31:15.120468+0000","last_unstale":"2024-03-04T07:31:16.124330+0000","last_undegraded":"2024-03-04T07:31:16.124330+0000","last_fullsized":"2024-03-04T07:31:16.124330+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.29","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124309+0000","last_change":"2024-03-04T07:31:15.123355+0000","last_active":"2024-03-04T07:31:16.124309+0000","last_peered":"2024-03-04T07:31:16.124309+0000","last_clean":"2024-03-04T07:31:16.124309+0000","last_became_active":"2024-03-04T07:31:15.123086+0000","last_became_peered":"2024-03-04T07:31:15.123086+0000","last_unstale":"2024-03-04T07:31:16.124309+0000","last_undegraded":"2024-03-04T07:31:16.124309+0000","last_fullsized":"2024-03-04T07:31:16.124309+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.28","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178187+0000","last_change":"2024-03-04T07:31:15.143879+0000","last_active":"2024-03-04T07:31:16.178187+0000","last_peered":"2024-03-04T07:31:16.178187+0000","last_clean":"2024-03-04T07:31:16.178187+0000","last_became_active":"2024-03-04T07:31:15.143507+0000","last_became_peered":"2024-03-04T07:31:15.143507+0000","last_unstale":"2024-03-04T07:31:16.178187+0000","last_undegraded":"2024-03-04T07:31:16.178187+0000","last_fullsized":"2024-03-04T07:31:16.178187+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:13:06.579050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.27","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177817+0000","last_change":"2024-03-04T07:31:15.123413+0000","last_active":"2024-03-04T07:31:16.177817+0000","last_peered":"2024-03-04T07:31:16.177817+0000","last_clean":"2024-03-04T07:31:16.177817+0000","last_became_active":"2024-03-04T07:31:15.123079+0000","last_became_peered":"2024-03-04T07:31:15.123079+0000","last_unstale":"2024-03-04T07:31:16.177817+0000","last_undegraded":"2024-03-04T07:31:16.177817+0000","last_fullsized":"2024-03-04T07:31:16.177817+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.26","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124608+0000","last_change":"2024-03-04T07:31:15.121220+0000","last_active":"2024-03-04T07:31:16.124608+0000","last_peered":"2024-03-04T07:31:16.124608+0000","last_clean":"2024-03-04T07:31:16.124608+0000","last_became_active":"2024-03-04T07:31:15.120627+0000","last_became_peered":"2024-03-04T07:31:15.120627+0000","last_unstale":"2024-03-04T07:31:16.124608+0000","last_undegraded":"2024-03-04T07:31:16.124608+0000","last_fullsized":"2024-03-04T07:31:16.124608+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:44:46.054516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.25","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177723+0000","last_change":"2024-03-04T07:31:15.138559+0000","last_active":"2024-03-04T07:31:16.177723+0000","last_peered":"2024-03-04T07:31:16.177723+0000","last_clean":"2024-03-04T07:31:16.177723+0000","last_became_active":"2024-03-04T07:31:15.138251+0000","last_became_peered":"2024-03-04T07:31:15.138251+0000","last_unstale":"2024-03-04T07:31:16.177723+0000","last_undegraded":"2024-03-04T07:31:16.177723+0000","last_fullsized":"2024-03-04T07:31:16.177723+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.24","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177847+0000","last_change":"2024-03-04T07:31:15.135906+0000","last_active":"2024-03-04T07:31:16.177847+0000","last_peered":"2024-03-04T07:31:16.177847+0000","last_clean":"2024-03-04T07:31:16.177847+0000","last_became_active":"2024-03-04T07:31:15.135315+0000","last_became_peered":"2024-03-04T07:31:15.135315+0000","last_unstale":"2024-03-04T07:31:16.177847+0000","last_undegraded":"2024-03-04T07:31:16.177847+0000","last_fullsized":"2024-03-04T07:31:16.177847+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.23","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124472+0000","last_change":"2024-03-04T07:31:15.124880+0000","last_active":"2024-03-04T07:31:16.124472+0000","last_peered":"2024-03-04T07:31:16.124472+0000","last_clean":"2024-03-04T07:31:16.124472+0000","last_became_active":"2024-03-04T07:31:15.124485+0000","last_became_peered":"2024-03-04T07:31:15.124485+0000","last_unstale":"2024-03-04T07:31:16.124472+0000","last_undegraded":"2024-03-04T07:31:16.124472+0000","last_fullsized":"2024-03-04T07:31:16.124472+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.22","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126645+0000","last_change":"2024-03-04T07:31:15.126445+0000","last_active":"2024-03-04T07:31:16.126645+0000","last_peered":"2024-03-04T07:31:16.126645+0000","last_clean":"2024-03-04T07:31:16.126645+0000","last_became_active":"2024-03-04T07:31:15.126159+0000","last_became_peered":"2024-03-04T07:31:15.126159+0000","last_unstale":"2024-03-04T07:31:16.126645+0000","last_undegraded":"2024-03-04T07:31:16.126645+0000","last_fullsized":"2024-03-04T07:31:16.126645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:06:10.440065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.21","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121921+0000","last_change":"2024-03-04T07:31:15.123321+0000","last_active":"2024-03-04T07:31:16.121921+0000","last_peered":"2024-03-04T07:31:16.121921+0000","last_clean":"2024-03-04T07:31:16.121921+0000","last_became_active":"2024-03-04T07:31:15.123027+0000","last_became_peered":"2024-03-04T07:31:15.123027+0000","last_unstale":"2024-03-04T07:31:16.121921+0000","last_undegraded":"2024-03-04T07:31:16.121921+0000","last_fullsized":"2024-03-04T07:31:16.121921+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.20","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176820+0000","last_change":"2024-03-04T07:31:15.124576+0000","last_active":"2024-03-04T07:31:16.176820+0000","last_peered":"2024-03-04T07:31:16.176820+0000","last_clean":"2024-03-04T07:31:16.176820+0000","last_became_active":"2024-03-04T07:31:15.124203+0000","last_became_peered":"2024-03-04T07:31:15.124203+0000","last_unstale":"2024-03-04T07:31:16.176820+0000","last_undegraded":"2024-03-04T07:31:16.176820+0000","last_fullsized":"2024-03-04T07:31:16.176820+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:25:53.272574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126921+0000","last_change":"2024-03-04T07:31:15.134190+0000","last_active":"2024-03-04T07:31:16.126921+0000","last_peered":"2024-03-04T07:31:16.126921+0000","last_clean":"2024-03-04T07:31:16.126921+0000","last_became_active":"2024-03-04T07:31:15.133882+0000","last_became_peered":"2024-03-04T07:31:15.133882+0000","last_unstale":"2024-03-04T07:31:16.126921+0000","last_undegraded":"2024-03-04T07:31:16.126921+0000","last_fullsized":"2024-03-04T07:31:16.126921+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:50.642628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1f","version":"89'4","reported_seq":18,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124659+0000","last_change":"2024-03-04T07:31:14.127398+0000","last_active":"2024-03-04T07:31:16.124659+0000","last_peered":"2024-03-04T07:31:16.124659+0000","last_clean":"2024-03-04T07:31:16.124659+0000","last_became_active":"2024-03-04T07:31:14.127144+0000","last_became_peered":"2024-03-04T07:31:14.127144+0000","last_unstale":"2024-03-04T07:31:16.124659+0000","last_undegraded":"2024-03-04T07:31:16.124659+0000","last_fullsized":"2024-03-04T07:31:16.124659+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:55:52.644367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122551+0000","last_change":"2024-03-04T07:31:15.121767+0000","last_active":"2024-03-04T07:31:16.122551+0000","last_peered":"2024-03-04T07:31:16.122551+0000","last_clean":"2024-03-04T07:31:16.122551+0000","last_became_active":"2024-03-04T07:31:15.121401+0000","last_became_peered":"2024-03-04T07:31:15.121401+0000","last_unstale":"2024-03-04T07:31:16.122551+0000","last_undegraded":"2024-03-04T07:31:16.122551+0000","last_fullsized":"2024-03-04T07:31:16.122551+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:01.876883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1e","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177655+0000","last_change":"2024-03-04T07:31:14.133406+0000","last_active":"2024-03-04T07:31:16.177655+0000","last_peered":"2024-03-04T07:31:16.177655+0000","last_clean":"2024-03-04T07:31:16.177655+0000","last_became_active":"2024-03-04T07:31:14.133110+0000","last_became_peered":"2024-03-04T07:31:14.133110+0000","last_unstale":"2024-03-04T07:31:16.177655+0000","last_undegraded":"2024-03-04T07:31:16.177655+0000","last_fullsized":"2024-03-04T07:31:16.177655+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:55.061136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177101+0000","last_change":"2024-03-04T07:31:15.133763+0000","last_active":"2024-03-04T07:31:16.177101+0000","last_peered":"2024-03-04T07:31:16.177101+0000","last_clean":"2024-03-04T07:31:16.177101+0000","last_became_active":"2024-03-04T07:31:15.133375+0000","last_became_peered":"2024-03-04T07:31:15.133375+0000","last_unstale":"2024-03-04T07:31:16.177101+0000","last_undegraded":"2024-03-04T07:31:16.177101+0000","last_fullsized":"2024-03-04T07:31:16.177101+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:38:07.254043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.1d","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124528+0000","last_change":"2024-03-04T07:31:14.122586+0000","last_active":"2024-03-04T07:31:16.124528+0000","last_peered":"2024-03-04T07:31:16.124528+0000","last_clean":"2024-03-04T07:31:16.124528+0000","last_became_active":"2024-03-04T07:31:14.122256+0000","last_became_peered":"2024-03-04T07:31:14.122256+0000","last_unstale":"2024-03-04T07:31:16.124528+0000","last_undegraded":"2024-03-04T07:31:16.124528+0000","last_fullsized":"2024-03-04T07:31:16.124528+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:30:41.922144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123988+0000","last_change":"2024-03-04T07:31:15.129468+0000","last_active":"2024-03-04T07:31:16.123988+0000","last_peered":"2024-03-04T07:31:16.123988+0000","last_clean":"2024-03-04T07:31:16.123988+0000","last_became_active":"2024-03-04T07:31:15.128904+0000","last_became_peered":"2024-03-04T07:31:15.128904+0000","last_unstale":"2024-03-04T07:31:16.123988+0000","last_undegraded":"2024-03-04T07:31:16.123988+0000","last_fullsized":"2024-03-04T07:31:16.123988+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:46.335494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1c","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124741+0000","last_change":"2024-03-04T07:31:14.127749+0000","last_active":"2024-03-04T07:31:16.124741+0000","last_peered":"2024-03-04T07:31:16.124741+0000","last_clean":"2024-03-04T07:31:16.124741+0000","last_became_active":"2024-03-04T07:31:14.126815+0000","last_became_peered":"2024-03-04T07:31:14.126815+0000","last_unstale":"2024-03-04T07:31:16.124741+0000","last_undegraded":"2024-03-04T07:31:16.124741+0000","last_fullsized":"2024-03-04T07:31:16.124741+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:40:01.393384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123517+0000","last_change":"2024-03-04T07:31:15.132434+0000","last_active":"2024-03-04T07:31:16.123517+0000","last_peered":"2024-03-04T07:31:16.123517+0000","last_clean":"2024-03-04T07:31:16.123517+0000","last_became_active":"2024-03-04T07:31:15.132156+0000","last_became_peered":"2024-03-04T07:31:15.132156+0000","last_unstale":"2024-03-04T07:31:16.123517+0000","last_undegraded":"2024-03-04T07:31:16.123517+0000","last_fullsized":"2024-03-04T07:31:16.123517+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:58:11.642356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1b","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176941+0000","last_change":"2024-03-04T07:31:14.129194+0000","last_active":"2024-03-04T07:31:16.176941+0000","last_peered":"2024-03-04T07:31:16.176941+0000","last_clean":"2024-03-04T07:31:16.176941+0000","last_became_active":"2024-03-04T07:31:14.128872+0000","last_became_peered":"2024-03-04T07:31:14.128872+0000","last_unstale":"2024-03-04T07:31:16.176941+0000","last_undegraded":"2024-03-04T07:31:16.176941+0000","last_fullsized":"2024-03-04T07:31:16.176941+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:27.130702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177477+0000","last_change":"2024-03-04T07:31:15.134189+0000","last_active":"2024-03-04T07:31:16.177477+0000","last_peered":"2024-03-04T07:31:16.177477+0000","last_clean":"2024-03-04T07:31:16.177477+0000","last_became_active":"2024-03-04T07:31:15.133733+0000","last_became_peered":"2024-03-04T07:31:15.133733+0000","last_unstale":"2024-03-04T07:31:16.177477+0000","last_undegraded":"2024-03-04T07:31:16.177477+0000","last_fullsized":"2024-03-04T07:31:16.177477+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:23:48.369845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1a","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124285+0000","last_change":"2024-03-04T07:31:14.129884+0000","last_active":"2024-03-04T07:31:16.124285+0000","last_peered":"2024-03-04T07:31:16.124285+0000","last_clean":"2024-03-04T07:31:16.124285+0000","last_became_active":"2024-03-04T07:31:14.129592+0000","last_became_peered":"2024-03-04T07:31:14.129592+0000","last_unstale":"2024-03-04T07:31:16.124285+0000","last_undegraded":"2024-03-04T07:31:16.124285+0000","last_fullsized":"2024-03-04T07:31:16.124285+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:58.727794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.18","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121821+0000","last_change":"2024-03-04T07:31:15.123097+0000","last_active":"2024-03-04T07:31:16.121821+0000","last_peered":"2024-03-04T07:31:16.121821+0000","last_clean":"2024-03-04T07:31:16.121821+0000","last_became_active":"2024-03-04T07:31:15.122827+0000","last_became_peered":"2024-03-04T07:31:15.122827+0000","last_unstale":"2024-03-04T07:31:16.121821+0000","last_undegraded":"2024-03-04T07:31:16.121821+0000","last_fullsized":"2024-03-04T07:31:16.121821+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.19","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177616+0000","last_change":"2024-03-04T07:31:14.131225+0000","last_active":"2024-03-04T07:31:16.177616+0000","last_peered":"2024-03-04T07:31:16.177616+0000","last_clean":"2024-03-04T07:31:16.177616+0000","last_became_active":"2024-03-04T07:31:14.130880+0000","last_became_peered":"2024-03-04T07:31:14.130880+0000","last_unstale":"2024-03-04T07:31:16.177616+0000","last_undegraded":"2024-03-04T07:31:16.177616+0000","last_fullsized":"2024-03-04T07:31:16.177616+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:08:27.793757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.19","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123785+0000","last_change":"2024-03-04T07:31:15.129803+0000","last_active":"2024-03-04T07:31:16.123785+0000","last_peered":"2024-03-04T07:31:16.123785+0000","last_clean":"2024-03-04T07:31:16.123785+0000","last_became_active":"2024-03-04T07:31:15.129441+0000","last_became_peered":"2024-03-04T07:31:15.129441+0000","last_unstale":"2024-03-04T07:31:16.123785+0000","last_undegraded":"2024-03-04T07:31:16.123785+0000","last_fullsized":"2024-03-04T07:31:16.123785+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:01:42.622168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.18","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176877+0000","last_change":"2024-03-04T07:31:14.135162+0000","last_active":"2024-03-04T07:31:16.176877+0000","last_peered":"2024-03-04T07:31:16.176877+0000","last_clean":"2024-03-04T07:31:16.176877+0000","last_became_active":"2024-03-04T07:31:14.134870+0000","last_became_peered":"2024-03-04T07:31:14.134870+0000","last_unstale":"2024-03-04T07:31:16.176877+0000","last_undegraded":"2024-03-04T07:31:16.176877+0000","last_fullsized":"2024-03-04T07:31:16.176877+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:49.640760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.16","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124763+0000","last_change":"2024-03-04T07:31:15.116159+0000","last_active":"2024-03-04T07:31:16.124763+0000","last_peered":"2024-03-04T07:31:16.124763+0000","last_clean":"2024-03-04T07:31:16.124763+0000","last_became_active":"2024-03-04T07:31:15.115861+0000","last_became_peered":"2024-03-04T07:31:15.115861+0000","last_unstale":"2024-03-04T07:31:16.124763+0000","last_undegraded":"2024-03-04T07:31:16.124763+0000","last_fullsized":"2024-03-04T07:31:16.124763+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:01.876883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.17","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177227+0000","last_change":"2024-03-04T07:31:14.129589+0000","last_active":"2024-03-04T07:31:16.177227+0000","last_peered":"2024-03-04T07:31:16.177227+0000","last_clean":"2024-03-04T07:31:16.177227+0000","last_became_active":"2024-03-04T07:31:14.129195+0000","last_became_peered":"2024-03-04T07:31:14.129195+0000","last_unstale":"2024-03-04T07:31:16.177227+0000","last_undegraded":"2024-03-04T07:31:16.177227+0000","last_fullsized":"2024-03-04T07:31:16.177227+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:30.998711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.17","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124168+0000","last_change":"2024-03-04T07:31:15.125869+0000","last_active":"2024-03-04T07:31:16.124168+0000","last_peered":"2024-03-04T07:31:16.124168+0000","last_clean":"2024-03-04T07:31:16.124168+0000","last_became_active":"2024-03-04T07:31:15.124885+0000","last_became_peered":"2024-03-04T07:31:15.124885+0000","last_unstale":"2024-03-04T07:31:16.124168+0000","last_undegraded":"2024-03-04T07:31:16.124168+0000","last_fullsized":"2024-03-04T07:31:16.124168+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:23:48.369845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122357+0000","last_change":"2024-03-04T07:31:14.129809+0000","last_active":"2024-03-04T07:31:16.122357+0000","last_peered":"2024-03-04T07:31:16.122357+0000","last_clean":"2024-03-04T07:31:16.122357+0000","last_became_active":"2024-03-04T07:31:14.129521+0000","last_became_peered":"2024-03-04T07:31:14.129521+0000","last_unstale":"2024-03-04T07:31:16.122357+0000","last_undegraded":"2024-03-04T07:31:16.122357+0000","last_fullsized":"2024-03-04T07:31:16.122357+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:40.635424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.14","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178000+0000","last_change":"2024-03-04T07:31:15.124154+0000","last_active":"2024-03-04T07:31:16.178000+0000","last_peered":"2024-03-04T07:31:16.178000+0000","last_clean":"2024-03-04T07:31:16.178000+0000","last_became_active":"2024-03-04T07:31:15.123765+0000","last_became_peered":"2024-03-04T07:31:15.123765+0000","last_unstale":"2024-03-04T07:31:16.178000+0000","last_undegraded":"2024-03-04T07:31:16.178000+0000","last_fullsized":"2024-03-04T07:31:16.178000+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.15","version":"89'3","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178140+0000","last_change":"2024-03-04T07:31:14.132032+0000","last_active":"2024-03-04T07:31:16.178140+0000","last_peered":"2024-03-04T07:31:16.178140+0000","last_clean":"2024-03-04T07:31:16.178140+0000","last_became_active":"2024-03-04T07:31:14.131696+0000","last_became_peered":"2024-03-04T07:31:14.131696+0000","last_unstale":"2024-03-04T07:31:16.178140+0000","last_undegraded":"2024-03-04T07:31:16.178140+0000","last_fullsized":"2024-03-04T07:31:16.178140+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:56.524264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.15","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177879+0000","last_change":"2024-03-04T07:31:15.137243+0000","last_active":"2024-03-04T07:31:16.177879+0000","last_peered":"2024-03-04T07:31:16.177879+0000","last_clean":"2024-03-04T07:31:16.177879+0000","last_became_active":"2024-03-04T07:31:15.137000+0000","last_became_peered":"2024-03-04T07:31:15.137000+0000","last_unstale":"2024-03-04T07:31:16.177879+0000","last_undegraded":"2024-03-04T07:31:16.177879+0000","last_fullsized":"2024-03-04T07:31:16.177879+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.14","version":"89'6","reported_seq":20,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123974+0000","last_change":"2024-03-04T07:31:14.134032+0000","last_active":"2024-03-04T07:31:16.123974+0000","last_peered":"2024-03-04T07:31:16.123974+0000","last_clean":"2024-03-04T07:31:16.123974+0000","last_became_active":"2024-03-04T07:31:14.133708+0000","last_became_peered":"2024-03-04T07:31:14.133708+0000","last_unstale":"2024-03-04T07:31:16.123974+0000","last_undegraded":"2024-03-04T07:31:16.123974+0000","last_fullsized":"2024-03-04T07:31:16.123974+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:40:01.393384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.12","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121991+0000","last_change":"2024-03-04T07:31:15.135192+0000","last_active":"2024-03-04T07:31:16.121991+0000","last_peered":"2024-03-04T07:31:16.121991+0000","last_clean":"2024-03-04T07:31:16.121991+0000","last_became_active":"2024-03-04T07:31:15.134853+0000","last_became_peered":"2024-03-04T07:31:15.134853+0000","last_unstale":"2024-03-04T07:31:16.121991+0000","last_undegraded":"2024-03-04T07:31:16.121991+0000","last_fullsized":"2024-03-04T07:31:16.121991+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126880+0000","last_change":"2024-03-04T07:31:14.126745+0000","last_active":"2024-03-04T07:31:16.126880+0000","last_peered":"2024-03-04T07:31:16.126880+0000","last_clean":"2024-03-04T07:31:16.126880+0000","last_became_active":"2024-03-04T07:31:14.126439+0000","last_became_peered":"2024-03-04T07:31:14.126439+0000","last_unstale":"2024-03-04T07:31:16.126880+0000","last_undegraded":"2024-03-04T07:31:16.126880+0000","last_fullsized":"2024-03-04T07:31:16.126880+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:11.062648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.13","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176892+0000","last_change":"2024-03-04T07:31:15.129119+0000","last_active":"2024-03-04T07:31:16.176892+0000","last_peered":"2024-03-04T07:31:16.176892+0000","last_clean":"2024-03-04T07:31:16.176892+0000","last_became_active":"2024-03-04T07:31:15.128780+0000","last_became_peered":"2024-03-04T07:31:15.128780+0000","last_unstale":"2024-03-04T07:31:16.176892+0000","last_undegraded":"2024-03-04T07:31:16.176892+0000","last_fullsized":"2024-03-04T07:31:16.176892+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:18:59.613735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.12","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126835+0000","last_change":"2024-03-04T07:31:14.134331+0000","last_active":"2024-03-04T07:31:16.126835+0000","last_peered":"2024-03-04T07:31:16.126835+0000","last_clean":"2024-03-04T07:31:16.126835+0000","last_became_active":"2024-03-04T07:31:14.133864+0000","last_became_peered":"2024-03-04T07:31:14.133864+0000","last_unstale":"2024-03-04T07:31:16.126835+0000","last_undegraded":"2024-03-04T07:31:16.126835+0000","last_fullsized":"2024-03-04T07:31:16.126835+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:28:03.119654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.10","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122878+0000","last_change":"2024-03-04T07:31:15.122062+0000","last_active":"2024-03-04T07:31:16.122878+0000","last_peered":"2024-03-04T07:31:16.122878+0000","last_clean":"2024-03-04T07:31:16.122878+0000","last_became_active":"2024-03-04T07:31:15.121765+0000","last_became_peered":"2024-03-04T07:31:15.121765+0000","last_unstale":"2024-03-04T07:31:16.122878+0000","last_undegraded":"2024-03-04T07:31:16.122878+0000","last_fullsized":"2024-03-04T07:31:16.122878+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:45:18.173216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.11","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123924+0000","last_change":"2024-03-04T07:31:14.128784+0000","last_active":"2024-03-04T07:31:16.123924+0000","last_peered":"2024-03-04T07:31:16.123924+0000","last_clean":"2024-03-04T07:31:16.123924+0000","last_became_active":"2024-03-04T07:31:14.128449+0000","last_became_peered":"2024-03-04T07:31:14.128449+0000","last_unstale":"2024-03-04T07:31:16.123924+0000","last_undegraded":"2024-03-04T07:31:16.123924+0000","last_fullsized":"2024-03-04T07:31:16.123924+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:58.727794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.11","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126535+0000","last_change":"2024-03-04T07:31:15.133344+0000","last_active":"2024-03-04T07:31:16.126535+0000","last_peered":"2024-03-04T07:31:16.126535+0000","last_clean":"2024-03-04T07:31:16.126535+0000","last_became_active":"2024-03-04T07:31:15.133003+0000","last_became_peered":"2024-03-04T07:31:15.133003+0000","last_unstale":"2024-03-04T07:31:16.126535+0000","last_undegraded":"2024-03-04T07:31:16.126535+0000","last_fullsized":"2024-03-04T07:31:16.126535+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:58:25.382715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.10","version":"89'8","reported_seq":22,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176949+0000","last_change":"2024-03-04T07:31:14.133109+0000","last_active":"2024-03-04T07:31:16.176949+0000","last_peered":"2024-03-04T07:31:16.176949+0000","last_clean":"2024-03-04T07:31:16.176949+0000","last_became_active":"2024-03-04T07:31:14.132859+0000","last_became_peered":"2024-03-04T07:31:14.132859+0000","last_unstale":"2024-03-04T07:31:16.176949+0000","last_undegraded":"2024-03-04T07:31:16.176949+0000","last_fullsized":"2024-03-04T07:31:16.176949+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:37.553768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177595+0000","last_change":"2024-03-04T07:31:15.132726+0000","last_active":"2024-03-04T07:31:16.177595+0000","last_peered":"2024-03-04T07:31:16.177595+0000","last_clean":"2024-03-04T07:31:16.177595+0000","last_became_active":"2024-03-04T07:31:15.132433+0000","last_became_peered":"2024-03-04T07:31:15.132433+0000","last_unstale":"2024-03-04T07:31:16.177595+0000","last_undegraded":"2024-03-04T07:31:16.177595+0000","last_fullsized":"2024-03-04T07:31:16.177595+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:23:48.369845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123556+0000","last_change":"2024-03-04T07:31:14.130857+0000","last_active":"2024-03-04T07:31:16.123556+0000","last_peered":"2024-03-04T07:31:16.123556+0000","last_clean":"2024-03-04T07:31:16.123556+0000","last_became_active":"2024-03-04T07:31:14.130500+0000","last_became_peered":"2024-03-04T07:31:14.130500+0000","last_unstale":"2024-03-04T07:31:16.123556+0000","last_undegraded":"2024-03-04T07:31:16.123556+0000","last_fullsized":"2024-03-04T07:31:16.123556+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:16.517725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123012+0000","last_change":"2024-03-04T07:31:15.121967+0000","last_active":"2024-03-04T07:31:16.123012+0000","last_peered":"2024-03-04T07:31:16.123012+0000","last_clean":"2024-03-04T07:31:16.123012+0000","last_became_active":"2024-03-04T07:31:15.121696+0000","last_became_peered":"2024-03-04T07:31:15.121696+0000","last_unstale":"2024-03-04T07:31:16.123012+0000","last_undegraded":"2024-03-04T07:31:16.123012+0000","last_fullsized":"2024-03-04T07:31:16.123012+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:42:26.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.d","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177917+0000","last_change":"2024-03-04T07:31:14.128357+0000","last_active":"2024-03-04T07:31:16.177917+0000","last_peered":"2024-03-04T07:31:16.177917+0000","last_clean":"2024-03-04T07:31:16.177917+0000","last_became_active":"2024-03-04T07:31:14.128047+0000","last_became_peered":"2024-03-04T07:31:14.128047+0000","last_unstale":"2024-03-04T07:31:16.177917+0000","last_undegraded":"2024-03-04T07:31:16.177917+0000","last_fullsized":"2024-03-04T07:31:16.177917+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:40:01.393384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178043+0000","last_change":"2024-03-04T07:31:15.139651+0000","last_active":"2024-03-04T07:31:16.178043+0000","last_peered":"2024-03-04T07:31:16.178043+0000","last_clean":"2024-03-04T07:31:16.178043+0000","last_became_active":"2024-03-04T07:31:15.139117+0000","last_became_peered":"2024-03-04T07:31:15.139117+0000","last_unstale":"2024-03-04T07:31:16.178043+0000","last_undegraded":"2024-03-04T07:31:16.178043+0000","last_fullsized":"2024-03-04T07:31:16.178043+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124827+0000","last_change":"2024-03-04T07:31:14.127553+0000","last_active":"2024-03-04T07:31:16.124827+0000","last_peered":"2024-03-04T07:31:16.124827+0000","last_clean":"2024-03-04T07:31:16.124827+0000","last_became_active":"2024-03-04T07:31:14.126598+0000","last_became_peered":"2024-03-04T07:31:14.126598+0000","last_unstale":"2024-03-04T07:31:16.124827+0000","last_undegraded":"2024-03-04T07:31:16.124827+0000","last_fullsized":"2024-03-04T07:31:16.124827+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:08:27.793757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126737+0000","last_change":"2024-03-04T07:31:15.132550+0000","last_active":"2024-03-04T07:31:16.126737+0000","last_peered":"2024-03-04T07:31:16.126737+0000","last_clean":"2024-03-04T07:31:16.126737+0000","last_became_active":"2024-03-04T07:31:15.132038+0000","last_became_peered":"2024-03-04T07:31:15.132038+0000","last_unstale":"2024-03-04T07:31:16.126737+0000","last_undegraded":"2024-03-04T07:31:16.126737+0000","last_fullsized":"2024-03-04T07:31:16.126737+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:54:00.967692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124458+0000","last_change":"2024-03-04T07:31:14.122956+0000","last_active":"2024-03-04T07:31:16.124458+0000","last_peered":"2024-03-04T07:31:16.124458+0000","last_clean":"2024-03-04T07:31:16.124458+0000","last_became_active":"2024-03-04T07:31:14.122682+0000","last_became_peered":"2024-03-04T07:31:14.122682+0000","last_unstale":"2024-03-04T07:31:16.124458+0000","last_undegraded":"2024-03-04T07:31:16.124458+0000","last_fullsized":"2024-03-04T07:31:16.124458+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:56.524264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126872+0000","last_change":"2024-03-04T07:31:15.123297+0000","last_active":"2024-03-04T07:31:16.126872+0000","last_peered":"2024-03-04T07:31:16.126872+0000","last_clean":"2024-03-04T07:31:16.126872+0000","last_became_active":"2024-03-04T07:31:15.123022+0000","last_became_peered":"2024-03-04T07:31:15.123022+0000","last_unstale":"2024-03-04T07:31:16.126872+0000","last_undegraded":"2024-03-04T07:31:16.126872+0000","last_fullsized":"2024-03-04T07:31:16.126872+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:52.140070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123624+0000","last_change":"2024-03-04T07:31:14.135693+0000","last_active":"2024-03-04T07:31:16.123624+0000","last_peered":"2024-03-04T07:31:16.123624+0000","last_clean":"2024-03-04T07:31:16.123624+0000","last_became_active":"2024-03-04T07:31:14.135302+0000","last_became_peered":"2024-03-04T07:31:14.135302+0000","last_unstale":"2024-03-04T07:31:16.123624+0000","last_undegraded":"2024-03-04T07:31:16.123624+0000","last_fullsized":"2024-03-04T07:31:16.123624+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:12:21.581341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.8","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126485+0000","last_change":"2024-03-04T07:31:15.132190+0000","last_active":"2024-03-04T07:31:16.126485+0000","last_peered":"2024-03-04T07:31:16.126485+0000","last_clean":"2024-03-04T07:31:16.126485+0000","last_became_active":"2024-03-04T07:31:15.131912+0000","last_became_peered":"2024-03-04T07:31:15.131912+0000","last_unstale":"2024-03-04T07:31:16.126485+0000","last_undegraded":"2024-03-04T07:31:16.126485+0000","last_fullsized":"2024-03-04T07:31:16.126485+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:43:35.898650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122824+0000","last_change":"2024-03-04T07:31:14.134205+0000","last_active":"2024-03-04T07:31:16.122824+0000","last_peered":"2024-03-04T07:31:16.122824+0000","last_clean":"2024-03-04T07:31:16.122824+0000","last_became_active":"2024-03-04T07:31:14.133786+0000","last_became_peered":"2024-03-04T07:31:14.133786+0000","last_unstale":"2024-03-04T07:31:16.122824+0000","last_undegraded":"2024-03-04T07:31:16.122824+0000","last_fullsized":"2024-03-04T07:31:16.122824+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:25:52.270706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.9","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177247+0000","last_change":"2024-03-04T07:31:15.127491+0000","last_active":"2024-03-04T07:31:16.177247+0000","last_peered":"2024-03-04T07:31:16.177247+0000","last_clean":"2024-03-04T07:31:16.177247+0000","last_became_active":"2024-03-04T07:31:15.127206+0000","last_became_peered":"2024-03-04T07:31:15.127206+0000","last_unstale":"2024-03-04T07:31:16.177247+0000","last_undegraded":"2024-03-04T07:31:16.177247+0000","last_fullsized":"2024-03-04T07:31:16.177247+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:01:42.622168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124252+0000","last_change":"2024-03-04T07:31:14.133376+0000","last_active":"2024-03-04T07:31:16.124252+0000","last_peered":"2024-03-04T07:31:16.124252+0000","last_clean":"2024-03-04T07:31:16.124252+0000","last_became_active":"2024-03-04T07:31:14.133082+0000","last_became_peered":"2024-03-04T07:31:14.133082+0000","last_unstale":"2024-03-04T07:31:16.124252+0000","last_undegraded":"2024-03-04T07:31:16.124252+0000","last_fullsized":"2024-03-04T07:31:16.124252+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:16.517725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.6","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124899+0000","last_change":"2024-03-04T07:31:15.120309+0000","last_active":"2024-03-04T07:31:16.124899+0000","last_peered":"2024-03-04T07:31:16.124899+0000","last_clean":"2024-03-04T07:31:16.124899+0000","last_became_active":"2024-03-04T07:31:15.120028+0000","last_became_peered":"2024-03-04T07:31:15.120028+0000","last_unstale":"2024-03-04T07:31:16.124899+0000","last_undegraded":"2024-03-04T07:31:16.124899+0000","last_fullsized":"2024-03-04T07:31:16.124899+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:24:19.440520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.7","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124150+0000","last_change":"2024-03-04T07:31:14.125768+0000","last_active":"2024-03-04T07:31:16.124150+0000","last_peered":"2024-03-04T07:31:16.124150+0000","last_clean":"2024-03-04T07:31:16.124150+0000","last_became_active":"2024-03-04T07:31:14.125467+0000","last_became_peered":"2024-03-04T07:31:14.125467+0000","last_unstale":"2024-03-04T07:31:16.124150+0000","last_undegraded":"2024-03-04T07:31:16.124150+0000","last_fullsized":"2024-03-04T07:31:16.124150+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:18:58.611867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.7","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177206+0000","last_change":"2024-03-04T07:31:15.125973+0000","last_active":"2024-03-04T07:31:16.177206+0000","last_peered":"2024-03-04T07:31:16.177206+0000","last_clean":"2024-03-04T07:31:16.177206+0000","last_became_active":"2024-03-04T07:31:15.125537+0000","last_became_peered":"2024-03-04T07:31:15.125537+0000","last_unstale":"2024-03-04T07:31:16.177206+0000","last_undegraded":"2024-03-04T07:31:16.177206+0000","last_fullsized":"2024-03-04T07:31:16.177206+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:58:11.642356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"90'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126022+0000","last_change":"2024-03-04T07:31:14.123270+0000","last_active":"2024-03-04T07:31:16.126022+0000","last_peered":"2024-03-04T07:31:16.126022+0000","last_clean":"2024-03-04T07:31:16.126022+0000","last_became_active":"2024-03-04T07:31:14.122998+0000","last_became_peered":"2024-03-04T07:31:14.122998+0000","last_unstale":"2024-03-04T07:31:16.126022+0000","last_undegraded":"2024-03-04T07:31:16.126022+0000","last_fullsized":"2024-03-04T07:31:16.126022+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:05:02.289401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.4","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177101+0000","last_change":"2024-03-04T07:31:15.124791+0000","last_active":"2024-03-04T07:31:16.177101+0000","last_peered":"2024-03-04T07:31:16.177101+0000","last_clean":"2024-03-04T07:31:16.177101+0000","last_became_active":"2024-03-04T07:31:15.124340+0000","last_became_peered":"2024-03-04T07:31:15.124340+0000","last_unstale":"2024-03-04T07:31:16.177101+0000","last_undegraded":"2024-03-04T07:31:16.177101+0000","last_fullsized":"2024-03-04T07:31:16.177101+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:18:52.983145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124481+0000","last_change":"2024-03-04T07:31:14.128646+0000","last_active":"2024-03-04T07:31:16.124481+0000","last_peered":"2024-03-04T07:31:16.124481+0000","last_clean":"2024-03-04T07:31:16.124481+0000","last_became_active":"2024-03-04T07:31:14.128385+0000","last_became_peered":"2024-03-04T07:31:14.128385+0000","last_unstale":"2024-03-04T07:31:16.124481+0000","last_undegraded":"2024-03-04T07:31:16.124481+0000","last_fullsized":"2024-03-04T07:31:16.124481+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:55.061136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.5","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177918+0000","last_change":"2024-03-04T07:31:15.131405+0000","last_active":"2024-03-04T07:31:16.177918+0000","last_peered":"2024-03-04T07:31:16.177918+0000","last_clean":"2024-03-04T07:31:16.177918+0000","last_became_active":"2024-03-04T07:31:15.131121+0000","last_became_peered":"2024-03-04T07:31:15.131121+0000","last_unstale":"2024-03-04T07:31:16.177918+0000","last_undegraded":"2024-03-04T07:31:16.177918+0000","last_fullsized":"2024-03-04T07:31:16.177918+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178090+0000","last_change":"2024-03-04T07:31:14.131821+0000","last_active":"2024-03-04T07:31:16.178090+0000","last_peered":"2024-03-04T07:31:16.178090+0000","last_clean":"2024-03-04T07:31:16.178090+0000","last_became_active":"2024-03-04T07:31:14.131489+0000","last_became_peered":"2024-03-04T07:31:14.131489+0000","last_unstale":"2024-03-04T07:31:16.178090+0000","last_undegraded":"2024-03-04T07:31:16.178090+0000","last_fullsized":"2024-03-04T07:31:16.178090+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:55.061136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.2","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126826+0000","last_change":"2024-03-04T07:31:15.134258+0000","last_active":"2024-03-04T07:31:16.126826+0000","last_peered":"2024-03-04T07:31:16.126826+0000","last_clean":"2024-03-04T07:31:16.126826+0000","last_became_active":"2024-03-04T07:31:15.133823+0000","last_became_peered":"2024-03-04T07:31:15.133823+0000","last_unstale":"2024-03-04T07:31:16.126826+0000","last_undegraded":"2024-03-04T07:31:16.126826+0000","last_fullsized":"2024-03-04T07:31:16.126826+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:02:19.871841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124566+0000","last_change":"2024-03-04T07:31:14.133865+0000","last_active":"2024-03-04T07:31:16.124566+0000","last_peered":"2024-03-04T07:31:16.124566+0000","last_clean":"2024-03-04T07:31:16.124566+0000","last_became_active":"2024-03-04T07:31:14.133589+0000","last_became_peered":"2024-03-04T07:31:14.133589+0000","last_unstale":"2024-03-04T07:31:16.124566+0000","last_undegraded":"2024-03-04T07:31:16.124566+0000","last_fullsized":"2024-03-04T07:31:16.124566+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:40.635424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123287+0000","last_change":"2024-03-04T07:31:15.123764+0000","last_active":"2024-03-04T07:31:16.123287+0000","last_peered":"2024-03-04T07:31:16.123287+0000","last_clean":"2024-03-04T07:31:16.123287+0000","last_became_active":"2024-03-04T07:31:15.123472+0000","last_became_peered":"2024-03-04T07:31:15.123472+0000","last_unstale":"2024-03-04T07:31:16.123287+0000","last_undegraded":"2024-03-04T07:31:16.123287+0000","last_fullsized":"2024-03-04T07:31:16.123287+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:56:17.317135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177239+0000","last_change":"2024-03-04T07:31:14.134964+0000","last_active":"2024-03-04T07:31:16.177239+0000","last_peered":"2024-03-04T07:31:16.177239+0000","last_clean":"2024-03-04T07:31:16.177239+0000","last_became_active":"2024-03-04T07:31:14.134607+0000","last_became_peered":"2024-03-04T07:31:14.134607+0000","last_unstale":"2024-03-04T07:31:16.177239+0000","last_undegraded":"2024-03-04T07:31:16.177239+0000","last_fullsized":"2024-03-04T07:31:16.177239+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:58.727794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.0","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126508+0000","last_change":"2024-03-04T07:31:15.133863+0000","last_active":"2024-03-04T07:31:16.126508+0000","last_peered":"2024-03-04T07:31:16.126508+0000","last_clean":"2024-03-04T07:31:16.126508+0000","last_became_active":"2024-03-04T07:31:15.133475+0000","last_became_peered":"2024-03-04T07:31:15.133475+0000","last_unstale":"2024-03-04T07:31:16.126508+0000","last_undegraded":"2024-03-04T07:31:16.126508+0000","last_fullsized":"2024-03-04T07:31:16.126508+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:01.998924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121894+0000","last_change":"2024-03-04T07:31:14.133663+0000","last_active":"2024-03-04T07:31:16.121894+0000","last_peered":"2024-03-04T07:31:16.121894+0000","last_clean":"2024-03-04T07:31:16.121894+0000","last_became_active":"2024-03-04T07:31:14.133367+0000","last_became_peered":"2024-03-04T07:31:14.133367+0000","last_unstale":"2024-03-04T07:31:16.121894+0000","last_undegraded":"2024-03-04T07:31:16.121894+0000","last_fullsized":"2024-03-04T07:31:16.121894+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:30:41.922144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.1","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177416+0000","last_change":"2024-03-04T07:31:15.135746+0000","last_active":"2024-03-04T07:31:16.177416+0000","last_peered":"2024-03-04T07:31:16.177416+0000","last_clean":"2024-03-04T07:31:16.177416+0000","last_became_active":"2024-03-04T07:31:15.135455+0000","last_became_peered":"2024-03-04T07:31:15.135455+0000","last_unstale":"2024-03-04T07:31:16.177416+0000","last_undegraded":"2024-03-04T07:31:16.177416+0000","last_fullsized":"2024-03-04T07:31:16.177416+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:38:12.157284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126587+0000","last_change":"2024-03-04T07:31:14.124436+0000","last_active":"2024-03-04T07:31:16.126587+0000","last_peered":"2024-03-04T07:31:16.126587+0000","last_clean":"2024-03-04T07:31:16.126587+0000","last_became_active":"2024-03-04T07:31:14.124125+0000","last_became_peered":"2024-03-04T07:31:14.124125+0000","last_unstale":"2024-03-04T07:31:16.126587+0000","last_undegraded":"2024-03-04T07:31:16.126587+0000","last_fullsized":"2024-03-04T07:31:16.126587+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:54.403600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126982+0000","last_change":"2024-03-04T07:31:15.132454+0000","last_active":"2024-03-04T07:31:16.126982+0000","last_peered":"2024-03-04T07:31:16.126982+0000","last_clean":"2024-03-04T07:31:16.126982+0000","last_became_active":"2024-03-04T07:31:15.132200+0000","last_became_peered":"2024-03-04T07:31:15.132200+0000","last_unstale":"2024-03-04T07:31:16.126982+0000","last_undegraded":"2024-03-04T07:31:16.126982+0000","last_fullsized":"2024-03-04T07:31:16.126982+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:28.132569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.f","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178272+0000","last_change":"2024-03-04T07:31:14.133893+0000","last_active":"2024-03-04T07:31:16.178272+0000","last_peered":"2024-03-04T07:31:16.178272+0000","last_clean":"2024-03-04T07:31:16.178272+0000","last_became_active":"2024-03-04T07:31:14.133545+0000","last_became_peered":"2024-03-04T07:31:14.133545+0000","last_unstale":"2024-03-04T07:31:16.178272+0000","last_undegraded":"2024-03-04T07:31:16.178272+0000","last_fullsized":"2024-03-04T07:31:16.178272+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:55:52.644367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":298,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178208+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:31:16.178208+0000","last_peered":"2024-03-04T07:31:16.178208+0000","last_clean":"2024-03-04T07:31:16.178208+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:31:16.178208+0000","last_undegraded":"2024-03-04T07:31:16.178208+0000","last_fullsized":"2024-03-04T07:31:16.178208+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":15,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":14,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574908,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32844,"kb_used_data":1664,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338996,"statfs":{"total":96636764160,"available":96603131904,"internally_reserved":0,"allocated":1703936,"data_stored":1058930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574907,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32264,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339576,"statfs":{"total":96636764160,"available":96603725824,"internally_reserved":0,"allocated":1110016,"data_stored":469102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574907,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":468520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574908,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32284,"kb_used_data":1096,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339556,"statfs":{"total":96636764160,"available":96603705344,"internally_reserved":0,"allocated":1122304,"data_stored":470186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574907,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32272,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339568,"statfs":{"total":96636764160,"available":96603717632,"internally_reserved":0,"allocated":1118208,"data_stored":470084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574907,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":468484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32256,"kb_used_data":1076,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339584,"statfs":{"total":96636764160,"available":96603734016,"internally_reserved":0,"allocated":1101824,"data_stored":468450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574907,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32844,"kb_used_data":1660,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338996,"statfs":{"total":96636764160,"available":96603131904,"internally_reserved":0,"allocated":1699840,"data_stored":1059400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:31:18.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:31:18.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.294+0000 7f87f5193640 1 Processor -- start 2024-03-04T07:31:18.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.294+0000 7f87f5193640 1 -- start start 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87f5193640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f87f0071c50 0x7f87f00d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87f5193640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f87f00ce970 con 0x7f87f00b2c50 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87f5193640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f87f00cef30 con 0x7f87f015e000 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87f5193640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f87f00cece0 con 0x7f87f0071c50 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87ee59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f87f0071c50 0x7f87f00d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87ee59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f87f0071c50 0x7f87f00d25d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38742/0 (socket says 172.21.15.64:38742) 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87ee59c640 1 -- 172.21.15.64:0/3825421702 learned_addr learned my addr 172.21.15.64:0/3825421702 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:18.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87ee59c640 1 -- 172.21.15.64:0/3825421702 >> v1:172.21.15.66:6789/0 conn(0x7f87f015e000 legacy=0x7f87f0071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87ee59c640 1 -- 172.21.15.64:0/3825421702 >> v1:172.21.15.64:6789/0 conn(0x7f87f00b2c50 legacy=0x7f87f00d2b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.295+0000 7f87ee59c640 1 -- 172.21.15.64:0/3825421702 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87f00cf210 con 0x7f87f0071c50 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.296+0000 7f87ee59c640 1 --2- 172.21.15.64:0/3825421702 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f00d25d0 secure :-1 s=READY pgs=1128 cs=0 l=1 rev1=1 crypto rx=0x7f87d8009870 tx=0x7f87d802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59a2edf348ddc56e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.296+0000 7f87edd9b640 1 -- 172.21.15.64:0/3825421702 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f87d8034070 con 0x7f87f0071c50 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.296+0000 7f87edd9b640 1 -- 172.21.15.64:0/3825421702 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f87d8030860 con 0x7f87f0071c50 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.296+0000 7f87edd9b640 1 -- 172.21.15.64:0/3825421702 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f87d8030c30 con 0x7f87f0071c50 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.297+0000 7f87f5193640 1 -- 172.21.15.64:0/3825421702 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 msgr2=0x7f87f00d25d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.297+0000 7f87f5193640 1 --2- 172.21.15.64:0/3825421702 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f00d25d0 secure :-1 s=READY pgs=1128 cs=0 l=1 rev1=1 crypto rx=0x7f87d8009870 tx=0x7f87d802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.297+0000 7f87f5193640 1 -- 172.21.15.64:0/3825421702 shutdown_connections 2024-03-04T07:31:18.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.297+0000 7f87f5193640 1 --2- 172.21.15.64:0/3825421702 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f00d25d0 unknown :-1 s=CLOSED pgs=1128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.297+0000 7f87f5193640 1 -- 172.21.15.64:0/3825421702 >> 172.21.15.64:0/3825421702 conn(0x7f87f00d3c80 msgr2=0x7f87f00d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:18.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.297+0000 7f87f5193640 1 -- 172.21.15.64:0/3825421702 shutdown_connections 2024-03-04T07:31:18.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.297+0000 7f87f5193640 1 -- 172.21.15.64:0/3825421702 wait complete. 2024-03-04T07:31:18.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.298+0000 7f87f5193640 1 Processor -- start 2024-03-04T07:31:18.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.298+0000 7f87f5193640 1 -- start start 2024-03-04T07:31:18.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87f5193640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f01450c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87f5193640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f87f00b2c50 0x7f87f0145600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87f5193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f87f015e000 0x7f87f00c99d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87f5193640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87f00b3320 con 0x7f87f0071c50 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87f5193640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87f00b31a0 con 0x7f87f00b2c50 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87f5193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f87f015e3e0 con 0x7f87f015e000 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f01450c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f01450c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38750/0 (socket says 172.21.15.64:38750) 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 -- 172.21.15.64:0/2968563564 learned_addr learned my addr 172.21.15.64:0/2968563564 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87ee59c640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f87f00b2c50 0x7f87f0145600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87ef59e640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f87f015e000 0x7f87f00c99d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 -- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f87f015e000 msgr2=0x7f87f00c99d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f87f015e000 0x7f87f00c99d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 -- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f87f00b2c50 msgr2=0x7f87f0145600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f87f00b2c50 0x7f87f0145600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87eed9d640 1 -- 172.21.15.64:0/2968563564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87f0157f70 con 0x7f87f0071c50 2024-03-04T07:31:18.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.299+0000 7f87ee59c640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f87f00b2c50 0x7f87f0145600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:18.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.300+0000 7f87eed9d640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f01450c0 secure :-1 s=READY pgs=1129 cs=0 l=1 rev1=1 crypto rx=0x7f87e800c450 tx=0x7f87e800c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:18.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.300+0000 7f87e77fe640 1 -- 172.21.15.64:0/2968563564 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f87e80052d0 con 0x7f87f0071c50 2024-03-04T07:31:18.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.300+0000 7f87e77fe640 1 -- 172.21.15.64:0/2968563564 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f87e8005470 con 0x7f87f0071c50 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.300+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f87f0158250 con 0x7f87f0071c50 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.300+0000 7f87e77fe640 1 -- 172.21.15.64:0/2968563564 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f87e8012870 con 0x7f87f0071c50 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.300+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f87f0158810 con 0x7f87f0071c50 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.301+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f87f0067ff0 con 0x7f87f0071c50 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.301+0000 7f87e77fe640 1 -- 172.21.15.64:0/2968563564 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f87e8012a10 con 0x7f87f0071c50 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.302+0000 7f87e77fe640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f87d006b500 0x7f87d006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.302+0000 7f87e77fe640 1 -- 172.21.15.64:0/2968563564 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f87e8089c90 con 0x7f87f0071c50 2024-03-04T07:31:18.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.302+0000 7f87ee59c640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f87d006b500 0x7f87d006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.302+0000 7f87ee59c640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f87d006b500 0x7f87d006d9c0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f87d8036000 tx=0x7f87d803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:18.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.306+0000 7f87e77fe640 1 -- 172.21.15.64:0/2968563564 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f87e80584c0 con 0x7f87f0071c50 2024-03-04T07:31:18.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.482+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f87f0143030 con 0x7f87d006b500 2024-03-04T07:31:18.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.488+0000 7f87e77fe640 1 -- 172.21.15.64:0/2968563564 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252082 (secure 0 0 0) 0x7f87f0143030 con 0x7f87d006b500 2024-03-04T07:31:18.491 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:18.492 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:31:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f87d006b500 msgr2=0x7f87d006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f87d006b500 0x7f87d006d9c0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f87d8036000 tx=0x7f87d803b040 comp rx=0 tx=0).stop 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 msgr2=0x7f87f01450c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f01450c0 secure :-1 s=READY pgs=1129 cs=0 l=1 rev1=1 crypto rx=0x7f87e800c450 tx=0x7f87e800c910 comp rx=0 tx=0).stop 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 shutdown_connections 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f87d006b500 0x7f87d006d9c0 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f87f015e000 0x7f87f00c99d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f87f00b2c50 0x7f87f0145600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 --2- 172.21.15.64:0/2968563564 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f87f0071c50 0x7f87f01450c0 unknown :-1 s=CLOSED pgs=1129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 >> 172.21.15.64:0/2968563564 conn(0x7f87f00d3c80 msgr2=0x7f87f00cf750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 shutdown_connections 2024-03-04T07:31:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.491+0000 7f87f5193640 1 -- 172.21.15.64:0/2968563564 wait complete. 2024-03-04T07:31:18.502 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2024-03-04T07:31:17.188752+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":97,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":259284,"kb_used_data":9832,"kb_used_omap":12,"kb_used_meta":249331,"kb_avail":754715436,"statfs":{"total":773094113280,"available":772828606464,"internally_reserved":0,"allocated":10067968,"data_stored":4933156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":255315504},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":27,"apply_latency_ms":27,"commit_latency_ns":27000000,"apply_latency_ns":27000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.081572"},"pg_stats":[{"pgid":"15.3f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177145+0000","last_change":"2024-03-04T07:31:15.133828+0000","last_active":"2024-03-04T07:31:16.177145+0000","last_peered":"2024-03-04T07:31:16.177145+0000","last_clean":"2024-03-04T07:31:16.177145+0000","last_became_active":"2024-03-04T07:31:15.130220+0000","last_became_peered":"2024-03-04T07:31:15.130220+0000","last_unstale":"2024-03-04T07:31:16.177145+0000","last_undegraded":"2024-03-04T07:31:16.177145+0000","last_fullsized":"2024-03-04T07:31:16.177145+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:42:23.551532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124793+0000","last_change":"2024-03-04T07:31:15.121443+0000","last_active":"2024-03-04T07:31:16.124793+0000","last_peered":"2024-03-04T07:31:16.124793+0000","last_clean":"2024-03-04T07:31:16.124793+0000","last_became_active":"2024-03-04T07:31:15.120975+0000","last_became_peered":"2024-03-04T07:31:15.120975+0000","last_unstale":"2024-03-04T07:31:16.124793+0000","last_undegraded":"2024-03-04T07:31:16.124793+0000","last_fullsized":"2024-03-04T07:31:16.124793+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:55:55.499344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177195+0000","last_change":"2024-03-04T07:31:15.128769+0000","last_active":"2024-03-04T07:31:16.177195+0000","last_peered":"2024-03-04T07:31:16.177195+0000","last_clean":"2024-03-04T07:31:16.177195+0000","last_became_active":"2024-03-04T07:31:15.128341+0000","last_became_peered":"2024-03-04T07:31:15.128341+0000","last_unstale":"2024-03-04T07:31:16.177195+0000","last_undegraded":"2024-03-04T07:31:16.177195+0000","last_fullsized":"2024-03-04T07:31:16.177195+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:42:26.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.3c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122338+0000","last_change":"2024-03-04T07:31:15.130417+0000","last_active":"2024-03-04T07:31:16.122338+0000","last_peered":"2024-03-04T07:31:16.122338+0000","last_clean":"2024-03-04T07:31:16.122338+0000","last_became_active":"2024-03-04T07:31:15.130092+0000","last_became_peered":"2024-03-04T07:31:15.130092+0000","last_unstale":"2024-03-04T07:31:16.122338+0000","last_undegraded":"2024-03-04T07:31:16.122338+0000","last_fullsized":"2024-03-04T07:31:16.122338+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:44:46.054516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.3b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177556+0000","last_change":"2024-03-04T07:31:15.131579+0000","last_active":"2024-03-04T07:31:16.177556+0000","last_peered":"2024-03-04T07:31:16.177556+0000","last_clean":"2024-03-04T07:31:16.177556+0000","last_became_active":"2024-03-04T07:31:15.131065+0000","last_became_peered":"2024-03-04T07:31:15.131065+0000","last_unstale":"2024-03-04T07:31:16.177556+0000","last_undegraded":"2024-03-04T07:31:16.177556+0000","last_fullsized":"2024-03-04T07:31:16.177556+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:46.335494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.3a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178089+0000","last_change":"2024-03-04T07:31:15.134359+0000","last_active":"2024-03-04T07:31:16.178089+0000","last_peered":"2024-03-04T07:31:16.178089+0000","last_clean":"2024-03-04T07:31:16.178089+0000","last_became_active":"2024-03-04T07:31:15.134065+0000","last_became_peered":"2024-03-04T07:31:15.134065+0000","last_unstale":"2024-03-04T07:31:16.178089+0000","last_undegraded":"2024-03-04T07:31:16.178089+0000","last_fullsized":"2024-03-04T07:31:16.178089+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:44:46.054516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.39","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177391+0000","last_change":"2024-03-04T07:31:15.133727+0000","last_active":"2024-03-04T07:31:16.177391+0000","last_peered":"2024-03-04T07:31:16.177391+0000","last_clean":"2024-03-04T07:31:16.177391+0000","last_became_active":"2024-03-04T07:31:15.133396+0000","last_became_peered":"2024-03-04T07:31:15.133396+0000","last_unstale":"2024-03-04T07:31:16.177391+0000","last_undegraded":"2024-03-04T07:31:16.177391+0000","last_fullsized":"2024-03-04T07:31:16.177391+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:46.335494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.38","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124401+0000","last_change":"2024-03-04T07:31:15.134334+0000","last_active":"2024-03-04T07:31:16.124401+0000","last_peered":"2024-03-04T07:31:16.124401+0000","last_clean":"2024-03-04T07:31:16.124401+0000","last_became_active":"2024-03-04T07:31:15.133991+0000","last_became_peered":"2024-03-04T07:31:15.133991+0000","last_unstale":"2024-03-04T07:31:16.124401+0000","last_undegraded":"2024-03-04T07:31:16.124401+0000","last_fullsized":"2024-03-04T07:31:16.124401+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.37","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124177+0000","last_change":"2024-03-04T07:31:15.126237+0000","last_active":"2024-03-04T07:31:16.124177+0000","last_peered":"2024-03-04T07:31:16.124177+0000","last_clean":"2024-03-04T07:31:16.124177+0000","last_became_active":"2024-03-04T07:31:15.125262+0000","last_became_peered":"2024-03-04T07:31:15.125262+0000","last_unstale":"2024-03-04T07:31:16.124177+0000","last_undegraded":"2024-03-04T07:31:16.124177+0000","last_fullsized":"2024-03-04T07:31:16.124177+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.36","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177737+0000","last_change":"2024-03-04T07:31:15.126272+0000","last_active":"2024-03-04T07:31:16.177737+0000","last_peered":"2024-03-04T07:31:16.177737+0000","last_clean":"2024-03-04T07:31:16.177737+0000","last_became_active":"2024-03-04T07:31:15.126005+0000","last_became_peered":"2024-03-04T07:31:15.126005+0000","last_unstale":"2024-03-04T07:31:16.177737+0000","last_undegraded":"2024-03-04T07:31:16.177737+0000","last_fullsized":"2024-03-04T07:31:16.177737+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.35","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177558+0000","last_change":"2024-03-04T07:31:15.138694+0000","last_active":"2024-03-04T07:31:16.177558+0000","last_peered":"2024-03-04T07:31:16.177558+0000","last_clean":"2024-03-04T07:31:16.177558+0000","last_became_active":"2024-03-04T07:31:15.138372+0000","last_became_peered":"2024-03-04T07:31:15.138372+0000","last_unstale":"2024-03-04T07:31:16.177558+0000","last_undegraded":"2024-03-04T07:31:16.177558+0000","last_fullsized":"2024-03-04T07:31:16.177558+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.34","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123704+0000","last_change":"2024-03-04T07:31:15.131929+0000","last_active":"2024-03-04T07:31:16.123704+0000","last_peered":"2024-03-04T07:31:16.123704+0000","last_clean":"2024-03-04T07:31:16.123704+0000","last_became_active":"2024-03-04T07:31:15.130469+0000","last_became_peered":"2024-03-04T07:31:15.130469+0000","last_unstale":"2024-03-04T07:31:16.123704+0000","last_undegraded":"2024-03-04T07:31:16.123704+0000","last_fullsized":"2024-03-04T07:31:16.123704+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:44:38.851234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.33","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122156+0000","last_change":"2024-03-04T07:31:15.135865+0000","last_active":"2024-03-04T07:31:16.122156+0000","last_peered":"2024-03-04T07:31:16.122156+0000","last_clean":"2024-03-04T07:31:16.122156+0000","last_became_active":"2024-03-04T07:31:15.135589+0000","last_became_peered":"2024-03-04T07:31:15.135589+0000","last_unstale":"2024-03-04T07:31:16.122156+0000","last_undegraded":"2024-03-04T07:31:16.122156+0000","last_fullsized":"2024-03-04T07:31:16.122156+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.32","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126596+0000","last_change":"2024-03-04T07:31:15.125996+0000","last_active":"2024-03-04T07:31:16.126596+0000","last_peered":"2024-03-04T07:31:16.126596+0000","last_clean":"2024-03-04T07:31:16.126596+0000","last_became_active":"2024-03-04T07:31:15.125691+0000","last_became_peered":"2024-03-04T07:31:15.125691+0000","last_unstale":"2024-03-04T07:31:16.126596+0000","last_undegraded":"2024-03-04T07:31:16.126596+0000","last_fullsized":"2024-03-04T07:31:16.126596+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:05.357706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.31","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121812+0000","last_change":"2024-03-04T07:31:15.122342+0000","last_active":"2024-03-04T07:31:16.121812+0000","last_peered":"2024-03-04T07:31:16.121812+0000","last_clean":"2024-03-04T07:31:16.121812+0000","last_became_active":"2024-03-04T07:31:15.122071+0000","last_became_peered":"2024-03-04T07:31:15.122071+0000","last_unstale":"2024-03-04T07:31:16.121812+0000","last_undegraded":"2024-03-04T07:31:16.121812+0000","last_fullsized":"2024-03-04T07:31:16.121812+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:04.955225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.30","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178027+0000","last_change":"2024-03-04T07:31:15.134097+0000","last_active":"2024-03-04T07:31:16.178027+0000","last_peered":"2024-03-04T07:31:16.178027+0000","last_clean":"2024-03-04T07:31:16.178027+0000","last_became_active":"2024-03-04T07:31:15.133790+0000","last_became_peered":"2024-03-04T07:31:15.133790+0000","last_unstale":"2024-03-04T07:31:16.178027+0000","last_undegraded":"2024-03-04T07:31:16.178027+0000","last_fullsized":"2024-03-04T07:31:16.178027+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.2f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124643+0000","last_change":"2024-03-04T07:31:15.125051+0000","last_active":"2024-03-04T07:31:16.124643+0000","last_peered":"2024-03-04T07:31:16.124643+0000","last_clean":"2024-03-04T07:31:16.124643+0000","last_became_active":"2024-03-04T07:31:15.124661+0000","last_became_peered":"2024-03-04T07:31:15.124661+0000","last_unstale":"2024-03-04T07:31:16.124643+0000","last_undegraded":"2024-03-04T07:31:16.124643+0000","last_fullsized":"2024-03-04T07:31:16.124643+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:13:06.579050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.2e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122337+0000","last_change":"2024-03-04T07:31:15.130315+0000","last_active":"2024-03-04T07:31:16.122337+0000","last_peered":"2024-03-04T07:31:16.122337+0000","last_clean":"2024-03-04T07:31:16.122337+0000","last_became_active":"2024-03-04T07:31:15.130060+0000","last_became_peered":"2024-03-04T07:31:15.130060+0000","last_unstale":"2024-03-04T07:31:16.122337+0000","last_undegraded":"2024-03-04T07:31:16.122337+0000","last_fullsized":"2024-03-04T07:31:16.122337+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:13:06.579050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.2d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126712+0000","last_change":"2024-03-04T07:31:15.125645+0000","last_active":"2024-03-04T07:31:16.126712+0000","last_peered":"2024-03-04T07:31:16.126712+0000","last_clean":"2024-03-04T07:31:16.126712+0000","last_became_active":"2024-03-04T07:31:15.124161+0000","last_became_peered":"2024-03-04T07:31:15.124161+0000","last_unstale":"2024-03-04T07:31:16.126712+0000","last_undegraded":"2024-03-04T07:31:16.126712+0000","last_fullsized":"2024-03-04T07:31:16.126712+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:56:42.837231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123902+0000","last_change":"2024-03-04T07:31:15.136716+0000","last_active":"2024-03-04T07:31:16.123902+0000","last_peered":"2024-03-04T07:31:16.123902+0000","last_clean":"2024-03-04T07:31:16.123902+0000","last_became_active":"2024-03-04T07:31:15.136413+0000","last_became_peered":"2024-03-04T07:31:15.136413+0000","last_unstale":"2024-03-04T07:31:16.123902+0000","last_undegraded":"2024-03-04T07:31:16.123902+0000","last_fullsized":"2024-03-04T07:31:16.123902+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:38:12.157284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.2b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126767+0000","last_change":"2024-03-04T07:31:15.125035+0000","last_active":"2024-03-04T07:31:16.126767+0000","last_peered":"2024-03-04T07:31:16.126767+0000","last_clean":"2024-03-04T07:31:16.126767+0000","last_became_active":"2024-03-04T07:31:15.124758+0000","last_became_peered":"2024-03-04T07:31:15.124758+0000","last_unstale":"2024-03-04T07:31:16.126767+0000","last_undegraded":"2024-03-04T07:31:16.126767+0000","last_fullsized":"2024-03-04T07:31:16.126767+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:02:02.097760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.2a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124330+0000","last_change":"2024-03-04T07:31:15.120756+0000","last_active":"2024-03-04T07:31:16.124330+0000","last_peered":"2024-03-04T07:31:16.124330+0000","last_clean":"2024-03-04T07:31:16.124330+0000","last_became_active":"2024-03-04T07:31:15.120468+0000","last_became_peered":"2024-03-04T07:31:15.120468+0000","last_unstale":"2024-03-04T07:31:16.124330+0000","last_undegraded":"2024-03-04T07:31:16.124330+0000","last_fullsized":"2024-03-04T07:31:16.124330+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.29","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124309+0000","last_change":"2024-03-04T07:31:15.123355+0000","last_active":"2024-03-04T07:31:16.124309+0000","last_peered":"2024-03-04T07:31:16.124309+0000","last_clean":"2024-03-04T07:31:16.124309+0000","last_became_active":"2024-03-04T07:31:15.123086+0000","last_became_peered":"2024-03-04T07:31:15.123086+0000","last_unstale":"2024-03-04T07:31:16.124309+0000","last_undegraded":"2024-03-04T07:31:16.124309+0000","last_fullsized":"2024-03-04T07:31:16.124309+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.28","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178187+0000","last_change":"2024-03-04T07:31:15.143879+0000","last_active":"2024-03-04T07:31:16.178187+0000","last_peered":"2024-03-04T07:31:16.178187+0000","last_clean":"2024-03-04T07:31:16.178187+0000","last_became_active":"2024-03-04T07:31:15.143507+0000","last_became_peered":"2024-03-04T07:31:15.143507+0000","last_unstale":"2024-03-04T07:31:16.178187+0000","last_undegraded":"2024-03-04T07:31:16.178187+0000","last_fullsized":"2024-03-04T07:31:16.178187+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:13:06.579050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.27","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177817+0000","last_change":"2024-03-04T07:31:15.123413+0000","last_active":"2024-03-04T07:31:16.177817+0000","last_peered":"2024-03-04T07:31:16.177817+0000","last_clean":"2024-03-04T07:31:16.177817+0000","last_became_active":"2024-03-04T07:31:15.123079+0000","last_became_peered":"2024-03-04T07:31:15.123079+0000","last_unstale":"2024-03-04T07:31:16.177817+0000","last_undegraded":"2024-03-04T07:31:16.177817+0000","last_fullsized":"2024-03-04T07:31:16.177817+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.26","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124608+0000","last_change":"2024-03-04T07:31:15.121220+0000","last_active":"2024-03-04T07:31:16.124608+0000","last_peered":"2024-03-04T07:31:16.124608+0000","last_clean":"2024-03-04T07:31:16.124608+0000","last_became_active":"2024-03-04T07:31:15.120627+0000","last_became_peered":"2024-03-04T07:31:15.120627+0000","last_unstale":"2024-03-04T07:31:16.124608+0000","last_undegraded":"2024-03-04T07:31:16.124608+0000","last_fullsized":"2024-03-04T07:31:16.124608+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:44:46.054516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.25","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177723+0000","last_change":"2024-03-04T07:31:15.138559+0000","last_active":"2024-03-04T07:31:16.177723+0000","last_peered":"2024-03-04T07:31:16.177723+0000","last_clean":"2024-03-04T07:31:16.177723+0000","last_became_active":"2024-03-04T07:31:15.138251+0000","last_became_peered":"2024-03-04T07:31:15.138251+0000","last_unstale":"2024-03-04T07:31:16.177723+0000","last_undegraded":"2024-03-04T07:31:16.177723+0000","last_fullsized":"2024-03-04T07:31:16.177723+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.24","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177847+0000","last_change":"2024-03-04T07:31:15.135906+0000","last_active":"2024-03-04T07:31:16.177847+0000","last_peered":"2024-03-04T07:31:16.177847+0000","last_clean":"2024-03-04T07:31:16.177847+0000","last_became_active":"2024-03-04T07:31:15.135315+0000","last_became_peered":"2024-03-04T07:31:15.135315+0000","last_unstale":"2024-03-04T07:31:16.177847+0000","last_undegraded":"2024-03-04T07:31:16.177847+0000","last_fullsized":"2024-03-04T07:31:16.177847+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.23","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124472+0000","last_change":"2024-03-04T07:31:15.124880+0000","last_active":"2024-03-04T07:31:16.124472+0000","last_peered":"2024-03-04T07:31:16.124472+0000","last_clean":"2024-03-04T07:31:16.124472+0000","last_became_active":"2024-03-04T07:31:15.124485+0000","last_became_peered":"2024-03-04T07:31:15.124485+0000","last_unstale":"2024-03-04T07:31:16.124472+0000","last_undegraded":"2024-03-04T07:31:16.124472+0000","last_fullsized":"2024-03-04T07:31:16.124472+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.22","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126645+0000","last_change":"2024-03-04T07:31:15.126445+0000","last_active":"2024-03-04T07:31:16.126645+0000","last_peered":"2024-03-04T07:31:16.126645+0000","last_clean":"2024-03-04T07:31:16.126645+0000","last_became_active":"2024-03-04T07:31:15.126159+0000","last_became_peered":"2024-03-04T07:31:15.126159+0000","last_unstale":"2024-03-04T07:31:16.126645+0000","last_undegraded":"2024-03-04T07:31:16.126645+0000","last_fullsized":"2024-03-04T07:31:16.126645+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:06:10.440065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.21","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121921+0000","last_change":"2024-03-04T07:31:15.123321+0000","last_active":"2024-03-04T07:31:16.121921+0000","last_peered":"2024-03-04T07:31:16.121921+0000","last_clean":"2024-03-04T07:31:16.121921+0000","last_became_active":"2024-03-04T07:31:15.123027+0000","last_became_peered":"2024-03-04T07:31:15.123027+0000","last_unstale":"2024-03-04T07:31:16.121921+0000","last_undegraded":"2024-03-04T07:31:16.121921+0000","last_fullsized":"2024-03-04T07:31:16.121921+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.20","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176820+0000","last_change":"2024-03-04T07:31:15.124576+0000","last_active":"2024-03-04T07:31:16.176820+0000","last_peered":"2024-03-04T07:31:16.176820+0000","last_clean":"2024-03-04T07:31:16.176820+0000","last_became_active":"2024-03-04T07:31:15.124203+0000","last_became_peered":"2024-03-04T07:31:15.124203+0000","last_unstale":"2024-03-04T07:31:16.176820+0000","last_undegraded":"2024-03-04T07:31:16.176820+0000","last_fullsized":"2024-03-04T07:31:16.176820+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:25:53.272574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126921+0000","last_change":"2024-03-04T07:31:15.134190+0000","last_active":"2024-03-04T07:31:16.126921+0000","last_peered":"2024-03-04T07:31:16.126921+0000","last_clean":"2024-03-04T07:31:16.126921+0000","last_became_active":"2024-03-04T07:31:15.133882+0000","last_became_peered":"2024-03-04T07:31:15.133882+0000","last_unstale":"2024-03-04T07:31:16.126921+0000","last_undegraded":"2024-03-04T07:31:16.126921+0000","last_fullsized":"2024-03-04T07:31:16.126921+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:50.642628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1f","version":"89'4","reported_seq":18,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124659+0000","last_change":"2024-03-04T07:31:14.127398+0000","last_active":"2024-03-04T07:31:16.124659+0000","last_peered":"2024-03-04T07:31:16.124659+0000","last_clean":"2024-03-04T07:31:16.124659+0000","last_became_active":"2024-03-04T07:31:14.127144+0000","last_became_peered":"2024-03-04T07:31:14.127144+0000","last_unstale":"2024-03-04T07:31:16.124659+0000","last_undegraded":"2024-03-04T07:31:16.124659+0000","last_fullsized":"2024-03-04T07:31:16.124659+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:55:52.644367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122551+0000","last_change":"2024-03-04T07:31:15.121767+0000","last_active":"2024-03-04T07:31:16.122551+0000","last_peered":"2024-03-04T07:31:16.122551+0000","last_clean":"2024-03-04T07:31:16.122551+0000","last_became_active":"2024-03-04T07:31:15.121401+0000","last_became_peered":"2024-03-04T07:31:15.121401+0000","last_unstale":"2024-03-04T07:31:16.122551+0000","last_undegraded":"2024-03-04T07:31:16.122551+0000","last_fullsized":"2024-03-04T07:31:16.122551+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:01.876883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1e","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177655+0000","last_change":"2024-03-04T07:31:14.133406+0000","last_active":"2024-03-04T07:31:16.177655+0000","last_peered":"2024-03-04T07:31:16.177655+0000","last_clean":"2024-03-04T07:31:16.177655+0000","last_became_active":"2024-03-04T07:31:14.133110+0000","last_became_peered":"2024-03-04T07:31:14.133110+0000","last_unstale":"2024-03-04T07:31:16.177655+0000","last_undegraded":"2024-03-04T07:31:16.177655+0000","last_fullsized":"2024-03-04T07:31:16.177655+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:55.061136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177101+0000","last_change":"2024-03-04T07:31:15.133763+0000","last_active":"2024-03-04T07:31:16.177101+0000","last_peered":"2024-03-04T07:31:16.177101+0000","last_clean":"2024-03-04T07:31:16.177101+0000","last_became_active":"2024-03-04T07:31:15.133375+0000","last_became_peered":"2024-03-04T07:31:15.133375+0000","last_unstale":"2024-03-04T07:31:16.177101+0000","last_undegraded":"2024-03-04T07:31:16.177101+0000","last_fullsized":"2024-03-04T07:31:16.177101+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:38:07.254043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.1d","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124528+0000","last_change":"2024-03-04T07:31:14.122586+0000","last_active":"2024-03-04T07:31:16.124528+0000","last_peered":"2024-03-04T07:31:16.124528+0000","last_clean":"2024-03-04T07:31:16.124528+0000","last_became_active":"2024-03-04T07:31:14.122256+0000","last_became_peered":"2024-03-04T07:31:14.122256+0000","last_unstale":"2024-03-04T07:31:16.124528+0000","last_undegraded":"2024-03-04T07:31:16.124528+0000","last_fullsized":"2024-03-04T07:31:16.124528+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:30:41.922144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123988+0000","last_change":"2024-03-04T07:31:15.129468+0000","last_active":"2024-03-04T07:31:16.123988+0000","last_peered":"2024-03-04T07:31:16.123988+0000","last_clean":"2024-03-04T07:31:16.123988+0000","last_became_active":"2024-03-04T07:31:15.128904+0000","last_became_peered":"2024-03-04T07:31:15.128904+0000","last_unstale":"2024-03-04T07:31:16.123988+0000","last_undegraded":"2024-03-04T07:31:16.123988+0000","last_fullsized":"2024-03-04T07:31:16.123988+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:29:46.335494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1c","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124741+0000","last_change":"2024-03-04T07:31:14.127749+0000","last_active":"2024-03-04T07:31:16.124741+0000","last_peered":"2024-03-04T07:31:16.124741+0000","last_clean":"2024-03-04T07:31:16.124741+0000","last_became_active":"2024-03-04T07:31:14.126815+0000","last_became_peered":"2024-03-04T07:31:14.126815+0000","last_unstale":"2024-03-04T07:31:16.124741+0000","last_undegraded":"2024-03-04T07:31:16.124741+0000","last_fullsized":"2024-03-04T07:31:16.124741+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:40:01.393384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.1a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123517+0000","last_change":"2024-03-04T07:31:15.132434+0000","last_active":"2024-03-04T07:31:16.123517+0000","last_peered":"2024-03-04T07:31:16.123517+0000","last_clean":"2024-03-04T07:31:16.123517+0000","last_became_active":"2024-03-04T07:31:15.132156+0000","last_became_peered":"2024-03-04T07:31:15.132156+0000","last_unstale":"2024-03-04T07:31:16.123517+0000","last_undegraded":"2024-03-04T07:31:16.123517+0000","last_fullsized":"2024-03-04T07:31:16.123517+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:58:11.642356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1b","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176941+0000","last_change":"2024-03-04T07:31:14.129194+0000","last_active":"2024-03-04T07:31:16.176941+0000","last_peered":"2024-03-04T07:31:16.176941+0000","last_clean":"2024-03-04T07:31:16.176941+0000","last_became_active":"2024-03-04T07:31:14.128872+0000","last_became_peered":"2024-03-04T07:31:14.128872+0000","last_unstale":"2024-03-04T07:31:16.176941+0000","last_undegraded":"2024-03-04T07:31:16.176941+0000","last_fullsized":"2024-03-04T07:31:16.176941+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:27.130702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177477+0000","last_change":"2024-03-04T07:31:15.134189+0000","last_active":"2024-03-04T07:31:16.177477+0000","last_peered":"2024-03-04T07:31:16.177477+0000","last_clean":"2024-03-04T07:31:16.177477+0000","last_became_active":"2024-03-04T07:31:15.133733+0000","last_became_peered":"2024-03-04T07:31:15.133733+0000","last_unstale":"2024-03-04T07:31:16.177477+0000","last_undegraded":"2024-03-04T07:31:16.177477+0000","last_fullsized":"2024-03-04T07:31:16.177477+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:23:48.369845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1a","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124285+0000","last_change":"2024-03-04T07:31:14.129884+0000","last_active":"2024-03-04T07:31:16.124285+0000","last_peered":"2024-03-04T07:31:16.124285+0000","last_clean":"2024-03-04T07:31:16.124285+0000","last_became_active":"2024-03-04T07:31:14.129592+0000","last_became_peered":"2024-03-04T07:31:14.129592+0000","last_unstale":"2024-03-04T07:31:16.124285+0000","last_undegraded":"2024-03-04T07:31:16.124285+0000","last_fullsized":"2024-03-04T07:31:16.124285+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:58.727794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.18","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121821+0000","last_change":"2024-03-04T07:31:15.123097+0000","last_active":"2024-03-04T07:31:16.121821+0000","last_peered":"2024-03-04T07:31:16.121821+0000","last_clean":"2024-03-04T07:31:16.121821+0000","last_became_active":"2024-03-04T07:31:15.122827+0000","last_became_peered":"2024-03-04T07:31:15.122827+0000","last_unstale":"2024-03-04T07:31:16.121821+0000","last_undegraded":"2024-03-04T07:31:16.121821+0000","last_fullsized":"2024-03-04T07:31:16.121821+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:11:43.834572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.19","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177616+0000","last_change":"2024-03-04T07:31:14.131225+0000","last_active":"2024-03-04T07:31:16.177616+0000","last_peered":"2024-03-04T07:31:16.177616+0000","last_clean":"2024-03-04T07:31:16.177616+0000","last_became_active":"2024-03-04T07:31:14.130880+0000","last_became_peered":"2024-03-04T07:31:14.130880+0000","last_unstale":"2024-03-04T07:31:16.177616+0000","last_undegraded":"2024-03-04T07:31:16.177616+0000","last_fullsized":"2024-03-04T07:31:16.177616+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:08:27.793757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.19","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123785+0000","last_change":"2024-03-04T07:31:15.129803+0000","last_active":"2024-03-04T07:31:16.123785+0000","last_peered":"2024-03-04T07:31:16.123785+0000","last_clean":"2024-03-04T07:31:16.123785+0000","last_became_active":"2024-03-04T07:31:15.129441+0000","last_became_peered":"2024-03-04T07:31:15.129441+0000","last_unstale":"2024-03-04T07:31:16.123785+0000","last_undegraded":"2024-03-04T07:31:16.123785+0000","last_fullsized":"2024-03-04T07:31:16.123785+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:01:42.622168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.18","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176877+0000","last_change":"2024-03-04T07:31:14.135162+0000","last_active":"2024-03-04T07:31:16.176877+0000","last_peered":"2024-03-04T07:31:16.176877+0000","last_clean":"2024-03-04T07:31:16.176877+0000","last_became_active":"2024-03-04T07:31:14.134870+0000","last_became_peered":"2024-03-04T07:31:14.134870+0000","last_unstale":"2024-03-04T07:31:16.176877+0000","last_undegraded":"2024-03-04T07:31:16.176877+0000","last_fullsized":"2024-03-04T07:31:16.176877+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:49:49.640760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.16","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124763+0000","last_change":"2024-03-04T07:31:15.116159+0000","last_active":"2024-03-04T07:31:16.124763+0000","last_peered":"2024-03-04T07:31:16.124763+0000","last_clean":"2024-03-04T07:31:16.124763+0000","last_became_active":"2024-03-04T07:31:15.115861+0000","last_became_peered":"2024-03-04T07:31:15.115861+0000","last_unstale":"2024-03-04T07:31:16.124763+0000","last_undegraded":"2024-03-04T07:31:16.124763+0000","last_fullsized":"2024-03-04T07:31:16.124763+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:01.876883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.17","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177227+0000","last_change":"2024-03-04T07:31:14.129589+0000","last_active":"2024-03-04T07:31:16.177227+0000","last_peered":"2024-03-04T07:31:16.177227+0000","last_clean":"2024-03-04T07:31:16.177227+0000","last_became_active":"2024-03-04T07:31:14.129195+0000","last_became_peered":"2024-03-04T07:31:14.129195+0000","last_unstale":"2024-03-04T07:31:16.177227+0000","last_undegraded":"2024-03-04T07:31:16.177227+0000","last_fullsized":"2024-03-04T07:31:16.177227+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:34:30.998711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.17","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124168+0000","last_change":"2024-03-04T07:31:15.125869+0000","last_active":"2024-03-04T07:31:16.124168+0000","last_peered":"2024-03-04T07:31:16.124168+0000","last_clean":"2024-03-04T07:31:16.124168+0000","last_became_active":"2024-03-04T07:31:15.124885+0000","last_became_peered":"2024-03-04T07:31:15.124885+0000","last_unstale":"2024-03-04T07:31:16.124168+0000","last_undegraded":"2024-03-04T07:31:16.124168+0000","last_fullsized":"2024-03-04T07:31:16.124168+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:23:48.369845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122357+0000","last_change":"2024-03-04T07:31:14.129809+0000","last_active":"2024-03-04T07:31:16.122357+0000","last_peered":"2024-03-04T07:31:16.122357+0000","last_clean":"2024-03-04T07:31:16.122357+0000","last_became_active":"2024-03-04T07:31:14.129521+0000","last_became_peered":"2024-03-04T07:31:14.129521+0000","last_unstale":"2024-03-04T07:31:16.122357+0000","last_undegraded":"2024-03-04T07:31:16.122357+0000","last_fullsized":"2024-03-04T07:31:16.122357+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:40.635424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.14","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178000+0000","last_change":"2024-03-04T07:31:15.124154+0000","last_active":"2024-03-04T07:31:16.178000+0000","last_peered":"2024-03-04T07:31:16.178000+0000","last_clean":"2024-03-04T07:31:16.178000+0000","last_became_active":"2024-03-04T07:31:15.123765+0000","last_became_peered":"2024-03-04T07:31:15.123765+0000","last_unstale":"2024-03-04T07:31:16.178000+0000","last_undegraded":"2024-03-04T07:31:16.178000+0000","last_fullsized":"2024-03-04T07:31:16.178000+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.15","version":"89'3","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178140+0000","last_change":"2024-03-04T07:31:14.132032+0000","last_active":"2024-03-04T07:31:16.178140+0000","last_peered":"2024-03-04T07:31:16.178140+0000","last_clean":"2024-03-04T07:31:16.178140+0000","last_became_active":"2024-03-04T07:31:14.131696+0000","last_became_peered":"2024-03-04T07:31:14.131696+0000","last_unstale":"2024-03-04T07:31:16.178140+0000","last_undegraded":"2024-03-04T07:31:16.178140+0000","last_fullsized":"2024-03-04T07:31:16.178140+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:56.524264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.15","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177879+0000","last_change":"2024-03-04T07:31:15.137243+0000","last_active":"2024-03-04T07:31:16.177879+0000","last_peered":"2024-03-04T07:31:16.177879+0000","last_clean":"2024-03-04T07:31:16.177879+0000","last_became_active":"2024-03-04T07:31:15.137000+0000","last_became_peered":"2024-03-04T07:31:15.137000+0000","last_unstale":"2024-03-04T07:31:16.177879+0000","last_undegraded":"2024-03-04T07:31:16.177879+0000","last_fullsized":"2024-03-04T07:31:16.177879+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:00.547485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.14","version":"89'6","reported_seq":20,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123974+0000","last_change":"2024-03-04T07:31:14.134032+0000","last_active":"2024-03-04T07:31:16.123974+0000","last_peered":"2024-03-04T07:31:16.123974+0000","last_clean":"2024-03-04T07:31:16.123974+0000","last_became_active":"2024-03-04T07:31:14.133708+0000","last_became_peered":"2024-03-04T07:31:14.133708+0000","last_unstale":"2024-03-04T07:31:16.123974+0000","last_undegraded":"2024-03-04T07:31:16.123974+0000","last_fullsized":"2024-03-04T07:31:16.123974+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:40:01.393384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.12","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121991+0000","last_change":"2024-03-04T07:31:15.135192+0000","last_active":"2024-03-04T07:31:16.121991+0000","last_peered":"2024-03-04T07:31:16.121991+0000","last_clean":"2024-03-04T07:31:16.121991+0000","last_became_active":"2024-03-04T07:31:15.134853+0000","last_became_peered":"2024-03-04T07:31:15.134853+0000","last_unstale":"2024-03-04T07:31:16.121991+0000","last_undegraded":"2024-03-04T07:31:16.121991+0000","last_fullsized":"2024-03-04T07:31:16.121991+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126880+0000","last_change":"2024-03-04T07:31:14.126745+0000","last_active":"2024-03-04T07:31:16.126880+0000","last_peered":"2024-03-04T07:31:16.126880+0000","last_clean":"2024-03-04T07:31:16.126880+0000","last_became_active":"2024-03-04T07:31:14.126439+0000","last_became_peered":"2024-03-04T07:31:14.126439+0000","last_unstale":"2024-03-04T07:31:16.126880+0000","last_undegraded":"2024-03-04T07:31:16.126880+0000","last_fullsized":"2024-03-04T07:31:16.126880+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:11.062648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.13","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176892+0000","last_change":"2024-03-04T07:31:15.129119+0000","last_active":"2024-03-04T07:31:16.176892+0000","last_peered":"2024-03-04T07:31:16.176892+0000","last_clean":"2024-03-04T07:31:16.176892+0000","last_became_active":"2024-03-04T07:31:15.128780+0000","last_became_peered":"2024-03-04T07:31:15.128780+0000","last_unstale":"2024-03-04T07:31:16.176892+0000","last_undegraded":"2024-03-04T07:31:16.176892+0000","last_fullsized":"2024-03-04T07:31:16.176892+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:18:59.613735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.12","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126835+0000","last_change":"2024-03-04T07:31:14.134331+0000","last_active":"2024-03-04T07:31:16.126835+0000","last_peered":"2024-03-04T07:31:16.126835+0000","last_clean":"2024-03-04T07:31:16.126835+0000","last_became_active":"2024-03-04T07:31:14.133864+0000","last_became_peered":"2024-03-04T07:31:14.133864+0000","last_unstale":"2024-03-04T07:31:16.126835+0000","last_undegraded":"2024-03-04T07:31:16.126835+0000","last_fullsized":"2024-03-04T07:31:16.126835+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:28:03.119654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.10","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122878+0000","last_change":"2024-03-04T07:31:15.122062+0000","last_active":"2024-03-04T07:31:16.122878+0000","last_peered":"2024-03-04T07:31:16.122878+0000","last_clean":"2024-03-04T07:31:16.122878+0000","last_became_active":"2024-03-04T07:31:15.121765+0000","last_became_peered":"2024-03-04T07:31:15.121765+0000","last_unstale":"2024-03-04T07:31:16.122878+0000","last_undegraded":"2024-03-04T07:31:16.122878+0000","last_fullsized":"2024-03-04T07:31:16.122878+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:45:18.173216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.11","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123924+0000","last_change":"2024-03-04T07:31:14.128784+0000","last_active":"2024-03-04T07:31:16.123924+0000","last_peered":"2024-03-04T07:31:16.123924+0000","last_clean":"2024-03-04T07:31:16.123924+0000","last_became_active":"2024-03-04T07:31:14.128449+0000","last_became_peered":"2024-03-04T07:31:14.128449+0000","last_unstale":"2024-03-04T07:31:16.123924+0000","last_undegraded":"2024-03-04T07:31:16.123924+0000","last_fullsized":"2024-03-04T07:31:16.123924+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:58.727794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.11","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126535+0000","last_change":"2024-03-04T07:31:15.133344+0000","last_active":"2024-03-04T07:31:16.126535+0000","last_peered":"2024-03-04T07:31:16.126535+0000","last_clean":"2024-03-04T07:31:16.126535+0000","last_became_active":"2024-03-04T07:31:15.133003+0000","last_became_peered":"2024-03-04T07:31:15.133003+0000","last_unstale":"2024-03-04T07:31:16.126535+0000","last_undegraded":"2024-03-04T07:31:16.126535+0000","last_fullsized":"2024-03-04T07:31:16.126535+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:58:25.382715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.10","version":"89'8","reported_seq":22,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.176949+0000","last_change":"2024-03-04T07:31:14.133109+0000","last_active":"2024-03-04T07:31:16.176949+0000","last_peered":"2024-03-04T07:31:16.176949+0000","last_clean":"2024-03-04T07:31:16.176949+0000","last_became_active":"2024-03-04T07:31:14.132859+0000","last_became_peered":"2024-03-04T07:31:14.132859+0000","last_unstale":"2024-03-04T07:31:16.176949+0000","last_undegraded":"2024-03-04T07:31:16.176949+0000","last_fullsized":"2024-03-04T07:31:16.176949+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:47:37.553768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.f","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177595+0000","last_change":"2024-03-04T07:31:15.132726+0000","last_active":"2024-03-04T07:31:16.177595+0000","last_peered":"2024-03-04T07:31:16.177595+0000","last_clean":"2024-03-04T07:31:16.177595+0000","last_became_active":"2024-03-04T07:31:15.132433+0000","last_became_peered":"2024-03-04T07:31:15.132433+0000","last_unstale":"2024-03-04T07:31:16.177595+0000","last_undegraded":"2024-03-04T07:31:16.177595+0000","last_fullsized":"2024-03-04T07:31:16.177595+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:23:48.369845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123556+0000","last_change":"2024-03-04T07:31:14.130857+0000","last_active":"2024-03-04T07:31:16.123556+0000","last_peered":"2024-03-04T07:31:16.123556+0000","last_clean":"2024-03-04T07:31:16.123556+0000","last_became_active":"2024-03-04T07:31:14.130500+0000","last_became_peered":"2024-03-04T07:31:14.130500+0000","last_unstale":"2024-03-04T07:31:16.123556+0000","last_undegraded":"2024-03-04T07:31:16.123556+0000","last_fullsized":"2024-03-04T07:31:16.123556+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:16.517725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.c","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123012+0000","last_change":"2024-03-04T07:31:15.121967+0000","last_active":"2024-03-04T07:31:16.123012+0000","last_peered":"2024-03-04T07:31:16.123012+0000","last_clean":"2024-03-04T07:31:16.123012+0000","last_became_active":"2024-03-04T07:31:15.121696+0000","last_became_peered":"2024-03-04T07:31:15.121696+0000","last_unstale":"2024-03-04T07:31:16.123012+0000","last_undegraded":"2024-03-04T07:31:16.123012+0000","last_fullsized":"2024-03-04T07:31:16.123012+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:42:26.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.d","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177917+0000","last_change":"2024-03-04T07:31:14.128357+0000","last_active":"2024-03-04T07:31:16.177917+0000","last_peered":"2024-03-04T07:31:16.177917+0000","last_clean":"2024-03-04T07:31:16.177917+0000","last_became_active":"2024-03-04T07:31:14.128047+0000","last_became_peered":"2024-03-04T07:31:14.128047+0000","last_unstale":"2024-03-04T07:31:16.177917+0000","last_undegraded":"2024-03-04T07:31:16.177917+0000","last_fullsized":"2024-03-04T07:31:16.177917+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:40:01.393384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.d","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178043+0000","last_change":"2024-03-04T07:31:15.139651+0000","last_active":"2024-03-04T07:31:16.178043+0000","last_peered":"2024-03-04T07:31:16.178043+0000","last_clean":"2024-03-04T07:31:16.178043+0000","last_became_active":"2024-03-04T07:31:15.139117+0000","last_became_peered":"2024-03-04T07:31:15.139117+0000","last_unstale":"2024-03-04T07:31:16.178043+0000","last_undegraded":"2024-03-04T07:31:16.178043+0000","last_fullsized":"2024-03-04T07:31:16.178043+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:06:23.000462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124827+0000","last_change":"2024-03-04T07:31:14.127553+0000","last_active":"2024-03-04T07:31:16.124827+0000","last_peered":"2024-03-04T07:31:16.124827+0000","last_clean":"2024-03-04T07:31:16.124827+0000","last_became_active":"2024-03-04T07:31:14.126598+0000","last_became_peered":"2024-03-04T07:31:14.126598+0000","last_unstale":"2024-03-04T07:31:16.124827+0000","last_undegraded":"2024-03-04T07:31:16.124827+0000","last_fullsized":"2024-03-04T07:31:16.124827+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:08:27.793757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.a","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126737+0000","last_change":"2024-03-04T07:31:15.132550+0000","last_active":"2024-03-04T07:31:16.126737+0000","last_peered":"2024-03-04T07:31:16.126737+0000","last_clean":"2024-03-04T07:31:16.126737+0000","last_became_active":"2024-03-04T07:31:15.132038+0000","last_became_peered":"2024-03-04T07:31:15.132038+0000","last_unstale":"2024-03-04T07:31:16.126737+0000","last_undegraded":"2024-03-04T07:31:16.126737+0000","last_fullsized":"2024-03-04T07:31:16.126737+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:54:00.967692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.b","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124458+0000","last_change":"2024-03-04T07:31:14.122956+0000","last_active":"2024-03-04T07:31:16.124458+0000","last_peered":"2024-03-04T07:31:16.124458+0000","last_clean":"2024-03-04T07:31:16.124458+0000","last_became_active":"2024-03-04T07:31:14.122682+0000","last_became_peered":"2024-03-04T07:31:14.122682+0000","last_unstale":"2024-03-04T07:31:16.124458+0000","last_undegraded":"2024-03-04T07:31:16.124458+0000","last_fullsized":"2024-03-04T07:31:16.124458+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:56.524264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.b","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126872+0000","last_change":"2024-03-04T07:31:15.123297+0000","last_active":"2024-03-04T07:31:16.126872+0000","last_peered":"2024-03-04T07:31:16.126872+0000","last_clean":"2024-03-04T07:31:16.126872+0000","last_became_active":"2024-03-04T07:31:15.123022+0000","last_became_peered":"2024-03-04T07:31:15.123022+0000","last_unstale":"2024-03-04T07:31:16.126872+0000","last_undegraded":"2024-03-04T07:31:16.126872+0000","last_fullsized":"2024-03-04T07:31:16.126872+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:41:52.140070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123624+0000","last_change":"2024-03-04T07:31:14.135693+0000","last_active":"2024-03-04T07:31:16.123624+0000","last_peered":"2024-03-04T07:31:16.123624+0000","last_clean":"2024-03-04T07:31:16.123624+0000","last_became_active":"2024-03-04T07:31:14.135302+0000","last_became_peered":"2024-03-04T07:31:14.135302+0000","last_unstale":"2024-03-04T07:31:16.123624+0000","last_undegraded":"2024-03-04T07:31:16.123624+0000","last_fullsized":"2024-03-04T07:31:16.123624+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:12:21.581341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.8","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126485+0000","last_change":"2024-03-04T07:31:15.132190+0000","last_active":"2024-03-04T07:31:16.126485+0000","last_peered":"2024-03-04T07:31:16.126485+0000","last_clean":"2024-03-04T07:31:16.126485+0000","last_became_active":"2024-03-04T07:31:15.131912+0000","last_became_peered":"2024-03-04T07:31:15.131912+0000","last_unstale":"2024-03-04T07:31:16.126485+0000","last_undegraded":"2024-03-04T07:31:16.126485+0000","last_fullsized":"2024-03-04T07:31:16.126485+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:43:35.898650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.122824+0000","last_change":"2024-03-04T07:31:14.134205+0000","last_active":"2024-03-04T07:31:16.122824+0000","last_peered":"2024-03-04T07:31:16.122824+0000","last_clean":"2024-03-04T07:31:16.122824+0000","last_became_active":"2024-03-04T07:31:14.133786+0000","last_became_peered":"2024-03-04T07:31:14.133786+0000","last_unstale":"2024-03-04T07:31:16.122824+0000","last_undegraded":"2024-03-04T07:31:16.122824+0000","last_fullsized":"2024-03-04T07:31:16.122824+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:25:52.270706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.9","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177247+0000","last_change":"2024-03-04T07:31:15.127491+0000","last_active":"2024-03-04T07:31:16.177247+0000","last_peered":"2024-03-04T07:31:16.177247+0000","last_clean":"2024-03-04T07:31:16.177247+0000","last_became_active":"2024-03-04T07:31:15.127206+0000","last_became_peered":"2024-03-04T07:31:15.127206+0000","last_unstale":"2024-03-04T07:31:16.177247+0000","last_undegraded":"2024-03-04T07:31:16.177247+0000","last_fullsized":"2024-03-04T07:31:16.177247+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:01:42.622168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124252+0000","last_change":"2024-03-04T07:31:14.133376+0000","last_active":"2024-03-04T07:31:16.124252+0000","last_peered":"2024-03-04T07:31:16.124252+0000","last_clean":"2024-03-04T07:31:16.124252+0000","last_became_active":"2024-03-04T07:31:14.133082+0000","last_became_peered":"2024-03-04T07:31:14.133082+0000","last_unstale":"2024-03-04T07:31:16.124252+0000","last_undegraded":"2024-03-04T07:31:16.124252+0000","last_fullsized":"2024-03-04T07:31:16.124252+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:41:16.517725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.6","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124899+0000","last_change":"2024-03-04T07:31:15.120309+0000","last_active":"2024-03-04T07:31:16.124899+0000","last_peered":"2024-03-04T07:31:16.124899+0000","last_clean":"2024-03-04T07:31:16.124899+0000","last_became_active":"2024-03-04T07:31:15.120028+0000","last_became_peered":"2024-03-04T07:31:15.120028+0000","last_unstale":"2024-03-04T07:31:16.124899+0000","last_undegraded":"2024-03-04T07:31:16.124899+0000","last_fullsized":"2024-03-04T07:31:16.124899+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:24:19.440520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.7","version":"89'2","reported_seq":16,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124150+0000","last_change":"2024-03-04T07:31:14.125768+0000","last_active":"2024-03-04T07:31:16.124150+0000","last_peered":"2024-03-04T07:31:16.124150+0000","last_clean":"2024-03-04T07:31:16.124150+0000","last_became_active":"2024-03-04T07:31:14.125467+0000","last_became_peered":"2024-03-04T07:31:14.125467+0000","last_unstale":"2024-03-04T07:31:16.124150+0000","last_undegraded":"2024-03-04T07:31:16.124150+0000","last_fullsized":"2024-03-04T07:31:16.124150+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:18:58.611867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.7","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177206+0000","last_change":"2024-03-04T07:31:15.125973+0000","last_active":"2024-03-04T07:31:16.177206+0000","last_peered":"2024-03-04T07:31:16.177206+0000","last_clean":"2024-03-04T07:31:16.177206+0000","last_became_active":"2024-03-04T07:31:15.125537+0000","last_became_peered":"2024-03-04T07:31:15.125537+0000","last_unstale":"2024-03-04T07:31:16.177206+0000","last_undegraded":"2024-03-04T07:31:16.177206+0000","last_fullsized":"2024-03-04T07:31:16.177206+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:58:11.642356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"90'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126022+0000","last_change":"2024-03-04T07:31:14.123270+0000","last_active":"2024-03-04T07:31:16.126022+0000","last_peered":"2024-03-04T07:31:16.126022+0000","last_clean":"2024-03-04T07:31:16.126022+0000","last_became_active":"2024-03-04T07:31:14.122998+0000","last_became_peered":"2024-03-04T07:31:14.122998+0000","last_unstale":"2024-03-04T07:31:16.126022+0000","last_undegraded":"2024-03-04T07:31:16.126022+0000","last_fullsized":"2024-03-04T07:31:16.126022+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:05:02.289401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.4","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177101+0000","last_change":"2024-03-04T07:31:15.124791+0000","last_active":"2024-03-04T07:31:16.177101+0000","last_peered":"2024-03-04T07:31:16.177101+0000","last_clean":"2024-03-04T07:31:16.177101+0000","last_became_active":"2024-03-04T07:31:15.124340+0000","last_became_peered":"2024-03-04T07:31:15.124340+0000","last_unstale":"2024-03-04T07:31:16.177101+0000","last_undegraded":"2024-03-04T07:31:16.177101+0000","last_fullsized":"2024-03-04T07:31:16.177101+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:18:52.983145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124481+0000","last_change":"2024-03-04T07:31:14.128646+0000","last_active":"2024-03-04T07:31:16.124481+0000","last_peered":"2024-03-04T07:31:16.124481+0000","last_clean":"2024-03-04T07:31:16.124481+0000","last_became_active":"2024-03-04T07:31:14.128385+0000","last_became_peered":"2024-03-04T07:31:14.128385+0000","last_unstale":"2024-03-04T07:31:16.124481+0000","last_undegraded":"2024-03-04T07:31:16.124481+0000","last_fullsized":"2024-03-04T07:31:16.124481+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:55.061136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.5","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177918+0000","last_change":"2024-03-04T07:31:15.131405+0000","last_active":"2024-03-04T07:31:16.177918+0000","last_peered":"2024-03-04T07:31:16.177918+0000","last_clean":"2024-03-04T07:31:16.177918+0000","last_became_active":"2024-03-04T07:31:15.131121+0000","last_became_peered":"2024-03-04T07:31:15.131121+0000","last_unstale":"2024-03-04T07:31:16.177918+0000","last_undegraded":"2024-03-04T07:31:16.177918+0000","last_fullsized":"2024-03-04T07:31:16.177918+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:06:13.431710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178090+0000","last_change":"2024-03-04T07:31:14.131821+0000","last_active":"2024-03-04T07:31:16.178090+0000","last_peered":"2024-03-04T07:31:16.178090+0000","last_clean":"2024-03-04T07:31:16.178090+0000","last_became_active":"2024-03-04T07:31:14.131489+0000","last_became_peered":"2024-03-04T07:31:14.131489+0000","last_unstale":"2024-03-04T07:31:16.178090+0000","last_undegraded":"2024-03-04T07:31:16.178090+0000","last_fullsized":"2024-03-04T07:31:16.178090+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:55.061136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.2","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126826+0000","last_change":"2024-03-04T07:31:15.134258+0000","last_active":"2024-03-04T07:31:16.126826+0000","last_peered":"2024-03-04T07:31:16.126826+0000","last_clean":"2024-03-04T07:31:16.126826+0000","last_became_active":"2024-03-04T07:31:15.133823+0000","last_became_peered":"2024-03-04T07:31:15.133823+0000","last_unstale":"2024-03-04T07:31:16.126826+0000","last_undegraded":"2024-03-04T07:31:16.126826+0000","last_fullsized":"2024-03-04T07:31:16.126826+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:02:19.871841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.124566+0000","last_change":"2024-03-04T07:31:14.133865+0000","last_active":"2024-03-04T07:31:16.124566+0000","last_peered":"2024-03-04T07:31:16.124566+0000","last_clean":"2024-03-04T07:31:16.124566+0000","last_became_active":"2024-03-04T07:31:14.133589+0000","last_became_peered":"2024-03-04T07:31:14.133589+0000","last_unstale":"2024-03-04T07:31:16.124566+0000","last_undegraded":"2024-03-04T07:31:16.124566+0000","last_fullsized":"2024-03-04T07:31:16.124566+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:26:40.635424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"15.3","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.123287+0000","last_change":"2024-03-04T07:31:15.123764+0000","last_active":"2024-03-04T07:31:16.123287+0000","last_peered":"2024-03-04T07:31:16.123287+0000","last_clean":"2024-03-04T07:31:16.123287+0000","last_became_active":"2024-03-04T07:31:15.123472+0000","last_became_peered":"2024-03-04T07:31:15.123472+0000","last_unstale":"2024-03-04T07:31:16.123287+0000","last_undegraded":"2024-03-04T07:31:16.123287+0000","last_fullsized":"2024-03-04T07:31:16.123287+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:56:17.317135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177239+0000","last_change":"2024-03-04T07:31:14.134964+0000","last_active":"2024-03-04T07:31:16.177239+0000","last_peered":"2024-03-04T07:31:16.177239+0000","last_clean":"2024-03-04T07:31:16.177239+0000","last_became_active":"2024-03-04T07:31:14.134607+0000","last_became_peered":"2024-03-04T07:31:14.134607+0000","last_unstale":"2024-03-04T07:31:16.177239+0000","last_undegraded":"2024-03-04T07:31:16.177239+0000","last_fullsized":"2024-03-04T07:31:16.177239+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:32:58.727794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.0","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126508+0000","last_change":"2024-03-04T07:31:15.133863+0000","last_active":"2024-03-04T07:31:16.126508+0000","last_peered":"2024-03-04T07:31:16.126508+0000","last_clean":"2024-03-04T07:31:16.126508+0000","last_became_active":"2024-03-04T07:31:15.133475+0000","last_became_peered":"2024-03-04T07:31:15.133475+0000","last_unstale":"2024-03-04T07:31:16.126508+0000","last_undegraded":"2024-03-04T07:31:16.126508+0000","last_fullsized":"2024-03-04T07:31:16.126508+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:01.998924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.121894+0000","last_change":"2024-03-04T07:31:14.133663+0000","last_active":"2024-03-04T07:31:16.121894+0000","last_peered":"2024-03-04T07:31:16.121894+0000","last_clean":"2024-03-04T07:31:16.121894+0000","last_became_active":"2024-03-04T07:31:14.133367+0000","last_became_peered":"2024-03-04T07:31:14.133367+0000","last_unstale":"2024-03-04T07:31:16.121894+0000","last_undegraded":"2024-03-04T07:31:16.121894+0000","last_fullsized":"2024-03-04T07:31:16.121894+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:30:41.922144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.1","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.177416+0000","last_change":"2024-03-04T07:31:15.135746+0000","last_active":"2024-03-04T07:31:16.177416+0000","last_peered":"2024-03-04T07:31:16.177416+0000","last_clean":"2024-03-04T07:31:16.177416+0000","last_became_active":"2024-03-04T07:31:15.135455+0000","last_became_peered":"2024-03-04T07:31:15.135455+0000","last_unstale":"2024-03-04T07:31:16.177416+0000","last_undegraded":"2024-03-04T07:31:16.177416+0000","last_fullsized":"2024-03-04T07:31:16.177416+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:38:12.157284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":14,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126587+0000","last_change":"2024-03-04T07:31:14.124436+0000","last_active":"2024-03-04T07:31:16.126587+0000","last_peered":"2024-03-04T07:31:16.126587+0000","last_clean":"2024-03-04T07:31:16.126587+0000","last_became_active":"2024-03-04T07:31:14.124125+0000","last_became_peered":"2024-03-04T07:31:14.124125+0000","last_unstale":"2024-03-04T07:31:16.126587+0000","last_undegraded":"2024-03-04T07:31:16.126587+0000","last_fullsized":"2024-03-04T07:31:16.126587+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:36:54.403600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.e","version":"0'0","reported_seq":12,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.126982+0000","last_change":"2024-03-04T07:31:15.132454+0000","last_active":"2024-03-04T07:31:16.126982+0000","last_peered":"2024-03-04T07:31:16.126982+0000","last_clean":"2024-03-04T07:31:16.126982+0000","last_became_active":"2024-03-04T07:31:15.132200+0000","last_became_peered":"2024-03-04T07:31:15.132200+0000","last_unstale":"2024-03-04T07:31:16.126982+0000","last_undegraded":"2024-03-04T07:31:16.126982+0000","last_fullsized":"2024-03-04T07:31:16.126982+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":88,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:14.106703+0000","last_clean_scrub_stamp":"2024-03-04T07:31:14.106703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:22:28.132569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.f","version":"89'1","reported_seq":15,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178272+0000","last_change":"2024-03-04T07:31:14.133893+0000","last_active":"2024-03-04T07:31:16.178272+0000","last_peered":"2024-03-04T07:31:16.178272+0000","last_clean":"2024-03-04T07:31:16.178272+0000","last_became_active":"2024-03-04T07:31:14.133545+0000","last_became_peered":"2024-03-04T07:31:14.133545+0000","last_unstale":"2024-03-04T07:31:16.178272+0000","last_undegraded":"2024-03-04T07:31:16.178272+0000","last_fullsized":"2024-03-04T07:31:16.178272+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":87,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:31:13.104836+0000","last_clean_scrub_stamp":"2024-03-04T07:31:13.104836+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:55:52.644367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":298,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-04T07:31:16.178208+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:31:16.178208+0000","last_peered":"2024-03-04T07:31:16.178208+0000","last_clean":"2024-03-04T07:31:16.178208+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:31:16.178208+0000","last_undegraded":"2024-03-04T07:31:16.178208+0000","last_fullsized":"2024-03-04T07:31:16.178208+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":15,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":14,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574908,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32844,"kb_used_data":1664,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338996,"statfs":{"total":96636764160,"available":96603131904,"internally_reserved":0,"allocated":1703936,"data_stored":1058930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574907,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32264,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339576,"statfs":{"total":96636764160,"available":96603725824,"internally_reserved":0,"allocated":1110016,"data_stored":469102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31914438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574907,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":468520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574908,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32284,"kb_used_data":1096,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339556,"statfs":{"total":96636764160,"available":96603705344,"internally_reserved":0,"allocated":1122304,"data_stored":470186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574907,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32272,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339568,"statfs":{"total":96636764160,"available":96603717632,"internally_reserved":0,"allocated":1118208,"data_stored":470084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574907,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32260,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339580,"statfs":{"total":96636764160,"available":96603729920,"internally_reserved":0,"allocated":1105920,"data_stored":468484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32256,"kb_used_data":1076,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339584,"statfs":{"total":96636764160,"available":96603734016,"internally_reserved":0,"allocated":1101824,"data_stored":468450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574907,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32844,"kb_used_data":1660,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94338996,"statfs":{"total":96636764160,"available":96603131904,"internally_reserved":0,"allocated":1699840,"data_stored":1059400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:31:18.506 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:31:18.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:31:18.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.651+0000 7fb8abcaf640 1 Processor -- start 2024-03-04T07:31:18.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.651+0000 7fb8abcaf640 1 -- start start 2024-03-04T07:31:18.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8abcaf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb8a40d3770 0x7fb8a4072680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8abcaf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb8a4072cf0 con 0x7fb8a415dd50 2024-03-04T07:31:18.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8abcaf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb8a4072fd0 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8abcaf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb8a40d1f60 con 0x7fb8a40d3770 2024-03-04T07:31:18.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8a9a4b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb8a415dd50 0x7fb8a40ce9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45744/0 (socket says 172.21.15.64:45744) 2024-03-04T07:31:18.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8a9a4b640 1 -- 172.21.15.64:0/3363081057 learned_addr learned my addr 172.21.15.64:0/3363081057 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:18.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8aa24c640 1 --2- 172.21.15.64:0/3363081057 >> v2:172.21.15.66:3300/0 conn(0x7fb8a40d3770 0x7fb8a4072680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8a8a49640 1 -- 172.21.15.64:0/3363081057 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2426912919 0 0) 0x7fb8a4072cf0 con 0x7fb8a415dd50 2024-03-04T07:31:18.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8a8a49640 1 -- 172.21.15.64:0/3363081057 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb88c003520 con 0x7fb8a415dd50 2024-03-04T07:31:18.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8aa24c640 1 -- 172.21.15.64:0/3363081057 >> v1:172.21.15.66:6789/0 conn(0x7fb8a40cf0f0 legacy=0x7fb8a4071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8aa24c640 1 -- 172.21.15.64:0/3363081057 >> v1:172.21.15.64:6789/0 conn(0x7fb8a415dd50 legacy=0x7fb8a40ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.652+0000 7fb8aa24c640 1 -- 172.21.15.64:0/3363081057 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8a40d2240 con 0x7fb8a40d3770 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.653+0000 7fb8aa24c640 1 --2- 172.21.15.64:0/3363081057 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 0x7fb8a4072680 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fb894009d90 tx=0x7fb89402be20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b020becb2b1bc988 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.653+0000 7fb8a8a49640 1 -- 172.21.15.64:0/3363081057 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb894039070 con 0x7fb8a40d3770 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.653+0000 7fb8a8a49640 1 -- 172.21.15.64:0/3363081057 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb89402cbd0 con 0x7fb8a40d3770 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.653+0000 7fb8a8a49640 1 -- 172.21.15.64:0/3363081057 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb894034440 con 0x7fb8a40d3770 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.654+0000 7fb8abcaf640 1 -- 172.21.15.64:0/3363081057 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 msgr2=0x7fb8a4072680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.654+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/3363081057 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 0x7fb8a4072680 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fb894009d90 tx=0x7fb89402be20 comp rx=0 tx=0).stop 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.654+0000 7fb8abcaf640 1 -- 172.21.15.64:0/3363081057 shutdown_connections 2024-03-04T07:31:18.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.654+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/3363081057 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 0x7fb8a4072680 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.654+0000 7fb8abcaf640 1 -- 172.21.15.64:0/3363081057 >> 172.21.15.64:0/3363081057 conn(0x7fb8a40d3ef0 msgr2=0x7fb8a40d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:18.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.654+0000 7fb8abcaf640 1 -- 172.21.15.64:0/3363081057 shutdown_connections 2024-03-04T07:31:18.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.654+0000 7fb8abcaf640 1 -- 172.21.15.64:0/3363081057 wait complete. 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.655+0000 7fb8abcaf640 1 Processor -- start 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.655+0000 7fb8abcaf640 1 -- start start 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8abcaf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8a40cf0f0 0x7fb8a414e420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8abcaf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 0x7fb8a41537a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8abcaf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8a415dd50 0x7fb8a40b7f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8abcaf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8a4168f10 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8abcaf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8a4168d90 con 0x7fb8a40d3770 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8abcaf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb8a4169090 con 0x7fb8a415dd50 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8a40cf0f0 0x7fb8a414e420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8a40cf0f0 0x7fb8a414e420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38762/0 (socket says 172.21.15.64:38762) 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 -- 172.21.15.64:0/662322778 learned_addr learned my addr 172.21.15.64:0/662322778 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a924a640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 0x7fb8a41537a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8aa24c640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8a415dd50 0x7fb8a40b7f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 -- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8a415dd50 msgr2=0x7fb8a40b7f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8a415dd50 0x7fb8a40b7f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 -- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 msgr2=0x7fb8a41537a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 0x7fb8a41537a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 -- 172.21.15.64:0/662322778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8a4160190 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a9a4b640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8a40cf0f0 0x7fb8a414e420 secure :-1 s=READY pgs=1131 cs=0 l=1 rev1=1 crypto rx=0x7fb89c00c530 tx=0x7fb89c00c9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a27fc640 1 -- 172.21.15.64:0/662322778 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb89c017070 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.656+0000 7fb8a27fc640 1 -- 172.21.15.64:0/662322778 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb89c005ce0 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.657+0000 7fb8a27fc640 1 -- 172.21.15.64:0/662322778 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb89c0072b0 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.657+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb8a41614d0 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.657+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb8a4160710 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.657+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb8a4161110 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.658+0000 7fb8a27fc640 1 -- 172.21.15.64:0/662322778 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb89c01a070 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.658+0000 7fb8a27fc640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb89006b500 0x7fb89006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:18.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.659+0000 7fb8a924a640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb89006b500 0x7fb89006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:18.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.659+0000 7fb8a27fc640 1 -- 172.21.15.64:0/662322778 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb89c089510 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.659+0000 7fb8a924a640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb89006b500 0x7fb89006d9c0 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7fb8a40ce5d0 tx=0x7fb898009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:18.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.663+0000 7fb8a27fc640 1 -- 172.21.15.64:0/662322778 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb89c057e00 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.912+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7fb8a406cdd0 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.914+0000 7fb8a27fc640 1 -- 172.21.15.64:0/662322778 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v20) v1 ==== 190+0+147 (secure 0 0 0) 0x7fb89c05bea0 con 0x7fb8a40cf0f0 2024-03-04T07:31:18.914 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:31:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:31:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:31:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:31:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:31:18.914 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:31:18.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb89006b500 msgr2=0x7fb89006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb89006b500 0x7fb89006d9c0 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7fb8a40ce5d0 tx=0x7fb898009040 comp rx=0 tx=0).stop 2024-03-04T07:31:18.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8a40cf0f0 msgr2=0x7fb8a414e420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:18.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8a40cf0f0 0x7fb8a414e420 secure :-1 s=READY pgs=1131 cs=0 l=1 rev1=1 crypto rx=0x7fb89c00c530 tx=0x7fb89c00c9f0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 shutdown_connections 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb89006b500 0x7fb89006d9c0 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8a415dd50 0x7fb8a40b7f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb8a40d3770 0x7fb8a41537a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 --2- 172.21.15.64:0/662322778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8a40cf0f0 0x7fb8a414e420 unknown :-1 s=CLOSED pgs=1131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 >> 172.21.15.64:0/662322778 conn(0x7fb8a40d3ef0 msgr2=0x7fb8a415cac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 shutdown_connections 2024-03-04T07:31:18.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:18.916+0000 7fb8abcaf640 1 -- 172.21.15.64:0/662322778 wait complete. 2024-03-04T07:31:18.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:19.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.029+0000 7f7a76bf6640 1 Processor -- start 2024-03-04T07:31:19.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.029+0000 7f7a76bf6640 1 -- start start 2024-03-04T07:31:19.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a76bf6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7a7015e060 0x7f7a700ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a76bf6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7a700d0dd0 con 0x7f7a7006fe20 2024-03-04T07:31:19.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a76bf6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7a700d10b0 con 0x7f7a700d0880 2024-03-04T07:31:19.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a76bf6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7a700cc0e0 con 0x7f7a7015e060 2024-03-04T07:31:19.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a6ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7a7006fe20 0x7f7a700ce850 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45758/0 (socket says 172.21.15.64:45758) 2024-03-04T07:31:19.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a6ffff640 1 -- 172.21.15.64:0/579770400 learned_addr learned my addr 172.21.15.64:0/579770400 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:19.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a74992640 1 --2- 172.21.15.64:0/579770400 >> v2:172.21.15.66:3300/0 conn(0x7f7a7015e060 0x7f7a700ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a6f7fe640 1 -- 172.21.15.64:0/579770400 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3718578942 0 0) 0x7f7a700d0dd0 con 0x7f7a7006fe20 2024-03-04T07:31:19.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a6f7fe640 1 -- 172.21.15.64:0/579770400 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7a60003520 con 0x7f7a7006fe20 2024-03-04T07:31:19.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a74992640 1 -- 172.21.15.64:0/579770400 >> v1:172.21.15.66:6789/0 conn(0x7f7a700d0880 legacy=0x7f7a700ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a74992640 1 -- 172.21.15.64:0/579770400 >> v1:172.21.15.64:6789/0 conn(0x7f7a7006fe20 legacy=0x7f7a700ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.030+0000 7f7a74992640 1 -- 172.21.15.64:0/579770400 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a700cc3c0 con 0x7f7a7015e060 2024-03-04T07:31:19.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.031+0000 7f7a74992640 1 --2- 172.21.15.64:0/579770400 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 0x7f7a700ce310 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f7a640097a0 tx=0x7f7a6402b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6ad82a8e0044ace3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.031+0000 7f7a6f7fe640 1 -- 172.21.15.64:0/579770400 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a64034070 con 0x7f7a7015e060 2024-03-04T07:31:19.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.031+0000 7f7a6f7fe640 1 -- 172.21.15.64:0/579770400 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a64030610 con 0x7f7a7015e060 2024-03-04T07:31:19.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.031+0000 7f7a6f7fe640 1 -- 172.21.15.64:0/579770400 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a640309c0 con 0x7f7a7015e060 2024-03-04T07:31:19.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.032+0000 7f7a76bf6640 1 -- 172.21.15.64:0/579770400 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 msgr2=0x7f7a700ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.032+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/579770400 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 0x7f7a700ce310 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f7a640097a0 tx=0x7f7a6402b960 comp rx=0 tx=0).stop 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.032+0000 7f7a76bf6640 1 -- 172.21.15.64:0/579770400 shutdown_connections 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.032+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/579770400 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 0x7f7a700ce310 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.032+0000 7f7a76bf6640 1 -- 172.21.15.64:0/579770400 >> 172.21.15.64:0/579770400 conn(0x7f7a700d1a90 msgr2=0x7f7a700d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.032+0000 7f7a76bf6640 1 -- 172.21.15.64:0/579770400 shutdown_connections 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.032+0000 7f7a76bf6640 1 -- 172.21.15.64:0/579770400 wait complete. 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.033+0000 7f7a76bf6640 1 Processor -- start 2024-03-04T07:31:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.033+0000 7f7a76bf6640 1 -- start start 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a76bf6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a7006fe20 0x7f7a70157e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a76bf6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a700d0880 0x7f7a701f49a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a76bf6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 0x7f7a701f7450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a76bf6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a700cef60 con 0x7f7a700d0880 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a76bf6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a7015e440 con 0x7f7a7015e060 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a76bf6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7a700cf0e0 con 0x7f7a7006fe20 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a700d0880 0x7f7a701f49a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a700d0880 0x7f7a701f49a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38766/0 (socket says 172.21.15.64:38766) 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a74992640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a7006fe20 0x7f7a70157e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 -- 172.21.15.64:0/815899158 learned_addr learned my addr 172.21.15.64:0/815899158 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a75193640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 0x7f7a701f7450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 -- 172.21.15.64:0/815899158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a7006fe20 msgr2=0x7f7a70157e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a7006fe20 0x7f7a70157e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 -- 172.21.15.64:0/815899158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 msgr2=0x7f7a701f7450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 0x7f7a701f7450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 -- 172.21.15.64:0/815899158 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a701583a0 con 0x7f7a700d0880 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6ffff640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a700d0880 0x7f7a701f49a0 secure :-1 s=READY pgs=1133 cs=0 l=1 rev1=1 crypto rx=0x7f7a5c00c4d0 tx=0x7f7a5c00c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6d7fa640 1 -- 172.21.15.64:0/815899158 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a5c017070 con 0x7f7a700d0880 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6d7fa640 1 -- 172.21.15.64:0/815899158 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a5c005ce0 con 0x7f7a700d0880 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a6d7fa640 1 -- 172.21.15.64:0/815899158 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a5c007270 con 0x7f7a700d0880 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.034+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a701f3ee0 con 0x7f7a700d0880 2024-03-04T07:31:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.035+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a701f8110 con 0x7f7a700d0880 2024-03-04T07:31:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.036+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7a70067ff0 con 0x7f7a700d0880 2024-03-04T07:31:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.036+0000 7f7a6d7fa640 1 -- 172.21.15.64:0/815899158 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7a5c01a070 con 0x7f7a700d0880 2024-03-04T07:31:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.036+0000 7f7a6d7fa640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5406b4d0 0x7f7a5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.037+0000 7f7a74992640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5406b4d0 0x7f7a5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.037+0000 7f7a6d7fa640 1 -- 172.21.15.64:0/815899158 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7a5c089410 con 0x7f7a700d0880 2024-03-04T07:31:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.037+0000 7f7a74992640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5406b4d0 0x7f7a5406d990 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f7a64036000 tx=0x7f7a64008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.041+0000 7f7a6d7fa640 1 -- 172.21.15.64:0/815899158 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7a5c057cd0 con 0x7f7a700d0880 2024-03-04T07:31:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.262+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7a700d2260 con 0x7f7a700d0880 2024-03-04T07:31:19.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.262+0000 7f7a6d7fa640 1 -- 172.21.15.64:0/815899158 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 50 v50) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f7a5c05bd70 con 0x7f7a700d0880 2024-03-04T07:31:19.262 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:19.262 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":50,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6619,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3500669041","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3500669041},{"type":"v1","addr":"172.21.15.64:6836","nonce":3500669041}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":46},{"gid":6634,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/362862056","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":362862056},{"type":"v1","addr":"172.21.15.66:6834","nonce":362862056}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":47},{"gid":7341,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/3848675976","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":3848675976},{"type":"v1","addr":"172.21.15.64:6837","nonce":3848675976}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":47}],"filesystems":[{"mdsmap":{"epoch":50,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:16.471582+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7353},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7353":{"gid":7353,"name":"b","rank":0,"incarnation":48,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/3397672669","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":3397672669},{"type":"v1","addr":"172.21.15.66:6835","nonce":3397672669}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7353,"qdb_cluster":[7353]},"id":7}]} 2024-03-04T07:31:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 50 2024-03-04T07:31:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5406b4d0 msgr2=0x7f7a5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5406b4d0 0x7f7a5406d990 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f7a64036000 tx=0x7f7a64008030 comp rx=0 tx=0).stop 2024-03-04T07:31:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a700d0880 msgr2=0x7f7a701f49a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a700d0880 0x7f7a701f49a0 secure :-1 s=READY pgs=1133 cs=0 l=1 rev1=1 crypto rx=0x7f7a5c00c4d0 tx=0x7f7a5c00c990 comp rx=0 tx=0).stop 2024-03-04T07:31:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 shutdown_connections 2024-03-04T07:31:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5406b4d0 0x7f7a5406d990 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a7015e060 0x7f7a701f7450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a700d0880 0x7f7a701f49a0 unknown :-1 s=CLOSED pgs=1133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 --2- 172.21.15.64:0/815899158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a7006fe20 0x7f7a70157e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 >> 172.21.15.64:0/815899158 conn(0x7f7a700d1a90 msgr2=0x7f7a700b2300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 shutdown_connections 2024-03-04T07:31:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.264+0000 7f7a76bf6640 1 -- 172.21.15.64:0/815899158 wait complete. 2024-03-04T07:31:19.678 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 50, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:31:14.475450+0000', 'modified': '2024-03-04T07:31:16.471582+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7353}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7353': {'gid': 7353, 'name': 'b', 'rank': 0, 'incarnation': 48, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6835/3397672669', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 3397672669}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 3397672669}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 7353, 'qdb_cluster': [7353]} 2024-03-04T07:31:19.678 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:31:19.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-03-04T07:31:19.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.786+0000 7f0df1c2a640 1 Processor -- start 2024-03-04T07:31:19.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.786+0000 7f0df1c2a640 1 -- start start 2024-03-04T07:31:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0df1c2a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0dec15df70 0x7f0dec0d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0df1c2a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0dec071c50 0x7f0dec0b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0df1c2a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0dec0b6f20 con 0x7f0dec0d18c0 2024-03-04T07:31:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0df1c2a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0dec0d3030 con 0x7f0dec15df70 2024-03-04T07:31:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0df1c2a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0dec0d3310 con 0x7f0dec071c50 2024-03-04T07:31:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0deb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0dec15df70 0x7f0dec0d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0deb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0dec15df70 0x7f0dec0d27b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38770/0 (socket says 172.21.15.64:38770) 2024-03-04T07:31:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0deb7fe640 1 -- 172.21.15.64:0/785853108 learned_addr learned my addr 172.21.15.64:0/785853108 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0debfff640 1 --2- 172.21.15.64:0/785853108 >> v2:172.21.15.66:3300/0 conn(0x7f0dec071c50 0x7f0dec0b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0deb7fe640 1 -- 172.21.15.64:0/785853108 >> v2:172.21.15.66:3300/0 conn(0x7f0dec071c50 msgr2=0x7f0dec0b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/785853108 >> v2:172.21.15.66:3300/0 conn(0x7f0dec071c50 0x7f0dec0b2700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0deb7fe640 1 -- 172.21.15.64:0/785853108 >> v1:172.21.15.64:6789/0 conn(0x7f0dec0d18c0 legacy=0x7f0dec071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.787+0000 7f0deb7fe640 1 -- 172.21.15.64:0/785853108 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0dec0d35f0 con 0x7f0dec15df70 2024-03-04T07:31:19.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.788+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/785853108 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec0d27b0 secure :-1 s=READY pgs=1135 cs=0 l=1 rev1=1 crypto rx=0x7f0dec08d130 tx=0x7f0ddc02ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad45db3d55649734 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.788+0000 7f0dea7fc640 1 -- 172.21.15.64:0/785853108 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ddc02f850 con 0x7f0dec15df70 2024-03-04T07:31:19.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.788+0000 7f0dea7fc640 1 -- 172.21.15.64:0/785853108 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ddc02f9f0 con 0x7f0dec15df70 2024-03-04T07:31:19.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.788+0000 7f0dea7fc640 1 -- 172.21.15.64:0/785853108 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ddc02fdc0 con 0x7f0dec15df70 2024-03-04T07:31:19.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 -- 172.21.15.64:0/785853108 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 msgr2=0x7f0dec0d27b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/785853108 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec0d27b0 secure :-1 s=READY pgs=1135 cs=0 l=1 rev1=1 crypto rx=0x7f0dec08d130 tx=0x7f0ddc02ba30 comp rx=0 tx=0).stop 2024-03-04T07:31:19.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 -- 172.21.15.64:0/785853108 shutdown_connections 2024-03-04T07:31:19.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/785853108 >> v2:172.21.15.66:3300/0 conn(0x7f0dec071c50 0x7f0dec0b2700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/785853108 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec0d27b0 unknown :-1 s=CLOSED pgs=1135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 -- 172.21.15.64:0/785853108 >> 172.21.15.64:0/785853108 conn(0x7f0dec0d3cf0 msgr2=0x7f0dec0d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 -- 172.21.15.64:0/785853108 shutdown_connections 2024-03-04T07:31:19.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.789+0000 7f0df1c2a640 1 -- 172.21.15.64:0/785853108 wait complete. 2024-03-04T07:31:19.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.790+0000 7f0df1c2a640 1 Processor -- start 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.790+0000 7f0df1c2a640 1 -- start start 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0df1c2a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0dec071c50 0x7f0dec1552c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0df1c2a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0dec0d18c0 0x7f0dec0ce670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0df1c2a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec1579f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0df1c2a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec15e350 con 0x7f0dec15df70 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0df1c2a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec0d2cf0 con 0x7f0dec0d18c0 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0df1c2a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec15e4d0 con 0x7f0dec071c50 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec1579f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec1579f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38786/0 (socket says 172.21.15.64:38786) 2024-03-04T07:31:19.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 -- 172.21.15.64:0/670562562 learned_addr learned my addr 172.21.15.64:0/670562562 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0dec071c50 0x7f0dec1552c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0deaffd640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0dec0d18c0 0x7f0dec0ce670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 -- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0dec071c50 msgr2=0x7f0dec1552c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0dec071c50 0x7f0dec1552c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 -- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0dec0d18c0 msgr2=0x7f0dec0ce670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0dec0d18c0 0x7f0dec0ce670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 -- 172.21.15.64:0/670562562 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0dec145f60 con 0x7f0dec15df70 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0debfff640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec1579f0 secure :-1 s=READY pgs=1136 cs=0 l=1 rev1=1 crypto rx=0x7f0de400efc0 tx=0x7f0de400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0dd3fff640 1 -- 172.21.15.64:0/670562562 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de4018020 con 0x7f0dec15df70 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0dd3fff640 1 -- 172.21.15.64:0/670562562 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0de4014480 con 0x7f0dec15df70 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0dd3fff640 1 -- 172.21.15.64:0/670562562 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de4014870 con 0x7f0dec15df70 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.791+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0dec146240 con 0x7f0dec15df70 2024-03-04T07:31:19.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.792+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0dec146800 con 0x7f0dec15df70 2024-03-04T07:31:19.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.793+0000 7f0dd3fff640 1 -- 172.21.15.64:0/670562562 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0de4005d10 con 0x7f0dec15df70 2024-03-04T07:31:19.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.794+0000 7f0dd3fff640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dd806b4d0 0x7f0dd806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.794+0000 7f0dd3fff640 1 -- 172.21.15.64:0/670562562 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0de408af10 con 0x7f0dec15df70 2024-03-04T07:31:19.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.794+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dd806b4d0 0x7f0dd806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.794+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dd806b4d0 0x7f0dd806d990 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f0ddc036000 tx=0x7f0ddc008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 Processor -- start 2024-03-04T07:31:19.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 -- start start 2024-03-04T07:31:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c550870 0x561e5c550c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c5548c0 0x561e5c556360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x561e5c4ba6b0 con 0x561e5c5548c0 2024-03-04T07:31:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x561e5c463a60 con 0x561e5c550870 2024-03-04T07:31:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x561e5c4b6750 con 0x561e5c580600 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.795+0000 7f0deaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c5548c0 0x561e5c556360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c5548c0 0x561e5c556360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38796/0 (socket says 172.21.15.64:38796) 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deb7fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c550870 0x561e5c550c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 -- 172.21.15.64:0/3892526551 learned_addr learned my addr 172.21.15.64:0/3892526551 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0debfff640 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 -- 172.21.15.64:0/3892526551 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 msgr2=0x561e5c580a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 -- 172.21.15.64:0/3892526551 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c550870 msgr2=0x561e5c550c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c550870 0x561e5c550c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 -- 172.21.15.64:0/3892526551 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561e5c481f30 con 0x561e5c5548c0 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0deaffd640 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c5548c0 0x561e5c556360 secure :-1 s=READY pgs=1137 cs=0 l=1 rev1=1 crypto rx=0x7f0de0001860 tx=0x7f0de0007e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd38a153ecf011b0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0dd17fa640 1 -- 172.21.15.64:0/3892526551 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de00052e0 con 0x561e5c5548c0 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0dd17fa640 1 -- 172.21.15.64:0/3892526551 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0de0018070 con 0x561e5c5548c0 2024-03-04T07:31:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.796+0000 7f0dd17fa640 1 -- 172.21.15.64:0/3892526551 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de000fab0 con 0x561e5c5548c0 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 -- 172.21.15.64:0/3892526551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c5548c0 msgr2=0x561e5c556360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c5548c0 0x561e5c556360 secure :-1 s=READY pgs=1137 cs=0 l=1 rev1=1 crypto rx=0x7f0de0001860 tx=0x7f0de0007e50 comp rx=0 tx=0).stop 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 -- 172.21.15.64:0/3892526551 shutdown_connections 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c5548c0 0x561e5c556360 unknown :-1 s=CLOSED pgs=1137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 --2- 172.21.15.64:0/3892526551 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c550870 0x561e5c550c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 -- 172.21.15.64:0/3892526551 >> 172.21.15.64:0/3892526551 conn(0x561e5c5528f0 msgr2=0x561e5c5541b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 -- 172.21.15.64:0/3892526551 shutdown_connections 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 -- 172.21.15.64:0/3892526551 wait complete. 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.797+0000 7f0df4812740 1 -- start start 2024-03-04T07:31:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 1 Processor -- start 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c59a8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57aed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c583350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x561e5c554d50 con 0x561e5c57b4b0 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x561e5c476ea0 con 0x561e5c582ed0 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x561e5c555030 con 0x561e5c580600 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0df4812740 10 client.0 _ll_register_callbacks cb 0x561e5c476870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0deaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57aed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0deaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57aed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38812/0 (socket says 172.21.15.64:38812) 2024-03-04T07:31:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0deaffd640 1 -- 172.21.15.64:0/1404051806 learned_addr learned my addr 172.21.15.64:0/1404051806 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0debfff640 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c583350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.798+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c59a8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0deaffd640 1 -- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 msgr2=0x561e5c59a8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0deaffd640 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c59a8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0deaffd640 1 -- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 msgr2=0x561e5c583350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0deaffd640 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c583350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0deaffd640 1 -- 172.21.15.64:0/1404051806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x561e5c477270 con 0x561e5c57b4b0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0deaffd640 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57aed0 secure :-1 s=READY pgs=1138 cs=0 l=1 rev1=1 crypto rx=0x7f0de0014000 tx=0x7f0de0000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=422a13972831391d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0dca7fc640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0dca7fc640 1 -- 172.21.15.64:0/1404051806 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0de000fc80 con 0x561e5c57b4b0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0dca7fc640 1 -- 172.21.15.64:0/1404051806 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0de00035f0 con 0x561e5c57b4b0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0de0018070 con 0x561e5c57b4b0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0dca7fc640 1 -- 172.21.15.64:0/1404051806 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=50) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0de000fe20 con 0x561e5c57b4b0 2024-03-04T07:31:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0df4812740 10 client.7485 fetch_fsmap learned FSMap version 50 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=50,monmap=2+}) v3 -- 0x561e5c1cf2d0 con 0x561e5c57b4b0 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0dca7fc640 1 -- 172.21.15.64:0/1404051806 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0de0018070 con 0x561e5c57b4b0 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0dca7fc640 1 -- 172.21.15.64:0/1404051806 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 50) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f0de00037f0 con 0x561e5c57b4b0 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0df4812740 10 client.7485 fetch_fsmap finished waiting for FSMap version 50 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0df4812740 10 client.7485 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.799+0000 7f0df4812740 20 client.7485 populate_metadata read hostname 'smithi064' 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.800+0000 7f0df4812740 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c596370 0x561e5c5967b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.800+0000 7f0df4812740 4 client.7485 mds_command: new command op to 7353 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.800+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 --> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x561e5c4c7bf0 con 0x561e5c596370 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.800+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c596370 0x561e5c5967b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.800+0000 7f0dca7fc640 10 client.7485 ms_handle_connect on v2:172.21.15.66:6833/3397672669 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.800+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c596370 0x561e5c5967b0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=141eaa7502177b1c server_cookie=ba227938f2020198 in_seq=0 out_seq=0 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0dca7fc640 1 -- 172.21.15.64:0/1404051806 <== mds.0 v2:172.21.15.66:6833/3397672669 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x561e5c4c7bf0 con 0x561e5c596370 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0dca7fc640 10 client.7485 handle_command_reply: tid=0 2024-03-04T07:31:19.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 client.7485 shutdown 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 msgr2=0x561e5c57aed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57aed0 secure :-1 s=READY pgs=1138 cs=0 l=1 rev1=1 crypto rx=0x7f0de0014000 tx=0x7f0de0000bc0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 shutdown_connections 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c596370 0x561e5c5967b0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c583350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57aed0 unknown :-1 s=CLOSED pgs=1138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0dca7fc640 0 client.7485 ms_handle_reset on v2:172.21.15.66:6833/3397672669 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 --2- 172.21.15.64:0/1404051806 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c59a8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 >> 172.21.15.64:0/1404051806 conn(0x561e5c550870 msgr2=0x561e5c57da40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 shutdown_connections 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 1 -- 172.21.15.64:0/1404051806 wait complete. 2024-03-04T07:31:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.801+0000 7f0df4812740 20 client.7485 trim_cache size 0 max 16384 2024-03-04T07:31:19.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.818+0000 7f0df4812740 1 Processor -- start 2024-03-04T07:31:19.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.818+0000 7f0df4812740 1 -- start start 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0df4812740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57c650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c554080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0df4812740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec148ac0 con 0x561e5c57b4b0 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec147ec0 con 0x561e5c582ed0 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0dec1484c0 con 0x561e5c580600 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57c650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57c650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38826/0 (socket says 172.21.15.64:38826) 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 -- 172.21.15.64:0/3866871770 learned_addr learned my addr 172.21.15.64:0/3866871770 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0debfff640 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c554080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deaffd640 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 -- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 msgr2=0x561e5c580f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 -- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 msgr2=0x561e5c554080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c554080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 -- 172.21.15.64:0/3866871770 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561e5c2996b0 con 0x561e5c57b4b0 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0debfff640 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c554080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57c650 secure :-1 s=READY pgs=1139 cs=0 l=1 rev1=1 crypto rx=0x7f0ddc02c040 tx=0x7f0ddc030000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4c2f5d08c24d92b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0dd0ff9640 1 -- 172.21.15.64:0/3866871770 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ddc057020 con 0x561e5c57b4b0 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0dd0ff9640 1 -- 172.21.15.64:0/3866871770 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ddc053070 con 0x561e5c57b4b0 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.819+0000 7f0dd0ff9640 1 -- 172.21.15.64:0/3866871770 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ddc042890 con 0x561e5c57b4b0 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 -- 172.21.15.64:0/3866871770 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 msgr2=0x561e5c57c650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57c650 secure :-1 s=READY pgs=1139 cs=0 l=1 rev1=1 crypto rx=0x7f0ddc02c040 tx=0x7f0ddc030000 comp rx=0 tx=0).stop 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 -- 172.21.15.64:0/3866871770 shutdown_connections 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c582ed0 0x561e5c554080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c580600 0x561e5c580f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 --2- 172.21.15.64:0/3866871770 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c57c650 unknown :-1 s=CLOSED pgs=1139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 -- 172.21.15.64:0/3866871770 >> 172.21.15.64:0/3866871770 conn(0x561e5c550870 msgr2=0x561e5c584120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 -- 172.21.15.64:0/3866871770 shutdown_connections 2024-03-04T07:31:19.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.820+0000 7f0df4812740 1 -- 172.21.15.64:0/3866871770 wait complete. 2024-03-04T07:31:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 -- start start 2024-03-04T07:31:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 Processor -- start 2024-03-04T07:31:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c5f4a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c580600 0x561e5c554f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c582ed0 0x561e5c579340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x561e5c554d50 con 0x561e5c57b4b0 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x561e5c57bcb0 con 0x561e5c580600 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x561e5c596e00 con 0x561e5c582ed0 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0df4812740 10 client.0 _ll_register_callbacks cb 0x7f0de00016d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0deb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c5f4a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0deb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c5f4a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38832/0 (socket says 172.21.15.64:38832) 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.821+0000 7f0deb7fe640 1 -- 172.21.15.64:0/839478107 learned_addr learned my addr 172.21.15.64:0/839478107 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0deaffd640 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c580600 0x561e5c554f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0debfff640 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c582ed0 0x561e5c579340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0deb7fe640 1 -- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c582ed0 msgr2=0x561e5c579340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c582ed0 0x561e5c579340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0deb7fe640 1 -- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c580600 msgr2=0x561e5c554f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c580600 0x561e5c554f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0deb7fe640 1 -- 172.21.15.64:0/839478107 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x561e5c57df20 con 0x561e5c57b4b0 2024-03-04T07:31:19.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0deb7fe640 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c5f4a20 secure :-1 s=READY pgs=1140 cs=0 l=1 rev1=1 crypto rx=0x7f0ddc02ba70 tx=0x7f0ddc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7f2911fac6146e3c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0dca7fc640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0dca7fc640 1 -- 172.21.15.64:0/839478107 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ddc057040 con 0x561e5c57b4b0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0dca7fc640 1 -- 172.21.15.64:0/839478107 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0ddc042d60 con 0x561e5c57b4b0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0ddc053070 con 0x561e5c57b4b0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0dca7fc640 1 -- 172.21.15.64:0/839478107 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=50) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0ddc063050 con 0x561e5c57b4b0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0df4812740 10 client.7491 fetch_fsmap learned FSMap version 50 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=50,monmap=2+}) v3 -- 0x561e5c579880 con 0x561e5c57b4b0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.822+0000 7f0dca7fc640 1 -- 172.21.15.64:0/839478107 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ddc053070 con 0x561e5c57b4b0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0dca7fc640 1 -- 172.21.15.64:0/839478107 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 50) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f0ddc066020 con 0x561e5c57b4b0 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 10 client.7491 fetch_fsmap finished waiting for FSMap version 50 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 10 client.7491 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 20 client.7491 populate_metadata read hostname 'smithi064' 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c60c510 0x561e5c617060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 4 client.7491 mds_command: new command op to 7353 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 --> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] -- command(tid 0: {"prefix": "status"}) v1 -- 0x561e5c4c1f40 con 0x561e5c60c510 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0deaffd640 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c60c510 0x561e5c617060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0dca7fc640 10 client.7491 ms_handle_connect on v2:172.21.15.66:6833/3397672669 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0deaffd640 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c60c510 0x561e5c617060 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d3f36e08d5afd380 server_cookie=7edb4466df679aa in_seq=0 out_seq=0 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0dca7fc640 1 -- 172.21.15.64:0/839478107 <== mds.0 v2:172.21.15.66:6833/3397672669 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x561e5c4c1f40 con 0x561e5c60c510 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0dca7fc640 10 client.7491 handle_command_reply: tid=0 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 client.7491 shutdown 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 msgr2=0x561e5c5f4a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c5f4a20 secure :-1 s=READY pgs=1140 cs=0 l=1 rev1=1 crypto rx=0x7f0ddc02ba70 tx=0x7f0ddc004730 comp rx=0 tx=0).stop 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 shutdown_connections 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:6833/3397672669,v1:172.21.15.66:6835/3397672669] conn(0x561e5c60c510 0x561e5c617060 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x561e5c582ed0 0x561e5c579340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0dca7fc640 0 client.7491 ms_handle_reset on v2:172.21.15.66:6833/3397672669 2024-03-04T07:31:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x561e5c580600 0x561e5c554f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 --2- 172.21.15.64:0/839478107 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x561e5c57b4b0 0x561e5c5f4a20 unknown :-1 s=CLOSED pgs=1140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.823+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 >> 172.21.15.64:0/839478107 conn(0x561e5c550870 msgr2=0x561e5c554080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 shutdown_connections 2024-03-04T07:31:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df4812740 1 -- 172.21.15.64:0/839478107 wait complete. 2024-03-04T07:31:19.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df4812740 20 client.7491 trim_cache size 0 max 16384 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 7353, 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 4.7067079959999996, 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 50, 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 90, 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 90, 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 9.7025506920000009 2024-03-04T07:31:19.830 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:31:19.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dd806b4d0 msgr2=0x7f0dd806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dd806b4d0 0x7f0dd806d990 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f0ddc036000 tx=0x7f0ddc008030 comp rx=0 tx=0).stop 2024-03-04T07:31:19.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 msgr2=0x7f0dec1579f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec1579f0 secure :-1 s=READY pgs=1136 cs=0 l=1 rev1=1 crypto rx=0x7f0de400efc0 tx=0x7f0de400c490 comp rx=0 tx=0).stop 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 shutdown_connections 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0dd806b4d0 0x7f0dd806d990 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0dec15df70 0x7f0dec1579f0 unknown :-1 s=CLOSED pgs=1136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0dec0d18c0 0x7f0dec0ce670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 --2- 172.21.15.64:0/670562562 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0dec071c50 0x7f0dec1552c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 >> 172.21.15.64:0/670562562 conn(0x7f0dec0d3cf0 msgr2=0x7f0dec0d1d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 shutdown_connections 2024-03-04T07:31:19.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.824+0000 7f0df1c2a640 1 -- 172.21.15.64:0/670562562 wait complete. 2024-03-04T07:31:19.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:31:19.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.937+0000 7fb4efedf640 1 Processor -- start 2024-03-04T07:31:19.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.937+0000 7fb4efedf640 1 -- start start 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4efedf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4e815e060 0x7fb4e80ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4efedf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb4e806fe20 0x7fb4e80ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4efedf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e80d0880 0x7fb4e80ced90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4efedf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4e80d0dd0 con 0x7fb4e815e060 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4efedf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4e80d10b0 con 0x7fb4e806fe20 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4efedf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4e80cc0e0 con 0x7fb4e80d0880 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4e815e060 0x7fb4e80ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4e815e060 0x7fb4e80ce310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38846/0 (socket says 172.21.15.64:38846) 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 -- 172.21.15.64:0/2974268696 learned_addr learned my addr 172.21.15.64:0/2974268696 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4ee47c640 1 --2- 172.21.15.64:0/2974268696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e80d0880 0x7fb4e80ced90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4ed47a640 1 --2- 172.21.15.64:0/2974268696 >> v2:172.21.15.66:3300/0 conn(0x7fb4e806fe20 0x7fb4e80ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 -- 172.21.15.64:0/2974268696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e80d0880 msgr2=0x7fb4e80ced90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 --2- 172.21.15.64:0/2974268696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e80d0880 0x7fb4e80ced90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 -- 172.21.15.64:0/2974268696 >> v2:172.21.15.66:3300/0 conn(0x7fb4e806fe20 msgr2=0x7fb4e80ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:19.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 --2- 172.21.15.64:0/2974268696 >> v2:172.21.15.66:3300/0 conn(0x7fb4e806fe20 0x7fb4e80ce850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.938+0000 7fb4edc7b640 1 -- 172.21.15.64:0/2974268696 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4e80cc3c0 con 0x7fb4e815e060 2024-03-04T07:31:19.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.939+0000 7fb4edc7b640 1 --2- 172.21.15.64:0/2974268696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e815e060 0x7fb4e80ce310 secure :-1 s=READY pgs=1141 cs=0 l=1 rev1=1 crypto rx=0x7fb4e8068cd0 tx=0x7fb4e402b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1619909f7865c4ed server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.939+0000 7fb4ecc79640 1 -- 172.21.15.64:0/2974268696 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4e4035020 con 0x7fb4e815e060 2024-03-04T07:31:19.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.939+0000 7fb4ecc79640 1 -- 172.21.15.64:0/2974268696 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb4e402d040 con 0x7fb4e815e060 2024-03-04T07:31:19.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.939+0000 7fb4ecc79640 1 -- 172.21.15.64:0/2974268696 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4e4031a80 con 0x7fb4e815e060 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.939+0000 7fb4efedf640 1 -- 172.21.15.64:0/2974268696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e815e060 msgr2=0x7fb4e80ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.939+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2974268696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e815e060 0x7fb4e80ce310 secure :-1 s=READY pgs=1141 cs=0 l=1 rev1=1 crypto rx=0x7fb4e8068cd0 tx=0x7fb4e402b900 comp rx=0 tx=0).stop 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.940+0000 7fb4efedf640 1 -- 172.21.15.64:0/2974268696 shutdown_connections 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.940+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2974268696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e80d0880 0x7fb4e80ced90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.940+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2974268696 >> v2:172.21.15.66:3300/0 conn(0x7fb4e806fe20 0x7fb4e80ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.940+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2974268696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e815e060 0x7fb4e80ce310 unknown :-1 s=CLOSED pgs=1141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.940+0000 7fb4efedf640 1 -- 172.21.15.64:0/2974268696 >> 172.21.15.64:0/2974268696 conn(0x7fb4e80d1a90 msgr2=0x7fb4e80d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.940+0000 7fb4efedf640 1 -- 172.21.15.64:0/2974268696 shutdown_connections 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.940+0000 7fb4efedf640 1 -- 172.21.15.64:0/2974268696 wait complete. 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 Processor -- start 2024-03-04T07:31:19.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 -- start start 2024-03-04T07:31:19.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4e806fe20 0x7fb4e80c7ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e80d0880 0x7fb4e80ca4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e815e060 0x7fb4e80caa00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4e806ed30 con 0x7fb4e80d0880 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4e815e440 con 0x7fb4e806fe20 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.941+0000 7fb4efedf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4e806eeb0 con 0x7fb4e815e060 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e80d0880 0x7fb4e80ca4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e80d0880 0x7fb4e80ca4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38862/0 (socket says 172.21.15.64:38862) 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 -- 172.21.15.64:0/2295370913 learned_addr learned my addr 172.21.15.64:0/2295370913 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4edc7b640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4e806fe20 0x7fb4e80c7ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ee47c640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e815e060 0x7fb4e80caa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 -- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e815e060 msgr2=0x7fb4e80caa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e815e060 0x7fb4e80caa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 -- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4e806fe20 msgr2=0x7fb4e80c7ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4e806fe20 0x7fb4e80c7ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:19.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 -- 172.21.15.64:0/2295370913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4e80b56c0 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4ed47a640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e80d0880 0x7fb4e80ca4c0 secure :-1 s=READY pgs=1142 cs=0 l=1 rev1=1 crypto rx=0x7fb4d800e980 tx=0x7fb4d800ee40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4de7fc640 1 -- 172.21.15.64:0/2295370913 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4d8017070 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4de7fc640 1 -- 172.21.15.64:0/2295370913 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb4d800ca00 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4de7fc640 1 -- 172.21.15.64:0/2295370913 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4d800cdd0 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb4e80b5960 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.942+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb4e80b5f20 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.943+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb4e80b63f0 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.944+0000 7fb4de7fc640 1 -- 172.21.15.64:0/2295370913 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb4d801a070 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.944+0000 7fb4de7fc640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb4cc06b4d0 0x7fb4cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.944+0000 7fb4de7fc640 1 -- 172.21.15.64:0/2295370913 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb4d808a150 con 0x7fb4e80d0880 2024-03-04T07:31:19.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.944+0000 7fb4edc7b640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb4cc06b4d0 0x7fb4cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:19.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.944+0000 7fb4edc7b640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb4cc06b4d0 0x7fb4cc06d990 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7fb4e4036000 tx=0x7fb4e403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:19.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:19.948+0000 7fb4de7fc640 1 -- 172.21.15.64:0/2295370913 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb4d8058980 con 0x7fb4e80d0880 2024-03-04T07:31:20.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.167+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fb4e8067ff0 con 0x7fb4e80d0880 2024-03-04T07:31:20.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.167+0000 7fb4de7fc640 1 -- 172.21.15.64:0/2295370913 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v50) v1 ==== 53+0+73 (secure 0 0 0) 0x7fb4d805ca20 con 0x7fb4e80d0880 2024-03-04T07:31:20.167 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:31:20.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb4cc06b4d0 msgr2=0x7fb4cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb4cc06b4d0 0x7fb4cc06d990 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7fb4e4036000 tx=0x7fb4e403c040 comp rx=0 tx=0).stop 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e80d0880 msgr2=0x7fb4e80ca4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e80d0880 0x7fb4e80ca4c0 secure :-1 s=READY pgs=1142 cs=0 l=1 rev1=1 crypto rx=0x7fb4d800e980 tx=0x7fb4d800ee40 comp rx=0 tx=0).stop 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 shutdown_connections 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb4cc06b4d0 0x7fb4cc06d990 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4e815e060 0x7fb4e80caa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4e80d0880 0x7fb4e80ca4c0 unknown :-1 s=CLOSED pgs=1142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 --2- 172.21.15.64:0/2295370913 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4e806fe20 0x7fb4e80c7ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 >> 172.21.15.64:0/2295370913 conn(0x7fb4e80d1a90 msgr2=0x7fb4e80cfe10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 shutdown_connections 2024-03-04T07:31:20.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:20.169+0000 7fb4efedf640 1 -- 172.21.15.64:0/2295370913 wait complete. 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:31:20.180 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:31:20.181 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.181 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:31:20.196 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42432sec preferred_lft 42432sec 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:31:20.197 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:31:20.198 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.199 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:31:20.261 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:31:20.263 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.263 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:31:20.319 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:31:20.319 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:31:20.319 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:31:20.319 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:31:20.319 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:31:20.319 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.319 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:31:20.320 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:31:20.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:31:20.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:31:20.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:31:20.320 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:31:20.413 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.413 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:31:20.428 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:31:20.428 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:31:20.428 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:31:20.428 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:31:20.494 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:31:20.494 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:31:20.495 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:31:20.549 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:31:20.549 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:31:20.549 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:31:20.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:31:20.671 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:31:20.726 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:31:20.727 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:31:20.728 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:31:20.729 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:31:20.729 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.729 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:31:20.781 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:31:20.781 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:31:20.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:31:20.851 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:31:20.906 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:31:20.906 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:31:20.906 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:31:20.906 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:31:20.906 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:31:20.906 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:31:20.906 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:31:20.907 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:31:20.908 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:31:20.909 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:31:20.909 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:31:20.909 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:31:20.909 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:31:20.909 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:31:20.909 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:20.909 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:31:20.910 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:31:20.910+0000 7f24a283d480 -1 init, newargv = 0x560c1c6185c0 newargc=15 2024-03-04T07:31:20.910 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[68753]: starting ceph client 2024-03-04T07:31:20.914 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[68753]: starting fuse 2024-03-04T07:31:20.925 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:31:20.925 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:31:20.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:31:20.925 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:31:20.925 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:31:20.925 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:31:20.925 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:31:20.926 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:31:20.927 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:31:20.927 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:31:20.927 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:31:20.927 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:31:20.927 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:31:20.927 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:31:20.927 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:31:21.046 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.68753.asok 2024-03-04T07:31:21.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:31:21.056 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.68753.asok 2024-03-04T07:31:21.056 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:21.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.68753.asok status 2024-03-04T07:31:21.195 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:31:21.195 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:31:21.195 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:31:21.195 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:31:21.195 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:31:21.195 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:31:21.195 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "68753", 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 7503, 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 7503 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:31:21.196 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 553993213 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 553993213 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.7503 192.168.0.1:0/553993213", 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/553993213", 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 50, 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 90, 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:31:21.197 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:31:21.198 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:31:21.198 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:31:21.203 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:21.204 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:31:21.204 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:31:21.219 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:31:21.219 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:31:21.219 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:21.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:31:21.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:21.329 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout: chdir = "" 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout: auth supported = cephx 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout: filestore xattr use omap = true 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.377 INFO:teuthology.orchestra.run.smithi064.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: auth debug = true 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on old message = true 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on bug = true 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:21.378 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default size = 2 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow pg remap = true 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on too few osds = false 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:21.379 INFO:teuthology.orchestra.run.smithi064.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: osd default data pool replay window = 5 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: mon allow pool delete = true 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: mon cluster log file level = debug 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: debug asserts on shutdown = true 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: mon health detail to clog = false 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.380 INFO:teuthology.orchestra.run.smithi064.stdout:[osd] 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd journal size = 100 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub max interval = 600 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd recover clone overlap = true 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug shutdown = true 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug op order = true 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:21.381 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug trim objects = true 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: osd open classes on start = true 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug pg log writeout = true 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: osd map max advance = 10 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: journal zero on create = true 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: bdev debug aio = true 2024-03-04T07:31:21.382 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug misdirected ops = true 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: bdev async discard = True 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: bdev enable discard = True 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore allocator = bitmap 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore block size = 96636764160 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore fsck on mount = True 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluefs = 1/20 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluestore = 1/20 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: debug osd = 20 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: debug rocksdb = 4/10 2024-03-04T07:31:21.383 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: osd objectstore = bluestore 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr] 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: debug mgr = 20 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:21.384 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout:[mon] 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: debug paxos = 20 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mon data avail warn = 5 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd prime pg temp = true 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.385 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: # 1m isn't quite enough 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:21.386 INFO:teuthology.orchestra.run.smithi064.stdout: mon op complaint time = 120 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout:[client] 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: rgw cache enabled = true 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable ops log = true 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable usage log = true 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: client mount timeout = 600 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: debug client = 20 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = "" 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.a] 2024-03-04T07:31:21.387 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.b] 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.c] 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout:[mds] 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds = 20 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds balancer = 20 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug frag = True 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug scatterstat = True 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout: mds op complaint time = 180 2024-03-04T07:31:21.388 INFO:teuthology.orchestra.run.smithi064.stdout: mds verify scatter = True 2024-03-04T07:31:21.389 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:21.389 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.389 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.389 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_files = 6 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout:[global] 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout: chdir = "" 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout: auth supported = cephx 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout: filestore xattr use omap = true 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.390 INFO:teuthology.orchestra.run.smithi066.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: auth debug = true 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on old message = true 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on bug = true 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.391 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default size = 2 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow pg remap = true 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on too few osds = false 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.392 INFO:teuthology.orchestra.run.smithi066.stdout: osd default data pool replay window = 5 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: mon allow pool delete = true 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: mon cluster log file level = debug 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: debug asserts on shutdown = true 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: mon health detail to clog = false 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout:[osd] 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: osd journal size = 100 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub max interval = 600 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:21.393 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd recover clone overlap = true 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug shutdown = true 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug op order = true 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug trim objects = true 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd open classes on start = true 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug pg log writeout = true 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: osd map max advance = 10 2024-03-04T07:31:21.394 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: journal zero on create = true 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: bdev debug aio = true 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug misdirected ops = true 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: bdev async discard = True 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: bdev enable discard = True 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore allocator = bitmap 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore block size = 96636764160 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore fsck on mount = True 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluefs = 1/20 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluestore = 1/20 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.395 INFO:teuthology.orchestra.run.smithi066.stdout: debug osd = 20 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: debug rocksdb = 4/10 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: osd objectstore = bluestore 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr] 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: debug mgr = 20 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.396 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout:[mon] 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: debug paxos = 20 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: mon data avail warn = 5 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd prime pg temp = true 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.397 INFO:teuthology.orchestra.run.smithi066.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: # 1m isn't quite enough 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: mon op complaint time = 120 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout:[client] 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: rgw cache enabled = true 2024-03-04T07:31:21.398 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable ops log = true 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable usage log = true 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: client mount timeout = 600 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: debug client = 20 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = "" 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.a] 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.b] 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.c] 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:31:21.399 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout:[mds] 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds = 20 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds balancer = 20 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug frag = True 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug scatterstat = True 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: mds op complaint time = 180 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: mds verify scatter = True 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.400 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_files = 6 2024-03-04T07:31:21.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:21.444 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:21.492 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-03-04T07:31:21.492 INFO:teuthology.orchestra.run.smithi064.stdout: chdir = "" 2024-03-04T07:31:21.492 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:21.492 INFO:teuthology.orchestra.run.smithi064.stdout: auth supported = cephx 2024-03-04T07:31:21.492 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.492 INFO:teuthology.orchestra.run.smithi064.stdout: filestore xattr use omap = true 2024-03-04T07:31:21.492 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: auth debug = true 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on old message = true 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on bug = true 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:21.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default size = 2 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow pg remap = true 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on too few osds = false 2024-03-04T07:31:21.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: osd default data pool replay window = 5 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon allow pool delete = true 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon cluster log file level = debug 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: debug asserts on shutdown = true 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon health detail to clog = false 2024-03-04T07:31:21.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout:[osd] 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd journal size = 100 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub max interval = 600 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd recover clone overlap = true 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug shutdown = true 2024-03-04T07:31:21.496 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug op order = true 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug trim objects = true 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: osd open classes on start = true 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug pg log writeout = true 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: osd map max advance = 10 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: journal zero on create = true 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.497 INFO:teuthology.orchestra.run.smithi064.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: bdev debug aio = true 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug misdirected ops = true 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: bdev async discard = True 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: bdev enable discard = True 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore allocator = bitmap 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore block size = 96636764160 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore fsck on mount = True 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluefs = 1/20 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluestore = 1/20 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.498 INFO:teuthology.orchestra.run.smithi064.stdout: debug osd = 20 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: debug rocksdb = 4/10 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: osd objectstore = bluestore 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr] 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: debug mgr = 20 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.499 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout:[mon] 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: debug paxos = 20 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: mon data avail warn = 5 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd prime pg temp = true 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.500 INFO:teuthology.orchestra.run.smithi064.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: # 1m isn't quite enough 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: mon op complaint time = 120 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.501 INFO:teuthology.orchestra.run.smithi064.stdout:[client] 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: rgw cache enabled = true 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable ops log = true 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable usage log = true 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: client mount timeout = 600 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: debug client = 20 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = "" 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.a] 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.b] 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.c] 2024-03-04T07:31:21.502 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout:[mds] 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds = 20 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds balancer = 20 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug frag = True 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug scatterstat = True 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: mds op complaint time = 180 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: mds verify scatter = True 2024-03-04T07:31:21.503 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:21.504 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.504 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.504 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_files = 6 2024-03-04T07:31:21.504 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_ops = 100000000 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout:[global] 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: chdir = "" 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: auth supported = cephx 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: filestore xattr use omap = true 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:21.506 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: auth debug = true 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on old message = true 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on bug = true 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default size = 2 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.507 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow pg remap = true 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on too few osds = false 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd default data pool replay window = 5 2024-03-04T07:31:21.508 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: mon allow pool delete = true 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: mon cluster log file level = debug 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: debug asserts on shutdown = true 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: mon health detail to clog = false 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout:[osd] 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: osd journal size = 100 2024-03-04T07:31:21.509 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:21.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub max interval = 600 2024-03-04T07:31:21.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:21.510 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd recover clone overlap = true 2024-03-04T07:31:21.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:21.510 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug shutdown = true 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug op order = true 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug trim objects = true 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd open classes on start = true 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug pg log writeout = true 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: osd map max advance = 10 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: journal zero on create = true 2024-03-04T07:31:21.511 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: bdev debug aio = true 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug misdirected ops = true 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: bdev async discard = True 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: bdev enable discard = True 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore allocator = bitmap 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore block size = 96636764160 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore fsck on mount = True 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluefs = 1/20 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluestore = 1/20 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.512 INFO:teuthology.orchestra.run.smithi066.stdout: debug osd = 20 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: debug rocksdb = 4/10 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: osd objectstore = bluestore 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr] 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: debug mgr = 20 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:21.513 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mon] 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: debug paxos = 20 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: mon data avail warn = 5 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:21.514 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd prime pg temp = true 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: # 1m isn't quite enough 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:21.515 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: mon op complaint time = 120 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout:[client] 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: rgw cache enabled = true 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable ops log = true 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable usage log = true 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: client mount timeout = 600 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: debug client = 20 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = "" 2024-03-04T07:31:21.516 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.a] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.b] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.c] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout:[mds] 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds = 20 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds balancer = 20 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug frag = True 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug scatterstat = True 2024-03-04T07:31:21.517 INFO:teuthology.orchestra.run.smithi066.stdout: mds op complaint time = 180 2024-03-04T07:31:21.518 INFO:teuthology.orchestra.run.smithi066.stdout: mds verify scatter = True 2024-03-04T07:31:21.518 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:21.518 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.518 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.518 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_files = 6 2024-03-04T07:31:21.518 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_ops = 100000000 2024-03-04T07:31:21.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:31:21.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.651+0000 7f02179e4640 1 Processor -- start 2024-03-04T07:31:21.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.651+0000 7f02179e4640 1 -- start start 2024-03-04T07:31:21.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f02179e4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f02100d1c80 0x7f02100d49c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:21.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f02179e4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f02100d1780 con 0x7f02100d28e0 2024-03-04T07:31:21.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f02179e4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f02100d0ee0 con 0x7f021015e370 2024-03-04T07:31:21.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f02179e4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f021015e750 con 0x7f02100d1c80 2024-03-04T07:31:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f0215f81640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f02100d1c80 0x7f02100d49c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f0214f7f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f02100d28e0 0x7f02100b3dc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45776/0 (socket says 172.21.15.64:45776) 2024-03-04T07:31:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f0214f7f640 1 -- 172.21.15.64:0/3054417052 learned_addr learned my addr 172.21.15.64:0/3054417052 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f0215f81640 1 -- 172.21.15.64:0/3054417052 >> v1:172.21.15.66:6789/0 conn(0x7f021015e370 legacy=0x7f02100d21d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f0215f81640 1 -- 172.21.15.64:0/3054417052 >> v1:172.21.15.64:6789/0 conn(0x7f02100d28e0 legacy=0x7f02100b3dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f020ffff640 1 -- 172.21.15.64:0/3054417052 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3037665079 0 0) 0x7f02100d1780 con 0x7f02100d28e0 2024-03-04T07:31:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.652+0000 7f0215f81640 1 -- 172.21.15.64:0/3054417052 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02100d11c0 con 0x7f02100d1c80 2024-03-04T07:31:21.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.653+0000 7f0215f81640 1 --2- 172.21.15.64:0/3054417052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f02100d1c80 0x7f02100d49c0 secure :-1 s=READY pgs=1146 cs=0 l=1 rev1=1 crypto rx=0x7f0200009d90 tx=0x7f020002beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a8d4fde7cba5be4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.653+0000 7f020ffff640 1 -- 172.21.15.64:0/3054417052 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0200039070 con 0x7f02100d1c80 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.653+0000 7f020ffff640 1 -- 172.21.15.64:0/3054417052 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f020002cc60 con 0x7f02100d1c80 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.653+0000 7f020ffff640 1 -- 172.21.15.64:0/3054417052 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0200034660 con 0x7f02100d1c80 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.654+0000 7f02179e4640 1 -- 172.21.15.64:0/3054417052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f02100d1c80 msgr2=0x7f02100d49c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.654+0000 7f02179e4640 1 --2- 172.21.15.64:0/3054417052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f02100d1c80 0x7f02100d49c0 secure :-1 s=READY pgs=1146 cs=0 l=1 rev1=1 crypto rx=0x7f0200009d90 tx=0x7f020002beb0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.654+0000 7f02179e4640 1 -- 172.21.15.64:0/3054417052 shutdown_connections 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.654+0000 7f02179e4640 1 --2- 172.21.15.64:0/3054417052 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f02100d1c80 0x7f02100d49c0 unknown :-1 s=CLOSED pgs=1146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.654+0000 7f02179e4640 1 -- 172.21.15.64:0/3054417052 >> 172.21.15.64:0/3054417052 conn(0x7f02100d3ea0 msgr2=0x7f02100d42b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.654+0000 7f02179e4640 1 -- 172.21.15.64:0/3054417052 shutdown_connections 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.654+0000 7f02179e4640 1 -- 172.21.15.64:0/3054417052 wait complete. 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.655+0000 7f02179e4640 1 Processor -- start 2024-03-04T07:31:21.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.655+0000 7f02179e4640 1 -- start start 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f02179e4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02100d1c80 0x7f02100cbe10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f02179e4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02100d28e0 0x7f021015f850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f02179e4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f021015e370 0x7f0210161480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f02179e4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02100cfca0 con 0x7f021015e370 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f02179e4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02100b38a0 con 0x7f02100d28e0 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f02179e4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f02100b44d0 con 0x7f02100d1c80 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f021015e370 0x7f0210161480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f021015e370 0x7f0210161480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38882/0 (socket says 172.21.15.64:38882) 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 -- 172.21.15.64:0/3494384733 learned_addr learned my addr 172.21.15.64:0/3494384733 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0214f7f640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02100d28e0 0x7f021015f850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215780640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02100d1c80 0x7f02100cbe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 -- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02100d1c80 msgr2=0x7f02100cbe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02100d1c80 0x7f02100cbe10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 -- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02100d28e0 msgr2=0x7f021015f850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02100d28e0 0x7f021015f850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 -- 172.21.15.64:0/3494384733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02100cc3f0 con 0x7f021015e370 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0214f7f640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02100d28e0 0x7f021015f850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215780640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02100d1c80 0x7f02100cbe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f0215f81640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f021015e370 0x7f0210161480 secure :-1 s=READY pgs=1147 cs=0 l=1 rev1=1 crypto rx=0x7f0200009d90 tx=0x7f0200004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.656+0000 7f020dffb640 1 -- 172.21.15.64:0/3494384733 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0200039050 con 0x7f021015e370 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.657+0000 7f020dffb640 1 -- 172.21.15.64:0/3494384733 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f020003a040 con 0x7f021015e370 2024-03-04T07:31:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.657+0000 7f020dffb640 1 -- 172.21.15.64:0/3494384733 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f020003e610 con 0x7f021015e370 2024-03-04T07:31:21.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.657+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f02100c7b30 con 0x7f021015e370 2024-03-04T07:31:21.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.657+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02100c80f0 con 0x7f021015e370 2024-03-04T07:31:21.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.658+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f02100751b0 con 0x7f021015e370 2024-03-04T07:31:21.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.658+0000 7f020dffb640 1 -- 172.21.15.64:0/3494384733 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0200047030 con 0x7f021015e370 2024-03-04T07:31:21.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.659+0000 7f020dffb640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01fc06b500 0x7f01fc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:21.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.659+0000 7f020dffb640 1 -- 172.21.15.64:0/3494384733 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f02000ac2e0 con 0x7f021015e370 2024-03-04T07:31:21.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.659+0000 7f0215780640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01fc06b500 0x7f01fc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:21.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.659+0000 7f0215780640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01fc06b500 0x7f01fc06d9c0 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f02100b4ef0 tx=0x7f0208009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:21.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.663+0000 7f020dffb640 1 -- 172.21.15.64:0/3494384733 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f020007b2b0 con 0x7f021015e370 2024-03-04T07:31:21.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.843+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f02100008d0 con 0x7f021015e370 2024-03-04T07:31:21.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.844+0000 7f020dffb640 1 -- 172.21.15.64:0/3494384733 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v90) v1 ==== 100+0+11 (secure 0 0 0) 0x7f020007abb0 con 0x7f021015e370 2024-03-04T07:31:21.843 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01fc06b500 msgr2=0x7f01fc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01fc06b500 0x7f01fc06d9c0 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f02100b4ef0 tx=0x7f0208009040 comp rx=0 tx=0).stop 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f021015e370 msgr2=0x7f0210161480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f021015e370 0x7f0210161480 secure :-1 s=READY pgs=1147 cs=0 l=1 rev1=1 crypto rx=0x7f0200009d90 tx=0x7f0200004a40 comp rx=0 tx=0).stop 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 shutdown_connections 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f01fc06b500 0x7f01fc06d9c0 unknown :-1 s=CLOSED pgs=521 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f021015e370 0x7f0210161480 unknown :-1 s=CLOSED pgs=1147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02100d28e0 0x7f021015f850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 --2- 172.21.15.64:0/3494384733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02100d1c80 0x7f02100cbe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 >> 172.21.15.64:0/3494384733 conn(0x7f02100d3ea0 msgr2=0x7f021006c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:21.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 shutdown_connections 2024-03-04T07:31:21.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:21.846+0000 7f02179e4640 1 -- 172.21.15.64:0/3494384733 wait complete. 2024-03-04T07:31:21.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:21.859 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: chdir = "" 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: auth supported = cephx 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: filestore xattr use omap = true 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: auth debug = true 2024-03-04T07:31:21.907 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on old message = true 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on bug = true 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default size = 2 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow pg remap = true 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:21.908 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on too few osds = false 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: osd default data pool replay window = 5 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon allow pool delete = true 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: mon cluster log file level = debug 2024-03-04T07:31:21.909 INFO:teuthology.orchestra.run.smithi064.stdout: debug asserts on shutdown = true 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: mon health detail to clog = false 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout:[osd] 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: osd journal size = 100 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub max interval = 600 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: osd recover clone overlap = true 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.910 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug shutdown = true 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug op order = true 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug trim objects = true 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: osd open classes on start = true 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug pg log writeout = true 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: osd map max advance = 10 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: journal zero on create = true 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:21.911 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: bdev debug aio = true 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug misdirected ops = true 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: bdev async discard = True 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: bdev enable discard = True 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore allocator = bitmap 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore block size = 96636764160 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore fsck on mount = True 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluefs = 1/20 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluestore = 1/20 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: debug osd = 20 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: debug rocksdb = 4/10 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:21.912 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: osd objectstore = bluestore 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr] 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: debug mgr = 20 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout:[mon] 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.913 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: debug paxos = 20 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: mon data avail warn = 5 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd prime pg temp = true 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.914 INFO:teuthology.orchestra.run.smithi064.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: # 1m isn't quite enough 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: mon op complaint time = 120 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout:[client] 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: rgw cache enabled = true 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable ops log = true 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable usage log = true 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:21.915 INFO:teuthology.orchestra.run.smithi064.stdout: client mount timeout = 600 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout: debug client = 20 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = "" 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.a] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.b] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.c] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout:[mds] 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds = 20 2024-03-04T07:31:21.916 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds balancer = 20 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug frag = True 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug scatterstat = True 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: mds op complaint time = 180 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: mds verify scatter = True 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_files = 6 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_ops = 100000000 2024-03-04T07:31:21.917 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-04T07:31:21.919 INFO:teuthology.orchestra.run.smithi066.stdout:[global] 2024-03-04T07:31:21.919 INFO:teuthology.orchestra.run.smithi066.stdout: chdir = "" 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: auth supported = cephx 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: filestore xattr use omap = true 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: auth debug = true 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on old message = true 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on bug = true 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.920 INFO:teuthology.orchestra.run.smithi066.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default size = 2 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow pg remap = true 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:21.921 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on too few osds = false 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: osd default data pool replay window = 5 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: mon allow pool delete = true 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: mon cluster log file level = debug 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: debug asserts on shutdown = true 2024-03-04T07:31:21.922 INFO:teuthology.orchestra.run.smithi066.stdout: mon health detail to clog = false 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout:[osd] 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: osd journal size = 100 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub max interval = 600 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: osd recover clone overlap = true 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.923 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug shutdown = true 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug op order = true 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug trim objects = true 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: osd open classes on start = true 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug pg log writeout = true 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: osd map max advance = 10 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: journal zero on create = true 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:21.924 INFO:teuthology.orchestra.run.smithi066.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: bdev debug aio = true 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug misdirected ops = true 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: bdev async discard = True 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: bdev enable discard = True 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore allocator = bitmap 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore block size = 96636764160 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore fsck on mount = True 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluefs = 1/20 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluestore = 1/20 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: debug osd = 20 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: debug rocksdb = 4/10 2024-03-04T07:31:21.925 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: osd objectstore = bluestore 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr] 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: debug mgr = 20 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:21.926 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout:[mon] 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: debug paxos = 20 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: mon data avail warn = 5 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd prime pg temp = true 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:21.927 INFO:teuthology.orchestra.run.smithi066.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: # 1m isn't quite enough 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: mon op complaint time = 120 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout:[client] 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: rgw cache enabled = true 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable ops log = true 2024-03-04T07:31:21.928 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable usage log = true 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: client mount timeout = 600 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: debug client = 20 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = "" 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.a] 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.b] 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.c] 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:31:21.929 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout:[mds] 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds = 20 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds balancer = 20 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug frag = True 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug scatterstat = True 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: mds op complaint time = 180 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: mds verify scatter = True 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_files = 6 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_ops = 100000000 2024-03-04T07:31:21.930 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-04T07:31:21.937 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:31:21.937 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:21.938 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:31:21.938 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:21.938 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:31:21.938 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:21.938 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:31:21.938 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:31:21.949 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:31:21.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail d 2024-03-04T07:31:21.950 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:31:21.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail b 2024-03-04T07:31:21.982 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:31:21.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail a 2024-03-04T07:31:21.983 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:31:21.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail c 2024-03-04T07:31:22.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.078+0000 7f0f006c3640 1 Processor -- start 2024-03-04T07:31:22.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.078+0000 7f0f006c3640 1 -- start start 2024-03-04T07:31:22.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.078+0000 7f4972d0d640 1 Processor -- start 2024-03-04T07:31:22.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.078+0000 7f4972d0d640 1 -- start start 2024-03-04T07:31:22.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f4972d0d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f496c15c370 0x7f496c0ccef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f0f006c3640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0ef815fcc0 0x7f0ef80b35e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f0f006c3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef8160610 0x7f0ef80b3b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f0f006c3640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0ef80cd320 con 0x7f0ef81600a0 2024-03-04T07:31:22.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f0f006c3640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0ef80cf740 con 0x7f0ef815fcc0 2024-03-04T07:31:22.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f0f006c3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ef80cfa20 con 0x7f0ef8160610 2024-03-04T07:31:22.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f4972d0d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f496c0cd430 0x7f496c06f1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f4972d0d640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f496c06ed00 con 0x7f496c15bf90 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f4972d0d640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f496c0cdbf0 con 0x7f496c15c370 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f4972d0d640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f496c0d0390 con 0x7f496c0cd430 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f496bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f496c15c370 0x7f496c0ccef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.079+0000 7f496bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f496c15c370 0x7f496c0ccef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38892/0 (socket says 172.21.15.64:38892) 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f496bfff640 1 -- 172.21.15.64:0/4161612122 learned_addr learned my addr 172.21.15.64:0/4161612122 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f49712aa640 1 --2- 172.21.15.64:0/4161612122 >> v2:172.21.15.66:3300/0 conn(0x7f496c0cd430 0x7f496c06f1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efdc5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef8160610 0x7f0ef80b3b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efe45f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0ef815fcc0 0x7f0ef80b35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efdc5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef8160610 0x7f0ef80b3b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:46932/0 (socket says 172.21.15.64:46932) 2024-03-04T07:31:22.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f496bfff640 1 -- 172.21.15.64:0/4161612122 >> v2:172.21.15.66:3300/0 conn(0x7f496c0cd430 msgr2=0x7f496c06f1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:22.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f496bfff640 1 --2- 172.21.15.64:0/4161612122 >> v2:172.21.15.66:3300/0 conn(0x7f496c0cd430 0x7f496c06f1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f496bfff640 1 -- 172.21.15.64:0/4161612122 >> v1:172.21.15.66:6789/0 conn(0x7f496c15bf90 legacy=0x7f496c0cc7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:22.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efe45f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0ef815fcc0 0x7f0ef80b35e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:34780/0 (socket says 172.21.15.64:34780) 2024-03-04T07:31:22.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efdc5e640 1 -- 172.21.15.64:0/325154135 learned_addr learned my addr 172.21.15.64:0/325154135 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:22.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f496bfff640 1 -- 172.21.15.64:0/4161612122 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f496c0d0670 con 0x7f496c15c370 2024-03-04T07:31:22.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f49712aa640 1 --2- 172.21.15.64:0/4161612122 >> v2:172.21.15.66:3300/0 conn(0x7f496c0cd430 0x7f496c06f1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:22.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efe45f640 1 -- 172.21.15.64:0/325154135 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef8160610 msgr2=0x7f0ef80b3b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:22.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efe45f640 1 --2- 172.21.15.64:0/325154135 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef8160610 0x7f0ef80b3b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efe45f640 1 -- 172.21.15.64:0/325154135 >> v1:172.21.15.66:6789/0 conn(0x7f0ef81600a0 legacy=0x7f0ef80b4060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.080+0000 7f0efe45f640 1 -- 172.21.15.64:0/325154135 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ef80cfd00 con 0x7f0ef815fcc0 2024-03-04T07:31:22.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f0efe45f640 1 --2- 172.21.15.64:0/325154135 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef815fcc0 0x7f0ef80b35e0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f0ef40097a0 tx=0x7f0ef402b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e55131d2d5fadbc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f496bfff640 1 --2- 172.21.15.64:0/4161612122 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c15c370 0x7f496c0ccef0 secure :-1 s=READY pgs=1148 cs=0 l=1 rev1=1 crypto rx=0x7f496c0cc360 tx=0x7f496002b7f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=23834c797416ef59 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f0efd45d640 1 -- 172.21.15.64:0/325154135 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ef40315a0 con 0x7f0ef815fcc0 2024-03-04T07:31:22.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f0efd45d640 1 -- 172.21.15.64:0/325154135 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ef4037070 con 0x7f0ef815fcc0 2024-03-04T07:31:22.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f0efd45d640 1 -- 172.21.15.64:0/325154135 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ef4031960 con 0x7f0ef815fcc0 2024-03-04T07:31:22.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f496b7fe640 1 -- 172.21.15.64:0/4161612122 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4960038070 con 0x7f496c15c370 2024-03-04T07:31:22.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f496b7fe640 1 -- 172.21.15.64:0/4161612122 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f496002f050 con 0x7f496c15c370 2024-03-04T07:31:22.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.081+0000 7f496b7fe640 1 -- 172.21.15.64:0/4161612122 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4960031970 con 0x7f496c15c370 2024-03-04T07:31:22.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 -- 172.21.15.64:0/325154135 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef815fcc0 msgr2=0x7f0ef80b35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 --2- 172.21.15.64:0/325154135 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef815fcc0 0x7f0ef80b35e0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f0ef40097a0 tx=0x7f0ef402b840 comp rx=0 tx=0).stop 2024-03-04T07:31:22.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 -- 172.21.15.64:0/4161612122 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c15c370 msgr2=0x7f496c0ccef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 -- 172.21.15.64:0/325154135 shutdown_connections 2024-03-04T07:31:22.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 --2- 172.21.15.64:0/325154135 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef8160610 0x7f0ef80b3b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 --2- 172.21.15.64:0/325154135 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef815fcc0 0x7f0ef80b35e0 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4161612122 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c15c370 0x7f496c0ccef0 secure :-1 s=READY pgs=1148 cs=0 l=1 rev1=1 crypto rx=0x7f496c0cc360 tx=0x7f496002b7f0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 -- 172.21.15.64:0/325154135 >> 172.21.15.64:0/325154135 conn(0x7f0ef80d0510 msgr2=0x7f0ef80d08f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:22.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 -- 172.21.15.64:0/4161612122 shutdown_connections 2024-03-04T07:31:22.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4161612122 >> v2:172.21.15.66:3300/0 conn(0x7f496c0cd430 0x7f496c06f1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4161612122 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c15c370 0x7f496c0ccef0 unknown :-1 s=CLOSED pgs=1148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 -- 172.21.15.64:0/4161612122 >> 172.21.15.64:0/4161612122 conn(0x7f496c0d1d10 msgr2=0x7f496c0d2120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:22.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 -- 172.21.15.64:0/4161612122 shutdown_connections 2024-03-04T07:31:22.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 -- 172.21.15.64:0/325154135 shutdown_connections 2024-03-04T07:31:22.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f4972d0d640 1 -- 172.21.15.64:0/4161612122 wait complete. 2024-03-04T07:31:22.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.082+0000 7f0f006c3640 1 -- 172.21.15.64:0/325154135 wait complete. 2024-03-04T07:31:22.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.084+0000 7f4972d0d640 1 Processor -- start 2024-03-04T07:31:22.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.084+0000 7f0f006c3640 1 Processor -- start 2024-03-04T07:31:22.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.084+0000 7f4972d0d640 1 -- start start 2024-03-04T07:31:22.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.084+0000 7f0f006c3640 1 -- start start 2024-03-04T07:31:22.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.085+0000 7f0f006c3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ef815fcc0 0x7f0ef806e990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.085+0000 7f0f006c3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef81600a0 0x7f0ef80aa910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.085+0000 7f0f006c3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef8160610 0x7f0ef8153990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.085+0000 7f0f006c3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ef80cd0c0 con 0x7f0ef815fcc0 2024-03-04T07:31:22.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.085+0000 7f0f006c3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ef80ccf40 con 0x7f0ef8160610 2024-03-04T07:31:22.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.085+0000 7f0f006c3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0ef80b4770 con 0x7f0ef81600a0 2024-03-04T07:31:22.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.085+0000 7f4972d0d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f496c0cd430 0x7f496c0cb8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f4972d0d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f496c15bf90 0x7f496c0b1a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efe45f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ef815fcc0 0x7f0ef806e990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efe45f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ef815fcc0 0x7f0ef806e990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38894/0 (socket says 172.21.15.64:38894) 2024-03-04T07:31:22.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efe45f640 1 -- 172.21.15.64:0/3067705499 learned_addr learned my addr 172.21.15.64:0/3067705499 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:22.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f4972d0d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c0c56e0 0x7f496c152560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f4972d0d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f496c0d2830 con 0x7f496c0c56e0 2024-03-04T07:31:22.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f4972d0d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f496c0d0c30 con 0x7f496c0cd430 2024-03-04T07:31:22.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f4972d0d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f496c0d29b0 con 0x7f496c15bf90 2024-03-04T07:31:22.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efec60640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef8160610 0x7f0ef8153990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efdc5e640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef81600a0 0x7f0ef80aa910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f496c15bf90 0x7f496c0b1a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f496c15bf90 0x7f496c0b1a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:46946/0 (socket says 172.21.15.64:46946) 2024-03-04T07:31:22.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 -- 172.21.15.64:0/4210111833 learned_addr learned my addr 172.21.15.64:0/4210111833 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:22.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f4970aa9640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f496c0cd430 0x7f496c0cb8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f49712aa640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c0c56e0 0x7f496c152560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efec60640 1 -- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef81600a0 msgr2=0x7f0ef80aa910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efec60640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef81600a0 0x7f0ef80aa910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efec60640 1 -- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ef815fcc0 msgr2=0x7f0ef806e990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efec60640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ef815fcc0 0x7f0ef806e990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efec60640 1 -- 172.21.15.64:0/3067705499 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ef806d610 con 0x7f0ef8160610 2024-03-04T07:31:22.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 -- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f496c0cd430 msgr2=0x7f496c0cb8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f496c0cd430 0x7f496c0cb8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 -- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c0c56e0 msgr2=0x7f496c152560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c0c56e0 0x7f496c152560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f496bfff640 1 -- 172.21.15.64:0/4210111833 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f496c0c5bf0 con 0x7f496c15bf90 2024-03-04T07:31:22.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.086+0000 7f0efec60640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef8160610 0x7f0ef8153990 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f0ef000c730 tx=0x7f0ef000cbf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f496bfff640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f496c15bf90 0x7f496c0b1a30 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f496002b830 tx=0x7f49600046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f0eeaffd640 1 -- 172.21.15.64:0/3067705499 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ef0016020 con 0x7f0ef8160610 2024-03-04T07:31:22.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f0eeaffd640 1 -- 172.21.15.64:0/3067705499 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ef0006d50 con 0x7f0ef8160610 2024-03-04T07:31:22.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0ef806d8f0 con 0x7f0ef8160610 2024-03-04T07:31:22.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f0eeaffd640 1 -- 172.21.15.64:0/3067705499 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ef0003bb0 con 0x7f0ef8160610 2024-03-04T07:31:22.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0ef806deb0 con 0x7f0ef8160610 2024-03-04T07:31:22.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f49697fa640 1 -- 172.21.15.64:0/4210111833 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4960042070 con 0x7f496c15bf90 2024-03-04T07:31:22.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f49697fa640 1 -- 172.21.15.64:0/4210111833 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f496003d890 con 0x7f496c15bf90 2024-03-04T07:31:22.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f49697fa640 1 -- 172.21.15.64:0/4210111833 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4960038040 con 0x7f496c15bf90 2024-03-04T07:31:22.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f496c0c5ed0 con 0x7f496c15bf90 2024-03-04T07:31:22.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.087+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f496c0c6490 con 0x7f496c15bf90 2024-03-04T07:31:22.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.088+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0ef806e680 con 0x7f0ef8160610 2024-03-04T07:31:22.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.088+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f496c0752d0 con 0x7f496c15bf90 2024-03-04T07:31:22.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.088+0000 7f0eeaffd640 1 -- 172.21.15.64:0/3067705499 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0ef00050d0 con 0x7f0ef8160610 2024-03-04T07:31:22.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.089+0000 7f0eeaffd640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0edc06b4d0 0x7f0edc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.089+0000 7f0eeaffd640 1 -- 172.21.15.64:0/3067705499 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0ef0089eb0 con 0x7f0ef8160610 2024-03-04T07:31:22.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.089+0000 7f0efe45f640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0edc06b4d0 0x7f0edc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.089+0000 7f49697fa640 1 -- 172.21.15.64:0/4210111833 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4960045070 con 0x7f496c15bf90 2024-03-04T07:31:22.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.089+0000 7f0efe45f640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0edc06b4d0 0x7f0edc06d990 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f0ef402b880 tx=0x7f0ef403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.089+0000 7f49697fa640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f495806b4d0 0x7f495806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.089+0000 7f49697fa640 1 -- 172.21.15.64:0/4210111833 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f49600ac090 con 0x7f496c15bf90 2024-03-04T07:31:22.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.090+0000 7f4970aa9640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f495806b4d0 0x7f495806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.090+0000 7f4970aa9640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f495806b4d0 0x7f495806d990 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f495c004f60 tx=0x7f495c007e30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.093+0000 7f0eeaffd640 1 -- 172.21.15.64:0/3067705499 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0ef0020020 con 0x7f0ef8160610 2024-03-04T07:31:22.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.094+0000 7f49697fa640 1 -- 172.21.15.64:0/4210111833 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f496004c030 con 0x7f496c15bf90 2024-03-04T07:31:22.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.134+0000 7f33031b6640 1 Processor -- start 2024-03-04T07:31:22.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.134+0000 7f33031b6640 1 -- start start 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f33031b6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f32fc0b0e90 0x7f32fc0d0660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f33031b6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f32fc0d2bb0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f33031b6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f32fc06ef10 con 0x7f32fc15e140 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f33031b6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f32fc0d28d0 con 0x7f32fc0b0e90 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f32fbfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f32fc06fbf0 0x7f32fc0cff50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45790/0 (socket says 172.21.15.64:45790) 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f32fbfff640 1 -- 172.21.15.64:0/1392367117 learned_addr learned my addr 172.21.15.64:0/1392367117 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f3301753640 1 --2- 172.21.15.64:0/1392367117 >> v2:172.21.15.66:3300/0 conn(0x7f32fc0b0e90 0x7f32fc0d0660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f32fb7fe640 1 -- 172.21.15.64:0/1392367117 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 431799659 0 0) 0x7f32fc0d2bb0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f32fb7fe640 1 -- 172.21.15.64:0/1392367117 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f32ec003520 con 0x7f32fc06fbf0 2024-03-04T07:31:22.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f3301753640 1 -- 172.21.15.64:0/1392367117 >> v1:172.21.15.66:6789/0 conn(0x7f32fc15e140 legacy=0x7f32fc06f4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f3301753640 1 -- 172.21.15.64:0/1392367117 >> v1:172.21.15.64:6789/0 conn(0x7f32fc06fbf0 legacy=0x7f32fc0cff50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.135+0000 7f3301753640 1 -- 172.21.15.64:0/1392367117 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32fc06f1b0 con 0x7f32fc0b0e90 2024-03-04T07:31:22.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.136+0000 7f3301753640 1 --2- 172.21.15.64:0/1392367117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc0b0e90 0x7f32fc0d0660 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f32f4009d90 tx=0x7f32f402beb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d91e1cb82b6a2608 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.137+0000 7f32fb7fe640 1 -- 172.21.15.64:0/1392367117 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32f4039070 con 0x7f32fc0b0e90 2024-03-04T07:31:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.137+0000 7f32fb7fe640 1 -- 172.21.15.64:0/1392367117 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f32f402cc60 con 0x7f32fc0b0e90 2024-03-04T07:31:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.137+0000 7f32fb7fe640 1 -- 172.21.15.64:0/1392367117 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32f4034660 con 0x7f32fc0b0e90 2024-03-04T07:31:22.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.138+0000 7f33031b6640 1 -- 172.21.15.64:0/1392367117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc0b0e90 msgr2=0x7f32fc0d0660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.138+0000 7f33031b6640 1 --2- 172.21.15.64:0/1392367117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc0b0e90 0x7f32fc0d0660 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f32f4009d90 tx=0x7f32f402beb0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.138+0000 7f33031b6640 1 -- 172.21.15.64:0/1392367117 shutdown_connections 2024-03-04T07:31:22.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.138+0000 7f33031b6640 1 --2- 172.21.15.64:0/1392367117 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc0b0e90 0x7f32fc0d0660 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.138+0000 7f33031b6640 1 -- 172.21.15.64:0/1392367117 >> 172.21.15.64:0/1392367117 conn(0x7f32fc0d1d10 msgr2=0x7f32fc0d2120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:22.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.138+0000 7f33031b6640 1 -- 172.21.15.64:0/1392367117 shutdown_connections 2024-03-04T07:31:22.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.138+0000 7f33031b6640 1 -- 172.21.15.64:0/1392367117 wait complete. 2024-03-04T07:31:22.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 Processor -- start 2024-03-04T07:31:22.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 -- start start 2024-03-04T07:31:22.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32fc06fbf0 0x7f32fc0cb800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32fc0b0e90 0x7f32fc0c97f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc15e140 0x7f32fc155ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32fc0b15f0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32fc0b1470 con 0x7f32fc15e140 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32fc06fbf0 0x7f32fc0cb800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.140+0000 7f33031b6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f32fc15e520 con 0x7f32fc0b0e90 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32fc06fbf0 0x7f32fc0cb800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38914/0 (socket says 172.21.15.64:38914) 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 -- 172.21.15.64:0/3314305751 learned_addr learned my addr 172.21.15.64:0/3314305751 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f32fbfff640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32fc0b0e90 0x7f32fc0c97f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3301753640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc15e140 0x7f32fc155ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 -- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32fc0b0e90 msgr2=0x7f32fc0c97f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32fc0b0e90 0x7f32fc0c97f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 -- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc15e140 msgr2=0x7f32fc155ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc15e140 0x7f32fc155ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 -- 172.21.15.64:0/3314305751 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32fc09cdb0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.141+0000 7f3300f52640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32fc06fbf0 0x7f32fc0cb800 secure :-1 s=READY pgs=1150 cs=0 l=1 rev1=1 crypto rx=0x7f32f000c3f0 tx=0x7f32f000c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.143+0000 7f32f97fa640 1 -- 172.21.15.64:0/3314305751 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32f00052d0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.143+0000 7f32f97fa640 1 -- 172.21.15.64:0/3314305751 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f32f0005490 con 0x7f32fc06fbf0 2024-03-04T07:31:22.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.143+0000 7f32f97fa640 1 -- 172.21.15.64:0/3314305751 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32f0007260 con 0x7f32fc06fbf0 2024-03-04T07:31:22.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.143+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32fc0ca2f0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.143+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f32fc0ca8b0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.145+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f32fc0752d0 con 0x7f32fc06fbf0 2024-03-04T07:31:22.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.145+0000 7f32f97fa640 1 -- 172.21.15.64:0/3314305751 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f32f0007400 con 0x7f32fc06fbf0 2024-03-04T07:31:22.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.145+0000 7f32f97fa640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32e006b520 0x7f32e006d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.146+0000 7f32fbfff640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32e006b520 0x7f32e006d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.146+0000 7f32f97fa640 1 -- 172.21.15.64:0/3314305751 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f32f0089480 con 0x7f32fc06fbf0 2024-03-04T07:31:22.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.146+0000 7f32fbfff640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32e006b520 0x7f32e006d9e0 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f32fc15cd60 tx=0x7f32e8007d30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.150+0000 7f32f97fa640 1 -- 172.21.15.64:0/3314305751 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f32f0057e20 con 0x7f32fc06fbf0 2024-03-04T07:31:22.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.151+0000 7f43d337a640 1 Processor -- start 2024-03-04T07:31:22.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.152+0000 7f43d337a640 1 -- start start 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d337a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f43cc06ec60 0x7f43cc165a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d337a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f43cc06f210 con 0x7f43cc0cf640 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d337a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f43cc06f4f0 con 0x7f43cc0b3a90 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d337a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43cc06f7d0 con 0x7f43cc06ec60 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d0915640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f43cc0cf640 0x7f43cc0b3e70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45796/0 (socket says 172.21.15.64:45796) 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d0915640 1 -- 172.21.15.64:0/1640731037 learned_addr learned my addr 172.21.15.64:0/1640731037 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d1917640 1 --2- 172.21.15.64:0/1640731037 >> v2:172.21.15.64:3300/0 conn(0x7f43cc06ec60 0x7f43cc165a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43c3fff640 1 -- 172.21.15.64:0/1640731037 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4259228442 0 0) 0x7f43cc06f210 con 0x7f43cc0cf640 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43c3fff640 1 -- 172.21.15.64:0/1640731037 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f43b8005730 con 0x7f43cc0cf640 2024-03-04T07:31:22.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d1917640 1 -- 172.21.15.64:0/1640731037 >> v1:172.21.15.66:6789/0 conn(0x7f43cc0b3a90 legacy=0x7f43cc06e550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d1917640 1 -- 172.21.15.64:0/1640731037 >> v1:172.21.15.64:6789/0 conn(0x7f43cc0cf640 legacy=0x7f43cc0b3e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.153+0000 7f43d1917640 1 -- 172.21.15.64:0/1640731037 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43cc06fab0 con 0x7f43cc06ec60 2024-03-04T07:31:22.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.154+0000 7f43d1917640 1 --2- 172.21.15.64:0/1640731037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc06ec60 0x7f43cc165a00 secure :-1 s=READY pgs=1152 cs=0 l=1 rev1=1 crypto rx=0x7f43c4009dc0 tx=0x7f43c402bf20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9c85ebffc6b1361 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.154+0000 7f43c3fff640 1 -- 172.21.15.64:0/1640731037 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43c4039070 con 0x7f43cc06ec60 2024-03-04T07:31:22.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.154+0000 7f43c3fff640 1 -- 172.21.15.64:0/1640731037 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43c402ccd0 con 0x7f43cc06ec60 2024-03-04T07:31:22.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.154+0000 7f43c3fff640 1 -- 172.21.15.64:0/1640731037 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43c4034660 con 0x7f43cc06ec60 2024-03-04T07:31:22.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.155+0000 7f43d337a640 1 -- 172.21.15.64:0/1640731037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc06ec60 msgr2=0x7f43cc165a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.155+0000 7f43d337a640 1 --2- 172.21.15.64:0/1640731037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc06ec60 0x7f43cc165a00 secure :-1 s=READY pgs=1152 cs=0 l=1 rev1=1 crypto rx=0x7f43c4009dc0 tx=0x7f43c402bf20 comp rx=0 tx=0).stop 2024-03-04T07:31:22.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.155+0000 7f43d337a640 1 -- 172.21.15.64:0/1640731037 shutdown_connections 2024-03-04T07:31:22.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.155+0000 7f43d337a640 1 --2- 172.21.15.64:0/1640731037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc06ec60 0x7f43cc165a00 unknown :-1 s=CLOSED pgs=1152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.155+0000 7f43d337a640 1 -- 172.21.15.64:0/1640731037 >> 172.21.15.64:0/1640731037 conn(0x7f43cc0d0be0 msgr2=0x7f43cc0d0ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:22.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.155+0000 7f43d337a640 1 -- 172.21.15.64:0/1640731037 shutdown_connections 2024-03-04T07:31:22.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.155+0000 7f43d337a640 1 -- 172.21.15.64:0/1640731037 wait complete. 2024-03-04T07:31:22.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.157+0000 7f43d337a640 1 Processor -- start 2024-03-04T07:31:22.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.157+0000 7f43d337a640 1 -- start start 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.157+0000 7f43d337a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43cc06ec60 0x7f43cc15c330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d337a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43cc0b3a90 0x7f43cc13c420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d337a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc0cf640 0x7f43cc15a730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d337a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43cc144b50 con 0x7f43cc0cf640 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d337a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43cc1449d0 con 0x7f43cc06ec60 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d337a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43cc06aec0 con 0x7f43cc0b3a90 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1116640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43cc06ec60 0x7f43cc15c330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1116640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43cc06ec60 0x7f43cc15c330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:34832/0 (socket says 172.21.15.64:34832) 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1116640 1 -- 172.21.15.64:0/1247602149 learned_addr learned my addr 172.21.15.64:0/1247602149 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:22.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d0915640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43cc0b3a90 0x7f43cc13c420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1917640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc0cf640 0x7f43cc15a730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1917640 1 -- 172.21.15.64:0/1247602149 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43cc0b3a90 msgr2=0x7f43cc13c420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1917640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43cc0b3a90 0x7f43cc13c420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1917640 1 -- 172.21.15.64:0/1247602149 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43cc06ec60 msgr2=0x7f43cc15c330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1917640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43cc06ec60 0x7f43cc15c330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1917640 1 -- 172.21.15.64:0/1247602149 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43cc13be20 con 0x7f43cc0cf640 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.158+0000 7f43d1917640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc0cf640 0x7f43cc15a730 secure :-1 s=READY pgs=1153 cs=0 l=1 rev1=1 crypto rx=0x7f43c4033000 tx=0x7f43c4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.159+0000 7f43c1ffb640 1 -- 172.21.15.64:0/1247602149 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43c4039050 con 0x7f43cc0cf640 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.159+0000 7f43c1ffb640 1 -- 172.21.15.64:0/1247602149 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43c4045070 con 0x7f43cc0cf640 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.159+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f43cc15c9c0 con 0x7f43cc0cf640 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.159+0000 7f43c1ffb640 1 -- 172.21.15.64:0/1247602149 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43c4009450 con 0x7f43cc0cf640 2024-03-04T07:31:22.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.159+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f43cc15cf80 con 0x7f43cc0cf640 2024-03-04T07:31:22.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.160+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f43cc0752d0 con 0x7f43cc0cf640 2024-03-04T07:31:22.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.161+0000 7f43c1ffb640 1 -- 172.21.15.64:0/1247602149 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f43c403e810 con 0x7f43cc0cf640 2024-03-04T07:31:22.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.161+0000 7f43c1ffb640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f43b006b4d0 0x7f43b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:22.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.161+0000 7f43c1ffb640 1 -- 172.21.15.64:0/1247602149 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f43c40accc0 con 0x7f43cc0cf640 2024-03-04T07:31:22.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.161+0000 7f43d1116640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f43b006b4d0 0x7f43b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:22.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.162+0000 7f43d1116640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f43b006b4d0 0x7f43b006d990 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f43c8001840 tx=0x7f43c8009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:22.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.166+0000 7f43c1ffb640 1 -- 172.21.15.64:0/1247602149 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f43c407b4f0 con 0x7f43cc0cf640 2024-03-04T07:31:22.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.376+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "d"} v 0) v1 -- 0x7f496c0cc020 con 0x7f496c15bf90 2024-03-04T07:31:22.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.382+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "b"} v 0) v1 -- 0x7f0ef8066eb0 con 0x7f0ef8160610 2024-03-04T07:31:22.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.422+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "c"} v 0) v1 -- 0x7f43cc0cd380 con 0x7f43cc0cf640 2024-03-04T07:31:22.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:22.429+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "a"} v 0) v1 -- 0x7f32fc0c87c0 con 0x7f32fc06fbf0 2024-03-04T07:31:23.154 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:31:23.154+0000 7ff841c6b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:31:23.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.156+0000 7f49697fa640 1 -- 172.21.15.64:0/4210111833 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "d"}]=0 failed mds gid 6634 v51) v1 ==== 95+0+0 (secure 0 0 0) 0x7f496007af10 con 0x7f496c15bf90 2024-03-04T07:31:23.155 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 6634 2024-03-04T07:31:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f495806b4d0 msgr2=0x7f495806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f495806b4d0 0x7f495806d990 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f495c004f60 tx=0x7f495c007e30 comp rx=0 tx=0).stop 2024-03-04T07:31:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f496c15bf90 msgr2=0x7f496c0b1a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f496c15bf90 0x7f496c0b1a30 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f496002b830 tx=0x7f49600046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 shutdown_connections 2024-03-04T07:31:23.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f495806b4d0 0x7f495806d990 unknown :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f496c0c56e0 0x7f496c152560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f496c15bf90 0x7f496c0b1a30 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 --2- 172.21.15.64:0/4210111833 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f496c0cd430 0x7f496c0cb8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 >> 172.21.15.64:0/4210111833 conn(0x7f496c0d1d10 msgr2=0x7f496c15c760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 shutdown_connections 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.158+0000 7f4972d0d640 1 -- 172.21.15.64:0/4210111833 wait complete. 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.160+0000 7f0eeaffd640 1 -- 172.21.15.64:0/3067705499 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "b"}]=0 failed mds gid 7353 v52) v1 ==== 95+0+0 (secure 0 0 0) 0x7f0ef00586f0 con 0x7f0ef8160610 2024-03-04T07:31:23.161 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 7353 2024-03-04T07:31:23.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.162+0000 7f43c1ffb640 1 -- 172.21.15.64:0/1247602149 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "c"}]=0 failed mds gid 6619 v53) v1 ==== 95+0+0 (secure 0 0 0) 0x7f43c407f590 con 0x7f43cc0cf640 2024-03-04T07:31:23.162 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 6619 2024-03-04T07:31:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0edc06b4d0 msgr2=0x7f0edc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0edc06b4d0 0x7f0edc06d990 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f0ef402b880 tx=0x7f0ef403d040 comp rx=0 tx=0).stop 2024-03-04T07:31:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef8160610 msgr2=0x7f0ef8153990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef8160610 0x7f0ef8153990 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f0ef000c730 tx=0x7f0ef000cbf0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 shutdown_connections 2024-03-04T07:31:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0edc06b4d0 0x7f0edc06d990 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0ef8160610 0x7f0ef8153990 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0ef81600a0 0x7f0ef80aa910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 --2- 172.21.15.64:0/3067705499 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0ef815fcc0 0x7f0ef806e990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 >> 172.21.15.64:0/3067705499 conn(0x7f0ef80d0510 msgr2=0x7f0ef80ceb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 shutdown_connections 2024-03-04T07:31:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.163+0000 7f0f006c3640 1 -- 172.21.15.64:0/3067705499 wait complete. 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f43b006b4d0 msgr2=0x7f43b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f43b006b4d0 0x7f43b006d990 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f43c8001840 tx=0x7f43c8009040 comp rx=0 tx=0).stop 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc0cf640 msgr2=0x7f43cc15a730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc0cf640 0x7f43cc15a730 secure :-1 s=READY pgs=1153 cs=0 l=1 rev1=1 crypto rx=0x7f43c4033000 tx=0x7f43c4004a40 comp rx=0 tx=0).stop 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 shutdown_connections 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f43b006b4d0 0x7f43b006d990 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43cc0cf640 0x7f43cc15a730 unknown :-1 s=CLOSED pgs=1153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43cc0b3a90 0x7f43cc13c420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 --2- 172.21.15.64:0/1247602149 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43cc06ec60 0x7f43cc15c330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 >> 172.21.15.64:0/1247602149 conn(0x7f43cc0d0be0 msgr2=0x7f43cc06e530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:23.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 shutdown_connections 2024-03-04T07:31:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:23.165+0000 7f43d337a640 1 -- 172.21.15.64:0/1247602149 wait complete. 2024-03-04T07:31:23.176 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:31:23.176 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:31:23.178 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:31:23.179 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:23.179 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:31:23.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:31:23.180 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:31:23.181 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:31:23.183 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:31:23.183 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:23.222 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:31:23.222 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:23.264 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:31:23.266 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:31:23.304 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:31:24.147 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:31:24.147+0000 7ff841c6b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:31:24.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.149+0000 7f32f97fa640 1 -- 172.21.15.64:0/3314305751 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "a"}]=0 failed mds gid 7341 v54) v1 ==== 95+0+0 (secure 0 0 0) 0x7f32f005bec0 con 0x7f32fc06fbf0 2024-03-04T07:31:24.149 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 7341 2024-03-04T07:31:24.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32e006b520 msgr2=0x7f32e006d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32e006b520 0x7f32e006d9e0 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f32fc15cd60 tx=0x7f32e8007d30 comp rx=0 tx=0).stop 2024-03-04T07:31:24.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32fc06fbf0 msgr2=0x7f32fc0cb800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32fc06fbf0 0x7f32fc0cb800 secure :-1 s=READY pgs=1150 cs=0 l=1 rev1=1 crypto rx=0x7f32f000c3f0 tx=0x7f32f000c8b0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 shutdown_connections 2024-03-04T07:31:24.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f32e006b520 0x7f32e006d9e0 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32fc15e140 0x7f32fc155ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f32fc0b0e90 0x7f32fc0c97f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 --2- 172.21.15.64:0/3314305751 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32fc06fbf0 0x7f32fc0cb800 unknown :-1 s=CLOSED pgs=1150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 >> 172.21.15.64:0/3314305751 conn(0x7f32fc0d1d10 msgr2=0x7f32fc0b04f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:24.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.152+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 shutdown_connections 2024-03-04T07:31:24.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.153+0000 7f33031b6640 1 -- 172.21.15.64:0/3314305751 wait complete. 2024-03-04T07:31:24.170 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:31:24.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:31:24.212 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:31:24.212 DEBUG:teuthology.parallel:result is None 2024-03-04T07:31:24.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:24.295 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:31:24.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.321+0000 7f054d193640 1 Processor -- start 2024-03-04T07:31:24.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.321+0000 7f054d193640 1 -- start start 2024-03-04T07:31:24.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054d193640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f054815e100 0x7f05480d1cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:24.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054d193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d2a30 0x7f05480b1070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054d193640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f05480d0100 con 0x7f05480d4c40 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054d193640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f05480cf860 con 0x7f054815e100 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054d193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f05480cfb40 con 0x7f05480d2a30 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f0546d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f054815e100 0x7f05480d1cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054659c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f05480d4c40 0x7f05480d2320 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57858/0 (socket says 172.21.15.64:57858) 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054659c640 1 -- 172.21.15.64:0/2615225709 learned_addr learned my addr 172.21.15.64:0/2615225709 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054759e640 1 --2- 172.21.15.64:0/2615225709 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d2a30 0x7f05480b1070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f0546d9d640 1 -- 172.21.15.64:0/2615225709 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d2a30 msgr2=0x7f05480b1070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f0546d9d640 1 --2- 172.21.15.64:0/2615225709 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d2a30 0x7f05480b1070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f0546d9d640 1 -- 172.21.15.64:0/2615225709 >> v1:172.21.15.64:6789/0 conn(0x7f05480d4c40 legacy=0x7f05480d2320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f0546d9d640 1 -- 172.21.15.64:0/2615225709 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05480cfe20 con 0x7f054815e100 2024-03-04T07:31:24.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.322+0000 7f054759e640 1 --2- 172.21.15.64:0/2615225709 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d2a30 0x7f05480b1070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.323+0000 7f0546d9d640 1 --2- 172.21.15.64:0/2615225709 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f054815e100 0x7f05480d1cd0 secure :-1 s=READY pgs=1161 cs=0 l=1 rev1=1 crypto rx=0x7f05380097a0 tx=0x7f053802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e8049b8eecf2e477 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.323+0000 7f0545d9b640 1 -- 172.21.15.64:0/2615225709 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0538035020 con 0x7f054815e100 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.323+0000 7f0545d9b640 1 -- 172.21.15.64:0/2615225709 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0538031830 con 0x7f054815e100 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.323+0000 7f0545d9b640 1 -- 172.21.15.64:0/2615225709 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0538031c00 con 0x7f054815e100 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 -- 172.21.15.64:0/2615225709 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f054815e100 msgr2=0x7f05480d1cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 --2- 172.21.15.64:0/2615225709 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f054815e100 0x7f05480d1cd0 secure :-1 s=READY pgs=1161 cs=0 l=1 rev1=1 crypto rx=0x7f05380097a0 tx=0x7f053802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 -- 172.21.15.64:0/2615225709 shutdown_connections 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 --2- 172.21.15.64:0/2615225709 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d2a30 0x7f05480b1070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 --2- 172.21.15.64:0/2615225709 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f054815e100 0x7f05480d1cd0 unknown :-1 s=CLOSED pgs=1161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 -- 172.21.15.64:0/2615225709 >> 172.21.15.64:0/2615225709 conn(0x7f05480d3ef0 msgr2=0x7f05480d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 -- 172.21.15.64:0/2615225709 shutdown_connections 2024-03-04T07:31:24.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.324+0000 7f054d193640 1 -- 172.21.15.64:0/2615225709 wait complete. 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.325+0000 7f054d193640 1 Processor -- start 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.325+0000 7f054d193640 1 -- start start 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f054d193640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05480d2a30 0x7f05481f6e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f054d193640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d4c40 0x7f0548160aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f054d193640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f054815e100 0x7f0548160fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f054d193640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f054815e4e0 con 0x7f05480d2a30 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f054d193640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f05480cf510 con 0x7f054815e100 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f054d193640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f054815e660 con 0x7f05480d4c40 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05480d2a30 0x7f05481f6e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05480d2a30 0x7f05481f6e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45062/0 (socket says 172.21.15.64:45062) 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 -- 172.21.15.64:0/370316282 learned_addr learned my addr 172.21.15.64:0/370316282 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:24.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 -- 172.21.15.64:0/370316282 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d4c40 msgr2=0x7f0548160aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d4c40 0x7f0548160aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 -- 172.21.15.64:0/370316282 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f054815e100 msgr2=0x7f0548160fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f054815e100 0x7f0548160fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 -- 172.21.15.64:0/370316282 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05481f5680 con 0x7f05480d2a30 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.326+0000 7f0546d9d640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05480d2a30 0x7f05481f6e10 secure :-1 s=READY pgs=1162 cs=0 l=1 rev1=1 crypto rx=0x7f0538009890 tx=0x7f05380046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.327+0000 7f053f7fe640 1 -- 172.21.15.64:0/370316282 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0538041070 con 0x7f05480d2a30 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.327+0000 7f053f7fe640 1 -- 172.21.15.64:0/370316282 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f053803aa30 con 0x7f05480d2a30 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.327+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f054809f220 con 0x7f05480d2a30 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.327+0000 7f053f7fe640 1 -- 172.21.15.64:0/370316282 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0538035040 con 0x7f05480d2a30 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.327+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f054809f7e0 con 0x7f05480d2a30 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.328+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f05480d3680 con 0x7f05480d2a30 2024-03-04T07:31:24.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.328+0000 7f053f7fe640 1 -- 172.21.15.64:0/370316282 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f053804c060 con 0x7f05480d2a30 2024-03-04T07:31:24.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.328+0000 7f053f7fe640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f052806b4d0 0x7f052806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:24.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.329+0000 7f053f7fe640 1 -- 172.21.15.64:0/370316282 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f05380abcf0 con 0x7f05480d2a30 2024-03-04T07:31:24.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.329+0000 7f054659c640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f052806b4d0 0x7f052806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:24.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.329+0000 7f054659c640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f052806b4d0 0x7f052806d990 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f0530000f60 tx=0x7f0530007a80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:24.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.333+0000 7f053f7fe640 1 -- 172.21.15.64:0/370316282 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f05480d3680 con 0x7f05480d2a30 2024-03-04T07:31:24.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.552+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f05480d1490 con 0x7f05480d2a30 2024-03-04T07:31:24.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.552+0000 7f053f7fe640 1 -- 172.21.15.64:0/370316282 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 55 v55) v1 ==== 94+0+4182 (secure 0 0 0) 0x7f05380b0070 con 0x7f05480d2a30 2024-03-04T07:31:24.552 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:24.552 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54}],"filesystems":[{"mdsmap":{"epoch":55,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:24.153587+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:replay","state_seq":1,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":7}]} 2024-03-04T07:31:24.553 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 55 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f052806b4d0 msgr2=0x7f052806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f052806b4d0 0x7f052806d990 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f0530000f60 tx=0x7f0530007a80 comp rx=0 tx=0).stop 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05480d2a30 msgr2=0x7f05481f6e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05480d2a30 0x7f05481f6e10 secure :-1 s=READY pgs=1162 cs=0 l=1 rev1=1 crypto rx=0x7f0538009890 tx=0x7f05380046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 shutdown_connections 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f052806b4d0 0x7f052806d990 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f054815e100 0x7f0548160fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f05480d4c40 0x7f0548160aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 --2- 172.21.15.64:0/370316282 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f05480d2a30 0x7f05481f6e10 unknown :-1 s=CLOSED pgs=1162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:24.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 >> 172.21.15.64:0/370316282 conn(0x7f05480d3ef0 msgr2=0x7f05480b0f70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:24.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 shutdown_connections 2024-03-04T07:31:24.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:24.554+0000 7f054d193640 1 -- 172.21.15.64:0/370316282 wait complete. 2024-03-04T07:31:24.967 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 55, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:31:14.475450+0000', 'modified': '2024-03-04T07:31:24.153587+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 94, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7557}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7557': {'gid': 7557, 'name': 'c', 'rank': 0, 'incarnation': 55, 'state': 'up:replay', 'state_seq': 1, 'addr': '172.21.15.64:6835/3393965936', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 3393965936}, {'type': 'v1', 'addr': '172.21.15.64:6835', 'nonce': 3393965936}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-04T07:31:24.968 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_7557:up:replay 2024-03-04T07:31:25.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:26.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.083+0000 7f2a6cd56640 1 Processor -- start 2024-03-04T07:31:26.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.083+0000 7f2a6cd56640 1 -- start start 2024-03-04T07:31:26.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6cd56640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 0x7f2a681691f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6cd56640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2a680b4ab0 con 0x7f2a6815e350 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6cd56640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2a6806c380 con 0x7f2a680b6580 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6cd56640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a680b50e0 con 0x7f2a680d0de0 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6659c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2a6815e350 0x7f2a68161ec0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57872/0 (socket says 172.21.15.64:57872) 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6659c640 1 -- 172.21.15.64:0/2824304118 learned_addr learned my addr 172.21.15.64:0/2824304118 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3444112749 0 0) 0x7f2a680b4ab0 con 0x7f2a6815e350 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2a50003520 con 0x7f2a6815e350 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a66d9d640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 0x7f2a681691f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3391276053 0 0) 0x7f2a50003520 con 0x7f2a6815e350 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 msgr2=0x7f2a681691f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 0x7f2a681691f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 >> v1:172.21.15.66:6789/0 conn(0x7f2a680b6580 legacy=0x7f2a681625d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:26.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a6806c660 con 0x7f2a6815e350 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.084+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2a58001670 con 0x7f2a6815e350 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 >> v1:172.21.15.64:6789/0 conn(0x7f2a6815e350 legacy=0x7f2a68161ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6559a640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a500039e0 0x7f2a50003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6559a640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a50004800 0x7f2a50004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6559a640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 0x7f2a5000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a680b50e0 con 0x7f2a500039e0 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a50005550 con 0x7f2a50004800 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a5000bf00 con 0x7f2a680d0de0 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6659c640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a500039e0 0x7f2a50003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a66d9d640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 0x7f2a5000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a65d9b640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a50004800 0x7f2a50004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6659c640 1 -- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 msgr2=0x7f2a5000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6659c640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 0x7f2a5000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6659c640 1 -- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a50004800 msgr2=0x7f2a50004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6659c640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a50004800 0x7f2a50004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a6659c640 1 -- 172.21.15.64:0/2824304118 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a50003520 con 0x7f2a500039e0 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a65d9b640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a50004800 0x7f2a50004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.085+0000 7f2a66d9d640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a680d0de0 0x7f2a5000b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.086+0000 7f2a6659c640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a500039e0 0x7f2a50003e00 secure :-1 s=READY pgs=1164 cs=0 l=1 rev1=1 crypto rx=0x7f2a5800c210 tx=0x7f2a50005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=edf6ae1204658f2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.086+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a58007d30 con 0x7f2a500039e0 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.086+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a58030070 con 0x7f2a500039e0 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.086+0000 7f2a6559a640 1 -- 172.21.15.64:0/2824304118 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a5802cd60 con 0x7f2a500039e0 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.086+0000 7f2a6cd56640 1 -- 172.21.15.64:0/2824304118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a500039e0 msgr2=0x7f2a50003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.086+0000 7f2a6cd56640 1 --2- 172.21.15.64:0/2824304118 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a500039e0 0x7f2a50003e00 secure :-1 s=READY pgs=1164 cs=0 l=1 rev1=1 crypto rx=0x7f2a5800c210 tx=0x7f2a50005830 comp rx=0 tx=0).stop 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.086+0000 7f2a66d9d640 1 -- 172.21.15.64:0/2824304118 reap_dead start 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.087+0000 7f2a6cd56640 1 -- 172.21.15.64:0/2824304118 shutdown_connections 2024-03-04T07:31:26.089 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.087+0000 7f2a6cd56640 1 -- 172.21.15.64:0/2824304118 >> 172.21.15.64:0/2824304118 conn(0x7f2a680d1c60 msgr2=0x7f2a680d2070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:26.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.087+0000 7f2a6cd56640 1 -- 172.21.15.64:0/2824304118 shutdown_connections 2024-03-04T07:31:26.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.087+0000 7f2a6cd56640 1 -- 172.21.15.64:0/2824304118 wait complete. 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.088+0000 7f2a6cd56640 1 Processor -- start 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.088+0000 7f2a6cd56640 1 -- start start 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6cd56640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a500039e0 0x7f2a6814ab00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6cd56640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a68157f70 0x7f2a681583f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6cd56640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a68156470 0x7f2a681568f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6cd56640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a6815e730 con 0x7f2a68156470 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6cd56640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a680d1270 con 0x7f2a500039e0 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6cd56640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2a68162ce0 con 0x7f2a68157f70 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a68156470 0x7f2a681568f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a68156470 0x7f2a681568f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45084/0 (socket says 172.21.15.64:45084) 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 -- 172.21.15.64:0/4227244230 learned_addr learned my addr 172.21.15.64:0/4227244230 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6659c640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a500039e0 0x7f2a6814ab00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a65d9b640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a68157f70 0x7f2a681583f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 -- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a68157f70 msgr2=0x7f2a681583f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a68157f70 0x7f2a681583f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 -- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a500039e0 msgr2=0x7f2a6814ab00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a500039e0 0x7f2a6814ab00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 -- 172.21.15.64:0/4227244230 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a681f3e10 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6659c640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a500039e0 0x7f2a6814ab00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a66d9d640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a68156470 0x7f2a681568f0 secure :-1 s=READY pgs=1165 cs=0 l=1 rev1=1 crypto rx=0x7f2a68071cb0 tx=0x7f2a60004840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a56ffd640 1 -- 172.21.15.64:0/4227244230 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a60007c50 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a56ffd640 1 -- 172.21.15.64:0/4227244230 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2a60019070 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.089+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a681571a0 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.090+0000 7f2a56ffd640 1 -- 172.21.15.64:0/4227244230 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a60014650 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.090+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a6815e950 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.091+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2a680752d0 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.091+0000 7f2a56ffd640 1 -- 172.21.15.64:0/4227244230 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2a600147f0 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.091+0000 7f2a56ffd640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4806b500 0x7f2a4806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.091+0000 7f2a56ffd640 1 -- 172.21.15.64:0/4227244230 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f2a6008a640 con 0x7f2a68156470 2024-03-04T07:31:26.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.092+0000 7f2a6659c640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4806b500 0x7f2a4806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:26.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.092+0000 7f2a6659c640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4806b500 0x7f2a4806d9c0 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f2a58034000 tx=0x7f2a58009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:26.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.096+0000 7f2a56ffd640 1 -- 172.21.15.64:0/4227244230 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2a60059410 con 0x7f2a68156470 2024-03-04T07:31:26.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.317+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2a68067a70 con 0x7f2a68156470 2024-03-04T07:31:26.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.317+0000 7f2a56ffd640 1 -- 172.21.15.64:0/4227244230 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 57 v57) v1 ==== 94+0+4973 (secure 0 0 0) 0x7f2a60058d10 con 0x7f2a68156470 2024-03-04T07:31:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:26.318 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":57,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":57,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:25.169968+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:rejoin","state_seq":3,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":7}]} 2024-03-04T07:31:26.318 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 57 2024-03-04T07:31:26.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4806b500 msgr2=0x7f2a4806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:26.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4806b500 0x7f2a4806d9c0 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f2a58034000 tx=0x7f2a58009030 comp rx=0 tx=0).stop 2024-03-04T07:31:26.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a68156470 msgr2=0x7f2a681568f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:26.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a68156470 0x7f2a681568f0 secure :-1 s=READY pgs=1165 cs=0 l=1 rev1=1 crypto rx=0x7f2a68071cb0 tx=0x7f2a60004840 comp rx=0 tx=0).stop 2024-03-04T07:31:26.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 shutdown_connections 2024-03-04T07:31:26.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2a4806b500 0x7f2a4806d9c0 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2a68156470 0x7f2a681568f0 unknown :-1 s=CLOSED pgs=1165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2a68157f70 0x7f2a681583f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 --2- 172.21.15.64:0/4227244230 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2a500039e0 0x7f2a6814ab00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:26.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 >> 172.21.15.64:0/4227244230 conn(0x7f2a680d1c60 msgr2=0x7f2a6815c590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:26.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 shutdown_connections 2024-03-04T07:31:26.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:26.319+0000 7f2a6cd56640 1 -- 172.21.15.64:0/4227244230 wait complete. 2024-03-04T07:31:26.733 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 57, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:31:14.475450+0000', 'modified': '2024-03-04T07:31:25.169968+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 94, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7557}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7557': {'gid': 7557, 'name': 'c', 'rank': 0, 'incarnation': 55, 'state': 'up:rejoin', 'state_seq': 3, 'addr': '172.21.15.64:6835/3393965936', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 3393965936}, {'type': 'v1', 'addr': '172.21.15.64:6835', 'nonce': 3393965936}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-04T07:31:26.733 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_7557:up:rejoin 2024-03-04T07:31:27.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.849+0000 7f655686f640 1 Processor -- start 2024-03-04T07:31:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.849+0000 7f655686f640 1 -- start start 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f655686f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f655006d320 0x7f655015eaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f655686f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655006d700 0x7f655015be70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f655686f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6550070aa0 con 0x7f655015c3b0 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f655686f640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f655015efe0 con 0x7f655006d320 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f655686f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6550070d40 con 0x7f655006d700 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f654ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f655006d320 0x7f655015eaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f6554e0c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f655015c3b0 0x7f655015c840 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57886/0 (socket says 172.21.15.64:57886) 2024-03-04T07:31:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f6554e0c640 1 -- 172.21.15.64:0/1382395115 learned_addr learned my addr 172.21.15.64:0/1382395115 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:27.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f654f7fe640 1 --2- 172.21.15.64:0/1382395115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655006d700 0x7f655015be70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:27.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f654ffff640 1 -- 172.21.15.64:0/1382395115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655006d700 msgr2=0x7f655015be70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:27.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f654ffff640 1 --2- 172.21.15.64:0/1382395115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655006d700 0x7f655015be70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:27.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f654ffff640 1 -- 172.21.15.64:0/1382395115 >> v1:172.21.15.64:6789/0 conn(0x7f655015c3b0 legacy=0x7f655015c840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:27.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f654ffff640 1 -- 172.21.15.64:0/1382395115 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6550071020 con 0x7f655006d320 2024-03-04T07:31:27.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.850+0000 7f654effd640 1 -- 172.21.15.64:0/1382395115 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 888079975 0 0) 0x7f6550070aa0 con 0x7f655015c3b0 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.851+0000 7f654ffff640 1 --2- 172.21.15.64:0/1382395115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f655015eaa0 secure :-1 s=READY pgs=1167 cs=0 l=1 rev1=1 crypto rx=0x7f65400097a0 tx=0x7f654002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a6c60b268cf6a4e3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.851+0000 7f654effd640 1 -- 172.21.15.64:0/1382395115 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6540035020 con 0x7f655006d320 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.851+0000 7f654effd640 1 -- 172.21.15.64:0/1382395115 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6540031830 con 0x7f655006d320 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.851+0000 7f654effd640 1 -- 172.21.15.64:0/1382395115 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6540031c00 con 0x7f655006d320 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 -- 172.21.15.64:0/1382395115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 msgr2=0x7f655015eaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 --2- 172.21.15.64:0/1382395115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f655015eaa0 secure :-1 s=READY pgs=1167 cs=0 l=1 rev1=1 crypto rx=0x7f65400097a0 tx=0x7f654002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 -- 172.21.15.64:0/1382395115 shutdown_connections 2024-03-04T07:31:27.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 --2- 172.21.15.64:0/1382395115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655006d700 0x7f655015be70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:27.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 --2- 172.21.15.64:0/1382395115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f655015eaa0 unknown :-1 s=CLOSED pgs=1167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:27.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 -- 172.21.15.64:0/1382395115 >> 172.21.15.64:0/1382395115 conn(0x7f65500b17a0 msgr2=0x7f65500b1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:27.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 -- 172.21.15.64:0/1382395115 shutdown_connections 2024-03-04T07:31:27.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.852+0000 7f655686f640 1 -- 172.21.15.64:0/1382395115 wait complete. 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.853+0000 7f655686f640 1 Processor -- start 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.853+0000 7f655686f640 1 -- start start 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f655686f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f6550153a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f655686f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f655006d700 0x7f65500cdf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f655686f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655015c3b0 0x7f655013f5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f655686f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65500718c0 con 0x7f655006d320 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f655686f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65500715e0 con 0x7f655006d700 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f655686f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f655006e610 con 0x7f655015c3b0 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f6550153a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f6550153a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45094/0 (socket says 172.21.15.64:45094) 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 -- 172.21.15.64:0/4254546838 learned_addr learned my addr 172.21.15.64:0/4254546838 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654f7fe640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f655006d700 0x7f65500cdf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:27.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 -- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655015c3b0 msgr2=0x7f655013f5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f6554e0c640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655015c3b0 0x7f655013f5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655015c3b0 0x7f655013f5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 -- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f655006d700 msgr2=0x7f65500cdf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f655006d700 0x7f65500cdf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.854+0000 7f654ffff640 1 -- 172.21.15.64:0/4254546838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6550140360 con 0x7f655006d320 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.855+0000 7f654ffff640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f6550153a10 secure :-1 s=READY pgs=1168 cs=0 l=1 rev1=1 crypto rx=0x7f6540000c00 tx=0x7f65400046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.855+0000 7f654cff9640 1 -- 172.21.15.64:0/4254546838 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6540035070 con 0x7f655006d320 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.855+0000 7f654cff9640 1 -- 172.21.15.64:0/4254546838 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6540004230 con 0x7f655006d320 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.855+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65501406a0 con 0x7f655006d320 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.855+0000 7f654cff9640 1 -- 172.21.15.64:0/4254546838 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f654003a6f0 con 0x7f655006d320 2024-03-04T07:31:27.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.855+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6550140ee0 con 0x7f655006d320 2024-03-04T07:31:27.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.856+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f65500752d0 con 0x7f655006d320 2024-03-04T07:31:27.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.856+0000 7f654cff9640 1 -- 172.21.15.64:0/4254546838 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6540038020 con 0x7f655006d320 2024-03-04T07:31:27.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.857+0000 7f654cff9640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f653006b4d0 0x7f653006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:27.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.857+0000 7f654cff9640 1 -- 172.21.15.64:0/4254546838 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f65400abfa0 con 0x7f655006d320 2024-03-04T07:31:27.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.857+0000 7f654f7fe640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f653006b4d0 0x7f653006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:27.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.857+0000 7f654f7fe640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f653006b4d0 0x7f653006d990 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f6538006fd0 tx=0x7f6538008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:27.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:27.861+0000 7f654cff9640 1 -- 172.21.15.64:0/4254546838 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f65501406a0 con 0x7f655006d320 2024-03-04T07:31:28.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.082+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f655006dd90 con 0x7f655006d320 2024-03-04T07:31:28.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.083+0000 7f654cff9640 1 -- 172.21.15.64:0/4254546838 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 58 v58) v1 ==== 94+0+4979 (secure 0 0 0) 0x7f6540040070 con 0x7f655006d320 2024-03-04T07:31:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:28.083 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":58,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":58,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:26.177365+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:clientreplay","state_seq":4,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":7}]} 2024-03-04T07:31:28.083 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 58 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f653006b4d0 msgr2=0x7f653006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f653006b4d0 0x7f653006d990 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f6538006fd0 tx=0x7f6538008040 comp rx=0 tx=0).stop 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 msgr2=0x7f6550153a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f6550153a10 secure :-1 s=READY pgs=1168 cs=0 l=1 rev1=1 crypto rx=0x7f6540000c00 tx=0x7f65400046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 shutdown_connections 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f653006b4d0 0x7f653006d990 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f655015c3b0 0x7f655013f5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f655006d700 0x7f65500cdf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 --2- 172.21.15.64:0/4254546838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f655006d320 0x7f6550153a10 unknown :-1 s=CLOSED pgs=1168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 >> 172.21.15.64:0/4254546838 conn(0x7f65500b17a0 msgr2=0x7f655014fa40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:28.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 shutdown_connections 2024-03-04T07:31:28.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:28.085+0000 7f655686f640 1 -- 172.21.15.64:0/4254546838 wait complete. 2024-03-04T07:31:28.498 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 58, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:31:14.475450+0000', 'modified': '2024-03-04T07:31:26.177365+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 94, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7557}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7557': {'gid': 7557, 'name': 'c', 'rank': 0, 'incarnation': 55, 'state': 'up:clientreplay', 'state_seq': 4, 'addr': '172.21.15.64:6835/3393965936', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 3393965936}, {'type': 'v1', 'addr': '172.21.15.64:6835', 'nonce': 3393965936}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-04T07:31:28.498 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_7557:up:clientreplay 2024-03-04T07:31:29.158 INFO:tasks.ceph.mds.c.smithi064.stderr:2024-03-04T07:31:29.159+0000 7f6b7ee82640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:31:29.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.601+0000 7fea3adf6640 1 Processor -- start 2024-03-04T07:31:29.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.601+0000 7fea3adf6640 1 -- start start 2024-03-04T07:31:29.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.602+0000 7fea3adf6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fea340b1ef0 0x7fea340b2380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.602+0000 7fea3adf6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 0x7fea340adbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.602+0000 7fea3adf6640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fea3406be50 con 0x7fea3415df80 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.602+0000 7fea3adf6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fea3406c130 con 0x7fea340b1ef0 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.602+0000 7fea3adf6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fea3406c410 con 0x7fea340ad760 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.602+0000 7fea38b92640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fea3415df80 0x7fea340b13a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57888/0 (socket says 172.21.15.64:57888) 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.602+0000 7fea38b92640 1 -- 172.21.15.64:0/3944051037 learned_addr learned my addr 172.21.15.64:0/3944051037 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea33fff640 1 --2- 172.21.15.64:0/3944051037 >> v2:172.21.15.66:3300/0 conn(0x7fea340b1ef0 0x7fea340b2380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea39393640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 0x7fea340adbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3814668009 0 0) 0x7fea3406be50 con 0x7fea3415df80 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fea24003520 con 0x7fea3415df80 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3212827814 0 0) 0x7fea24003520 con 0x7fea3415df80 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 msgr2=0x7fea340adbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 0x7fea340adbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 >> v2:172.21.15.66:3300/0 conn(0x7fea340b1ef0 msgr2=0x7fea340b2380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:29.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 --2- 172.21.15.64:0/3944051037 >> v2:172.21.15.66:3300/0 conn(0x7fea340b1ef0 0x7fea340b2380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea3406c6f0 con 0x7fea3415df80 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fea28001670 con 0x7fea3415df80 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 >> v1:172.21.15.64:6789/0 conn(0x7fea3415df80 legacy=0x7fea340b13a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea240039b0 0x7fea24003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 0x7fea24004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea24004d40 0x7fea240051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea3406c130 con 0x7fea24004d40 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea3406c410 con 0x7fea240039b0 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fea2400b9c0 con 0x7fea340ad760 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea39393640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea24004d40 0x7fea240051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea33fff640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 0x7fea24004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea38b92640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea240039b0 0x7fea24003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.603+0000 7fea39393640 1 -- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 msgr2=0x7fea24004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.604+0000 7fea39393640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea340ad760 0x7fea24004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.604+0000 7fea39393640 1 -- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea240039b0 msgr2=0x7fea24003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.604+0000 7fea39393640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea240039b0 0x7fea24003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.604+0000 7fea39393640 1 -- 172.21.15.64:0/3944051037 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea24003520 con 0x7fea24004d40 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.604+0000 7fea38b92640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea240039b0 0x7fea24003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea39393640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea24004d40 0x7fea240051c0 secure :-1 s=READY pgs=1170 cs=0 l=1 rev1=1 crypto rx=0x7fea2c009fc0 tx=0x7fea24005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3b80bc9013b5339b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea2c039070 con 0x7fea24004d40 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fea2c02cc20 con 0x7fea24004d40 2024-03-04T07:31:29.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea337fe640 1 -- 172.21.15.64:0/3944051037 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea2c009c10 con 0x7fea24004d40 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea3adf6640 1 -- 172.21.15.64:0/3944051037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea24004d40 msgr2=0x7fea240051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea3adf6640 1 --2- 172.21.15.64:0/3944051037 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea24004d40 0x7fea240051c0 secure :-1 s=READY pgs=1170 cs=0 l=1 rev1=1 crypto rx=0x7fea2c009fc0 tx=0x7fea24005830 comp rx=0 tx=0).stop 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea39393640 1 -- 172.21.15.64:0/3944051037 reap_dead start 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea3adf6640 1 -- 172.21.15.64:0/3944051037 shutdown_connections 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea3adf6640 1 -- 172.21.15.64:0/3944051037 >> 172.21.15.64:0/3944051037 conn(0x7fea340b50c0 msgr2=0x7fea340b5780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea3adf6640 1 -- 172.21.15.64:0/3944051037 shutdown_connections 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.605+0000 7fea3adf6640 1 -- 172.21.15.64:0/3944051037 wait complete. 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.607+0000 7fea3adf6640 1 Processor -- start 2024-03-04T07:31:29.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.607+0000 7fea3adf6640 1 -- start start 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.608+0000 7fea3adf6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea34099920 0x7fea3415cc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.608+0000 7fea3adf6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea34159d70 0x7fea34158ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.608+0000 7fea3adf6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea3415a1d0 0x7fea34159020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.608+0000 7fea3adf6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea3406bc10 con 0x7fea3415a1d0 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.608+0000 7fea3adf6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea3406ba90 con 0x7fea34159d70 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.608+0000 7fea3adf6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fea3415e360 con 0x7fea34099920 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea3415a1d0 0x7fea34159020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea3415a1d0 0x7fea34159020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45126/0 (socket says 172.21.15.64:45126) 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 -- 172.21.15.64:0/3266746414 learned_addr learned my addr 172.21.15.64:0/3266746414 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea38b92640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea34099920 0x7fea3415cc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea33fff640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea34159d70 0x7fea34158ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 -- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea34099920 msgr2=0x7fea3415cc90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea34099920 0x7fea3415cc90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 -- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea34159d70 msgr2=0x7fea34158ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea34159d70 0x7fea34158ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 -- 172.21.15.64:0/3266746414 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea3415bbf0 con 0x7fea3415a1d0 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea38b92640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea34099920 0x7fea3415cc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea39393640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea3415a1d0 0x7fea34159020 secure :-1 s=READY pgs=1171 cs=0 l=1 rev1=1 crypto rx=0x7fea2c002230 tx=0x7fea2c009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea317fa640 1 -- 172.21.15.64:0/3266746414 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea2c039030 con 0x7fea3415a1d0 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea317fa640 1 -- 172.21.15.64:0/3266746414 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fea2c043070 con 0x7fea3415a1d0 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea3415bed0 con 0x7fea3415a1d0 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea317fa640 1 -- 172.21.15.64:0/3266746414 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea2c009c10 con 0x7fea3415a1d0 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.609+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea3415c490 con 0x7fea3415a1d0 2024-03-04T07:31:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.610+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fea340752d0 con 0x7fea3415a1d0 2024-03-04T07:31:29.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.611+0000 7fea317fa640 1 -- 172.21.15.64:0/3266746414 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fea2c03d7f0 con 0x7fea3415a1d0 2024-03-04T07:31:29.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.611+0000 7fea317fa640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fea1806b500 0x7fea1806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:29.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.611+0000 7fea317fa640 1 -- 172.21.15.64:0/3266746414 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fea2c0ac8a0 con 0x7fea3415a1d0 2024-03-04T07:31:29.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.611+0000 7fea38b92640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fea1806b500 0x7fea1806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:29.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.612+0000 7fea38b92640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fea1806b500 0x7fea1806d9c0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7fea28000c00 tx=0x7fea28009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:29.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.616+0000 7fea317fa640 1 -- 172.21.15.64:0/3266746414 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fea2c07b6a0 con 0x7fea3415a1d0 2024-03-04T07:31:29.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.834+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fea34158100 con 0x7fea3415a1d0 2024-03-04T07:31:29.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.834+0000 7fea317fa640 1 -- 172.21.15.64:0/3266746414 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fea2c07afa0 con 0x7fea3415a1d0 2024-03-04T07:31:29.834 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:29.835 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:29.835 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:29.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fea1806b500 msgr2=0x7fea1806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:29.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fea1806b500 0x7fea1806d9c0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7fea28000c00 tx=0x7fea28009040 comp rx=0 tx=0).stop 2024-03-04T07:31:29.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea3415a1d0 msgr2=0x7fea34159020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea3415a1d0 0x7fea34159020 secure :-1 s=READY pgs=1171 cs=0 l=1 rev1=1 crypto rx=0x7fea2c002230 tx=0x7fea2c009750 comp rx=0 tx=0).stop 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 shutdown_connections 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fea1806b500 0x7fea1806d9c0 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fea3415a1d0 0x7fea34159020 unknown :-1 s=CLOSED pgs=1171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fea34159d70 0x7fea34158ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 --2- 172.21.15.64:0/3266746414 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fea34099920 0x7fea3415cc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 >> 172.21.15.64:0/3266746414 conn(0x7fea340b50c0 msgr2=0x7fea3406d2e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 shutdown_connections 2024-03-04T07:31:29.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:29.836+0000 7fea3adf6640 1 -- 172.21.15.64:0/3266746414 wait complete. 2024-03-04T07:31:30.250 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 59, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:31:14.475450+0000', 'modified': '2024-03-04T07:31:28.181905+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 94, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7557}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7557': {'gid': 7557, 'name': 'c', 'rank': 0, 'incarnation': 55, 'state': 'up:active', 'state_seq': 5, 'addr': '172.21.15.64:6835/3393965936', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 3393965936}, {'type': 'v1', 'addr': '172.21.15.64:6835', 'nonce': 3393965936}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [15], 'metadata_pool': 14, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 7557, 'qdb_cluster': [7557]} 2024-03-04T07:31:30.250 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:31:30.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-04T07:31:30.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.354+0000 7f6bcf04e640 1 Processor -- start 2024-03-04T07:31:30.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.354+0000 7f6bcf04e640 1 -- start start 2024-03-04T07:31:30.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.354+0000 7f6bcf04e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6bc80d2680 0x7f6bc80d4a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcf04e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6bc80d2a60 0x7f6bc80cf810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcf04e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6bc80ceab0 con 0x7f6bc80b3050 2024-03-04T07:31:30.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcf04e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6bc80cf380 con 0x7f6bc80d2a60 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcf04e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6bc80d1d10 con 0x7f6bc80d2680 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcd5eb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6bc80d2a60 0x7f6bc80cf810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bc7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6bc80b3050 0x7f6bc80b0b00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:57904/0 (socket says 172.21.15.64:57904) 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bc7fff640 1 -- 172.21.15.64:0/3405566955 learned_addr learned my addr 172.21.15.64:0/3405566955 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bccdea640 1 --2- 172.21.15.64:0/3405566955 >> v2:172.21.15.66:3300/0 conn(0x7f6bc80d2680 0x7f6bc80d4a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcd5eb640 1 -- 172.21.15.64:0/3405566955 >> v2:172.21.15.66:3300/0 conn(0x7f6bc80d2680 msgr2=0x7f6bc80d4a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcd5eb640 1 --2- 172.21.15.64:0/3405566955 >> v2:172.21.15.66:3300/0 conn(0x7f6bc80d2680 0x7f6bc80d4a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcd5eb640 1 -- 172.21.15.64:0/3405566955 >> v1:172.21.15.64:6789/0 conn(0x7f6bc80b3050 legacy=0x7f6bc80b0b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.355+0000 7f6bcd5eb640 1 -- 172.21.15.64:0/3405566955 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6bc80d1ff0 con 0x7f6bc80d2a60 2024-03-04T07:31:30.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bcd5eb640 1 --2- 172.21.15.64:0/3405566955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80d2a60 0x7f6bc80cf810 secure :-1 s=READY pgs=1173 cs=0 l=1 rev1=1 crypto rx=0x7f6bc808c4c0 tx=0x7f6bc002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7d190824f8691242 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:30.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bc77fe640 1 -- 172.21.15.64:0/3405566955 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bc0039070 con 0x7f6bc80d2a60 2024-03-04T07:31:30.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bc77fe640 1 -- 172.21.15.64:0/3405566955 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bc002cc80 con 0x7f6bc80d2a60 2024-03-04T07:31:30.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bc77fe640 1 -- 172.21.15.64:0/3405566955 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bc0034660 con 0x7f6bc80d2a60 2024-03-04T07:31:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bcf04e640 1 -- 172.21.15.64:0/3405566955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80d2a60 msgr2=0x7f6bc80cf810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/3405566955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80d2a60 0x7f6bc80cf810 secure :-1 s=READY pgs=1173 cs=0 l=1 rev1=1 crypto rx=0x7f6bc808c4c0 tx=0x7f6bc002bed0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bcf04e640 1 -- 172.21.15.64:0/3405566955 shutdown_connections 2024-03-04T07:31:30.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/3405566955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80d2a60 0x7f6bc80cf810 unknown :-1 s=CLOSED pgs=1173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/3405566955 >> v2:172.21.15.66:3300/0 conn(0x7f6bc80d2680 0x7f6bc80d4a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.356+0000 7f6bcf04e640 1 -- 172.21.15.64:0/3405566955 >> 172.21.15.64:0/3405566955 conn(0x7f6bc80d3f30 msgr2=0x7f6bc80d4340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.357+0000 7f6bcf04e640 1 -- 172.21.15.64:0/3405566955 shutdown_connections 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.357+0000 7f6bcf04e640 1 -- 172.21.15.64:0/3405566955 wait complete. 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 Processor -- start 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 -- start start 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80b3050 0x7f6bc81e9cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bc80d2680 0x7f6bc80c75c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bc80d2a60 0x7f6bc80b20f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bc80b1210 con 0x7f6bc80b3050 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bc80cf010 con 0x7f6bc80d2680 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.358+0000 7f6bcf04e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6bc80cfd50 con 0x7f6bc80d2a60 2024-03-04T07:31:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80b3050 0x7f6bc81e9cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80b3050 0x7f6bc81e9cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45146/0 (socket says 172.21.15.64:45146) 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 -- 172.21.15.64:0/2683446819 learned_addr learned my addr 172.21.15.64:0/2683446819 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bc80d2680 0x7f6bc80c75c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bcd5eb640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bc80d2a60 0x7f6bc80b20f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 -- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bc80d2a60 msgr2=0x7f6bc80b20f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bc80d2a60 0x7f6bc80b20f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 -- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bc80d2680 msgr2=0x7f6bc80c75c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bc80d2680 0x7f6bc80c75c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 -- 172.21.15.64:0/2683446819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6bc81e8940 con 0x7f6bc80b3050 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bccdea640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80b3050 0x7f6bc81e9cc0 secure :-1 s=READY pgs=1174 cs=0 l=1 rev1=1 crypto rx=0x7f6bbc00e9b0 tx=0x7f6bbc00ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bc57fa640 1 -- 172.21.15.64:0/2683446819 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bbc017070 con 0x7f6bc80b3050 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bc57fa640 1 -- 172.21.15.64:0/2683446819 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bbc00c9a0 con 0x7f6bc80b3050 2024-03-04T07:31:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bc57fa640 1 -- 172.21.15.64:0/2683446819 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bbc00cd90 con 0x7f6bc80b3050 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6bc81e8c20 con 0x7f6bc80b3050 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.359+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6bc81e91e0 con 0x7f6bc80b3050 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.361+0000 7f6bc57fa640 1 -- 172.21.15.64:0/2683446819 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6bbc01a070 con 0x7f6bc80b3050 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.361+0000 7f6bc57fa640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bac06b4d0 0x7f6bac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.362+0000 7f6bc57fa640 1 -- 172.21.15.64:0/2683446819 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f6bbc08a660 con 0x7f6bc80b3050 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.362+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bac06b4d0 0x7f6bac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.362+0000 7f6bd1c3c740 1 Processor -- start 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.362+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bac06b4d0 0x7f6bac06d990 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f6bc80c7380 tx=0x7f6bb4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.362+0000 7f6bd1c3c740 1 -- start start 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f99f4870 0x5634f99f4c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f99f88c0 0x5634f99fa360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a24600 0x5634f9a24a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634f995e6b0 con 0x5634f99f4870 2024-03-04T07:31:30.378 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634f9907a60 con 0x5634f99f88c0 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5634f995a750 con 0x5634f9a24600 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f99f4870 0x5634f99f4c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f99f4870 0x5634f99f4c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45148/0 (socket says 172.21.15.64:45148) 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 -- 172.21.15.64:0/2784320824 learned_addr learned my addr 172.21.15.64:0/2784320824 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 -- 172.21.15.64:0/2784320824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a24600 msgr2=0x5634f9a24a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a24600 0x5634f9a24a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 -- 172.21.15.64:0/2784320824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f99f88c0 msgr2=0x5634f99fa360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f99f88c0 0x5634f99fa360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f99f88c0 0x5634f99fa360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 -- 172.21.15.64:0/2784320824 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5634f9925f30 con 0x5634f99f4870 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.363+0000 7f6bccdea640 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f99f4870 0x5634f99f4c50 secure :-1 s=READY pgs=1175 cs=0 l=1 rev1=1 crypto rx=0x7f6bbc058cc0 tx=0x7f6bbc058d30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=70ae72ff5747b82f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bb27fc640 1 -- 172.21.15.64:0/2784320824 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bbc021070 con 0x5634f99f4870 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bb27fc640 1 -- 172.21.15.64:0/2784320824 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bbc061440 con 0x5634f99f4870 2024-03-04T07:31:30.379 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bb27fc640 1 -- 172.21.15.64:0/2784320824 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bbc057030 con 0x5634f99f4870 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2784320824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f99f4870 msgr2=0x5634f99f4c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f99f4870 0x5634f99f4c50 secure :-1 s=READY pgs=1175 cs=0 l=1 rev1=1 crypto rx=0x7f6bbc058cc0 tx=0x7f6bbc058d30 comp rx=0 tx=0).stop 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2784320824 shutdown_connections 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a24600 0x5634f9a24a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f99f88c0 0x5634f99fa360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2784320824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f99f4870 0x5634f99f4c50 unknown :-1 s=CLOSED pgs=1175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2784320824 >> 172.21.15.64:0/2784320824 conn(0x5634f99f68f0 msgr2=0x5634f99f81b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2784320824 shutdown_connections 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.364+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2784320824 wait complete. 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.365+0000 7f6bd1c3c740 1 -- start start 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.365+0000 7f6bd1c3c740 1 Processor -- start 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a24600 0x5634f9a39c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a26180 0x5634f9a3fc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 0x5634f9a28370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634f99f8d50 con 0x5634f9a24600 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634f991aea0 con 0x5634f9a26180 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5634f99f9030 con 0x5634f9a27ef0 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 10 client.0 _ll_register_callbacks cb 0x5634f991a870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a24600 0x5634f9a39c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a24600 0x5634f9a39c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45152/0 (socket says 172.21.15.64:45152) 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 -- 172.21.15.64:0/173176539 learned_addr learned my addr 172.21.15.64:0/173176539 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 -- 172.21.15.64:0/173176539 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 msgr2=0x5634f9a28370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 0x5634f9a28370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 -- 172.21.15.64:0/173176539 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a26180 msgr2=0x5634f9a3fc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a26180 0x5634f9a3fc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 -- 172.21.15.64:0/173176539 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5634f991b270 con 0x5634f9a24600 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bccdea640 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a24600 0x5634f9a39c30 secure :-1 s=READY pgs=1176 cs=0 l=1 rev1=1 crypto rx=0x7f6bbc05d040 tx=0x7f6bbc000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c86a04212ec5729a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6ba77fe640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6ba77fe640 1 -- 172.21.15.64:0/173176539 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bbc075060 con 0x5634f9a24600 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6ba77fe640 1 -- 172.21.15.64:0/173176539 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f6bbc058180 con 0x5634f9a24600 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.366+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6bbc061440 con 0x5634f9a24600 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6ba77fe640 1 -- 172.21.15.64:0/173176539 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=59) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6bbc058380 con 0x5634f9a24600 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 10 client.7608 fetch_fsmap learned FSMap version 59 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=59,monmap=2+}) v3 -- 0x5634f96732d0 con 0x5634f9a24600 2024-03-04T07:31:30.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6ba77fe640 1 -- 172.21.15.64:0/173176539 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bbc061440 con 0x5634f9a24600 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6ba77fe640 1 -- 172.21.15.64:0/173176539 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 59) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f6bbc054a80 con 0x5634f9a24600 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 10 client.7608 fetch_fsmap finished waiting for FSMap version 59 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 10 client.7608 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 20 client.7608 populate_metadata read hostname 'smithi064' 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f9a3a1c0 0x5634f9a1f1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 4 client.7608 mds_command: new command op to 7557 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 --> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5634f996bbf0 con 0x5634f9a3a1c0 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f9a3a1c0 0x5634f9a1f1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6ba77fe640 10 client.7608 ms_handle_connect on v2:172.21.15.64:6834/3393965936 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.367+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f9a3a1c0 0x5634f9a1f1e0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5f11ce6997c95e0 server_cookie=658e4fc12df48134 in_seq=0 out_seq=0 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6ba77fe640 1 -- 172.21.15.64:0/173176539 <== mds.0 v2:172.21.15.64:6834/3393965936 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x5634f996bbf0 con 0x5634f9a3a1c0 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6ba77fe640 10 client.7608 handle_command_reply: tid=0 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 client.7608 shutdown 2024-03-04T07:31:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a24600 msgr2=0x5634f9a39c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a24600 0x5634f9a39c30 secure :-1 s=READY pgs=1176 cs=0 l=1 rev1=1 crypto rx=0x7f6bbc05d040 tx=0x7f6bbc000bc0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 shutdown_connections 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 0x5634f9a28370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f9a3a1c0 0x5634f9a1f1e0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a26180 0x5634f9a3fc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/173176539 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a24600 0x5634f9a39c30 unknown :-1 s=CLOSED pgs=1176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6ba77fe640 0 client.7608 ms_handle_reset on v2:172.21.15.64:6834/3393965936 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 >> 172.21.15.64:0/173176539 conn(0x5634f99f4870 msgr2=0x5634f9a225c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 shutdown_connections 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/173176539 wait complete. 2024-03-04T07:31:30.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.368+0000 7f6bd1c3c740 20 client.7608 trim_cache size 0 max 16384 2024-03-04T07:31:30.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.385+0000 7f6bd1c3c740 1 Processor -- start 2024-03-04T07:31:30.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.385+0000 7f6bd1c3c740 1 -- start start 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a249e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a26180 0x5634f9a24f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 0x5634f9a52070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bc815e570 con 0x5634f9a26180 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bc815d970 con 0x5634f9a24600 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6bc815df70 con 0x5634f9a27ef0 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a26180 0x5634f9a24f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a26180 0x5634f9a24f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45162/0 (socket says 172.21.15.64:45162) 2024-03-04T07:31:30.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 -- 172.21.15.64:0/2225877743 learned_addr learned my addr 172.21.15.64:0/2225877743 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bccdea640 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a249e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bcd5eb640 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 0x5634f9a52070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 -- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 msgr2=0x5634f9a52070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 0x5634f9a52070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 -- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 msgr2=0x5634f9a249e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a249e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.386+0000 7f6bc7fff640 1 -- 172.21.15.64:0/2225877743 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5634f973d6b0 con 0x5634f9a26180 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a26180 0x5634f9a24f20 secure :-1 s=READY pgs=1177 cs=0 l=1 rev1=1 crypto rx=0x7f6bb4000f60 tx=0x7f6bb4017040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29b25a3ed1bcbfa2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bb1ffb640 1 -- 172.21.15.64:0/2225877743 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bb4007660 con 0x5634f9a26180 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bb1ffb640 1 -- 172.21.15.64:0/2225877743 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bb4028070 con 0x5634f9a26180 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bb1ffb640 1 -- 172.21.15.64:0/2225877743 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bb4023910 con 0x5634f9a26180 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2225877743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a26180 msgr2=0x5634f9a24f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a26180 0x5634f9a24f20 secure :-1 s=READY pgs=1177 cs=0 l=1 rev1=1 crypto rx=0x7f6bb4000f60 tx=0x7f6bb4017040 comp rx=0 tx=0).stop 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2225877743 shutdown_connections 2024-03-04T07:31:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a27ef0 0x5634f9a52070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a26180 0x5634f9a24f20 unknown :-1 s=CLOSED pgs=1177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/2225877743 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a249e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2225877743 >> 172.21.15.64:0/2225877743 conn(0x5634f99f4870 msgr2=0x5634f9a1f1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:30.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2225877743 shutdown_connections 2024-03-04T07:31:30.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.387+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/2225877743 wait complete. 2024-03-04T07:31:30.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.388+0000 7f6bd1c3c740 1 -- start start 2024-03-04T07:31:30.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.388+0000 7f6bd1c3c740 1 Processor -- start 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a20640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a26180 0x5634f9a20b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bd1c3c740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a27ef0 0x5634f9a19f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634f99f8d50 con 0x5634f9a27ef0 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634f99f9030 con 0x5634f9a24600 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bd1c3c740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5634f9a27710 con 0x5634f9a26180 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bd1c3c740 10 client.0 _ll_register_callbacks cb 0x7f6bbc00cf50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a27ef0 0x5634f9a19f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a27ef0 0x5634f9a19f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:45170/0 (socket says 172.21.15.64:45170) 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 -- 172.21.15.64:0/3506060050 learned_addr learned my addr 172.21.15.64:0/3506060050 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:30.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a26180 0x5634f9a20b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bccdea640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a20640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 -- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a26180 msgr2=0x5634f9a20b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a26180 0x5634f9a20b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 -- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 msgr2=0x5634f9a20640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a20640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bcd5eb640 1 -- 172.21.15.64:0/3506060050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5634f9a1f820 con 0x5634f9a27ef0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.389+0000 7f6bc7fff640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a26180 0x5634f9a20b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bcd5eb640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a27ef0 0x5634f9a19f90 secure :-1 s=READY pgs=1178 cs=0 l=1 rev1=1 crypto rx=0x7f6bc0009790 tx=0x7f6bc0034480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2561a4b7c5842d2f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6ba77fe640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6ba77fe640 1 -- 172.21.15.64:0/3506060050 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6bc0039040 con 0x5634f9a27ef0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6ba77fe640 1 -- 172.21.15.64:0/3506060050 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f6bc0034ac0 con 0x5634f9a27ef0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6bb4028070 con 0x5634f9a27ef0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6ba77fe640 1 -- 172.21.15.64:0/3506060050 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=59) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6bc0050050 con 0x5634f9a27ef0 2024-03-04T07:31:30.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 10 client.7614 fetch_fsmap learned FSMap version 59 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=59,monmap=2+}) v3 -- 0x5634f9aaf950 con 0x5634f9a27ef0 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6ba77fe640 1 -- 172.21.15.64:0/3506060050 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6bb4028070 con 0x5634f9a27ef0 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6ba77fe640 1 -- 172.21.15.64:0/3506060050 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 59) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f6bc0053020 con 0x5634f9a27ef0 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 10 client.7614 fetch_fsmap finished waiting for FSMap version 59 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 10 client.7614 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 20 client.7614 populate_metadata read hostname 'smithi064' 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f99f96d0 0x5634f99f9ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 4 client.7614 mds_command: new command op to 7557 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 --> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5634f9965f40 con 0x5634f99f96d0 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.390+0000 7f6bccdea640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f99f96d0 0x5634f99f9ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6ba77fe640 10 client.7614 ms_handle_connect on v2:172.21.15.64:6834/3393965936 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bccdea640 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f99f96d0 0x5634f99f9ab0 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8721861a375267ab server_cookie=14b5ed2394f00da8 in_seq=0 out_seq=0 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6ba77fe640 1 -- 172.21.15.64:0/3506060050 <== mds.0 v2:172.21.15.64:6834/3393965936 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+331 (crc 0 0 0) 0x5634f9965f40 con 0x5634f99f96d0 2024-03-04T07:31:30.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6ba77fe640 10 client.7614 handle_command_reply: tid=0 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 client.7614 shutdown 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a27ef0 msgr2=0x5634f9a19f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a27ef0 0x5634f9a19f90 secure :-1 s=READY pgs=1178 cs=0 l=1 rev1=1 crypto rx=0x7f6bc0009790 tx=0x7f6bc0034480 comp rx=0 tx=0).stop 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 shutdown_connections 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5634f9a27ef0 0x5634f9a19f90 unknown :-1 s=CLOSED pgs=1178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5634f9a26180 0x5634f9a20b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.64:6834/3393965936,v1:172.21.15.64:6835/3393965936] conn(0x5634f99f96d0 0x5634f99f9ab0 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 --2- 172.21.15.64:0/3506060050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5634f9a24600 0x5634f9a20640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 >> 172.21.15.64:0/3506060050 conn(0x5634f99f4870 msgr2=0x5634f99f3510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6ba77fe640 0 client.7614 ms_handle_reset on v2:172.21.15.64:6834/3393965936 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 shutdown_connections 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 1 -- 172.21.15.64:0/3506060050 wait complete. 2024-03-04T07:31:30.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.391+0000 7f6bd1c3c740 20 client.7614 trim_cache size 0 max 16384 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 7557, 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 6.2334247420000004, 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 59, 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 94, 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 94, 2024-03-04T07:31:30.398 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 7.0849746820000004 2024-03-04T07:31:30.399 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:31:30.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bac06b4d0 msgr2=0x7f6bac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bac06b4d0 0x7f6bac06d990 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f6bc80c7380 tx=0x7f6bb4009040 comp rx=0 tx=0).stop 2024-03-04T07:31:30.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80b3050 msgr2=0x7f6bc81e9cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80b3050 0x7f6bc81e9cc0 secure :-1 s=READY pgs=1174 cs=0 l=1 rev1=1 crypto rx=0x7f6bbc00e9b0 tx=0x7f6bbc00ee70 comp rx=0 tx=0).stop 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 shutdown_connections 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bac06b4d0 0x7f6bac06d990 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bc80d2a60 0x7f6bc80b20f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bc80d2680 0x7f6bc80c75c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 --2- 172.21.15.64:0/2683446819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bc80b3050 0x7f6bc81e9cc0 unknown :-1 s=CLOSED pgs=1174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 >> 172.21.15.64:0/2683446819 conn(0x7f6bc80d3f30 msgr2=0x7f6bc806c3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 shutdown_connections 2024-03-04T07:31:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:30.392+0000 7f6bcf04e640 1 -- 172.21.15.64:0/2683446819 wait complete. 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> subdir = "delete_me" 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> size_unit = 1024 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> file_multiplier = 200 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> os.mkdir(os.path.join(mountpoint, subdir)) 2024-03-04T07:31:30.406 DEBUG:teuthology.orchestra.run.smithi064:> for i in range(0, file_multiplier): 2024-03-04T07:31:30.407 DEBUG:teuthology.orchestra.run.smithi064:> for size in range(0, 16*size_unit, size_unit): 2024-03-04T07:31:30.407 DEBUG:teuthology.orchestra.run.smithi064:> filename = "{0}_{1}.bin".format(i, size // size_unit) 2024-03-04T07:31:30.407 DEBUG:teuthology.orchestra.run.smithi064:> with open(os.path.join(mountpoint, subdir, filename), '"'"'w'"'"') as f: 2024-03-04T07:31:30.407 DEBUG:teuthology.orchestra.run.smithi064:> f.write(size * '"'"'x'"'"') 2024-03-04T07:31:30.407 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:31:37.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:31:37.244 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json config get mds_max_purge_ops 2024-03-04T07:31:37.244 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:37.244 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -rf delete_me) 2024-03-04T07:31:37.366 INFO:teuthology.orchestra.run.smithi066.stdout:{"mds_max_purge_ops":"100000000"} 2024-03-04T07:31:37.367 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_max_purge_ops": "100000000" } 2024-03-04T07:31:37.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json config get mds_max_purge_files 2024-03-04T07:31:37.501 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_max_purge_files":"6"} 2024-03-04T07:31:37.501 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_max_purge_files": "6" } 2024-03-04T07:31:37.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:37.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.607+0000 7f6226a84640 1 Processor -- start 2024-03-04T07:31:37.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.607+0000 7f6226a84640 1 -- start start 2024-03-04T07:31:37.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6226a84640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f622015ad70 0x7f62200b2aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:37.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6226a84640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f622015f2d0 0x7f62200cd070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:37.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6226a84640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62200b4120 0x7f62200b45b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:37.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6226a84640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f622015b150 con 0x7f62200b4120 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6226a84640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f62200cd5b0 con 0x7f622015f2d0 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6226a84640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62200b3080 con 0x7f622015ad70 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62200b4120 0x7f62200b45b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f62200b4120 0x7f62200b45b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49620/0 (socket says 172.21.15.64:49620) 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 -- 172.21.15.64:0/2086570129 learned_addr learned my addr 172.21.15.64:0/2086570129 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6224820640 1 --2- 172.21.15.64:0/2086570129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f622015ad70 0x7f62200b2aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f621ffff640 1 --2- 172.21.15.64:0/2086570129 >> v2:172.21.15.66:3300/0 conn(0x7f622015f2d0 0x7f62200cd070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 -- 172.21.15.64:0/2086570129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f622015ad70 msgr2=0x7f62200b2aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 --2- 172.21.15.64:0/2086570129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f622015ad70 0x7f62200b2aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 -- 172.21.15.64:0/2086570129 >> v2:172.21.15.66:3300/0 conn(0x7f622015f2d0 msgr2=0x7f62200cd070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:37.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 --2- 172.21.15.64:0/2086570129 >> v2:172.21.15.66:3300/0 conn(0x7f622015f2d0 0x7f62200cd070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.608+0000 7f6225021640 1 -- 172.21.15.64:0/2086570129 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62200b3360 con 0x7f62200b4120 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.609+0000 7f6225021640 1 --2- 172.21.15.64:0/2086570129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62200b4120 0x7f62200b45b0 secure :-1 s=READY pgs=1179 cs=0 l=1 rev1=1 crypto rx=0x7f62200706d0 tx=0x7f621802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e89f9bb5ead4e55c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.609+0000 7f621f7fe640 1 -- 172.21.15.64:0/2086570129 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f621802ca70 con 0x7f62200b4120 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.609+0000 7f621f7fe640 1 -- 172.21.15.64:0/2086570129 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f621802cc10 con 0x7f62200b4120 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.609+0000 7f621f7fe640 1 -- 172.21.15.64:0/2086570129 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62180343f0 con 0x7f62200b4120 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 -- 172.21.15.64:0/2086570129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62200b4120 msgr2=0x7f62200b45b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 --2- 172.21.15.64:0/2086570129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62200b4120 0x7f62200b45b0 secure :-1 s=READY pgs=1179 cs=0 l=1 rev1=1 crypto rx=0x7f62200706d0 tx=0x7f621802bed0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 -- 172.21.15.64:0/2086570129 shutdown_connections 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 --2- 172.21.15.64:0/2086570129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f62200b4120 0x7f62200b45b0 unknown :-1 s=CLOSED pgs=1179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 --2- 172.21.15.64:0/2086570129 >> v2:172.21.15.66:3300/0 conn(0x7f622015f2d0 0x7f62200cd070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 --2- 172.21.15.64:0/2086570129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f622015ad70 0x7f62200b2aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 -- 172.21.15.64:0/2086570129 >> 172.21.15.64:0/2086570129 conn(0x7f62200b17a0 msgr2=0x7f62200b1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 -- 172.21.15.64:0/2086570129 shutdown_connections 2024-03-04T07:31:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.610+0000 7f6226a84640 1 -- 172.21.15.64:0/2086570129 wait complete. 2024-03-04T07:31:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.611+0000 7f6226a84640 1 Processor -- start 2024-03-04T07:31:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.611+0000 7f6226a84640 1 -- start start 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6226a84640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62200b4120 0x7f62201394a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6226a84640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f622015ad70 0x7f622015d130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6226a84640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f622015f2d0 0x7f622006d090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6226a84640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62200b3c00 con 0x7f622015ad70 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6226a84640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62200b3920 con 0x7f622015f2d0 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6226a84640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f622015f730 con 0x7f62200b4120 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f622015ad70 0x7f622015d130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f622015ad70 0x7f622015d130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49626/0 (socket says 172.21.15.64:49626) 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 -- 172.21.15.64:0/202650302 learned_addr learned my addr 172.21.15.64:0/202650302 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6224820640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62200b4120 0x7f62201394a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f6225021640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f622015f2d0 0x7f622006d090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 -- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62200b4120 msgr2=0x7f62201394a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62200b4120 0x7f62201394a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 -- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f622015f2d0 msgr2=0x7f622006d090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f622015f2d0 0x7f622006d090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 -- 172.21.15.64:0/202650302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f622015da30 con 0x7f622015ad70 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621ffff640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f622015ad70 0x7f622015d130 secure :-1 s=READY pgs=1180 cs=0 l=1 rev1=1 crypto rx=0x7f620c00d5f0 tx=0x7f620c00dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.613+0000 7f621d7fa640 1 -- 172.21.15.64:0/202650302 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f620c0125f0 con 0x7f622015ad70 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.614+0000 7f621d7fa640 1 -- 172.21.15.64:0/202650302 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f620c019070 con 0x7f622015ad70 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.614+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f622015dd10 con 0x7f622015ad70 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.614+0000 7f621d7fa640 1 -- 172.21.15.64:0/202650302 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f620c0129d0 con 0x7f622015ad70 2024-03-04T07:31:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.614+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f622015e2d0 con 0x7f622015ad70 2024-03-04T07:31:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.615+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f62200752d0 con 0x7f622015ad70 2024-03-04T07:31:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.616+0000 7f621d7fa640 1 -- 172.21.15.64:0/202650302 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f620c012b70 con 0x7f622015ad70 2024-03-04T07:31:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.616+0000 7f621d7fa640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f620406b4d0 0x7f620406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.616+0000 7f621d7fa640 1 -- 172.21.15.64:0/202650302 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f620c08a570 con 0x7f622015ad70 2024-03-04T07:31:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.616+0000 7f6224820640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f620406b4d0 0x7f620406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:37.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.617+0000 7f6224820640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f620406b4d0 0x7f620406d990 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f6214006fd0 tx=0x7f6214008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:37.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.624+0000 7f621d7fa640 1 -- 172.21.15.64:0/202650302 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f620c058cf0 con 0x7f622015ad70 2024-03-04T07:31:37.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.958+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f622006c9d0 con 0x7f622015ad70 2024-03-04T07:31:37.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.959+0000 7f621d7fa640 1 -- 172.21.15.64:0/202650302 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f620c05cd90 con 0x7f622015ad70 2024-03-04T07:31:37.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:37.959 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:37.959 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f620406b4d0 msgr2=0x7f620406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f620406b4d0 0x7f620406d990 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f6214006fd0 tx=0x7f6214008040 comp rx=0 tx=0).stop 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f622015ad70 msgr2=0x7f622015d130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f622015ad70 0x7f622015d130 secure :-1 s=READY pgs=1180 cs=0 l=1 rev1=1 crypto rx=0x7f620c00d5f0 tx=0x7f620c00dab0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 shutdown_connections 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f620406b4d0 0x7f620406d990 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f622015f2d0 0x7f622006d090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f622015ad70 0x7f622015d130 unknown :-1 s=CLOSED pgs=1180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 --2- 172.21.15.64:0/202650302 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f62200b4120 0x7f62201394a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:37.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 >> 172.21.15.64:0/202650302 conn(0x7f62200b17a0 msgr2=0x7f6220155f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:37.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 shutdown_connections 2024-03-04T07:31:37.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:37.961+0000 7f6226a84640 1 -- 172.21.15.64:0/202650302 wait complete. 2024-03-04T07:31:37.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:38.133 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":37,"msgr_send_messages":32,"msgr_recv_bytes":114535,"msgr_send_bytes":4228165,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.013713374,"msgr_running_send_time":0.007057857,"msgr_running_recv_time":0.004582684,"msgr_running_fast_dispatch_time":0.001896378,"msgr_send_messages_queue_lat":{"avgcount":32,"sum":0.004205150,"avgtime":0.000131410},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":107584,"msgr_send_encrypted_bytes":6672},"AsyncMessenger::Worker-1":{"msgr_recv_messages":4885,"msgr_send_messages":11095,"msgr_recv_bytes":1695753,"msgr_send_bytes":10703040,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.525355478,"msgr_running_send_time":0.289799667,"msgr_running_recv_time":0.275942487,"msgr_running_fast_dispatch_time":0.004079559,"msgr_send_messages_queue_lat":{"avgcount":11094,"sum":0.124864323,"avgtime":0.000011255},"msgr_handle_ack_lat":{"avgcount":15749,"sum":0.002760647,"avgtime":0.000000175},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":92672},"AsyncMessenger::Worker-2":{"msgr_recv_messages":11,"msgr_send_messages":13,"msgr_recv_bytes":10104,"msgr_send_bytes":3970,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.002612604,"msgr_running_send_time":0.000228277,"msgr_running_recv_time":0.001958449,"msgr_running_fast_dispatch_time":0.000269694,"msgr_send_messages_queue_lat":{"avgcount":11,"sum":0.004100936,"avgtime":0.000372812},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":75,"sum":1.134300432,"avgtime":0.015124005}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":4,"sum":0.000280526,"avgtime":0.000070131}},"mds":{"request":3381,"reply":3379,"reply_latency":{"avgcount":3379,"sum":3.797972611,"avgtime":0.001123993},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":3380,"inodes_top":0,"inodes_bottom":3380,"inodes_pin_tail":0,"inodes_pinned":3380,"inodes_expired":0,"inodes_with_caps":3202,"caps":3202,"subtrees":2,"traverse":3387,"traverse_hit":3386,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":3199,"root_rbytes":6251520,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":132,"ceph_cap_op_grant":3390,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":1107,"handle_client_caps_dirty":1029,"handle_client_cap_release":0,"process_request_cap_release":3539},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":169,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":169,"strays_enqueued":0,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":4480,"evex":0,"evtrm":0,"ev":4483,"evexg":0,"evexd":0,"segadd":6,"segex":0,"segtrm":0,"seg":8,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":13166977,"rdpos":4195244,"jlat":{"avgcount":11,"sum":0.051728722,"avgtime":0.004702611},"replayed":3},"mds_mem":{"ino":3214,"ino+":3214,"ino-":0,"dir":13,"dir+":13,"dir-":0,"dn":3380,"dn+":3380,"dn-":0,"cap":3202,"cap+":3202,"cap-":0,"rss":83760,"heap":248164},"mds_server":{"dispatch_client_request":3384,"dispatch_server_request":0,"handle_client_request":3381,"handle_client_session":0,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":169,"sum":0.240377700,"avgtime":0.001422353},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":3219,"average_load":3219,"avg_session_uptime":802,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":374909,"buffer_anon_items":595,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":5313688,"mds_co_items":25350,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":64,"op_send_bytes":8733931,"op_resend":0,"op_reply":64,"op_latency":{"avgcount":64,"sum":0.139997326,"avgtime":0.002187458},"op_inflight":0,"oplen_avg":{"avgcount":64,"sum":119},"op":64,"op_r":36,"op_w":28,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":8,"osdop_write":11,"osdop_writefull":1,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":7,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":15,"omap_rd":48,"omap_del":2},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":0,"pq_executing":0,"pq_executing_high_water":0,"pq_executed_ops":0,"pq_executed":0,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":89218,"max":104857600,"get_started":0,"get":4929,"get_sum":1374111,"get_or_fail_fail":0,"get_or_fail_success":4929,"take":0,"take_sum":0,"put":4586,"put_sum":1284893,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":64,"take_sum":8735048,"put":64,"put_sum":8735048,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":64,"take_sum":64,"put":64,"put_sum":64,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":0,"put_sum":0,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":584065,"max":3758096384,"get_started":0,"get":4480,"get_sum":8971292,"get_or_fail_fail":0,"get_or_fail_success":4480,"take":0,"take_sum":0,"put":11,"put_sum":8387227,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:38.134 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 114535, "msgr_recv_encrypted_bytes": 107584, "msgr_recv_messages": 37, "msgr_running_fast_dispatch_time": 0.001896378, "msgr_running_recv_time": 0.004582684, "msgr_running_send_time": 0.007057857, "msgr_running_total_time": 0.013713374, "msgr_send_bytes": 4228165, "msgr_send_encrypted_bytes": 6672, "msgr_send_messages": 32, "msgr_send_messages_queue_lat": { "avgcount": 32, "avgtime": 0.00013141, "sum": 0.00420515 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 15749, "avgtime": 1.75e-07, "sum": 0.002760647 }, "msgr_recv_bytes": 1695753, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 4885, "msgr_running_fast_dispatch_time": 0.004079559, "msgr_running_recv_time": 0.275942487, "msgr_running_send_time": 0.289799667, "msgr_running_total_time": 0.525355478, "msgr_send_bytes": 10703040, "msgr_send_encrypted_bytes": 92672, "msgr_send_messages": 11095, "msgr_send_messages_queue_lat": { "avgcount": 11094, "avgtime": 1.1255e-05, "sum": 0.124864323 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 10104, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 11, "msgr_running_fast_dispatch_time": 0.000269694, "msgr_running_recv_time": 0.001958449, "msgr_running_send_time": 0.000228277, "msgr_running_total_time": 0.002612604, "msgr_send_bytes": 3970, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 13, "msgr_send_messages_queue_lat": { "avgcount": 11, "avgtime": 0.000372812, "sum": 0.004100936 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 75, "avgtime": 0.015124005, "sum": 1.134300432 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 4, "avgtime": 7.0131e-05, "sum": 0.000280526 }, "queue_len": 0 }, "mds": { "caps": 3202, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 3390, "ceph_cap_op_revoke": 132, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 0, "handle_client_caps": 1107, "handle_client_caps_dirty": 1029, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 3380, "inodes_bottom": 3380, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 3380, "inodes_top": 0, "inodes_with_caps": 3202, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 3539, "q": 0, "reply": 3379, "reply_latency": { "avgcount": 3379, "avgtime": 0.001123993, "sum": 3.797972611 }, "request": 3381, "root_rbytes": 6251520, "root_rfiles": 3199, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 3387, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 3386, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 169, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 169, "strays_enqueued": 0, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 4483, "evadd": 4480, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 11, "avgtime": 0.004702611, "sum": 0.051728722 }, "rdpos": 4195244, "replayed": 3, "seg": 8, "segadd": 6, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 13166977 }, "mds_mem": { "cap": 3202, "cap+": 3202, "cap-": 0, "dir": 13, "dir+": 13, "dir-": 0, "dn": 3380, "dn+": 3380, "dn-": 0, "heap": 248164, "ino": 3214, "ino+": 3214, "ino-": 0, "rss": 83760 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 3384, "dispatch_server_request": 0, "handle_client_request": 3381, "handle_client_session": 0, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 169, "avgtime": 0.001422353, "sum": 0.2403777 } }, "mds_sessions": { "average_load": 3219, "avg_session_uptime": 802, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 3219 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 374909, "buffer_anon_items": 595, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 5313688, "mds_co_items": 25350, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 2, "omap_rd": 48, "omap_wr": 15, "op": 64, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 64, "avgtime": 0.002187458, "sum": 0.139997326 }, "op_pg": 0, "op_r": 36, "op_reply": 64, "op_resend": 0, "op_rmw": 0, "op_send": 64, "op_send_bytes": 8733931, "op_w": 28, "oplen_avg": { "avgcount": 64, "sum": 119 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 7, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 8, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 11, "osdop_writefull": 1, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 0, "pq_executed_ops": 0, "pq_executing": 0, "pq_executing_high_water": 0, "pq_executing_ops": 0, "pq_executing_ops_high_water": 0, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 4929, "get_or_fail_fail": 0, "get_or_fail_success": 4929, "get_started": 0, "get_sum": 1374111, "max": 104857600, "put": 4586, "put_sum": 1284893, "take": 0, "take_sum": 0, "val": 89218, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 64, "put_sum": 8735048, "take": 64, "take_sum": 8735048, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 64, "put_sum": 64, "take": 64, "take_sum": 64, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 3758096384, "put": 0, "put_sum": 0, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 4480, "get_or_fail_fail": 0, "get_or_fail_success": 4480, "get_started": 0, "get_sum": 8971292, "max": 3758096384, "put": 11, "put_sum": 8387227, "take": 0, "take_sum": 0, "val": 584065, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:38.134 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/169, 0/169 2024-03-04T07:31:39.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.267+0000 7f541a55e640 1 Processor -- start 2024-03-04T07:31:39.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.267+0000 7f541a55e640 1 -- start start 2024-03-04T07:31:39.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f541a55e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140d1710 0x7f541406aec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:39.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f541a55e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f54140cd0f0 0x7f541406b400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f541a55e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f54140d0990 con 0x7f5414162470 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f541a55e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f54140cfe60 con 0x7f54140cd0f0 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f541a55e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f54140d0410 con 0x7f54140d1710 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f5418afb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f54140cd0f0 0x7f541406b400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f5418afb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f54140cd0f0 0x7f541406b400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:48298/0 (socket says 172.21.15.64:48298) 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f5418afb640 1 -- 172.21.15.64:0/1634885308 learned_addr learned my addr 172.21.15.64:0/1634885308 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f54137fe640 1 --2- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140d1710 0x7f541406aec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f5418afb640 1 -- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140d1710 msgr2=0x7f541406aec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f5418afb640 1 --2- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140d1710 0x7f541406aec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.269+0000 7f5418afb640 1 -- 172.21.15.64:0/1634885308 >> v1:172.21.15.66:6789/0 conn(0x7f5414162470 legacy=0x7f54140cc9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:39.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.270+0000 7f5418afb640 1 -- 172.21.15.64:0/1634885308 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54140d06b0 con 0x7f54140cd0f0 2024-03-04T07:31:39.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.271+0000 7f5418afb640 1 --2- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140cd0f0 0x7f541406b400 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f53fc009870 tx=0x7f53fc02ba20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=477c4f674691f8b2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:39.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.271+0000 7f5412ffd640 1 -- 172.21.15.64:0/1634885308 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53fc02f840 con 0x7f54140cd0f0 2024-03-04T07:31:39.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.271+0000 7f5412ffd640 1 -- 172.21.15.64:0/1634885308 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f53fc02f9e0 con 0x7f54140cd0f0 2024-03-04T07:31:39.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.271+0000 7f5412ffd640 1 -- 172.21.15.64:0/1634885308 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53fc02fdb0 con 0x7f54140cd0f0 2024-03-04T07:31:39.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.272+0000 7f541a55e640 1 -- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140cd0f0 msgr2=0x7f541406b400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:39.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.272+0000 7f541a55e640 1 --2- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140cd0f0 0x7f541406b400 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f53fc009870 tx=0x7f53fc02ba20 comp rx=0 tx=0).stop 2024-03-04T07:31:39.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.272+0000 7f541a55e640 1 -- 172.21.15.64:0/1634885308 shutdown_connections 2024-03-04T07:31:39.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.272+0000 7f541a55e640 1 --2- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140cd0f0 0x7f541406b400 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.272+0000 7f541a55e640 1 --2- 172.21.15.64:0/1634885308 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140d1710 0x7f541406aec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.272+0000 7f541a55e640 1 -- 172.21.15.64:0/1634885308 >> 172.21.15.64:0/1634885308 conn(0x7f54140d0bf0 msgr2=0x7f54140d1000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:39.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.272+0000 7f541a55e640 1 -- 172.21.15.64:0/1634885308 shutdown_connections 2024-03-04T07:31:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.273+0000 7f541a55e640 1 -- 172.21.15.64:0/1634885308 wait complete. 2024-03-04T07:31:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.274+0000 7f541a55e640 1 Processor -- start 2024-03-04T07:31:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.274+0000 7f541a55e640 1 -- start start 2024-03-04T07:31:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f541a55e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140cd0f0 0x7f54141eca40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f541a55e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140d1710 0x7f54141fbb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f541a55e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5414162470 0x7f54141fc0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:39.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f541a55e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54140a1e70 con 0x7f5414162470 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f541a55e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f541406b940 con 0x7f54140d1710 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f541a55e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f54140a1ff0 con 0x7f54140cd0f0 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f5418afb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5414162470 0x7f54141fc0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f54137fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140d1710 0x7f54141fbb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f54137fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140d1710 0x7f54141fbb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:48310/0 (socket says 172.21.15.64:48310) 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f5413fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140cd0f0 0x7f54141eca40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f5418afb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5414162470 0x7f54141fc0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49640/0 (socket says 172.21.15.64:49640) 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.275+0000 7f54137fe640 1 -- 172.21.15.64:0/793783403 learned_addr learned my addr 172.21.15.64:0/793783403 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5413fff640 1 -- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140d1710 msgr2=0x7f54141fbb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5413fff640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140d1710 0x7f54141fbb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5413fff640 1 -- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5414162470 msgr2=0x7f54141fc0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5413fff640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5414162470 0x7f54141fc0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5413fff640 1 -- 172.21.15.64:0/793783403 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54140c4c00 con 0x7f54140cd0f0 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f54137fe640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140d1710 0x7f54141fbb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:39.277 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5413fff640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140cd0f0 0x7f54141eca40 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f540c00c770 tx=0x7f540c00cc30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:39.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5410ff9640 1 -- 172.21.15.64:0/793783403 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f540c003b70 con 0x7f54140cd0f0 2024-03-04T07:31:39.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f54141ed3c0 con 0x7f54140cd0f0 2024-03-04T07:31:39.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54140c4ea0 con 0x7f54140cd0f0 2024-03-04T07:31:39.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.276+0000 7f5410ff9640 1 -- 172.21.15.64:0/793783403 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f540c003d10 con 0x7f54140cd0f0 2024-03-04T07:31:39.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.277+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f53f8005290 con 0x7f54140cd0f0 2024-03-04T07:31:39.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.277+0000 7f5410ff9640 1 -- 172.21.15.64:0/793783403 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f540c00aa30 con 0x7f54140cd0f0 2024-03-04T07:31:39.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.278+0000 7f5410ff9640 1 -- 172.21.15.64:0/793783403 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f540c00ac90 con 0x7f54140cd0f0 2024-03-04T07:31:39.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.278+0000 7f5410ff9640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f53f406b4d0 0x7f53f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:39.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.279+0000 7f54137fe640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f53f406b4d0 0x7f53f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:39.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.279+0000 7f5410ff9640 1 -- 172.21.15.64:0/793783403 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f540c089fb0 con 0x7f54140cd0f0 2024-03-04T07:31:39.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.279+0000 7f54137fe640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f53f406b4d0 0x7f53f406d990 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f54140c4820 tx=0x7f5404009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:39.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.283+0000 7f5410ff9640 1 -- 172.21.15.64:0/793783403 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f540c0587c0 con 0x7f54140cd0f0 2024-03-04T07:31:39.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.567+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f53f8005850 con 0x7f54140cd0f0 2024-03-04T07:31:39.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.568+0000 7f5410ff9640 1 -- 172.21.15.64:0/793783403 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f540c05c860 con 0x7f54140cd0f0 2024-03-04T07:31:39.568 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:39.568 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:39.568 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.570+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f53f406b4d0 msgr2=0x7f53f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.570+0000 7f541a55e640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f53f406b4d0 0x7f53f406d990 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f54140c4820 tx=0x7f5404009190 comp rx=0 tx=0).stop 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140cd0f0 msgr2=0x7f54141eca40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140cd0f0 0x7f54141eca40 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f540c00c770 tx=0x7f540c00cc30 comp rx=0 tx=0).stop 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 shutdown_connections 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f53f406b4d0 0x7f53f406d990 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5414162470 0x7f54141fc0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54140d1710 0x7f54141fbb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 --2- 172.21.15.64:0/793783403 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54140cd0f0 0x7f54141eca40 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 >> 172.21.15.64:0/793783403 conn(0x7f54140d0bf0 msgr2=0x7f54140ce750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 shutdown_connections 2024-03-04T07:31:39.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:39.571+0000 7f541a55e640 1 -- 172.21.15.64:0/793783403 wait complete. 2024-03-04T07:31:39.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:39.733 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":58,"msgr_send_messages":53,"msgr_recv_bytes":119879,"msgr_send_bytes":4235301,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.016615198,"msgr_running_send_time":0.008753677,"msgr_running_recv_time":0.005274726,"msgr_running_fast_dispatch_time":0.002291693,"msgr_send_messages_queue_lat":{"avgcount":53,"sum":0.005616304,"avgtime":0.000105968},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":107856,"msgr_send_encrypted_bytes":7296},"AsyncMessenger::Worker-1":{"msgr_recv_messages":6043,"msgr_send_messages":13805,"msgr_recv_bytes":2116721,"msgr_send_bytes":14985269,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.645586530,"msgr_running_send_time":0.359331054,"msgr_running_recv_time":0.329626685,"msgr_running_fast_dispatch_time":0.006674847,"msgr_send_messages_queue_lat":{"avgcount":13804,"sum":0.166398425,"avgtime":0.000012054},"msgr_handle_ack_lat":{"avgcount":19133,"sum":0.003453565,"avgtime":0.000000180},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":94464},"AsyncMessenger::Worker-2":{"msgr_recv_messages":46,"msgr_send_messages":48,"msgr_recv_bytes":25099,"msgr_send_bytes":21665,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.006236111,"msgr_running_send_time":0.002236612,"msgr_running_recv_time":0.002680386,"msgr_running_fast_dispatch_time":0.001016632,"msgr_send_messages_queue_lat":{"avgcount":46,"sum":0.005157982,"avgtime":0.000112130},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":83,"sum":2.513781975,"avgtime":0.030286529}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":113,"sum":0.072999001,"avgtime":0.000646008}},"mds":{"request":3830,"reply":3827,"reply_latency":{"avgcount":3827,"sum":5.211895307,"avgtime":0.001361874},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":3717,"inodes_top":0,"inodes_bottom":3717,"inodes_pin_tail":0,"inodes_pinned":3717,"inodes_expired":0,"inodes_with_caps":3141,"caps":3141,"subtrees":2,"traverse":3836,"traverse_hit":3835,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":2769,"root_rbytes":9039872,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":367,"ceph_cap_op_grant":4028,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":2133,"handle_client_caps_dirty":1997,"handle_client_cap_release":1,"process_request_cap_release":4437},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":557,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":618,"strays_enqueued":61,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":6019,"evex":0,"evtrm":0,"ev":6022,"evexg":0,"evexd":0,"segadd":7,"segex":0,"segtrm":0,"seg":9,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":16631601,"rdpos":4195244,"jlat":{"avgcount":14,"sum":0.069774886,"avgtime":0.004983920},"replayed":3},"mds_mem":{"ino":3153,"ino+":3214,"ino-":61,"dir":13,"dir+":13,"dir-":0,"dn":3767,"dn+":3828,"dn-":61,"cap":3141,"cap+":3202,"cap-":61,"rss":84144,"heap":248164},"mds_server":{"dispatch_client_request":3833,"dispatch_server_request":0,"handle_client_request":3830,"handle_client_session":0,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":617,"sum":1.654300396,"avgtime":0.002681199},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":3598,"average_load":3598,"avg_session_uptime":807,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":7963311,"buffer_anon_items":2079,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":11679473,"mds_co_items":69670,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":142,"op_send_bytes":12067709,"op_resend":0,"op_reply":142,"op_latency":{"avgcount":142,"sum":0.323993819,"avgtime":0.002281646},"op_inflight":0,"oplen_avg":{"avgcount":142,"sum":201},"op":142,"op_r":38,"op_w":104,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":10,"osdop_write":16,"osdop_writefull":3,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":74,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":19,"omap_rd":48,"omap_del":4},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":61,"pq_executed":61,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":296,"max":104857600,"get_started":0,"get":6143,"get_sum":1706106,"get_or_fail_fail":0,"get_or_fail_success":6143,"take":0,"take_sum":0,"put":6142,"put_sum":1705810,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":142,"take_sum":12074987,"put":142,"put_sum":12074987,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":142,"take_sum":142,"put":142,"put_sum":142,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":61,"get_sum":6161,"get_or_fail_fail":0,"get_or_fail_success":61,"take":0,"take_sum":0,"put":2,"put_sum":6161,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":748539,"max":3758096384,"get_started":0,"get":6019,"get_sum":12435916,"get_or_fail_fail":0,"get_or_fail_success":6019,"take":0,"take_sum":0,"put":14,"put_sum":11687377,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:39.734 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 119879, "msgr_recv_encrypted_bytes": 107856, "msgr_recv_messages": 58, "msgr_running_fast_dispatch_time": 0.002291693, "msgr_running_recv_time": 0.005274726, "msgr_running_send_time": 0.008753677, "msgr_running_total_time": 0.016615198, "msgr_send_bytes": 4235301, "msgr_send_encrypted_bytes": 7296, "msgr_send_messages": 53, "msgr_send_messages_queue_lat": { "avgcount": 53, "avgtime": 0.000105968, "sum": 0.005616304 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 19133, "avgtime": 1.8e-07, "sum": 0.003453565 }, "msgr_recv_bytes": 2116721, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 6043, "msgr_running_fast_dispatch_time": 0.006674847, "msgr_running_recv_time": 0.329626685, "msgr_running_send_time": 0.359331054, "msgr_running_total_time": 0.64558653, "msgr_send_bytes": 14985269, "msgr_send_encrypted_bytes": 94464, "msgr_send_messages": 13805, "msgr_send_messages_queue_lat": { "avgcount": 13804, "avgtime": 1.2054e-05, "sum": 0.166398425 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 25099, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 46, "msgr_running_fast_dispatch_time": 0.001016632, "msgr_running_recv_time": 0.002680386, "msgr_running_send_time": 0.002236612, "msgr_running_total_time": 0.006236111, "msgr_send_bytes": 21665, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 48, "msgr_send_messages_queue_lat": { "avgcount": 46, "avgtime": 0.00011213, "sum": 0.005157982 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 83, "avgtime": 0.030286529, "sum": 2.513781975 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 113, "avgtime": 0.000646008, "sum": 0.072999001 }, "queue_len": 0 }, "mds": { "caps": 3141, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 4028, "ceph_cap_op_revoke": 367, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 1, "handle_client_caps": 2133, "handle_client_caps_dirty": 1997, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 3717, "inodes_bottom": 3717, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 3717, "inodes_top": 0, "inodes_with_caps": 3141, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 4437, "q": 0, "reply": 3827, "reply_latency": { "avgcount": 3827, "avgtime": 0.001361874, "sum": 5.211895307 }, "request": 3830, "root_rbytes": 9039872, "root_rfiles": 2769, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 3836, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 3835, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 557, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 618, "strays_enqueued": 61, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 6022, "evadd": 6019, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 14, "avgtime": 0.00498392, "sum": 0.069774886 }, "rdpos": 4195244, "replayed": 3, "seg": 9, "segadd": 7, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 16631601 }, "mds_mem": { "cap": 3141, "cap+": 3202, "cap-": 61, "dir": 13, "dir+": 13, "dir-": 0, "dn": 3767, "dn+": 3828, "dn-": 61, "heap": 248164, "ino": 3153, "ino+": 3214, "ino-": 61, "rss": 84144 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 3833, "dispatch_server_request": 0, "handle_client_request": 3830, "handle_client_session": 0, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 617, "avgtime": 0.002681199, "sum": 1.654300396 } }, "mds_sessions": { "average_load": 3598, "avg_session_uptime": 807, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 3598 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 7963311, "buffer_anon_items": 2079, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 11679473, "mds_co_items": 69670, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 4, "omap_rd": 48, "omap_wr": 19, "op": 142, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 142, "avgtime": 0.002281646, "sum": 0.323993819 }, "op_pg": 0, "op_r": 38, "op_reply": 142, "op_resend": 0, "op_rmw": 0, "op_send": 142, "op_send_bytes": 12067709, "op_w": 104, "oplen_avg": { "avgcount": 142, "sum": 201 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 74, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 10, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 16, "osdop_writefull": 3, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 61, "pq_executed_ops": 61, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 6143, "get_or_fail_fail": 0, "get_or_fail_success": 6143, "get_started": 0, "get_sum": 1706106, "max": 104857600, "put": 6142, "put_sum": 1705810, "take": 0, "take_sum": 0, "val": 296, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 142, "put_sum": 12074987, "take": 142, "take_sum": 12074987, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 142, "put_sum": 142, "take": 142, "take_sum": 142, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 61, "get_or_fail_fail": 0, "get_or_fail_success": 61, "get_started": 0, "get_sum": 6161, "max": 3758096384, "put": 2, "put_sum": 6161, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 6019, "get_or_fail_fail": 0, "get_or_fail_success": 6019, "get_started": 0, "get_sum": 12435916, "max": 3758096384, "put": 14, "put_sum": 11687377, "take": 0, "take_sum": 0, "val": 748539, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:39.735 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/557, 61/618 2024-03-04T07:31:40.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:40.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.854+0000 7f3f5742a640 1 Processor -- start 2024-03-04T07:31:40.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.854+0000 7f3f5742a640 1 -- start start 2024-03-04T07:31:40.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.855+0000 7f3f5742a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3f5006b5d0 0x7f3f500726f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.855+0000 7f3f5742a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 0x7f3f5015eaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.855+0000 7f3f5742a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3f5015f5e0 con 0x7f3f5015bd90 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.855+0000 7f3f5742a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3f5015f300 con 0x7f3f5006b5d0 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.855+0000 7f3f5742a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f50071c10 con 0x7f3f5015e610 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.855+0000 7f3f551c6640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3f5015bd90 0x7f3f5006aec0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:46806/0 (socket says 172.21.15.64:46806) 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.855+0000 7f3f551c6640 1 -- 172.21.15.64:0/2854619485 learned_addr learned my addr 172.21.15.64:0/2854619485 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f559c7640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 0x7f3f5015eaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f549c5640 1 --2- 172.21.15.64:0/2854619485 >> v2:172.21.15.66:3300/0 conn(0x7f3f5006b5d0 0x7f3f500726f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 543959155 0 0) 0x7f3f5015f5e0 con 0x7f3f5015bd90 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3f3c003520 con 0x7f3f5015bd90 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2765267106 0 0) 0x7f3f3c003520 con 0x7f3f5015bd90 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 msgr2=0x7f3f5015eaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 0x7f3f5015eaa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:40.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 >> v2:172.21.15.66:3300/0 conn(0x7f3f5006b5d0 msgr2=0x7f3f500726f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 --2- 172.21.15.64:0/2854619485 >> v2:172.21.15.66:3300/0 conn(0x7f3f5006b5d0 0x7f3f500726f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f50071ef0 con 0x7f3f5015bd90 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f3f4c001650 con 0x7f3f5015bd90 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f559c7640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 0x7f3f5015eaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f549c5640 1 --2- 172.21.15.64:0/2854619485 >> v2:172.21.15.66:3300/0 conn(0x7f3f5006b5d0 0x7f3f500726f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 >> v1:172.21.15.64:6789/0 conn(0x7f3f5015bd90 legacy=0x7f3f5006aec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 0x7f3f3c0039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f3c003ef0 0x7f3f3c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f3c004d40 0x7f3f3c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f5015f300 con 0x7f3f3c003ef0 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f50071c10 con 0x7f3f3c004d40 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f3c00b9c0 con 0x7f3f5015e610 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f549c5640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f3c003ef0 0x7f3f3c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f551c6640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 0x7f3f3c0039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.856+0000 7f3f559c7640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f3c004d40 0x7f3f3c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.857+0000 7f3f549c5640 1 -- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 msgr2=0x7f3f3c0039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.857+0000 7f3f549c5640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f5015e610 0x7f3f3c0039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.857+0000 7f3f549c5640 1 -- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f3c004d40 msgr2=0x7f3f3c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.857+0000 7f3f549c5640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f3c004d40 0x7f3f3c0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.857+0000 7f3f549c5640 1 -- 172.21.15.64:0/2854619485 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f3c003520 con 0x7f3f3c003ef0 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.858+0000 7f3f549c5640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f3c003ef0 0x7f3f3c004800 secure :-1 s=READY pgs=1182 cs=0 l=1 rev1=1 crypto rx=0x7f3f40009cc0 tx=0x7f3f3c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b1165b11a1130a65 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.858+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f40039070 con 0x7f3f3c003ef0 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.858+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f4002c960 con 0x7f3f3c003ef0 2024-03-04T07:31:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.858+0000 7f3f47fff640 1 -- 172.21.15.64:0/2854619485 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f40009860 con 0x7f3f3c003ef0 2024-03-04T07:31:40.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.859+0000 7f3f5742a640 1 -- 172.21.15.64:0/2854619485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f3c003ef0 msgr2=0x7f3f3c004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:40.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.859+0000 7f3f5742a640 1 --2- 172.21.15.64:0/2854619485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f3c003ef0 0x7f3f3c004800 secure :-1 s=READY pgs=1182 cs=0 l=1 rev1=1 crypto rx=0x7f3f40009cc0 tx=0x7f3f3c005830 comp rx=0 tx=0).stop 2024-03-04T07:31:40.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.859+0000 7f3f559c7640 1 -- 172.21.15.64:0/2854619485 reap_dead start 2024-03-04T07:31:40.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.859+0000 7f3f5742a640 1 -- 172.21.15.64:0/2854619485 shutdown_connections 2024-03-04T07:31:40.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.859+0000 7f3f5742a640 1 -- 172.21.15.64:0/2854619485 >> 172.21.15.64:0/2854619485 conn(0x7f3f500d0c90 msgr2=0x7f3f500d1070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:40.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.859+0000 7f3f5742a640 1 -- 172.21.15.64:0/2854619485 shutdown_connections 2024-03-04T07:31:40.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.859+0000 7f3f5742a640 1 -- 172.21.15.64:0/2854619485 wait complete. 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.860+0000 7f3f5742a640 1 Processor -- start 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.860+0000 7f3f5742a640 1 -- start start 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f5742a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f50151a80 0x7f3f5015b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f5742a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f500c8ee0 0x7f3f500b0d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f5742a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f50151e60 0x7f3f500ae260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f5742a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f5015ec20 con 0x7f3f50151a80 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f5742a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f5015eaa0 con 0x7f3f50151e60 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f5742a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f5015eda0 con 0x7f3f500c8ee0 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f50151a80 0x7f3f5015b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f50151a80 0x7f3f5015b390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49664/0 (socket says 172.21.15.64:49664) 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 -- 172.21.15.64:0/3570840635 learned_addr learned my addr 172.21.15.64:0/3570840635 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f549c5640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f500c8ee0 0x7f3f500b0d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f559c7640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f50151e60 0x7f3f500ae260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 -- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f500c8ee0 msgr2=0x7f3f500b0d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f500c8ee0 0x7f3f500b0d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 -- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f50151e60 msgr2=0x7f3f500ae260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f50151e60 0x7f3f500ae260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.861+0000 7f3f551c6640 1 -- 172.21.15.64:0/3570840635 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f500b1500 con 0x7f3f50151a80 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.862+0000 7f3f549c5640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f500c8ee0 0x7f3f500b0d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.862+0000 7f3f551c6640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f50151a80 0x7f3f5015b390 secure :-1 s=READY pgs=1183 cs=0 l=1 rev1=1 crypto rx=0x7f3f4c007fd0 tx=0x7f3f4c007720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.862+0000 7f3f45ffb640 1 -- 172.21.15.64:0/3570840635 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f4c00f940 con 0x7f3f50151a80 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.862+0000 7f3f45ffb640 1 -- 172.21.15.64:0/3570840635 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f4c018070 con 0x7f3f50151a80 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.862+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f5006dc90 con 0x7f3f50151a80 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.862+0000 7f3f45ffb640 1 -- 172.21.15.64:0/3570840635 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f4c00fd60 con 0x7f3f50151a80 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.862+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f5006e250 con 0x7f3f50151a80 2024-03-04T07:31:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.863+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3f5006e450 con 0x7f3f50151a80 2024-03-04T07:31:40.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.864+0000 7f3f45ffb640 1 -- 172.21.15.64:0/3570840635 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3f4c01d460 con 0x7f3f50151a80 2024-03-04T07:31:40.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.865+0000 7f3f45ffb640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f3406b500 0x7f3f3406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:40.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.865+0000 7f3f45ffb640 1 -- 172.21.15.64:0/3570840635 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f3f4c08a350 con 0x7f3f50151a80 2024-03-04T07:31:40.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.865+0000 7f3f549c5640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f3406b500 0x7f3f3406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:40.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.866+0000 7f3f549c5640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f3406b500 0x7f3f3406d9c0 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f3f40036000 tx=0x7f3f40030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:40.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:40.871+0000 7f3f45ffb640 1 -- 172.21.15.64:0/3570840635 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3f5006e450 con 0x7f3f50151a80 2024-03-04T07:31:41.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.200+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3f500752d0 con 0x7f3f50151a80 2024-03-04T07:31:41.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.200+0000 7f3f45ffb640 1 -- 172.21.15.64:0/3570840635 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f3f4c0591b0 con 0x7f3f50151a80 2024-03-04T07:31:41.201 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:41.201 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:41.201 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:41.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f3406b500 msgr2=0x7f3f3406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:41.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f3406b500 0x7f3f3406d9c0 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f3f40036000 tx=0x7f3f40030000 comp rx=0 tx=0).stop 2024-03-04T07:31:41.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f50151a80 msgr2=0x7f3f5015b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f50151a80 0x7f3f5015b390 secure :-1 s=READY pgs=1183 cs=0 l=1 rev1=1 crypto rx=0x7f3f4c007fd0 tx=0x7f3f4c007720 comp rx=0 tx=0).stop 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 shutdown_connections 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f3406b500 0x7f3f3406d9c0 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f50151e60 0x7f3f500ae260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f500c8ee0 0x7f3f500b0d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 --2- 172.21.15.64:0/3570840635 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f50151a80 0x7f3f5015b390 unknown :-1 s=CLOSED pgs=1183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 >> 172.21.15.64:0/3570840635 conn(0x7f3f500d0c90 msgr2=0x7f3f5015c220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 shutdown_connections 2024-03-04T07:31:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:41.203+0000 7f3f5742a640 1 -- 172.21.15.64:0/3570840635 wait complete. 2024-03-04T07:31:41.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:41.379 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":151,"msgr_send_messages":146,"msgr_recv_bytes":143501,"msgr_send_bytes":4265619,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.028165664,"msgr_running_send_time":0.014579308,"msgr_running_recv_time":0.007074101,"msgr_running_fast_dispatch_time":0.005372935,"msgr_send_messages_queue_lat":{"avgcount":146,"sum":0.011429800,"avgtime":0.000078286},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":107856,"msgr_send_encrypted_bytes":7296},"AsyncMessenger::Worker-1":{"msgr_recv_messages":7725,"msgr_send_messages":15381,"msgr_recv_bytes":2736585,"msgr_send_bytes":16569066,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.749391947,"msgr_running_send_time":0.412575098,"msgr_running_recv_time":0.392247981,"msgr_running_fast_dispatch_time":0.009704563,"msgr_send_messages_queue_lat":{"avgcount":15380,"sum":0.192047580,"avgtime":0.000012486},"msgr_handle_ack_lat":{"avgcount":22040,"sum":0.003876836,"avgtime":0.000000175},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":94464},"AsyncMessenger::Worker-2":{"msgr_recv_messages":222,"msgr_send_messages":227,"msgr_recv_bytes":124736,"msgr_send_bytes":135042,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.026011419,"msgr_running_send_time":0.012624848,"msgr_running_recv_time":0.006523213,"msgr_running_fast_dispatch_time":0.005481864,"msgr_send_messages_queue_lat":{"avgcount":225,"sum":0.014453908,"avgtime":0.000064239},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":88,"sum":2.843663148,"avgtime":0.032314353}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":663,"sum":0.342171624,"avgtime":0.000516095}},"mds":{"request":4555,"reply":4552,"reply_latency":{"avgcount":4552,"sum":6.436676831,"avgtime":0.001414032},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":3944,"inodes_top":0,"inodes_bottom":3944,"inodes_pin_tail":0,"inodes_pinned":3944,"inodes_expired":0,"inodes_with_caps":2954,"caps":2954,"subtrees":2,"traverse":4561,"traverse_hit":4560,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":2591,"root_rbytes":11752448,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":578,"ceph_cap_op_grant":4227,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":2999,"handle_client_caps_dirty":2807,"handle_client_cap_release":3,"process_request_cap_release":5887},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":737,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":1343,"strays_enqueued":606,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":8158,"evex":0,"evtrm":0,"ev":8161,"evexg":0,"evexd":0,"segadd":10,"segex":0,"segtrm":0,"seg":12,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":20953301,"rdpos":4195244,"jlat":{"avgcount":15,"sum":0.075373888,"avgtime":0.005024925},"replayed":3},"mds_mem":{"ino":3153,"ino+":3214,"ino-":61,"dir":13,"dir+":13,"dir-":0,"dn":4492,"dn+":4553,"dn-":61,"cap":2596,"cap+":3202,"cap-":606,"rss":102960,"heap":248164},"mds_server":{"dispatch_client_request":4558,"dispatch_server_request":0,"handle_client_request":4555,"handle_client_session":0,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":1342,"sum":2.879081920,"avgtime":0.002145366},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":4249,"average_load":4249,"avg_session_uptime":807,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":7963311,"buffer_anon_items":2079,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":11679473,"mds_co_items":69670,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":6,"op_laggy":0,"op_send":504,"op_send_bytes":13033436,"op_resend":0,"op_reply":498,"op_latency":{"avgcount":498,"sum":0.957981725,"avgtime":0.001923658},"op_inflight":5,"oplen_avg":{"avgcount":504,"sum":566},"op":504,"op_r":44,"op_w":460,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":16,"osdop_write":23,"osdop_writefull":4,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":420,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":22,"omap_rd":48,"omap_del":6},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":6,"pq_executing_ops_high_water":6,"pq_executing":6,"pq_executing_high_water":6,"pq_executed_ops":400,"pq_executed":400,"pq_item_in_journal":206},"throttle-msgr_dispatch_throttler-mds":{"val":297,"max":104857600,"get_started":0,"get":8094,"get_sum":2273639,"get_or_fail_fail":0,"get_or_fail_success":8094,"take":0,"take_sum":0,"put":8093,"put_sum":2273342,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":504,"take_sum":13095759,"put":498,"put_sum":13095759,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":6,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":504,"take_sum":504,"put":498,"put_sum":498,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":606,"get_sum":61206,"get_or_fail_fail":0,"get_or_fail_success":606,"take":0,"take_sum":0,"put":8,"put_sum":61206,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":4176085,"max":3758096384,"get_started":0,"get":8158,"get_sum":16757616,"get_or_fail_fail":0,"get_or_fail_success":8158,"take":0,"take_sum":0,"put":15,"put_sum":12581531,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:41.380 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 143501, "msgr_recv_encrypted_bytes": 107856, "msgr_recv_messages": 151, "msgr_running_fast_dispatch_time": 0.005372935, "msgr_running_recv_time": 0.007074101, "msgr_running_send_time": 0.014579308, "msgr_running_total_time": 0.028165664, "msgr_send_bytes": 4265619, "msgr_send_encrypted_bytes": 7296, "msgr_send_messages": 146, "msgr_send_messages_queue_lat": { "avgcount": 146, "avgtime": 7.8286e-05, "sum": 0.0114298 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 22040, "avgtime": 1.75e-07, "sum": 0.003876836 }, "msgr_recv_bytes": 2736585, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 7725, "msgr_running_fast_dispatch_time": 0.009704563, "msgr_running_recv_time": 0.392247981, "msgr_running_send_time": 0.412575098, "msgr_running_total_time": 0.749391947, "msgr_send_bytes": 16569066, "msgr_send_encrypted_bytes": 94464, "msgr_send_messages": 15381, "msgr_send_messages_queue_lat": { "avgcount": 15380, "avgtime": 1.2486e-05, "sum": 0.19204758 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 124736, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 222, "msgr_running_fast_dispatch_time": 0.005481864, "msgr_running_recv_time": 0.006523213, "msgr_running_send_time": 0.012624848, "msgr_running_total_time": 0.026011419, "msgr_send_bytes": 135042, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 227, "msgr_send_messages_queue_lat": { "avgcount": 225, "avgtime": 6.4239e-05, "sum": 0.014453908 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 88, "avgtime": 0.032314353, "sum": 2.843663148 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 663, "avgtime": 0.000516095, "sum": 0.342171624 }, "queue_len": 0 }, "mds": { "caps": 2954, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 4227, "ceph_cap_op_revoke": 578, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 3, "handle_client_caps": 2999, "handle_client_caps_dirty": 2807, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 3944, "inodes_bottom": 3944, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 3944, "inodes_top": 0, "inodes_with_caps": 2954, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 5887, "q": 0, "reply": 4552, "reply_latency": { "avgcount": 4552, "avgtime": 0.001414032, "sum": 6.436676831 }, "request": 4555, "root_rbytes": 11752448, "root_rfiles": 2591, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 4561, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 4560, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 737, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 1343, "strays_enqueued": 606, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 8161, "evadd": 8158, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 15, "avgtime": 0.005024925, "sum": 0.075373888 }, "rdpos": 4195244, "replayed": 3, "seg": 12, "segadd": 10, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 20953301 }, "mds_mem": { "cap": 2596, "cap+": 3202, "cap-": 606, "dir": 13, "dir+": 13, "dir-": 0, "dn": 4492, "dn+": 4553, "dn-": 61, "heap": 248164, "ino": 3153, "ino+": 3214, "ino-": 61, "rss": 102960 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 4558, "dispatch_server_request": 0, "handle_client_request": 4555, "handle_client_session": 0, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 1342, "avgtime": 0.002145366, "sum": 2.87908192 } }, "mds_sessions": { "average_load": 4249, "avg_session_uptime": 807, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 4249 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 7963311, "buffer_anon_items": 2079, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 11679473, "mds_co_items": 69670, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 6, "omap_rd": 48, "omap_wr": 22, "op": 504, "op_Infinitylight": 5, "op_active": 6, "op_laggy": 0, "op_latency": { "avgcount": 498, "avgtime": 0.001923658, "sum": 0.957981725 }, "op_pg": 0, "op_r": 44, "op_reply": 498, "op_resend": 0, "op_rmw": 0, "op_send": 504, "op_send_bytes": 13033436, "op_w": 460, "oplen_avg": { "avgcount": 504, "sum": 566 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 420, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 16, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 23, "osdop_writefull": 4, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 400, "pq_executed_ops": 400, "pq_executing": 6, "pq_executing_high_water": 6, "pq_executing_ops": 6, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 206 }, "throttle-msgr_dispatch_throttler-mds": { "get": 8094, "get_or_fail_fail": 0, "get_or_fail_success": 8094, "get_started": 0, "get_sum": 2273639, "max": 104857600, "put": 8093, "put_sum": 2273342, "take": 0, "take_sum": 0, "val": 297, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 498, "put_sum": 13095759, "take": 504, "take_sum": 13095759, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 498, "put_sum": 498, "take": 504, "take_sum": 504, "val": 6, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 606, "get_or_fail_fail": 0, "get_or_fail_success": 606, "get_started": 0, "get_sum": 61206, "max": 3758096384, "put": 8, "put_sum": 61206, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 8158, "get_or_fail_fail": 0, "get_or_fail_success": 8158, "get_started": 0, "get_sum": 16757616, "max": 3758096384, "put": 15, "put_sum": 12581531, "take": 0, "take_sum": 0, "val": 4176085, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:41.380 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 6/737, 400/1343 2024-03-04T07:31:42.381 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:42.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.511+0000 7f288ff3e640 1 Processor -- start 2024-03-04T07:31:42.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.511+0000 7f288ff3e640 1 -- start start 2024-03-04T07:31:42.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ff3e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f28881458c0 0x7f288815d080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:42.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ff3e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28880728e0 con 0x7f288806aec0 2024-03-04T07:31:42.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ff3e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f288815ff00 con 0x7f2888070240 2024-03-04T07:31:42.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ff3e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f288815d5c0 con 0x7f28881458c0 2024-03-04T07:31:42.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288d4d9640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f288806aec0 0x7f2888071ac0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:46818/0 (socket says 172.21.15.64:46818) 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288d4d9640 1 -- 172.21.15.64:0/3562436916 learned_addr learned my addr 172.21.15.64:0/3562436916 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288dcda640 1 --2- 172.21.15.64:0/3562436916 >> v2:172.21.15.66:3300/0 conn(0x7f28881458c0 0x7f288815d080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3540479372 0 0) 0x7f28880728e0 con 0x7f288806aec0 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2874003520 con 0x7f288806aec0 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 938009112 0 0) 0x7f288815ff00 con 0x7f2888070240 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f28880728e0 con 0x7f2888070240 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 135121021 0 0) 0x7f2874003520 con 0x7f288806aec0 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288dcda640 1 -- 172.21.15.64:0/3562436916 >> v1:172.21.15.66:6789/0 conn(0x7f2888070240 legacy=0x7f28880721d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.513+0000 7f288dcda640 1 -- 172.21.15.64:0/3562436916 >> v1:172.21.15.64:6789/0 conn(0x7f288806aec0 legacy=0x7f2888071ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:42.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.514+0000 7f288dcda640 1 -- 172.21.15.64:0/3562436916 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28881601e0 con 0x7f28881458c0 2024-03-04T07:31:42.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.514+0000 7f288dcda640 1 --2- 172.21.15.64:0/3562436916 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 0x7f288815d080 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f28840097a0 tx=0x7f288402b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=52623f0f16ac4270 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:42.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.515+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2884038070 con 0x7f28881458c0 2024-03-04T07:31:42.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.515+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2884031500 con 0x7f28881458c0 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.515+0000 7f288ccd8640 1 -- 172.21.15.64:0/3562436916 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2878001670 con 0x7f28881458c0 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.515+0000 7f288ff3e640 1 -- 172.21.15.64:0/3562436916 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 msgr2=0x7f288815d080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.515+0000 7f288ff3e640 1 --2- 172.21.15.64:0/3562436916 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 0x7f288815d080 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f28840097a0 tx=0x7f288402b820 comp rx=0 tx=0).stop 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.516+0000 7f288ff3e640 1 -- 172.21.15.64:0/3562436916 shutdown_connections 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.516+0000 7f288ff3e640 1 --2- 172.21.15.64:0/3562436916 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 0x7f288815d080 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.516+0000 7f288ff3e640 1 -- 172.21.15.64:0/3562436916 >> 172.21.15.64:0/3562436916 conn(0x7f28880d0c30 msgr2=0x7f28880d1040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.516+0000 7f288ff3e640 1 -- 172.21.15.64:0/3562436916 shutdown_connections 2024-03-04T07:31:42.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.516+0000 7f288ff3e640 1 -- 172.21.15.64:0/3562436916 wait complete. 2024-03-04T07:31:42.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.517+0000 7f288ff3e640 1 Processor -- start 2024-03-04T07:31:42.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.517+0000 7f288ff3e640 1 -- start start 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288ff3e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f288806aec0 0x7f28880b5370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288ff3e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2888070240 0x7f28880ae930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288ff3e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 0x7f28880aee70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288ff3e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2888160a80 con 0x7f2888070240 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288ff3e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28881607a0 con 0x7f28881458c0 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288ff3e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f288815fbb0 con 0x7f288806aec0 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2888070240 0x7f28880ae930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2888070240 0x7f28880ae930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:49676/0 (socket says 172.21.15.64:49676) 2024-03-04T07:31:42.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 -- 172.21.15.64:0/1434888747 learned_addr learned my addr 172.21.15.64:0/1434888747 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288e4db640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 0x7f28880aee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288dcda640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f288806aec0 0x7f28880b5370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 -- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f288806aec0 msgr2=0x7f28880b5370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f288806aec0 0x7f28880b5370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 -- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 msgr2=0x7f28880aee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 0x7f28880aee70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.518+0000 7f288d4d9640 1 -- 172.21.15.64:0/1434888747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28880ad8b0 con 0x7f2888070240 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.519+0000 7f288dcda640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f288806aec0 0x7f28880b5370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.519+0000 7f288d4d9640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2888070240 0x7f28880ae930 secure :-1 s=READY pgs=1185 cs=0 l=1 rev1=1 crypto rx=0x7f287800c780 tx=0x7f287800cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.519+0000 7f287e7fc640 1 -- 172.21.15.64:0/1434888747 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2878017070 con 0x7f2888070240 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.519+0000 7f287e7fc640 1 -- 172.21.15.64:0/1434888747 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2878005af0 con 0x7f2888070240 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.519+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28880adb70 con 0x7f2888070240 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.519+0000 7f287e7fc640 1 -- 172.21.15.64:0/1434888747 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2878007070 con 0x7f2888070240 2024-03-04T07:31:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.519+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28880ae130 con 0x7f2888070240 2024-03-04T07:31:42.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.520+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f28880752d0 con 0x7f2888070240 2024-03-04T07:31:42.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.521+0000 7f287e7fc640 1 -- 172.21.15.64:0/1434888747 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f287801a070 con 0x7f2888070240 2024-03-04T07:31:42.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.521+0000 7f287e7fc640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f286c06b4d0 0x7f286c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:42.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.521+0000 7f287e7fc640 1 -- 172.21.15.64:0/1434888747 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f2878089720 con 0x7f2888070240 2024-03-04T07:31:42.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.522+0000 7f288dcda640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f286c06b4d0 0x7f286c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:42.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.522+0000 7f288dcda640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f286c06b4d0 0x7f286c06d990 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f2884035000 tx=0x7f2884008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.529+0000 7f287e7fc640 1 -- 172.21.15.64:0/1434888747 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2878057f30 con 0x7f2888070240 2024-03-04T07:31:42.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.829+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f28880b0510 con 0x7f2888070240 2024-03-04T07:31:42.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.830+0000 7f287e7fc640 1 -- 172.21.15.64:0/1434888747 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f287805bfd0 con 0x7f2888070240 2024-03-04T07:31:42.830 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:42.830 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:42.830 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f286c06b4d0 msgr2=0x7f286c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f286c06b4d0 0x7f286c06d990 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f2884035000 tx=0x7f2884008030 comp rx=0 tx=0).stop 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2888070240 msgr2=0x7f28880ae930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2888070240 0x7f28880ae930 secure :-1 s=READY pgs=1185 cs=0 l=1 rev1=1 crypto rx=0x7f287800c780 tx=0x7f287800cc40 comp rx=0 tx=0).stop 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 shutdown_connections 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f286c06b4d0 0x7f286c06d990 unknown :-1 s=CLOSED pgs=538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28881458c0 0x7f28880aee70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2888070240 0x7f28880ae930 unknown :-1 s=CLOSED pgs=1185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 --2- 172.21.15.64:0/1434888747 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f288806aec0 0x7f28880b5370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 >> 172.21.15.64:0/1434888747 conn(0x7f28880d0c30 msgr2=0x7f2888143680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 shutdown_connections 2024-03-04T07:31:42.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:42.832+0000 7f288ff3e640 1 -- 172.21.15.64:0/1434888747 wait complete. 2024-03-04T07:31:42.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:42.996 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":216,"msgr_send_messages":211,"msgr_recv_bytes":160011,"msgr_send_bytes":4286809,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.033492720,"msgr_running_send_time":0.016645909,"msgr_running_recv_time":0.008146852,"msgr_running_fast_dispatch_time":0.007224880,"msgr_send_messages_queue_lat":{"avgcount":211,"sum":0.014471430,"avgtime":0.000068584},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":107856,"msgr_send_encrypted_bytes":7296},"AsyncMessenger::Worker-1":{"msgr_recv_messages":8731,"msgr_send_messages":18514,"msgr_recv_bytes":3110333,"msgr_send_bytes":21870389,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.885591207,"msgr_running_send_time":0.493614505,"msgr_running_recv_time":0.448921437,"msgr_running_fast_dispatch_time":0.013501102,"msgr_send_messages_queue_lat":{"avgcount":18513,"sum":0.266164520,"avgtime":0.000014377},"msgr_handle_ack_lat":{"avgcount":25332,"sum":0.004565135,"avgtime":0.000000180},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":94464},"AsyncMessenger::Worker-2":{"msgr_recv_messages":355,"msgr_send_messages":357,"msgr_recv_bytes":163193,"msgr_send_bytes":182277,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.038842597,"msgr_running_send_time":0.018820754,"msgr_running_recv_time":0.008912028,"msgr_running_fast_dispatch_time":0.008933161,"msgr_send_messages_queue_lat":{"avgcount":355,"sum":0.021400133,"avgtime":0.000060282},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":2096,"complete_latency":{"avgcount":90,"sum":2.844162946,"avgtime":0.031601810}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":1079,"sum":0.415468345,"avgtime":0.000385049}},"mds":{"request":5293,"reply":5291,"reply_latency":{"avgcount":5291,"sum":7.623762063,"avgtime":0.001440892},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":4928,"inodes_top":0,"inodes_bottom":4928,"inodes_pin_tail":0,"inodes_pinned":4928,"inodes_expired":0,"inodes_with_caps":2549,"caps":2549,"subtrees":2,"traverse":5299,"traverse_hit":5298,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":2133,"root_rbytes":12225536,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":619,"ceph_cap_op_grant":5077,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3198,"handle_client_caps_dirty":2994,"handle_client_cap_release":4,"process_request_cap_release":7363},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":1428,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":2081,"strays_enqueued":653,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":9144,"evex":0,"evtrm":0,"ev":9147,"evexg":0,"evexd":0,"segadd":11,"segex":0,"segtrm":0,"seg":13,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":23723693,"rdpos":4195244,"jlat":{"avgcount":16,"sum":0.101555711,"avgtime":0.006347231},"replayed":3},"mds_mem":{"ino":2866,"ino+":3214,"ino-":348,"dir":13,"dir+":13,"dir-":0,"dn":4944,"dn+":5292,"dn-":348,"cap":2549,"cap+":3202,"cap-":653,"rss":111792,"heap":248164},"mds_server":{"dispatch_client_request":5296,"dispatch_server_request":0,"handle_client_request":5293,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":2081,"sum":4.066167152,"avgtime":0.001953948},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":4902,"average_load":4902,"avg_session_uptime":807,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":7963311,"buffer_anon_items":2079,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":11679473,"mds_co_items":69670,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":762,"op_send_bytes":17236418,"op_resend":0,"op_reply":762,"op_latency":{"avgcount":762,"sum":1.455972229,"avgtime":0.001910724},"op_inflight":0,"oplen_avg":{"avgcount":762,"sum":826},"op":762,"op_r":47,"op_w":715,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":19,"osdop_write":27,"osdop_writefull":6,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":668,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":24,"omap_rd":48,"omap_del":7},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":653,"pq_executed":653,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":9298,"get_sum":2593994,"get_or_fail_fail":0,"get_or_fail_success":9298,"take":0,"take_sum":0,"put":9298,"put_sum":2593994,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":762,"take_sum":17303488,"put":762,"put_sum":17303488,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":762,"take_sum":762,"put":762,"put_sum":762,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":653,"get_sum":65953,"get_or_fail_fail":0,"get_or_fail_success":653,"take":0,"take_sum":0,"put":11,"put_sum":65953,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":2752173,"max":3758096384,"get_started":0,"get":9144,"get_sum":19528008,"get_or_fail_fail":0,"get_or_fail_success":9144,"take":0,"take_sum":0,"put":16,"put_sum":16775835,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:42.997 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 160011, "msgr_recv_encrypted_bytes": 107856, "msgr_recv_messages": 216, "msgr_running_fast_dispatch_time": 0.00722488, "msgr_running_recv_time": 0.008146852, "msgr_running_send_time": 0.016645909, "msgr_running_total_time": 0.03349272, "msgr_send_bytes": 4286809, "msgr_send_encrypted_bytes": 7296, "msgr_send_messages": 211, "msgr_send_messages_queue_lat": { "avgcount": 211, "avgtime": 6.8584e-05, "sum": 0.01447143 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 25332, "avgtime": 1.8e-07, "sum": 0.004565135 }, "msgr_recv_bytes": 3110333, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 8731, "msgr_running_fast_dispatch_time": 0.013501102, "msgr_running_recv_time": 0.448921437, "msgr_running_send_time": 0.493614505, "msgr_running_total_time": 0.885591207, "msgr_send_bytes": 21870389, "msgr_send_encrypted_bytes": 94464, "msgr_send_messages": 18514, "msgr_send_messages_queue_lat": { "avgcount": 18513, "avgtime": 1.4377e-05, "sum": 0.26616452 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 163193, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 355, "msgr_running_fast_dispatch_time": 0.008933161, "msgr_running_recv_time": 0.008912028, "msgr_running_send_time": 0.018820754, "msgr_running_total_time": 0.038842597, "msgr_send_bytes": 182277, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 357, "msgr_send_messages_queue_lat": { "avgcount": 355, "avgtime": 6.0282e-05, "sum": 0.021400133 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 90, "avgtime": 0.03160181, "sum": 2.844162946 }, "queue_len": 2096 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 1079, "avgtime": 0.000385049, "sum": 0.415468345 }, "queue_len": 0 }, "mds": { "caps": 2549, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 5077, "ceph_cap_op_revoke": 619, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 4, "handle_client_caps": 3198, "handle_client_caps_dirty": 2994, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 4928, "inodes_bottom": 4928, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 4928, "inodes_top": 0, "inodes_with_caps": 2549, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 7363, "q": 0, "reply": 5291, "reply_latency": { "avgcount": 5291, "avgtime": 0.001440892, "sum": 7.623762063 }, "request": 5293, "root_rbytes": 12225536, "root_rfiles": 2133, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 5299, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 5298, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 1428, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 2081, "strays_enqueued": 653, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 9147, "evadd": 9144, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 16, "avgtime": 0.006347231, "sum": 0.101555711 }, "rdpos": 4195244, "replayed": 3, "seg": 13, "segadd": 11, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 23723693 }, "mds_mem": { "cap": 2549, "cap+": 3202, "cap-": 653, "dir": 13, "dir+": 13, "dir-": 0, "dn": 4944, "dn+": 5292, "dn-": 348, "heap": 248164, "ino": 2866, "ino+": 3214, "ino-": 348, "rss": 111792 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 5296, "dispatch_server_request": 0, "handle_client_request": 5293, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 2081, "avgtime": 0.001953948, "sum": 4.066167152 } }, "mds_sessions": { "average_load": 4902, "avg_session_uptime": 807, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 4902 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 7963311, "buffer_anon_items": 2079, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 11679473, "mds_co_items": 69670, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 7, "omap_rd": 48, "omap_wr": 24, "op": 762, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 762, "avgtime": 0.001910724, "sum": 1.455972229 }, "op_pg": 0, "op_r": 47, "op_reply": 762, "op_resend": 0, "op_rmw": 0, "op_send": 762, "op_send_bytes": 17236418, "op_w": 715, "oplen_avg": { "avgcount": 762, "sum": 826 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 668, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 19, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 27, "osdop_writefull": 6, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 653, "pq_executed_ops": 653, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 9298, "get_or_fail_fail": 0, "get_or_fail_success": 9298, "get_started": 0, "get_sum": 2593994, "max": 104857600, "put": 9298, "put_sum": 2593994, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 762, "put_sum": 17303488, "take": 762, "take_sum": 17303488, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 762, "put_sum": 762, "take": 762, "take_sum": 762, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 653, "get_or_fail_fail": 0, "get_or_fail_success": 653, "get_started": 0, "get_sum": 65953, "max": 3758096384, "put": 11, "put_sum": 65953, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 9144, "get_or_fail_fail": 0, "get_or_fail_success": 9144, "get_started": 0, "get_sum": 19528008, "max": 3758096384, "put": 16, "put_sum": 16775835, "take": 0, "take_sum": 0, "val": 2752173, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:42.998 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/1428, 653/2081 2024-03-04T07:31:43.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:44.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.129+0000 7f55b2078640 1 Processor -- start 2024-03-04T07:31:44.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.129+0000 7f55b2078640 1 -- start start 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55b2078640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f55ac0ceae0 0x7f55ac0cfa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55b2078640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0b2f70 0x7f55ac0cffc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55b2078640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55ac15c6c0 con 0x7f55ac15c2e0 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55b2078640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f55ac0cd7a0 con 0x7f55ac0ceae0 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55b2078640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55ac0cda80 con 0x7f55ac0b2f70 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55aaffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f55ac0ceae0 0x7f55ac0cfa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55ab7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f55ac15c2e0 0x7f55ac0ce3d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58210/0 (socket says 172.21.15.64:58210) 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55ab7fe640 1 -- 172.21.15.64:0/1990449202 learned_addr learned my addr 172.21.15.64:0/1990449202 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55abfff640 1 --2- 172.21.15.64:0/1990449202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0b2f70 0x7f55ac0cffc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55aaffd640 1 -- 172.21.15.64:0/1990449202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0b2f70 msgr2=0x7f55ac0cffc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:44.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55aaffd640 1 --2- 172.21.15.64:0/1990449202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0b2f70 0x7f55ac0cffc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55aaffd640 1 -- 172.21.15.64:0/1990449202 >> v1:172.21.15.64:6789/0 conn(0x7f55ac15c2e0 legacy=0x7f55ac0ce3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:44.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55aa7fc640 1 -- 172.21.15.64:0/1990449202 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3296730522 0 0) 0x7f55ac15c6c0 con 0x7f55ac15c2e0 2024-03-04T07:31:44.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55aaffd640 1 -- 172.21.15.64:0/1990449202 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55ac0b1760 con 0x7f55ac0ceae0 2024-03-04T07:31:44.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.130+0000 7f55abfff640 1 --2- 172.21.15.64:0/1990449202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0b2f70 0x7f55ac0cffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:44.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.131+0000 7f55aaffd640 1 --2- 172.21.15.64:0/1990449202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac0ceae0 0x7f55ac0cfa80 secure :-1 s=READY pgs=1187 cs=0 l=1 rev1=1 crypto rx=0x7f55a00097a0 tx=0x7f55a002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c17e0e656fe5b31 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:44.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55aa7fc640 1 -- 172.21.15.64:0/1990449202 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a0035020 con 0x7f55ac0ceae0 2024-03-04T07:31:44.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55aa7fc640 1 -- 172.21.15.64:0/1990449202 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f55a0031830 con 0x7f55ac0ceae0 2024-03-04T07:31:44.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55aa7fc640 1 -- 172.21.15.64:0/1990449202 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a0031c00 con 0x7f55ac0ceae0 2024-03-04T07:31:44.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55b2078640 1 -- 172.21.15.64:0/1990449202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac0ceae0 msgr2=0x7f55ac0cfa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55b2078640 1 --2- 172.21.15.64:0/1990449202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac0ceae0 0x7f55ac0cfa80 secure :-1 s=READY pgs=1187 cs=0 l=1 rev1=1 crypto rx=0x7f55a00097a0 tx=0x7f55a002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55b2078640 1 -- 172.21.15.64:0/1990449202 shutdown_connections 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55b2078640 1 --2- 172.21.15.64:0/1990449202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0b2f70 0x7f55ac0cffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55b2078640 1 --2- 172.21.15.64:0/1990449202 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac0ceae0 0x7f55ac0cfa80 unknown :-1 s=CLOSED pgs=1187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55b2078640 1 -- 172.21.15.64:0/1990449202 >> 172.21.15.64:0/1990449202 conn(0x7f55ac0d1ca0 msgr2=0x7f55ac0d20b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.132+0000 7f55b2078640 1 -- 172.21.15.64:0/1990449202 shutdown_connections 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.133+0000 7f55b2078640 1 -- 172.21.15.64:0/1990449202 wait complete. 2024-03-04T07:31:44.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.134+0000 7f55b2078640 1 Processor -- start 2024-03-04T07:31:44.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.134+0000 7f55b2078640 1 -- start start 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55b2078640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55ac0b2f70 0x7f55ac0c7ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55b2078640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0ceae0 0x7f55ac0b3790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55b2078640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac15c2e0 0x7f55ac0b3cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55b2078640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55ac0cd540 con 0x7f55ac15c2e0 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55b2078640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55ac0cd3c0 con 0x7f55ac0b2f70 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55b2078640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55ac0d27c0 con 0x7f55ac0ceae0 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55abfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac15c2e0 0x7f55ac0b3cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55abfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac15c2e0 0x7f55ac0b3cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37650/0 (socket says 172.21.15.64:37650) 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55abfff640 1 -- 172.21.15.64:0/2793488653 learned_addr learned my addr 172.21.15.64:0/2793488653 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55aaffd640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0ceae0 0x7f55ac0b3790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.135+0000 7f55ab7fe640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55ac0b2f70 0x7f55ac0c7ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55abfff640 1 -- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0ceae0 msgr2=0x7f55ac0b3790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:44.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55abfff640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0ceae0 0x7f55ac0b3790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55abfff640 1 -- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55ac0b2f70 msgr2=0x7f55ac0c7ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55abfff640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55ac0b2f70 0x7f55ac0c7ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55abfff640 1 -- 172.21.15.64:0/2793488653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55ac0b55d0 con 0x7f55ac15c2e0 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55aaffd640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0ceae0 0x7f55ac0b3790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55abfff640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac15c2e0 0x7f55ac0b3cd0 secure :-1 s=READY pgs=1188 cs=0 l=1 rev1=1 crypto rx=0x7f55a400bfb0 tx=0x7f55a400e4f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f5593fff640 1 -- 172.21.15.64:0/2793488653 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a401a020 con 0x7f55ac15c2e0 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f5593fff640 1 -- 172.21.15.64:0/2793488653 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f55a4016640 con 0x7f55ac15c2e0 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f5593fff640 1 -- 172.21.15.64:0/2793488653 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a4016a30 con 0x7f55ac15c2e0 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f55ac0b58b0 con 0x7f55ac15c2e0 2024-03-04T07:31:44.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.136+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55ac0b5e70 con 0x7f55ac15c2e0 2024-03-04T07:31:44.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.138+0000 7f5593fff640 1 -- 172.21.15.64:0/2793488653 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f55a401d020 con 0x7f55ac15c2e0 2024-03-04T07:31:44.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.138+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f558c005290 con 0x7f55ac15c2e0 2024-03-04T07:31:44.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.139+0000 7f5593fff640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f559806b520 0x7f559806d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:44.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.139+0000 7f5593fff640 1 -- 172.21.15.64:0/2793488653 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f55a408ec50 con 0x7f55ac15c2e0 2024-03-04T07:31:44.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.139+0000 7f55ab7fe640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f559806b520 0x7f559806d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:44.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.139+0000 7f55ab7fe640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f559806b520 0x7f559806d9e0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f559c000f60 tx=0x7f559c007c00 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:44.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.147+0000 7f5593fff640 1 -- 172.21.15.64:0/2793488653 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f55a4024020 con 0x7f55ac15c2e0 2024-03-04T07:31:44.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.479+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f558c005580 con 0x7f55ac15c2e0 2024-03-04T07:31:44.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.480+0000 7f5593fff640 1 -- 172.21.15.64:0/2793488653 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f55a405db20 con 0x7f55ac15c2e0 2024-03-04T07:31:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:44.480 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:44.480 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:44.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f559806b520 msgr2=0x7f559806d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:44.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f559806b520 0x7f559806d9e0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f559c000f60 tx=0x7f559c007c00 comp rx=0 tx=0).stop 2024-03-04T07:31:44.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac15c2e0 msgr2=0x7f55ac0b3cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:44.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac15c2e0 0x7f55ac0b3cd0 secure :-1 s=READY pgs=1188 cs=0 l=1 rev1=1 crypto rx=0x7f55a400bfb0 tx=0x7f55a400e4f0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 shutdown_connections 2024-03-04T07:31:44.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f559806b520 0x7f559806d9e0 unknown :-1 s=CLOSED pgs=539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55ac15c2e0 0x7f55ac0b3cd0 unknown :-1 s=CLOSED pgs=1188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55ac0ceae0 0x7f55ac0b3790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 --2- 172.21.15.64:0/2793488653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55ac0b2f70 0x7f55ac0c7ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:44.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 >> 172.21.15.64:0/2793488653 conn(0x7f55ac0d1ca0 msgr2=0x7f55ac158350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:44.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 shutdown_connections 2024-03-04T07:31:44.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:44.482+0000 7f55b2078640 1 -- 172.21.15.64:0/2793488653 wait complete. 2024-03-04T07:31:44.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:44.659 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":273,"msgr_send_messages":268,"msgr_recv_bytes":174499,"msgr_send_bytes":4305681,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.041294039,"msgr_running_send_time":0.020640735,"msgr_running_recv_time":0.009632886,"msgr_running_fast_dispatch_time":0.009208026,"msgr_send_messages_queue_lat":{"avgcount":268,"sum":0.017341641,"avgtime":0.000064707},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":108128,"msgr_send_encrypted_bytes":7920},"AsyncMessenger::Worker-1":{"msgr_recv_messages":9717,"msgr_send_messages":21428,"msgr_recv_bytes":3497693,"msgr_send_bytes":27084318,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.007554864,"msgr_running_send_time":0.564355959,"msgr_running_recv_time":0.504146540,"msgr_running_fast_dispatch_time":0.017598582,"msgr_send_messages_queue_lat":{"avgcount":21427,"sum":0.321792706,"avgtime":0.000015018},"msgr_handle_ack_lat":{"avgcount":28157,"sum":0.005148547,"avgtime":0.000000182},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":96256},"AsyncMessenger::Worker-2":{"msgr_recv_messages":488,"msgr_send_messages":490,"msgr_recv_bytes":266965,"msgr_send_bytes":295715,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.053936147,"msgr_running_send_time":0.026647280,"msgr_running_recv_time":0.011922851,"msgr_running_fast_dispatch_time":0.011865550,"msgr_send_messages_queue_lat":{"avgcount":488,"sum":0.026866386,"avgtime":0.000055054},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":1424,"complete_latency":{"avgcount":96,"sum":4.534907593,"avgtime":0.047238620}},"finisher-PurgeQueue":{"queue_len":412,"complete_latency":{"avgcount":1491,"sum":0.536170345,"avgtime":0.000359604}},"mds":{"request":6193,"reply":6190,"reply_latency":{"avgcount":6190,"sum":8.787710832,"avgtime":0.001419662},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":0,"dir_split":0,"dir_merge":0,"inodes":5538,"inodes_top":0,"inodes_bottom":5538,"inodes_pin_tail":0,"inodes_pinned":5538,"inodes_expired":0,"inodes_with_caps":1855,"caps":1855,"subtrees":2,"traverse":6198,"traverse_hit":6197,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":1259,"root_rbytes":9657344,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":619,"ceph_cap_op_grant":5968,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3204,"handle_client_caps_dirty":3000,"handle_client_cap_release":6,"process_request_cap_release":9163},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":1812,"num_strays_delayed":0,"num_strays_enqueuing":179,"strays_created":2980,"strays_enqueued":1168,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":10567,"evex":0,"evtrm":0,"ev":10570,"evexg":0,"evexd":0,"segadd":13,"segex":0,"segtrm":0,"seg":15,"segmjr":2,"segexg":0,"segexd":0,"expos":4194304,"wrpos":26914109,"rdpos":4195244,"jlat":{"avgcount":18,"sum":0.134257440,"avgtime":0.007458746},"replayed":3},"mds_mem":{"ino":2561,"ino+":3214,"ino-":653,"dir":13,"dir+":13,"dir-":0,"dn":5538,"dn+":6191,"dn-":653,"cap":1855,"cap+":3202,"cap-":1347,"rss":105180,"heap":248164},"mds_server":{"dispatch_client_request":6196,"dispatch_server_request":0,"handle_client_request":6193,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":2980,"sum":5.230115921,"avgtime":0.001755072},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5699,"average_load":5699,"avg_session_uptime":812,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":23895845,"buffer_anon_items":6820,"buffer_meta_bytes":88,"buffer_meta_items":1,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":13551660,"mds_co_items":136018,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":1027,"op_send_bytes":21502680,"op_resend":0,"op_reply":1027,"op_latency":{"avgcount":1027,"sum":1.938963015,"avgtime":0.001887987},"op_inflight":0,"oplen_avg":{"avgcount":1027,"sum":1095},"op":1027,"op_r":53,"op_w":974,"op_rmw":0,"op_pg":0,"osdop_stat":2,"osdop_create":0,"osdop_read":25,"osdop_write":36,"osdop_writefull":8,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":914,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":0,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":28,"omap_rd":48,"omap_del":9},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":1,"pq_executing_ops_high_water":6,"pq_executing":1,"pq_executing_high_water":6,"pq_executed_ops":897,"pq_executed":897,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":297,"max":104857600,"get_started":0,"get":10474,"get_sum":2993722,"get_or_fail_fail":0,"get_or_fail_success":10474,"take":0,"take_sum":0,"put":10473,"put_sum":2993425,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1027,"take_sum":21639844,"put":1027,"put_sum":21639844,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1027,"take_sum":1027,"put":1027,"put_sum":1027,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":1347,"get_sum":136047,"get_or_fail_fail":0,"get_or_fail_success":1347,"take":0,"take_sum":0,"put":17,"put_sum":136047,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":1748285,"max":3758096384,"get_started":0,"get":10567,"get_sum":22718424,"get_or_fail_fail":0,"get_or_fail_success":10567,"take":0,"take_sum":0,"put":19,"put_sum":20970139,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:44.661 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 174499, "msgr_recv_encrypted_bytes": 108128, "msgr_recv_messages": 273, "msgr_running_fast_dispatch_time": 0.009208026, "msgr_running_recv_time": 0.009632886, "msgr_running_send_time": 0.020640735, "msgr_running_total_time": 0.041294039, "msgr_send_bytes": 4305681, "msgr_send_encrypted_bytes": 7920, "msgr_send_messages": 268, "msgr_send_messages_queue_lat": { "avgcount": 268, "avgtime": 6.4707e-05, "sum": 0.017341641 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 28157, "avgtime": 1.82e-07, "sum": 0.005148547 }, "msgr_recv_bytes": 3497693, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 9717, "msgr_running_fast_dispatch_time": 0.017598582, "msgr_running_recv_time": 0.50414654, "msgr_running_send_time": 0.564355959, "msgr_running_total_time": 1.007554864, "msgr_send_bytes": 27084318, "msgr_send_encrypted_bytes": 96256, "msgr_send_messages": 21428, "msgr_send_messages_queue_lat": { "avgcount": 21427, "avgtime": 1.5018e-05, "sum": 0.321792706 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 266965, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 488, "msgr_running_fast_dispatch_time": 0.01186555, "msgr_running_recv_time": 0.011922851, "msgr_running_send_time": 0.02664728, "msgr_running_total_time": 0.053936147, "msgr_send_bytes": 295715, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 490, "msgr_send_messages_queue_lat": { "avgcount": 488, "avgtime": 5.5054e-05, "sum": 0.026866386 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 96, "avgtime": 0.04723862, "sum": 4.534907593 }, "queue_len": 1424 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 1491, "avgtime": 0.000359604, "sum": 0.536170345 }, "queue_len": 412 }, "mds": { "caps": 1855, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 5968, "ceph_cap_op_revoke": 619, "ceph_cap_op_trunc": 0, "dir_commit": 0, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 6, "handle_client_caps": 3204, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 5538, "inodes_bottom": 5538, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 5538, "inodes_top": 0, "inodes_with_caps": 1855, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 9163, "q": 0, "reply": 6190, "reply_latency": { "avgcount": 6190, "avgtime": 0.001419662, "sum": 8.787710832 }, "request": 6193, "root_rbytes": 9657344, "root_rfiles": 1259, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6198, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6197, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 1812, "num_strays_delayed": 0, "num_strays_enqueuing": 179, "recovery_completed": 0, "recovery_started": 0, "strays_created": 2980, "strays_enqueued": 1168, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 10570, "evadd": 10567, "evex": 0, "evexd": 0, "evexg": 0, "evlrg": 0, "evtrm": 0, "expos": 4194304, "jlat": { "avgcount": 18, "avgtime": 0.007458746, "sum": 0.13425744 }, "rdpos": 4195244, "replayed": 3, "seg": 15, "segadd": 13, "segex": 0, "segexd": 0, "segexg": 0, "segmjr": 2, "segtrm": 0, "wrpos": 26914109 }, "mds_mem": { "cap": 1855, "cap+": 3202, "cap-": 1347, "dir": 13, "dir+": 13, "dir-": 0, "dn": 5538, "dn+": 6191, "dn-": 653, "heap": 248164, "ino": 2561, "ino+": 3214, "ino-": 653, "rss": 105180 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6196, "dispatch_server_request": 0, "handle_client_request": 6193, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 2980, "avgtime": 0.001755072, "sum": 5.230115921 } }, "mds_sessions": { "average_load": 5699, "avg_session_uptime": 812, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5699 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 23895845, "buffer_anon_items": 6820, "buffer_meta_bytes": 88, "buffer_meta_items": 1, "mds_co_bytes": 13551660, "mds_co_items": 136018, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 9, "omap_rd": 48, "omap_wr": 28, "op": 1027, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 1027, "avgtime": 0.001887987, "sum": 1.938963015 }, "op_pg": 0, "op_r": 53, "op_reply": 1027, "op_resend": 0, "op_rmw": 0, "op_send": 1027, "op_send_bytes": 21502680, "op_w": 974, "oplen_avg": { "avgcount": 1027, "sum": 1095 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 0, "osdop_delete": 914, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 25, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 0, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 2, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 36, "osdop_writefull": 8, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 897, "pq_executed_ops": 897, "pq_executing": 1, "pq_executing_high_water": 6, "pq_executing_ops": 1, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 10474, "get_or_fail_fail": 0, "get_or_fail_success": 10474, "get_started": 0, "get_sum": 2993722, "max": 104857600, "put": 10473, "put_sum": 2993425, "take": 0, "take_sum": 0, "val": 297, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 1027, "put_sum": 21639844, "take": 1027, "take_sum": 21639844, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 1027, "put_sum": 1027, "take": 1027, "take_sum": 1027, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 1347, "get_or_fail_fail": 0, "get_or_fail_success": 1347, "get_started": 0, "get_sum": 136047, "max": 3758096384, "put": 17, "put_sum": 136047, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 10567, "get_or_fail_fail": 0, "get_or_fail_success": 10567, "get_started": 0, "get_sum": 22718424, "max": 3758096384, "put": 19, "put_sum": 20970139, "take": 0, "take_sum": 0, "val": 1748285, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:44.661 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 1/1812, 897/2980 2024-03-04T07:31:45.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:45.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:45.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.744+0000 7f2c7115c640 1 Processor -- start 2024-03-04T07:31:45.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.744+0000 7f2c7115c640 1 -- start start 2024-03-04T07:31:45.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c7115c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c15f7d0 0x7f2c6c06aec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c7115c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2c6c0b5030 0x7f2c6c0ad1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c7115c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2c6c0b5b80 con 0x7f2c6c0b6030 2024-03-04T07:31:45.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c7115c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2c6c0b1d20 con 0x7f2c6c0b5030 2024-03-04T07:31:45.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c7115c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2c6c0b55b0 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6b59e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2c6c0b6030 0x7f2c6c0ccd10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58212/0 (socket says 172.21.15.64:58212) 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6b59e640 1 -- 172.21.15.64:0/4120300901 learned_addr learned my addr 172.21.15.64:0/4120300901 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6ad9d640 1 --2- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c15f7d0 0x7f2c6c06aec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6a59c640 1 --2- 172.21.15.64:0/4120300901 >> v2:172.21.15.66:3300/0 conn(0x7f2c6c0b5030 0x7f2c6c0ad1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c69d9b640 1 -- 172.21.15.64:0/4120300901 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2056386172 0 0) 0x7f2c64001490 con 0x7f2c6c0b6030 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c69d9b640 1 -- 172.21.15.64:0/4120300901 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2c54003520 con 0x7f2c6c0b6030 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c69d9b640 1 -- 172.21.15.64:0/4120300901 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1831138918 0 0) 0x7f2c54003520 con 0x7f2c6c0b6030 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6a59c640 1 -- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c15f7d0 msgr2=0x7f2c6c06aec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6a59c640 1 --2- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c15f7d0 0x7f2c6c06aec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6a59c640 1 -- 172.21.15.64:0/4120300901 >> v1:172.21.15.64:6789/0 conn(0x7f2c6c0b6030 legacy=0x7f2c6c0ccd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.746+0000 7f2c6a59c640 1 -- 172.21.15.64:0/4120300901 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2c6c0732f0 con 0x7f2c6c0b5030 2024-03-04T07:31:45.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.747+0000 7f2c6ad9d640 1 --2- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c15f7d0 0x7f2c6c06aec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:45.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.747+0000 7f2c6a59c640 1 --2- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 0x7f2c6c0ad1b0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f2c600097a0 tx=0x7f2c6002b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e35f085cdac37b99 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:45.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.747+0000 7f2c69d9b640 1 -- 172.21.15.64:0/4120300901 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c60038070 con 0x7f2c6c0b5030 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.747+0000 7f2c69d9b640 1 -- 172.21.15.64:0/4120300901 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2c60031500 con 0x7f2c6c0b5030 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.747+0000 7f2c69d9b640 1 -- 172.21.15.64:0/4120300901 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c64001bc0 con 0x7f2c6c0b5030 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 -- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 msgr2=0x7f2c6c0ad1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 --2- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 0x7f2c6c0ad1b0 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f2c600097a0 tx=0x7f2c6002b820 comp rx=0 tx=0).stop 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 -- 172.21.15.64:0/4120300901 shutdown_connections 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 --2- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 0x7f2c6c0ad1b0 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 --2- 172.21.15.64:0/4120300901 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c15f7d0 0x7f2c6c06aec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 -- 172.21.15.64:0/4120300901 >> 172.21.15.64:0/4120300901 conn(0x7f2c6c072650 msgr2=0x7f2c6c072a60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 -- 172.21.15.64:0/4120300901 shutdown_connections 2024-03-04T07:31:45.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.748+0000 7f2c7115c640 1 -- 172.21.15.64:0/4120300901 wait complete. 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.749+0000 7f2c7115c640 1 Processor -- start 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c7115c640 1 -- start start 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c7115c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 0x7f2c6c0c5ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c7115c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c0b6030 0x7f2c6c159230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c7115c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2c6c15f7d0 0x7f2c6c0ce6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c7115c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c6c06b580 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c7115c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c6c06b400 con 0x7f2c6c0b5030 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c7115c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2c6c06b700 con 0x7f2c6c0b6030 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c6b59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2c6c15f7d0 0x7f2c6c0ce6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c6b59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2c6c15f7d0 0x7f2c6c0ce6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37666/0 (socket says 172.21.15.64:37666) 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c6b59e640 1 -- 172.21.15.64:0/1087723813 learned_addr learned my addr 172.21.15.64:0/1087723813 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.750+0000 7f2c6ad9d640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 0x7f2c6c0c5ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c6a59c640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c0b6030 0x7f2c6c159230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c6b59e640 1 -- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c0b6030 msgr2=0x7f2c6c159230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c6b59e640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c0b6030 0x7f2c6c159230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c6b59e640 1 -- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 msgr2=0x7f2c6c0c5ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c6b59e640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 0x7f2c6c0c5ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c6b59e640 1 -- 172.21.15.64:0/1087723813 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2c6c158200 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c6b59e640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2c6c15f7d0 0x7f2c6c0ce6c0 secure :-1 s=READY pgs=1190 cs=0 l=1 rev1=1 crypto rx=0x7f2c64001a60 tx=0x7f2c6400a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c537fe640 1 -- 172.21.15.64:0/1087723813 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c64012400 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c537fe640 1 -- 172.21.15.64:0/1087723813 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2c640125c0 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c537fe640 1 -- 172.21.15.64:0/1087723813 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c640129b0 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2c6c158470 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.751+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2c6c158a30 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.752+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2c6c0752d0 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.753+0000 7f2c537fe640 1 -- 172.21.15.64:0/1087723813 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2c64012b50 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.753+0000 7f2c537fe640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2c5806b4d0 0x7f2c5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.753+0000 7f2c537fe640 1 -- 172.21.15.64:0/1087723813 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f2c6408aa10 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.753+0000 7f2c6ad9d640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2c5806b4d0 0x7f2c5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.754+0000 7f2c6ad9d640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2c5806b4d0 0x7f2c5806d990 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f2c6c0b18d0 tx=0x7f2c5c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:45.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.761+0000 7f2c537fe640 1 -- 172.21.15.64:0/1087723813 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2c6c0b5b80 con 0x7f2c6c15f7d0 2024-03-04T07:31:45.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.790+0000 7fa283cb5640 1 Processor -- start 2024-03-04T07:31:45.790 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.790+0000 7fa283cb5640 1 -- start start 2024-03-04T07:31:45.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa283cb5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa27c15e100 0x7fa27c0d1cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa283cb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c0b1070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa283cb5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa27c0d0100 con 0x7fa27c0d4c40 2024-03-04T07:31:45.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa283cb5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa27c0cf860 con 0x7fa27c15e100 2024-03-04T07:31:45.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa283cb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa27c0cfb40 con 0x7fa27c0d2a30 2024-03-04T07:31:45.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa282252640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c0b1070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa281a51640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa27c15e100 0x7fa27c0d1cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa281250640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fa27c0d4c40 0x7fa27c0d2320 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43630/0 (socket says 172.21.15.64:43630) 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa282252640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c0b1070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:51952/0 (socket says 172.21.15.64:51952) 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa281250640 1 -- 172.21.15.64:0/320059202 learned_addr learned my addr 172.21.15.64:0/320059202 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa281a51640 1 -- 172.21.15.64:0/320059202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 msgr2=0x7fa27c0b1070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa281a51640 1 --2- 172.21.15.64:0/320059202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c0b1070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa281a51640 1 -- 172.21.15.64:0/320059202 >> v1:172.21.15.66:6789/0 conn(0x7fa27c0d4c40 legacy=0x7fa27c0d2320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa281a51640 1 -- 172.21.15.64:0/320059202 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa27c0cfe20 con 0x7fa27c15e100 2024-03-04T07:31:45.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.791+0000 7fa280a4f640 1 -- 172.21.15.64:0/320059202 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3171677225 0 0) 0x7fa27c0d0100 con 0x7fa27c0d4c40 2024-03-04T07:31:45.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.792+0000 7fa281a51640 1 --2- 172.21.15.64:0/320059202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 0x7fa27c0d1cd0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7fa2780097a0 tx=0x7fa27802b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6e6c783dece4356f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.792+0000 7fa280a4f640 1 -- 172.21.15.64:0/320059202 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa278035020 con 0x7fa27c15e100 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.792+0000 7fa280a4f640 1 -- 172.21.15.64:0/320059202 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa278031830 con 0x7fa27c15e100 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.792+0000 7fa280a4f640 1 -- 172.21.15.64:0/320059202 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa278031c00 con 0x7fa27c15e100 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 -- 172.21.15.64:0/320059202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 msgr2=0x7fa27c0d1cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 --2- 172.21.15.64:0/320059202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 0x7fa27c0d1cd0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7fa2780097a0 tx=0x7fa27802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 -- 172.21.15.64:0/320059202 shutdown_connections 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 --2- 172.21.15.64:0/320059202 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c0b1070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 --2- 172.21.15.64:0/320059202 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 0x7fa27c0d1cd0 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 -- 172.21.15.64:0/320059202 >> 172.21.15.64:0/320059202 conn(0x7fa27c0d3ef0 msgr2=0x7fa27c0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 -- 172.21.15.64:0/320059202 shutdown_connections 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.793+0000 7fa283cb5640 1 -- 172.21.15.64:0/320059202 wait complete. 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.794+0000 7fa283cb5640 1 Processor -- start 2024-03-04T07:31:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.794+0000 7fa283cb5640 1 -- start start 2024-03-04T07:31:45.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa283cb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c15ca40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa283cb5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa27c0d4c40 0x7fa27c09ded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa283cb5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 0x7fa27c09e410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa283cb5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa27c15e4e0 con 0x7fa27c0d4c40 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa283cb5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa27c0cf510 con 0x7fa27c15e100 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa283cb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa27c15e660 con 0x7fa27c0d2a30 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa27c0d4c40 0x7fa27c09ded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa27c0d4c40 0x7fa27c09ded0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37670/0 (socket says 172.21.15.64:37670) 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 -- 172.21.15.64:0/3849040070 learned_addr learned my addr 172.21.15.64:0/3849040070 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281a51640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c15ca40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa282252640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 0x7fa27c09e410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 -- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 msgr2=0x7fa27c15ca40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c15ca40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 -- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 msgr2=0x7fa27c09e410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 0x7fa27c09e410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.795+0000 7fa281250640 1 -- 172.21.15.64:0/3849040070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa27c09ea70 con 0x7fa27c0d4c40 2024-03-04T07:31:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.796+0000 7fa281250640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa27c0d4c40 0x7fa27c09ded0 secure :-1 s=READY pgs=1191 cs=0 l=1 rev1=1 crypto rx=0x7fa26c00c4d0 tx=0x7fa26c00c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:45.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.796+0000 7fa2727fc640 1 -- 172.21.15.64:0/3849040070 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa26c018070 con 0x7fa27c0d4c40 2024-03-04T07:31:45.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.796+0000 7fa2727fc640 1 -- 172.21.15.64:0/3849040070 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa26c007500 con 0x7fa27c0d4c40 2024-03-04T07:31:45.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.796+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa27c09ed50 con 0x7fa27c0d4c40 2024-03-04T07:31:45.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.796+0000 7fa2727fc640 1 -- 172.21.15.64:0/3849040070 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa26c004a20 con 0x7fa27c0d4c40 2024-03-04T07:31:45.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.796+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa27c09f310 con 0x7fa27c0d4c40 2024-03-04T07:31:45.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.797+0000 7fa2727fc640 1 -- 172.21.15.64:0/3849040070 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa26c01b070 con 0x7fa27c0d4c40 2024-03-04T07:31:45.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.797+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa27c0752d0 con 0x7fa27c0d4c40 2024-03-04T07:31:45.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.797+0000 7fa2727fc640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa26006b4d0 0x7fa26006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:45.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.798+0000 7fa2727fc640 1 -- 172.21.15.64:0/3849040070 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fa26c089520 con 0x7fa27c0d4c40 2024-03-04T07:31:45.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.798+0000 7fa281a51640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa26006b4d0 0x7fa26006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:45.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.798+0000 7fa281a51640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa26006b4d0 0x7fa26006d990 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7fa278036000 tx=0x7fa278008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:45.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:45.802+0000 7fa2727fc640 1 -- 172.21.15.64:0/3849040070 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa26c058380 con 0x7fa27c0d4c40 2024-03-04T07:31:46.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.016+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2c6c0b02b0 con 0x7f2c6c15f7d0 2024-03-04T07:31:46.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.016+0000 7f2c537fe640 1 -- 172.21.15.64:0/1087723813 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f2c64059190 con 0x7f2c6c15f7d0 2024-03-04T07:31:46.016 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:46.017 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:46.017 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:46.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2c5806b4d0 msgr2=0x7f2c5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:46.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2c5806b4d0 0x7f2c5806d990 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f2c6c0b18d0 tx=0x7f2c5c009190 comp rx=0 tx=0).stop 2024-03-04T07:31:46.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2c6c15f7d0 msgr2=0x7f2c6c0ce6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2c6c15f7d0 0x7f2c6c0ce6c0 secure :-1 s=READY pgs=1190 cs=0 l=1 rev1=1 crypto rx=0x7f2c64001a60 tx=0x7f2c6400a430 comp rx=0 tx=0).stop 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 shutdown_connections 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2c5806b4d0 0x7f2c5806d990 unknown :-1 s=CLOSED pgs=540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2c6c15f7d0 0x7f2c6c0ce6c0 unknown :-1 s=CLOSED pgs=1190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2c6c0b6030 0x7f2c6c159230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 --2- 172.21.15.64:0/1087723813 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2c6c0b5030 0x7f2c6c0c5ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 >> 172.21.15.64:0/1087723813 conn(0x7f2c6c072650 msgr2=0x7f2c6c0af1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 shutdown_connections 2024-03-04T07:31:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.018+0000 7f2c7115c640 1 -- 172.21.15.64:0/1087723813 wait complete. 2024-03-04T07:31:46.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-04T07:31:46.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.138+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa27c0ce2b0 con 0x7fa27c0d4c40 2024-03-04T07:31:46.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.139+0000 7fa2727fc640 1 -- 172.21.15.64:0/3849040070 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fa26c057c80 con 0x7fa27c0d4c40 2024-03-04T07:31:46.139 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:46.139 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:46.139 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:46.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa26006b4d0 msgr2=0x7fa26006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:46.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa26006b4d0 0x7fa26006d990 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7fa278036000 tx=0x7fa278008030 comp rx=0 tx=0).stop 2024-03-04T07:31:46.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa27c0d4c40 msgr2=0x7fa27c09ded0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:46.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa27c0d4c40 0x7fa27c09ded0 secure :-1 s=READY pgs=1191 cs=0 l=1 rev1=1 crypto rx=0x7fa26c00c4d0 tx=0x7fa26c00c990 comp rx=0 tx=0).stop 2024-03-04T07:31:46.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 shutdown_connections 2024-03-04T07:31:46.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa26006b4d0 0x7fa26006d990 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa27c15e100 0x7fa27c09e410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa27c0d4c40 0x7fa27c09ded0 unknown :-1 s=CLOSED pgs=1191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 --2- 172.21.15.64:0/3849040070 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa27c0d2a30 0x7fa27c15ca40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:46.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 >> 172.21.15.64:0/3849040070 conn(0x7fa27c0d3ef0 msgr2=0x7fa27c06bfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:46.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 shutdown_connections 2024-03-04T07:31:46.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:46.141+0000 7fa283cb5640 1 -- 172.21.15.64:0/3849040070 wait complete. 2024-03-04T07:31:46.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:46.443 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:31:46.444 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:31:46.458 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":845,"msgr_send_messages":840,"msgr_recv_bytes":349577,"msgr_send_bytes":5245108,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.099270278,"msgr_running_send_time":0.050844403,"msgr_running_recv_time":0.017195370,"msgr_running_fast_dispatch_time":0.026870462,"msgr_send_messages_queue_lat":{"avgcount":840,"sum":1.937744879,"avgtime":0.002306839},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":108128,"msgr_send_encrypted_bytes":7920},"AsyncMessenger::Worker-1":{"msgr_recv_messages":10591,"msgr_send_messages":24396,"msgr_recv_bytes":3809417,"msgr_send_bytes":28013043,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.135524635,"msgr_running_send_time":0.639903089,"msgr_running_recv_time":0.534530685,"msgr_running_fast_dispatch_time":0.034875794,"msgr_send_messages_queue_lat":{"avgcount":24395,"sum":0.862717778,"avgtime":0.000035364},"msgr_handle_ack_lat":{"avgcount":29678,"sum":0.005558815,"avgtime":0.000000187},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":96256},"AsyncMessenger::Worker-2":{"msgr_recv_messages":1621,"msgr_send_messages":1623,"msgr_recv_bytes":657995,"msgr_send_bytes":1632715,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.148018542,"msgr_running_send_time":0.074066118,"msgr_running_recv_time":0.026687203,"msgr_running_fast_dispatch_time":0.038042007,"msgr_send_messages_queue_lat":{"avgcount":1621,"sum":1.322785151,"avgtime":0.000816030},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":126,"sum":7.063435129,"avgtime":0.056059008}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":2995,"sum":1.326862221,"avgtime":0.000443025}},"mds":{"request":6413,"reply":6411,"reply_latency":{"avgcount":6411,"sum":9.655326614,"avgtime":0.001506056},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":12,"dir_split":0,"dir_merge":0,"inodes":4586,"inodes_top":0,"inodes_bottom":4586,"inodes_pin_tail":0,"inodes_pinned":4586,"inodes_expired":0,"inodes_with_caps":694,"caps":694,"subtrees":2,"traverse":6419,"traverse_hit":6418,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":0,"root_rbytes":0,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":621,"ceph_cap_op_grant":7014,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3206,"handle_client_caps_dirty":3000,"handle_client_cap_release":8,"process_request_cap_release":9603},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":1375,"num_strays_delayed":1375,"num_strays_enqueuing":0,"strays_created":3201,"strays_enqueued":1826,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":11450,"evex":11450,"evtrm":11450,"ev":11452,"evexg":0,"evexd":11449,"segadd":15,"segex":16,"segtrm":16,"seg":1,"segmjr":1,"segexg":0,"segexd":15,"expos":28013210,"wrpos":28017404,"rdpos":4195244,"jlat":{"avgcount":23,"sum":1.498343863,"avgtime":0.065145385},"replayed":3},"mds_mem":{"ino":1388,"ino+":3214,"ino-":1826,"dir":13,"dir+":13,"dir-":0,"dn":1386,"dn+":6412,"dn-":5026,"cap":694,"cap+":3202,"cap-":2508,"rss":105820,"heap":248164},"mds_server":{"dispatch_client_request":6419,"dispatch_server_request":0,"handle_client_request":6413,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":1,"sum":0.545126756,"avgtime":0.545126756},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":3200,"sum":5.552604947,"avgtime":0.001735189},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5853,"average_load":5853,"avg_session_uptime":812,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":23895845,"buffer_anon_items":6820,"buffer_meta_bytes":88,"buffer_meta_items":1,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":13551660,"mds_co_items":136018,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":3380,"op_send_bytes":25370633,"op_resend":0,"op_reply":3380,"op_latency":{"avgcount":3380,"sum":122.070672134,"avgtime":0.036115583},"op_inflight":0,"oplen_avg":{"avgcount":3380,"sum":6245},"op":3380,"op_r":56,"op_w":3324,"op_rmw":0,"op_pg":0,"osdop_stat":12,"osdop_create":1387,"osdop_read":28,"osdop_write":43,"osdop_writefull":15,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":1847,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":2774,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":53,"omap_rd":48,"omap_del":13},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":1826,"pq_executed":1826,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":13053,"get_sum":3639444,"get_or_fail_fail":0,"get_or_fail_success":13053,"take":0,"take_sum":0,"put":13053,"put_sum":3639444,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":3380,"take_sum":25556176,"put":3380,"put_sum":25556176,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":3380,"take_sum":3380,"put":3380,"put_sum":3380,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":1826,"get_sum":184426,"get_or_fail_fail":0,"get_or_fail_success":1826,"take":0,"take_sum":0,"put":20,"put_sum":184426,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":0,"max":3758096384,"get_started":0,"get":11450,"get_sum":23821719,"get_or_fail_fail":0,"get_or_fail_success":11450,"take":0,"take_sum":0,"put":23,"put_sum":23821719,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:46.460 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 349577, "msgr_recv_encrypted_bytes": 108128, "msgr_recv_messages": 845, "msgr_running_fast_dispatch_time": 0.026870462, "msgr_running_recv_time": 0.01719537, "msgr_running_send_time": 0.050844403, "msgr_running_total_time": 0.099270278, "msgr_send_bytes": 5245108, "msgr_send_encrypted_bytes": 7920, "msgr_send_messages": 840, "msgr_send_messages_queue_lat": { "avgcount": 840, "avgtime": 0.002306839, "sum": 1.937744879 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 29678, "avgtime": 1.87e-07, "sum": 0.005558815 }, "msgr_recv_bytes": 3809417, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 10591, "msgr_running_fast_dispatch_time": 0.034875794, "msgr_running_recv_time": 0.534530685, "msgr_running_send_time": 0.639903089, "msgr_running_total_time": 1.135524635, "msgr_send_bytes": 28013043, "msgr_send_encrypted_bytes": 96256, "msgr_send_messages": 24396, "msgr_send_messages_queue_lat": { "avgcount": 24395, "avgtime": 3.5364e-05, "sum": 0.862717778 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 657995, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 1621, "msgr_running_fast_dispatch_time": 0.038042007, "msgr_running_recv_time": 0.026687203, "msgr_running_send_time": 0.074066118, "msgr_running_total_time": 0.148018542, "msgr_send_bytes": 1632715, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 1623, "msgr_send_messages_queue_lat": { "avgcount": 1621, "avgtime": 0.00081603, "sum": 1.322785151 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 126, "avgtime": 0.056059008, "sum": 7.063435129 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 2995, "avgtime": 0.000443025, "sum": 1.326862221 }, "queue_len": 0 }, "mds": { "caps": 694, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 7014, "ceph_cap_op_revoke": 621, "ceph_cap_op_trunc": 0, "dir_commit": 12, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 8, "handle_client_caps": 3206, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 4586, "inodes_bottom": 4586, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 4586, "inodes_top": 0, "inodes_with_caps": 694, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 9603, "q": 0, "reply": 6411, "reply_latency": { "avgcount": 6411, "avgtime": 0.001506056, "sum": 9.655326614 }, "request": 6413, "root_rbytes": 0, "root_rfiles": 0, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6419, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6418, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 1375, "num_strays_delayed": 1375, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3201, "strays_enqueued": 1826, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 11452, "evadd": 11450, "evex": 11450, "evexd": 11449, "evexg": 0, "evlrg": 0, "evtrm": 11450, "expos": 28013210, "jlat": { "avgcount": 23, "avgtime": 0.065145385, "sum": 1.498343863 }, "rdpos": 4195244, "replayed": 3, "seg": 1, "segadd": 15, "segex": 16, "segexd": 15, "segexg": 0, "segmjr": 1, "segtrm": 16, "wrpos": 28017404 }, "mds_mem": { "cap": 694, "cap+": 3202, "cap-": 2508, "dir": 13, "dir+": 13, "dir-": 0, "dn": 1386, "dn+": 6412, "dn-": 5026, "heap": 248164, "ino": 1388, "ino+": 3214, "ino-": 1826, "rss": 105820 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6419, "dispatch_server_request": 0, "handle_client_request": 6413, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 1, "avgtime": 0.545126756, "sum": 0.545126756 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 3200, "avgtime": 0.001735189, "sum": 5.552604947 } }, "mds_sessions": { "average_load": 5853, "avg_session_uptime": 812, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5853 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 23895845, "buffer_anon_items": 6820, "buffer_meta_bytes": 88, "buffer_meta_items": 1, "mds_co_bytes": 13551660, "mds_co_items": 136018, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 13, "omap_rd": 48, "omap_wr": 53, "op": 3380, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 3380, "avgtime": 0.036115583, "sum": 122.070672134 }, "op_pg": 0, "op_r": 56, "op_reply": 3380, "op_resend": 0, "op_rmw": 0, "op_send": 3380, "op_send_bytes": 25370633, "op_w": 3324, "oplen_avg": { "avgcount": 3380, "sum": 6245 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 1387, "osdop_delete": 1847, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 28, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 2774, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 12, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 43, "osdop_writefull": 15, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 1826, "pq_executed_ops": 1826, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 13053, "get_or_fail_fail": 0, "get_or_fail_success": 13053, "get_started": 0, "get_sum": 3639444, "max": 104857600, "put": 13053, "put_sum": 3639444, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 3380, "put_sum": 25556176, "take": 3380, "take_sum": 25556176, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 3380, "put_sum": 3380, "take": 3380, "take_sum": 3380, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 1826, "get_or_fail_fail": 0, "get_or_fail_success": 1826, "get_started": 0, "get_sum": 184426, "max": 3758096384, "put": 20, "put_sum": 184426, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 11450, "get_or_fail_fail": 0, "get_or_fail_success": 11450, "get_started": 0, "get_sum": 23821719, "max": 3758096384, "put": 23, "put_sum": 23821719, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:46.460 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/1375, 1826/3201 2024-03-04T07:31:47.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:47.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.621+0000 7fc92f2b8640 1 Processor -- start 2024-03-04T07:31:47.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.622+0000 7fc92f2b8640 1 -- start start 2024-03-04T07:31:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.623+0000 7fc92f2b8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc92815ad70 0x7fc9280b2aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.623+0000 7fc92f2b8640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc92815b150 con 0x7fc928071ab0 2024-03-04T07:31:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.623+0000 7fc92f2b8640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9280b3080 con 0x7fc92815f2d0 2024-03-04T07:31:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.623+0000 7fc92f2b8640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9280b3360 con 0x7fc92815ad70 2024-03-04T07:31:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.623+0000 7fc92d855640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc928071ab0 0x7fc9280b4120 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58228/0 (socket says 172.21.15.64:58228) 2024-03-04T07:31:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.623+0000 7fc92d855640 1 -- 172.21.15.64:0/226891009 learned_addr learned my addr 172.21.15.64:0/226891009 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:47.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.623+0000 7fc92d054640 1 --2- 172.21.15.64:0/226891009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc92815ad70 0x7fc9280b2aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.624+0000 7fc91ffff640 1 -- 172.21.15.64:0/226891009 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4257469959 0 0) 0x7fc92815b150 con 0x7fc928071ab0 2024-03-04T07:31:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.624+0000 7fc91ffff640 1 -- 172.21.15.64:0/226891009 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc914003520 con 0x7fc928071ab0 2024-03-04T07:31:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.624+0000 7fc91ffff640 1 -- 172.21.15.64:0/226891009 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3686894649 0 0) 0x7fc9280b3080 con 0x7fc92815f2d0 2024-03-04T07:31:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.624+0000 7fc91ffff640 1 -- 172.21.15.64:0/226891009 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc92815b150 con 0x7fc92815f2d0 2024-03-04T07:31:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.624+0000 7fc92d054640 1 -- 172.21.15.64:0/226891009 >> v1:172.21.15.66:6789/0 conn(0x7fc92815f2d0 legacy=0x7fc9280cd070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.624+0000 7fc92d054640 1 -- 172.21.15.64:0/226891009 >> v1:172.21.15.64:6789/0 conn(0x7fc928071ab0 legacy=0x7fc9280b4120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.624+0000 7fc92d054640 1 -- 172.21.15.64:0/226891009 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9280b3640 con 0x7fc92815ad70 2024-03-04T07:31:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.626+0000 7fc92d054640 1 --2- 172.21.15.64:0/226891009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc92815ad70 0x7fc9280b2aa0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fc9240097d0 tx=0x7fc92402b8a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=641d32da6177eff3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.626+0000 7fc91ffff640 1 -- 172.21.15.64:0/226891009 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc924035020 con 0x7fc92815ad70 2024-03-04T07:31:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.626+0000 7fc91ffff640 1 -- 172.21.15.64:0/226891009 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc924031800 con 0x7fc92815ad70 2024-03-04T07:31:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.626+0000 7fc91ffff640 1 -- 172.21.15.64:0/226891009 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc924031bd0 con 0x7fc92815ad70 2024-03-04T07:31:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.627+0000 7fc92f2b8640 1 -- 172.21.15.64:0/226891009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc92815ad70 msgr2=0x7fc9280b2aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.627+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/226891009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc92815ad70 0x7fc9280b2aa0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fc9240097d0 tx=0x7fc92402b8a0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.627+0000 7fc92f2b8640 1 -- 172.21.15.64:0/226891009 shutdown_connections 2024-03-04T07:31:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.627+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/226891009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc92815ad70 0x7fc9280b2aa0 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.627+0000 7fc92f2b8640 1 -- 172.21.15.64:0/226891009 >> 172.21.15.64:0/226891009 conn(0x7fc9280b17a0 msgr2=0x7fc9280b1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.627+0000 7fc92f2b8640 1 -- 172.21.15.64:0/226891009 shutdown_connections 2024-03-04T07:31:47.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.627+0000 7fc92f2b8640 1 -- 172.21.15.64:0/226891009 wait complete. 2024-03-04T07:31:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.631+0000 7fc92f2b8640 1 Processor -- start 2024-03-04T07:31:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.631+0000 7fc92f2b8640 1 -- start start 2024-03-04T07:31:47.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92f2b8640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc928071ab0 0x7fc928153af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:47.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92f2b8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc92815f2d0 0x7fc928150810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:47.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92f2b8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9280cbf80 0x7fc92815c7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:47.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92f2b8640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc92815f730 con 0x7fc92815f2d0 2024-03-04T07:31:47.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92f2b8640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9280b3c00 con 0x7fc928071ab0 2024-03-04T07:31:47.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92f2b8640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc928071f40 con 0x7fc9280cbf80 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92d855640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9280cbf80 0x7fc92815c7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92c853640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc92815f2d0 0x7fc928150810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92c853640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc92815f2d0 0x7fc928150810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37680/0 (socket says 172.21.15.64:37680) 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.632+0000 7fc92c853640 1 -- 172.21.15.64:0/960763873 learned_addr learned my addr 172.21.15.64:0/960763873 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc92d054640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc928071ab0 0x7fc928153af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc92c853640 1 -- 172.21.15.64:0/960763873 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9280cbf80 msgr2=0x7fc92815c7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc92c853640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9280cbf80 0x7fc92815c7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc92c853640 1 -- 172.21.15.64:0/960763873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc928071ab0 msgr2=0x7fc928153af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc92c853640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc928071ab0 0x7fc928153af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc92c853640 1 -- 172.21.15.64:0/960763873 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc928150d50 con 0x7fc92815f2d0 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc92c853640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc92815f2d0 0x7fc928150810 secure :-1 s=READY pgs=1193 cs=0 l=1 rev1=1 crypto rx=0x7fc91800c460 tx=0x7fc91800c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc91dffb640 1 -- 172.21.15.64:0/960763873 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc918017070 con 0x7fc92815f2d0 2024-03-04T07:31:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.633+0000 7fc91dffb640 1 -- 172.21.15.64:0/960763873 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc9180035f0 con 0x7fc92815f2d0 2024-03-04T07:31:47.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.634+0000 7fc91dffb640 1 -- 172.21.15.64:0/960763873 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc918003a20 con 0x7fc92815f2d0 2024-03-04T07:31:47.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.634+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc9281540f0 con 0x7fc92815f2d0 2024-03-04T07:31:47.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.635+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc9281546b0 con 0x7fc92815f2d0 2024-03-04T07:31:47.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.637+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc9280752d0 con 0x7fc92815f2d0 2024-03-04T07:31:47.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.638+0000 7fc91dffb640 1 -- 172.21.15.64:0/960763873 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc91801a070 con 0x7fc92815f2d0 2024-03-04T07:31:47.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.638+0000 7fc91dffb640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc90c06b500 0x7fc90c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:47.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.638+0000 7fc91dffb640 1 -- 172.21.15.64:0/960763873 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fc918089660 con 0x7fc92815f2d0 2024-03-04T07:31:47.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.638+0000 7fc92d054640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc90c06b500 0x7fc90c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:47.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.639+0000 7fc92d054640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc90c06b500 0x7fc90c06d9c0 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7fc924036000 tx=0x7fc924008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:47.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.645+0000 7fc91dffb640 1 -- 172.21.15.64:0/960763873 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc928150d50 con 0x7fc92815f2d0 2024-03-04T07:31:47.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:31:47.855 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:31:47.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.920+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc92806c9d0 con 0x7fc92815f2d0 2024-03-04T07:31:47.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.920+0000 7fc91dffb640 1 -- 172.21.15.64:0/960763873 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fc918058390 con 0x7fc92815f2d0 2024-03-04T07:31:47.920 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:47.920 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:47.920 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:47.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc90c06b500 msgr2=0x7fc90c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:47.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc90c06b500 0x7fc90c06d9c0 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7fc924036000 tx=0x7fc924008030 comp rx=0 tx=0).stop 2024-03-04T07:31:47.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc92815f2d0 msgr2=0x7fc928150810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:47.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc92815f2d0 0x7fc928150810 secure :-1 s=READY pgs=1193 cs=0 l=1 rev1=1 crypto rx=0x7fc91800c460 tx=0x7fc91800c920 comp rx=0 tx=0).stop 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 shutdown_connections 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc90c06b500 0x7fc90c06d9c0 unknown :-1 s=CLOSED pgs=542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9280cbf80 0x7fc92815c7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc92815f2d0 0x7fc928150810 unknown :-1 s=CLOSED pgs=1193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 --2- 172.21.15.64:0/960763873 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc928071ab0 0x7fc928153af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 >> 172.21.15.64:0/960763873 conn(0x7fc9280b17a0 msgr2=0x7fc9280af710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 shutdown_connections 2024-03-04T07:31:47.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:47.923+0000 7fc92f2b8640 1 -- 172.21.15.64:0/960763873 wait complete. 2024-03-04T07:31:47.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:48.105 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":1193,"msgr_send_messages":1188,"msgr_recv_bytes":437987,"msgr_send_bytes":5358854,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.126051565,"msgr_running_send_time":0.061205626,"msgr_running_recv_time":0.023051869,"msgr_running_fast_dispatch_time":0.034334610,"msgr_send_messages_queue_lat":{"avgcount":1188,"sum":1.951416941,"avgtime":0.001642606},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":108400,"msgr_send_encrypted_bytes":8544},"AsyncMessenger::Worker-1":{"msgr_recv_messages":10970,"msgr_send_messages":24773,"msgr_recv_bytes":3922299,"msgr_send_bytes":28135945,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.164280468,"msgr_running_send_time":0.651471909,"msgr_running_recv_time":0.541057317,"msgr_running_fast_dispatch_time":0.042578032,"msgr_send_messages_queue_lat":{"avgcount":24772,"sum":0.874875464,"avgtime":0.000035317},"msgr_handle_ack_lat":{"avgcount":29680,"sum":0.005558999,"avgtime":0.000000187},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":96256},"AsyncMessenger::Worker-2":{"msgr_recv_messages":2278,"msgr_send_messages":2280,"msgr_recv_bytes":963591,"msgr_send_bytes":1985705,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.198731295,"msgr_running_send_time":0.095124076,"msgr_running_recv_time":0.038007698,"msgr_running_fast_dispatch_time":0.053158136,"msgr_send_messages_queue_lat":{"avgcount":2278,"sum":1.352004547,"avgtime":0.000593505},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":126,"sum":7.063435129,"avgtime":0.056059008}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":5376,"sum":1.461218325,"avgtime":0.000271804}},"mds":{"request":6413,"reply":6411,"reply_latency":{"avgcount":6411,"sum":9.655326614,"avgtime":0.001506056},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":12,"dir_split":0,"dir_merge":0,"inodes":4586,"inodes_top":0,"inodes_bottom":4586,"inodes_pin_tail":0,"inodes_pinned":4586,"inodes_expired":0,"inodes_with_caps":694,"caps":694,"subtrees":2,"traverse":6419,"traverse_hit":6418,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":0,"root_rbytes":0,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":621,"ceph_cap_op_grant":7014,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3206,"handle_client_caps_dirty":3000,"handle_client_cap_release":9,"process_request_cap_release":9603},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":1,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":3201,"strays_enqueued":3200,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":12825,"evex":11450,"evtrm":11450,"ev":1378,"evexg":0,"evexd":11449,"segadd":16,"segex":16,"segtrm":16,"seg":2,"segmjr":1,"segexg":0,"segexd":15,"expos":28013210,"wrpos":28842176,"rdpos":4195244,"jlat":{"avgcount":23,"sum":1.498343863,"avgtime":0.065145385},"replayed":3},"mds_mem":{"ino":1388,"ino+":3214,"ino-":1826,"dir":13,"dir+":13,"dir-":0,"dn":1386,"dn+":6412,"dn-":5026,"cap":2,"cap+":3202,"cap-":3200,"rss":111708,"heap":248164},"mds_server":{"dispatch_client_request":6419,"dispatch_server_request":0,"handle_client_request":6413,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":1,"sum":0.545126756,"avgtime":0.545126756},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":3200,"sum":5.552604947,"avgtime":0.001735189},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5853,"average_load":5853,"avg_session_uptime":812,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":23895845,"buffer_anon_items":6820,"buffer_meta_bytes":88,"buffer_meta_items":1,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":13551660,"mds_co_items":136018,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":4761,"op_send_bytes":25509497,"op_resend":0,"op_reply":4761,"op_latency":{"avgcount":4761,"sum":124.269630198,"avgtime":0.026101581},"op_inflight":0,"oplen_avg":{"avgcount":4761,"sum":7626},"op":4761,"op_r":59,"op_w":4702,"op_rmw":0,"op_pg":0,"osdop_stat":12,"osdop_create":1387,"osdop_read":31,"osdop_write":46,"osdop_writefull":16,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":3221,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":2774,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":53,"omap_rd":48,"omap_del":13},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":3200,"pq_executed":3200,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":14437,"get_sum":4021720,"get_or_fail_fail":0,"get_or_fail_success":14437,"take":0,"take_sum":0,"put":14437,"put_sum":4021720,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":4761,"take_sum":25833814,"put":4761,"put_sum":25833814,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":4761,"take_sum":4761,"put":4761,"put_sum":4761,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":3200,"get_sum":323200,"get_or_fail_fail":0,"get_or_fail_success":3200,"take":0,"take_sum":0,"put":23,"put_sum":323200,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":824772,"max":3758096384,"get_started":0,"get":12825,"get_sum":24646491,"get_or_fail_fail":0,"get_or_fail_success":12825,"take":0,"take_sum":0,"put":23,"put_sum":23821719,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:48.107 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 437987, "msgr_recv_encrypted_bytes": 108400, "msgr_recv_messages": 1193, "msgr_running_fast_dispatch_time": 0.03433461, "msgr_running_recv_time": 0.023051869, "msgr_running_send_time": 0.061205626, "msgr_running_total_time": 0.126051565, "msgr_send_bytes": 5358854, "msgr_send_encrypted_bytes": 8544, "msgr_send_messages": 1188, "msgr_send_messages_queue_lat": { "avgcount": 1188, "avgtime": 0.001642606, "sum": 1.951416941 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 29680, "avgtime": 1.87e-07, "sum": 0.005558999 }, "msgr_recv_bytes": 3922299, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 10970, "msgr_running_fast_dispatch_time": 0.042578032, "msgr_running_recv_time": 0.541057317, "msgr_running_send_time": 0.651471909, "msgr_running_total_time": 1.164280468, "msgr_send_bytes": 28135945, "msgr_send_encrypted_bytes": 96256, "msgr_send_messages": 24773, "msgr_send_messages_queue_lat": { "avgcount": 24772, "avgtime": 3.5317e-05, "sum": 0.874875464 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 963591, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 2278, "msgr_running_fast_dispatch_time": 0.053158136, "msgr_running_recv_time": 0.038007698, "msgr_running_send_time": 0.095124076, "msgr_running_total_time": 0.198731295, "msgr_send_bytes": 1985705, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 2280, "msgr_send_messages_queue_lat": { "avgcount": 2278, "avgtime": 0.000593505, "sum": 1.352004547 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 126, "avgtime": 0.056059008, "sum": 7.063435129 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 5376, "avgtime": 0.000271804, "sum": 1.461218325 }, "queue_len": 0 }, "mds": { "caps": 694, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 7014, "ceph_cap_op_revoke": 621, "ceph_cap_op_trunc": 0, "dir_commit": 12, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 9, "handle_client_caps": 3206, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 4586, "inodes_bottom": 4586, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 4586, "inodes_top": 0, "inodes_with_caps": 694, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 9603, "q": 0, "reply": 6411, "reply_latency": { "avgcount": 6411, "avgtime": 0.001506056, "sum": 9.655326614 }, "request": 6413, "root_rbytes": 0, "root_rfiles": 0, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6419, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6418, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 1, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3201, "strays_enqueued": 3200, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 1378, "evadd": 12825, "evex": 11450, "evexd": 11449, "evexg": 0, "evlrg": 0, "evtrm": 11450, "expos": 28013210, "jlat": { "avgcount": 23, "avgtime": 0.065145385, "sum": 1.498343863 }, "rdpos": 4195244, "replayed": 3, "seg": 2, "segadd": 16, "segex": 16, "segexd": 15, "segexg": 0, "segmjr": 1, "segtrm": 16, "wrpos": 28842176 }, "mds_mem": { "cap": 2, "cap+": 3202, "cap-": 3200, "dir": 13, "dir+": 13, "dir-": 0, "dn": 1386, "dn+": 6412, "dn-": 5026, "heap": 248164, "ino": 1388, "ino+": 3214, "ino-": 1826, "rss": 111708 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6419, "dispatch_server_request": 0, "handle_client_request": 6413, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 1, "avgtime": 0.545126756, "sum": 0.545126756 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 3200, "avgtime": 0.001735189, "sum": 5.552604947 } }, "mds_sessions": { "average_load": 5853, "avg_session_uptime": 812, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5853 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 23895845, "buffer_anon_items": 6820, "buffer_meta_bytes": 88, "buffer_meta_items": 1, "mds_co_bytes": 13551660, "mds_co_items": 136018, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 13, "omap_rd": 48, "omap_wr": 53, "op": 4761, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 4761, "avgtime": 0.026101581, "sum": 124.269630198 }, "op_pg": 0, "op_r": 59, "op_reply": 4761, "op_resend": 0, "op_rmw": 0, "op_send": 4761, "op_send_bytes": 25509497, "op_w": 4702, "oplen_avg": { "avgcount": 4761, "sum": 7626 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 1387, "osdop_delete": 3221, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 31, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 2774, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 12, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 46, "osdop_writefull": 16, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 3200, "pq_executed_ops": 3200, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 14437, "get_or_fail_fail": 0, "get_or_fail_success": 14437, "get_started": 0, "get_sum": 4021720, "max": 104857600, "put": 14437, "put_sum": 4021720, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 4761, "put_sum": 25833814, "take": 4761, "take_sum": 25833814, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 4761, "put_sum": 4761, "take": 4761, "take_sum": 4761, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 3200, "get_or_fail_fail": 0, "get_or_fail_success": 3200, "get_started": 0, "get_sum": 323200, "max": 3758096384, "put": 23, "put_sum": 323200, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 12825, "get_or_fail_fail": 0, "get_or_fail_success": 12825, "get_started": 0, "get_sum": 24646491, "max": 3758096384, "put": 23, "put_sum": 23821719, "take": 0, "take_sum": 0, "val": 824772, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:48.107 INFO:tasks.cephfs.test_strays:Waiting for purge to complete 0/1, 3200/3201 2024-03-04T07:31:49.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:49.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.211+0000 7fbd636b5640 1 Processor -- start 2024-03-04T07:31:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.211+0000 7fbd636b5640 1 -- start start 2024-03-04T07:31:49.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd636b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c0d4c40 0x7fbd5c0d1cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd636b5640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbd5c0d0100 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd636b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbd5c0cf860 con 0x7fbd5c15e100 2024-03-04T07:31:49.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd636b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbd5c0cfb40 con 0x7fbd5c0d4c40 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd61c52640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbd5c0d2a30 0x7fbd5c0b1070 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58240/0 (socket says 172.21.15.64:58240) 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd61c52640 1 -- 172.21.15.64:0/1896551197 learned_addr learned my addr 172.21.15.64:0/1896551197 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd60c50640 1 --2- 172.21.15.64:0/1896551197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c0d4c40 0x7fbd5c0d1cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 301238066 0 0) 0x7fbd5c0d0100 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbd48003520 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2737220641 0 0) 0x7fbd5c0cf860 con 0x7fbd5c15e100 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbd5c0d0100 con 0x7fbd5c15e100 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2420958123 0 0) 0x7fbd48003520 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd60c50640 1 -- 172.21.15.64:0/1896551197 >> v1:172.21.15.66:6789/0 conn(0x7fbd5c15e100 legacy=0x7fbd5c0d2320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd60c50640 1 -- 172.21.15.64:0/1896551197 >> v1:172.21.15.64:6789/0 conn(0x7fbd5c0d2a30 legacy=0x7fbd5c0b1070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.212+0000 7fbd60c50640 1 -- 172.21.15.64:0/1896551197 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd5c0cfe20 con 0x7fbd5c0d4c40 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd60c50640 1 --2- 172.21.15.64:0/1896551197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c0d4c40 0x7fbd5c0d1cd0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fbd4c0097d0 tx=0x7fbd4c02b800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2eded1769198e321 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd4c037070 con 0x7fbd5c0d4c40 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbd4c030510 con 0x7fbd5c0d4c40 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd53fff640 1 -- 172.21.15.64:0/1896551197 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd540019f0 con 0x7fbd5c0d4c40 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd636b5640 1 -- 172.21.15.64:0/1896551197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c0d4c40 msgr2=0x7fbd5c0d1cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1896551197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c0d4c40 0x7fbd5c0d1cd0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fbd4c0097d0 tx=0x7fbd4c02b800 comp rx=0 tx=0).stop 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd636b5640 1 -- 172.21.15.64:0/1896551197 shutdown_connections 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1896551197 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c0d4c40 0x7fbd5c0d1cd0 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd636b5640 1 -- 172.21.15.64:0/1896551197 >> 172.21.15.64:0/1896551197 conn(0x7fbd5c0d3ef0 msgr2=0x7fbd5c0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd636b5640 1 -- 172.21.15.64:0/1896551197 shutdown_connections 2024-03-04T07:31:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.214+0000 7fbd636b5640 1 -- 172.21.15.64:0/1896551197 wait complete. 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.215+0000 7fbd636b5640 1 Processor -- start 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd636b5640 1 -- start start 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd636b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd5c0d2a30 0x7fbd5c0b8cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd636b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd5c0d4c40 0x7fbd5c0b6770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd636b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c15e100 0x7fbd5c15c7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd636b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd5c15e4e0 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd636b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd5c0cf510 con 0x7fbd5c0d4c40 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd636b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbd5c15e660 con 0x7fbd5c15e100 2024-03-04T07:31:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd61451640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd5c0d2a30 0x7fbd5c0b8cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd61451640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd5c0d2a30 0x7fbd5c0b8cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37694/0 (socket says 172.21.15.64:37694) 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd61451640 1 -- 172.21.15.64:0/1626397365 learned_addr learned my addr 172.21.15.64:0/1626397365 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd60c50640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd5c0d4c40 0x7fbd5c0b6770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.216+0000 7fbd61c52640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c15e100 0x7fbd5c15c7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd61451640 1 -- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c15e100 msgr2=0x7fbd5c15c7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd61451640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c15e100 0x7fbd5c15c7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd61451640 1 -- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd5c0d4c40 msgr2=0x7fbd5c0b6770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd61451640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd5c0d4c40 0x7fbd5c0b6770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd61451640 1 -- 172.21.15.64:0/1626397365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd5c0b6cb0 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd60c50640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd5c0d4c40 0x7fbd5c0b6770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd61451640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd5c0d2a30 0x7fbd5c0b8cc0 secure :-1 s=READY pgs=1195 cs=0 l=1 rev1=1 crypto rx=0x7fbd5800c4c0 tx=0x7fbd5800c980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd51ffb640 1 -- 172.21.15.64:0/1626397365 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd580052d0 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd51ffb640 1 -- 172.21.15.64:0/1626397365 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbd58005470 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd51ffb640 1 -- 172.21.15.64:0/1626397365 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd58012870 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbd5c0b6f50 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.217+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd5c0b7510 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.218+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbd5c0752d0 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.218+0000 7fbd51ffb640 1 -- 172.21.15.64:0/1626397365 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbd58012a10 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.219+0000 7fbd51ffb640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd4006b4d0 0x7fbd4006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.219+0000 7fbd60c50640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd4006b4d0 0x7fbd4006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.219+0000 7fbd51ffb640 1 -- 172.21.15.64:0/1626397365 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fbd58089f60 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.219+0000 7fbd60c50640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd4006b4d0 0x7fbd4006d990 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7fbd4c034000 tx=0x7fbd4c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:49.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.224+0000 7fbd51ffb640 1 -- 172.21.15.64:0/1626397365 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbd58058cb0 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.443+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbd5c0d10b0 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.444+0000 7fbd51ffb640 1 -- 172.21.15.64:0/1626397365 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fbd580585b0 con 0x7fbd5c0d2a30 2024-03-04T07:31:49.444 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:49.444 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:49.445 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd4006b4d0 msgr2=0x7fbd4006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd4006b4d0 0x7fbd4006d990 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7fbd4c034000 tx=0x7fbd4c008030 comp rx=0 tx=0).stop 2024-03-04T07:31:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd5c0d2a30 msgr2=0x7fbd5c0b8cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd5c0d2a30 0x7fbd5c0b8cc0 secure :-1 s=READY pgs=1195 cs=0 l=1 rev1=1 crypto rx=0x7fbd5800c4c0 tx=0x7fbd5800c980 comp rx=0 tx=0).stop 2024-03-04T07:31:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 shutdown_connections 2024-03-04T07:31:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd4006b4d0 0x7fbd4006d990 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd5c15e100 0x7fbd5c15c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd5c0d4c40 0x7fbd5c0b6770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 --2- 172.21.15.64:0/1626397365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd5c0d2a30 0x7fbd5c0b8cc0 unknown :-1 s=CLOSED pgs=1195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 >> 172.21.15.64:0/1626397365 conn(0x7fbd5c0d3ef0 msgr2=0x7fbd5c15d4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 shutdown_connections 2024-03-04T07:31:49.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.446+0000 7fbd636b5640 1 -- 172.21.15.64:0/1626397365 wait complete. 2024-03-04T07:31:49.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:49.628 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":1194,"msgr_send_messages":1189,"msgr_recv_bytes":438241,"msgr_send_bytes":5359180,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.126107882,"msgr_running_send_time":0.061228038,"msgr_running_recv_time":0.023064417,"msgr_running_fast_dispatch_time":0.034350967,"msgr_send_messages_queue_lat":{"avgcount":1189,"sum":1.951432970,"avgtime":0.001641238},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":108400,"msgr_send_encrypted_bytes":8544},"AsyncMessenger::Worker-1":{"msgr_recv_messages":10975,"msgr_send_messages":24776,"msgr_recv_bytes":3923841,"msgr_send_bytes":28138585,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.164692329,"msgr_running_send_time":0.651652810,"msgr_running_recv_time":0.541173508,"msgr_running_fast_dispatch_time":0.042694236,"msgr_send_messages_queue_lat":{"avgcount":24775,"sum":0.874958981,"avgtime":0.000035316},"msgr_handle_ack_lat":{"avgcount":29683,"sum":0.005559292,"avgtime":0.000000187},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":98048},"AsyncMessenger::Worker-2":{"msgr_recv_messages":2282,"msgr_send_messages":2284,"msgr_recv_bytes":964676,"msgr_send_bytes":2812539,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.201774323,"msgr_running_send_time":0.097403715,"msgr_running_recv_time":0.038060939,"msgr_running_fast_dispatch_time":0.053850941,"msgr_send_messages_queue_lat":{"avgcount":2282,"sum":1.352343849,"avgtime":0.000592613},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":130,"sum":7.116505977,"avgtime":0.054742353}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":5383,"sum":1.461372435,"avgtime":0.000271479}},"mds":{"request":6413,"reply":6411,"reply_latency":{"avgcount":6411,"sum":9.655326614,"avgtime":0.001506056},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":12,"dir_split":0,"dir_merge":0,"inodes":1386,"inodes_top":0,"inodes_bottom":1386,"inodes_pin_tail":0,"inodes_pinned":1386,"inodes_expired":0,"inodes_with_caps":1,"caps":1,"subtrees":2,"traverse":6419,"traverse_hit":6418,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":0,"root_rbytes":0,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":621,"ceph_cap_op_grant":7014,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3206,"handle_client_caps_dirty":3000,"handle_client_cap_release":10,"process_request_cap_release":9603},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":0,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":3201,"strays_enqueued":3201,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":12826,"evex":11450,"evtrm":11450,"ev":1379,"evexg":0,"evexd":11449,"segadd":16,"segex":16,"segtrm":16,"seg":2,"segmjr":1,"segexg":0,"segexd":15,"expos":28013210,"wrpos":28842775,"rdpos":4195244,"jlat":{"avgcount":24,"sum":1.505652818,"avgtime":0.062735534},"replayed":3},"mds_mem":{"ino":13,"ino+":3214,"ino-":3201,"dir":12,"dir+":13,"dir-":1,"dn":1386,"dn+":6412,"dn-":5026,"cap":1,"cap+":3202,"cap-":3201,"rss":113244,"heap":248164},"mds_server":{"dispatch_client_request":6419,"dispatch_server_request":0,"handle_client_request":6413,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":1,"sum":0.545126756,"avgtime":0.545126756},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":3200,"sum":5.552604947,"avgtime":0.001735189},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5853,"average_load":5853,"avg_session_uptime":817,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":11319594,"buffer_anon_items":2850,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":4407832,"mds_co_items":73121,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":4768,"op_send_bytes":26335187,"op_resend":0,"op_reply":4768,"op_latency":{"avgcount":4768,"sum":124.282629950,"avgtime":0.026065987},"op_inflight":0,"oplen_avg":{"avgcount":4768,"sum":7635},"op":4768,"op_r":60,"op_w":4708,"op_rmw":0,"op_pg":0,"osdop_stat":12,"osdop_create":1387,"osdop_read":32,"osdop_write":48,"osdop_writefull":17,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":3222,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":2774,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":55,"omap_rd":48,"omap_del":15},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":3201,"pq_executed":3201,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":14447,"get_sum":4023701,"get_or_fail_fail":0,"get_or_fail_success":14447,"take":0,"take_sum":0,"put":14447,"put_sum":4023701,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":4768,"take_sum":26659605,"put":4768,"put_sum":26659605,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":4768,"take_sum":4768,"put":4768,"put_sum":4768,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":3201,"get_sum":323301,"get_or_fail_fail":0,"get_or_fail_success":3201,"take":0,"take_sum":0,"put":24,"put_sum":323301,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":0,"max":3758096384,"get_started":0,"get":12826,"get_sum":24647090,"get_or_fail_fail":0,"get_or_fail_success":12826,"take":0,"take_sum":0,"put":24,"put_sum":24647090,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:49.630 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 438241, "msgr_recv_encrypted_bytes": 108400, "msgr_recv_messages": 1194, "msgr_running_fast_dispatch_time": 0.034350967, "msgr_running_recv_time": 0.023064417, "msgr_running_send_time": 0.061228038, "msgr_running_total_time": 0.126107882, "msgr_send_bytes": 5359180, "msgr_send_encrypted_bytes": 8544, "msgr_send_messages": 1189, "msgr_send_messages_queue_lat": { "avgcount": 1189, "avgtime": 0.001641238, "sum": 1.95143297 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 29683, "avgtime": 1.87e-07, "sum": 0.005559292 }, "msgr_recv_bytes": 3923841, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 10975, "msgr_running_fast_dispatch_time": 0.042694236, "msgr_running_recv_time": 0.541173508, "msgr_running_send_time": 0.65165281, "msgr_running_total_time": 1.164692329, "msgr_send_bytes": 28138585, "msgr_send_encrypted_bytes": 98048, "msgr_send_messages": 24776, "msgr_send_messages_queue_lat": { "avgcount": 24775, "avgtime": 3.5316e-05, "sum": 0.874958981 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 964676, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 2282, "msgr_running_fast_dispatch_time": 0.053850941, "msgr_running_recv_time": 0.038060939, "msgr_running_send_time": 0.097403715, "msgr_running_total_time": 0.201774323, "msgr_send_bytes": 2812539, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 2284, "msgr_send_messages_queue_lat": { "avgcount": 2282, "avgtime": 0.000592613, "sum": 1.352343849 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 130, "avgtime": 0.054742353, "sum": 7.116505977 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 5383, "avgtime": 0.000271479, "sum": 1.461372435 }, "queue_len": 0 }, "mds": { "caps": 1, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 7014, "ceph_cap_op_revoke": 621, "ceph_cap_op_trunc": 0, "dir_commit": 12, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 10, "handle_client_caps": 3206, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 1386, "inodes_bottom": 1386, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 1386, "inodes_top": 0, "inodes_with_caps": 1, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 9603, "q": 0, "reply": 6411, "reply_latency": { "avgcount": 6411, "avgtime": 0.001506056, "sum": 9.655326614 }, "request": 6413, "root_rbytes": 0, "root_rfiles": 0, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6419, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6418, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 0, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3201, "strays_enqueued": 3201, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 1379, "evadd": 12826, "evex": 11450, "evexd": 11449, "evexg": 0, "evlrg": 0, "evtrm": 11450, "expos": 28013210, "jlat": { "avgcount": 24, "avgtime": 0.062735534, "sum": 1.505652818 }, "rdpos": 4195244, "replayed": 3, "seg": 2, "segadd": 16, "segex": 16, "segexd": 15, "segexg": 0, "segmjr": 1, "segtrm": 16, "wrpos": 28842775 }, "mds_mem": { "cap": 1, "cap+": 3202, "cap-": 3201, "dir": 12, "dir+": 13, "dir-": 1, "dn": 1386, "dn+": 6412, "dn-": 5026, "heap": 248164, "ino": 13, "ino+": 3214, "ino-": 3201, "rss": 113244 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6419, "dispatch_server_request": 0, "handle_client_request": 6413, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 1, "avgtime": 0.545126756, "sum": 0.545126756 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 3200, "avgtime": 0.001735189, "sum": 5.552604947 } }, "mds_sessions": { "average_load": 5853, "avg_session_uptime": 817, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5853 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 11319594, "buffer_anon_items": 2850, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 4407832, "mds_co_items": 73121, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 15, "omap_rd": 48, "omap_wr": 55, "op": 4768, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 4768, "avgtime": 0.026065987, "sum": 124.28262995 }, "op_pg": 0, "op_r": 60, "op_reply": 4768, "op_resend": 0, "op_rmw": 0, "op_send": 4768, "op_send_bytes": 26335187, "op_w": 4708, "oplen_avg": { "avgcount": 4768, "sum": 7635 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 1387, "osdop_delete": 3222, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 32, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 2774, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 12, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 48, "osdop_writefull": 17, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 3201, "pq_executed_ops": 3201, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 14447, "get_or_fail_fail": 0, "get_or_fail_success": 14447, "get_started": 0, "get_sum": 4023701, "max": 104857600, "put": 14447, "put_sum": 4023701, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 4768, "put_sum": 26659605, "take": 4768, "take_sum": 26659605, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 4768, "put_sum": 4768, "take": 4768, "take_sum": 4768, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 3201, "get_or_fail_fail": 0, "get_or_fail_success": 3201, "get_started": 0, "get_sum": 323301, "max": 3758096384, "put": 24, "put_sum": 323301, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 12826, "get_or_fail_fail": 0, "get_or_fail_success": 12826, "get_started": 0, "get_sum": 24647090, "max": 3758096384, "put": 24, "put_sum": 24647090, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:49.630 INFO:tasks.cephfs.test_strays:Complete purge in 7 seconds 2024-03-04T07:31:49.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:49.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.732+0000 7f4e4b25c640 1 Processor -- start 2024-03-04T07:31:49.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.732+0000 7f4e4b25c640 1 -- start start 2024-03-04T07:31:49.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.732+0000 7f4e4b25c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4e440d2a30 0x7f4e440d1cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.732+0000 7f4e4b25c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4e440d0100 con 0x7f4e440d4c40 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.732+0000 7f4e4b25c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4e440cf860 con 0x7f4e4415e100 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.732+0000 7f4e4b25c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4e440cfb40 con 0x7f4e440d2a30 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e43fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4e440d4c40 0x7f4e440b1070 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58252/0 (socket says 172.21.15.64:58252) 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e43fff640 1 -- 172.21.15.64:0/2587249479 learned_addr learned my addr 172.21.15.64:0/2587249479 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e497f9640 1 --2- 172.21.15.64:0/2587249479 >> v2:172.21.15.66:3300/0 conn(0x7f4e440d2a30 0x7f4e440d1cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3713411120 0 0) 0x7f4e440d0100 con 0x7f4e440d4c40 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4e30003520 con 0x7f4e440d4c40 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3143537744 0 0) 0x7f4e30003520 con 0x7f4e440d4c40 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 >> v2:172.21.15.66:3300/0 conn(0x7f4e440d2a30 msgr2=0x7f4e440d1cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 --2- 172.21.15.64:0/2587249479 >> v2:172.21.15.66:3300/0 conn(0x7f4e440d2a30 0x7f4e440d1cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 >> v1:172.21.15.66:6789/0 conn(0x7f4e4415e100 legacy=0x7f4e440d2320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e440cfe20 con 0x7f4e440d4c40 2024-03-04T07:31:49.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f4e2c001670 con 0x7f4e440d4c40 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e497f9640 1 --2- 172.21.15.64:0/2587249479 >> v2:172.21.15.66:3300/0 conn(0x7f4e440d2a30 0x7f4e440d1cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 >> v1:172.21.15.64:6789/0 conn(0x7f4e440d4c40 legacy=0x7f4e440b1070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e300039b0 0x7f4e30003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300043b0 0x7f4e30004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e30005080 0x7f4e3000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e440cfb40 con 0x7f4e300039b0 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e3000bfe0 con 0x7f4e300043b0 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4e3000c1e0 con 0x7f4e30005080 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.733+0000 7f4e48ff8640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e300039b0 0x7f4e30003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e43fff640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300043b0 0x7f4e30004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e497f9640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e30005080 0x7f4e3000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e48ff8640 1 -- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e30005080 msgr2=0x7f4e3000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e48ff8640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e30005080 0x7f4e3000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e48ff8640 1 -- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300043b0 msgr2=0x7f4e30004880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:49.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e48ff8640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300043b0 0x7f4e30004880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e48ff8640 1 -- 172.21.15.64:0/2587249479 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e30003520 con 0x7f4e300039b0 2024-03-04T07:31:49.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e497f9640 1 -- 172.21.15.64:0/2587249479 reap_dead start 2024-03-04T07:31:49.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.734+0000 7f4e48ff8640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e300039b0 0x7f4e30003d90 secure :-1 s=READY pgs=1197 cs=0 l=1 rev1=1 crypto rx=0x7f4e3c001f60 tx=0x7f4e30005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=671c019aa2ec382b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:49.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e3c039070 con 0x7f4e300039b0 2024-03-04T07:31:49.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4e3c02c730 con 0x7f4e300039b0 2024-03-04T07:31:49.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e437fe640 1 -- 172.21.15.64:0/2587249479 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e3c001b30 con 0x7f4e300039b0 2024-03-04T07:31:49.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e4b25c640 1 -- 172.21.15.64:0/2587249479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e300039b0 msgr2=0x7f4e30003d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e300039b0 0x7f4e30003d90 secure :-1 s=READY pgs=1197 cs=0 l=1 rev1=1 crypto rx=0x7f4e3c001f60 tx=0x7f4e30005830 comp rx=0 tx=0).stop 2024-03-04T07:31:49.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e4b25c640 1 -- 172.21.15.64:0/2587249479 shutdown_connections 2024-03-04T07:31:49.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/2587249479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e300039b0 0x7f4e30003d90 unknown :-1 s=CLOSED pgs=1197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e4b25c640 1 -- 172.21.15.64:0/2587249479 >> 172.21.15.64:0/2587249479 conn(0x7f4e440d3ef0 msgr2=0x7f4e440d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:49.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e4b25c640 1 -- 172.21.15.64:0/2587249479 shutdown_connections 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.735+0000 7f4e4b25c640 1 -- 172.21.15.64:0/2587249479 wait complete. 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 Processor -- start 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 -- start start 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300039b0 0x7f4e440cd9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e441f33b0 0x7f4e440b7eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e4406bcf0 0x7f4e4406c170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e4415e4e0 con 0x7f4e441f33b0 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e440cf510 con 0x7f4e300039b0 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e4b25c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4e4415e660 con 0x7f4e4406bcf0 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.737+0000 7f4e43fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e441f33b0 0x7f4e440b7eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e441f33b0 0x7f4e440b7eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37722/0 (socket says 172.21.15.64:37722) 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 -- 172.21.15.64:0/3532814447 learned_addr learned my addr 172.21.15.64:0/3532814447 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e48ff8640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300039b0 0x7f4e440cd9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e497f9640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e4406bcf0 0x7f4e4406c170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 -- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e4406bcf0 msgr2=0x7f4e4406c170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e4406bcf0 0x7f4e4406c170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 -- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300039b0 msgr2=0x7f4e440cd9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300039b0 0x7f4e440cd9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 -- 172.21.15.64:0/3532814447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e440cdef0 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e43fff640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e441f33b0 0x7f4e440b7eb0 secure :-1 s=READY pgs=1198 cs=0 l=1 rev1=1 crypto rx=0x7f4e2c001df0 tx=0x7f4e2c007430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e417fa640 1 -- 172.21.15.64:0/3532814447 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e2c00f780 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e417fa640 1 -- 172.21.15.64:0/3532814447 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4e2c018070 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4e440b83f0 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e417fa640 1 -- 172.21.15.64:0/3532814447 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e2c00fb80 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.738+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4e440cb130 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.739+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4e440752d0 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.740+0000 7f4e417fa640 1 -- 172.21.15.64:0/3532814447 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4e2c00fd20 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.740+0000 7f4e417fa640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e2406b500 0x7f4e2406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.740+0000 7f4e417fa640 1 -- 172.21.15.64:0/3532814447 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f4e2c089f70 con 0x7f4e441f33b0 2024-03-04T07:31:49.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.740+0000 7f4e48ff8640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e2406b500 0x7f4e2406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:49.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.741+0000 7f4e48ff8640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e2406b500 0x7f4e2406d9c0 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f4e3c036000 tx=0x7f4e3c009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:49.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.744+0000 7f4e417fa640 1 -- 172.21.15.64:0/3532814447 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4e2c058780 con 0x7f4e441f33b0 2024-03-04T07:31:49.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.964+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4e4406b7d0 con 0x7f4e441f33b0 2024-03-04T07:31:49.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.965+0000 7f4e417fa640 1 -- 172.21.15.64:0/3532814447 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f4e2c05c820 con 0x7f4e441f33b0 2024-03-04T07:31:49.965 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:49.965 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:49.965 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:49.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e2406b500 msgr2=0x7f4e2406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e2406b500 0x7f4e2406d9c0 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f4e3c036000 tx=0x7f4e3c009030 comp rx=0 tx=0).stop 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e441f33b0 msgr2=0x7f4e440b7eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e441f33b0 0x7f4e440b7eb0 secure :-1 s=READY pgs=1198 cs=0 l=1 rev1=1 crypto rx=0x7f4e2c001df0 tx=0x7f4e2c007430 comp rx=0 tx=0).stop 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 shutdown_connections 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4e2406b500 0x7f4e2406d9c0 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4e4406bcf0 0x7f4e4406c170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4e441f33b0 0x7f4e440b7eb0 unknown :-1 s=CLOSED pgs=1198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 --2- 172.21.15.64:0/3532814447 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4e300039b0 0x7f4e440cd9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 >> 172.21.15.64:0/3532814447 conn(0x7f4e440d3ef0 msgr2=0x7f4e440b2100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 shutdown_connections 2024-03-04T07:31:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:49.967+0000 7f4e4b25c640 1 -- 172.21.15.64:0/3532814447 wait complete. 2024-03-04T07:31:49.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump 2024-03-04T07:31:50.095 INFO:teuthology.orchestra.run.smithi064.stdout:{"AsyncMessenger::Worker-0":{"msgr_recv_messages":1194,"msgr_send_messages":1189,"msgr_recv_bytes":438241,"msgr_send_bytes":5359180,"msgr_created_connections":8,"msgr_active_connections":4,"msgr_running_total_time":0.126107882,"msgr_running_send_time":0.061228038,"msgr_running_recv_time":0.023064417,"msgr_running_fast_dispatch_time":0.034350967,"msgr_send_messages_queue_lat":{"avgcount":1189,"sum":1.951432970,"avgtime":0.001641238},"msgr_handle_ack_lat":{"avgcount":2,"sum":0.000000167,"avgtime":0.000000083},"msgr_recv_encrypted_bytes":108400,"msgr_send_encrypted_bytes":8544},"AsyncMessenger::Worker-1":{"msgr_recv_messages":10975,"msgr_send_messages":24776,"msgr_recv_bytes":3923841,"msgr_send_bytes":28138585,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":1.164692329,"msgr_running_send_time":0.651652810,"msgr_running_recv_time":0.541173508,"msgr_running_fast_dispatch_time":0.042694236,"msgr_send_messages_queue_lat":{"avgcount":24775,"sum":0.874958981,"avgtime":0.000035316},"msgr_handle_ack_lat":{"avgcount":29683,"sum":0.005559292,"avgtime":0.000000187},"msgr_recv_encrypted_bytes":1728,"msgr_send_encrypted_bytes":98048},"AsyncMessenger::Worker-2":{"msgr_recv_messages":2282,"msgr_send_messages":2284,"msgr_recv_bytes":964676,"msgr_send_bytes":2812539,"msgr_created_connections":6,"msgr_active_connections":5,"msgr_running_total_time":0.201774323,"msgr_running_send_time":0.097403715,"msgr_running_recv_time":0.038060939,"msgr_running_fast_dispatch_time":0.053850941,"msgr_send_messages_queue_lat":{"avgcount":2282,"sum":1.352343849,"avgtime":0.000592613},"msgr_handle_ack_lat":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"msgr_recv_encrypted_bytes":0,"msgr_send_encrypted_bytes":0},"cct":{"total_workers":1,"unhealthy_workers":0},"finisher-MDSRank":{"queue_len":0,"complete_latency":{"avgcount":130,"sum":7.116505977,"avgtime":0.054742353}},"finisher-PurgeQueue":{"queue_len":0,"complete_latency":{"avgcount":5383,"sum":1.461372435,"avgtime":0.000271479}},"mds":{"request":6413,"reply":6411,"reply_latency":{"avgcount":6411,"sum":9.655326614,"avgtime":0.001506056},"slow_reply":0,"forward":0,"dir_fetch_complete":12,"dir_fetch_keys":10,"dir_commit":12,"dir_split":0,"dir_merge":0,"inodes":1386,"inodes_top":0,"inodes_bottom":1386,"inodes_pin_tail":0,"inodes_pinned":1386,"inodes_expired":0,"inodes_with_caps":1,"caps":1,"subtrees":2,"traverse":6419,"traverse_hit":6418,"traverse_forward":0,"traverse_discover":0,"traverse_dir_fetch":0,"traverse_remote_ino":0,"traverse_lock":0,"load_cent":0,"q":0,"exported":0,"exported_inodes":0,"imported":0,"imported_inodes":0,"openino_dir_fetch":0,"openino_backtrace_fetch":0,"openino_peer_discover":0,"root_rfiles":0,"root_rbytes":0,"root_rsnaps":0,"scrub_backtrace_fetch":0,"scrub_set_tag":0,"scrub_backtrace_repaired":0,"scrub_inotable_repaired":0,"scrub_dir_inodes":0,"scrub_dir_base_inodes":0,"scrub_dirfrag_rstats":0,"scrub_file_inodes":0,"handle_inode_file_caps":0,"ceph_cap_op_revoke":621,"ceph_cap_op_grant":7014,"ceph_cap_op_trunc":0,"ceph_cap_op_flushsnap_ack":0,"ceph_cap_op_flush_ack":0,"handle_client_caps":3206,"handle_client_caps_dirty":3000,"handle_client_cap_release":10,"process_request_cap_release":9603},"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":0,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":3201,"strays_enqueued":3201,"strays_reintegrated":0,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0},"mds_log":{"evlrg":0,"evadd":12826,"evex":11450,"evtrm":11450,"ev":1379,"evexg":0,"evexd":11449,"segadd":16,"segex":16,"segtrm":16,"seg":2,"segmjr":1,"segexg":0,"segexd":15,"expos":28013210,"wrpos":28842775,"rdpos":4195244,"jlat":{"avgcount":24,"sum":1.505652818,"avgtime":0.062735534},"replayed":3},"mds_mem":{"ino":13,"ino+":3214,"ino-":3201,"dir":12,"dir+":13,"dir-":1,"dn":1386,"dn+":6412,"dn-":5026,"cap":1,"cap+":3202,"cap-":3201,"rss":113244,"heap":248164},"mds_server":{"dispatch_client_request":6419,"dispatch_server_request":0,"handle_client_request":6413,"handle_client_session":1,"handle_peer_request":0,"req_create_latency":{"avgcount":3200,"sum":3.353721224,"avgtime":0.001048037},"req_getattr_latency":{"avgcount":6,"sum":0.001296405,"avgtime":0.000216067},"req_getfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_link_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookup_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookuphash_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupino_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupname_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupparent_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lookupsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_lssnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mkdir_latency":{"avgcount":1,"sum":0.002420542,"avgtime":0.002420542},"req_mknod_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_mksnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_open_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_readdir_latency":{"avgcount":3,"sum":0.200156740,"avgtime":0.066718913},"req_rename_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_renamesnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_snapdiff_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmdir_latency":{"avgcount":1,"sum":0.545126756,"avgtime":0.545126756},"req_rmsnap_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_rmxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setdirlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setfilelock_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setlayout_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_setxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_symlink_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"req_unlink_latency":{"avgcount":3200,"sum":5.552604947,"avgtime":0.001735189},"cap_revoke_eviction":0,"cap_acquisition_throttle":0,"req_getvxattr_latency":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"mds_sessions":{"session_count":1,"session_add":1,"session_remove":0,"sessions_open":1,"sessions_stale":0,"total_load":5853,"average_load":5853,"avg_session_uptime":817,"mdthresh_evicted":0},"mempool":{"bloom_filter_bytes":0,"bloom_filter_items":0,"bluestore_alloc_bytes":0,"bluestore_alloc_items":0,"bluestore_cache_data_bytes":0,"bluestore_cache_data_items":0,"bluestore_cache_onode_bytes":0,"bluestore_cache_onode_items":0,"bluestore_cache_meta_bytes":0,"bluestore_cache_meta_items":0,"bluestore_cache_other_bytes":0,"bluestore_cache_other_items":0,"bluestore_cache_buffer_bytes":0,"bluestore_cache_buffer_items":0,"bluestore_extent_bytes":0,"bluestore_extent_items":0,"bluestore_blob_bytes":0,"bluestore_blob_items":0,"bluestore_shared_blob_bytes":0,"bluestore_shared_blob_items":0,"bluestore_inline_bl_bytes":0,"bluestore_inline_bl_items":0,"bluestore_fsck_bytes":0,"bluestore_fsck_items":0,"bluestore_txc_bytes":0,"bluestore_txc_items":0,"bluestore_writing_deferred_bytes":0,"bluestore_writing_deferred_items":0,"bluestore_writing_bytes":0,"bluestore_writing_items":0,"bluefs_bytes":0,"bluefs_items":0,"bluefs_file_reader_bytes":0,"bluefs_file_reader_items":0,"bluefs_file_writer_bytes":0,"bluefs_file_writer_items":0,"buffer_anon_bytes":11319594,"buffer_anon_items":2850,"buffer_meta_bytes":0,"buffer_meta_items":0,"osd_bytes":0,"osd_items":0,"osd_mapbl_bytes":0,"osd_mapbl_items":0,"osd_pglog_bytes":0,"osd_pglog_items":0,"osdmap_bytes":5672,"osdmap_items":92,"osdmap_mapping_bytes":0,"osdmap_mapping_items":0,"pgmap_bytes":0,"pgmap_items":0,"mds_co_bytes":4407832,"mds_co_items":73121,"unittest_1_bytes":0,"unittest_1_items":0,"unittest_2_bytes":0,"unittest_2_items":0},"objecter":{"op_active":0,"op_laggy":0,"op_send":4768,"op_send_bytes":26335187,"op_resend":0,"op_reply":4768,"op_latency":{"avgcount":4768,"sum":124.282629950,"avgtime":0.026065987},"op_inflight":0,"oplen_avg":{"avgcount":4768,"sum":7635},"op":4768,"op_r":60,"op_w":4708,"op_rmw":0,"op_pg":0,"osdop_stat":12,"osdop_create":1387,"osdop_read":32,"osdop_write":48,"osdop_writefull":17,"osdop_writesame":0,"osdop_append":0,"osdop_zero":1,"osdop_truncate":0,"osdop_delete":3222,"osdop_mapext":0,"osdop_sparse_read":0,"osdop_clonerange":0,"osdop_getxattr":24,"osdop_setxattr":2774,"osdop_cmpxattr":0,"osdop_rmxattr":0,"osdop_resetxattrs":0,"osdop_call":0,"osdop_watch":0,"osdop_notify":0,"osdop_src_cmpxattr":0,"osdop_pgls":0,"osdop_pgls_filter":0,"osdop_other":0,"linger_active":0,"linger_send":0,"linger_resend":0,"linger_ping":0,"poolop_active":0,"poolop_send":0,"poolop_resend":0,"poolstat_active":0,"poolstat_send":0,"poolstat_resend":0,"statfs_active":0,"statfs_send":0,"statfs_resend":0,"command_active":0,"command_send":0,"command_resend":0,"map_epoch":0,"map_full":0,"map_inc":0,"osd_sessions":8,"osd_session_open":8,"osd_session_close":0,"osd_laggy":0,"omap_wr":55,"omap_rd":48,"omap_del":15},"oft":{"omap_total_objs":0,"omap_total_kv_pairs":0,"omap_total_updates":0,"omap_total_removes":0},"osd-slow-ops":{"slow_ops_count":0},"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":6,"pq_executing":0,"pq_executing_high_water":6,"pq_executed_ops":3201,"pq_executed":3201,"pq_item_in_journal":0},"throttle-msgr_dispatch_throttler-mds":{"val":0,"max":104857600,"get_started":0,"get":14447,"get_sum":4023701,"get_or_fail_fail":0,"get_or_fail_success":14447,"take":0,"take_sum":0,"put":14447,"put_sum":4023701,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_bytes":{"val":0,"max":104857600,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":4768,"take_sum":26659605,"put":4768,"put_sum":26659605,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-objecter_ops":{"val":0,"max":1024,"get_started":0,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":4768,"take_sum":4768,"put":4768,"put_sum":4768,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle":{"val":0,"max":3758096384,"get_started":0,"get":3201,"get_sum":323301,"get_or_fail_fail":0,"get_or_fail_success":3201,"take":0,"take_sum":0,"put":24,"put_sum":323301,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}},"throttle-write_buf_throttle-0x562d284d2a00":{"val":0,"max":3758096384,"get_started":0,"get":12826,"get_sum":24647090,"get_or_fail_fail":0,"get_or_fail_success":12826,"take":0,"take_sum":0,"put":24,"put_sum":24647090,"wait":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000}}} 2024-03-04T07:31:50.096 DEBUG:tasks.cephfs.filesystem:_json_asok output { "AsyncMessenger::Worker-0": { "msgr_active_connections": 4, "msgr_created_connections": 8, "msgr_handle_ack_lat": { "avgcount": 2, "avgtime": 8.3e-08, "sum": 1.67e-07 }, "msgr_recv_bytes": 438241, "msgr_recv_encrypted_bytes": 108400, "msgr_recv_messages": 1194, "msgr_running_fast_dispatch_time": 0.034350967, "msgr_running_recv_time": 0.023064417, "msgr_running_send_time": 0.061228038, "msgr_running_total_time": 0.126107882, "msgr_send_bytes": 5359180, "msgr_send_encrypted_bytes": 8544, "msgr_send_messages": 1189, "msgr_send_messages_queue_lat": { "avgcount": 1189, "avgtime": 0.001641238, "sum": 1.95143297 } }, "AsyncMessenger::Worker-1": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 29683, "avgtime": 1.87e-07, "sum": 0.005559292 }, "msgr_recv_bytes": 3923841, "msgr_recv_encrypted_bytes": 1728, "msgr_recv_messages": 10975, "msgr_running_fast_dispatch_time": 0.042694236, "msgr_running_recv_time": 0.541173508, "msgr_running_send_time": 0.65165281, "msgr_running_total_time": 1.164692329, "msgr_send_bytes": 28138585, "msgr_send_encrypted_bytes": 98048, "msgr_send_messages": 24776, "msgr_send_messages_queue_lat": { "avgcount": 24775, "avgtime": 3.5316e-05, "sum": 0.874958981 } }, "AsyncMessenger::Worker-2": { "msgr_active_connections": 5, "msgr_created_connections": 6, "msgr_handle_ack_lat": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "msgr_recv_bytes": 964676, "msgr_recv_encrypted_bytes": 0, "msgr_recv_messages": 2282, "msgr_running_fast_dispatch_time": 0.053850941, "msgr_running_recv_time": 0.038060939, "msgr_running_send_time": 0.097403715, "msgr_running_total_time": 0.201774323, "msgr_send_bytes": 2812539, "msgr_send_encrypted_bytes": 0, "msgr_send_messages": 2284, "msgr_send_messages_queue_lat": { "avgcount": 2282, "avgtime": 0.000592613, "sum": 1.352343849 } }, "cct": { "total_workers": 1, "unhealthy_workers": 0 }, "finisher-MDSRank": { "complete_latency": { "avgcount": 130, "avgtime": 0.054742353, "sum": 7.116505977 }, "queue_len": 0 }, "finisher-PurgeQueue": { "complete_latency": { "avgcount": 5383, "avgtime": 0.000271479, "sum": 1.461372435 }, "queue_len": 0 }, "mds": { "caps": 1, "ceph_cap_op_flush_ack": 0, "ceph_cap_op_flushsnap_ack": 0, "ceph_cap_op_grant": 7014, "ceph_cap_op_revoke": 621, "ceph_cap_op_trunc": 0, "dir_commit": 12, "dir_fetch_complete": 12, "dir_fetch_keys": 10, "dir_merge": 0, "dir_split": 0, "exported": 0, "exported_inodes": 0, "forward": 0, "handle_client_cap_release": 10, "handle_client_caps": 3206, "handle_client_caps_dirty": 3000, "handle_inode_file_caps": 0, "imported": 0, "imported_inodes": 0, "inodes": 1386, "inodes_bottom": 1386, "inodes_expired": 0, "inodes_pin_tail": 0, "inodes_pinned": 1386, "inodes_top": 0, "inodes_with_caps": 1, "load_cent": 0, "openino_backtrace_fetch": 0, "openino_dir_fetch": 0, "openino_peer_discover": 0, "process_request_cap_release": 9603, "q": 0, "reply": 6411, "reply_latency": { "avgcount": 6411, "avgtime": 0.001506056, "sum": 9.655326614 }, "request": 6413, "root_rbytes": 0, "root_rfiles": 0, "root_rsnaps": 0, "scrub_backtrace_fetch": 0, "scrub_backtrace_repaired": 0, "scrub_dir_base_inodes": 0, "scrub_dir_inodes": 0, "scrub_dirfrag_rstats": 0, "scrub_file_inodes": 0, "scrub_inotable_repaired": 0, "scrub_set_tag": 0, "slow_reply": 0, "subtrees": 2, "traverse": 6419, "traverse_dir_fetch": 0, "traverse_discover": 0, "traverse_forward": 0, "traverse_hit": 6418, "traverse_lock": 0, "traverse_remote_ino": 0 }, "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 0, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3201, "strays_enqueued": 3201, "strays_migrated": 0, "strays_reintegrated": 0 }, "mds_log": { "ev": 1379, "evadd": 12826, "evex": 11450, "evexd": 11449, "evexg": 0, "evlrg": 0, "evtrm": 11450, "expos": 28013210, "jlat": { "avgcount": 24, "avgtime": 0.062735534, "sum": 1.505652818 }, "rdpos": 4195244, "replayed": 3, "seg": 2, "segadd": 16, "segex": 16, "segexd": 15, "segexg": 0, "segmjr": 1, "segtrm": 16, "wrpos": 28842775 }, "mds_mem": { "cap": 1, "cap+": 3202, "cap-": 3201, "dir": 12, "dir+": 13, "dir-": 1, "dn": 1386, "dn+": 6412, "dn-": 5026, "heap": 248164, "ino": 13, "ino+": 3214, "ino-": 3201, "rss": 113244 }, "mds_server": { "cap_acquisition_throttle": 0, "cap_revoke_eviction": 0, "dispatch_client_request": 6419, "dispatch_server_request": 0, "handle_client_request": 6413, "handle_client_session": 1, "handle_peer_request": 0, "req_create_latency": { "avgcount": 3200, "avgtime": 0.001048037, "sum": 3.353721224 }, "req_getattr_latency": { "avgcount": 6, "avgtime": 0.000216067, "sum": 0.001296405 }, "req_getfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_getvxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_link_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookup_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookuphash_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupino_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupname_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupparent_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lookupsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_lssnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mkdir_latency": { "avgcount": 1, "avgtime": 0.002420542, "sum": 0.002420542 }, "req_mknod_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_mksnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_open_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_readdir_latency": { "avgcount": 3, "avgtime": 0.066718913, "sum": 0.20015674 }, "req_rename_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_renamesnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmdir_latency": { "avgcount": 1, "avgtime": 0.545126756, "sum": 0.545126756 }, "req_rmsnap_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_rmxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setdirlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setfilelock_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setlayout_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_setxattr_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_snapdiff_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_symlink_latency": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 }, "req_unlink_latency": { "avgcount": 3200, "avgtime": 0.001735189, "sum": 5.552604947 } }, "mds_sessions": { "average_load": 5853, "avg_session_uptime": 817, "mdthresh_evicted": 0, "session_add": 1, "session_count": 1, "session_remove": 0, "sessions_open": 1, "sessions_stale": 0, "total_load": 5853 }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluefs_bytes": 0, "bluefs_file_reader_bytes": 0, "bluefs_file_reader_items": 0, "bluefs_file_writer_bytes": 0, "bluefs_file_writer_items": 0, "bluefs_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_blob_bytes": 0, "bluestore_blob_items": 0, "bluestore_cache_buffer_bytes": 0, "bluestore_cache_buffer_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_meta_bytes": 0, "bluestore_cache_meta_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_extent_bytes": 0, "bluestore_extent_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_inline_bl_bytes": 0, "bluestore_inline_bl_items": 0, "bluestore_shared_blob_bytes": 0, "bluestore_shared_blob_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_items": 0, "buffer_anon_bytes": 11319594, "buffer_anon_items": 2850, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "mds_co_bytes": 4407832, "mds_co_items": 73121, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 5672, "osdmap_items": 92, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "command_active": 0, "command_resend": 0, "command_send": 0, "linger_active": 0, "linger_ping": 0, "linger_resend": 0, "linger_send": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "omap_del": 15, "omap_rd": 48, "omap_wr": 55, "op": 4768, "op_Infinitylight": 0, "op_active": 0, "op_laggy": 0, "op_latency": { "avgcount": 4768, "avgtime": 0.026065987, "sum": 124.28262995 }, "op_pg": 0, "op_r": 60, "op_reply": 4768, "op_resend": 0, "op_rmw": 0, "op_send": 4768, "op_send_bytes": 26335187, "op_w": 4708, "oplen_avg": { "avgcount": 4768, "sum": 7635 }, "osd_laggy": 0, "osd_session_close": 0, "osd_session_open": 8, "osd_sessions": 8, "osdop_append": 0, "osdop_call": 0, "osdop_clonerange": 0, "osdop_cmpxattr": 0, "osdop_create": 1387, "osdop_delete": 3222, "osdop_getxattr": 24, "osdop_mapext": 0, "osdop_notify": 0, "osdop_other": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_read": 32, "osdop_resetxattrs": 0, "osdop_rmxattr": 0, "osdop_setxattr": 2774, "osdop_sparse_read": 0, "osdop_src_cmpxattr": 0, "osdop_stat": 12, "osdop_truncate": 0, "osdop_watch": 0, "osdop_write": 48, "osdop_writefull": 17, "osdop_writesame": 0, "osdop_zero": 1, "poolop_active": 0, "poolop_resend": 0, "poolop_send": 0, "poolstat_active": 0, "poolstat_resend": 0, "poolstat_send": 0, "statfs_active": 0, "statfs_resend": 0, "statfs_send": 0 }, "oft": { "omap_total_kv_pairs": 0, "omap_total_objs": 0, "omap_total_removes": 0, "omap_total_updates": 0 }, "osd-slow-ops": { "slow_ops_count": 0 }, "purge_queue": { "pq_executed": 3201, "pq_executed_ops": 3201, "pq_executing": 0, "pq_executing_high_water": 6, "pq_executing_ops": 0, "pq_executing_ops_high_water": 6, "pq_item_in_journal": 0 }, "throttle-msgr_dispatch_throttler-mds": { "get": 14447, "get_or_fail_fail": 0, "get_or_fail_success": 14447, "get_started": 0, "get_sum": 4023701, "max": 104857600, "put": 14447, "put_sum": 4023701, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_bytes": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 104857600, "put": 4768, "put_sum": 26659605, "take": 4768, "take_sum": 26659605, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-objecter_ops": { "get": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "get_started": 0, "get_sum": 0, "max": 1024, "put": 4768, "put_sum": 4768, "take": 4768, "take_sum": 4768, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle": { "get": 3201, "get_or_fail_fail": 0, "get_or_fail_success": 3201, "get_started": 0, "get_sum": 323301, "max": 3758096384, "put": 24, "put_sum": 323301, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } }, "throttle-write_buf_throttle-0x562d284d2a00": { "get": 12826, "get_or_fail_fail": 0, "get_or_fail_success": 12826, "get_started": 0, "get_sum": 24647090, "max": 3758096384, "put": 24, "put_sum": 24647090, "take": 0, "take_sum": 0, "val": 0, "wait": { "avgcount": 0, "avgtime": 0.0, "sum": 0.0 } } } 2024-03-04T07:31:50.097 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:31:50.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:31:50.145 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:31:50.195 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:31:50.196 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:31:50.196 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:50.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:50.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.353+0000 7fd5c70eb640 1 Processor -- start 2024-03-04T07:31:50.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.353+0000 7fd5c70eb640 1 -- start start 2024-03-04T07:31:50.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5c70eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 0x7fd5c00cd070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5c70eb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd5c015b150 con 0x7fd5c015ad70 2024-03-04T07:31:50.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5c70eb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd5c00cd5b0 con 0x7fd5c0071ab0 2024-03-04T07:31:50.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5c70eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd5c00b3080 con 0x7fd5c00b2aa0 2024-03-04T07:31:50.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5c4e87640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd5c015ad70 0x7fd5c015f2d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58266/0 (socket says 172.21.15.64:58266) 2024-03-04T07:31:50.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5c4e87640 1 -- 172.21.15.64:0/3473543553 learned_addr learned my addr 172.21.15.64:0/3473543553 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bffff640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 0x7fd5c00cd070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4278181012 0 0) 0x7fd5c015b150 con 0x7fd5c015ad70 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd5b0003520 con 0x7fd5c015ad70 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3524751322 0 0) 0x7fd5b0003520 con 0x7fd5c015ad70 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 msgr2=0x7fd5c00cd070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 0x7fd5c00cd070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 >> v1:172.21.15.66:6789/0 conn(0x7fd5c0071ab0 legacy=0x7fd5c00b4120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5c00b3360 con 0x7fd5c015ad70 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fd5b4001670 con 0x7fd5c015ad70 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 >> v1:172.21.15.64:6789/0 conn(0x7fd5c015ad70 legacy=0x7fd5c015f2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5b00039e0 0x7fd5b0003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5b0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 0x7fd5b00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5c00b3080 con 0x7fd5b0004800 2024-03-04T07:31:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b000ba70 con 0x7fd5b00039e0 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.354+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b000bc70 con 0x7fd5c00b2aa0 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5bffff640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5b0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5c4e87640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5b00039e0 0x7fd5b0003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5c5688640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 0x7fd5b00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5bffff640 1 -- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 msgr2=0x7fd5b00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5bffff640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c00b2aa0 0x7fd5b00051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5bffff640 1 -- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5b00039e0 msgr2=0x7fd5b0003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5bffff640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5b00039e0 0x7fd5b0003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5bffff640 1 -- 172.21.15.64:0/3473543553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5b0003520 con 0x7fd5b0004800 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.355+0000 7fd5c4e87640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5b00039e0 0x7fd5b0003e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5bffff640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5b0004c80 secure :-1 s=READY pgs=1200 cs=0 l=1 rev1=1 crypto rx=0x7fd5ac009a90 tx=0x7fd5b0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=16157c3060c30fd3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5ac039070 con 0x7fd5b0004800 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd5ac02c680 con 0x7fd5b0004800 2024-03-04T07:31:50.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5bf7fe640 1 -- 172.21.15.64:0/3473543553 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5ac009690 con 0x7fd5b0004800 2024-03-04T07:31:50.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5c70eb640 1 -- 172.21.15.64:0/3473543553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 msgr2=0x7fd5b0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5c70eb640 1 --2- 172.21.15.64:0/3473543553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5b0004c80 secure :-1 s=READY pgs=1200 cs=0 l=1 rev1=1 crypto rx=0x7fd5ac009a90 tx=0x7fd5b0005830 comp rx=0 tx=0).stop 2024-03-04T07:31:50.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5c5688640 1 -- 172.21.15.64:0/3473543553 reap_dead start 2024-03-04T07:31:50.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5c70eb640 1 -- 172.21.15.64:0/3473543553 shutdown_connections 2024-03-04T07:31:50.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5c70eb640 1 -- 172.21.15.64:0/3473543553 >> 172.21.15.64:0/3473543553 conn(0x7fd5c00b17a0 msgr2=0x7fd5c00b1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:50.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5c70eb640 1 -- 172.21.15.64:0/3473543553 shutdown_connections 2024-03-04T07:31:50.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.356+0000 7fd5c70eb640 1 -- 172.21.15.64:0/3473543553 wait complete. 2024-03-04T07:31:50.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.358+0000 7fd5c70eb640 1 Processor -- start 2024-03-04T07:31:50.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.358+0000 7fd5c70eb640 1 -- start start 2024-03-04T07:31:50.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.359+0000 7fd5c70eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5c01f0550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.359+0000 7fd5c70eb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5c01f0a90 0x7fd5c014e9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.359+0000 7fd5c70eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c014eef0 0x7fd5c014f370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.359+0000 7fd5c70eb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5c00b3c00 con 0x7fd5b0004800 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.359+0000 7fd5c70eb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5c00b3920 con 0x7fd5c01f0a90 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.359+0000 7fd5c70eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd5c00cd070 con 0x7fd5c014eef0 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5c01f0550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5c01f0550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37746/0 (socket says 172.21.15.64:37746) 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 -- 172.21.15.64:0/978783565 learned_addr learned my addr 172.21.15.64:0/978783565 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5bffff640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5c01f0a90 0x7fd5c014e9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c5688640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c014eef0 0x7fd5c014f370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 -- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c014eef0 msgr2=0x7fd5c014f370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c014eef0 0x7fd5c014f370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 -- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5c01f0a90 msgr2=0x7fd5c014e9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5c01f0a90 0x7fd5c014e9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 -- 172.21.15.64:0/978783565 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5c01e4a50 con 0x7fd5b0004800 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c4e87640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5c01f0550 secure :-1 s=READY pgs=1201 cs=0 l=1 rev1=1 crypto rx=0x7fd5b400e7e0 tx=0x7fd5b400eca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5bd7fa640 1 -- 172.21.15.64:0/978783565 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5b400c8d0 con 0x7fd5b0004800 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5bd7fa640 1 -- 172.21.15.64:0/978783565 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd5b4018070 con 0x7fd5b0004800 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd5c01e4d30 con 0x7fd5b0004800 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5bd7fa640 1 -- 172.21.15.64:0/978783565 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5b400ccb0 con 0x7fd5b0004800 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.360+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd5c01e52f0 con 0x7fd5b0004800 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.361+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd5c01e54f0 con 0x7fd5b0004800 2024-03-04T07:31:50.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.362+0000 7fd5bd7fa640 1 -- 172.21.15.64:0/978783565 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd5b401d460 con 0x7fd5b0004800 2024-03-04T07:31:50.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.362+0000 7fd5bd7fa640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd5a406b500 0x7fd5a406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.362+0000 7fd5bffff640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd5a406b500 0x7fd5a406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.362+0000 7fd5bd7fa640 1 -- 172.21.15.64:0/978783565 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fd5b408a0d0 con 0x7fd5b0004800 2024-03-04T07:31:50.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.363+0000 7fd5bffff640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd5a406b500 0x7fd5a406d9c0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7fd5ac036000 tx=0x7fd5ac030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.367+0000 7fd5bd7fa640 1 -- 172.21.15.64:0/978783565 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd5c01e54f0 con 0x7fd5b0004800 2024-03-04T07:31:50.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.548+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fd5c00752d0 con 0x7fd5b0004800 2024-03-04T07:31:50.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.549+0000 7fd5bd7fa640 1 -- 172.21.15.64:0/978783565 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) v1 ==== 74+0+16819 (secure 0 0 0) 0x7fd5b40587c0 con 0x7fd5b0004800 2024-03-04T07:31:50.549 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:50.550 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":94,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:24.148060+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:31:12.750651+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-04T07:31:13.444529+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"90","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.64:6835/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.66:6835/3397672669":"2024-03-05T07:31:23.154858+0000","172.21.15.66:6833/3397672669":"2024-03-05T07:31:23.154858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:50.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd5a406b500 msgr2=0x7fd5a406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd5a406b500 0x7fd5a406d9c0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7fd5ac036000 tx=0x7fd5ac030000 comp rx=0 tx=0).stop 2024-03-04T07:31:50.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 msgr2=0x7fd5c01f0550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5c01f0550 secure :-1 s=READY pgs=1201 cs=0 l=1 rev1=1 crypto rx=0x7fd5b400e7e0 tx=0x7fd5b400eca0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 shutdown_connections 2024-03-04T07:31:50.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd5a406b500 0x7fd5a406d9c0 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd5c014eef0 0x7fd5c014f370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd5c01f0a90 0x7fd5c014e9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 --2- 172.21.15.64:0/978783565 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd5b0004800 0x7fd5c01f0550 unknown :-1 s=CLOSED pgs=1201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 >> 172.21.15.64:0/978783565 conn(0x7fd5c00b17a0 msgr2=0x7fd5c00cee90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:50.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 shutdown_connections 2024-03-04T07:31:50.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.551+0000 7fd5c70eb640 1 -- 172.21.15.64:0/978783565 wait complete. 2024-03-04T07:31:50.565 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 14, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:31:12.750651+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 15, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:31:13.444529+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '90', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:31:50.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:31:50.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.673+0000 7f434e822640 1 Processor -- start 2024-03-04T07:31:50.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.673+0000 7f434e822640 1 -- start start 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f434e822640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f434815afb0 0x7f434815f5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f434e822640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f434806e2d0 0x7f43480cedc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f434e822640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806bfd0 0x7f434806c460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f434e822640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43480cf300 con 0x7f434815afb0 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f434e822640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43480cf5a0 con 0x7f434806e2d0 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f434e822640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43480cf880 con 0x7f434806bfd0 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f434815afb0 0x7f434815f5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f434815afb0 0x7f434815f5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37754/0 (socket says 172.21.15.64:37754) 2024-03-04T07:31:50.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 -- 172.21.15.64:0/1111816533 learned_addr learned my addr 172.21.15.64:0/1111816533 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:50.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 -- 172.21.15.64:0/1111816533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806bfd0 msgr2=0x7f434806c460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:50.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 --2- 172.21.15.64:0/1111816533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806bfd0 0x7f434806c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 -- 172.21.15.64:0/1111816533 >> v2:172.21.15.66:3300/0 conn(0x7f434806e2d0 msgr2=0x7f43480cedc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:50.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f43477fe640 1 --2- 172.21.15.64:0/1111816533 >> v2:172.21.15.66:3300/0 conn(0x7f434806e2d0 0x7f43480cedc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 --2- 172.21.15.64:0/1111816533 >> v2:172.21.15.66:3300/0 conn(0x7f434806e2d0 0x7f43480cedc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.675+0000 7f4347fff640 1 -- 172.21.15.64:0/1111816533 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43480cfb60 con 0x7f434815afb0 2024-03-04T07:31:50.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.676+0000 7f4347fff640 1 --2- 172.21.15.64:0/1111816533 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434815afb0 0x7f434815f5c0 secure :-1 s=READY pgs=1202 cs=0 l=1 rev1=1 crypto rx=0x7f43480c4840 tx=0x7f433802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b775ab90bed1944a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.676+0000 7f4346ffd640 1 -- 172.21.15.64:0/1111816533 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4338035020 con 0x7f434815afb0 2024-03-04T07:31:50.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.676+0000 7f4346ffd640 1 -- 172.21.15.64:0/1111816533 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4338031870 con 0x7f434815afb0 2024-03-04T07:31:50.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.676+0000 7f4346ffd640 1 -- 172.21.15.64:0/1111816533 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4338031c40 con 0x7f434815afb0 2024-03-04T07:31:50.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 -- 172.21.15.64:0/1111816533 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434815afb0 msgr2=0x7f434815f5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 --2- 172.21.15.64:0/1111816533 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434815afb0 0x7f434815f5c0 secure :-1 s=READY pgs=1202 cs=0 l=1 rev1=1 crypto rx=0x7f43480c4840 tx=0x7f433802b910 comp rx=0 tx=0).stop 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 -- 172.21.15.64:0/1111816533 shutdown_connections 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 --2- 172.21.15.64:0/1111816533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806bfd0 0x7f434806c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 --2- 172.21.15.64:0/1111816533 >> v2:172.21.15.66:3300/0 conn(0x7f434806e2d0 0x7f43480cedc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 --2- 172.21.15.64:0/1111816533 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434815afb0 0x7f434815f5c0 unknown :-1 s=CLOSED pgs=1202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 -- 172.21.15.64:0/1111816533 >> 172.21.15.64:0/1111816533 conn(0x7f43480d0ec0 msgr2=0x7f43480d12a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 -- 172.21.15.64:0/1111816533 shutdown_connections 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.677+0000 7f434e822640 1 -- 172.21.15.64:0/1111816533 wait complete. 2024-03-04T07:31:50.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.678+0000 7f434e822640 1 Processor -- start 2024-03-04T07:31:50.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.678+0000 7f434e822640 1 -- start start 2024-03-04T07:31:50.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434e822640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434806bfd0 0x7f43480b3ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434e822640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806e2d0 0x7f43481f3fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434e822640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434815afb0 0x7f43481f4520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434e822640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43480cbb80 con 0x7f434806bfd0 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434e822640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f434815b390 con 0x7f434815afb0 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434e822640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43480cbd00 con 0x7f434806e2d0 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434806bfd0 0x7f43480b3ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434806bfd0 0x7f43480b3ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37770/0 (socket says 172.21.15.64:37770) 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 -- 172.21.15.64:0/130470979 learned_addr learned my addr 172.21.15.64:0/130470979 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434cdbf640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434815afb0 0x7f43481f4520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 -- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806e2d0 msgr2=0x7f43481f3fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f43477fe640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806e2d0 0x7f43481f3fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806e2d0 0x7f43481f3fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 -- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434815afb0 msgr2=0x7f43481f4520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434815afb0 0x7f43481f4520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 -- 172.21.15.64:0/130470979 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43480c59a0 con 0x7f434806bfd0 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f434cdbf640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434815afb0 0x7f43481f4520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.679+0000 7f4347fff640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434806bfd0 0x7f43480b3ae0 secure :-1 s=READY pgs=1203 cs=0 l=1 rev1=1 crypto rx=0x7f433802b950 tx=0x7f43380046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.680+0000 7f4344ff9640 1 -- 172.21.15.64:0/130470979 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4338040070 con 0x7f434806bfd0 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.680+0000 7f4344ff9640 1 -- 172.21.15.64:0/130470979 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f433803a420 con 0x7f434806bfd0 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.680+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f43480c5c10 con 0x7f434806bfd0 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.680+0000 7f4344ff9640 1 -- 172.21.15.64:0/130470979 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4338035040 con 0x7f434806bfd0 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.680+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f43480b4020 con 0x7f434806bfd0 2024-03-04T07:31:50.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.681+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4348067830 con 0x7f434806bfd0 2024-03-04T07:31:50.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.681+0000 7f4344ff9640 1 -- 172.21.15.64:0/130470979 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f433804b060 con 0x7f434806bfd0 2024-03-04T07:31:50.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.682+0000 7f4344ff9640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432806b4d0 0x7f432806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.682+0000 7f43477fe640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432806b4d0 0x7f432806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.682+0000 7f4344ff9640 1 -- 172.21.15.64:0/130470979 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f43380ac200 con 0x7f434806bfd0 2024-03-04T07:31:50.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.682+0000 7f43477fe640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432806b4d0 0x7f432806d990 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f4330006fd0 tx=0x7f4330008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.686+0000 7f4344ff9640 1 -- 172.21.15.64:0/130470979 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4348067830 con 0x7f434806bfd0 2024-03-04T07:31:50.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.869+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f4348068760 con 0x7f434806bfd0 2024-03-04T07:31:50.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.869+0000 7f4344ff9640 1 -- 172.21.15.64:0/130470979 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v94) v1 ==== 93+0+10 (secure 0 0 0) 0x7f433807afd0 con 0x7f434806bfd0 2024-03-04T07:31:50.869 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432806b4d0 msgr2=0x7f432806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432806b4d0 0x7f432806d990 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f4330006fd0 tx=0x7f4330008040 comp rx=0 tx=0).stop 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434806bfd0 msgr2=0x7f43480b3ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434806bfd0 0x7f43480b3ae0 secure :-1 s=READY pgs=1203 cs=0 l=1 rev1=1 crypto rx=0x7f433802b950 tx=0x7f43380046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 shutdown_connections 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f432806b4d0 0x7f432806d990 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f434815afb0 0x7f43481f4520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f434806e2d0 0x7f43481f3fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 --2- 172.21.15.64:0/130470979 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f434806bfd0 0x7f43480b3ae0 unknown :-1 s=CLOSED pgs=1203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 >> 172.21.15.64:0/130470979 conn(0x7f43480d0ec0 msgr2=0x7f43480b1c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:50.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 shutdown_connections 2024-03-04T07:31:50.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.872+0000 7f434e822640 1 -- 172.21.15.64:0/130470979 wait complete. 2024-03-04T07:31:50.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:31:50.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.991+0000 7f5a141e2640 1 Processor -- start 2024-03-04T07:31:50.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.991+0000 7f5a141e2640 1 -- start start 2024-03-04T07:31:50.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a141e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0d24f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a141e2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5a0c161c10 0x7f5a0c0d2a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a141e2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5a0c0d2100 con 0x7f5a0c0d0fc0 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a141e2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5a0c0b52c0 con 0x7f5a0c161c10 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a141e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a0c0d4bd0 con 0x7f5a0c161830 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a1177d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5a0c161c10 0x7f5a0c0d2a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a1277f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5a0c0d0fc0 0x7f5a0c0c7a60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58282/0 (socket says 172.21.15.64:58282) 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a1277f640 1 -- 172.21.15.64:0/3054468842 learned_addr learned my addr 172.21.15.64:0/3054468842 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a11f7e640 1 --2- 172.21.15.64:0/3054468842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0d24f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.992+0000 7f5a1177d640 1 -- 172.21.15.64:0/3054468842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 msgr2=0x7f5a0c0d24f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.993+0000 7f5a10f7c640 1 -- 172.21.15.64:0/3054468842 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1101460435 0 0) 0x7f5a0c0d2100 con 0x7f5a0c0d0fc0 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.993+0000 7f5a1177d640 1 --2- 172.21.15.64:0/3054468842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0d24f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.993+0000 7f5a1177d640 1 -- 172.21.15.64:0/3054468842 >> v1:172.21.15.64:6789/0 conn(0x7f5a0c0d0fc0 legacy=0x7f5a0c0c7a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.993+0000 7f5a1177d640 1 -- 172.21.15.64:0/3054468842 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a0c0b5000 con 0x7f5a0c161c10 2024-03-04T07:31:50.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.993+0000 7f5a11f7e640 1 --2- 172.21.15.64:0/3054468842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0d24f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.993+0000 7f5a1177d640 1 --2- 172.21.15.64:0/3054468842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c161c10 0x7f5a0c0d2a30 secure :-1 s=READY pgs=1205 cs=0 l=1 rev1=1 crypto rx=0x7f5a080097a0 tx=0x7f5a0802b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a5863d710a414ca4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.993+0000 7f5a10f7c640 1 -- 172.21.15.64:0/3054468842 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a080315a0 con 0x7f5a0c161c10 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a10f7c640 1 -- 172.21.15.64:0/3054468842 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a08037070 con 0x7f5a0c161c10 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a10f7c640 1 -- 172.21.15.64:0/3054468842 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a08031960 con 0x7f5a0c161c10 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 -- 172.21.15.64:0/3054468842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c161c10 msgr2=0x7f5a0c0d2a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 --2- 172.21.15.64:0/3054468842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c161c10 0x7f5a0c0d2a30 secure :-1 s=READY pgs=1205 cs=0 l=1 rev1=1 crypto rx=0x7f5a080097a0 tx=0x7f5a0802b840 comp rx=0 tx=0).stop 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 -- 172.21.15.64:0/3054468842 shutdown_connections 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 --2- 172.21.15.64:0/3054468842 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c161c10 0x7f5a0c0d2a30 unknown :-1 s=CLOSED pgs=1205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 --2- 172.21.15.64:0/3054468842 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0d24f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 -- 172.21.15.64:0/3054468842 >> 172.21.15.64:0/3054468842 conn(0x7f5a0c0d3f10 msgr2=0x7f5a0c0d4320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:50.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 -- 172.21.15.64:0/3054468842 shutdown_connections 2024-03-04T07:31:50.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.994+0000 7f5a141e2640 1 -- 172.21.15.64:0/3054468842 wait complete. 2024-03-04T07:31:50.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.995+0000 7f5a141e2640 1 Processor -- start 2024-03-04T07:31:50.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.995+0000 7f5a141e2640 1 -- start start 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a141e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a0c0d0fc0 0x7f5a0c0b3c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a141e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0b2100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a141e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c0cd770 0x7f5a0c1f4570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a141e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a0c0b55d0 con 0x7f5a0c0cd770 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a141e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a0c0cfb30 con 0x7f5a0c0d0fc0 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a141e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a0c0b42a0 con 0x7f5a0c161830 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c0cd770 0x7f5a0c1f4570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c0cd770 0x7f5a0c1f4570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37786/0 (socket says 172.21.15.64:37786) 2024-03-04T07:31:50.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 -- 172.21.15.64:0/1551297166 learned_addr learned my addr 172.21.15.64:0/1551297166 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a11f7e640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a0c0d0fc0 0x7f5a0c0b3c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1177d640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0b2100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 -- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 msgr2=0x7f5a0c0b2100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0b2100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 -- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a0c0d0fc0 msgr2=0x7f5a0c0b3c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a0c0d0fc0 0x7f5a0c0b3c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1277f640 1 -- 172.21.15.64:0/1551297166 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a0c0b4580 con 0x7f5a0c0cd770 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a11f7e640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a0c0d0fc0 0x7f5a0c0b3c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.996+0000 7f5a1177d640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0b2100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.997+0000 7f5a1277f640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c0cd770 0x7f5a0c1f4570 secure :-1 s=READY pgs=1206 cs=0 l=1 rev1=1 crypto rx=0x7f5a0400c7a0 tx=0x7f5a0400cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.997+0000 7f5a027fc640 1 -- 172.21.15.64:0/1551297166 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a04018070 con 0x7f5a0c0cd770 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.997+0000 7f5a027fc640 1 -- 172.21.15.64:0/1551297166 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a0400a640 con 0x7f5a0c0cd770 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.997+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5a0c0b4860 con 0x7f5a0c0cd770 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.997+0000 7f5a027fc640 1 -- 172.21.15.64:0/1551297166 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a0400aab0 con 0x7f5a0c0cd770 2024-03-04T07:31:50.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.997+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5a0c1f3d40 con 0x7f5a0c0cd770 2024-03-04T07:31:51.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.998+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5a0c0b2700 con 0x7f5a0c0cd770 2024-03-04T07:31:51.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.998+0000 7f5a027fc640 1 -- 172.21.15.64:0/1551297166 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5a0401b070 con 0x7f5a0c0cd770 2024-03-04T07:31:51.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.999+0000 7f5a027fc640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59f006b4d0 0x7f59f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.999+0000 7f5a027fc640 1 -- 172.21.15.64:0/1551297166 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f5a040898e0 con 0x7f5a0c0cd770 2024-03-04T07:31:51.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.999+0000 7f5a11f7e640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59f006b4d0 0x7f59f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:50.999+0000 7f5a11f7e640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59f006b4d0 0x7f59f006d990 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f5a0c0d3960 tx=0x7f59fc008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:51.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.003+0000 7f5a027fc640 1 -- 172.21.15.64:0/1551297166 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5a040586b0 con 0x7f5a0c0cd770 2024-03-04T07:31:51.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.185+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f5a0c068050 con 0x7f5a0c0cd770 2024-03-04T07:31:51.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.185+0000 7f5a027fc640 1 -- 172.21.15.64:0/1551297166 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v94) v1 ==== 104+0+11 (secure 0 0 0) 0x7f5a04057fb0 con 0x7f5a0c0cd770 2024-03-04T07:31:51.185 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59f006b4d0 msgr2=0x7f59f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59f006b4d0 0x7f59f006d990 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f5a0c0d3960 tx=0x7f59fc008040 comp rx=0 tx=0).stop 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c0cd770 msgr2=0x7f5a0c1f4570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c0cd770 0x7f5a0c1f4570 secure :-1 s=READY pgs=1206 cs=0 l=1 rev1=1 crypto rx=0x7f5a0400c7a0 tx=0x7f5a0400cc60 comp rx=0 tx=0).stop 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 shutdown_connections 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59f006b4d0 0x7f59f006d990 unknown :-1 s=CLOSED pgs=547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a0c0cd770 0x7f5a0c1f4570 unknown :-1 s=CLOSED pgs=1206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a0c161830 0x7f5a0c0b2100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 --2- 172.21.15.64:0/1551297166 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a0c0d0fc0 0x7f5a0c0b3c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 >> 172.21.15.64:0/1551297166 conn(0x7f5a0c0d3f10 msgr2=0x7f5a0c15e590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:51.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 shutdown_connections 2024-03-04T07:31:51.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.187+0000 7f5a141e2640 1 -- 172.21.15.64:0/1551297166 wait complete. 2024-03-04T07:31:51.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:31:51.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.308+0000 7f2334b3a640 1 Processor -- start 2024-03-04T07:31:51.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.308+0000 7f2334b3a640 1 -- start start 2024-03-04T07:31:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f2334b3a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f23300d2bb0 0x7f23300d49e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f2334b3a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d1d10 0x7f23300b46e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f2334b3a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f23300b0af0 con 0x7f233015e2d0 2024-03-04T07:31:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f2334b3a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f23300b3380 con 0x7f23300d2bb0 2024-03-04T07:31:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f2334b3a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f23300b3640 con 0x7f23300d1d10 2024-03-04T07:31:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232e59c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f233015e2d0 0x7f23300d24a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43678/0 (socket says 172.21.15.64:43678) 2024-03-04T07:31:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232e59c640 1 -- 172.21.15.64:0/3759612413 learned_addr learned my addr 172.21.15.64:0/3759612413 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232ed9d640 1 --2- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d1d10 0x7f23300b46e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232dd9b640 1 --2- 172.21.15.64:0/3759612413 >> v2:172.21.15.66:3300/0 conn(0x7f23300d2bb0 0x7f23300d49e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232d59a640 1 -- 172.21.15.64:0/3759612413 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4183273695 0 0) 0x7f23300b0af0 con 0x7f233015e2d0 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232d59a640 1 -- 172.21.15.64:0/3759612413 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2318003520 con 0x7f233015e2d0 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232dd9b640 1 -- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d1d10 msgr2=0x7f23300b46e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232dd9b640 1 --2- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d1d10 0x7f23300b46e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232dd9b640 1 -- 172.21.15.64:0/3759612413 >> v1:172.21.15.66:6789/0 conn(0x7f233015e2d0 legacy=0x7f23300d24a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232dd9b640 1 -- 172.21.15.64:0/3759612413 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f23300b3920 con 0x7f23300d2bb0 2024-03-04T07:31:51.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.310+0000 7f232ed9d640 1 --2- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d1d10 0x7f23300b46e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:51.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.311+0000 7f232dd9b640 1 --2- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d2bb0 0x7f23300d49e0 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f23240097a0 tx=0x7f232402b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1bc56f690f154d7d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:51.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.311+0000 7f232d59a640 1 -- 172.21.15.64:0/3759612413 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2324035020 con 0x7f23300d2bb0 2024-03-04T07:31:51.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.311+0000 7f232d59a640 1 -- 172.21.15.64:0/3759612413 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2324031830 con 0x7f23300d2bb0 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.311+0000 7f232d59a640 1 -- 172.21.15.64:0/3759612413 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2324031be0 con 0x7f23300d2bb0 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 -- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d2bb0 msgr2=0x7f23300d49e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 --2- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d2bb0 0x7f23300d49e0 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f23240097a0 tx=0x7f232402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 -- 172.21.15.64:0/3759612413 shutdown_connections 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 --2- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d1d10 0x7f23300b46e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 --2- 172.21.15.64:0/3759612413 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d2bb0 0x7f23300d49e0 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 -- 172.21.15.64:0/3759612413 >> 172.21.15.64:0/3759612413 conn(0x7f23300d3ec0 msgr2=0x7f23300d42d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 -- 172.21.15.64:0/3759612413 shutdown_connections 2024-03-04T07:31:51.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.312+0000 7f2334b3a640 1 -- 172.21.15.64:0/3759612413 wait complete. 2024-03-04T07:31:51.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.314+0000 7f2334b3a640 1 Processor -- start 2024-03-04T07:31:51.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.314+0000 7f2334b3a640 1 -- start start 2024-03-04T07:31:51.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.315+0000 7f2334b3a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d1d10 0x7f233015c690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.315+0000 7f2334b3a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d2bb0 0x7f23300b8390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.315+0000 7f2334b3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f233015e2d0 0x7f23300b88d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.315+0000 7f2334b3a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f23300b41c0 con 0x7f233015e2d0 2024-03-04T07:31:51.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.315+0000 7f2334b3a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f23300b3ee0 con 0x7f23300d1d10 2024-03-04T07:31:51.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.315+0000 7f2334b3a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f233015e6b0 con 0x7f23300d2bb0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f233015e2d0 0x7f23300b88d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f233015e2d0 0x7f23300b88d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37802/0 (socket says 172.21.15.64:37802) 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 -- 172.21.15.64:0/1306676698 learned_addr learned my addr 172.21.15.64:0/1306676698 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232e59c640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d1d10 0x7f233015c690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232dd9b640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d2bb0 0x7f23300b8390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 -- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d2bb0 msgr2=0x7f23300b8390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d2bb0 0x7f23300b8390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 -- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d1d10 msgr2=0x7f233015c690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d1d10 0x7f233015c690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 -- 172.21.15.64:0/1306676698 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f233015cbd0 con 0x7f233015e2d0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f232ed9d640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f233015e2d0 0x7f23300b88d0 secure :-1 s=READY pgs=1207 cs=0 l=1 rev1=1 crypto rx=0x7f232800efb0 tx=0x7f232800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f231effd640 1 -- 172.21.15.64:0/1306676698 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2328018020 con 0x7f233015e2d0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f231effd640 1 -- 172.21.15.64:0/1306676698 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2328014480 con 0x7f233015e2d0 2024-03-04T07:31:51.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f23300b7b20 con 0x7f233015e2d0 2024-03-04T07:31:51.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f231effd640 1 -- 172.21.15.64:0/1306676698 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2328014850 con 0x7f233015e2d0 2024-03-04T07:31:51.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.316+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f233015ea80 con 0x7f233015e2d0 2024-03-04T07:31:51.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.318+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f23300b8040 con 0x7f233015e2d0 2024-03-04T07:31:51.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.318+0000 7f231effd640 1 -- 172.21.15.64:0/1306676698 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2328005d10 con 0x7f233015e2d0 2024-03-04T07:31:51.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.318+0000 7f231effd640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231006b4d0 0x7f231006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.318+0000 7f232e59c640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231006b4d0 0x7f231006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.318+0000 7f231effd640 1 -- 172.21.15.64:0/1306676698 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f232808b080 con 0x7f233015e2d0 2024-03-04T07:31:51.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.319+0000 7f232e59c640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231006b4d0 0x7f231006d990 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f2320005880 tx=0x7f2320009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:51.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.323+0000 7f231effd640 1 -- 172.21.15.64:0/1306676698 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f23300b8040 con 0x7f233015e2d0 2024-03-04T07:31:51.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.580+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f2330068050 con 0x7f233015e2d0 2024-03-04T07:31:51.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.580+0000 7f231effd640 1 -- 172.21.15.64:0/1306676698 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v94) v1 ==== 100+0+11 (secure 0 0 0) 0x7f2328022030 con 0x7f233015e2d0 2024-03-04T07:31:51.581 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.582+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231006b4d0 msgr2=0x7f231006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.582+0000 7f2334b3a640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231006b4d0 0x7f231006d990 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f2320005880 tx=0x7f2320009040 comp rx=0 tx=0).stop 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f233015e2d0 msgr2=0x7f23300b88d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f233015e2d0 0x7f23300b88d0 secure :-1 s=READY pgs=1207 cs=0 l=1 rev1=1 crypto rx=0x7f232800efb0 tx=0x7f232800c490 comp rx=0 tx=0).stop 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 shutdown_connections 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231006b4d0 0x7f231006d990 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f233015e2d0 0x7f23300b88d0 unknown :-1 s=CLOSED pgs=1207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23300d2bb0 0x7f23300b8390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 --2- 172.21.15.64:0/1306676698 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f23300d1d10 0x7f233015c690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 >> 172.21.15.64:0/1306676698 conn(0x7f23300d3ec0 msgr2=0x7f23300d24a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 shutdown_connections 2024-03-04T07:31:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.583+0000 7f2334b3a640 1 -- 172.21.15.64:0/1306676698 wait complete. 2024-03-04T07:31:51.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:31:51.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f807dd13640 1 Processor -- start 2024-03-04T07:31:51.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f807dd13640 1 -- start start 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f807dd13640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f80780ceae0 0x7f80780cfa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f807dd13640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f80780b2f70 0x7f80780cffc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f807dd13640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f807815c6c0 con 0x7f807815c2e0 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f807dd13640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f80780cd7a0 con 0x7f80780ceae0 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f807dd13640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f80780cda80 con 0x7f80780b2f70 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f8076ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f80780ceae0 0x7f80780cfa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.697+0000 7f8076ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f80780ceae0 0x7f80780cfa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37816/0 (socket says 172.21.15.64:37816) 2024-03-04T07:31:51.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8076ffd640 1 -- 172.21.15.64:0/3311433595 learned_addr learned my addr 172.21.15.64:0/3311433595 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8077fff640 1 --2- 172.21.15.64:0/3311433595 >> v2:172.21.15.66:3300/0 conn(0x7f80780b2f70 0x7f80780cffc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8076ffd640 1 -- 172.21.15.64:0/3311433595 >> v2:172.21.15.66:3300/0 conn(0x7f80780b2f70 msgr2=0x7f80780cffc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8076ffd640 1 --2- 172.21.15.64:0/3311433595 >> v2:172.21.15.66:3300/0 conn(0x7f80780b2f70 0x7f80780cffc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8076ffd640 1 -- 172.21.15.64:0/3311433595 >> v1:172.21.15.66:6789/0 conn(0x7f807815c2e0 legacy=0x7f80780ce3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8076ffd640 1 -- 172.21.15.64:0/3311433595 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80780b1760 con 0x7f80780ceae0 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8077fff640 1 --2- 172.21.15.64:0/3311433595 >> v2:172.21.15.66:3300/0 conn(0x7f80780b2f70 0x7f80780cffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.698+0000 7f8076ffd640 1 --2- 172.21.15.64:0/3311433595 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780cfa80 secure :-1 s=READY pgs=1208 cs=0 l=1 rev1=1 crypto rx=0x7f8078073790 tx=0x7f806c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=74220c00fdfdbaa8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f80767fc640 1 -- 172.21.15.64:0/3311433595 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f806c035020 con 0x7f80780ceae0 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f80767fc640 1 -- 172.21.15.64:0/3311433595 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f806c031870 con 0x7f80780ceae0 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f80767fc640 1 -- 172.21.15.64:0/3311433595 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f806c031c40 con 0x7f80780ceae0 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 -- 172.21.15.64:0/3311433595 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 msgr2=0x7f80780cfa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 --2- 172.21.15.64:0/3311433595 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780cfa80 secure :-1 s=READY pgs=1208 cs=0 l=1 rev1=1 crypto rx=0x7f8078073790 tx=0x7f806c02b910 comp rx=0 tx=0).stop 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 -- 172.21.15.64:0/3311433595 shutdown_connections 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 --2- 172.21.15.64:0/3311433595 >> v2:172.21.15.66:3300/0 conn(0x7f80780b2f70 0x7f80780cffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 --2- 172.21.15.64:0/3311433595 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780cfa80 unknown :-1 s=CLOSED pgs=1208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 -- 172.21.15.64:0/3311433595 >> 172.21.15.64:0/3311433595 conn(0x7f80780d1ca0 msgr2=0x7f80780d20b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:51.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 -- 172.21.15.64:0/3311433595 shutdown_connections 2024-03-04T07:31:51.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.699+0000 7f807dd13640 1 -- 172.21.15.64:0/3311433595 wait complete. 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.700+0000 7f807dd13640 1 Processor -- start 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f807dd13640 1 -- start start 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f807dd13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80780b2f70 0x7f807814ca70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f807dd13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780b68d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f807dd13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f807815c2e0 0x7f80780b6e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f807dd13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80780cd540 con 0x7f80780ceae0 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f807dd13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80780cd3c0 con 0x7f807815c2e0 2024-03-04T07:31:51.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f807dd13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80780d27c0 con 0x7f80780b2f70 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f8076ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780b68d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f8076ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780b68d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37830/0 (socket says 172.21.15.64:37830) 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.701+0000 7f8076ffd640 1 -- 172.21.15.64:0/1186758214 learned_addr learned my addr 172.21.15.64:0/1186758214 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f8076ffd640 1 -- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80780b2f70 msgr2=0x7f807814ca70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f80777fe640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80780b2f70 0x7f807814ca70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f8077fff640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f807815c2e0 0x7f80780b6e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f8076ffd640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80780b2f70 0x7f807814ca70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f8076ffd640 1 -- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f807815c2e0 msgr2=0x7f80780b6e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f8076ffd640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f807815c2e0 0x7f80780b6e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f8076ffd640 1 -- 172.21.15.64:0/1186758214 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80781500b0 con 0x7f80780ceae0 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f8076ffd640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780b68d0 secure :-1 s=READY pgs=1209 cs=0 l=1 rev1=1 crypto rx=0x7f806c000c00 tx=0x7f806c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f805ffff640 1 -- 172.21.15.64:0/1186758214 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f806c040070 con 0x7f80780ceae0 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f805ffff640 1 -- 172.21.15.64:0/1186758214 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f806c03a420 con 0x7f80780ceae0 2024-03-04T07:31:51.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80781f3cf0 con 0x7f80780ceae0 2024-03-04T07:31:51.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f805ffff640 1 -- 172.21.15.64:0/1186758214 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f806c035040 con 0x7f80780ceae0 2024-03-04T07:31:51.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.702+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80781f42b0 con 0x7f80780ceae0 2024-03-04T07:31:51.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.703+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f80780cf280 con 0x7f80780ceae0 2024-03-04T07:31:51.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.704+0000 7f805ffff640 1 -- 172.21.15.64:0/1186758214 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f806c04b060 con 0x7f80780ceae0 2024-03-04T07:31:51.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.704+0000 7f805ffff640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f806406b4d0 0x7f806406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:51.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.704+0000 7f805ffff640 1 -- 172.21.15.64:0/1186758214 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f806c0ac250 con 0x7f80780ceae0 2024-03-04T07:31:51.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.704+0000 7f80777fe640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f806406b4d0 0x7f806406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:51.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.705+0000 7f80777fe640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f806406b4d0 0x7f806406d990 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f8068006fd0 tx=0x7f8068008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:51.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:51.709+0000 7f805ffff640 1 -- 172.21.15.64:0/1186758214 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f80780cf280 con 0x7f80780ceae0 2024-03-04T07:31:52.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.070+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f80780cefa0 con 0x7f80780ceae0 2024-03-04T07:31:52.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.071+0000 7f805ffff640 1 -- 172.21.15.64:0/1186758214 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 4 entries v94) v1 ==== 80+0+244 (secure 0 0 0) 0x7f806c07b020 con 0x7f80780ceae0 2024-03-04T07:31:52.072 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.64:6837/3848675976 2024-03-05T07:31:24.148036+0000 2024-03-04T07:31:52.072 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.64:6835/3848675976 2024-03-05T07:31:24.148036+0000 2024-03-04T07:31:52.072 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.66:6835/3397672669 2024-03-05T07:31:23.154858+0000 2024-03-04T07:31:52.072 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.66:6833/3397672669 2024-03-05T07:31:23.154858+0000 2024-03-04T07:31:52.072 INFO:teuthology.orchestra.run.smithi064.stderr:listed 4 entries 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f806406b4d0 msgr2=0x7f806406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f806406b4d0 0x7f806406d990 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f8068006fd0 tx=0x7f8068008040 comp rx=0 tx=0).stop 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 msgr2=0x7f80780b68d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780b68d0 secure :-1 s=READY pgs=1209 cs=0 l=1 rev1=1 crypto rx=0x7f806c000c00 tx=0x7f806c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 shutdown_connections 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f806406b4d0 0x7f806406d990 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f807815c2e0 0x7f80780b6e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80780ceae0 0x7f80780b68d0 unknown :-1 s=CLOSED pgs=1209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 --2- 172.21.15.64:0/1186758214 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80780b2f70 0x7f807814ca70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 >> 172.21.15.64:0/1186758214 conn(0x7f80780d1ca0 msgr2=0x7f80780b02c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:52.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 shutdown_connections 2024-03-04T07:31:52.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.073+0000 7f807dd13640 1 -- 172.21.15.64:0/1186758214 wait complete. 2024-03-04T07:31:52.084 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:31:52.084 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:52.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:31:52.109 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[68753]: fuse finished with error 0 and tester_r 0 2024-03-04T07:31:52.111 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:31:52.111 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:52.111 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:31:52.280 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:31:52.280 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:52.297 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:31:52.297 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:31:52.297 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:52.298 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:31:52.353 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:31:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:31:52.354 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:31:52.355 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:52.369 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:31:52.369 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:31:52.369 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:52.369 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:31:52.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:31:52.424 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:31:52.425 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:31:52.425 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:31:52.442 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:31:52.442 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:31:52.442 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:31:52.443 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:31:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:31:52.499 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:31:52.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:52.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.600+0000 7fb42a84e640 1 Processor -- start 2024-03-04T07:31:52.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.600+0000 7fb42a84e640 1 -- start start 2024-03-04T07:31:52.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb42a84e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb42415e100 0x7fb4240d1cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb42a84e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4240d4c40 0x7fb4240d2320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb42a84e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb4240d0100 con 0x7fb42415e4e0 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb42a84e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4240cf860 con 0x7fb4240d4c40 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb42a84e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4240cfb40 con 0x7fb42415e100 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb4237fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4240d4c40 0x7fb4240d2320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb4237fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4240d4c40 0x7fb4240d2320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37836/0 (socket says 172.21.15.64:37836) 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb4237fe640 1 -- 172.21.15.64:0/2894310296 learned_addr learned my addr 172.21.15.64:0/2894310296 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb423fff640 1 --2- 172.21.15.64:0/2894310296 >> v2:172.21.15.66:3300/0 conn(0x7fb42415e100 0x7fb4240d1cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb4237fe640 1 -- 172.21.15.64:0/2894310296 >> v2:172.21.15.66:3300/0 conn(0x7fb42415e100 msgr2=0x7fb4240d1cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb4237fe640 1 --2- 172.21.15.64:0/2894310296 >> v2:172.21.15.66:3300/0 conn(0x7fb42415e100 0x7fb4240d1cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb4237fe640 1 -- 172.21.15.64:0/2894310296 >> v1:172.21.15.66:6789/0 conn(0x7fb42415e4e0 legacy=0x7fb4240d0650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.601+0000 7fb4237fe640 1 -- 172.21.15.64:0/2894310296 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4240cfe20 con 0x7fb4240d4c40 2024-03-04T07:31:52.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.602+0000 7fb4237fe640 1 --2- 172.21.15.64:0/2894310296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4240d4c40 0x7fb4240d2320 secure :-1 s=READY pgs=1210 cs=0 l=1 rev1=1 crypto rx=0x7fb42408e3a0 tx=0x7fb41802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=35bf94f09298d973 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.602+0000 7fb422ffd640 1 -- 172.21.15.64:0/2894310296 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb418034070 con 0x7fb4240d4c40 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.602+0000 7fb422ffd640 1 -- 172.21.15.64:0/2894310296 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb418030860 con 0x7fb4240d4c40 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.602+0000 7fb422ffd640 1 -- 172.21.15.64:0/2894310296 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb418030c30 con 0x7fb4240d4c40 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 -- 172.21.15.64:0/2894310296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4240d4c40 msgr2=0x7fb4240d2320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 --2- 172.21.15.64:0/2894310296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4240d4c40 0x7fb4240d2320 secure :-1 s=READY pgs=1210 cs=0 l=1 rev1=1 crypto rx=0x7fb42408e3a0 tx=0x7fb41802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 -- 172.21.15.64:0/2894310296 shutdown_connections 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 --2- 172.21.15.64:0/2894310296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4240d4c40 0x7fb4240d2320 unknown :-1 s=CLOSED pgs=1210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 --2- 172.21.15.64:0/2894310296 >> v2:172.21.15.66:3300/0 conn(0x7fb42415e100 0x7fb4240d1cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 -- 172.21.15.64:0/2894310296 >> 172.21.15.64:0/2894310296 conn(0x7fb4240d3ef0 msgr2=0x7fb4240d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 -- 172.21.15.64:0/2894310296 shutdown_connections 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.603+0000 7fb42a84e640 1 -- 172.21.15.64:0/2894310296 wait complete. 2024-03-04T07:31:52.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.604+0000 7fb42a84e640 1 Processor -- start 2024-03-04T07:31:52.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.604+0000 7fb42a84e640 1 -- start start 2024-03-04T07:31:52.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb42a84e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4240d4c40 0x7fb4241535c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb42a84e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb42415e100 0x7fb42415f600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb42a84e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb42415e4e0 0x7fb424156b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb42a84e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4240b2080 con 0x7fb42415e100 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb42a84e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4240cf510 con 0x7fb42415e4e0 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb42a84e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4240b2200 con 0x7fb4240d4c40 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb42415e100 0x7fb42415f600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb42415e100 0x7fb42415f600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37842/0 (socket says 172.21.15.64:37842) 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 -- 172.21.15.64:0/895316750 learned_addr learned my addr 172.21.15.64:0/895316750 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb428deb640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb42415e4e0 0x7fb424156b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb423fff640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4240d4c40 0x7fb4241535c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 -- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4240d4c40 msgr2=0x7fb4241535c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4240d4c40 0x7fb4241535c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 -- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb42415e4e0 msgr2=0x7fb424156b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb42415e4e0 0x7fb424156b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.605+0000 7fb4237fe640 1 -- 172.21.15.64:0/895316750 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb424157040 con 0x7fb42415e100 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.606+0000 7fb4237fe640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb42415e100 0x7fb42415f600 secure :-1 s=READY pgs=1211 cs=0 l=1 rev1=1 crypto rx=0x7fb418000c00 tx=0x7fb418004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:52.608 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.606+0000 7fb420ff9640 1 -- 172.21.15.64:0/895316750 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb418048070 con 0x7fb42415e100 2024-03-04T07:31:52.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.606+0000 7fb420ff9640 1 -- 172.21.15.64:0/895316750 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb418030660 con 0x7fb42415e100 2024-03-04T07:31:52.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.606+0000 7fb420ff9640 1 -- 172.21.15.64:0/895316750 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb418034040 con 0x7fb42415e100 2024-03-04T07:31:52.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.606+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb424154be0 con 0x7fb42415e100 2024-03-04T07:31:52.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.606+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb4240cc490 con 0x7fb42415e100 2024-03-04T07:31:52.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.607+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb4240752d0 con 0x7fb42415e100 2024-03-04T07:31:52.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.607+0000 7fb420ff9640 1 -- 172.21.15.64:0/895316750 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb41804b070 con 0x7fb42415e100 2024-03-04T07:31:52.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.608+0000 7fb420ff9640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb41006b4d0 0x7fb41006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.608+0000 7fb420ff9640 1 -- 172.21.15.64:0/895316750 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fb4180ac570 con 0x7fb42415e100 2024-03-04T07:31:52.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.608+0000 7fb423fff640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb41006b4d0 0x7fb41006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.608+0000 7fb423fff640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb41006b4d0 0x7fb41006d990 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7fb414006fd0 tx=0x7fb414008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:52.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.612+0000 7fb420ff9640 1 -- 172.21.15.64:0/895316750 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb41807b4f0 con 0x7fb42415e100 2024-03-04T07:31:52.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.832+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb4240d1510 con 0x7fb42415e100 2024-03-04T07:31:52.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.833+0000 7fb420ff9640 1 -- 172.21.15.64:0/895316750 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7fb41807adf0 con 0x7fb42415e100 2024-03-04T07:31:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:52.834 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:52.834 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb41006b4d0 msgr2=0x7fb41006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb41006b4d0 0x7fb41006d990 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7fb414006fd0 tx=0x7fb414008040 comp rx=0 tx=0).stop 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb42415e100 msgr2=0x7fb42415f600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb42415e100 0x7fb42415f600 secure :-1 s=READY pgs=1211 cs=0 l=1 rev1=1 crypto rx=0x7fb418000c00 tx=0x7fb418004730 comp rx=0 tx=0).stop 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 shutdown_connections 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb41006b4d0 0x7fb41006d990 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb42415e4e0 0x7fb424156b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb42415e100 0x7fb42415f600 unknown :-1 s=CLOSED pgs=1211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 --2- 172.21.15.64:0/895316750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4240d4c40 0x7fb4241535c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 >> 172.21.15.64:0/895316750 conn(0x7fb4240d3ef0 msgr2=0x7fb4240d02f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:52.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 shutdown_connections 2024-03-04T07:31:52.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.835+0000 7fb42a84e640 1 -- 172.21.15.64:0/895316750 wait complete. 2024-03-04T07:31:52.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:52.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.949+0000 7fa5a9202640 1 Processor -- start 2024-03-04T07:31:52.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.949+0000 7fa5a9202640 1 -- start start 2024-03-04T07:31:52.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a9202640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a40cea80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a9202640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa5a40d0780 0x7fa5a40b1920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a9202640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa5a40d2750 con 0x7fa5a415c040 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a9202640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa5a40b1120 con 0x7fa5a40d0780 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a9202640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa5a40b1400 con 0x7fa5a40cf970 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a359e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa5a40d0780 0x7fa5a40b1920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a359e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa5a40d0780 0x7fa5a40b1920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37850/0 (socket says 172.21.15.64:37850) 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a359e640 1 -- 172.21.15.64:0/3532777911 learned_addr learned my addr 172.21.15.64:0/3532777911 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa59bfff640 1 --2- 172.21.15.64:0/3532777911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a40cea80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a359e640 1 -- 172.21.15.64:0/3532777911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 msgr2=0x7fa5a40cea80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a359e640 1 --2- 172.21.15.64:0/3532777911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a40cea80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a359e640 1 -- 172.21.15.64:0/3532777911 >> v1:172.21.15.64:6789/0 conn(0x7fa5a415c040 legacy=0x7fa5a40d0070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.950+0000 7fa5a359e640 1 -- 172.21.15.64:0/3532777911 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa5a40cd6b0 con 0x7fa5a40d0780 2024-03-04T07:31:52.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.951+0000 7fa5a359e640 1 --2- 172.21.15.64:0/3532777911 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a40d0780 0x7fa5a40b1920 secure :-1 s=READY pgs=1213 cs=0 l=1 rev1=1 crypto rx=0x7fa59c009e50 tx=0x7fa59c02bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=92b5f739061e0382 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:52.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.951+0000 7fa5a259c640 1 -- 172.21.15.64:0/3532777911 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa59c02cb00 con 0x7fa5a40d0780 2024-03-04T07:31:52.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.951+0000 7fa5a259c640 1 -- 172.21.15.64:0/3532777911 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa59c02cca0 con 0x7fa5a40d0780 2024-03-04T07:31:52.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.951+0000 7fa5a259c640 1 -- 172.21.15.64:0/3532777911 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa59c034610 con 0x7fa5a40d0780 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 -- 172.21.15.64:0/3532777911 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a40d0780 msgr2=0x7fa5a40b1920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 --2- 172.21.15.64:0/3532777911 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a40d0780 0x7fa5a40b1920 secure :-1 s=READY pgs=1213 cs=0 l=1 rev1=1 crypto rx=0x7fa59c009e50 tx=0x7fa59c02bf60 comp rx=0 tx=0).stop 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 -- 172.21.15.64:0/3532777911 shutdown_connections 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 --2- 172.21.15.64:0/3532777911 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a40d0780 0x7fa5a40b1920 unknown :-1 s=CLOSED pgs=1213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 --2- 172.21.15.64:0/3532777911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a40cea80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 -- 172.21.15.64:0/3532777911 >> 172.21.15.64:0/3532777911 conn(0x7fa5a40d1a90 msgr2=0x7fa5a40d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 -- 172.21.15.64:0/3532777911 shutdown_connections 2024-03-04T07:31:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.952+0000 7fa5a9202640 1 -- 172.21.15.64:0/3532777911 wait complete. 2024-03-04T07:31:52.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.953+0000 7fa5a9202640 1 Processor -- start 2024-03-04T07:31:52.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.954+0000 7fa5a9202640 1 -- start start 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a9202640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a4157de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a9202640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5a40d0780 0x7fa5a415d420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a9202640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a415c040 0x7fa5a415d960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a9202640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa5a40b21f0 con 0x7fa5a415c040 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a9202640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa5a40b2070 con 0x7fa5a40d0780 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a9202640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa5a40b2370 con 0x7fa5a40cf970 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a415c040 0x7fa5a415d960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a415c040 0x7fa5a415d960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37854/0 (socket says 172.21.15.64:37854) 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 -- 172.21.15.64:0/1277680336 learned_addr learned my addr 172.21.15.64:0/1277680336 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa59bfff640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5a40d0780 0x7fa5a415d420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a2d9d640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a4157de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 -- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 msgr2=0x7fa5a4157de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a4157de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 -- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5a40d0780 msgr2=0x7fa5a415d420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5a40d0780 0x7fa5a415d420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 -- 172.21.15.64:0/1277680336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa5a41596b0 con 0x7fa5a415c040 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa59bfff640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5a40d0780 0x7fa5a415d420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.955+0000 7fa5a359e640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a415c040 0x7fa5a415d960 secure :-1 s=READY pgs=1214 cs=0 l=1 rev1=1 crypto rx=0x7fa59c036040 tx=0x7fa59c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.956+0000 7fa59b7fe640 1 -- 172.21.15.64:0/1277680336 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa59c043020 con 0x7fa5a415c040 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.956+0000 7fa59b7fe640 1 -- 172.21.15.64:0/1277680336 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa59c02ce50 con 0x7fa5a415c040 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.956+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa5a4158610 con 0x7fa5a415c040 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.956+0000 7fa59b7fe640 1 -- 172.21.15.64:0/1277680336 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa59c03e670 con 0x7fa5a415c040 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.956+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa5a4158bd0 con 0x7fa5a415c040 2024-03-04T07:31:52.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.957+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa5a4075260 con 0x7fa5a415c040 2024-03-04T07:31:52.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.958+0000 7fa59b7fe640 1 -- 172.21.15.64:0/1277680336 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa59c046020 con 0x7fa5a415c040 2024-03-04T07:31:52.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.958+0000 7fa59b7fe640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa58406b4d0 0x7fa58406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:52.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.958+0000 7fa59b7fe640 1 -- 172.21.15.64:0/1277680336 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fa59c0aca40 con 0x7fa5a415c040 2024-03-04T07:31:52.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.958+0000 7fa5a2d9d640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa58406b4d0 0x7fa58406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:52.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.959+0000 7fa5a2d9d640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa58406b4d0 0x7fa58406d990 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7fa594007fd0 tx=0x7fa594009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:52.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:52.962+0000 7fa59b7fe640 1 -- 172.21.15.64:0/1277680336 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa59c04d030 con 0x7fa5a415c040 2024-03-04T07:31:53.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.148+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa5a40ceda0 con 0x7fa5a415c040 2024-03-04T07:31:53.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.149+0000 7fa59b7fe640 1 -- 172.21.15.64:0/1277680336 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) v1 ==== 74+0+16819 (secure 0 0 0) 0x7fa59c07b810 con 0x7fa5a415c040 2024-03-04T07:31:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:53.150 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":94,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:24.148060+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:31:12.750651+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-04T07:31:13.444529+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"90","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.64:6835/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.66:6835/3397672669":"2024-03-05T07:31:23.154858+0000","172.21.15.66:6833/3397672669":"2024-03-05T07:31:23.154858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa58406b4d0 msgr2=0x7fa58406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa58406b4d0 0x7fa58406d990 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7fa594007fd0 tx=0x7fa594009040 comp rx=0 tx=0).stop 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a415c040 msgr2=0x7fa5a415d960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a415c040 0x7fa5a415d960 secure :-1 s=READY pgs=1214 cs=0 l=1 rev1=1 crypto rx=0x7fa59c036040 tx=0x7fa59c004a40 comp rx=0 tx=0).stop 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 shutdown_connections 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa58406b4d0 0x7fa58406d990 unknown :-1 s=CLOSED pgs=551 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa5a415c040 0x7fa5a415d960 unknown :-1 s=CLOSED pgs=1214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa5a40d0780 0x7fa5a415d420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 --2- 172.21.15.64:0/1277680336 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa5a40cf970 0x7fa5a4157de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 >> 172.21.15.64:0/1277680336 conn(0x7fa5a40d1a90 msgr2=0x7fa5a40d0b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:53.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 shutdown_connections 2024-03-04T07:31:53.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.151+0000 7fa5a9202640 1 -- 172.21.15.64:0/1277680336 wait complete. 2024-03-04T07:31:53.164 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 14, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:31:12.750651+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.5, 'score_stable': 2.5, 'optimal_score': 1, 'raw_score_acting': 2.5, 'raw_score_stable': 2.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 15, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:31:13.444529+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '90', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:31:53.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:31:53.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.268+0000 7f157aafb640 1 Processor -- start 2024-03-04T07:31:53.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.269+0000 7f157aafb640 1 -- start start 2024-03-04T07:31:53.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.269+0000 7f157aafb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f157415e4f0 0x7f15740d1c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.269+0000 7f157aafb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f15740b0c80 con 0x7f15740b1a40 2024-03-04T07:31:53.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.269+0000 7f157aafb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f15740b0f60 con 0x7f15740d2490 2024-03-04T07:31:53.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.269+0000 7f157aafb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f15740d4e40 con 0x7f157415e4f0 2024-03-04T07:31:53.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.269+0000 7f1579098640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f15740b1a40 0x7f15740b1eb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:58306/0 (socket says 172.21.15.64:58306) 2024-03-04T07:31:53.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.269+0000 7f1579098640 1 -- 172.21.15.64:0/2058660353 learned_addr learned my addr 172.21.15.64:0/2058660353 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f1573fff640 1 --2- 172.21.15.64:0/2058660353 >> v2:172.21.15.66:3300/0 conn(0x7f157415e4f0 0x7f15740d1c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1648798743 0 0) 0x7f15740b0c80 con 0x7f15740b1a40 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1560003520 con 0x7f15740b1a40 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2575045563 0 0) 0x7f1560003520 con 0x7f15740b1a40 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 >> v2:172.21.15.66:3300/0 conn(0x7f157415e4f0 msgr2=0x7f15740d1c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 --2- 172.21.15.64:0/2058660353 >> v2:172.21.15.66:3300/0 conn(0x7f157415e4f0 0x7f15740d1c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 >> v1:172.21.15.66:6789/0 conn(0x7f15740d2490 legacy=0x7f15740d2870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15740b1240 con 0x7f15740b1a40 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f1564001670 con 0x7f15740b1a40 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 >> v1:172.21.15.64:6789/0 conn(0x7f15740b1a40 legacy=0x7f15740b1eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15600039b0 0x7f1560003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15600043b0 0x7f1560004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1560005080 0x7f156000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.274 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15740d4e40 con 0x7f1560005080 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f156000bfe0 con 0x7f15600043b0 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f156000c1e0 con 0x7f15600039b0 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f1579098640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1560005080 0x7f156000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f1573fff640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15600043b0 0x7f1560004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.270+0000 7f1578897640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15600039b0 0x7f1560003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1579098640 1 -- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15600039b0 msgr2=0x7f1560003d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1579098640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15600039b0 0x7f1560003d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1579098640 1 -- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15600043b0 msgr2=0x7f1560004880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1579098640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f15600043b0 0x7f1560004880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1579098640 1 -- 172.21.15.64:0/2058660353 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1560003520 con 0x7f1560005080 2024-03-04T07:31:53.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1578897640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f15600039b0 0x7f1560003d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1579098640 1 -- 172.21.15.64:0/2058660353 reap_dead start 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f1579098640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1560005080 0x7f156000b9c0 secure :-1 s=READY pgs=1216 cs=0 l=1 rev1=1 crypto rx=0x7f156400c190 tx=0x7f1560005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eef74e4fa09318bc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1564007cc0 con 0x7f1560005080 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1564030070 con 0x7f1560005080 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.271+0000 7f15737fe640 1 -- 172.21.15.64:0/2058660353 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1564005d50 con 0x7f1560005080 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.272+0000 7f157aafb640 1 -- 172.21.15.64:0/2058660353 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1560005080 msgr2=0x7f156000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.272+0000 7f157aafb640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1560005080 0x7f156000b9c0 secure :-1 s=READY pgs=1216 cs=0 l=1 rev1=1 crypto rx=0x7f156400c190 tx=0x7f1560005830 comp rx=0 tx=0).stop 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.272+0000 7f157aafb640 1 -- 172.21.15.64:0/2058660353 shutdown_connections 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.272+0000 7f157aafb640 1 --2- 172.21.15.64:0/2058660353 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1560005080 0x7f156000b9c0 unknown :-1 s=CLOSED pgs=1216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.272+0000 7f157aafb640 1 -- 172.21.15.64:0/2058660353 >> 172.21.15.64:0/2058660353 conn(0x7f15740d3eb0 msgr2=0x7f15740d42c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.272+0000 7f157aafb640 1 -- 172.21.15.64:0/2058660353 shutdown_connections 2024-03-04T07:31:53.276 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.272+0000 7f157aafb640 1 -- 172.21.15.64:0/2058660353 wait complete. 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.273+0000 7f157aafb640 1 Processor -- start 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.273+0000 7f157aafb640 1 -- start start 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f157aafb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1560005080 0x7f15741511f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f157aafb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f157409dc60 0x7f157409e0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f157aafb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f157409f960 0x7f157409fde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f157aafb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15740d49d0 con 0x7f157409dc60 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f157aafb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15740b15b0 con 0x7f157409f960 2024-03-04T07:31:53.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f157aafb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f15740d4b50 con 0x7f1560005080 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f157409dc60 0x7f157409e0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f157409dc60 0x7f157409e0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37876/0 (socket says 172.21.15.64:37876) 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 -- 172.21.15.64:0/15133955 learned_addr learned my addr 172.21.15.64:0/15133955 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1579098640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f157409f960 0x7f157409fde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1578897640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1560005080 0x7f15741511f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 -- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1560005080 msgr2=0x7f15741511f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1560005080 0x7f15741511f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 -- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f157409f960 msgr2=0x7f157409fde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f157409f960 0x7f157409fde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.274+0000 7f1573fff640 1 -- 172.21.15.64:0/15133955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15740a0830 con 0x7f157409dc60 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.275+0000 7f1573fff640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f157409dc60 0x7f157409e0e0 secure :-1 s=READY pgs=1217 cs=0 l=1 rev1=1 crypto rx=0x7f155c0119c0 tx=0x7f155c011e80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.275+0000 7f15717fa640 1 -- 172.21.15.64:0/15133955 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f155c019070 con 0x7f157409dc60 2024-03-04T07:31:53.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.275+0000 7f15717fa640 1 -- 172.21.15.64:0/15133955 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f155c004240 con 0x7f157409dc60 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.275+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f15740a0b10 con 0x7f157409dc60 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.275+0000 7f15717fa640 1 -- 172.21.15.64:0/15133955 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f155c00f8e0 con 0x7f157409dc60 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.275+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15740a10d0 con 0x7f157409dc60 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.276+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f15740b0f60 con 0x7f157409dc60 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.276+0000 7f15717fa640 1 -- 172.21.15.64:0/15133955 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f155c01c070 con 0x7f157409dc60 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.277+0000 7f15717fa640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f155406b4d0 0x7f155406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.277+0000 7f15717fa640 1 -- 172.21.15.64:0/15133955 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f155c08a640 con 0x7f157409dc60 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.277+0000 7f1578897640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f155406b4d0 0x7f155406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.277+0000 7f1578897640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f155406b4d0 0x7f155406d990 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7f156c001db0 tx=0x7f156c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:53.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.281+0000 7f15717fa640 1 -- 172.21.15.64:0/15133955 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f15740b0f60 con 0x7f157409dc60 2024-03-04T07:31:53.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.543+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f1574068050 con 0x7f157409dc60 2024-03-04T07:31:53.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.544+0000 7f15717fa640 1 -- 172.21.15.64:0/15133955 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v94) v1 ==== 93+0+10 (secure 0 0 0) 0x7f155c0594a0 con 0x7f157409dc60 2024-03-04T07:31:53.545 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f155406b4d0 msgr2=0x7f155406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f155406b4d0 0x7f155406d990 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7f156c001db0 tx=0x7f156c009040 comp rx=0 tx=0).stop 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f157409dc60 msgr2=0x7f157409e0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f157409dc60 0x7f157409e0e0 secure :-1 s=READY pgs=1217 cs=0 l=1 rev1=1 crypto rx=0x7f155c0119c0 tx=0x7f155c011e80 comp rx=0 tx=0).stop 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 shutdown_connections 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f155406b4d0 0x7f155406d990 unknown :-1 s=CLOSED pgs=552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f157409f960 0x7f157409fde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f157409dc60 0x7f157409e0e0 unknown :-1 s=CLOSED pgs=1217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 --2- 172.21.15.64:0/15133955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1560005080 0x7f15741511f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 >> 172.21.15.64:0/15133955 conn(0x7f15740d3eb0 msgr2=0x7f157415e900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:53.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 shutdown_connections 2024-03-04T07:31:53.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.546+0000 7f157aafb640 1 -- 172.21.15.64:0/15133955 wait complete. 2024-03-04T07:31:53.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:31:53.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.666+0000 7f200efc9640 1 Processor -- start 2024-03-04T07:31:53.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.666+0000 7f200efc9640 1 -- start start 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f200efc9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f200806e510 0x7f2008070aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f200efc9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2008144c40 con 0x7f2008144860 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f200efc9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f20080ce740 con 0x7f2008070fe0 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f200efc9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f20080cea20 con 0x7f200806e510 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f1ffffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f200806e510 0x7f2008070aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f1ffffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f200806e510 0x7f2008070aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44652/0 (socket says 172.21.15.64:44652) 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f1ffffff640 1 -- 172.21.15.64:0/1629962560 learned_addr learned my addr 172.21.15.64:0/1629962560 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:53.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f200cd65640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2008144860 0x7f20080cde40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35342/0 (socket says 172.21.15.64:35342) 2024-03-04T07:31:53.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f1ffffff640 1 -- 172.21.15.64:0/1629962560 >> v1:172.21.15.66:6789/0 conn(0x7f2008070fe0 legacy=0x7f2008071410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:53.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f1ffffff640 1 -- 172.21.15.64:0/1629962560 >> v1:172.21.15.64:6789/0 conn(0x7f2008144860 legacy=0x7f20080cde40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.667+0000 7f1ffffff640 1 -- 172.21.15.64:0/1629962560 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20080ced00 con 0x7f200806e510 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.668+0000 7f1ffffff640 1 --2- 172.21.15.64:0/1629962560 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f200806e510 0x7f2008070aa0 secure :-1 s=READY pgs=1219 cs=0 l=1 rev1=1 crypto rx=0x7f1ff8009870 tx=0x7f1ff802dbb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8fee671599ac2be5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.668+0000 7f1fff7fe640 1 -- 172.21.15.64:0/1629962560 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ff803b070 con 0x7f200806e510 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.668+0000 7f1fff7fe640 1 -- 172.21.15.64:0/1629962560 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ff802e960 con 0x7f200806e510 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.668+0000 7f1fff7fe640 1 -- 172.21.15.64:0/1629962560 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ff802ed30 con 0x7f200806e510 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.669+0000 7f200efc9640 1 -- 172.21.15.64:0/1629962560 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f200806e510 msgr2=0x7f2008070aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.669+0000 7f200efc9640 1 --2- 172.21.15.64:0/1629962560 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f200806e510 0x7f2008070aa0 secure :-1 s=READY pgs=1219 cs=0 l=1 rev1=1 crypto rx=0x7f1ff8009870 tx=0x7f1ff802dbb0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.669+0000 7f200efc9640 1 -- 172.21.15.64:0/1629962560 shutdown_connections 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.669+0000 7f200efc9640 1 --2- 172.21.15.64:0/1629962560 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f200806e510 0x7f2008070aa0 unknown :-1 s=CLOSED pgs=1219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.669+0000 7f200efc9640 1 -- 172.21.15.64:0/1629962560 >> 172.21.15.64:0/1629962560 conn(0x7f20080b17a0 msgr2=0x7f20080b1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:53.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.669+0000 7f200efc9640 1 -- 172.21.15.64:0/1629962560 shutdown_connections 2024-03-04T07:31:53.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.669+0000 7f200efc9640 1 -- 172.21.15.64:0/1629962560 wait complete. 2024-03-04T07:31:53.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.670+0000 7f200efc9640 1 Processor -- start 2024-03-04T07:31:53.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.670+0000 7f200efc9640 1 -- start start 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f200efc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f200806e510 0x7f2008140090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f200efc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2008070fe0 0x7f2008138de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f200efc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2008144860 0x7f20081385a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f200efc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20080b3ab0 con 0x7f2008070fe0 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f200efc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20080cf2c0 con 0x7f200806e510 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f200efc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f20080b3c30 con 0x7f2008144860 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2008070fe0 0x7f2008138de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2008070fe0 0x7f2008138de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44658/0 (socket says 172.21.15.64:44658) 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 -- 172.21.15.64:0/2385709686 learned_addr learned my addr 172.21.15.64:0/2385709686 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 -- 172.21.15.64:0/2385709686 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2008144860 msgr2=0x7f20081385a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2008144860 0x7f20081385a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 -- 172.21.15.64:0/2385709686 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f200806e510 msgr2=0x7f2008140090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f200806e510 0x7f2008140090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 -- 172.21.15.64:0/2385709686 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2008139320 con 0x7f2008070fe0 2024-03-04T07:31:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffffff640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2008070fe0 0x7f2008138de0 secure :-1 s=READY pgs=1220 cs=0 l=1 rev1=1 crypto rx=0x7f1ff8009870 tx=0x7f1ff80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffd7fa640 1 -- 172.21.15.64:0/2385709686 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ff803b070 con 0x7f2008070fe0 2024-03-04T07:31:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffd7fa640 1 -- 172.21.15.64:0/2385709686 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ff8036d80 con 0x7f2008070fe0 2024-03-04T07:31:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20081f3f90 con 0x7f2008070fe0 2024-03-04T07:31:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.671+0000 7f1ffd7fa640 1 -- 172.21.15.64:0/2385709686 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ff8041660 con 0x7f2008070fe0 2024-03-04T07:31:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.672+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20081f4550 con 0x7f2008070fe0 2024-03-04T07:31:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.673+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2008068050 con 0x7f2008070fe0 2024-03-04T07:31:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.673+0000 7f1ffd7fa640 1 -- 172.21.15.64:0/2385709686 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1ff8041800 con 0x7f2008070fe0 2024-03-04T07:31:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.673+0000 7f1ffd7fa640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1ff406b4d0 0x7f1ff406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.673+0000 7f1ffd7fa640 1 -- 172.21.15.64:0/2385709686 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f1ff80ac5c0 con 0x7f2008070fe0 2024-03-04T07:31:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.673+0000 7f200cd65640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1ff406b4d0 0x7f1ff406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.674+0000 7f200cd65640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1ff406b4d0 0x7f1ff406d990 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f2004000f60 tx=0x7f2004007a00 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.678+0000 7f1ffd7fa640 1 -- 172.21.15.64:0/2385709686 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1ff807b390 con 0x7f2008070fe0 2024-03-04T07:31:53.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.857+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f20080cf880 con 0x7f2008070fe0 2024-03-04T07:31:53.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.857+0000 7f1ffd7fa640 1 -- 172.21.15.64:0/2385709686 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v94) v1 ==== 104+0+11 (secure 0 0 0) 0x7f1ff807ac90 con 0x7f2008070fe0 2024-03-04T07:31:53.858 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:31:53.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1ff406b4d0 msgr2=0x7f1ff406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1ff406b4d0 0x7f1ff406d990 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f2004000f60 tx=0x7f2004007a00 comp rx=0 tx=0).stop 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2008070fe0 msgr2=0x7f2008138de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2008070fe0 0x7f2008138de0 secure :-1 s=READY pgs=1220 cs=0 l=1 rev1=1 crypto rx=0x7f1ff8009870 tx=0x7f1ff80046c0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 shutdown_connections 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1ff406b4d0 0x7f1ff406d990 unknown :-1 s=CLOSED pgs=553 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2008144860 0x7f20081385a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2008070fe0 0x7f2008138de0 unknown :-1 s=CLOSED pgs=1220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 --2- 172.21.15.64:0/2385709686 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f200806e510 0x7f2008140090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 >> 172.21.15.64:0/2385709686 conn(0x7f20080b17a0 msgr2=0x7f20080b47b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 shutdown_connections 2024-03-04T07:31:53.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:53.859+0000 7f200efc9640 1 -- 172.21.15.64:0/2385709686 wait complete. 2024-03-04T07:31:53.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:31:54.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.007+0000 7feece55e640 1 Processor -- start 2024-03-04T07:31:54.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.008+0000 7feece55e640 1 -- start start 2024-03-04T07:31:54.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.008+0000 7feece55e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feec8161830 0x7feec80d24f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.008+0000 7feece55e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feec80d2100 con 0x7feec8161c10 2024-03-04T07:31:54.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.008+0000 7feece55e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feec80b5000 con 0x7feec80d0fc0 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.008+0000 7feece55e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feec80b52c0 con 0x7feec8161830 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.008+0000 7feec77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7feec8161c10 0x7feec80c7a60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35348/0 (socket says 172.21.15.64:35348) 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.008+0000 7feec77fe640 1 -- 172.21.15.64:0/1188061856 learned_addr learned my addr 172.21.15.64:0/1188061856 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec7fff640 1 --2- 172.21.15.64:0/1188061856 >> v2:172.21.15.66:3300/0 conn(0x7feec8161830 0x7feec80d24f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1802020444 0 0) 0x7feec80d2100 con 0x7feec8161c10 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7feeb0003520 con 0x7feec8161c10 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 914867616 0 0) 0x7feeb0003520 con 0x7feec8161c10 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 >> v2:172.21.15.66:3300/0 conn(0x7feec8161830 msgr2=0x7feec80d24f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 --2- 172.21.15.64:0/1188061856 >> v2:172.21.15.66:3300/0 conn(0x7feec8161830 0x7feec80d24f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 >> v1:172.21.15.66:6789/0 conn(0x7feec80d0fc0 legacy=0x7feec80c8170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feec80d1e20 con 0x7feec8161c10 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7feebc001670 con 0x7feec8161c10 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 >> v1:172.21.15.64:6789/0 conn(0x7feec8161c10 legacy=0x7feec80c7a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeb00039b0 0x7feeb0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeb0004800 0x7feeb0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feeb000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feec80b52c0 con 0x7feeb00051c0 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feeb000c160 con 0x7feeb0004800 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feeb000c360 con 0x7feeb00039b0 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feeccafb640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feeb000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec7fff640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeb00039b0 0x7feeb0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feec77fe640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeb0004800 0x7feeb0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feeccafb640 1 -- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeb00039b0 msgr2=0x7feeb0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feeccafb640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeb00039b0 0x7feeb0003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feeccafb640 1 -- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeb0004800 msgr2=0x7feeb0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.009+0000 7feeccafb640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeb0004800 0x7feeb0004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.010+0000 7feeccafb640 1 -- 172.21.15.64:0/1188061856 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feeb0003520 con 0x7feeb00051c0 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.010+0000 7feec77fe640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeb0004800 0x7feeb0004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:54.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.010+0000 7feeccafb640 1 -- 172.21.15.64:0/1188061856 reap_dead start 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.010+0000 7feeccafb640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feeb000b9c0 secure :-1 s=READY pgs=1222 cs=0 l=1 rev1=1 crypto rx=0x7feec0001e50 tx=0x7feeb0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d5953a3b1a07fc2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.010+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feec0001a70 con 0x7feeb00051c0 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.010+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feec0003bb0 con 0x7feeb00051c0 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.010+0000 7feec6ffd640 1 -- 172.21.15.64:0/1188061856 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feec00050d0 con 0x7feeb00051c0 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.011+0000 7feece55e640 1 -- 172.21.15.64:0/1188061856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 msgr2=0x7feeb000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.011+0000 7feece55e640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feeb000b9c0 secure :-1 s=READY pgs=1222 cs=0 l=1 rev1=1 crypto rx=0x7feec0001e50 tx=0x7feeb0005830 comp rx=0 tx=0).stop 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.011+0000 7feece55e640 1 -- 172.21.15.64:0/1188061856 shutdown_connections 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.011+0000 7feece55e640 1 --2- 172.21.15.64:0/1188061856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feeb000b9c0 unknown :-1 s=CLOSED pgs=1222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.011+0000 7feece55e640 1 -- 172.21.15.64:0/1188061856 >> 172.21.15.64:0/1188061856 conn(0x7feec80d3f10 msgr2=0x7feec80d4320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.011+0000 7feece55e640 1 -- 172.21.15.64:0/1188061856 shutdown_connections 2024-03-04T07:31:54.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.011+0000 7feece55e640 1 -- 172.21.15.64:0/1188061856 wait complete. 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.012+0000 7feece55e640 1 Processor -- start 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.012+0000 7feece55e640 1 -- start start 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feece55e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feec8156ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feece55e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec81f7fa0 0x7feec81439c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feece55e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec8143f00 0x7feec8154bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feece55e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feec80d24f0 con 0x7feeb00051c0 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feece55e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feec80d4bd0 con 0x7feec81f7fa0 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feece55e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feec80d2670 con 0x7feec8143f00 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feec8156ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feec8156ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44680/0 (socket says 172.21.15.64:44680) 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 -- 172.21.15.64:0/1578194001 learned_addr learned my addr 172.21.15.64:0/1578194001 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feeccafb640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec8143f00 0x7feec8154bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec77fe640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec81f7fa0 0x7feec81439c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 -- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec8143f00 msgr2=0x7feec8154bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec8143f00 0x7feec8154bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 -- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec81f7fa0 msgr2=0x7feec81439c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec81f7fa0 0x7feec81439c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.013+0000 7feec7fff640 1 -- 172.21.15.64:0/1578194001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feec8157560 con 0x7feeb00051c0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.014+0000 7feec7fff640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feec8156ec0 secure :-1 s=READY pgs=1223 cs=0 l=1 rev1=1 crypto rx=0x7feeb8011910 tx=0x7feeb8011dd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.014+0000 7feec4ff9640 1 -- 172.21.15.64:0/1578194001 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feeb800faf0 con 0x7feeb00051c0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.014+0000 7feec4ff9640 1 -- 172.21.15.64:0/1578194001 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feeb800fc90 con 0x7feeb00051c0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.014+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feec8157840 con 0x7feeb00051c0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.014+0000 7feec4ff9640 1 -- 172.21.15.64:0/1578194001 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feeb8014440 con 0x7feeb00051c0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.014+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feec8157e00 con 0x7feeb00051c0 2024-03-04T07:31:54.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.015+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feec80ce700 con 0x7feeb00051c0 2024-03-04T07:31:54.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.015+0000 7feec4ff9640 1 -- 172.21.15.64:0/1578194001 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feeb8012040 con 0x7feeb00051c0 2024-03-04T07:31:54.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.016+0000 7feec4ff9640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeb406b4d0 0x7feeb406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.016+0000 7feec4ff9640 1 -- 172.21.15.64:0/1578194001 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7feeb808a880 con 0x7feeb00051c0 2024-03-04T07:31:54.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.016+0000 7feec77fe640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeb406b4d0 0x7feeb406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.016+0000 7feec77fe640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeb406b4d0 0x7feeb406d990 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7feebc001eb0 tx=0x7feebc000ed0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.020+0000 7feec4ff9640 1 -- 172.21.15.64:0/1578194001 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feec80ce700 con 0x7feeb00051c0 2024-03-04T07:31:54.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.204+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7feec80d0da0 con 0x7feeb00051c0 2024-03-04T07:31:54.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.205+0000 7feec4ff9640 1 -- 172.21.15.64:0/1578194001 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v94) v1 ==== 100+0+11 (secure 0 0 0) 0x7feec80d0da0 con 0x7feeb00051c0 2024-03-04T07:31:54.205 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeb406b4d0 msgr2=0x7feeb406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeb406b4d0 0x7feeb406d990 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7feebc001eb0 tx=0x7feebc000ed0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 msgr2=0x7feec8156ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feec8156ec0 secure :-1 s=READY pgs=1223 cs=0 l=1 rev1=1 crypto rx=0x7feeb8011910 tx=0x7feeb8011dd0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 shutdown_connections 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeb406b4d0 0x7feeb406d990 unknown :-1 s=CLOSED pgs=554 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec8143f00 0x7feec8154bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec81f7fa0 0x7feec81439c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 --2- 172.21.15.64:0/1578194001 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeb00051c0 0x7feec8156ec0 unknown :-1 s=CLOSED pgs=1223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 >> 172.21.15.64:0/1578194001 conn(0x7feec80d3f10 msgr2=0x7feec80b8f30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:54.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 shutdown_connections 2024-03-04T07:31:54.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.207+0000 7feece55e640 1 -- 172.21.15.64:0/1578194001 wait complete. 2024-03-04T07:31:54.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:54.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.322+0000 7f554d676640 1 Processor -- start 2024-03-04T07:31:54.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.322+0000 7f554d676640 1 -- start start 2024-03-04T07:31:54.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.322+0000 7f554d676640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55480b6580 0x7f55481625d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f554d676640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5548162b10 0x7f55480cfdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f554d676640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55480b4ab0 con 0x7f554815e350 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f554d676640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f55480d1320 con 0x7f5548162b10 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f554d676640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55480b50e0 con 0x7f55480b6580 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55477fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5548162b10 0x7f55480cfdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55477fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5548162b10 0x7f55480cfdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44682/0 (socket says 172.21.15.64:44682) 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55477fe640 1 -- 172.21.15.64:0/4196141849 learned_addr learned my addr 172.21.15.64:0/4196141849 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55467fc640 1 --2- 172.21.15.64:0/4196141849 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55480b6580 0x7f55481625d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55477fe640 1 -- 172.21.15.64:0/4196141849 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55480b6580 msgr2=0x7f55481625d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55477fe640 1 --2- 172.21.15.64:0/4196141849 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55480b6580 0x7f55481625d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55477fe640 1 -- 172.21.15.64:0/4196141849 >> v1:172.21.15.66:6789/0 conn(0x7f554815e350 legacy=0x7f5548161ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.323+0000 7f55477fe640 1 -- 172.21.15.64:0/4196141849 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f554806c380 con 0x7f5548162b10 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f55477fe640 1 --2- 172.21.15.64:0/4196141849 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f55480cfdd0 secure :-1 s=READY pgs=1224 cs=0 l=1 rev1=1 crypto rx=0x7f5540009e80 tx=0x7f554002beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=613fcc2dc1edf9a6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f5545ffb640 1 -- 172.21.15.64:0/4196141849 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f554002ca50 con 0x7f5548162b10 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f5545ffb640 1 -- 172.21.15.64:0/4196141849 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f554002cbf0 con 0x7f5548162b10 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f5545ffb640 1 -- 172.21.15.64:0/4196141849 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55400343f0 con 0x7f5548162b10 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f554d676640 1 -- 172.21.15.64:0/4196141849 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 msgr2=0x7f55480cfdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f554d676640 1 --2- 172.21.15.64:0/4196141849 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f55480cfdd0 secure :-1 s=READY pgs=1224 cs=0 l=1 rev1=1 crypto rx=0x7f5540009e80 tx=0x7f554002beb0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f554d676640 1 -- 172.21.15.64:0/4196141849 shutdown_connections 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f554d676640 1 --2- 172.21.15.64:0/4196141849 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f55480cfdd0 unknown :-1 s=CLOSED pgs=1224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f554d676640 1 --2- 172.21.15.64:0/4196141849 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f55480b6580 0x7f55481625d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.324+0000 7f554d676640 1 -- 172.21.15.64:0/4196141849 >> 172.21.15.64:0/4196141849 conn(0x7f55480d1c60 msgr2=0x7f55480d2070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.325+0000 7f554d676640 1 -- 172.21.15.64:0/4196141849 shutdown_connections 2024-03-04T07:31:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.325+0000 7f554d676640 1 -- 172.21.15.64:0/4196141849 wait complete. 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.326+0000 7f554d676640 1 Processor -- start 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.327+0000 7f554d676640 1 -- start start 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f554d676640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55480b6580 0x7f55481f1380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f554d676640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f554815e350 0x7f554809db70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f554d676640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f554809e0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f554d676640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f554815e730 con 0x7f5548162b10 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f554d676640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55480b4d70 con 0x7f55480b6580 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f554d676640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55480d0310 con 0x7f554815e350 2024-03-04T07:31:54.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f554809e0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f554809e0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44684/0 (socket says 172.21.15.64:44684) 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 -- 172.21.15.64:0/3270384598 learned_addr learned my addr 172.21.15.64:0/3270384598 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f5546ffd640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55480b6580 0x7f55481f1380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55467fc640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f554815e350 0x7f554809db70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 -- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f554815e350 msgr2=0x7f554809db70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f554815e350 0x7f554809db70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 -- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55480b6580 msgr2=0x7f55481f1380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55480b6580 0x7f55481f1380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 -- 172.21.15.64:0/3270384598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f554809e6c0 con 0x7f5548162b10 2024-03-04T07:31:54.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f55477fe640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f554809e0b0 secure :-1 s=READY pgs=1225 cs=0 l=1 rev1=1 crypto rx=0x7f5540036040 tx=0x7f5540004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.328+0000 7f552f7fe640 1 -- 172.21.15.64:0/3270384598 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5540044020 con 0x7f5548162b10 2024-03-04T07:31:54.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.329+0000 7f552f7fe640 1 -- 172.21.15.64:0/3270384598 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5540040670 con 0x7f5548162b10 2024-03-04T07:31:54.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.329+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f554809e9a0 con 0x7f5548162b10 2024-03-04T07:31:54.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.329+0000 7f552f7fe640 1 -- 172.21.15.64:0/3270384598 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5540009510 con 0x7f5548162b10 2024-03-04T07:31:54.332 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.329+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55480c6910 con 0x7f5548162b10 2024-03-04T07:31:54.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.330+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f55480752d0 con 0x7f5548162b10 2024-03-04T07:31:54.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.330+0000 7f552f7fe640 1 -- 172.21.15.64:0/3270384598 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5540047020 con 0x7f5548162b10 2024-03-04T07:31:54.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.331+0000 7f552f7fe640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553406b4d0 0x7f553406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.331+0000 7f5546ffd640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553406b4d0 0x7f553406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.331+0000 7f552f7fe640 1 -- 172.21.15.64:0/3270384598 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f55400accc0 con 0x7f5548162b10 2024-03-04T07:31:54.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.331+0000 7f5546ffd640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553406b4d0 0x7f553406d990 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f55480c8800 tx=0x7f5538009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.338+0000 7f552f7fe640 1 -- 172.21.15.64:0/3270384598 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f554004e030 con 0x7f5548162b10 2024-03-04T07:31:54.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.559+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5548067a70 con 0x7f5548162b10 2024-03-04T07:31:54.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.560+0000 7f552f7fe640 1 -- 172.21.15.64:0/3270384598 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f554007bb20 con 0x7f5548162b10 2024-03-04T07:31:54.561 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:54.561 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:54.561 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553406b4d0 msgr2=0x7f553406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553406b4d0 0x7f553406d990 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f55480c8800 tx=0x7f5538009040 comp rx=0 tx=0).stop 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 msgr2=0x7f554809e0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f554809e0b0 secure :-1 s=READY pgs=1225 cs=0 l=1 rev1=1 crypto rx=0x7f5540036040 tx=0x7f5540004a40 comp rx=0 tx=0).stop 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 shutdown_connections 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f553406b4d0 0x7f553406d990 unknown :-1 s=CLOSED pgs=555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5548162b10 0x7f554809e0b0 unknown :-1 s=CLOSED pgs=1225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f554815e350 0x7f554809db70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 --2- 172.21.15.64:0/3270384598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55480b6580 0x7f55481f1380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 >> 172.21.15.64:0/3270384598 conn(0x7f55480d1c60 msgr2=0x7f554806d340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 shutdown_connections 2024-03-04T07:31:54.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.562+0000 7f554d676640 1 -- 172.21.15.64:0/3270384598 wait complete. 2024-03-04T07:31:54.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:54.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.675+0000 7f6bf57ac640 1 Processor -- start 2024-03-04T07:31:54.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.675+0000 7f6bf57ac640 1 -- start start 2024-03-04T07:31:54.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.675+0000 7f6bf57ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf0145980 0x7f6bf00714b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.675+0000 7f6bf57ac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6bf01600f0 0x7f6bf006d0c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.675+0000 7f6bf57ac640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6bf00720c0 con 0x7f6bf00728a0 2024-03-04T07:31:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.675+0000 7f6bf57ac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6bf006cc30 con 0x7f6bf01600f0 2024-03-04T07:31:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.675+0000 7f6bf57ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6bf0071b00 con 0x7f6bf0145980 2024-03-04T07:31:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6be7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6bf00728a0 0x7f6bf015f9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35364/0 (socket says 172.21.15.64:35364) 2024-03-04T07:31:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6be7fff640 1 -- 172.21.15.64:0/1665745751 learned_addr learned my addr 172.21.15.64:0/1665745751 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6bef7fe640 1 --2- 172.21.15.64:0/1665745751 >> v2:172.21.15.66:3300/0 conn(0x7f6bf01600f0 0x7f6bf006d0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6beeffd640 1 --2- 172.21.15.64:0/1665745751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf0145980 0x7f6bf00714b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6bee7fc640 1 -- 172.21.15.64:0/1665745751 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1165445499 0 0) 0x7f6bf00720c0 con 0x7f6bf00728a0 2024-03-04T07:31:54.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6bee7fc640 1 -- 172.21.15.64:0/1665745751 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6bdc003520 con 0x7f6bf00728a0 2024-03-04T07:31:54.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6beeffd640 1 -- 172.21.15.64:0/1665745751 >> v2:172.21.15.66:3300/0 conn(0x7f6bf01600f0 msgr2=0x7f6bf006d0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:54.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6beeffd640 1 --2- 172.21.15.64:0/1665745751 >> v2:172.21.15.66:3300/0 conn(0x7f6bf01600f0 0x7f6bf006d0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6beeffd640 1 -- 172.21.15.64:0/1665745751 >> v1:172.21.15.64:6789/0 conn(0x7f6bf00728a0 legacy=0x7f6bf015f9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6bee7fc640 1 -- 172.21.15.64:0/1665745751 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2063192490 0 0) 0x7f6bdc003520 con 0x7f6bf00728a0 2024-03-04T07:31:54.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6beeffd640 1 -- 172.21.15.64:0/1665745751 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6bf0071de0 con 0x7f6bf0145980 2024-03-04T07:31:54.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.676+0000 7f6bef7fe640 1 --2- 172.21.15.64:0/1665745751 >> v2:172.21.15.66:3300/0 conn(0x7f6bf01600f0 0x7f6bf006d0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:54.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.677+0000 7f6beeffd640 1 --2- 172.21.15.64:0/1665745751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf0145980 0x7f6bf00714b0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f6be00097a0 tx=0x7f6be002b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=414cbe8125ac5dde server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.677+0000 7f6bee7fc640 1 -- 172.21.15.64:0/1665745751 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6be0035020 con 0x7f6bf0145980 2024-03-04T07:31:54.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.677+0000 7f6bee7fc640 1 -- 172.21.15.64:0/1665745751 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6be0031830 con 0x7f6bf0145980 2024-03-04T07:31:54.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.677+0000 7f6bee7fc640 1 -- 172.21.15.64:0/1665745751 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6be0031be0 con 0x7f6bf0145980 2024-03-04T07:31:54.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1665745751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf0145980 msgr2=0x7f6bf00714b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1665745751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf0145980 0x7f6bf00714b0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f6be00097a0 tx=0x7f6be002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1665745751 shutdown_connections 2024-03-04T07:31:54.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1665745751 >> v2:172.21.15.66:3300/0 conn(0x7f6bf01600f0 0x7f6bf006d0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1665745751 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf0145980 0x7f6bf00714b0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1665745751 >> 172.21.15.64:0/1665745751 conn(0x7f6bf00b17a0 msgr2=0x7f6bf00b1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:54.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1665745751 shutdown_connections 2024-03-04T07:31:54.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.678+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1665745751 wait complete. 2024-03-04T07:31:54.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.679+0000 7f6bf57ac640 1 Processor -- start 2024-03-04T07:31:54.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.679+0000 7f6bf57ac640 1 -- start start 2024-03-04T07:31:54.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bf57ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf00728a0 0x7f6bf015b470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bf57ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bf0145980 0x7f6bf00aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bf57ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bf01600f0 0x7f6bf00ce190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bf57ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bf006d600 con 0x7f6bf01600f0 2024-03-04T07:31:54.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bf57ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6bf0072cc0 con 0x7f6bf0145980 2024-03-04T07:31:54.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bf57ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6bf006d780 con 0x7f6bf00728a0 2024-03-04T07:31:54.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bf01600f0 0x7f6bf00ce190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bf01600f0 0x7f6bf00ce190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44700/0 (socket says 172.21.15.64:44700) 2024-03-04T07:31:54.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 -- 172.21.15.64:0/1315433870 learned_addr learned my addr 172.21.15.64:0/1315433870 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:54.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6beeffd640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf00728a0 0x7f6bf015b470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6be7fff640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bf0145980 0x7f6bf00aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 -- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf00728a0 msgr2=0x7f6bf015b470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf00728a0 0x7f6bf015b470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 -- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bf0145980 msgr2=0x7f6bf00aea00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bf0145980 0x7f6bf00aea00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 -- 172.21.15.64:0/1315433870 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6bf00a9680 con 0x7f6bf01600f0 2024-03-04T07:31:54.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6be7fff640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bf0145980 0x7f6bf00aea00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:54.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6bef7fe640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bf01600f0 0x7f6bf00ce190 secure :-1 s=READY pgs=1227 cs=0 l=1 rev1=1 crypto rx=0x7f6be8009e10 tx=0x7f6be800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6be77fe640 1 -- 172.21.15.64:0/1315433870 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6be8018020 con 0x7f6bf01600f0 2024-03-04T07:31:54.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.680+0000 7f6be77fe640 1 -- 172.21.15.64:0/1315433870 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6be8014480 con 0x7f6bf01600f0 2024-03-04T07:31:54.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.681+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6bf00a9960 con 0x7f6bf01600f0 2024-03-04T07:31:54.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.681+0000 7f6be77fe640 1 -- 172.21.15.64:0/1315433870 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6be8014850 con 0x7f6bf01600f0 2024-03-04T07:31:54.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.681+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6bf00ac0c0 con 0x7f6bf01600f0 2024-03-04T07:31:54.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.681+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6bf00752d0 con 0x7f6bf01600f0 2024-03-04T07:31:54.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.682+0000 7f6be77fe640 1 -- 172.21.15.64:0/1315433870 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6be8005d10 con 0x7f6bf01600f0 2024-03-04T07:31:54.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.683+0000 7f6be77fe640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bd006b500 0x7f6bd006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:54.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.683+0000 7f6be77fe640 1 -- 172.21.15.64:0/1315433870 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f6be808aff0 con 0x7f6bf01600f0 2024-03-04T07:31:54.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.683+0000 7f6beeffd640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bd006b500 0x7f6bd006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:54.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.683+0000 7f6beeffd640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bd006b500 0x7f6bd006d9c0 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f6be0036000 tx=0x7f6be0008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:54.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.687+0000 7f6be77fe640 1 -- 172.21.15.64:0/1315433870 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6be8022030 con 0x7f6bf01600f0 2024-03-04T07:31:54.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.872+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6bf015f480 con 0x7f6bf01600f0 2024-03-04T07:31:54.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.873+0000 7f6be77fe640 1 -- 172.21.15.64:0/1315433870 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) v1 ==== 74+0+16819 (secure 0 0 0) 0x7f6be8010040 con 0x7f6bf01600f0 2024-03-04T07:31:54.874 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:54.874 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":94,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:24.148060+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:31:12.750651+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-04T07:31:13.444529+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"90","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.64:6835/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.66:6835/3397672669":"2024-03-05T07:31:23.154858+0000","172.21.15.66:6833/3397672669":"2024-03-05T07:31:23.154858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:54.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bd006b500 msgr2=0x7f6bd006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bd006b500 0x7f6bd006d9c0 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f6be0036000 tx=0x7f6be0008030 comp rx=0 tx=0).stop 2024-03-04T07:31:54.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bf01600f0 msgr2=0x7f6bf00ce190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:54.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bf01600f0 0x7f6bf00ce190 secure :-1 s=READY pgs=1227 cs=0 l=1 rev1=1 crypto rx=0x7f6be8009e10 tx=0x7f6be800c490 comp rx=0 tx=0).stop 2024-03-04T07:31:54.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 shutdown_connections 2024-03-04T07:31:54.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6bd006b500 0x7f6bd006d9c0 unknown :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6bf01600f0 0x7f6bf00ce190 unknown :-1 s=CLOSED pgs=1227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6bf0145980 0x7f6bf00aea00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 --2- 172.21.15.64:0/1315433870 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6bf00728a0 0x7f6bf015b470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:54.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 >> 172.21.15.64:0/1315433870 conn(0x7f6bf00b17a0 msgr2=0x7f6bf00b0a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:54.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 shutdown_connections 2024-03-04T07:31:54.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:54.875+0000 7f6bf57ac640 1 -- 172.21.15.64:0/1315433870 wait complete. 2024-03-04T07:31:54.891 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:31:54.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:55.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.023+0000 7f66fa415640 1 Processor -- start 2024-03-04T07:31:55.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.023+0000 7f66fa415640 1 -- start start 2024-03-04T07:31:55.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66fa415640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 0x7f66f415d940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66fa415640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66f4162a60 con 0x7f66f415de80 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66fa415640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f66f40d0ee0 con 0x7f66f406cfc0 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66fa415640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66f40d11c0 con 0x7f66f40cebc0 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66f37fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f66f415de80 0x7f66f4161c40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35372/0 (socket says 172.21.15.64:35372) 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66f37fe640 1 -- 172.21.15.64:0/1176235821 learned_addr learned my addr 172.21.15.64:0/1176235821 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66f3fff640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 0x7f66f415d940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2429287136 0 0) 0x7f66f4162a60 con 0x7f66f415de80 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f66dc003520 con 0x7f66f415de80 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2995495165 0 0) 0x7f66f40d0ee0 con 0x7f66f406cfc0 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.024+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f66f4162a60 con 0x7f66f406cfc0 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 364766912 0 0) 0x7f66dc003520 con 0x7f66f415de80 2024-03-04T07:31:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 msgr2=0x7f66f415d940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 0x7f66f415d940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 >> v1:172.21.15.66:6789/0 conn(0x7f66f406cfc0 legacy=0x7f66f4162350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66f40b1820 con 0x7f66f415de80 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f66e4001670 con 0x7f66f415de80 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 >> v1:172.21.15.64:6789/0 conn(0x7f66f415de80 legacy=0x7f66f4161c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc003b70 0x7f66dc003f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66dc004800 0x7f66dc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 0x7f66dc0051e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66f40d11c0 con 0x7f66dc004800 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66dc00bad0 con 0x7f66dc003b70 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66dc00bcd0 con 0x7f66f40cebc0 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f37fe640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66dc004800 0x7f66dc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f3fff640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc003b70 0x7f66dc003f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f89b2640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 0x7f66dc0051e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f37fe640 1 -- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 msgr2=0x7f66dc0051e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f37fe640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 0x7f66dc0051e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f37fe640 1 -- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc003b70 msgr2=0x7f66dc003f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f37fe640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc003b70 0x7f66dc003f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f37fe640 1 -- 172.21.15.64:0/1176235821 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66dc003520 con 0x7f66dc004800 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.025+0000 7f66f89b2640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cebc0 0x7f66dc0051e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.026+0000 7f66f37fe640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66dc004800 0x7f66dc004c80 secure :-1 s=READY pgs=1229 cs=0 l=1 rev1=1 crypto rx=0x7f66e400c190 tx=0x7f66dc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7785d6080937b928 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.026+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66e4007cc0 con 0x7f66dc004800 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.026+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66e4030070 con 0x7f66dc004800 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.026+0000 7f66f2ffd640 1 -- 172.21.15.64:0/1176235821 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66e4005d30 con 0x7f66dc004800 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.026+0000 7f66fa415640 1 -- 172.21.15.64:0/1176235821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66dc004800 msgr2=0x7f66dc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.026+0000 7f66fa415640 1 --2- 172.21.15.64:0/1176235821 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66dc004800 0x7f66dc004c80 secure :-1 s=READY pgs=1229 cs=0 l=1 rev1=1 crypto rx=0x7f66e400c190 tx=0x7f66dc005830 comp rx=0 tx=0).stop 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.026+0000 7f66f89b2640 1 -- 172.21.15.64:0/1176235821 reap_dead start 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.027+0000 7f66fa415640 1 -- 172.21.15.64:0/1176235821 shutdown_connections 2024-03-04T07:31:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.027+0000 7f66fa415640 1 -- 172.21.15.64:0/1176235821 >> 172.21.15.64:0/1176235821 conn(0x7f66f40d1cd0 msgr2=0x7f66f40d20e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.027+0000 7f66fa415640 1 -- 172.21.15.64:0/1176235821 shutdown_connections 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.027+0000 7f66fa415640 1 -- 172.21.15.64:0/1176235821 wait complete. 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.028+0000 7f66fa415640 1 Processor -- start 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.028+0000 7f66fa415640 1 -- start start 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc004800 0x7f66f415c880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cb620 0x7f66f415bfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66f41e7a60 0x7f66f41e7ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66f40d0a90 con 0x7f66f41e7a60 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66f40cefa0 con 0x7f66dc004800 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f66f40d0c10 con 0x7f66f40cb620 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66f41e7a60 0x7f66f41e7ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66f41e7a60 0x7f66f41e7ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44718/0 (socket says 172.21.15.64:44718) 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 -- 172.21.15.64:0/4231428619 learned_addr learned my addr 172.21.15.64:0/4231428619 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f3fff640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc004800 0x7f66f415c880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f37fe640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cb620 0x7f66f415bfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 -- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cb620 msgr2=0x7f66f415bfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cb620 0x7f66f415bfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 -- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc004800 msgr2=0x7f66f415c880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc004800 0x7f66f415c880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 -- 172.21.15.64:0/4231428619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66f40cbaa0 con 0x7f66f41e7a60 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f89b2640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66f41e7a60 0x7f66f41e7ee0 secure :-1 s=READY pgs=1230 cs=0 l=1 rev1=1 crypto rx=0x7f66ec001330 tx=0x7f66ec00ec00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f0ff9640 1 -- 172.21.15.64:0/4231428619 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66ec00ee50 con 0x7f66f41e7a60 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f0ff9640 1 -- 172.21.15.64:0/4231428619 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f66ec003b80 con 0x7f66f41e7a60 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66f0ff9640 1 -- 172.21.15.64:0/4231428619 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66ec003d70 con 0x7f66f41e7a60 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66f40ca2f0 con 0x7f66f41e7a60 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.029+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66f40ca8b0 con 0x7f66f41e7a60 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.031+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f66f40752d0 con 0x7f66f41e7a60 2024-03-04T07:31:55.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.031+0000 7f66f0ff9640 1 -- 172.21.15.64:0/4231428619 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f66ec01a020 con 0x7f66f41e7a60 2024-03-04T07:31:55.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.031+0000 7f66f0ff9640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f66e006b500 0x7f66e006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.031+0000 7f66f3fff640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f66e006b500 0x7f66e006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.031+0000 7f66f0ff9640 1 -- 172.21.15.64:0/4231428619 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f66ec089700 con 0x7f66f41e7a60 2024-03-04T07:31:55.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.032+0000 7f66f3fff640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f66e006b500 0x7f66e006d9c0 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f66e8009900 tx=0x7f66e8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.036+0000 7f66f0ff9640 1 -- 172.21.15.64:0/4231428619 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f66ec021020 con 0x7f66f41e7a60 2024-03-04T07:31:55.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.351+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f66f40b1140 con 0x7f66f41e7a60 2024-03-04T07:31:55.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.351+0000 7f66f0ff9640 1 -- 172.21.15.64:0/4231428619 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f66ec058560 con 0x7f66f41e7a60 2024-03-04T07:31:55.352 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:55.353 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:55.353 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f66e006b500 msgr2=0x7f66e006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f66e006b500 0x7f66e006d9c0 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f66e8009900 tx=0x7f66e8008040 comp rx=0 tx=0).stop 2024-03-04T07:31:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66f41e7a60 msgr2=0x7f66f41e7ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66f41e7a60 0x7f66f41e7ee0 secure :-1 s=READY pgs=1230 cs=0 l=1 rev1=1 crypto rx=0x7f66ec001330 tx=0x7f66ec00ec00 comp rx=0 tx=0).stop 2024-03-04T07:31:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 shutdown_connections 2024-03-04T07:31:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f66e006b500 0x7f66e006d9c0 unknown :-1 s=CLOSED pgs=557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f66f41e7a60 0x7f66f41e7ee0 unknown :-1 s=CLOSED pgs=1230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f66f40cb620 0x7f66f415bfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 --2- 172.21.15.64:0/4231428619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f66dc004800 0x7f66f415c880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 >> 172.21.15.64:0/4231428619 conn(0x7f66f40d1cd0 msgr2=0x7f66f40d02c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 shutdown_connections 2024-03-04T07:31:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.353+0000 7f66fa415640 1 -- 172.21.15.64:0/4231428619 wait complete. 2024-03-04T07:31:55.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:31:55.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.468+0000 7f81d452b640 1 Processor -- start 2024-03-04T07:31:55.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.468+0000 7f81d452b640 1 -- start start 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.469+0000 7f81d452b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f81cc15c2e0 0x7f81cc0ce3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.469+0000 7f81d452b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81cc0d0a90 0x7f81cc0ce910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.469+0000 7f81d452b640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81cc15c6c0 con 0x7f81cc0d27c0 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.469+0000 7f81d452b640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f81cc0cd7a0 con 0x7f81cc0d0a90 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.469+0000 7f81d452b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f81cc0cda80 con 0x7f81cc15c2e0 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d2ac8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81cc0d0a90 0x7f81cc0ce910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d2ac8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81cc0d0a90 0x7f81cc0ce910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44726/0 (socket says 172.21.15.64:44726) 2024-03-04T07:31:55.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d1ac6640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f81cc0d27c0 0x7f81cc0b1f60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35380/0 (socket says 172.21.15.64:35380) 2024-03-04T07:31:55.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2291139158 learned_addr learned my addr 172.21.15.64:0/2291139158 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:55.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d22c7640 1 --2- 172.21.15.64:0/2291139158 >> v2:172.21.15.66:3300/0 conn(0x7f81cc15c2e0 0x7f81cc0ce3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2291139158 >> v2:172.21.15.66:3300/0 conn(0x7f81cc15c2e0 msgr2=0x7f81cc0ce3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:55.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d2ac8640 1 --2- 172.21.15.64:0/2291139158 >> v2:172.21.15.66:3300/0 conn(0x7f81cc15c2e0 0x7f81cc0ce3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2291139158 >> v1:172.21.15.64:6789/0 conn(0x7f81cc0d27c0 legacy=0x7f81cc0b1f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.470+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2291139158 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81cc0b1760 con 0x7f81cc0d0a90 2024-03-04T07:31:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d2ac8640 1 --2- 172.21.15.64:0/2291139158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc0d0a90 0x7f81cc0ce910 secure :-1 s=READY pgs=1232 cs=0 l=1 rev1=1 crypto rx=0x7f81bc009870 tx=0x7f81bc02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=76dba884c2bcc95 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d12c5640 1 -- 172.21.15.64:0/2291139158 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81bc034070 con 0x7f81cc0d0a90 2024-03-04T07:31:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d12c5640 1 -- 172.21.15.64:0/2291139158 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81bc030860 con 0x7f81cc0d0a90 2024-03-04T07:31:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d12c5640 1 -- 172.21.15.64:0/2291139158 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81bc030c30 con 0x7f81cc0d0a90 2024-03-04T07:31:55.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 -- 172.21.15.64:0/2291139158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc0d0a90 msgr2=0x7f81cc0ce910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 --2- 172.21.15.64:0/2291139158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc0d0a90 0x7f81cc0ce910 secure :-1 s=READY pgs=1232 cs=0 l=1 rev1=1 crypto rx=0x7f81bc009870 tx=0x7f81bc02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 -- 172.21.15.64:0/2291139158 shutdown_connections 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 --2- 172.21.15.64:0/2291139158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc0d0a90 0x7f81cc0ce910 unknown :-1 s=CLOSED pgs=1232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 --2- 172.21.15.64:0/2291139158 >> v2:172.21.15.66:3300/0 conn(0x7f81cc15c2e0 0x7f81cc0ce3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 -- 172.21.15.64:0/2291139158 >> 172.21.15.64:0/2291139158 conn(0x7f81cc0d1ca0 msgr2=0x7f81cc0d20b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 -- 172.21.15.64:0/2291139158 shutdown_connections 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.471+0000 7f81d452b640 1 -- 172.21.15.64:0/2291139158 wait complete. 2024-03-04T07:31:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.473+0000 7f81d452b640 1 Processor -- start 2024-03-04T07:31:55.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.473+0000 7f81d452b640 1 -- start start 2024-03-04T07:31:55.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.474+0000 7f81d452b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81cc0d0a90 0x7f81cc1e8050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d452b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81cc0d27c0 0x7f81cc06d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d452b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc15c2e0 0x7f81cc0b3c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d452b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81cc0cd540 con 0x7f81cc15c2e0 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d452b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81cc0cd3c0 con 0x7f81cc0d27c0 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d452b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81cc0b2670 con 0x7f81cc0d0a90 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc15c2e0 0x7f81cc0b3c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc15c2e0 0x7f81cc0b3c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44738/0 (socket says 172.21.15.64:44738) 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2926136734 learned_addr learned my addr 172.21.15.64:0/2926136734 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d1ac6640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81cc0d27c0 0x7f81cc06d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d22c7640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81cc0d0a90 0x7f81cc1e8050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81cc0d0a90 msgr2=0x7f81cc1e8050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81cc0d0a90 0x7f81cc1e8050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81cc0d27c0 msgr2=0x7f81cc06d850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81cc0d27c0 0x7f81cc06d850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 -- 172.21.15.64:0/2926136734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81cc06c2b0 con 0x7f81cc15c2e0 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d1ac6640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81cc0d27c0 0x7f81cc06d850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d22c7640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81cc0d0a90 0x7f81cc1e8050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d2ac8640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc15c2e0 0x7f81cc0b3c00 secure :-1 s=READY pgs=1233 cs=0 l=1 rev1=1 crypto rx=0x7f81bc009870 tx=0x7f81bc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81caffd640 1 -- 172.21.15.64:0/2926136734 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81bc048070 con 0x7f81cc15c2e0 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81caffd640 1 -- 172.21.15.64:0/2926136734 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81bc030660 con 0x7f81cc15c2e0 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81caffd640 1 -- 172.21.15.64:0/2926136734 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81bc034040 con 0x7f81cc15c2e0 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81cc06c590 con 0x7f81cc15c2e0 2024-03-04T07:31:55.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.475+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81cc06cb50 con 0x7f81cc15c2e0 2024-03-04T07:31:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.477+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81cc0752d0 con 0x7f81cc15c2e0 2024-03-04T07:31:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.477+0000 7f81caffd640 1 -- 172.21.15.64:0/2926136734 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f81bc04b070 con 0x7f81cc15c2e0 2024-03-04T07:31:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.477+0000 7f81caffd640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81b806b4d0 0x7f81b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.477+0000 7f81caffd640 1 -- 172.21.15.64:0/2926136734 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f81bc0ac3d0 con 0x7f81cc15c2e0 2024-03-04T07:31:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.477+0000 7f81d22c7640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81b806b4d0 0x7f81b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.478+0000 7f81d22c7640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81b806b4d0 0x7f81b806d990 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f81c4009f50 tx=0x7f81c400b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.482+0000 7f81caffd640 1 -- 172.21.15.64:0/2926136734 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f81bc07b380 con 0x7f81cc15c2e0 2024-03-04T07:31:55.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.705+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f81cc0748b0 con 0x7f81cc15c2e0 2024-03-04T07:31:55.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.706+0000 7f81caffd640 1 -- 172.21.15.64:0/2926136734 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 59 v59) v1 ==== 94+0+4980 (secure 0 0 0) 0x7f81bc07ac80 con 0x7f81cc15c2e0 2024-03-04T07:31:55.707 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:55.707 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":59,"default_fscid":7,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56}],"filesystems":[{"mdsmap":{"epoch":59,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:31:14.475450+0000","modified":"2024-03-04T07:31:28.181905+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":94,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7557},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7557":{"gid":7557,"name":"c","rank":0,"incarnation":55,"state":"up:active","state_seq":5,"addr":"172.21.15.64:6835/3393965936","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3393965936},{"type":"v1","addr":"172.21.15.64:6835","nonce":3393965936}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[15],"metadata_pool":14,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7557,"qdb_cluster":[7557]},"id":7}]} 2024-03-04T07:31:55.707 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 59 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.707+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81b806b4d0 msgr2=0x7f81b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81b806b4d0 0x7f81b806d990 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f81c4009f50 tx=0x7f81c400b040 comp rx=0 tx=0).stop 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc15c2e0 msgr2=0x7f81cc0b3c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc15c2e0 0x7f81cc0b3c00 secure :-1 s=READY pgs=1233 cs=0 l=1 rev1=1 crypto rx=0x7f81bc009870 tx=0x7f81bc004730 comp rx=0 tx=0).stop 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 shutdown_connections 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81b806b4d0 0x7f81b806d990 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81cc15c2e0 0x7f81cc0b3c00 unknown :-1 s=CLOSED pgs=1233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81cc0d27c0 0x7f81cc06d850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 --2- 172.21.15.64:0/2926136734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81cc0d0a90 0x7f81cc1e8050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 >> 172.21.15.64:0/2926136734 conn(0x7f81cc0d1ca0 msgr2=0x7f81cc06f910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:55.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 shutdown_connections 2024-03-04T07:31:55.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.708+0000 7f81d452b640 1 -- 172.21.15.64:0/2926136734 wait complete. 2024-03-04T07:31:55.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:31:55.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.821+0000 7f1abf3c7640 1 Processor -- start 2024-03-04T07:31:55.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.821+0000 7f1abf3c7640 1 -- start start 2024-03-04T07:31:55.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.822+0000 7f1abf3c7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1ab815c2e0 0x7f1ab80ce3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.822+0000 7f1abf3c7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab80d0a90 0x7f1ab80ce910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.822+0000 7f1abf3c7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1ab815c6c0 con 0x7f1ab80d27c0 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.822+0000 7f1abf3c7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1ab80cd7a0 con 0x7f1ab815c2e0 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.822+0000 7f1abf3c7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ab80cda80 con 0x7f1ab80d0a90 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.822+0000 7f1abd964640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab80d0a90 0x7f1ab80ce910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.822+0000 7f1abd163640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1ab815c2e0 0x7f1ab80ce3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abc962640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f1ab80d27c0 0x7f1ab80b1f60 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43770/0 (socket says 172.21.15.64:43770) 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abd163640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1ab815c2e0 0x7f1ab80ce3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:48472/0 (socket says 172.21.15.64:48472) 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abc962640 1 -- 172.21.15.64:0/2856635165 learned_addr learned my addr 172.21.15.64:0/2856635165 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abd964640 1 -- 172.21.15.64:0/2856635165 >> v2:172.21.15.66:3300/0 conn(0x7f1ab815c2e0 msgr2=0x7f1ab80ce3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abd964640 1 --2- 172.21.15.64:0/2856635165 >> v2:172.21.15.66:3300/0 conn(0x7f1ab815c2e0 0x7f1ab80ce3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abd964640 1 -- 172.21.15.64:0/2856635165 >> v1:172.21.15.66:6789/0 conn(0x7f1ab80d27c0 legacy=0x7f1ab80b1f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abd964640 1 -- 172.21.15.64:0/2856635165 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ab80b1760 con 0x7f1ab80d0a90 2024-03-04T07:31:55.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.823+0000 7f1abd163640 1 --2- 172.21.15.64:0/2856635165 >> v2:172.21.15.66:3300/0 conn(0x7f1ab815c2e0 0x7f1ab80ce3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:55.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1abd964640 1 --2- 172.21.15.64:0/2856635165 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab80d0a90 0x7f1ab80ce910 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f1ab0009dc0 tx=0x7f1ab002be90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e89ed969872f95be server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1aabfff640 1 -- 172.21.15.64:0/2856635165 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ab0039070 con 0x7f1ab80d0a90 2024-03-04T07:31:55.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1aabfff640 1 -- 172.21.15.64:0/2856635165 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ab002cc40 con 0x7f1ab80d0a90 2024-03-04T07:31:55.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1aabfff640 1 -- 172.21.15.64:0/2856635165 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ab0034440 con 0x7f1ab80d0a90 2024-03-04T07:31:55.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1abf3c7640 1 -- 172.21.15.64:0/2856635165 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab80d0a90 msgr2=0x7f1ab80ce910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/2856635165 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab80d0a90 0x7f1ab80ce910 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f1ab0009dc0 tx=0x7f1ab002be90 comp rx=0 tx=0).stop 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1abf3c7640 1 -- 172.21.15.64:0/2856635165 shutdown_connections 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/2856635165 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab80d0a90 0x7f1ab80ce910 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/2856635165 >> v2:172.21.15.66:3300/0 conn(0x7f1ab815c2e0 0x7f1ab80ce3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.824+0000 7f1abf3c7640 1 -- 172.21.15.64:0/2856635165 >> 172.21.15.64:0/2856635165 conn(0x7f1ab80d1ca0 msgr2=0x7f1ab80d20b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.825+0000 7f1abf3c7640 1 -- 172.21.15.64:0/2856635165 shutdown_connections 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.825+0000 7f1abf3c7640 1 -- 172.21.15.64:0/2856635165 wait complete. 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 Processor -- start 2024-03-04T07:31:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 -- start start 2024-03-04T07:31:55.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ab80d0a90 0x7f1ab81e90f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ab80d27c0 0x7f1ab8158ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab815c2e0 0x7f1ab8159410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ab80cd540 con 0x7f1ab80d0a90 2024-03-04T07:31:55.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ab80cd3c0 con 0x7f1ab80d27c0 2024-03-04T07:31:55.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.826+0000 7f1abf3c7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1ab80b2670 con 0x7f1ab815c2e0 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ab80d0a90 0x7f1ab81e90f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ab80d0a90 0x7f1ab81e90f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44748/0 (socket says 172.21.15.64:44748) 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 -- 172.21.15.64:0/4244868523 learned_addr learned my addr 172.21.15.64:0/4244868523 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abc962640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ab80d27c0 0x7f1ab8158ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd964640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab815c2e0 0x7f1ab8159410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 -- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab815c2e0 msgr2=0x7f1ab8159410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab815c2e0 0x7f1ab8159410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 -- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ab80d27c0 msgr2=0x7f1ab8158ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ab80d27c0 0x7f1ab8158ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 -- 172.21.15.64:0/4244868523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ab80ca500 con 0x7f1ab80d0a90 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abd163640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ab80d0a90 0x7f1ab81e90f0 secure :-1 s=READY pgs=1234 cs=0 l=1 rev1=1 crypto rx=0x7f1ab400e990 tx=0x7f1ab400ee50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1aa9ffb640 1 -- 172.21.15.64:0/4244868523 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ab4017070 con 0x7f1ab80d0a90 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1aa9ffb640 1 -- 172.21.15.64:0/4244868523 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1ab400c9e0 con 0x7f1ab80d0a90 2024-03-04T07:31:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ab80ca7e0 con 0x7f1ab80d0a90 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.827+0000 7f1aa9ffb640 1 -- 172.21.15.64:0/4244868523 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ab400cdb0 con 0x7f1ab80d0a90 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.828+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ab80cada0 con 0x7f1ab80d0a90 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.829+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1ab80b60b0 con 0x7f1ab80d0a90 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.829+0000 7f1aa9ffb640 1 -- 172.21.15.64:0/4244868523 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1ab401a070 con 0x7f1ab80d0a90 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.829+0000 7f1aa9ffb640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a9c06b4d0 0x7f1a9c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.829+0000 7f1aa9ffb640 1 -- 172.21.15.64:0/4244868523 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f1ab408a440 con 0x7f1ab80d0a90 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.830+0000 7f1abc962640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a9c06b4d0 0x7f1a9c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:55.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.830+0000 7f1abc962640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a9c06b4d0 0x7f1a9c06d990 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f1ab81f7d50 tx=0x7f1aac007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:55.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:55.834+0000 7f1aa9ffb640 1 -- 172.21.15.64:0/4244868523 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1ab80b60b0 con 0x7f1ab80d0a90 2024-03-04T07:31:56.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.017+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f1ab80752d0 con 0x7f1ab80d0a90 2024-03-04T07:31:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.018+0000 7f1aa9ffb640 1 -- 172.21.15.64:0/4244868523 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v94) v1 ==== 74+0+16819 (secure 0 0 0) 0x7f1ab4058bc0 con 0x7f1ab80d0a90 2024-03-04T07:31:56.019 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:56.019 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":94,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:24.148060+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":14,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:31:12.750651+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":15,"pool_name":"cephfs_data","create_time":"2024-03-04T07:31:13.444529+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"90","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.64:6835/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.66:6835/3397672669":"2024-03-05T07:31:23.154858+0000","172.21.15.66:6833/3397672669":"2024-03-05T07:31:23.154858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:31:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a9c06b4d0 msgr2=0x7f1a9c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a9c06b4d0 0x7f1a9c06d990 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f1ab81f7d50 tx=0x7f1aac007e80 comp rx=0 tx=0).stop 2024-03-04T07:31:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ab80d0a90 msgr2=0x7f1ab81e90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ab80d0a90 0x7f1ab81e90f0 secure :-1 s=READY pgs=1234 cs=0 l=1 rev1=1 crypto rx=0x7f1ab400e990 tx=0x7f1ab400ee50 comp rx=0 tx=0).stop 2024-03-04T07:31:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 shutdown_connections 2024-03-04T07:31:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a9c06b4d0 0x7f1a9c06d990 unknown :-1 s=CLOSED pgs=559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1ab815c2e0 0x7f1ab8159410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1ab80d27c0 0x7f1ab8158ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 --2- 172.21.15.64:0/4244868523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1ab80d0a90 0x7f1ab81e90f0 unknown :-1 s=CLOSED pgs=1234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 >> 172.21.15.64:0/4244868523 conn(0x7f1ab80d1ca0 msgr2=0x7f1ab80cc870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 shutdown_connections 2024-03-04T07:31:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.020+0000 7f1abf3c7640 1 -- 172.21.15.64:0/4244868523 wait complete. 2024-03-04T07:31:56.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:31:56.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.160+0000 7fc94aa1c640 1 Processor -- start 2024-03-04T07:31:56.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.160+0000 7fc94aa1c640 1 -- start start 2024-03-04T07:31:56.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc94aa1c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc9440d2680 0x7fc9440d4a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc94aa1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b3050 0x7fc9440b0b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc94aa1c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9440ceab0 con 0x7fc9440b1040 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc94aa1c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc9440cf380 con 0x7fc9440d2680 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc94aa1c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9440d1d10 con 0x7fc9440b3050 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc943fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc9440d2680 0x7fc9440d4a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc943fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc9440d2680 0x7fc9440d4a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44762/0 (socket says 172.21.15.64:44762) 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc943fff640 1 -- 172.21.15.64:0/782206354 learned_addr learned my addr 172.21.15.64:0/782206354 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.161+0000 7fc9437fe640 1 --2- 172.21.15.64:0/782206354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b3050 0x7fc9440b0b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.162+0000 7fc943fff640 1 -- 172.21.15.64:0/782206354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b3050 msgr2=0x7fc9440b0b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.162+0000 7fc943fff640 1 --2- 172.21.15.64:0/782206354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b3050 0x7fc9440b0b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.162+0000 7fc943fff640 1 -- 172.21.15.64:0/782206354 >> v1:172.21.15.64:6789/0 conn(0x7fc9440b1040 legacy=0x7fc9440cf810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.162+0000 7fc943fff640 1 -- 172.21.15.64:0/782206354 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9440d1ff0 con 0x7fc9440d2680 2024-03-04T07:31:56.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc943fff640 1 --2- 172.21.15.64:0/782206354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440d2680 0x7fc9440d4a50 secure :-1 s=READY pgs=1236 cs=0 l=1 rev1=1 crypto rx=0x7fc934009870 tx=0x7fc93402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=24596c9c446de1ad server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:56.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc942ffd640 1 -- 172.21.15.64:0/782206354 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc934034070 con 0x7fc9440d2680 2024-03-04T07:31:56.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc942ffd640 1 -- 172.21.15.64:0/782206354 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc934030840 con 0x7fc9440d2680 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc942ffd640 1 -- 172.21.15.64:0/782206354 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc934030c10 con 0x7fc9440d2680 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 -- 172.21.15.64:0/782206354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440d2680 msgr2=0x7fc9440d4a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/782206354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440d2680 0x7fc9440d4a50 secure :-1 s=READY pgs=1236 cs=0 l=1 rev1=1 crypto rx=0x7fc934009870 tx=0x7fc93402b980 comp rx=0 tx=0).stop 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 -- 172.21.15.64:0/782206354 shutdown_connections 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/782206354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b3050 0x7fc9440b0b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/782206354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440d2680 0x7fc9440d4a50 unknown :-1 s=CLOSED pgs=1236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 -- 172.21.15.64:0/782206354 >> 172.21.15.64:0/782206354 conn(0x7fc9440d3f30 msgr2=0x7fc9440d4340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 -- 172.21.15.64:0/782206354 shutdown_connections 2024-03-04T07:31:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.163+0000 7fc94aa1c640 1 -- 172.21.15.64:0/782206354 wait complete. 2024-03-04T07:31:56.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.164+0000 7fc94aa1c640 1 Processor -- start 2024-03-04T07:31:56.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc94aa1c640 1 -- start start 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc94aa1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b1040 0x7fc9441535e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc94aa1c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440b3050 0x7fc944161df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc94aa1c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9440d2680 0x7fc944162330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc94aa1c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9440cff20 con 0x7fc9440b3050 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc94aa1c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9440cf010 con 0x7fc9440d2680 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc94aa1c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9440d00a0 con 0x7fc9440b1040 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.165+0000 7fc9437fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440b3050 0x7fc944161df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440b3050 0x7fc944161df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44778/0 (socket says 172.21.15.64:44778) 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 -- 172.21.15.64:0/2870593619 learned_addr learned my addr 172.21.15.64:0/2870593619 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc948fb9640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9440d2680 0x7fc944162330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc943fff640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b1040 0x7fc9441535e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 -- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b1040 msgr2=0x7fc9441535e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b1040 0x7fc9441535e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 -- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9440d2680 msgr2=0x7fc944162330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9440d2680 0x7fc944162330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 -- 172.21.15.64:0/2870593619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc94415d2a0 con 0x7fc9440b3050 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc943fff640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b1040 0x7fc9441535e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc948fb9640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9440d2680 0x7fc944162330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc9437fe640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440b3050 0x7fc944161df0 secure :-1 s=READY pgs=1237 cs=0 l=1 rev1=1 crypto rx=0x7fc93800e9b0 tx=0x7fc93800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc940ff9640 1 -- 172.21.15.64:0/2870593619 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc938017070 con 0x7fc9440b3050 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc940ff9640 1 -- 172.21.15.64:0/2870593619 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc93800c9a0 con 0x7fc9440b3050 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc94415d580 con 0x7fc9440b3050 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc940ff9640 1 -- 172.21.15.64:0/2870593619 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc93800cd70 con 0x7fc9440b3050 2024-03-04T07:31:56.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.166+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc94415db40 con 0x7fc9440b3050 2024-03-04T07:31:56.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.167+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc9440752d0 con 0x7fc9440b3050 2024-03-04T07:31:56.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.168+0000 7fc940ff9640 1 -- 172.21.15.64:0/2870593619 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc93801a070 con 0x7fc9440b3050 2024-03-04T07:31:56.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.168+0000 7fc940ff9640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc93006b4d0 0x7fc93006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.168+0000 7fc940ff9640 1 -- 172.21.15.64:0/2870593619 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fc93808a570 con 0x7fc9440b3050 2024-03-04T07:31:56.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.168+0000 7fc943fff640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc93006b4d0 0x7fc93006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.169+0000 7fc943fff640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc93006b4d0 0x7fc93006d990 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7fc934036000 tx=0x7fc93403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:56.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.173+0000 7fc940ff9640 1 -- 172.21.15.64:0/2870593619 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc9380593d0 con 0x7fc9440b3050 2024-03-04T07:31:56.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.395+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7fc9440b9d10 con 0x7fc9440b3050 2024-03-04T07:31:56.396 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:31:56.395+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:31:56.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.397+0000 7fc940ff9640 1 -- 172.21.15.64:0/2870593619 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v60) v1 ==== 161+0+0 (secure 0 0 0) 0x7fc938058cd0 con 0x7fc9440b3050 2024-03-04T07:31:56.398 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:31:56.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.400+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc93006b4d0 msgr2=0x7fc93006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.400+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc93006b4d0 0x7fc93006d990 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7fc934036000 tx=0x7fc93403b040 comp rx=0 tx=0).stop 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.400+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440b3050 msgr2=0x7fc944161df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.400+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440b3050 0x7fc944161df0 secure :-1 s=READY pgs=1237 cs=0 l=1 rev1=1 crypto rx=0x7fc93800e9b0 tx=0x7fc93800ee70 comp rx=0 tx=0).stop 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 shutdown_connections 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc93006b4d0 0x7fc93006d990 unknown :-1 s=CLOSED pgs=560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9440d2680 0x7fc944162330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9440b3050 0x7fc944161df0 unknown :-1 s=CLOSED pgs=1237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 --2- 172.21.15.64:0/2870593619 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9440b1040 0x7fc9441535e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 >> 172.21.15.64:0/2870593619 conn(0x7fc9440d3f30 msgr2=0x7fc94406b7e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 shutdown_connections 2024-03-04T07:31:56.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.401+0000 7fc94aa1c640 1 -- 172.21.15.64:0/2870593619 wait complete. 2024-03-04T07:31:56.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:31:56.439 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:31:56.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.523+0000 7f0501956640 1 Processor -- start 2024-03-04T07:31:56.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.523+0000 7f0501956640 1 -- start start 2024-03-04T07:31:56.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.524+0000 7f0501956640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 0x7f04fc06e0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.524+0000 7f0501956640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f04fc06daa0 0x7f04fc0b3540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.524+0000 7f0501956640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f04fc0b2aa0 con 0x7f04fc0714b0 2024-03-04T07:31:56.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.524+0000 7f0501956640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f04fc0b30b0 con 0x7f04fc06daa0 2024-03-04T07:31:56.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.524+0000 7f0501956640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04fc071990 con 0x7f04fc15a470 2024-03-04T07:31:56.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.524+0000 7f04fa7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f04fc0714b0 0x7f04fc06d390 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35394/0 (socket says 172.21.15.64:35394) 2024-03-04T07:31:56.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.524+0000 7f04fa7fc640 1 -- 172.21.15.64:0/1699472084 learned_addr learned my addr 172.21.15.64:0/1699472084 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fb7fe640 1 --2- 172.21.15.64:0/1699472084 >> v2:172.21.15.66:3300/0 conn(0x7f04fc06daa0 0x7f04fc0b3540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04faffd640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 0x7f04fc06e0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2060978442 0 0) 0x7f04fc0b2aa0 con 0x7f04fc0714b0 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f04e4003520 con 0x7f04fc0714b0 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2305986119 0 0) 0x7f04e4003520 con 0x7f04fc0714b0 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 msgr2=0x7f04fc06e0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 0x7f04fc06e0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 >> v2:172.21.15.66:3300/0 conn(0x7f04fc06daa0 msgr2=0x7f04fc0b3540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 --2- 172.21.15.64:0/1699472084 >> v2:172.21.15.66:3300/0 conn(0x7f04fc06daa0 0x7f04fc0b3540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04fc071c70 con 0x7f04fc0714b0 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f04f0001670 con 0x7f04fc0714b0 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fb7fe640 1 --2- 172.21.15.64:0/1699472084 >> v2:172.21.15.66:3300/0 conn(0x7f04fc06daa0 0x7f04fc0b3540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 >> v1:172.21.15.64:6789/0 conn(0x7f04fc0714b0 legacy=0x7f04fc06d390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04e40039b0 0x7f04e4003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04e4004800 0x7f04e4004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 0x7f04e40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04fc0b30b0 con 0x7f04e4004800 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04fc071990 con 0x7f04e40039b0 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04e400b9c0 con 0x7f04fc15a470 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fa7fc640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04e4004800 0x7f04e4004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04faffd640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04e40039b0 0x7f04e4003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fb7fe640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 0x7f04e40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fa7fc640 1 -- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 msgr2=0x7f04e40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fa7fc640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04fc15a470 0x7f04e40051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fa7fc640 1 -- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04e40039b0 msgr2=0x7f04e4003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fa7fc640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04e40039b0 0x7f04e4003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04fa7fc640 1 -- 172.21.15.64:0/1699472084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04e4003520 con 0x7f04e4004800 2024-03-04T07:31:56.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.525+0000 7f04faffd640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04e40039b0 0x7f04e4003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.526+0000 7f04fa7fc640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04e4004800 0x7f04e4004c80 secure :-1 s=READY pgs=1241 cs=0 l=1 rev1=1 crypto rx=0x7f04f0007cf0 tx=0x7f04e4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a7275ce375ebfce9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.526+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04f0007d30 con 0x7f04e4004800 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.526+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f04f0032070 con 0x7f04e4004800 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.526+0000 7f04f9ffb640 1 -- 172.21.15.64:0/1699472084 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04f00058d0 con 0x7f04e4004800 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.527+0000 7f0501956640 1 -- 172.21.15.64:0/1699472084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04e4004800 msgr2=0x7f04e4004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.527+0000 7f0501956640 1 --2- 172.21.15.64:0/1699472084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04e4004800 0x7f04e4004c80 secure :-1 s=READY pgs=1241 cs=0 l=1 rev1=1 crypto rx=0x7f04f0007cf0 tx=0x7f04e4005830 comp rx=0 tx=0).stop 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.527+0000 7f04fb7fe640 1 -- 172.21.15.64:0/1699472084 reap_dead start 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.527+0000 7f0501956640 1 -- 172.21.15.64:0/1699472084 shutdown_connections 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.527+0000 7f0501956640 1 -- 172.21.15.64:0/1699472084 >> 172.21.15.64:0/1699472084 conn(0x7f04fc0b17a0 msgr2=0x7f04fc0b1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.527+0000 7f0501956640 1 -- 172.21.15.64:0/1699472084 shutdown_connections 2024-03-04T07:31:56.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.527+0000 7f0501956640 1 -- 172.21.15.64:0/1699472084 wait complete. 2024-03-04T07:31:56.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.528+0000 7f0501956640 1 Processor -- start 2024-03-04T07:31:56.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.528+0000 7f0501956640 1 -- start start 2024-03-04T07:31:56.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f0501956640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04e4004800 0x7f04fc150110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f0501956640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04fc1565f0 0x7f04fc13d880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f0501956640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04fc1df890 0x7f04fc1dfd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f0501956640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04fc0cf850 con 0x7f04fc1df890 2024-03-04T07:31:56.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f0501956640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04fc0cf6d0 con 0x7f04fc1565f0 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f0501956640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f04fc15a850 con 0x7f04e4004800 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04fc1df890 0x7f04fc1dfd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04fc1df890 0x7f04fc1dfd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44814/0 (socket says 172.21.15.64:44814) 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 -- 172.21.15.64:0/2177140537 learned_addr learned my addr 172.21.15.64:0/2177140537 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04faffd640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04e4004800 0x7f04fc150110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fa7fc640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04fc1565f0 0x7f04fc13d880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 -- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04e4004800 msgr2=0x7f04fc150110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04e4004800 0x7f04fc150110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 -- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04fc1565f0 msgr2=0x7f04fc13d880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04fc1565f0 0x7f04fc13d880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fb7fe640 1 -- 172.21.15.64:0/2177140537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04fc150990 con 0x7f04fc1df890 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04faffd640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04e4004800 0x7f04fc150110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.529+0000 7f04fa7fc640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04fc1565f0 0x7f04fc13d880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.530+0000 7f04fb7fe640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04fc1df890 0x7f04fc1dfd10 secure :-1 s=READY pgs=1242 cs=0 l=1 rev1=1 crypto rx=0x7f04fc0af6c0 tx=0x7f04f4010580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.530+0000 7f04e3fff640 1 -- 172.21.15.64:0/2177140537 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04f4010c40 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.530+0000 7f04e3fff640 1 -- 172.21.15.64:0/2177140537 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f04f401a070 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.530+0000 7f04e3fff640 1 -- 172.21.15.64:0/2177140537 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04f4003990 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.530+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04fc156a70 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.530+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04fc156fd0 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.531+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f04fc068050 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.531+0000 7f04e3fff640 1 -- 172.21.15.64:0/2177140537 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f04f4005d10 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.532+0000 7f04e3fff640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f04e806b4d0 0x7f04e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.532+0000 7f04e3fff640 1 -- 172.21.15.64:0/2177140537 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(95..95 src has 1..95) v4 ==== 5967+0+0 (secure 0 0 0) 0x7f04f408a9e0 con 0x7f04fc1df890 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.532+0000 7f04faffd640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f04e806b4d0 0x7f04e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:56.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.532+0000 7f04faffd640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f04e806b4d0 0x7f04e806d990 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7f04fc152250 tx=0x7f04ec008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:56.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.536+0000 7f04e3fff640 1 -- 172.21.15.64:0/2177140537 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f04f4024020 con 0x7f04fc1df890 2024-03-04T07:31:56.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:56.755+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f04fc0b4da0 con 0x7f04fc1df890 2024-03-04T07:31:57.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.405+0000 7f04e3fff640 1 -- 172.21.15.64:0/2177140537 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v61) v1 ==== 104+0+0 (secure 0 0 0) 0x7f04fc156a70 con 0x7f04fc1df890 2024-03-04T07:31:57.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f04e806b4d0 msgr2=0x7f04e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:57.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f04e806b4d0 0x7f04e806d990 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7f04fc152250 tx=0x7f04ec008040 comp rx=0 tx=0).stop 2024-03-04T07:31:57.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04fc1df890 msgr2=0x7f04fc1dfd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04fc1df890 0x7f04fc1dfd10 secure :-1 s=READY pgs=1242 cs=0 l=1 rev1=1 crypto rx=0x7f04fc0af6c0 tx=0x7f04f4010580 comp rx=0 tx=0).stop 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 shutdown_connections 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f04e806b4d0 0x7f04e806d990 unknown :-1 s=CLOSED pgs=561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f04fc1df890 0x7f04fc1dfd10 unknown :-1 s=CLOSED pgs=1242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f04fc1565f0 0x7f04fc13d880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 --2- 172.21.15.64:0/2177140537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f04e4004800 0x7f04fc150110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 >> 172.21.15.64:0/2177140537 conn(0x7f04fc0b17a0 msgr2=0x7f04fc0b3420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 shutdown_connections 2024-03-04T07:31:57.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.407+0000 7f0501956640 1 -- 172.21.15.64:0/2177140537 wait complete. 2024-03-04T07:31:57.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:31:57.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.571+0000 7f4b7f25c640 1 Processor -- start 2024-03-04T07:31:57.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.571+0000 7f4b7f25c640 1 -- start start 2024-03-04T07:31:57.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7f25c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b78161720 0x7f4b780d1020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7f25c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b780d2c10 con 0x7f4b780b3ec0 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7f25c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b780b5180 con 0x7f4b780b2d10 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7f25c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b780d2ef0 con 0x7f4b78161720 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7d7f9640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4b780b3ec0 0x7f4b780d1d90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35408/0 (socket says 172.21.15.64:35408) 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7d7f9640 1 -- 172.21.15.64:0/145248883 learned_addr learned my addr 172.21.15.64:0/145248883 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7cff8640 1 --2- 172.21.15.64:0/145248883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b78161720 0x7f4b780d1020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2292908452 0 0) 0x7f4b780d2c10 con 0x7f4b780b3ec0 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4b68003520 con 0x7f4b780b3ec0 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 189774229 0 0) 0x7f4b780b5180 con 0x7f4b780b2d10 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4b780d2c10 con 0x7f4b780b2d10 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7cff8640 1 -- 172.21.15.64:0/145248883 >> v1:172.21.15.66:6789/0 conn(0x7f4b780b2d10 legacy=0x7f4b780b37b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b7cff8640 1 -- 172.21.15.64:0/145248883 >> v1:172.21.15.64:6789/0 conn(0x7f4b780b3ec0 legacy=0x7f4b780d1d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:57.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.572+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4269574543 0 0) 0x7f4b68003520 con 0x7f4b780b3ec0 2024-03-04T07:31:57.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.573+0000 7f4b7cff8640 1 -- 172.21.15.64:0/145248883 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b780b5460 con 0x7f4b78161720 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.573+0000 7f4b7cff8640 1 --2- 172.21.15.64:0/145248883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b78161720 0x7f4b780d1020 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f4b6c0097d0 tx=0x7f4b6c02b770 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a3b078c59ed2aeaf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.573+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b6c037070 con 0x7f4b78161720 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b6c030510 con 0x7f4b78161720 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b777fe640 1 -- 172.21.15.64:0/145248883 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b700019f0 con 0x7f4b78161720 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b7f25c640 1 -- 172.21.15.64:0/145248883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b78161720 msgr2=0x7f4b780d1020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/145248883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b78161720 0x7f4b780d1020 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f4b6c0097d0 tx=0x7f4b6c02b770 comp rx=0 tx=0).stop 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b7f25c640 1 -- 172.21.15.64:0/145248883 shutdown_connections 2024-03-04T07:31:57.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/145248883 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b78161720 0x7f4b780d1020 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b7f25c640 1 -- 172.21.15.64:0/145248883 >> 172.21.15.64:0/145248883 conn(0x7f4b780d3e90 msgr2=0x7f4b780d42a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b7f25c640 1 -- 172.21.15.64:0/145248883 shutdown_connections 2024-03-04T07:31:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.574+0000 7f4b7f25c640 1 -- 172.21.15.64:0/145248883 wait complete. 2024-03-04T07:31:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.575+0000 7f4b7f25c640 1 Processor -- start 2024-03-04T07:31:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.575+0000 7f4b7f25c640 1 -- start start 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7f25c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b780b2d10 0x7f4b78154310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7f25c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b780b3ec0 0x7f4b78160340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7f25c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b78161720 0x7f4b78160880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7f25c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b78161b00 con 0x7f4b78161720 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7f25c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b780b57d0 con 0x7f4b780b2d10 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7f25c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b78161c80 con 0x7f4b780b3ec0 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7d7f9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b78161720 0x7f4b78160880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7d7f9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b78161720 0x7f4b78160880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44824/0 (socket says 172.21.15.64:44824) 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.576+0000 7f4b7d7f9640 1 -- 172.21.15.64:0/3229926547 learned_addr learned my addr 172.21.15.64:0/3229926547 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b77fff640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b780b3ec0 0x7f4b78160340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7cff8640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b780b2d10 0x7f4b78154310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7d7f9640 1 -- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b780b3ec0 msgr2=0x7f4b78160340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7d7f9640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b780b3ec0 0x7f4b78160340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7d7f9640 1 -- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b780b2d10 msgr2=0x7f4b78154310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7d7f9640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b780b2d10 0x7f4b78154310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7d7f9640 1 -- 172.21.15.64:0/3229926547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b780c8f10 con 0x7f4b78161720 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7cff8640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b780b2d10 0x7f4b78154310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b77fff640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b780b3ec0 0x7f4b78160340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7d7f9640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b78161720 0x7f4b78160880 secure :-1 s=READY pgs=1244 cs=0 l=1 rev1=1 crypto rx=0x7f4b70001fb0 tx=0x7f4b7000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b757fa640 1 -- 172.21.15.64:0/3229926547 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b70017070 con 0x7f4b78161720 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b757fa640 1 -- 172.21.15.64:0/3229926547 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b70012400 con 0x7f4b78161720 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b780c91f0 con 0x7f4b78161720 2024-03-04T07:31:57.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b757fa640 1 -- 172.21.15.64:0/3229926547 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b70012830 con 0x7f4b78161720 2024-03-04T07:31:57.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.577+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b780c97b0 con 0x7f4b78161720 2024-03-04T07:31:57.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.578+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b780752d0 con 0x7f4b78161720 2024-03-04T07:31:57.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.578+0000 7f4b757fa640 1 -- 172.21.15.64:0/3229926547 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b7001a070 con 0x7f4b78161720 2024-03-04T07:31:57.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.579+0000 7f4b757fa640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b5c06b4d0 0x7f4b5c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:57.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.579+0000 7f4b757fa640 1 -- 172.21.15.64:0/3229926547 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(95..95 src has 1..95) v4 ==== 5967+0+0 (secure 0 0 0) 0x7f4b7008a310 con 0x7f4b78161720 2024-03-04T07:31:57.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.579+0000 7f4b7cff8640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b5c06b4d0 0x7f4b5c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:57.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.579+0000 7f4b7cff8640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b5c06b4d0 0x7f4b5c06d990 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f4b6c034000 tx=0x7f4b6c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:57.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.583+0000 7f4b757fa640 1 -- 172.21.15.64:0/3229926547 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b70059090 con 0x7f4b78161720 2024-03-04T07:31:57.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:57.764+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f4b7815da30 con 0x7f4b78161720 2024-03-04T07:31:58.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.407+0000 7f4b757fa640 1 -- 172.21.15.64:0/3229926547 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v96) v1 ==== 181+0+0 (secure 0 0 0) 0x7f4b70058990 con 0x7f4b78161720 2024-03-04T07:31:58.408 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b5c06b4d0 msgr2=0x7f4b5c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b5c06b4d0 0x7f4b5c06d990 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f4b6c034000 tx=0x7f4b6c008030 comp rx=0 tx=0).stop 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b78161720 msgr2=0x7f4b78160880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b78161720 0x7f4b78160880 secure :-1 s=READY pgs=1244 cs=0 l=1 rev1=1 crypto rx=0x7f4b70001fb0 tx=0x7f4b7000a430 comp rx=0 tx=0).stop 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 shutdown_connections 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b5c06b4d0 0x7f4b5c06d990 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b78161720 0x7f4b78160880 unknown :-1 s=CLOSED pgs=1244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b780b3ec0 0x7f4b78160340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 --2- 172.21.15.64:0/3229926547 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b780b2d10 0x7f4b78154310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 >> 172.21.15.64:0/3229926547 conn(0x7f4b780d3e90 msgr2=0x7f4b780cebc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:58.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 shutdown_connections 2024-03-04T07:31:58.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.410+0000 7f4b7f25c640 1 -- 172.21.15.64:0/3229926547 wait complete. 2024-03-04T07:31:58.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:31:58.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.540+0000 7f0cd7fa1640 1 Processor -- start 2024-03-04T07:31:58.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.540+0000 7f0cd7fa1640 1 -- start start 2024-03-04T07:31:58.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd7fa1640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0cd00b5310 0x7f0cd00ceb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd7fa1640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0cd00cf160 con 0x7f0cd00d0fc0 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd7fa1640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0cd00cf440 con 0x7f0cd00d49c0 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd7fa1640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0cd00d4e00 con 0x7f0cd00b5310 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd653e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0cd00d0fc0 0x7f0cd00b32a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35424/0 (socket says 172.21.15.64:35424) 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd653e640 1 -- 172.21.15.64:0/814585444 learned_addr learned my addr 172.21.15.64:0/814585444 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd5d3d640 1 --2- 172.21.15.64:0/814585444 >> v2:172.21.15.66:3300/0 conn(0x7f0cd00b5310 0x7f0cd00ceb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1257963083 0 0) 0x7f0cd00cf160 con 0x7f0cd00d0fc0 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0cbc003520 con 0x7f0cd00d0fc0 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 475686997 0 0) 0x7f0cbc003520 con 0x7f0cd00d0fc0 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 >> v2:172.21.15.66:3300/0 conn(0x7f0cd00b5310 msgr2=0x7f0cd00ceb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 --2- 172.21.15.64:0/814585444 >> v2:172.21.15.66:3300/0 conn(0x7f0cd00b5310 0x7f0cd00ceb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 >> v1:172.21.15.66:6789/0 conn(0x7f0cd00d49c0 legacy=0x7f0cd00d08b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cd00cf720 con 0x7f0cd00d0fc0 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f0cc80019f0 con 0x7f0cd00d0fc0 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 >> v1:172.21.15.64:6789/0 conn(0x7f0cd00d0fc0 legacy=0x7f0cd00b32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cbc003a00 0x7f0cbc003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cbc004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.541+0000 7f0cd4d3b640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cbc0051c0 0x7f0cbc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cd00d4e00 con 0x7f0cbc004800 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cbc00c0c0 con 0x7f0cbc003a00 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0cbc00c2c0 con 0x7f0cbc0051c0 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd553c640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cbc004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd5d3d640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cbc003a00 0x7f0cbc003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd653e640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cbc0051c0 0x7f0cbc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd553c640 1 -- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cbc0051c0 msgr2=0x7f0cbc00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd553c640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cbc0051c0 0x7f0cbc00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd553c640 1 -- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cbc003a00 msgr2=0x7f0cbc003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd553c640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cbc003a00 0x7f0cbc003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd553c640 1 -- 172.21.15.64:0/814585444 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cbc003520 con 0x7f0cbc004800 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd653e640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cbc0051c0 0x7f0cbc00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.542+0000 7f0cd653e640 1 -- 172.21.15.64:0/814585444 reap_dead start 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd553c640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cbc004c80 secure :-1 s=READY pgs=1246 cs=0 l=1 rev1=1 crypto rx=0x7f0cc0001750 tx=0x7f0cbc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d04ba166e990613 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cc0001370 con 0x7f0cbc004800 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0cc00052d0 con 0x7f0cbc004800 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd4d3b640 1 -- 172.21.15.64:0/814585444 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cc0003830 con 0x7f0cbc004800 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/814585444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 msgr2=0x7f0cbc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cbc004c80 secure :-1 s=READY pgs=1246 cs=0 l=1 rev1=1 crypto rx=0x7f0cc0001750 tx=0x7f0cbc005830 comp rx=0 tx=0).stop 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/814585444 shutdown_connections 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/814585444 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cbc004c80 unknown :-1 s=CLOSED pgs=1246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/814585444 >> 172.21.15.64:0/814585444 conn(0x7f0cd00d3ea0 msgr2=0x7f0cd00d42b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:58.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/814585444 shutdown_connections 2024-03-04T07:31:58.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.543+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/814585444 wait complete. 2024-03-04T07:31:58.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.544+0000 7f0cd7fa1640 1 Processor -- start 2024-03-04T07:31:58.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd7fa1640 1 -- start start 2024-03-04T07:31:58.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd7fa1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cd00b93b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd7fa1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cd00b6a90 0x7f0cd00b6f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd7fa1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cd00cbfc0 0x7f0cd00cc440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd7fa1640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cd00d1fb0 con 0x7f0cbc004800 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd7fa1640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cd00d1e30 con 0x7f0cd00b6a90 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd7fa1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0cd00ceb60 con 0x7f0cd00cbfc0 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd5d3d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cd00b93b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd5d3d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cd00b93b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44842/0 (socket says 172.21.15.64:44842) 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.545+0000 7f0cd5d3d640 1 -- 172.21.15.64:0/2867690111 learned_addr learned my addr 172.21.15.64:0/2867690111 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd5d3d640 1 -- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cd00cbfc0 msgr2=0x7f0cd00cc440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd653e640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cd00cbfc0 0x7f0cd00cc440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd553c640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cd00b6a90 0x7f0cd00b6f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd5d3d640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cd00cbfc0 0x7f0cd00cc440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd5d3d640 1 -- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cd00b6a90 msgr2=0x7f0cd00b6f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd5d3d640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cd00b6a90 0x7f0cd00b6f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd5d3d640 1 -- 172.21.15.64:0/2867690111 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cd00cdd50 con 0x7f0cbc004800 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd553c640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cd00b6a90 0x7f0cd00b6f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:58.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd653e640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cd00cbfc0 0x7f0cd00cc440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd5d3d640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cd00b93b0 secure :-1 s=READY pgs=1247 cs=0 l=1 rev1=1 crypto rx=0x7f0ccc011a70 tx=0x7f0ccc011f30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cc67fc640 1 -- 172.21.15.64:0/2867690111 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ccc00fb50 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cc67fc640 1 -- 172.21.15.64:0/2867690111 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0ccc00fcf0 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0cd00b98f0 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cc67fc640 1 -- 172.21.15.64:0/2867690111 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ccc00fea0 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.546+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0cd00ccf90 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.547+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0cd00752d0 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.547+0000 7f0cc67fc640 1 -- 172.21.15.64:0/2867690111 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0ccc012040 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.548+0000 7f0cc67fc640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0cb406b4d0 0x7f0cb406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.548+0000 7f0cd553c640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0cb406b4d0 0x7f0cb406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.548+0000 7f0cc67fc640 1 -- 172.21.15.64:0/2867690111 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 5529+0+0 (secure 0 0 0) 0x7f0ccc08a870 con 0x7f0cbc004800 2024-03-04T07:31:58.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.548+0000 7f0cd553c640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0cb406b4d0 0x7f0cb406d990 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f0cc0033000 tx=0x7f0cc0009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:58.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.552+0000 7f0cc67fc640 1 -- 172.21.15.64:0/2867690111 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0ccc019030 con 0x7f0cbc004800 2024-03-04T07:31:58.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:58.733+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f0cd00d2260 con 0x7f0cbc004800 2024-03-04T07:31:59.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.413+0000 7f0cc67fc640 1 -- 172.21.15.64:0/2867690111 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v97) v1 ==== 169+0+0 (secure 0 0 0) 0x7f0ccc0597a0 con 0x7f0cbc004800 2024-03-04T07:31:59.414 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0cb406b4d0 msgr2=0x7f0cb406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0cb406b4d0 0x7f0cb406d990 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f0cc0033000 tx=0x7f0cc0009030 comp rx=0 tx=0).stop 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 msgr2=0x7f0cd00b93b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cd00b93b0 secure :-1 s=READY pgs=1247 cs=0 l=1 rev1=1 crypto rx=0x7f0ccc011a70 tx=0x7f0ccc011f30 comp rx=0 tx=0).stop 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 shutdown_connections 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0cb406b4d0 0x7f0cb406d990 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cd00cbfc0 0x7f0cd00cc440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cd00b6a90 0x7f0cd00b6f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 --2- 172.21.15.64:0/2867690111 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cbc004800 0x7f0cd00b93b0 unknown :-1 s=CLOSED pgs=1247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 >> 172.21.15.64:0/2867690111 conn(0x7f0cd00d3ea0 msgr2=0x7f0cd00b0d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:59.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 shutdown_connections 2024-03-04T07:31:59.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.417+0000 7f0cd7fa1640 1 -- 172.21.15.64:0/2867690111 wait complete. 2024-03-04T07:31:59.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:59.437 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:59.488 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-03-04T07:31:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: chdir = "" 2024-03-04T07:31:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: auth supported = cephx 2024-03-04T07:31:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: filestore xattr use omap = true 2024-03-04T07:31:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: auth debug = true 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on old message = true 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on bug = true 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default size = 2 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow pg remap = true 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on too few osds = false 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: osd default data pool replay window = 5 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: mon allow pool delete = true 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: mon cluster log file level = debug 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: debug asserts on shutdown = true 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: mon health detail to clog = false 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout:[osd] 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: osd journal size = 100 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub max interval = 600 2024-03-04T07:31:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd recover clone overlap = true 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug shutdown = true 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug op order = true 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug trim objects = true 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd open classes on start = true 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug pg log writeout = true 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: osd map max advance = 10 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: journal zero on create = true 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: bdev debug aio = true 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug misdirected ops = true 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: bdev async discard = True 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: bdev enable discard = True 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore allocator = bitmap 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore block size = 96636764160 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore fsck on mount = True 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluefs = 1/20 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluestore = 1/20 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: debug osd = 20 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: debug rocksdb = 4/10 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: osd objectstore = bluestore 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr] 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: debug mgr = 20 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout:[mon] 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: debug paxos = 20 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon data avail warn = 5 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd prime pg temp = true 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: # 1m isn't quite enough 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: mon op complaint time = 120 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout:[client] 2024-03-04T07:31:59.496 INFO:teuthology.orchestra.run.smithi064.stdout: rgw cache enabled = true 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable ops log = true 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable usage log = true 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: client mount timeout = 600 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: debug client = 20 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = "" 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.a] 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.b] 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.c] 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:31:59.497 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout:[mds] 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds = 20 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds balancer = 20 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug frag = True 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug scatterstat = True 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: mds op complaint time = 180 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: mds verify scatter = True 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_ops = 100000000 2024-03-04T07:31:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout:[global] 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: chdir = "" 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: auth supported = cephx 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: filestore xattr use omap = true 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:59.504 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: auth debug = true 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on old message = true 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on bug = true 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default size = 2 2024-03-04T07:31:59.505 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow pg remap = true 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on too few osds = false 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.506 INFO:teuthology.orchestra.run.smithi066.stdout: osd default data pool replay window = 5 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: mon allow pool delete = true 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: mon cluster log file level = debug 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: debug asserts on shutdown = true 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: mon health detail to clog = false 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout:[osd] 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: osd journal size = 100 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub max interval = 600 2024-03-04T07:31:59.507 INFO:teuthology.orchestra.run.smithi066.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd recover clone overlap = true 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug shutdown = true 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug op order = true 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug trim objects = true 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd open classes on start = true 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug pg log writeout = true 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.508 INFO:teuthology.orchestra.run.smithi066.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: osd map max advance = 10 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: journal zero on create = true 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: bdev debug aio = true 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug misdirected ops = true 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: bdev async discard = True 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: bdev enable discard = True 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore allocator = bitmap 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore block size = 96636764160 2024-03-04T07:31:59.509 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore fsck on mount = True 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluefs = 1/20 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluestore = 1/20 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: debug osd = 20 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: debug rocksdb = 4/10 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd objectstore = bluestore 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr] 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.510 INFO:teuthology.orchestra.run.smithi066.stdout: debug mgr = 20 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout:[mon] 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: debug paxos = 20 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: mon data avail warn = 5 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:59.511 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd prime pg temp = true 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: # 1m isn't quite enough 2024-03-04T07:31:59.512 INFO:teuthology.orchestra.run.smithi066.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: mon op complaint time = 120 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout:[client] 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: rgw cache enabled = true 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable ops log = true 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable usage log = true 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: client mount timeout = 600 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: debug client = 20 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.513 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = "" 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.a] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.b] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.c] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout:[mds] 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds = 20 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds balancer = 20 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.514 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug frag = True 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug scatterstat = True 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: mds op complaint time = 180 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: mds verify scatter = True 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_ops = 100000000 2024-03-04T07:31:59.515 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-04T07:31:59.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:59.545 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout: chdir = "" 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout: auth supported = cephx 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout: filestore xattr use omap = true 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.594 INFO:teuthology.orchestra.run.smithi064.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: auth debug = true 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on old message = true 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on bug = true 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default size = 2 2024-03-04T07:31:59.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow pg remap = true 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on too few osds = false 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: osd default data pool replay window = 5 2024-03-04T07:31:59.596 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: mon allow pool delete = true 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: mon cluster log file level = debug 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: debug asserts on shutdown = true 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: mon health detail to clog = false 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout:[osd] 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: osd journal size = 100 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub max interval = 600 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.597 INFO:teuthology.orchestra.run.smithi064.stdout: osd recover clone overlap = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug shutdown = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug op order = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug trim objects = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd open classes on start = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug pg log writeout = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: osd map max advance = 10 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: journal zero on create = true 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.598 INFO:teuthology.orchestra.run.smithi064.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: bdev debug aio = true 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug misdirected ops = true 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: bdev async discard = True 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: bdev enable discard = True 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore allocator = bitmap 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore block size = 96636764160 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore fsck on mount = True 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluefs = 1/20 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluestore = 1/20 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: debug osd = 20 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: debug rocksdb = 4/10 2024-03-04T07:31:59.599 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: osd objectstore = bluestore 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr] 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: debug mgr = 20 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:59.600 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout:[mon] 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: debug paxos = 20 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: mon data avail warn = 5 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd prime pg temp = true 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:59.601 INFO:teuthology.orchestra.run.smithi064.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: # 1m isn't quite enough 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: mon op complaint time = 120 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout:[client] 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: rgw cache enabled = true 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable ops log = true 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable usage log = true 2024-03-04T07:31:59.602 INFO:teuthology.orchestra.run.smithi064.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout: client mount timeout = 600 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout: debug client = 20 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = "" 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.a] 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.b] 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.c] 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:31:59.603 INFO:teuthology.orchestra.run.smithi064.stdout:[mds] 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds = 20 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds balancer = 20 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug frag = True 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug scatterstat = True 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: mds op complaint time = 180 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: mds verify scatter = True 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.604 INFO:teuthology.orchestra.run.smithi064.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout:[global] 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: chdir = "" 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: auth supported = cephx 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: filestore xattr use omap = true 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:59.607 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: auth debug = true 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on old message = true 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on bug = true 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default size = 2 2024-03-04T07:31:59.608 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow pg remap = true 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on too few osds = false 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: osd default data pool replay window = 5 2024-03-04T07:31:59.609 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: mon allow pool delete = true 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: mon cluster log file level = debug 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: debug asserts on shutdown = true 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: mon health detail to clog = false 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout:[osd] 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: osd journal size = 100 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.610 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub max interval = 600 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd recover clone overlap = true 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug shutdown = true 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug op order = true 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug trim objects = true 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd open classes on start = true 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug pg log writeout = true 2024-03-04T07:31:59.611 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: osd map max advance = 10 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: journal zero on create = true 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: bdev debug aio = true 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug misdirected ops = true 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: bdev async discard = True 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: bdev enable discard = True 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore allocator = bitmap 2024-03-04T07:31:59.612 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore block size = 96636764160 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore fsck on mount = True 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluefs = 1/20 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluestore = 1/20 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: debug osd = 20 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: debug rocksdb = 4/10 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: osd objectstore = bluestore 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr] 2024-03-04T07:31:59.613 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: debug mgr = 20 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout:[mon] 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: debug paxos = 20 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: mon data avail warn = 5 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:59.614 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd prime pg temp = true 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: # 1m isn't quite enough 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:59.615 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: mon op complaint time = 120 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout:[client] 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: rgw cache enabled = true 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable ops log = true 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable usage log = true 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: client mount timeout = 600 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: debug client = 20 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = "" 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.616 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.a] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.b] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.c] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout:[mds] 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds = 20 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds balancer = 20 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug frag = True 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug scatterstat = True 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout: mds op complaint time = 180 2024-03-04T07:31:59.617 INFO:teuthology.orchestra.run.smithi066.stdout: mds verify scatter = True 2024-03-04T07:31:59.618 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:59.618 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.618 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.618 INFO:teuthology.orchestra.run.smithi066.stdout: mds_max_purge_ops_per_pg = 1562500.0 2024-03-04T07:31:59.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:59.661 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-03-04T07:31:59.709 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-03-04T07:31:59.709 INFO:teuthology.orchestra.run.smithi064.stdout: chdir = "" 2024-03-04T07:31:59.709 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:59.709 INFO:teuthology.orchestra.run.smithi064.stdout: auth supported = cephx 2024-03-04T07:31:59.709 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.709 INFO:teuthology.orchestra.run.smithi064.stdout: filestore xattr use omap = true 2024-03-04T07:31:59.709 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: auth debug = true 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on old message = true 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: ms die on bug = true 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:59.710 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default size = 2 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd allow pg remap = true 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon warn on too few osds = false 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.711 INFO:teuthology.orchestra.run.smithi064.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:59.712 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.712 INFO:teuthology.orchestra.run.smithi064.stdout: osd default data pool replay window = 5 2024-03-04T07:31:59.712 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: mon allow pool delete = true 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: mon cluster log file level = debug 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: debug asserts on shutdown = true 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: mon health detail to clog = false 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout:[osd] 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: osd journal size = 100 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: osd scrub max interval = 600 2024-03-04T07:31:59.713 INFO:teuthology.orchestra.run.smithi064.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd recover clone overlap = true 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug shutdown = true 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug op order = true 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug trim objects = true 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd open classes on start = true 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug pg log writeout = true 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:59.714 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: osd map max advance = 10 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: journal zero on create = true 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: bdev debug aio = true 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: osd debug misdirected ops = true 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: bdev async discard = True 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: bdev enable discard = True 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore allocator = bitmap 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore block size = 96636764160 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: bluestore fsck on mount = True 2024-03-04T07:31:59.715 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluefs = 1/20 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: debug bluestore = 1/20 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: debug osd = 20 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: debug rocksdb = 4/10 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: osd objectstore = bluestore 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout:[mgr] 2024-03-04T07:31:59.716 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: debug mgr = 20 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout:[mon] 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: debug mon = 20 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: debug paxos = 20 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: debug auth = 20 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: mon data avail warn = 5 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.717 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: mon osd prime pg temp = true 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: # 1m isn't quite enough 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.718 INFO:teuthology.orchestra.run.smithi064.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: mon op complaint time = 120 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout:[client] 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: rgw cache enabled = true 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable ops log = true 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: rgw enable usage log = true 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: client mount timeout = 600 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: debug client = 20 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: pid file = "" 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.a] 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.b] 2024-03-04T07:31:59.719 INFO:teuthology.orchestra.run.smithi064.stdout:[mon.c] 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.a] 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.c] 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.b] 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout:[mds.d] 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout:[mds] 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds = 20 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: debug mds balancer = 20 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: debug ms = 1 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug frag = True 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: mds debug scatterstat = True 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: mds op complaint time = 180 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: mds verify scatter = True 2024-03-04T07:31:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: osd op complaint time = 180 2024-03-04T07:31:59.721 INFO:teuthology.orchestra.run.smithi064.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.721 INFO:teuthology.orchestra.run.smithi064.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout:[global] 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout: chdir = "" 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout: auth supported = cephx 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout: filestore xattr use omap = true 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.723 INFO:teuthology.orchestra.run.smithi066.stdout: mon clock drift allowed = 1.000 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: osd crush chooseleaf type = 0 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: auth debug = true 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on old message = true 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: ms die on bug = true 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: mon max pg per osd = 10000 # >= luminous 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: mon pg warn max object skew = 0 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: # disable pg_autoscaler by default for new pools 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: osd_pool_default_pg_autoscale_mode = off 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.724 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default size = 2 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow primary affinity = true 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd allow pg remap = true 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on legacy crush tunables = false 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on crush straw calc version zero = false 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on no sortbitwise = false 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on osd down out interval zero = false 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon warn on too few osds = false 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_pool_no_redundancy = false 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: mon_allow_pool_size_one = true 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-03-04T07:31:59.725 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: osd default data pool replay window = 5 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: mon allow pool delete = true 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: mon cluster log file level = debug 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: debug asserts on shutdown = true 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: mon health detail to clog = false 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: mon host = "172.21.15.64,172.21.15.66,[v2:172.21.15.66:3301,v1:172.21.15.66:6790]" 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: fsid = c3ef085c-fd40-4f87-8c92-fdb42a102a8c 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout:[osd] 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: osd journal size = 100 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub load threshold = 5.0 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: osd scrub max interval = 600 2024-03-04T07:31:59.726 INFO:teuthology.orchestra.run.smithi066.stdout: osd mclock profile = high_recovery_ops 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd recover clone overlap = true 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd recovery max chunk = 1048576 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug shutdown = true 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug op order = true 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug verify stray on activate = true 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug trim objects = true 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd open classes on start = true 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug pg log writeout = true 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd deep scrub update digest min age = 30 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.727 INFO:teuthology.orchestra.run.smithi066.stdout: osd map max advance = 10 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: journal zero on create = true 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: filestore ondisk finisher threads = 3 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: filestore apply finisher threads = 3 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: bdev debug aio = true 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: osd debug misdirected ops = true 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: bdev async discard = True 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: bdev enable discard = True 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore allocator = bitmap 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore block size = 96636764160 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: bluestore fsck on mount = True 2024-03-04T07:31:59.728 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluefs = 1/20 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: debug bluestore = 1/20 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: debug osd = 20 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: debug rocksdb = 4/10 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd backfillfull_ratio = 0.85 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd full ratio = 0.9 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd nearfull ratio = 0.8 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: osd failsafe full ratio = 0.95 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: osd objectstore = bluestore 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout:[mgr] 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: debug mgr = 20 2024-03-04T07:31:59.729 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: mgr/telemetry/nag = false 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout:[mon] 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: debug mon = 20 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: debug paxos = 20 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: debug auth = 20 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: mon data avail warn = 5 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: mon mgr mkfs grace = 240 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min pgs per osd = 4 2024-03-04T07:31:59.730 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd reporter subtree level = osd 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: mon osd prime pg temp = true 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: mon reweight min bytes per osd = 10 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: auth mon ticket ttl = 660 # 11m 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: auth service ticket ttl = 240 # 4m 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: # don't complain about insecure global_id in the test suite 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: # 1m isn't quite enough 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: mon_down_mkfs_grace = 2m 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.731 INFO:teuthology.orchestra.run.smithi066.stdout: mon_warn_on_filestore_osds = false 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: mon op complaint time = 120 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout:[client] 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: rgw cache enabled = true 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable ops log = true 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: rgw enable usage log = true 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: client mount timeout = 600 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: debug client = 20 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: pid file = "" 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.732 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.a] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.b] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:[mon.c] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.a] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.c] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.b] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:[mds.d] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:[mds] 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds = 20 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: debug mds balancer = 20 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: debug ms = 1 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug frag = True 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: mds debug scatterstat = True 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: mds op complaint time = 180 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: mds verify scatter = True 2024-03-04T07:31:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: osd op complaint time = 180 2024-03-04T07:31:59.734 INFO:teuthology.orchestra.run.smithi066.stdout: rados mon op timeout = 900 2024-03-04T07:31:59.734 INFO:teuthology.orchestra.run.smithi066.stdout: rados osd op timeout = 900 2024-03-04T07:31:59.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.867+0000 7f5721e39640 1 Processor -- start 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.867+0000 7f5721e39640 1 -- start start 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f5721e39640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0717b0 0x7f571c071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f5721e39640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f571c0b2b10 0x7f571c0d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f5721e39640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f571c15e4b0 con 0x7f571c0b3060 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f5721e39640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f571c0d2e10 con 0x7f571c0b2b10 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f5721e39640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f571c0b97e0 con 0x7f571c0717b0 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f571c0b2b10 0x7f571c0d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f571c0b2b10 0x7f571c0d32a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44856/0 (socket says 172.21.15.64:44856) 2024-03-04T07:31:59.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571bfff640 1 -- 172.21.15.64:0/1706447156 learned_addr learned my addr 172.21.15.64:0/1706447156 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571b7fe640 1 --2- 172.21.15.64:0/1706447156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0717b0 0x7f571c071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571bfff640 1 -- 172.21.15.64:0/1706447156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0717b0 msgr2=0x7f571c071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571bfff640 1 --2- 172.21.15.64:0/1706447156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0717b0 0x7f571c071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571bfff640 1 -- 172.21.15.64:0/1706447156 >> v1:172.21.15.66:6789/0 conn(0x7f571c0b3060 legacy=0x7f571c0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.868+0000 7f571bfff640 1 -- 172.21.15.64:0/1706447156 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f571c0b9a80 con 0x7f571c0b2b10 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.869+0000 7f571bfff640 1 --2- 172.21.15.64:0/1706447156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c0d32a0 secure :-1 s=READY pgs=1248 cs=0 l=1 rev1=1 crypto rx=0x7f5714009e50 tx=0x7f571402bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f5c8aca7bc5faed0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.869+0000 7f571a7fc640 1 -- 172.21.15.64:0/1706447156 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f571402cb00 con 0x7f571c0b2b10 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.869+0000 7f571a7fc640 1 -- 172.21.15.64:0/1706447156 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f571402cca0 con 0x7f571c0b2b10 2024-03-04T07:31:59.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.869+0000 7f571a7fc640 1 -- 172.21.15.64:0/1706447156 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5714034610 con 0x7f571c0b2b10 2024-03-04T07:31:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 -- 172.21.15.64:0/1706447156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 msgr2=0x7f571c0d32a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 --2- 172.21.15.64:0/1706447156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c0d32a0 secure :-1 s=READY pgs=1248 cs=0 l=1 rev1=1 crypto rx=0x7f5714009e50 tx=0x7f571402bf60 comp rx=0 tx=0).stop 2024-03-04T07:31:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 -- 172.21.15.64:0/1706447156 shutdown_connections 2024-03-04T07:31:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 --2- 172.21.15.64:0/1706447156 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c0d32a0 unknown :-1 s=CLOSED pgs=1248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 --2- 172.21.15.64:0/1706447156 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0717b0 0x7f571c071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 -- 172.21.15.64:0/1706447156 >> 172.21.15.64:0/1706447156 conn(0x7f571c0d3d20 msgr2=0x7f571c0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:31:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 -- 172.21.15.64:0/1706447156 shutdown_connections 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.870+0000 7f5721e39640 1 -- 172.21.15.64:0/1706447156 wait complete. 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.871+0000 7f5721e39640 1 Processor -- start 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.871+0000 7f5721e39640 1 -- start start 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f571c0717b0 0x7f571c1577a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c155700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0b3060 0x7f571c158850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f571c0b7560 con 0x7f571c0b2b10 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f571c0d4bf0 con 0x7f571c0717b0 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f571c0b76e0 con 0x7f571c0b3060 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c155700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c155700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44860/0 (socket says 172.21.15.64:44860) 2024-03-04T07:31:59.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 -- 172.21.15.64:0/1683838279 learned_addr learned my addr 172.21.15.64:0/1683838279 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571b7fe640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f571c0717b0 0x7f571c1577a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571bfff640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0b3060 0x7f571c158850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 -- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0b3060 msgr2=0x7f571c158850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0b3060 0x7f571c158850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 -- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f571c0717b0 msgr2=0x7f571c1577a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f571c0717b0 0x7f571c1577a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 -- 172.21.15.64:0/1683838279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f571c155c40 con 0x7f571c0b2b10 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f571affd640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c155700 secure :-1 s=READY pgs=1249 cs=0 l=1 rev1=1 crypto rx=0x7f571000c3c0 tx=0x7f571000c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5703fff640 1 -- 172.21.15.64:0/1683838279 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57100052d0 con 0x7f571c0b2b10 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5703fff640 1 -- 172.21.15.64:0/1683838279 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5710005490 con 0x7f571c0b2b10 2024-03-04T07:31:59.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f571c159150 con 0x7f571c0b2b10 2024-03-04T07:31:59.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5703fff640 1 -- 172.21.15.64:0/1683838279 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5710007260 con 0x7f571c0b2b10 2024-03-04T07:31:59.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.872+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f571c156fa0 con 0x7f571c0b2b10 2024-03-04T07:31:59.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.873+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f571c06ce40 con 0x7f571c0b2b10 2024-03-04T07:31:59.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.874+0000 7f5703fff640 1 -- 172.21.15.64:0/1683838279 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5710007400 con 0x7f571c0b2b10 2024-03-04T07:31:59.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.874+0000 7f5703fff640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570806b4d0 0x7f570806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:31:59.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.874+0000 7f5703fff640 1 -- 172.21.15.64:0/1683838279 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f57100893d0 con 0x7f571c0b2b10 2024-03-04T07:31:59.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.874+0000 7f571b7fe640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570806b4d0 0x7f570806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:31:59.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.875+0000 7f571b7fe640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570806b4d0 0x7f570806d990 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f570c009b40 tx=0x7f570c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:31:59.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:31:59.879+0000 7f5703fff640 1 -- 172.21.15.64:0/1683838279 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f571c15e4b0 con 0x7f571c0b2b10 2024-03-04T07:32:00.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.061+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f571c0b5d20 con 0x7f571c0b2b10 2024-03-04T07:32:00.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.064+0000 7f5703fff640 1 -- 172.21.15.64:0/1683838279 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v29) v1 ==== 104+0+0 (secure 0 0 0) 0x7f5710057e50 con 0x7f571c0b2b10 2024-03-04T07:32:00.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570806b4d0 msgr2=0x7f570806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:00.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570806b4d0 0x7f570806d990 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f570c009b40 tx=0x7f570c009190 comp rx=0 tx=0).stop 2024-03-04T07:32:00.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 msgr2=0x7f571c155700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:00.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c155700 secure :-1 s=READY pgs=1249 cs=0 l=1 rev1=1 crypto rx=0x7f571000c3c0 tx=0x7f571000c880 comp rx=0 tx=0).stop 2024-03-04T07:32:00.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 shutdown_connections 2024-03-04T07:32:00.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f570806b4d0 0x7f570806d990 unknown :-1 s=CLOSED pgs=565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f571c0b3060 0x7f571c158850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f571c0b2b10 0x7f571c155700 unknown :-1 s=CLOSED pgs=1249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 --2- 172.21.15.64:0/1683838279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f571c0717b0 0x7f571c1577a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 >> 172.21.15.64:0/1683838279 conn(0x7f571c0d3d20 msgr2=0x7f571c0d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:00.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 shutdown_connections 2024-03-04T07:32:00.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.066+0000 7f5721e39640 1 -- 172.21.15.64:0/1683838279 wait complete. 2024-03-04T07:32:00.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_files_throttle' 2024-03-04T07:32:00.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.187+0000 7fa43b59e640 1 Processor -- start 2024-03-04T07:32:00.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.187+0000 7fa43b59e640 1 -- start start 2024-03-04T07:32:00.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43b59e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c15dd00 0x7fa43c0ceb30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:00.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43b59e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa43c06af20 0x7fa43c06b930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:00.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43b59e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa43c06be70 0x7fa43c0b25c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43b59e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa43c06c2c0 con 0x7fa43c06be70 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43b59e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa43c06b670 con 0x7fa43c06af20 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43b59e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa43c0cbfc0 con 0x7fa43c15dd00 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa43c06be70 0x7fa43c0b25c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa43c06be70 0x7fa43c0b25c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44870/0 (socket says 172.21.15.64:44870) 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43a59c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c15dd00 0x7fa43c0ceb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 -- 172.21.15.64:0/3327335824 learned_addr learned my addr 172.21.15.64:0/3327335824 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa439d9b640 1 --2- 172.21.15.64:0/3327335824 >> v2:172.21.15.66:3300/0 conn(0x7fa43c06af20 0x7fa43c06b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 -- 172.21.15.64:0/3327335824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c15dd00 msgr2=0x7fa43c0ceb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 --2- 172.21.15.64:0/3327335824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c15dd00 0x7fa43c0ceb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 -- 172.21.15.64:0/3327335824 >> v2:172.21.15.66:3300/0 conn(0x7fa43c06af20 msgr2=0x7fa43c06b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 --2- 172.21.15.64:0/3327335824 >> v2:172.21.15.66:3300/0 conn(0x7fa43c06af20 0x7fa43c06b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.189+0000 7fa43ad9d640 1 -- 172.21.15.64:0/3327335824 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa43c0cc2a0 con 0x7fa43c06be70 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.190+0000 7fa43ad9d640 1 --2- 172.21.15.64:0/3327335824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06be70 0x7fa43c0b25c0 secure :-1 s=READY pgs=1250 cs=0 l=1 rev1=1 crypto rx=0x7fa43c068410 tx=0x7fa43402be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=83418f3cfd41f0d0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.190+0000 7fa43959a640 1 -- 172.21.15.64:0/3327335824 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa43402c9e0 con 0x7fa43c06be70 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.190+0000 7fa43959a640 1 -- 172.21.15.64:0/3327335824 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa43402cb80 con 0x7fa43c06be70 2024-03-04T07:32:00.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.190+0000 7fa43959a640 1 -- 172.21.15.64:0/3327335824 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4340343f0 con 0x7fa43c06be70 2024-03-04T07:32:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 -- 172.21.15.64:0/3327335824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06be70 msgr2=0x7fa43c0b25c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 --2- 172.21.15.64:0/3327335824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06be70 0x7fa43c0b25c0 secure :-1 s=READY pgs=1250 cs=0 l=1 rev1=1 crypto rx=0x7fa43c068410 tx=0x7fa43402be40 comp rx=0 tx=0).stop 2024-03-04T07:32:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 -- 172.21.15.64:0/3327335824 shutdown_connections 2024-03-04T07:32:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 --2- 172.21.15.64:0/3327335824 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06be70 0x7fa43c0b25c0 unknown :-1 s=CLOSED pgs=1250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 --2- 172.21.15.64:0/3327335824 >> v2:172.21.15.66:3300/0 conn(0x7fa43c06af20 0x7fa43c06b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 --2- 172.21.15.64:0/3327335824 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c15dd00 0x7fa43c0ceb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 -- 172.21.15.64:0/3327335824 >> 172.21.15.64:0/3327335824 conn(0x7fa43c0b34a0 msgr2=0x7fa43c0b38b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:00.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 -- 172.21.15.64:0/3327335824 shutdown_connections 2024-03-04T07:32:00.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.191+0000 7fa43b59e640 1 -- 172.21.15.64:0/3327335824 wait complete. 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.192+0000 7fa43b59e640 1 Processor -- start 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.192+0000 7fa43b59e640 1 -- start start 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43b59e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06af20 0x7fa43c0cb350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43b59e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c06be70 0x7fa43c1f75c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43b59e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa43c15dd00 0x7fa43c0b6a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43b59e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa43c06b380 con 0x7fa43c06af20 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43b59e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa43c0cc860 con 0x7fa43c15dd00 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43b59e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa43c0b1310 con 0x7fa43c06be70 2024-03-04T07:32:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06af20 0x7fa43c0cb350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06af20 0x7fa43c0cb350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44876/0 (socket says 172.21.15.64:44876) 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43a59c640 1 -- 172.21.15.64:0/2314515798 learned_addr learned my addr 172.21.15.64:0/2314515798 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa439d9b640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c06be70 0x7fa43c1f75c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.193+0000 7fa43ad9d640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa43c15dd00 0x7fa43c0b6a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43a59c640 1 -- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c06be70 msgr2=0x7fa43c1f75c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43a59c640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c06be70 0x7fa43c1f75c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43a59c640 1 -- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa43c15dd00 msgr2=0x7fa43c0b6a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43a59c640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa43c15dd00 0x7fa43c0b6a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43a59c640 1 -- 172.21.15.64:0/2314515798 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa43c160b10 con 0x7fa43c06af20 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43a59c640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06af20 0x7fa43c0cb350 secure :-1 s=READY pgs=1251 cs=0 l=1 rev1=1 crypto rx=0x7fa42c00e8a0 tx=0x7fa42c00ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa42affd640 1 -- 172.21.15.64:0/2314515798 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa42c017070 con 0x7fa43c06af20 2024-03-04T07:32:00.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa42affd640 1 -- 172.21.15.64:0/2314515798 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa42c00c940 con 0x7fa43c06af20 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa43c0ca2a0 con 0x7fa43c06af20 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa42affd640 1 -- 172.21.15.64:0/2314515798 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa42c00cd30 con 0x7fa43c06af20 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.194+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa43c0c97c0 con 0x7fa43c06af20 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.195+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa43c04f330 con 0x7fa43c06af20 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.195+0000 7fa42affd640 1 -- 172.21.15.64:0/2314515798 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa42c01a070 con 0x7fa43c06af20 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.196+0000 7fa42affd640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa41c06b4d0 0x7fa41c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.196+0000 7fa42affd640 1 -- 172.21.15.64:0/2314515798 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fa42c089e40 con 0x7fa43c06af20 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.196+0000 7fa439d9b640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa41c06b4d0 0x7fa41c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.197+0000 7fa439d9b640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa41c06b4d0 0x7fa41c06d990 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7fa43c0d2610 tx=0x7fa430008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:00.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.200+0000 7fa42affd640 1 -- 172.21.15.64:0/2314515798 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa43c160b10 con 0x7fa43c06af20 2024-03-04T07:32:00.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:00.450+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]} v 0) v1 -- 0x7fa43c0b10d0 con 0x7fa43c06af20 2024-03-04T07:32:01.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.418+0000 7fa42affd640 1 -- 172.21.15.64:0/2314515798 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_files_throttle"]}]=0 v286) v1 ==== 134+0+0 (secure 0 0 0) 0x7fa42c058f10 con 0x7fa43c06af20 2024-03-04T07:32:01.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa41c06b4d0 msgr2=0x7fa41c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa41c06b4d0 0x7fa41c06d990 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7fa43c0d2610 tx=0x7fa430008040 comp rx=0 tx=0).stop 2024-03-04T07:32:01.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06af20 msgr2=0x7fa43c0cb350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06af20 0x7fa43c0cb350 secure :-1 s=READY pgs=1251 cs=0 l=1 rev1=1 crypto rx=0x7fa42c00e8a0 tx=0x7fa42c00ed60 comp rx=0 tx=0).stop 2024-03-04T07:32:01.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 shutdown_connections 2024-03-04T07:32:01.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa41c06b4d0 0x7fa41c06d990 unknown :-1 s=CLOSED pgs=566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa43c15dd00 0x7fa43c0b6a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa43c06be70 0x7fa43c1f75c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 --2- 172.21.15.64:0/2314515798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa43c06af20 0x7fa43c0cb350 unknown :-1 s=CLOSED pgs=1251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 >> 172.21.15.64:0/2314515798 conn(0x7fa43c0b34a0 msgr2=0x7fa43c0b2370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:01.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 shutdown_connections 2024-03-04T07:32:01.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.422+0000 7fa43b59e640 1 -- 172.21.15.64:0/2314515798 wait complete. 2024-03-04T07:32:01.437 INFO:tasks.cephfs_test_runner:test_files_throttle (tasks.cephfs.test_strays.TestStrays) ... ok 2024-03-04T07:32:01.437 INFO:tasks.cephfs_test_runner:Starting test: test_hardlink_reintegration (tasks.cephfs.test_strays.TestStrays) That removal of primary dentry of hardlinked inode results 2024-03-04T07:32:01.437 INFO:tasks.cephfs_test_runner:test_hardlink_reintegration (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:32:01.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:32:01.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.544+0000 7fc057088640 1 Processor -- start 2024-03-04T07:32:01.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.544+0000 7fc057088640 1 -- start start 2024-03-04T07:32:01.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc057088640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc0500b1020 0x7fc05006af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc057088640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc0500cc0e0 con 0x7fc05006ed10 2024-03-04T07:32:01.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc057088640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc0500cc3c0 con 0x7fc0500cd890 2024-03-04T07:32:01.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc057088640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc0500d28b0 con 0x7fc0500b1020 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc05006ed10 0x7fc0500cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35434/0 (socket says 172.21.15.64:35434) 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04ffff640 1 -- 172.21.15.64:0/1382721187 learned_addr learned my addr 172.21.15.64:0/1382721187 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 727677196 0 0) 0x7fc0500cc0e0 con 0x7fc05006ed10 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc040003520 con 0x7fc05006ed10 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc054e24640 1 --2- 172.21.15.64:0/1382721187 >> v2:172.21.15.66:3300/0 conn(0x7fc0500b1020 0x7fc05006af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 478813964 0 0) 0x7fc040003520 con 0x7fc05006ed10 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 >> v2:172.21.15.66:3300/0 conn(0x7fc0500b1020 msgr2=0x7fc05006af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 --2- 172.21.15.64:0/1382721187 >> v2:172.21.15.66:3300/0 conn(0x7fc0500b1020 0x7fc05006af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 >> v1:172.21.15.66:6789/0 conn(0x7fc0500cd890 legacy=0x7fc05015d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0500cc6a0 con 0x7fc05006ed10 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fc03c001670 con 0x7fc05006ed10 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 >> v1:172.21.15.64:6789/0 conn(0x7fc05006ed10 legacy=0x7fc0500cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc040003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc040004800 0x7fc040004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.545+0000 7fc04f7fe640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0400051c0 0x7fc04000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0500d28b0 con 0x7fc040003a00 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc04000c0c0 con 0x7fc0400051c0 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc04000c2c0 con 0x7fc040004800 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc054e24640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc040003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc04ffff640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc040004800 0x7fc040004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc055625640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0400051c0 0x7fc04000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc054e24640 1 -- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc040004800 msgr2=0x7fc040004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc054e24640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc040004800 0x7fc040004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc054e24640 1 -- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0400051c0 msgr2=0x7fc04000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc054e24640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0400051c0 0x7fc04000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc054e24640 1 -- 172.21.15.64:0/1382721187 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc040003520 con 0x7fc040003a00 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc055625640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0400051c0 0x7fc04000b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.546+0000 7fc055625640 1 -- 172.21.15.64:0/1382721187 reap_dead start 2024-03-04T07:32:01.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc054e24640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc040003e20 secure :-1 s=READY pgs=1253 cs=0 l=1 rev1=1 crypto rx=0x7fc044009ac0 tx=0x7fc040005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f5037b359b11cb6c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc044039070 con 0x7fc040003a00 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc04402c6b0 con 0x7fc040003a00 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc04f7fe640 1 -- 172.21.15.64:0/1382721187 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc044009690 con 0x7fc040003a00 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc057088640 1 -- 172.21.15.64:0/1382721187 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 msgr2=0x7fc040003e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc057088640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc040003e20 secure :-1 s=READY pgs=1253 cs=0 l=1 rev1=1 crypto rx=0x7fc044009ac0 tx=0x7fc040005830 comp rx=0 tx=0).stop 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc057088640 1 -- 172.21.15.64:0/1382721187 shutdown_connections 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc057088640 1 --2- 172.21.15.64:0/1382721187 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc040003e20 unknown :-1 s=CLOSED pgs=1253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc057088640 1 -- 172.21.15.64:0/1382721187 >> 172.21.15.64:0/1382721187 conn(0x7fc0500d1a90 msgr2=0x7fc0500d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc057088640 1 -- 172.21.15.64:0/1382721187 shutdown_connections 2024-03-04T07:32:01.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.547+0000 7fc057088640 1 -- 172.21.15.64:0/1382721187 wait complete. 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 Processor -- start 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 -- start start 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc05015dbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0500c9c20 0x7fc0500ca0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0500c70a0 0x7fc0500c7520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0500cdca0 con 0x7fc040003a00 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0500ccc60 con 0x7fc0500c9c20 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.549+0000 7fc057088640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc0500cde20 con 0x7fc0500c70a0 2024-03-04T07:32:01.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc05015dbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc05015dbf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44886/0 (socket says 172.21.15.64:44886) 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 -- 172.21.15.64:0/924619023 learned_addr learned my addr 172.21.15.64:0/924619023 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc04ffff640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0500c9c20 0x7fc0500ca0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc055625640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0500c70a0 0x7fc0500c7520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 -- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0500c70a0 msgr2=0x7fc0500c7520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0500c70a0 0x7fc0500c7520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 -- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0500c9c20 msgr2=0x7fc0500ca0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0500c9c20 0x7fc0500ca0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 -- 172.21.15.64:0/924619023 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0500b4c50 con 0x7fc040003a00 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc054e24640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc05015dbf0 secure :-1 s=READY pgs=1254 cs=0 l=1 rev1=1 crypto rx=0x7fc044000c00 tx=0x7fc044004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc04d7fa640 1 -- 172.21.15.64:0/924619023 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc044042020 con 0x7fc040003a00 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc04d7fa640 1 -- 172.21.15.64:0/924619023 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc04403a040 con 0x7fc040003a00 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0500b4f30 con 0x7fc040003a00 2024-03-04T07:32:01.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc04d7fa640 1 -- 172.21.15.64:0/924619023 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc044039070 con 0x7fc040003a00 2024-03-04T07:32:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.550+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0500b54f0 con 0x7fc040003a00 2024-03-04T07:32:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.551+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc050067ff0 con 0x7fc040003a00 2024-03-04T07:32:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.552+0000 7fc04d7fa640 1 -- 172.21.15.64:0/924619023 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc044045020 con 0x7fc040003a00 2024-03-04T07:32:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.552+0000 7fc04d7fa640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc03406b500 0x7fc03406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.552+0000 7fc04d7fa640 1 -- 172.21.15.64:0/924619023 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fc0440ab310 con 0x7fc040003a00 2024-03-04T07:32:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.552+0000 7fc04ffff640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc03406b500 0x7fc03406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.553+0000 7fc04ffff640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc03406b500 0x7fc03406d9c0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7fc03c001eb0 tx=0x7fc03c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:01.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.557+0000 7fc04d7fa640 1 -- 172.21.15.64:0/924619023 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc04404c070 con 0x7fc040003a00 2024-03-04T07:32:01.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.740+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc05006f2f0 con 0x7fc040003a00 2024-03-04T07:32:01.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.740+0000 7fc04d7fa640 1 -- 172.21.15.64:0/924619023 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v97) v1 ==== 74+0+13233 (secure 0 0 0) 0x7fc04407a3e0 con 0x7fc040003a00 2024-03-04T07:32:01.741 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:01.741 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":97,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:31:59.409874+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6834/3393965936":"2024-03-05T07:31:56.397335+0000","172.21.15.64:6837/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.64:6835/3848675976":"2024-03-05T07:31:24.148036+0000","172.21.15.66:6835/3397672669":"2024-03-05T07:31:23.154858+0000","172.21.15.64:6835/3393965936":"2024-03-05T07:31:56.397335+0000","172.21.15.66:6833/3397672669":"2024-03-05T07:31:23.154858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:32:01.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc03406b500 msgr2=0x7fc03406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc03406b500 0x7fc03406d9c0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7fc03c001eb0 tx=0x7fc03c009040 comp rx=0 tx=0).stop 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 msgr2=0x7fc05015dbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc05015dbf0 secure :-1 s=READY pgs=1254 cs=0 l=1 rev1=1 crypto rx=0x7fc044000c00 tx=0x7fc044004730 comp rx=0 tx=0).stop 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 shutdown_connections 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc03406b500 0x7fc03406d9c0 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0500c70a0 0x7fc0500c7520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0500c9c20 0x7fc0500ca0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 --2- 172.21.15.64:0/924619023 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc040003a00 0x7fc05015dbf0 unknown :-1 s=CLOSED pgs=1254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 >> 172.21.15.64:0/924619023 conn(0x7fc0500d1a90 msgr2=0x7fc0500ae840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 shutdown_connections 2024-03-04T07:32:01.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.742+0000 7fc057088640 1 -- 172.21.15.64:0/924619023 wait complete. 2024-03-04T07:32:01.754 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:32:01.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:32:01.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.858+0000 7f0b43a9a640 1 Processor -- start 2024-03-04T07:32:01.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.858+0000 7f0b43a9a640 1 -- start start 2024-03-04T07:32:01.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b43a9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 0x7f0b3c072680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b43a9a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0b3c072cf0 con 0x7f0b3c0cf0f0 2024-03-04T07:32:01.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b43a9a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0b3c072fd0 con 0x7f0b3c15dd50 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b43a9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b3c0d1f60 con 0x7f0b3c0d3770 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b41035640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0b3c0cf0f0 0x7f0b3c071f70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35446/0 (socket says 172.21.15.64:35446) 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b41035640 1 -- 172.21.15.64:0/2632390155 learned_addr learned my addr 172.21.15.64:0/2632390155 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b42037640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 0x7f0b3c072680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3731381166 0 0) 0x7f0b3c072cf0 con 0x7f0b3c0cf0f0 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0b28003520 con 0x7f0b3c0cf0f0 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2207885260 0 0) 0x7f0b28003520 con 0x7f0b3c0cf0f0 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 msgr2=0x7f0b3c072680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 0x7f0b3c072680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 >> v1:172.21.15.66:6789/0 conn(0x7f0b3c15dd50 legacy=0x7f0b3c0ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b3c0d2240 con 0x7f0b3c0cf0f0 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f0b2c001670 con 0x7f0b3c0cf0f0 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 >> v1:172.21.15.64:6789/0 conn(0x7f0b3c0cf0f0 legacy=0x7f0b3c071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b280039e0 0x7f0b28003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 0x7f0b28004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b280051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b3c0d1f60 con 0x7f0b28004d40 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b2800bb50 con 0x7f0b280039e0 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b2800bd20 con 0x7f0b3c0d3770 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.859+0000 7f0b42037640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b280051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b41836640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b280039e0 0x7f0b28003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b41035640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 0x7f0b28004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b42037640 1 -- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 msgr2=0x7f0b28004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b42037640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0d3770 0x7f0b28004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b42037640 1 -- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b280039e0 msgr2=0x7f0b28003dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b42037640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b280039e0 0x7f0b28003dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b42037640 1 -- 172.21.15.64:0/2632390155 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b28003520 con 0x7f0b28004d40 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.860+0000 7f0b42037640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b280051c0 secure :-1 s=READY pgs=1256 cs=0 l=1 rev1=1 crypto rx=0x7f0b3400e170 tx=0x7f0b28005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f263cfdc243b745c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b34039070 con 0x7f0b28004d40 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b3402cbf0 con 0x7f0b28004d40 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b40834640 1 -- 172.21.15.64:0/2632390155 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b34009c10 con 0x7f0b28004d40 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b43a9a640 1 -- 172.21.15.64:0/2632390155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 msgr2=0x7f0b280051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b43a9a640 1 --2- 172.21.15.64:0/2632390155 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b280051c0 secure :-1 s=READY pgs=1256 cs=0 l=1 rev1=1 crypto rx=0x7f0b3400e170 tx=0x7f0b28005830 comp rx=0 tx=0).stop 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b42037640 1 -- 172.21.15.64:0/2632390155 reap_dead start 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b43a9a640 1 -- 172.21.15.64:0/2632390155 shutdown_connections 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b43a9a640 1 -- 172.21.15.64:0/2632390155 >> 172.21.15.64:0/2632390155 conn(0x7f0b3c0d3ef0 msgr2=0x7f0b3c0d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b43a9a640 1 -- 172.21.15.64:0/2632390155 shutdown_connections 2024-03-04T07:32:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.861+0000 7f0b43a9a640 1 -- 172.21.15.64:0/2632390155 wait complete. 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.862+0000 7f0b43a9a640 1 Processor -- start 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.862+0000 7f0b43a9a640 1 -- start start 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b43a9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b3c158030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b43a9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b3c0acee0 0x7f0b3c0ad600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b43a9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0adb40 0x7f0b3c1f3fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b43a9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b3c072680 con 0x7f0b28004d40 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b43a9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b3c15e340 con 0x7f0b3c0acee0 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b43a9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b3c072800 con 0x7f0b3c0adb40 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b3c158030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b3c158030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44908/0 (socket says 172.21.15.64:44908) 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 -- 172.21.15.64:0/1408223646 learned_addr learned my addr 172.21.15.64:0/1408223646 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 -- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0adb40 msgr2=0x7f0b3c1f3fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41035640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b3c0acee0 0x7f0b3c0ad600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b42037640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0adb40 0x7f0b3c1f3fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0adb40 0x7f0b3c1f3fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 -- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b3c0acee0 msgr2=0x7f0b3c0ad600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b3c0acee0 0x7f0b3c0ad600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41836640 1 -- 172.21.15.64:0/1408223646 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b3c0ac310 con 0x7f0b28004d40 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b41035640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b3c0acee0 0x7f0b3c0ad600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.863+0000 7f0b42037640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0adb40 0x7f0b3c1f3fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.864+0000 7f0b41836640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b3c158030 secure :-1 s=READY pgs=1257 cs=0 l=1 rev1=1 crypto rx=0x7f0b3800f820 tx=0x7f0b3800fce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.864+0000 7f0b327fc640 1 -- 172.21.15.64:0/1408223646 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b380052e0 con 0x7f0b28004d40 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.864+0000 7f0b327fc640 1 -- 172.21.15.64:0/1408223646 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0b38018070 con 0x7f0b28004d40 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.864+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0b3c15b480 con 0x7f0b28004d40 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.864+0000 7f0b327fc640 1 -- 172.21.15.64:0/1408223646 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b3800d910 con 0x7f0b28004d40 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.864+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0b3c15ba40 con 0x7f0b28004d40 2024-03-04T07:32:01.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.865+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0b3c06cdd0 con 0x7f0b28004d40 2024-03-04T07:32:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.865+0000 7f0b327fc640 1 -- 172.21.15.64:0/1408223646 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0b3800dab0 con 0x7f0b28004d40 2024-03-04T07:32:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.866+0000 7f0b327fc640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b2006b4d0 0x7f0b2006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.866+0000 7f0b41035640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b2006b4d0 0x7f0b2006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.866+0000 7f0b327fc640 1 -- 172.21.15.64:0/1408223646 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f0b38089ee0 con 0x7f0b28004d40 2024-03-04T07:32:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.866+0000 7f0b41035640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b2006b4d0 0x7f0b2006d990 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7f0b3c0b85f0 tx=0x7f0b2c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:01.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:01.870+0000 7f0b327fc640 1 -- 172.21.15.64:0/1408223646 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0b38058fb0 con 0x7f0b28004d40 2024-03-04T07:32:02.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.052+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f0b3c0adf20 con 0x7f0b28004d40 2024-03-04T07:32:02.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.053+0000 7f0b327fc640 1 -- 172.21.15.64:0/1408223646 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v97) v1 ==== 93+0+10 (secure 0 0 0) 0x7f0b380588b0 con 0x7f0b28004d40 2024-03-04T07:32:02.053 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b2006b4d0 msgr2=0x7f0b2006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b2006b4d0 0x7f0b2006d990 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7f0b3c0b85f0 tx=0x7f0b2c009040 comp rx=0 tx=0).stop 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 msgr2=0x7f0b3c158030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b3c158030 secure :-1 s=READY pgs=1257 cs=0 l=1 rev1=1 crypto rx=0x7f0b3800f820 tx=0x7f0b3800fce0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 shutdown_connections 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0b2006b4d0 0x7f0b2006d990 unknown :-1 s=CLOSED pgs=568 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0b3c0adb40 0x7f0b3c1f3fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0b3c0acee0 0x7f0b3c0ad600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 --2- 172.21.15.64:0/1408223646 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0b28004d40 0x7f0b3c158030 unknown :-1 s=CLOSED pgs=1257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.054+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 >> 172.21.15.64:0/1408223646 conn(0x7f0b3c0d3ef0 msgr2=0x7f0b3c15faa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.055+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 shutdown_connections 2024-03-04T07:32:02.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.055+0000 7f0b43a9a640 1 -- 172.21.15.64:0/1408223646 wait complete. 2024-03-04T07:32:02.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration' 2024-03-04T07:32:02.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.168+0000 7f9a9e0df640 1 Processor -- start 2024-03-04T07:32:02.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.168+0000 7f9a9e0df640 1 -- start start 2024-03-04T07:32:02.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.168+0000 7f9a9e0df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a9806ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.168+0000 7f9a9e0df640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9a980b2460 0x7f9a9806d210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.168+0000 7f9a9e0df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9a9815e410 con 0x7f9a9806ef70 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.168+0000 7f9a9e0df640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9a98162700 con 0x7f9a980b2460 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.168+0000 7f9a9e0df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a98161af0 con 0x7f9a9815e030 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a97fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9a980b2460 0x7f9a9806d210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a96ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9a9806ef70 0x7f9a980b1d50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35460/0 (socket says 172.21.15.64:35460) 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a96ffd640 1 -- 172.21.15.64:0/3190161456 learned_addr learned my addr 172.21.15.64:0/3190161456 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a97fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9a980b2460 0x7f9a9806d210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44914/0 (socket says 172.21.15.64:44914) 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a977fe640 1 --2- 172.21.15.64:0/3190161456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a9806ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a97fff640 1 -- 172.21.15.64:0/3190161456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 msgr2=0x7f9a9806ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a97fff640 1 --2- 172.21.15.64:0/3190161456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a9806ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a97fff640 1 -- 172.21.15.64:0/3190161456 >> v1:172.21.15.64:6789/0 conn(0x7f9a9806ef70 legacy=0x7f9a980b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a97fff640 1 -- 172.21.15.64:0/3190161456 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a98161dd0 con 0x7f9a980b2460 2024-03-04T07:32:02.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.169+0000 7f9a967fc640 1 -- 172.21.15.64:0/3190161456 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2778887481 0 0) 0x7f9a8c001110 con 0x7f9a9806ef70 2024-03-04T07:32:02.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a97fff640 1 --2- 172.21.15.64:0/3190161456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a9806d210 secure :-1 s=READY pgs=1259 cs=0 l=1 rev1=1 crypto rx=0x7f9a90009d90 tx=0x7f9a9002bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d4e7a8ac164a3098 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:02.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a967fc640 1 -- 172.21.15.64:0/3190161456 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a90039070 con 0x7f9a980b2460 2024-03-04T07:32:02.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a967fc640 1 -- 172.21.15.64:0/3190161456 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9a9002cc50 con 0x7f9a980b2460 2024-03-04T07:32:02.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a967fc640 1 -- 172.21.15.64:0/3190161456 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a90034440 con 0x7f9a980b2460 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 -- 172.21.15.64:0/3190161456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 msgr2=0x7f9a9806d210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/3190161456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a9806d210 secure :-1 s=READY pgs=1259 cs=0 l=1 rev1=1 crypto rx=0x7f9a90009d90 tx=0x7f9a9002bea0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 -- 172.21.15.64:0/3190161456 shutdown_connections 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/3190161456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a9806d210 unknown :-1 s=CLOSED pgs=1259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/3190161456 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a9806ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 -- 172.21.15.64:0/3190161456 >> 172.21.15.64:0/3190161456 conn(0x7f9a980b55e0 msgr2=0x7f9a980b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 -- 172.21.15.64:0/3190161456 shutdown_connections 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.170+0000 7f9a9e0df640 1 -- 172.21.15.64:0/3190161456 wait complete. 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.172+0000 7f9a9e0df640 1 Processor -- start 2024-03-04T07:32:02.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.172+0000 7f9a9e0df640 1 -- start start 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.172+0000 7f9a9e0df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9a9806ef70 0x7f9a980ad0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.172+0000 7f9a9e0df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a98154ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.172+0000 7f9a9e0df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a981553e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.172+0000 7f9a9e0df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a980b31e0 con 0x7f9a980b2460 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.172+0000 7f9a9e0df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a98162390 con 0x7f9a9806ef70 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a9e0df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a980b3360 con 0x7f9a9815e030 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a98154ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a98154ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:44930/0 (socket says 172.21.15.64:44930) 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 -- 172.21.15.64:0/2799987274 learned_addr learned my addr 172.21.15.64:0/2799987274 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a977fe640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9a9806ef70 0x7f9a980ad0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 -- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 msgr2=0x7f9a981553e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a97fff640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a981553e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a981553e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 -- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9a9806ef70 msgr2=0x7f9a980ad0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9a9806ef70 0x7f9a980ad0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 -- 172.21.15.64:0/2799987274 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a980ada80 con 0x7f9a980b2460 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a97fff640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a981553e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a96ffd640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a98154ea0 secure :-1 s=READY pgs=1260 cs=0 l=1 rev1=1 crypto rx=0x7f9a8c00c7b0 tx=0x7f9a8c00cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a7ffff640 1 -- 172.21.15.64:0/2799987274 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a8c018070 con 0x7f9a980b2460 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a7ffff640 1 -- 172.21.15.64:0/2799987274 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9a8c0072d0 con 0x7f9a980b2460 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9a980add60 con 0x7f9a980b2460 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a7ffff640 1 -- 172.21.15.64:0/2799987274 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9a8c0047e0 con 0x7f9a980b2460 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.173+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9a980ae320 con 0x7f9a980b2460 2024-03-04T07:32:02.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.174+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9a98067ff0 con 0x7f9a980b2460 2024-03-04T07:32:02.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.175+0000 7f9a7ffff640 1 -- 172.21.15.64:0/2799987274 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9a8c01b070 con 0x7f9a980b2460 2024-03-04T07:32:02.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.175+0000 7f9a7ffff640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9a8406b4d0 0x7f9a8406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:02.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.175+0000 7f9a7ffff640 1 -- 172.21.15.64:0/2799987274 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f9a8c0893b0 con 0x7f9a980b2460 2024-03-04T07:32:02.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.175+0000 7f9a977fe640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9a8406b4d0 0x7f9a8406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:02.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.176+0000 7f9a977fe640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9a8406b4d0 0x7f9a8406d990 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f9a88009a30 tx=0x7f9a88008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:02.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.180+0000 7f9a7ffff640 1 -- 172.21.15.64:0/2799987274 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9a9815e410 con 0x7f9a980b2460 2024-03-04T07:32:02.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:02.429+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]} v 0) v1 -- 0x7f9a980b2910 con 0x7f9a980b2460 2024-03-04T07:32:03.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.424+0000 7f9a7ffff640 1 -- 172.21.15.64:0/2799987274 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]}]=0 v288) v1 ==== 145+0+0 (secure 0 0 0) 0x7f9a8c0583f0 con 0x7f9a980b2460 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9a8406b4d0 msgr2=0x7f9a8406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9a8406b4d0 0x7f9a8406d990 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f9a88009a30 tx=0x7f9a88008040 comp rx=0 tx=0).stop 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 msgr2=0x7f9a98154ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a98154ea0 secure :-1 s=READY pgs=1260 cs=0 l=1 rev1=1 crypto rx=0x7f9a8c00c7b0 tx=0x7f9a8c00cc70 comp rx=0 tx=0).stop 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 shutdown_connections 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9a8406b4d0 0x7f9a8406d990 unknown :-1 s=CLOSED pgs=569 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9a9815e030 0x7f9a981553e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9a980b2460 0x7f9a98154ea0 unknown :-1 s=CLOSED pgs=1260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 --2- 172.21.15.64:0/2799987274 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9a9806ef70 0x7f9a980ad0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 >> 172.21.15.64:0/2799987274 conn(0x7f9a980b55e0 msgr2=0x7f9a980b1100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:03.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 shutdown_connections 2024-03-04T07:32:03.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.427+0000 7f9a9e0df640 1 -- 172.21.15.64:0/2799987274 wait complete. 2024-03-04T07:32:03.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:32:03.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.552+0000 7f2f9d49a640 1 Processor -- start 2024-03-04T07:32:03.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.552+0000 7f2f9d49a640 1 -- start start 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.552+0000 7f2f9d49a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2f980cf0f0 0x7f2f98071f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.552+0000 7f2f9d49a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2f980724b0 0x7f2f980d2760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.552+0000 7f2f9d49a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2f98072cf0 con 0x7f2f9815dd50 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.552+0000 7f2f9d49a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2f98072fd0 con 0x7f2f980cf0f0 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.552+0000 7f2f9d49a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2f980d1f60 con 0x7f2f980724b0 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f8ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2f980cf0f0 0x7f2f98071f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f8ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2f980cf0f0 0x7f2f98071f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52638/0 (socket says 172.21.15.64:52638) 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f8ffff640 1 -- 172.21.15.64:0/3943300047 learned_addr learned my addr 172.21.15.64:0/3943300047 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f96ffd640 1 --1- 172.21.15.64:0/3943300047 >> v1:172.21.15.64:6789/0 conn(0x7f2f9815dd50 0x7f2f980ce9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44608/0 (socket says 172.21.15.64:44608) 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f8ffff640 1 -- 172.21.15.64:0/3943300047 >> v2:172.21.15.66:3300/0 conn(0x7f2f980724b0 msgr2=0x7f2f980d2760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f8ffff640 1 --2- 172.21.15.64:0/3943300047 >> v2:172.21.15.66:3300/0 conn(0x7f2f980724b0 0x7f2f980d2760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f8ffff640 1 -- 172.21.15.64:0/3943300047 >> v1:172.21.15.64:6789/0 conn(0x7f2f9815dd50 legacy=0x7f2f980ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f967fc640 1 -- 172.21.15.64:0/3943300047 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2526113307 0 0) 0x7f2f98072cf0 con 0x7f2f9815dd50 2024-03-04T07:32:03.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.553+0000 7f2f8ffff640 1 -- 172.21.15.64:0/3943300047 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f980d2240 con 0x7f2f980cf0f0 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f8ffff640 1 --2- 172.21.15.64:0/3943300047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f980cf0f0 0x7f2f98071f70 secure :-1 s=READY pgs=1262 cs=0 l=1 rev1=1 crypto rx=0x7f2f980d03f0 tx=0x7f2f8002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fa8c2b25269c7c7a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f967fc640 1 -- 172.21.15.64:0/3943300047 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f80035020 con 0x7f2f980cf0f0 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f967fc640 1 -- 172.21.15.64:0/3943300047 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f80031870 con 0x7f2f980cf0f0 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f967fc640 1 -- 172.21.15.64:0/3943300047 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f80031c40 con 0x7f2f980cf0f0 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f9d49a640 1 -- 172.21.15.64:0/3943300047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f980cf0f0 msgr2=0x7f2f98071f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/3943300047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f980cf0f0 0x7f2f98071f70 secure :-1 s=READY pgs=1262 cs=0 l=1 rev1=1 crypto rx=0x7f2f980d03f0 tx=0x7f2f8002b910 comp rx=0 tx=0).stop 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f9d49a640 1 -- 172.21.15.64:0/3943300047 shutdown_connections 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/3943300047 >> v2:172.21.15.66:3300/0 conn(0x7f2f980724b0 0x7f2f980d2760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/3943300047 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f980cf0f0 0x7f2f98071f70 unknown :-1 s=CLOSED pgs=1262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.554+0000 7f2f9d49a640 1 -- 172.21.15.64:0/3943300047 >> 172.21.15.64:0/3943300047 conn(0x7f2f980d3ef0 msgr2=0x7f2f980d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.555+0000 7f2f9d49a640 1 -- 172.21.15.64:0/3943300047 shutdown_connections 2024-03-04T07:32:03.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.555+0000 7f2f9d49a640 1 -- 172.21.15.64:0/3943300047 wait complete. 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 Processor -- start 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 -- start start 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2f980724b0 0x7f2f980ccf10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2f980cf0f0 0x7f2f981e7120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f9815dd50 0x7f2f98156ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f9815e2b0 con 0x7f2f9815dd50 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f9815e130 con 0x7f2f980cf0f0 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f9d49a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2f980d2ca0 con 0x7f2f980724b0 2024-03-04T07:32:03.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.556+0000 7f2f977fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f9815dd50 0x7f2f98156ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f9815dd50 0x7f2f98156ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52644/0 (socket says 172.21.15.64:52644) 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 -- 172.21.15.64:0/2569245358 learned_addr learned my addr 172.21.15.64:0/2569245358 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f8ffff640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2f980cf0f0 0x7f2f981e7120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f96ffd640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2f980724b0 0x7f2f980ccf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 -- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2f980724b0 msgr2=0x7f2f980ccf10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2f980724b0 0x7f2f980ccf10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 -- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2f980cf0f0 msgr2=0x7f2f981e7120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2f980cf0f0 0x7f2f981e7120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 -- 172.21.15.64:0/2569245358 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f981e8a10 con 0x7f2f9815dd50 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f977fe640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f9815dd50 0x7f2f98156ad0 secure :-1 s=READY pgs=1263 cs=0 l=1 rev1=1 crypto rx=0x7f2f9000b970 tx=0x7f2f9000be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f8f7fe640 1 -- 172.21.15.64:0/2569245358 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f90019070 con 0x7f2f9815dd50 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f8f7fe640 1 -- 172.21.15.64:0/2569245358 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2f9000cb00 con 0x7f2f9815dd50 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f981e7970 con 0x7f2f9815dd50 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f8f7fe640 1 -- 172.21.15.64:0/2569245358 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f90014440 con 0x7f2f9815dd50 2024-03-04T07:32:03.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.557+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f981e7f30 con 0x7f2f9815dd50 2024-03-04T07:32:03.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.558+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2f9806cdd0 con 0x7f2f9815dd50 2024-03-04T07:32:03.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.559+0000 7f2f8f7fe640 1 -- 172.21.15.64:0/2569245358 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2f9001c070 con 0x7f2f9815dd50 2024-03-04T07:32:03.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.559+0000 7f2f8f7fe640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2f7806b4d0 0x7f2f7806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:03.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.559+0000 7f2f8f7fe640 1 -- 172.21.15.64:0/2569245358 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f2f90089b50 con 0x7f2f9815dd50 2024-03-04T07:32:03.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.559+0000 7f2f96ffd640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2f7806b4d0 0x7f2f7806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:03.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.559+0000 7f2f96ffd640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2f7806b4d0 0x7f2f7806d990 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f2f88007fd0 tx=0x7f2f88007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:03.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.563+0000 7f2f8f7fe640 1 -- 172.21.15.64:0/2569245358 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2f90058660 con 0x7f2f9815dd50 2024-03-04T07:32:03.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.746+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7f2f9804f410 con 0x7f2f9815dd50 2024-03-04T07:32:03.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.749+0000 7f2f8f7fe640 1 -- 172.21.15.64:0/2569245358 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v30)=0 v30) v1 ==== 122+0+0 (secure 0 0 0) 0x7f2f9005c700 con 0x7f2f9815dd50 2024-03-04T07:32:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2f7806b4d0 msgr2=0x7f2f7806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2f7806b4d0 0x7f2f7806d990 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f2f88007fd0 tx=0x7f2f88007de0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f9815dd50 msgr2=0x7f2f98156ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f9815dd50 0x7f2f98156ad0 secure :-1 s=READY pgs=1263 cs=0 l=1 rev1=1 crypto rx=0x7f2f9000b970 tx=0x7f2f9000be30 comp rx=0 tx=0).stop 2024-03-04T07:32:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 shutdown_connections 2024-03-04T07:32:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2f7806b4d0 0x7f2f7806d990 unknown :-1 s=CLOSED pgs=570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2f9815dd50 0x7f2f98156ad0 unknown :-1 s=CLOSED pgs=1263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2f980cf0f0 0x7f2f981e7120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 --2- 172.21.15.64:0/2569245358 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2f980724b0 0x7f2f980ccf10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 >> 172.21.15.64:0/2569245358 conn(0x7f2f980d3ef0 msgr2=0x7f2f9815bd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 shutdown_connections 2024-03-04T07:32:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:03.751+0000 7f2f9d49a640 1 -- 172.21.15.64:0/2569245358 wait complete. 2024-03-04T07:32:03.763 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:32:03.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:32:03.801 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:32:03.850 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:32:03.850 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:32:03.850 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:32:03.865 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:32:03.865 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:32:03.865 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:32:03.883 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:32:03.883 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:32:03.883 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:32:03.920 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:32:03.921 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:32:03.921 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:32:03.940 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:32:03.940 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:32:03.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:04.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.061+0000 7ff280e9d640 1 Processor -- start 2024-03-04T07:32:04.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.061+0000 7ff280e9d640 1 -- start start 2024-03-04T07:32:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff280e9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff27c06df30 0x7ff27c0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff280e9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff27c0aec40 0x7ff27c0b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff280e9d640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff27c071240 con 0x7ff27c158520 2024-03-04T07:32:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff280e9d640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff27c0af0f0 con 0x7ff27c06df30 2024-03-04T07:32:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff280e9d640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff27c071b70 con 0x7ff27c0aec40 2024-03-04T07:32:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff271d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff27c06df30 0x7ff27c0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff27a59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff27c158520 0x7ff27c06d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44620/0 (socket says 172.21.15.64:44620) 2024-03-04T07:32:04.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff27a59c640 1 -- 172.21.15.64:0/1442249127 learned_addr learned my addr 172.21.15.64:0/1442249127 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:04.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff271d9b640 1 -- 172.21.15.64:0/1442249127 >> v2:172.21.15.66:3300/0 conn(0x7ff27c0aec40 msgr2=0x7ff27c0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:04.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff271d9b640 1 --2- 172.21.15.64:0/1442249127 >> v2:172.21.15.66:3300/0 conn(0x7ff27c0aec40 0x7ff27c0b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff271d9b640 1 -- 172.21.15.64:0/1442249127 >> v1:172.21.15.64:6789/0 conn(0x7ff27c158520 legacy=0x7ff27c06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.062+0000 7ff271d9b640 1 -- 172.21.15.64:0/1442249127 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff27c071520 con 0x7ff27c06df30 2024-03-04T07:32:04.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.063+0000 7ff271d9b640 1 --2- 172.21.15.64:0/1442249127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c06df30 0x7ff27c0ae700 secure :-1 s=READY pgs=1265 cs=0 l=1 rev1=1 crypto rx=0x7ff2640097a0 tx=0x7ff26402b710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=20351dc77d9814db server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:04.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.063+0000 7ff279d9b640 1 -- 172.21.15.64:0/1442249127 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff264037070 con 0x7ff27c06df30 2024-03-04T07:32:04.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.063+0000 7ff279d9b640 1 -- 172.21.15.64:0/1442249127 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff264030710 con 0x7ff27c06df30 2024-03-04T07:32:04.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.063+0000 7ff279d9b640 1 -- 172.21.15.64:0/1442249127 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff264030ae0 con 0x7ff27c06df30 2024-03-04T07:32:04.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.063+0000 7ff280e9d640 1 -- 172.21.15.64:0/1442249127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c06df30 msgr2=0x7ff27c0ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.063+0000 7ff280e9d640 1 --2- 172.21.15.64:0/1442249127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c06df30 0x7ff27c0ae700 secure :-1 s=READY pgs=1265 cs=0 l=1 rev1=1 crypto rx=0x7ff2640097a0 tx=0x7ff26402b710 comp rx=0 tx=0).stop 2024-03-04T07:32:04.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.063+0000 7ff280e9d640 1 -- 172.21.15.64:0/1442249127 shutdown_connections 2024-03-04T07:32:04.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.064+0000 7ff280e9d640 1 --2- 172.21.15.64:0/1442249127 >> v2:172.21.15.66:3300/0 conn(0x7ff27c0aec40 0x7ff27c0b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.064+0000 7ff280e9d640 1 --2- 172.21.15.64:0/1442249127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c06df30 0x7ff27c0ae700 unknown :-1 s=CLOSED pgs=1265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.064+0000 7ff280e9d640 1 -- 172.21.15.64:0/1442249127 >> 172.21.15.64:0/1442249127 conn(0x7ff27c066d00 msgr2=0x7ff27c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:04.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.064+0000 7ff280e9d640 1 -- 172.21.15.64:0/1442249127 shutdown_connections 2024-03-04T07:32:04.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.064+0000 7ff280e9d640 1 -- 172.21.15.64:0/1442249127 wait complete. 2024-03-04T07:32:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.065+0000 7ff280e9d640 1 Processor -- start 2024-03-04T07:32:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.065+0000 7ff280e9d640 1 -- start start 2024-03-04T07:32:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff280e9d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff27c06df30 0x7ff27c148f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff280e9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c0aec40 0x7ff27c14c890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff280e9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff27c158520 0x7ff27c149c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff280e9d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff27c158900 con 0x7ff27c0aec40 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff280e9d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff27c072000 con 0x7ff27c06df30 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff280e9d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff27c158a80 con 0x7ff27c158520 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c0aec40 0x7ff27c14c890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c0aec40 0x7ff27c14c890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52660/0 (socket says 172.21.15.64:52660) 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 -- 172.21.15.64:0/54813304 learned_addr learned my addr 172.21.15.64:0/54813304 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff27a59c640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff27c06df30 0x7ff27c148f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff27ad9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff27c158520 0x7ff27c149c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 -- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff27c158520 msgr2=0x7ff27c149c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff27c158520 0x7ff27c149c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 -- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff27c06df30 msgr2=0x7ff27c148f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff27c06df30 0x7ff27c148f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 -- 172.21.15.64:0/54813304 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff27c15d750 con 0x7ff27c0aec40 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff27ad9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff27c158520 0x7ff27c149c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.066+0000 7ff271d9b640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c0aec40 0x7ff27c14c890 secure :-1 s=READY pgs=1266 cs=0 l=1 rev1=1 crypto rx=0x7ff2640097a0 tx=0x7ff2640046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.067+0000 7ff2737fe640 1 -- 172.21.15.64:0/54813304 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff264042070 con 0x7ff27c0aec40 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.067+0000 7ff2737fe640 1 -- 172.21.15.64:0/54813304 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff264032a40 con 0x7ff27c0aec40 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.067+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff27c15da30 con 0x7ff27c0aec40 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.067+0000 7ff2737fe640 1 -- 172.21.15.64:0/54813304 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff264037040 con 0x7ff27c0aec40 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.067+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff27c14d220 con 0x7ff27c0aec40 2024-03-04T07:32:04.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.068+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff27c067ff0 con 0x7ff27c0aec40 2024-03-04T07:32:04.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.068+0000 7ff2737fe640 1 -- 172.21.15.64:0/54813304 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff264045070 con 0x7ff27c0aec40 2024-03-04T07:32:04.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.068+0000 7ff2737fe640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff25c06b4d0 0x7ff25c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.069+0000 7ff2737fe640 1 -- 172.21.15.64:0/54813304 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7ff2640abf00 con 0x7ff27c0aec40 2024-03-04T07:32:04.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.069+0000 7ff27a59c640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff25c06b4d0 0x7ff25c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.069+0000 7ff27a59c640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff25c06b4d0 0x7ff25c06d990 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7ff26c000f60 tx=0x7ff26c007e20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:04.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.073+0000 7ff2737fe640 1 -- 172.21.15.64:0/54813304 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff26407aaa0 con 0x7ff27c0aec40 2024-03-04T07:32:04.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.293+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff27c0b31f0 con 0x7ff27c0aec40 2024-03-04T07:32:04.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.293+0000 7ff2737fe640 1 -- 172.21.15.64:0/54813304 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 61 v61) v1 ==== 94+0+3699 (secure 0 0 0) 0x7ff26407eb40 con 0x7ff27c0aec40 2024-03-04T07:32:04.294 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:04.295 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":61,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6802,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2535134006","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2535134006},{"type":"v1","addr":"172.21.15.66:6833","nonce":2535134006}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":6805,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3175436483","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3175436483},{"type":"v1","addr":"172.21.15.66:6835","nonce":3175436483}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":54},{"gid":7563,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2938742806","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":2938742806},{"type":"v1","addr":"172.21.15.64:6837","nonce":2938742806}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":56},{"gid":7776,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/1945462689","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1945462689},{"type":"v1","addr":"172.21.15.64:6835","nonce":1945462689}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":61}],"filesystems":[]} 2024-03-04T07:32:04.295 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 61 2024-03-04T07:32:04.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff25c06b4d0 msgr2=0x7ff25c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff25c06b4d0 0x7ff25c06d990 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7ff26c000f60 tx=0x7ff26c007e20 comp rx=0 tx=0).stop 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c0aec40 msgr2=0x7ff27c14c890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c0aec40 0x7ff27c14c890 secure :-1 s=READY pgs=1266 cs=0 l=1 rev1=1 crypto rx=0x7ff2640097a0 tx=0x7ff2640046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 shutdown_connections 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff25c06b4d0 0x7ff25c06d990 unknown :-1 s=CLOSED pgs=571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff27c158520 0x7ff27c149c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff27c0aec40 0x7ff27c14c890 unknown :-1 s=CLOSED pgs=1266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 --2- 172.21.15.64:0/54813304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff27c06df30 0x7ff27c148f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 >> 172.21.15.64:0/54813304 conn(0x7ff27c066d00 msgr2=0x7ff27c15a390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.295+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 shutdown_connections 2024-03-04T07:32:04.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.296+0000 7ff280e9d640 1 -- 172.21.15.64:0/54813304 wait complete. 2024-03-04T07:32:04.308 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:32:04.308 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:32:04.308 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:32:04.309 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:04.309 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:32:04.309 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:32:04.309 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:32:04.309 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:04.309 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:32:04.309 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:32:04.310 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:32:04.310 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:04.310 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:32:04.310 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:32:04.310 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:32:04.310 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:04.319 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:32:04.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:32:04.320 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:32:04.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:32:04.321 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:32:04.322 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:32:04.322 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:32:04.322 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:32:04.362 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:32:04.362 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:04.362 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:32:04.363 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:04.365 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:32:04.365 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:04.366 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:32:04.366 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:04.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:32:04.438 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:32:04.438 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:32:04.454 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:32:04.455 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:32:04.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.472+0000 7f3e1b5d0640 1 Processor -- start 2024-03-04T07:32:04.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.472+0000 7f3e1b5d0640 1 -- start start 2024-03-04T07:32:04.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.473+0000 7f3e1b5d0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e140b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.473+0000 7f3e1b5d0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3e140cea30 con 0x7f3e140b9a10 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.473+0000 7f3e1b5d0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3e140cf3c0 con 0x7f3e1415df00 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.473+0000 7f3e1b5d0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3e140cf0e0 con 0x7f3e140b95a0 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.473+0000 7f3e19b6d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3e140b9a10 0x7f3e140d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44642/0 (socket says 172.21.15.64:44642) 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.473+0000 7f3e19b6d640 1 -- 172.21.15.64:0/854365982 learned_addr learned my addr 172.21.15.64:0/854365982 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.473+0000 7f3e18b6b640 1 --2- 172.21.15.64:0/854365982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e140b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.474+0000 7f3e0bfff640 1 -- 172.21.15.64:0/854365982 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2532708654 0 0) 0x7f3e140cea30 con 0x7f3e140b9a10 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.474+0000 7f3e0bfff640 1 -- 172.21.15.64:0/854365982 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3e00003520 con 0x7f3e140b9a10 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.474+0000 7f3e0bfff640 1 -- 172.21.15.64:0/854365982 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3452325882 0 0) 0x7f3e140cf3c0 con 0x7f3e1415df00 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.474+0000 7f3e18b6b640 1 -- 172.21.15.64:0/854365982 >> v1:172.21.15.66:6789/0 conn(0x7f3e1415df00 legacy=0x7f3e140d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.474+0000 7f3e18b6b640 1 -- 172.21.15.64:0/854365982 >> v1:172.21.15.64:6789/0 conn(0x7f3e140b9a10 legacy=0x7f3e140d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.474+0000 7f3e18b6b640 1 -- 172.21.15.64:0/854365982 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e140ce470 con 0x7f3e140b95a0 2024-03-04T07:32:04.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e18b6b640 1 --2- 172.21.15.64:0/854365982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e140b5700 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f3e100097d0 tx=0x7f3e1002b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e0966035deb6e878 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:04.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e0bfff640 1 -- 172.21.15.64:0/854365982 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e10035020 con 0x7f3e140b95a0 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e0bfff640 1 -- 172.21.15.64:0/854365982 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e10031890 con 0x7f3e140b95a0 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e0bfff640 1 -- 172.21.15.64:0/854365982 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e10031c60 con 0x7f3e140b95a0 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/854365982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 msgr2=0x7f3e140b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/854365982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e140b5700 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f3e100097d0 tx=0x7f3e1002b930 comp rx=0 tx=0).stop 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/854365982 shutdown_connections 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/854365982 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e140b5700 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/854365982 >> 172.21.15.64:0/854365982 conn(0x7f3e14071960 msgr2=0x7f3e14071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/854365982 shutdown_connections 2024-03-04T07:32:04.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.475+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/854365982 wait complete. 2024-03-04T07:32:04.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.476+0000 7f3e1b5d0640 1 Processor -- start 2024-03-04T07:32:04.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.476+0000 7f3e1b5d0640 1 -- start start 2024-03-04T07:32:04.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e1b5d0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e14158260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e1b5d0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3e140b9a10 0x7f3e141f3780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e1b5d0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3e1415df00 0x7f3e141f3cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e1b5d0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e140cedf0 con 0x7f3e140b9a10 2024-03-04T07:32:04.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e1b5d0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e140cec70 con 0x7f3e1415df00 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e1b5d0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3e140b5c40 con 0x7f3e140b95a0 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e18b6b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3e140b9a10 0x7f3e141f3780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e18b6b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3e140b9a10 0x7f3e141f3780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52724/0 (socket says 172.21.15.64:52724) 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e18b6b640 1 -- 172.21.15.64:0/1219671996 learned_addr learned my addr 172.21.15.64:0/1219671996 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e1936c640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e14158260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.477+0000 7f3e19b6d640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3e1415df00 0x7f3e141f3cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e18b6b640 1 -- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 msgr2=0x7f3e14158260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e18b6b640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e14158260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e18b6b640 1 -- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3e1415df00 msgr2=0x7f3e141f3cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e18b6b640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3e1415df00 0x7f3e141f3cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e18b6b640 1 -- 172.21.15.64:0/1219671996 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e141f42a0 con 0x7f3e140b9a10 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e18b6b640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3e140b9a10 0x7f3e141f3780 secure :-1 s=READY pgs=1276 cs=0 l=1 rev1=1 crypto rx=0x7f3e100098f0 tx=0x7f3e100046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:04.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e09ffb640 1 -- 172.21.15.64:0/1219671996 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e10035040 con 0x7f3e140b9a10 2024-03-04T07:32:04.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e09ffb640 1 -- 172.21.15.64:0/1219671996 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e10030ca0 con 0x7f3e140b9a10 2024-03-04T07:32:04.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e141f4580 con 0x7f3e140b9a10 2024-03-04T07:32:04.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e09ffb640 1 -- 172.21.15.64:0/1219671996 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e10004270 con 0x7f3e140b9a10 2024-03-04T07:32:04.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.478+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e141f4e40 con 0x7f3e140b9a10 2024-03-04T07:32:04.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.479+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3e14067ff0 con 0x7f3e140b9a10 2024-03-04T07:32:04.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.480+0000 7f3e09ffb640 1 -- 172.21.15.64:0/1219671996 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3e10038020 con 0x7f3e140b9a10 2024-03-04T07:32:04.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.480+0000 7f3e09ffb640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3df806b4d0 0x7f3df806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:04.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.480+0000 7f3e09ffb640 1 -- 172.21.15.64:0/1219671996 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f3e100ac200 con 0x7f3e140b9a10 2024-03-04T07:32:04.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.480+0000 7f3e1936c640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3df806b4d0 0x7f3df806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:04.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.481+0000 7f3e1936c640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3df806b4d0 0x7f3df806d990 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7f3e04000f60 tx=0x7f3e04009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:04.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.484+0000 7f3e09ffb640 1 -- 172.21.15.64:0/1219671996 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3e1002d040 con 0x7f3e140b9a10 2024-03-04T07:32:04.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:04.677+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f3e140d2a80 con 0x7f3e140b9a10 2024-03-04T07:32:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.437+0000 7f3e09ffb640 1 -- 172.21.15.64:0/1219671996 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v99) v1 ==== 97+0+0 (secure 0 0 0) 0x7f3e1007ac80 con 0x7f3e140b9a10 2024-03-04T07:32:05.438 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3df806b4d0 msgr2=0x7f3df806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3df806b4d0 0x7f3df806d990 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7f3e04000f60 tx=0x7f3e04009040 comp rx=0 tx=0).stop 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3e140b9a10 msgr2=0x7f3e141f3780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3e140b9a10 0x7f3e141f3780 secure :-1 s=READY pgs=1276 cs=0 l=1 rev1=1 crypto rx=0x7f3e100098f0 tx=0x7f3e100046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 shutdown_connections 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3df806b4d0 0x7f3df806d990 unknown :-1 s=CLOSED pgs=572 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3e1415df00 0x7f3e141f3cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3e140b9a10 0x7f3e141f3780 unknown :-1 s=CLOSED pgs=1276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 --2- 172.21.15.64:0/1219671996 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3e140b95a0 0x7f3e14158260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 >> 172.21.15.64:0/1219671996 conn(0x7f3e14071960 msgr2=0x7f3e14067450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 shutdown_connections 2024-03-04T07:32:05.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.441+0000 7f3e1b5d0640 1 -- 172.21.15.64:0/1219671996 wait complete. 2024-03-04T07:32:05.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:32:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.564+0000 7fc43c4a4640 1 Processor -- start 2024-03-04T07:32:05.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.564+0000 7fc43c4a4640 1 -- start start 2024-03-04T07:32:05.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.565+0000 7fc43c4a4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc43415df00 0x7fc4340d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.565+0000 7fc43c4a4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc4340cea30 con 0x7fc4340b9a30 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.565+0000 7fc43c4a4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc4340d2af0 con 0x7fc4340b95a0 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.565+0000 7fc43c4a4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc4340cf3c0 con 0x7fc43415df00 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.565+0000 7fc43a240640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc43415df00 0x7fc4340d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.565+0000 7fc43aa41640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc4340b9a30 0x7fc4340d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44648/0 (socket says 172.21.15.64:44648) 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.565+0000 7fc43aa41640 1 -- 172.21.15.64:0/1628182870 learned_addr learned my addr 172.21.15.64:0/1628182870 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.566+0000 7fc43a240640 1 -- 172.21.15.64:0/1628182870 >> v1:172.21.15.66:6789/0 conn(0x7fc4340b95a0 legacy=0x7fc4340b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.566+0000 7fc43a240640 1 -- 172.21.15.64:0/1628182870 >> v1:172.21.15.64:6789/0 conn(0x7fc4340b9a30 legacy=0x7fc4340d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.566+0000 7fc43a240640 1 -- 172.21.15.64:0/1628182870 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4340cf0e0 con 0x7fc43415df00 2024-03-04T07:32:05.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.566+0000 7fc43923e640 1 -- 172.21.15.64:0/1628182870 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3814503933 0 0) 0x7fc4340cea30 con 0x7fc4340b9a30 2024-03-04T07:32:05.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43a240640 1 --2- 172.21.15.64:0/1628182870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415df00 0x7fc4340d25b0 secure :-1 s=READY pgs=1278 cs=0 l=1 rev1=1 crypto rx=0x7fc42c0097a0 tx=0x7fc42c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b26dfe6cdd0016b9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:05.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43923e640 1 -- 172.21.15.64:0/1628182870 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc42c035020 con 0x7fc43415df00 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43923e640 1 -- 172.21.15.64:0/1628182870 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc42c031830 con 0x7fc43415df00 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43923e640 1 -- 172.21.15.64:0/1628182870 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc42c031c00 con 0x7fc43415df00 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43c4a4640 1 -- 172.21.15.64:0/1628182870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415df00 msgr2=0x7fc4340d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/1628182870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415df00 0x7fc4340d25b0 secure :-1 s=READY pgs=1278 cs=0 l=1 rev1=1 crypto rx=0x7fc42c0097a0 tx=0x7fc42c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43c4a4640 1 -- 172.21.15.64:0/1628182870 shutdown_connections 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/1628182870 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415df00 0x7fc4340d25b0 unknown :-1 s=CLOSED pgs=1278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43c4a4640 1 -- 172.21.15.64:0/1628182870 >> 172.21.15.64:0/1628182870 conn(0x7fc434071960 msgr2=0x7fc434071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43c4a4640 1 -- 172.21.15.64:0/1628182870 shutdown_connections 2024-03-04T07:32:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.567+0000 7fc43c4a4640 1 -- 172.21.15.64:0/1628182870 wait complete. 2024-03-04T07:32:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.568+0000 7fc43c4a4640 1 Processor -- start 2024-03-04T07:32:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.568+0000 7fc43c4a4640 1 -- start start 2024-03-04T07:32:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43c4a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4340b95a0 0x7fc43415ac80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43c4a4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc43415df00 0x7fc43415a5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43c4a4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415b240 0x7fc4340d3970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43c4a4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4340cec70 con 0x7fc43415b240 2024-03-04T07:32:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43c4a4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4340ce750 con 0x7fc43415df00 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43c4a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc4340cedf0 con 0x7fc4340b95a0 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43aa41640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415b240 0x7fc4340d3970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43aa41640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415b240 0x7fc4340d3970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52740/0 (socket says 172.21.15.64:52740) 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43aa41640 1 -- 172.21.15.64:0/2474032859 learned_addr learned my addr 172.21.15.64:0/2474032859 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc439a3f640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc43415df00 0x7fc43415a5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.569+0000 7fc43a240640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4340b95a0 0x7fc43415ac80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43aa41640 1 -- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4340b95a0 msgr2=0x7fc43415ac80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43aa41640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4340b95a0 0x7fc43415ac80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43aa41640 1 -- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc43415df00 msgr2=0x7fc43415a5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43aa41640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc43415df00 0x7fc43415a5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43aa41640 1 -- 172.21.15.64:0/2474032859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc43415e6c0 con 0x7fc43415b240 2024-03-04T07:32:05.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43a240640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4340b95a0 0x7fc43415ac80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43aa41640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415b240 0x7fc4340d3970 secure :-1 s=READY pgs=1279 cs=0 l=1 rev1=1 crypto rx=0x7fc42400c7d0 tx=0x7fc42400cc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc432ffd640 1 -- 172.21.15.64:0/2474032859 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc424018070 con 0x7fc43415b240 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc432ffd640 1 -- 172.21.15.64:0/2474032859 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc42400a640 con 0x7fc43415b240 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc43415e9a0 con 0x7fc43415b240 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc432ffd640 1 -- 172.21.15.64:0/2474032859 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc42400aa70 con 0x7fc43415b240 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.570+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc43415ef60 con 0x7fc43415b240 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.571+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc434067ff0 con 0x7fc43415b240 2024-03-04T07:32:05.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.571+0000 7fc432ffd640 1 -- 172.21.15.64:0/2474032859 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc42401b070 con 0x7fc43415b240 2024-03-04T07:32:05.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.572+0000 7fc432ffd640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc42006b4d0 0x7fc42006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.572+0000 7fc432ffd640 1 -- 172.21.15.64:0/2474032859 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fc424089390 con 0x7fc43415b240 2024-03-04T07:32:05.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.572+0000 7fc43a240640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc42006b4d0 0x7fc42006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.572+0000 7fc43a240640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc42006b4d0 0x7fc42006d990 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fc42c036000 tx=0x7fc42c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:05.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.576+0000 7fc432ffd640 1 -- 172.21.15.64:0/2474032859 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc424057fa0 con 0x7fc43415b240 2024-03-04T07:32:05.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.824+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7fc43406b780 con 0x7fc43415b240 2024-03-04T07:32:05.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.824+0000 7fc432ffd640 1 -- 172.21.15.64:0/2474032859 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v20) v1 ==== 80+0+5304 (secure 0 0 0) 0x7fc42405c040 con 0x7fc43415b240 2024-03-04T07:32:05.825 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:05.825 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:32:05.825 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:32:05.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:32:05.826 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:32:05.827 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.829 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:32:05.830 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.831 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.832 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow r", 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.833 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.834 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:32:05.835 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:32:05.836 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:32:05.837 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:32:05.837 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:32:05.837 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:32:05.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:05.837 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:32:05.837 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:32:05.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc42006b4d0 msgr2=0x7fc42006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc42006b4d0 0x7fc42006d990 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fc42c036000 tx=0x7fc42c03c040 comp rx=0 tx=0).stop 2024-03-04T07:32:05.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415b240 msgr2=0x7fc4340d3970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415b240 0x7fc4340d3970 secure :-1 s=READY pgs=1279 cs=0 l=1 rev1=1 crypto rx=0x7fc42400c7d0 tx=0x7fc42400cc90 comp rx=0 tx=0).stop 2024-03-04T07:32:05.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 shutdown_connections 2024-03-04T07:32:05.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc42006b4d0 0x7fc42006d990 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc43415b240 0x7fc4340d3970 unknown :-1 s=CLOSED pgs=1279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc43415df00 0x7fc43415a5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 --2- 172.21.15.64:0/2474032859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4340b95a0 0x7fc43415ac80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 >> 172.21.15.64:0/2474032859 conn(0x7fc434071960 msgr2=0x7fc4340cad50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:05.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 shutdown_connections 2024-03-04T07:32:05.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.827+0000 7fc43c4a4640 1 -- 172.21.15.64:0/2474032859 wait complete. 2024-03-04T07:32:05.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:32:05.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.941+0000 7fe3a47c9640 1 Processor -- start 2024-03-04T07:32:05.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.941+0000 7fe3a47c9640 1 -- start start 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a47c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe39c15e060 0x7fe39c0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a47c9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe39c06fe20 0x7fe39c0ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a47c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe39c0d0dd0 con 0x7fe39c0d0880 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a47c9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe39c0d10b0 con 0x7fe39c06fe20 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a47c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe39c0cc0e0 con 0x7fe39c15e060 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a1d64640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe39c06fe20 0x7fe39c0ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a1d64640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe39c06fe20 0x7fe39c0ce850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52746/0 (socket says 172.21.15.64:52746) 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a1d64640 1 -- 172.21.15.64:0/1240880990 learned_addr learned my addr 172.21.15.64:0/1240880990 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a2565640 1 --2- 172.21.15.64:0/1240880990 >> v2:172.21.15.66:3300/0 conn(0x7fe39c15e060 0x7fe39c0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a1d64640 1 -- 172.21.15.64:0/1240880990 >> v2:172.21.15.66:3300/0 conn(0x7fe39c15e060 msgr2=0x7fe39c0ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a1d64640 1 --2- 172.21.15.64:0/1240880990 >> v2:172.21.15.66:3300/0 conn(0x7fe39c15e060 0x7fe39c0ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a1d64640 1 -- 172.21.15.64:0/1240880990 >> v1:172.21.15.66:6789/0 conn(0x7fe39c0d0880 legacy=0x7fe39c06ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:05.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.942+0000 7fe3a1d64640 1 -- 172.21.15.64:0/1240880990 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe39c0cc3c0 con 0x7fe39c06fe20 2024-03-04T07:32:05.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.943+0000 7fe3a1d64640 1 --2- 172.21.15.64:0/1240880990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c06fe20 0x7fe39c0ce850 secure :-1 s=READY pgs=1280 cs=0 l=1 rev1=1 crypto rx=0x7fe39c0722f0 tx=0x7fe38c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f4e1bce3cced133b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:05.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.943+0000 7fe3a1563640 1 -- 172.21.15.64:0/1240880990 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe38c034070 con 0x7fe39c06fe20 2024-03-04T07:32:05.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.943+0000 7fe3a1563640 1 -- 172.21.15.64:0/1240880990 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe38c030860 con 0x7fe39c06fe20 2024-03-04T07:32:05.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.943+0000 7fe3a1563640 1 -- 172.21.15.64:0/1240880990 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe38c030c30 con 0x7fe39c06fe20 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 -- 172.21.15.64:0/1240880990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c06fe20 msgr2=0x7fe39c0ce850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/1240880990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c06fe20 0x7fe39c0ce850 secure :-1 s=READY pgs=1280 cs=0 l=1 rev1=1 crypto rx=0x7fe39c0722f0 tx=0x7fe38c02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 -- 172.21.15.64:0/1240880990 shutdown_connections 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/1240880990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c06fe20 0x7fe39c0ce850 unknown :-1 s=CLOSED pgs=1280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/1240880990 >> v2:172.21.15.66:3300/0 conn(0x7fe39c15e060 0x7fe39c0ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 -- 172.21.15.64:0/1240880990 >> 172.21.15.64:0/1240880990 conn(0x7fe39c0d1a90 msgr2=0x7fe39c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 -- 172.21.15.64:0/1240880990 shutdown_connections 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.944+0000 7fe3a47c9640 1 -- 172.21.15.64:0/1240880990 wait complete. 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.945+0000 7fe3a47c9640 1 Processor -- start 2024-03-04T07:32:05.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.945+0000 7fe3a47c9640 1 -- start start 2024-03-04T07:32:05.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe39c06fe20 0x7fe39c151ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe39c0d0880 0x7fe39c1f4880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c15e060 0x7fe39c15bad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe39c0cdbd0 con 0x7fe39c15e060 2024-03-04T07:32:05.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe39c15e440 con 0x7fe39c0d0880 2024-03-04T07:32:05.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe39c0cdd50 con 0x7fe39c06fe20 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c15e060 0x7fe39c15bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c15e060 0x7fe39c15bad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52752/0 (socket says 172.21.15.64:52752) 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 -- 172.21.15.64:0/3599541855 learned_addr learned my addr 172.21.15.64:0/3599541855 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2565640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe39c06fe20 0x7fe39c151ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a1d64640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe39c0d0880 0x7fe39c1f4880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 -- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe39c06fe20 msgr2=0x7fe39c151ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe39c06fe20 0x7fe39c151ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 -- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe39c0d0880 msgr2=0x7fe39c1f4880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe39c0d0880 0x7fe39c1f4880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 -- 172.21.15.64:0/3599541855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe39c0c6450 con 0x7fe39c15e060 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a2d66640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c15e060 0x7fe39c15bad0 secure :-1 s=READY pgs=1281 cs=0 l=1 rev1=1 crypto rx=0x7fe39400c7a0 tx=0x7fe39400cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe392ffd640 1 -- 172.21.15.64:0/3599541855 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe394003b70 con 0x7fe39c15e060 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe392ffd640 1 -- 172.21.15.64:0/3599541855 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe394003d30 con 0x7fe39c15e060 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe39c0c58d0 con 0x7fe39c15e060 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe392ffd640 1 -- 172.21.15.64:0/3599541855 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39400aa90 con 0x7fe39c15e060 2024-03-04T07:32:05.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.946+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe39c0c5e90 con 0x7fe39c15e060 2024-03-04T07:32:05.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.948+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe39c067ff0 con 0x7fe39c15e060 2024-03-04T07:32:05.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.948+0000 7fe392ffd640 1 -- 172.21.15.64:0/3599541855 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe39400ac30 con 0x7fe39c15e060 2024-03-04T07:32:05.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.948+0000 7fe392ffd640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe38006b4d0 0x7fe38006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:05.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.948+0000 7fe3a2565640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe38006b4d0 0x7fe38006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:05.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.949+0000 7fe392ffd640 1 -- 172.21.15.64:0/3599541855 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fe394089cb0 con 0x7fe39c15e060 2024-03-04T07:32:05.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.949+0000 7fe3a2565640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe38006b4d0 0x7fe38006d990 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7fe398006fd0 tx=0x7fe398008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:05.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:05.953+0000 7fe392ffd640 1 -- 172.21.15.64:0/3599541855 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe394058830 con 0x7fe39c15e060 2024-03-04T07:32:06.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.133+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe39c1f3720 con 0x7fe39c15e060 2024-03-04T07:32:06.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.134+0000 7fe392ffd640 1 -- 172.21.15.64:0/3599541855 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v101) v1 ==== 74+0+12845 (secure 0 0 0) 0x7fe39405c8d0 con 0x7fe39c15e060 2024-03-04T07:32:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:06.135 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":101,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:05.449821+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe38006b4d0 msgr2=0x7fe38006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe38006b4d0 0x7fe38006d990 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7fe398006fd0 tx=0x7fe398008040 comp rx=0 tx=0).stop 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c15e060 msgr2=0x7fe39c15bad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c15e060 0x7fe39c15bad0 secure :-1 s=READY pgs=1281 cs=0 l=1 rev1=1 crypto rx=0x7fe39400c7a0 tx=0x7fe39400cc60 comp rx=0 tx=0).stop 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 shutdown_connections 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe38006b4d0 0x7fe38006d990 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe39c15e060 0x7fe39c15bad0 unknown :-1 s=CLOSED pgs=1281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe39c0d0880 0x7fe39c1f4880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 --2- 172.21.15.64:0/3599541855 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe39c06fe20 0x7fe39c151ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 >> 172.21.15.64:0/3599541855 conn(0x7fe39c0d1a90 msgr2=0x7fe39c0b3df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:06.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 shutdown_connections 2024-03-04T07:32:06.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.136+0000 7fe3a47c9640 1 -- 172.21.15.64:0/3599541855 wait complete. 2024-03-04T07:32:06.148 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:32:06.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:32:06.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.252+0000 7f6ba1c84640 1 Processor -- start 2024-03-04T07:32:06.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.252+0000 7f6ba1c84640 1 -- start start 2024-03-04T07:32:06.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6ba1c84640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0717b0 0x7f6b9c071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6ba1c84640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b9c0b3060 0x7f6b9c0d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6ba1c84640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6b9c0b2b10 0x7f6b9c0b6e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6ba1c84640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b9c0d2e10 con 0x7f6b9c0b3060 2024-03-04T07:32:06.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6ba1c84640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6b9c15e4b0 con 0x7f6b9c0b2b10 2024-03-04T07:32:06.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6ba1c84640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b9c0b97e0 con 0x7f6b9c0717b0 2024-03-04T07:32:06.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b9c0b3060 0x7f6b9c0d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6b9c0b3060 0x7f6b9c0d32a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52754/0 (socket says 172.21.15.64:52754) 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 -- 172.21.15.64:0/3586425021 learned_addr learned my addr 172.21.15.64:0/3586425021 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9b7fe640 1 --2- 172.21.15.64:0/3586425021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0717b0 0x7f6b9c071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9bfff640 1 --2- 172.21.15.64:0/3586425021 >> v2:172.21.15.66:3300/0 conn(0x7f6b9c0b2b10 0x7f6b9c0b6e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 -- 172.21.15.64:0/3586425021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0717b0 msgr2=0x7f6b9c071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 --2- 172.21.15.64:0/3586425021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0717b0 0x7f6b9c071b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 -- 172.21.15.64:0/3586425021 >> v2:172.21.15.66:3300/0 conn(0x7f6b9c0b2b10 msgr2=0x7f6b9c0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 --2- 172.21.15.64:0/3586425021 >> v2:172.21.15.66:3300/0 conn(0x7f6b9c0b2b10 0x7f6b9c0b6e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.253+0000 7f6b9affd640 1 -- 172.21.15.64:0/3586425021 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b9c0b9a80 con 0x7f6b9c0b3060 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.254+0000 7f6b9bfff640 1 --2- 172.21.15.64:0/3586425021 >> v2:172.21.15.66:3300/0 conn(0x7f6b9c0b2b10 0x7f6b9c0b6e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.254+0000 7f6b9affd640 1 --2- 172.21.15.64:0/3586425021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0b3060 0x7f6b9c0d32a0 secure :-1 s=READY pgs=1282 cs=0 l=1 rev1=1 crypto rx=0x7f6b9c08c750 tx=0x7f6b9002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f0c61821f7a028f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.254+0000 7f6b9a7fc640 1 -- 172.21.15.64:0/3586425021 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b90035020 con 0x7f6b9c0b3060 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.254+0000 7f6b9a7fc640 1 -- 172.21.15.64:0/3586425021 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b90031870 con 0x7f6b9c0b3060 2024-03-04T07:32:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.254+0000 7f6b9a7fc640 1 -- 172.21.15.64:0/3586425021 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b90031c40 con 0x7f6b9c0b3060 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 -- 172.21.15.64:0/3586425021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0b3060 msgr2=0x7f6b9c0d32a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/3586425021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0b3060 0x7f6b9c0d32a0 secure :-1 s=READY pgs=1282 cs=0 l=1 rev1=1 crypto rx=0x7f6b9c08c750 tx=0x7f6b9002b910 comp rx=0 tx=0).stop 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 -- 172.21.15.64:0/3586425021 shutdown_connections 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/3586425021 >> v2:172.21.15.66:3300/0 conn(0x7f6b9c0b2b10 0x7f6b9c0b6e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/3586425021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0b3060 0x7f6b9c0d32a0 unknown :-1 s=CLOSED pgs=1282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/3586425021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0717b0 0x7f6b9c071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 -- 172.21.15.64:0/3586425021 >> 172.21.15.64:0/3586425021 conn(0x7f6b9c0d3d20 msgr2=0x7f6b9c0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 -- 172.21.15.64:0/3586425021 shutdown_connections 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.255+0000 7f6ba1c84640 1 -- 172.21.15.64:0/3586425021 wait complete. 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.256+0000 7f6ba1c84640 1 Processor -- start 2024-03-04T07:32:06.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.256+0000 7f6ba1c84640 1 -- start start 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6ba1c84640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0717b0 0x7f6b9c0cc820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6ba1c84640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b9c0b2b10 0x7f6b9c0c9430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6ba1c84640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0b3060 0x7f6b9c149ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6ba1c84640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b9c0b7390 con 0x7f6b9c0717b0 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6ba1c84640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b9c0d4bf0 con 0x7f6b9c0b2b10 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6ba1c84640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6b9c0b7510 con 0x7f6b9c0b3060 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0717b0 0x7f6b9c0cc820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0717b0 0x7f6b9c0cc820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52760/0 (socket says 172.21.15.64:52760) 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 -- 172.21.15.64:0/2645262955 learned_addr learned my addr 172.21.15.64:0/2645262955 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9bfff640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0b3060 0x7f6b9c149ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9affd640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b9c0b2b10 0x7f6b9c0c9430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 -- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0b3060 msgr2=0x7f6b9c149ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0b3060 0x7f6b9c149ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 -- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b9c0b2b10 msgr2=0x7f6b9c0c9430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b9c0b2b10 0x7f6b9c0c9430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9b7fe640 1 -- 172.21.15.64:0/2645262955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b9c0cce30 con 0x7f6b9c0717b0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9bfff640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0b3060 0x7f6b9c149ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.257+0000 7f6b9affd640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b9c0b2b10 0x7f6b9c0c9430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.258+0000 7f6b9b7fe640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0717b0 0x7f6b9c0cc820 secure :-1 s=READY pgs=1283 cs=0 l=1 rev1=1 crypto rx=0x7f6b8c00c7f0 tx=0x7f6b8c00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.258+0000 7f6b83fff640 1 -- 172.21.15.64:0/2645262955 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b8c015070 con 0x7f6b9c0717b0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.258+0000 7f6b83fff640 1 -- 172.21.15.64:0/2645262955 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b8c004220 con 0x7f6b9c0717b0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.258+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b9c0cd110 con 0x7f6b9c0717b0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.258+0000 7f6b83fff640 1 -- 172.21.15.64:0/2645262955 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b8c007620 con 0x7f6b9c0717b0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.258+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b9c0cd6d0 con 0x7f6b9c0717b0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.259+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b9c06ce40 con 0x7f6b9c0717b0 2024-03-04T07:32:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.259+0000 7f6b83fff640 1 -- 172.21.15.64:0/2645262955 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b8c0079d0 con 0x7f6b9c0717b0 2024-03-04T07:32:06.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.259+0000 7f6b83fff640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.260+0000 7f6b9affd640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.260+0000 7f6b83fff640 1 -- 172.21.15.64:0/2645262955 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f6b8c089fb0 con 0x7f6b9c0717b0 2024-03-04T07:32:06.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.260+0000 7f6b9affd640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7f6b90036000 tx=0x7f6b90008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.264+0000 7f6b83fff640 1 -- 172.21.15.64:0/2645262955 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6b8c0590f0 con 0x7f6b9c0717b0 2024-03-04T07:32:06.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.446+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f6b9c04f410 con 0x7f6b9c0717b0 2024-03-04T07:32:06.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.446+0000 7f6b83fff640 1 -- 172.21.15.64:0/2645262955 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v101) v1 ==== 93+0+10 (secure 0 0 0) 0x7f6b8c0589f0 con 0x7f6b9c0717b0 2024-03-04T07:32:06.447 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:32:06.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 msgr2=0x7f6b8806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7f6b90036000 tx=0x7f6b90008030 comp rx=0 tx=0).stop 2024-03-04T07:32:06.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0717b0 msgr2=0x7f6b9c0cc820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0717b0 0x7f6b9c0cc820 secure :-1 s=READY pgs=1283 cs=0 l=1 rev1=1 crypto rx=0x7f6b8c00c7f0 tx=0x7f6b8c00ccb0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 shutdown_connections 2024-03-04T07:32:06.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 unknown :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6b9c0b3060 0x7f6b9c149ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6b9c0b2b10 0x7f6b9c0c9430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 --2- 172.21.15.64:0/2645262955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6b9c0717b0 0x7f6b9c0cc820 unknown :-1 s=CLOSED pgs=1283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 >> 172.21.15.64:0/2645262955 conn(0x7f6b9c0d3d20 msgr2=0x7f6b9c071b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:06.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 shutdown_connections 2024-03-04T07:32:06.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.448+0000 7f6ba1c84640 1 -- 172.21.15.64:0/2645262955 wait complete. 2024-03-04T07:32:06.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:32:06.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8168f9a640 1 Processor -- start 2024-03-04T07:32:06.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8168f9a640 1 -- start start 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8168f9a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f816415df00 0x7f81640d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8168f9a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81640b9a30 0x7f81640d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8168f9a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81640cea30 con 0x7f81640b95a0 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8168f9a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f81640d2af0 con 0x7f81640b9a30 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8168f9a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f81640cf3c0 con 0x7f816415df00 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8162d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81640b9a30 0x7f81640d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8161d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f81640b95a0 0x7f81640b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44664/0 (socket says 172.21.15.64:44664) 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8162d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81640b9a30 0x7f81640d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52770/0 (socket says 172.21.15.64:52770) 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.561+0000 7f8161d9b640 1 -- 172.21.15.64:0/1667381603 learned_addr learned my addr 172.21.15.64:0/1667381603 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.562+0000 7f816259c640 1 --2- 172.21.15.64:0/1667381603 >> v2:172.21.15.66:3300/0 conn(0x7f816415df00 0x7f81640d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.562+0000 7f8162d9d640 1 -- 172.21.15.64:0/1667381603 >> v2:172.21.15.66:3300/0 conn(0x7f816415df00 msgr2=0x7f81640d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.562+0000 7f8162d9d640 1 --2- 172.21.15.64:0/1667381603 >> v2:172.21.15.66:3300/0 conn(0x7f816415df00 0x7f81640d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.562+0000 7f8162d9d640 1 -- 172.21.15.64:0/1667381603 >> v1:172.21.15.64:6789/0 conn(0x7f81640b95a0 legacy=0x7f81640b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.562+0000 7f816159a640 1 -- 172.21.15.64:0/1667381603 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3636982728 0 0) 0x7f81640cea30 con 0x7f81640b95a0 2024-03-04T07:32:06.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.562+0000 7f8162d9d640 1 -- 172.21.15.64:0/1667381603 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81640cf0e0 con 0x7f81640b9a30 2024-03-04T07:32:06.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.562+0000 7f8162d9d640 1 --2- 172.21.15.64:0/1667381603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81640b9a30 0x7f81640d3220 secure :-1 s=READY pgs=1285 cs=0 l=1 rev1=1 crypto rx=0x7f81640d11b0 tx=0x7f815402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7eb8df5cbbbef21d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f816159a640 1 -- 172.21.15.64:0/1667381603 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8154039070 con 0x7f81640b9a30 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f816159a640 1 -- 172.21.15.64:0/1667381603 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f815402cc80 con 0x7f81640b9a30 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f816159a640 1 -- 172.21.15.64:0/1667381603 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8154034660 con 0x7f81640b9a30 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 -- 172.21.15.64:0/1667381603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81640b9a30 msgr2=0x7f81640d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 --2- 172.21.15.64:0/1667381603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81640b9a30 0x7f81640d3220 secure :-1 s=READY pgs=1285 cs=0 l=1 rev1=1 crypto rx=0x7f81640d11b0 tx=0x7f815402bed0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 -- 172.21.15.64:0/1667381603 shutdown_connections 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 --2- 172.21.15.64:0/1667381603 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81640b9a30 0x7f81640d3220 unknown :-1 s=CLOSED pgs=1285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 --2- 172.21.15.64:0/1667381603 >> v2:172.21.15.66:3300/0 conn(0x7f816415df00 0x7f81640d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 -- 172.21.15.64:0/1667381603 >> 172.21.15.64:0/1667381603 conn(0x7f8164071960 msgr2=0x7f8164071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 -- 172.21.15.64:0/1667381603 shutdown_connections 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.563+0000 7f8168f9a640 1 -- 172.21.15.64:0/1667381603 wait complete. 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 Processor -- start 2024-03-04T07:32:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 -- start start 2024-03-04T07:32:06.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81640b95a0 0x7f816414e0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816415df00 0x7f81640b4570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81640b4ab0 0x7f816415e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81640cec70 con 0x7f816415df00 2024-03-04T07:32:06.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81640ce750 con 0x7f81640b4ab0 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8168f9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81640cedf0 con 0x7f81640b95a0 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8161d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816415df00 0x7f81640b4570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8161d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816415df00 0x7f81640b4570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52772/0 (socket says 172.21.15.64:52772) 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8161d9b640 1 -- 172.21.15.64:0/674228209 learned_addr learned my addr 172.21.15.64:0/674228209 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.565+0000 7f8162d9d640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81640b4ab0 0x7f816415e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f816259c640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81640b95a0 0x7f816414e0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8161d9b640 1 -- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81640b95a0 msgr2=0x7f816414e0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8161d9b640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81640b95a0 0x7f816414e0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8161d9b640 1 -- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81640b4ab0 msgr2=0x7f816415e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8161d9b640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81640b4ab0 0x7f816415e2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8161d9b640 1 -- 172.21.15.64:0/674228209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f816414e600 con 0x7f816415df00 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8161d9b640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816415df00 0x7f81640b4570 secure :-1 s=READY pgs=1286 cs=0 l=1 rev1=1 crypto rx=0x7f814c00c4a0 tx=0x7f814c00c960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f815affd640 1 -- 172.21.15.64:0/674228209 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f814c018070 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f815affd640 1 -- 172.21.15.64:0/674228209 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f814c0074e0 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f816415eef0 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f815affd640 1 -- 172.21.15.64:0/674228209 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f814c004a20 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.566+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81641f3d30 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.567+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81640b4ec0 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.568+0000 7f815affd640 1 -- 172.21.15.64:0/674228209 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f814c01b070 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.568+0000 7f815affd640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814406b4d0 0x7f814406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.568+0000 7f816259c640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814406b4d0 0x7f814406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.568+0000 7f815affd640 1 -- 172.21.15.64:0/674228209 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f814c089050 con 0x7f816415df00 2024-03-04T07:32:06.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.568+0000 7f816259c640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814406b4d0 0x7f814406d990 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f815c009aa0 tx=0x7f815c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.572+0000 7f815affd640 1 -- 172.21.15.64:0/674228209 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f81640b4ec0 con 0x7f816415df00 2024-03-04T07:32:06.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.755+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f8164067ff0 con 0x7f816415df00 2024-03-04T07:32:06.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.756+0000 7f815affd640 1 -- 172.21.15.64:0/674228209 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v101) v1 ==== 84+0+65 (secure 0 0 0) 0x7f816414e600 con 0x7f816415df00 2024-03-04T07:32:06.756 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:06.756 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:32:06.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:06.756 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:32:06.756 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:32:06.756 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:06.756 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:32:06.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814406b4d0 msgr2=0x7f814406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814406b4d0 0x7f814406d990 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f815c009aa0 tx=0x7f815c009190 comp rx=0 tx=0).stop 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816415df00 msgr2=0x7f81640b4570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816415df00 0x7f81640b4570 secure :-1 s=READY pgs=1286 cs=0 l=1 rev1=1 crypto rx=0x7f814c00c4a0 tx=0x7f814c00c960 comp rx=0 tx=0).stop 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 shutdown_connections 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814406b4d0 0x7f814406d990 unknown :-1 s=CLOSED pgs=580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81640b4ab0 0x7f816415e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816415df00 0x7f81640b4570 unknown :-1 s=CLOSED pgs=1286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 --2- 172.21.15.64:0/674228209 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81640b95a0 0x7f816414e0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.757+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 >> 172.21.15.64:0/674228209 conn(0x7f8164071960 msgr2=0x7f81640d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.758+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 shutdown_connections 2024-03-04T07:32:06.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.758+0000 7f8168f9a640 1 -- 172.21.15.64:0/674228209 wait complete. 2024-03-04T07:32:06.768 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:32:06.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:32:06.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.874+0000 7f2b9cd72640 1 Processor -- start 2024-03-04T07:32:06.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.875+0000 7f2b9cd72640 1 -- start start 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.875+0000 7f2b9cd72640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2b980b7420 0x7f2b980d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.875+0000 7f2b9cd72640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2b980d3740 con 0x7f2b980ceea0 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.875+0000 7f2b9cd72640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2b980d3120 con 0x7f2b9815dd90 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.875+0000 7f2b9cd72640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2b980d3400 con 0x7f2b980b7420 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.875+0000 7f2b8ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2b980ceea0 0x7f2b980b6d10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44668/0 (socket says 172.21.15.64:44668) 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.875+0000 7f2b8ffff640 1 -- 172.21.15.64:0/1374341437 learned_addr learned my addr 172.21.15.64:0/1374341437 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 624678764 0 0) 0x7f2b980d3740 con 0x7f2b980ceea0 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2b84003520 con 0x7f2b980ceea0 2024-03-04T07:32:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b96d9d640 1 --2- 172.21.15.64:0/1374341437 >> v2:172.21.15.66:3300/0 conn(0x7f2b980b7420 0x7f2b980d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 437872690 0 0) 0x7f2b84003520 con 0x7f2b980ceea0 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 >> v2:172.21.15.66:3300/0 conn(0x7f2b980b7420 msgr2=0x7f2b980d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 --2- 172.21.15.64:0/1374341437 >> v2:172.21.15.66:3300/0 conn(0x7f2b980b7420 0x7f2b980d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 >> v1:172.21.15.66:6789/0 conn(0x7f2b9815dd90 legacy=0x7f2b980d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b980d39e0 con 0x7f2b980ceea0 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2b80001690 con 0x7f2b980ceea0 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b96d9d640 1 --2- 172.21.15.64:0/1374341437 >> v2:172.21.15.66:3300/0 conn(0x7f2b980b7420 0x7f2b980d29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 >> v1:172.21.15.64:6789/0 conn(0x7f2b980ceea0 legacy=0x7f2b980b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b840039b0 0x7f2b84003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b84004800 0x7f2b84004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b840051c0 0x7f2b8400b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b980d3400 con 0x7f2b84004800 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b8400c110 con 0x7f2b840039b0 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2b8400c310 con 0x7f2b840051c0 2024-03-04T07:32:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b8ffff640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b84004800 0x7f2b84004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b9659c640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b840039b0 0x7f2b84003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b96d9d640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b840051c0 0x7f2b8400b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b8ffff640 1 -- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b840051c0 msgr2=0x7f2b8400b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b8ffff640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b840051c0 0x7f2b8400b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b8ffff640 1 -- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b840039b0 msgr2=0x7f2b84003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b8ffff640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b840039b0 0x7f2b84003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b8ffff640 1 -- 172.21.15.64:0/1374341437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b84003520 con 0x7f2b84004800 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.876+0000 7f2b96d9d640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b840051c0 0x7f2b8400b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.877+0000 7f2b96d9d640 1 -- 172.21.15.64:0/1374341437 reap_dead start 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.877+0000 7f2b8ffff640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b84004800 0x7f2b84004c80 secure :-1 s=READY pgs=1288 cs=0 l=1 rev1=1 crypto rx=0x7f2b8000c210 tx=0x7f2b84005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c0c1d532e823ce4e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.877+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b80007d10 con 0x7f2b84004800 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.877+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2b80030070 con 0x7f2b84004800 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.877+0000 7f2b95d9b640 1 -- 172.21.15.64:0/1374341437 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b80005d70 con 0x7f2b84004800 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.878+0000 7f2b9cd72640 1 -- 172.21.15.64:0/1374341437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b84004800 msgr2=0x7f2b84004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.878+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b84004800 0x7f2b84004c80 secure :-1 s=READY pgs=1288 cs=0 l=1 rev1=1 crypto rx=0x7f2b8000c210 tx=0x7f2b84005830 comp rx=0 tx=0).stop 2024-03-04T07:32:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.878+0000 7f2b9cd72640 1 -- 172.21.15.64:0/1374341437 shutdown_connections 2024-03-04T07:32:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.878+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/1374341437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b84004800 0x7f2b84004c80 unknown :-1 s=CLOSED pgs=1288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.878+0000 7f2b9cd72640 1 -- 172.21.15.64:0/1374341437 >> 172.21.15.64:0/1374341437 conn(0x7f2b98071960 msgr2=0x7f2b98071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.878+0000 7f2b9cd72640 1 -- 172.21.15.64:0/1374341437 shutdown_connections 2024-03-04T07:32:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.878+0000 7f2b9cd72640 1 -- 172.21.15.64:0/1374341437 wait complete. 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.879+0000 7f2b9cd72640 1 Processor -- start 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.879+0000 7f2b9cd72640 1 -- start start 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b9cd72640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b84004800 0x7f2b9815f900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b9cd72640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b98160f80 0x7f2b98149b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b9cd72640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b980ccb30 0x7f2b980d4030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b9cd72640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b980d29d0 con 0x7f2b980ccb30 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b9cd72640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b9815e170 con 0x7f2b84004800 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b9cd72640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2b980d2b50 con 0x7f2b98160f80 2024-03-04T07:32:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b980ccb30 0x7f2b980d4030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b980ccb30 0x7f2b980d4030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52778/0 (socket says 172.21.15.64:52778) 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 -- 172.21.15.64:0/695545116 learned_addr learned my addr 172.21.15.64:0/695545116 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b9659c640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b84004800 0x7f2b9815f900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 -- 172.21.15.64:0/695545116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b98160f80 msgr2=0x7f2b98149b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b98160f80 0x7f2b98149b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 -- 172.21.15.64:0/695545116 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b84004800 msgr2=0x7f2b9815f900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b84004800 0x7f2b9815f900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 -- 172.21.15.64:0/695545116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b9815fee0 con 0x7f2b980ccb30 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.880+0000 7f2b96d9d640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b980ccb30 0x7f2b980d4030 secure :-1 s=READY pgs=1289 cs=0 l=1 rev1=1 crypto rx=0x7f2b980b9df0 tx=0x7f2b90004840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.881+0000 7f2b8effd640 1 -- 172.21.15.64:0/695545116 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b90007d30 con 0x7f2b980ccb30 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.881+0000 7f2b8effd640 1 -- 172.21.15.64:0/695545116 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2b90019070 con 0x7f2b980ccb30 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.881+0000 7f2b8effd640 1 -- 172.21.15.64:0/695545116 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b90014650 con 0x7f2b980ccb30 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.881+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2b981601c0 con 0x7f2b980ccb30 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.881+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2b98160780 con 0x7f2b980ccb30 2024-03-04T07:32:06.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.882+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2b98072a30 con 0x7f2b980ccb30 2024-03-04T07:32:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.882+0000 7f2b8effd640 1 -- 172.21.15.64:0/695545116 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2b900147f0 con 0x7f2b980ccb30 2024-03-04T07:32:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.883+0000 7f2b8effd640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b7806b500 0x7f2b7806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.883+0000 7f2b9659c640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b7806b500 0x7f2b7806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.883+0000 7f2b8effd640 1 -- 172.21.15.64:0/695545116 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f2b9008a680 con 0x7f2b980ccb30 2024-03-04T07:32:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.883+0000 7f2b9659c640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b7806b500 0x7f2b7806d9c0 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f2b88000c00 tx=0x7f2b88009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:06.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:06.887+0000 7f2b8effd640 1 -- 172.21.15.64:0/695545116 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2b90023020 con 0x7f2b980ccb30 2024-03-04T07:32:07.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.067+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f2b98072020 con 0x7f2b980ccb30 2024-03-04T07:32:07.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.444+0000 7f2b8effd640 1 -- 172.21.15.64:0/695545116 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v102) v1 ==== 138+0+0 (secure 0 0 0) 0x7f2b900598e0 con 0x7f2b980ccb30 2024-03-04T07:32:07.444 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:32:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b7806b500 msgr2=0x7f2b7806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b7806b500 0x7f2b7806d9c0 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f2b88000c00 tx=0x7f2b88009040 comp rx=0 tx=0).stop 2024-03-04T07:32:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b980ccb30 msgr2=0x7f2b980d4030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b980ccb30 0x7f2b980d4030 secure :-1 s=READY pgs=1289 cs=0 l=1 rev1=1 crypto rx=0x7f2b980b9df0 tx=0x7f2b90004840 comp rx=0 tx=0).stop 2024-03-04T07:32:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 shutdown_connections 2024-03-04T07:32:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b7806b500 0x7f2b7806d9c0 unknown :-1 s=CLOSED pgs=581 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b980ccb30 0x7f2b980d4030 unknown :-1 s=CLOSED pgs=1289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b98160f80 0x7f2b98149b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 --2- 172.21.15.64:0/695545116 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b84004800 0x7f2b9815f900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 >> 172.21.15.64:0/695545116 conn(0x7f2b98071960 msgr2=0x7f2b980d1f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:07.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 shutdown_connections 2024-03-04T07:32:07.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.446+0000 7f2b9cd72640 1 -- 172.21.15.64:0/695545116 wait complete. 2024-03-04T07:32:07.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:32:07.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.572+0000 7f805021c640 1 Processor -- start 2024-03-04T07:32:07.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.572+0000 7f805021c640 1 -- start start 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.573+0000 7f805021c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dbd0 0x7f804815ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.573+0000 7f805021c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f80480b5840 0x7f80480b39b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.573+0000 7f805021c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f804815f3b0 con 0x7f804815dfb0 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.573+0000 7f805021c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f804806f070 con 0x7f80480b5840 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.573+0000 7f805021c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8048071940 con 0x7f804815dbd0 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.573+0000 7f804d7b7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f80480b5840 0x7f80480b39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.573+0000 7f804dfb8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dbd0 0x7f804815ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804dfb8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dbd0 0x7f804815ee70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:34602/0 (socket says 172.21.15.64:34602) 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804e7b9640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f804815dfb0 0x7f8048161a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:56834/0 (socket says 172.21.15.64:56834) 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804d7b7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f80480b5840 0x7f80480b39b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:51382/0 (socket says 172.21.15.64:51382) 2024-03-04T07:32:07.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804dfb8640 1 -- 172.21.15.64:0/3523376818 learned_addr learned my addr 172.21.15.64:0/3523376818 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:07.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804dfb8640 1 -- 172.21.15.64:0/3523376818 >> v2:172.21.15.66:3300/0 conn(0x7f80480b5840 msgr2=0x7f80480b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:07.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804dfb8640 1 --2- 172.21.15.64:0/3523376818 >> v2:172.21.15.66:3300/0 conn(0x7f80480b5840 0x7f80480b39b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804dfb8640 1 -- 172.21.15.64:0/3523376818 >> v1:172.21.15.66:6789/0 conn(0x7f804815dfb0 legacy=0x7f8048161a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:07.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804dfb8640 1 -- 172.21.15.64:0/3523376818 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80480b5db0 con 0x7f804815dbd0 2024-03-04T07:32:07.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.574+0000 7f804d7b7640 1 --2- 172.21.15.64:0/3523376818 >> v2:172.21.15.66:3300/0 conn(0x7f80480b5840 0x7f80480b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:07.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f804dfb8640 1 --2- 172.21.15.64:0/3523376818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dbd0 0x7f804815ee70 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f8040009870 tx=0x7f804002b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7588b1beec7d348 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:07.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f804cfb6640 1 -- 172.21.15.64:0/3523376818 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8040034070 con 0x7f804815dbd0 2024-03-04T07:32:07.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f804cfb6640 1 -- 172.21.15.64:0/3523376818 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8040030840 con 0x7f804815dbd0 2024-03-04T07:32:07.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f804cfb6640 1 -- 172.21.15.64:0/3523376818 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8040030c10 con 0x7f804815dbd0 2024-03-04T07:32:07.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f805021c640 1 -- 172.21.15.64:0/3523376818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dbd0 msgr2=0x7f804815ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:07.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f805021c640 1 --2- 172.21.15.64:0/3523376818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dbd0 0x7f804815ee70 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f8040009870 tx=0x7f804002b980 comp rx=0 tx=0).stop 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f805021c640 1 -- 172.21.15.64:0/3523376818 shutdown_connections 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f805021c640 1 --2- 172.21.15.64:0/3523376818 >> v2:172.21.15.66:3300/0 conn(0x7f80480b5840 0x7f80480b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f805021c640 1 --2- 172.21.15.64:0/3523376818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dbd0 0x7f804815ee70 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.575+0000 7f805021c640 1 -- 172.21.15.64:0/3523376818 >> 172.21.15.64:0/3523376818 conn(0x7f80480b26b0 msgr2=0x7f80480b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.576+0000 7f805021c640 1 -- 172.21.15.64:0/3523376818 shutdown_connections 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.576+0000 7f805021c640 1 -- 172.21.15.64:0/3523376818 wait complete. 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 Processor -- start 2024-03-04T07:32:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 -- start start 2024-03-04T07:32:07.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80480b5840 0x7f804815b4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:07.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f804815dbd0 0x7f80480ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:07.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dfb0 0x7f80480ae450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:07.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80480721c0 con 0x7f804815dbd0 2024-03-04T07:32:07.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8048071ee0 con 0x7f80480b5840 2024-03-04T07:32:07.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f805021c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80480726e0 con 0x7f804815dfb0 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.577+0000 7f804d7b7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f804815dbd0 0x7f80480ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80480b5840 0x7f804815b4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804d7b7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f804815dbd0 0x7f80480ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52786/0 (socket says 172.21.15.64:52786) 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80480b5840 0x7f804815b4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:51384/0 (socket says 172.21.15.64:51384) 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804d7b7640 1 -- 172.21.15.64:0/3248251519 learned_addr learned my addr 172.21.15.64:0/3248251519 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804e7b9640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dfb0 0x7f80480ae450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 -- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dfb0 msgr2=0x7f80480ae450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dfb0 0x7f80480ae450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 -- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f804815dbd0 msgr2=0x7f80480ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f804815dbd0 0x7f80480ad8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 -- 172.21.15.64:0/3248251519 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f804815bad0 con 0x7f80480b5840 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804d7b7640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f804815dbd0 0x7f80480ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:07.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f804dfb8640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80480b5840 0x7f804815b4f0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f8040033000 tx=0x7f8040004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f803e7fc640 1 -- 172.21.15.64:0/3248251519 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8040048070 con 0x7f80480b5840 2024-03-04T07:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f803e7fc640 1 -- 172.21.15.64:0/3248251519 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8040030640 con 0x7f80480b5840 2024-03-04T07:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f804815bdb0 con 0x7f80480b5840 2024-03-04T07:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f803e7fc640 1 -- 172.21.15.64:0/3248251519 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8040034040 con 0x7f80480b5840 2024-03-04T07:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.578+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f804815c370 con 0x7f80480b5840 2024-03-04T07:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.579+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8048073380 con 0x7f80480b5840 2024-03-04T07:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.580+0000 7f803e7fc640 1 -- 172.21.15.64:0/3248251519 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8040037070 con 0x7f80480b5840 2024-03-04T07:32:07.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.580+0000 7f803e7fc640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f802c06b4d0 0x7f802c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:07.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.580+0000 7f803e7fc640 1 -- 172.21.15.64:0/3248251519 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 5227+0+0 (secure 0 0 0) 0x7f80400abda0 con 0x7f80480b5840 2024-03-04T07:32:07.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.580+0000 7f804d7b7640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f802c06b4d0 0x7f802c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:07.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.581+0000 7f804d7b7640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f802c06b4d0 0x7f802c06d990 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f804815aa10 tx=0x7f8038009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:07.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.585+0000 7f803e7fc640 1 -- 172.21.15.64:0/3248251519 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f804007a960 con 0x7f80480b5840 2024-03-04T07:32:07.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:07.765+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f804815a440 con 0x7f80480b5840 2024-03-04T07:32:08.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.456+0000 7f803e7fc640 1 -- 172.21.15.64:0/3248251519 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v103) v1 ==== 170+0+0 (secure 0 0 0) 0x7f804007ea00 con 0x7f80480b5840 2024-03-04T07:32:08.458 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f802c06b4d0 msgr2=0x7f802c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f802c06b4d0 0x7f802c06d990 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f804815aa10 tx=0x7f8038009190 comp rx=0 tx=0).stop 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80480b5840 msgr2=0x7f804815b4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80480b5840 0x7f804815b4f0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f8040033000 tx=0x7f8040004730 comp rx=0 tx=0).stop 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 shutdown_connections 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f802c06b4d0 0x7f802c06d990 secure :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f804815aa10 tx=0x7f8038009190 comp rx=0 tx=0).stop 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f804815dfb0 0x7f80480ae450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f804815dbd0 0x7f80480ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 --2- 172.21.15.64:0/3248251519 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80480b5840 0x7f804815b4f0 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 >> 172.21.15.64:0/3248251519 conn(0x7f80480b26b0 msgr2=0x7f804815ac40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 shutdown_connections 2024-03-04T07:32:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.471+0000 7f805021c640 1 -- 172.21.15.64:0/3248251519 wait complete. 2024-03-04T07:32:08.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:32:08.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.592+0000 7f611a302640 1 Processor -- start 2024-03-04T07:32:08.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.592+0000 7f611a302640 1 -- start start 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611a302640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6114162360 0x7f611415f250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611a302640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6114141550 con 0x7f61140d2000 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611a302640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6114162860 con 0x7f6114141170 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611a302640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f61140d1320 con 0x7f6114162360 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611889f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6114162360 0x7f611415f250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611889f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6114162360 0x7f611415f250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52800/0 (socket says 172.21.15.64:52800) 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611889f640 1 -- 172.21.15.64:0/662095797 learned_addr learned my addr 172.21.15.64:0/662095797 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f61137fe640 1 --1- 172.21.15.64:0/662095797 >> v1:172.21.15.64:6789/0 conn(0x7f61140d2000 0x7f6114161c50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44684/0 (socket says 172.21.15.64:44684) 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611889f640 1 -- 172.21.15.64:0/662095797 >> v1:172.21.15.66:6789/0 conn(0x7f6114141170 legacy=0x7f61140d18f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:08.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611889f640 1 -- 172.21.15.64:0/662095797 >> v1:172.21.15.64:6789/0 conn(0x7f61140d2000 legacy=0x7f6114161c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:08.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f611889f640 1 -- 172.21.15.64:0/662095797 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6114162b40 con 0x7f6114162360 2024-03-04T07:32:08.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.593+0000 7f6112ffd640 1 -- 172.21.15.64:0/662095797 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 516660366 0 0) 0x7f6114141550 con 0x7f61140d2000 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.594+0000 7f611889f640 1 --2- 172.21.15.64:0/662095797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6114162360 0x7f611415f250 secure :-1 s=READY pgs=1291 cs=0 l=1 rev1=1 crypto rx=0x7f610c009e50 tx=0x7f610c02c010 comp rx=0 tx=0).ready entity=mon.0 client_cookie=97f9d39cc582e9d1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.594+0000 7f6112ffd640 1 -- 172.21.15.64:0/662095797 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f610c02cbb0 con 0x7f6114162360 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.594+0000 7f6112ffd640 1 -- 172.21.15.64:0/662095797 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f610c02cd50 con 0x7f6114162360 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.594+0000 7f6112ffd640 1 -- 172.21.15.64:0/662095797 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f610c034610 con 0x7f6114162360 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.595+0000 7f611a302640 1 -- 172.21.15.64:0/662095797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6114162360 msgr2=0x7f611415f250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.595+0000 7f611a302640 1 --2- 172.21.15.64:0/662095797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6114162360 0x7f611415f250 secure :-1 s=READY pgs=1291 cs=0 l=1 rev1=1 crypto rx=0x7f610c009e50 tx=0x7f610c02c010 comp rx=0 tx=0).stop 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.595+0000 7f611a302640 1 -- 172.21.15.64:0/662095797 shutdown_connections 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.595+0000 7f611a302640 1 --2- 172.21.15.64:0/662095797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6114162360 0x7f611415f250 unknown :-1 s=CLOSED pgs=1291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.595+0000 7f611a302640 1 -- 172.21.15.64:0/662095797 >> 172.21.15.64:0/662095797 conn(0x7f611406eb00 msgr2=0x7f611406ef10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:08.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.595+0000 7f611a302640 1 -- 172.21.15.64:0/662095797 shutdown_connections 2024-03-04T07:32:08.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.595+0000 7f611a302640 1 -- 172.21.15.64:0/662095797 wait complete. 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.596+0000 7f611a302640 1 Processor -- start 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.596+0000 7f611a302640 1 -- start start 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f611a302640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61140d2000 0x7f611415ba50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f611a302640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6114141170 0x7f61140cff00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f611a302640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6114162360 0x7f61140d0440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f611a302640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f611415ea50 con 0x7f61140d2000 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f611a302640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f611415edc0 con 0x7f6114141170 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f611a302640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f61140702d0 con 0x7f6114162360 2024-03-04T07:32:08.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61140d2000 0x7f611415ba50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61140d2000 0x7f611415ba50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52804/0 (socket says 172.21.15.64:52804) 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 -- 172.21.15.64:0/559541894 learned_addr learned my addr 172.21.15.64:0/559541894 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f61137fe640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6114141170 0x7f61140cff00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f611889f640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6114162360 0x7f61140d0440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 -- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6114162360 msgr2=0x7f61140d0440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6114162360 0x7f61140d0440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 -- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6114141170 msgr2=0x7f61140cff00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6114141170 0x7f61140cff00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 -- 172.21.15.64:0/559541894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61141f4630 con 0x7f61140d2000 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f61137fe640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6114141170 0x7f61140cff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.597+0000 7f6113fff640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61140d2000 0x7f611415ba50 secure :-1 s=READY pgs=1292 cs=0 l=1 rev1=1 crypto rx=0x7f610400c4d0 tx=0x7f610400c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.598+0000 7f6110ff9640 1 -- 172.21.15.64:0/559541894 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61040052d0 con 0x7f61140d2000 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.598+0000 7f6110ff9640 1 -- 172.21.15.64:0/559541894 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6104005470 con 0x7f61140d2000 2024-03-04T07:32:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.598+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61141f4910 con 0x7f61140d2000 2024-03-04T07:32:08.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.598+0000 7f6110ff9640 1 -- 172.21.15.64:0/559541894 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6104012850 con 0x7f61140d2000 2024-03-04T07:32:08.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.598+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61141f4300 con 0x7f61140d2000 2024-03-04T07:32:08.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.599+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f61140700c0 con 0x7f61140d2000 2024-03-04T07:32:08.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.599+0000 7f6110ff9640 1 -- 172.21.15.64:0/559541894 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f61040129f0 con 0x7f61140d2000 2024-03-04T07:32:08.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.599+0000 7f6110ff9640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60f406b4d0 0x7f60f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:08.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.600+0000 7f6110ff9640 1 -- 172.21.15.64:0/559541894 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(103..103 src has 1..103) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f6104089d90 con 0x7f61140d2000 2024-03-04T07:32:08.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.600+0000 7f61137fe640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60f406b4d0 0x7f60f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:08.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.600+0000 7f61137fe640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60f406b4d0 0x7f60f406d990 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f611413e260 tx=0x7f60fc007c50 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:08.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.604+0000 7f6110ff9640 1 -- 172.21.15.64:0/559541894 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6104058c70 con 0x7f61140d2000 2024-03-04T07:32:08.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:08.824+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f61140b6ac0 con 0x7f61140d2000 2024-03-04T07:32:09.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.469+0000 7f6110ff9640 1 -- 172.21.15.64:0/559541894 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 16 and data pool 17 v64) v1 ==== 174+0+0 (secure 0 0 0) 0x7f6104058570 con 0x7f61140d2000 2024-03-04T07:32:09.471 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 16 and data pool 17 2024-03-04T07:32:09.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.475+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60f406b4d0 msgr2=0x7f60f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.475+0000 7f611a302640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60f406b4d0 0x7f60f406d990 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f611413e260 tx=0x7f60fc007c50 comp rx=0 tx=0).stop 2024-03-04T07:32:09.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.475+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61140d2000 msgr2=0x7f611415ba50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.475+0000 7f611a302640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61140d2000 0x7f611415ba50 secure :-1 s=READY pgs=1292 cs=0 l=1 rev1=1 crypto rx=0x7f610400c4d0 tx=0x7f610400c990 comp rx=0 tx=0).stop 2024-03-04T07:32:09.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.476+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 shutdown_connections 2024-03-04T07:32:09.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.476+0000 7f611a302640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f60f406b4d0 0x7f60f406d990 secure :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f611413e260 tx=0x7f60fc007c50 comp rx=0 tx=0).stop 2024-03-04T07:32:09.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.476+0000 7f611a302640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6114162360 0x7f61140d0440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.477+0000 7f611a302640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6114141170 0x7f61140cff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.477+0000 7f611a302640 1 --2- 172.21.15.64:0/559541894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f61140d2000 0x7f611415ba50 unknown :-1 s=CLOSED pgs=1292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.477+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 >> 172.21.15.64:0/559541894 conn(0x7f611406eb00 msgr2=0x7f611415dfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:09.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.477+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 shutdown_connections 2024-03-04T07:32:09.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.482+0000 7f611a302640 1 -- 172.21.15.64:0/559541894 wait complete. 2024-03-04T07:32:09.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:32:09.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.604+0000 7ff9c3437640 1 Processor -- start 2024-03-04T07:32:09.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.604+0000 7ff9c3437640 1 -- start start 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.604+0000 7ff9c3437640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff9bc0b1020 0x7ff9bc06af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.604+0000 7ff9c3437640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc06ed10 0x7ff9bc0cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.604+0000 7ff9c3437640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff9bc0cc0e0 con 0x7ff9bc0cd6c0 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.604+0000 7ff9c3437640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff9bc0d28b0 con 0x7ff9bc0b1020 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.604+0000 7ff9c3437640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9bc0cf320 con 0x7ff9bc06ed10 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c11d3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff9bc0b1020 0x7ff9bc06af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c11d3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff9bc0b1020 0x7ff9bc06af20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52820/0 (socket says 172.21.15.64:52820) 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c11d3640 1 -- 172.21.15.64:0/2223372495 learned_addr learned my addr 172.21.15.64:0/2223372495 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c09d2640 1 --2- 172.21.15.64:0/2223372495 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc06ed10 0x7ff9bc0cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c11d3640 1 -- 172.21.15.64:0/2223372495 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc06ed10 msgr2=0x7ff9bc0cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c11d3640 1 --2- 172.21.15.64:0/2223372495 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc06ed10 0x7ff9bc0cd180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c11d3640 1 -- 172.21.15.64:0/2223372495 >> v1:172.21.15.66:6789/0 conn(0x7ff9bc0cd6c0 legacy=0x7ff9bc15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:09.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.605+0000 7ff9c11d3640 1 -- 172.21.15.64:0/2223372495 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9bc0cc3c0 con 0x7ff9bc0b1020 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.606+0000 7ff9c11d3640 1 --2- 172.21.15.64:0/2223372495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc06af20 secure :-1 s=READY pgs=1293 cs=0 l=1 rev1=1 crypto rx=0x7ff9b8009870 tx=0x7ff9b802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bd3cb8f34634e030 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.606+0000 7ff9affff640 1 -- 172.21.15.64:0/2223372495 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9b8035020 con 0x7ff9bc0b1020 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.606+0000 7ff9affff640 1 -- 172.21.15.64:0/2223372495 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff9b8031640 con 0x7ff9bc0b1020 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.606+0000 7ff9affff640 1 -- 172.21.15.64:0/2223372495 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9b802d040 con 0x7ff9bc0b1020 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.606+0000 7ff9c3437640 1 -- 172.21.15.64:0/2223372495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc0b1020 msgr2=0x7ff9bc06af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.606+0000 7ff9c3437640 1 --2- 172.21.15.64:0/2223372495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc06af20 secure :-1 s=READY pgs=1293 cs=0 l=1 rev1=1 crypto rx=0x7ff9b8009870 tx=0x7ff9b802b8f0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.606+0000 7ff9c3437640 1 -- 172.21.15.64:0/2223372495 shutdown_connections 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.607+0000 7ff9c3437640 1 --2- 172.21.15.64:0/2223372495 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc06ed10 0x7ff9bc0cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.607+0000 7ff9c3437640 1 --2- 172.21.15.64:0/2223372495 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc06af20 unknown :-1 s=CLOSED pgs=1293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.607+0000 7ff9c3437640 1 -- 172.21.15.64:0/2223372495 >> 172.21.15.64:0/2223372495 conn(0x7ff9bc0d1a90 msgr2=0x7ff9bc0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.607+0000 7ff9c3437640 1 -- 172.21.15.64:0/2223372495 shutdown_connections 2024-03-04T07:32:09.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.607+0000 7ff9c3437640 1 -- 172.21.15.64:0/2223372495 wait complete. 2024-03-04T07:32:09.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 Processor -- start 2024-03-04T07:32:09.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 -- start start 2024-03-04T07:32:09.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc06ed10 0x7ff9bc0b6210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc0cad30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc0cd6c0 0x7ff9bc0cb270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9bc0ccc60 con 0x7ff9bc06ed10 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9bc0cc980 con 0x7ff9bc0b1020 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.608+0000 7ff9c3437640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9bc0cdb10 con 0x7ff9bc0cd6c0 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc06ed10 0x7ff9bc0b6210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc06ed10 0x7ff9bc0b6210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52826/0 (socket says 172.21.15.64:52826) 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 -- 172.21.15.64:0/365032195 learned_addr learned my addr 172.21.15.64:0/365032195 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c09d2640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc0cad30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c19d4640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc0cd6c0 0x7ff9bc0cb270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 -- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc0cd6c0 msgr2=0x7ff9bc0cb270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc0cd6c0 0x7ff9bc0cb270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 -- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9bc0b1020 msgr2=0x7ff9bc0cad30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc0cad30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 -- 172.21.15.64:0/365032195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9bc0b6b40 con 0x7ff9bc06ed10 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c19d4640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc0cd6c0 0x7ff9bc0cb270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c09d2640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc0cad30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c11d3640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc06ed10 0x7ff9bc0b6210 secure :-1 s=READY pgs=1294 cs=0 l=1 rev1=1 crypto rx=0x7ff9b802b930 tx=0x7ff9b80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9adffb640 1 -- 172.21.15.64:0/365032195 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9b8040070 con 0x7ff9bc06ed10 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9adffb640 1 -- 172.21.15.64:0/365032195 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff9b80399f0 con 0x7ff9bc06ed10 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9adffb640 1 -- 172.21.15.64:0/365032195 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9b8035040 con 0x7ff9bc06ed10 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff9bc0b6e20 con 0x7ff9bc06ed10 2024-03-04T07:32:09.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.609+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff9bc0b73e0 con 0x7ff9bc06ed10 2024-03-04T07:32:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.610+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff9bc067ff0 con 0x7ff9bc06ed10 2024-03-04T07:32:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.611+0000 7ff9adffb640 1 -- 172.21.15.64:0/365032195 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff9b804b060 con 0x7ff9bc06ed10 2024-03-04T07:32:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.611+0000 7ff9adffb640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9a006b500 0x7ff9a006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.611+0000 7ff9adffb640 1 -- 172.21.15.64:0/365032195 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff9b80abaa0 con 0x7ff9bc06ed10 2024-03-04T07:32:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.611+0000 7ff9c09d2640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9a006b500 0x7ff9a006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.612+0000 7ff9c09d2640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9a006b500 0x7ff9a006d9c0 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7ff9bc0cb920 tx=0x7ff9b0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:09.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.615+0000 7ff9adffb640 1 -- 172.21.15.64:0/365032195 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff9b807a9b0 con 0x7ff9bc06ed10 2024-03-04T07:32:09.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.798+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7ff9bc06f2f0 con 0x7ff9bc06ed10 2024-03-04T07:32:09.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.799+0000 7ff9adffb640 1 -- 172.21.15.64:0/365032195 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v104) v1 ==== 74+0+16592 (secure 0 0 0) 0x7ff9b807a2b0 con 0x7ff9bc06ed10 2024-03-04T07:32:09.799 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:09.799 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":104,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:08.825861+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9a006b500 msgr2=0x7ff9a006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9a006b500 0x7ff9a006d9c0 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7ff9bc0cb920 tx=0x7ff9b0008040 comp rx=0 tx=0).stop 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc06ed10 msgr2=0x7ff9bc0b6210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc06ed10 0x7ff9bc0b6210 secure :-1 s=READY pgs=1294 cs=0 l=1 rev1=1 crypto rx=0x7ff9b802b930 tx=0x7ff9b80046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 shutdown_connections 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9a006b500 0x7ff9a006d9c0 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9bc0cd6c0 0x7ff9bc0cb270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9bc0b1020 0x7ff9bc0cad30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 --2- 172.21.15.64:0/365032195 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9bc06ed10 0x7ff9bc0b6210 unknown :-1 s=CLOSED pgs=1294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 >> 172.21.15.64:0/365032195 conn(0x7ff9bc0d1a90 msgr2=0x7ff9bc06bfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 shutdown_connections 2024-03-04T07:32:09.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.801+0000 7ff9c3437640 1 -- 172.21.15.64:0/365032195 wait complete. 2024-03-04T07:32:09.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:32:09.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.913+0000 7f9392735640 1 Processor -- start 2024-03-04T07:32:09.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f9392735640 1 -- start start 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f9392735640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f938c15bda0 0x7f938c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f9392735640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f938c0b0ed0 0x7f938c0ce360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f9392735640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f938c0cd350 con 0x7f938c0ce8a0 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f9392735640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f938c06eb20 con 0x7f938c0b0ed0 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f9392735640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f938c0b1330 con 0x7f938c15bda0 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f938b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f938c0b0ed0 0x7f938c0ce360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f938b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f938c0b0ed0 0x7f938c0ce360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52832/0 (socket says 172.21.15.64:52832) 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.914+0000 7f938b7fe640 1 -- 172.21.15.64:0/341245369 learned_addr learned my addr 172.21.15.64:0/341245369 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.915+0000 7f938bfff640 1 --2- 172.21.15.64:0/341245369 >> v2:172.21.15.66:3300/0 conn(0x7f938c15bda0 0x7f938c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.915+0000 7f938b7fe640 1 -- 172.21.15.64:0/341245369 >> v2:172.21.15.66:3300/0 conn(0x7f938c15bda0 msgr2=0x7f938c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:09.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.915+0000 7f938b7fe640 1 --2- 172.21.15.64:0/341245369 >> v2:172.21.15.66:3300/0 conn(0x7f938c15bda0 0x7f938c0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.915+0000 7f938b7fe640 1 -- 172.21.15.64:0/341245369 >> v1:172.21.15.66:6789/0 conn(0x7f938c0ce8a0 legacy=0x7f938c0cfc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:09.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.915+0000 7f938b7fe640 1 -- 172.21.15.64:0/341245369 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f938c0cd630 con 0x7f938c0b0ed0 2024-03-04T07:32:09.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f938b7fe640 1 --2- 172.21.15.64:0/341245369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c0ce360 secure :-1 s=READY pgs=1295 cs=0 l=1 rev1=1 crypto rx=0x7f938c071ed0 tx=0x7f937c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=807ccdc58161daee server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f938affd640 1 -- 172.21.15.64:0/341245369 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f937c035020 con 0x7f938c0b0ed0 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f938affd640 1 -- 172.21.15.64:0/341245369 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f937c031870 con 0x7f938c0b0ed0 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f938affd640 1 -- 172.21.15.64:0/341245369 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f937c031c40 con 0x7f938c0b0ed0 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 -- 172.21.15.64:0/341245369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 msgr2=0x7f938c0ce360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 --2- 172.21.15.64:0/341245369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c0ce360 secure :-1 s=READY pgs=1295 cs=0 l=1 rev1=1 crypto rx=0x7f938c071ed0 tx=0x7f937c02b910 comp rx=0 tx=0).stop 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 -- 172.21.15.64:0/341245369 shutdown_connections 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 --2- 172.21.15.64:0/341245369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c0ce360 unknown :-1 s=CLOSED pgs=1295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 --2- 172.21.15.64:0/341245369 >> v2:172.21.15.66:3300/0 conn(0x7f938c15bda0 0x7f938c0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 -- 172.21.15.64:0/341245369 >> 172.21.15.64:0/341245369 conn(0x7f938c0d1a90 msgr2=0x7f938c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 -- 172.21.15.64:0/341245369 shutdown_connections 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.916+0000 7f9392735640 1 -- 172.21.15.64:0/341245369 wait complete. 2024-03-04T07:32:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.917+0000 7f9392735640 1 Processor -- start 2024-03-04T07:32:09.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f9392735640 1 -- start start 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f9392735640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c157c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f9392735640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f938c0ce8a0 0x7f938c0c42d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f9392735640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f938c15bda0 0x7f938c15cd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f9392735640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f938c15c180 con 0x7f938c0b0ed0 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f9392735640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f938c06ed70 con 0x7f938c15bda0 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f9392735640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f938c15c300 con 0x7f938c0ce8a0 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f938bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c157c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f938bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c157c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52840/0 (socket says 172.21.15.64:52840) 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.918+0000 7f938bfff640 1 -- 172.21.15.64:0/3032963095 learned_addr learned my addr 172.21.15.64:0/3032963095 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f9390cd2640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f938c15bda0 0x7f938c15cd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f938b7fe640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f938c0ce8a0 0x7f938c0c42d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f938bfff640 1 -- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f938c0ce8a0 msgr2=0x7f938c0c42d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f938bfff640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f938c0ce8a0 0x7f938c0c42d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f938bfff640 1 -- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f938c15bda0 msgr2=0x7f938c15cd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f938bfff640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f938c15bda0 0x7f938c15cd60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f938bfff640 1 -- 172.21.15.64:0/3032963095 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f938c15dbe0 con 0x7f938c0b0ed0 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f938bfff640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c157c60 secure :-1 s=READY pgs=1296 cs=0 l=1 rev1=1 crypto rx=0x7f938000d800 tx=0x7f938000dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f9388ff9640 1 -- 172.21.15.64:0/3032963095 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f938000b660 con 0x7f938c0b0ed0 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f9388ff9640 1 -- 172.21.15.64:0/3032963095 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9380019070 con 0x7f938c0b0ed0 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f9388ff9640 1 -- 172.21.15.64:0/3032963095 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f938000ba20 con 0x7f938c0b0ed0 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f938c0c4b20 con 0x7f938c0b0ed0 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.919+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f938c06b620 con 0x7f938c0b0ed0 2024-03-04T07:32:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.920+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f938c067ff0 con 0x7f938c0b0ed0 2024-03-04T07:32:09.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.920+0000 7f9388ff9640 1 -- 172.21.15.64:0/3032963095 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f938000bbc0 con 0x7f938c0b0ed0 2024-03-04T07:32:09.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.921+0000 7f9388ff9640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f937806b4d0 0x7f937806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:09.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.921+0000 7f938b7fe640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f937806b4d0 0x7f937806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:09.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.921+0000 7f9388ff9640 1 -- 172.21.15.64:0/3032963095 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f938008a4d0 con 0x7f938c0b0ed0 2024-03-04T07:32:09.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.921+0000 7f938b7fe640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f937806b4d0 0x7f937806d990 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f937c036000 tx=0x7f937c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:09.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:09.925+0000 7f9388ff9640 1 -- 172.21.15.64:0/3032963095 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9380058d00 con 0x7f938c0b0ed0 2024-03-04T07:32:10.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.110+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f938c0cf870 con 0x7f938c0b0ed0 2024-03-04T07:32:10.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.111+0000 7f9388ff9640 1 -- 172.21.15.64:0/3032963095 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v104) v1 ==== 74+0+16592 (secure 0 0 0) 0x7f9380088e20 con 0x7f938c0b0ed0 2024-03-04T07:32:10.111 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:10.111 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":104,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:08.825861+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:32:10.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f937806b4d0 msgr2=0x7f937806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f937806b4d0 0x7f937806d990 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f937c036000 tx=0x7f937c03c040 comp rx=0 tx=0).stop 2024-03-04T07:32:10.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 msgr2=0x7f938c157c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c157c60 secure :-1 s=READY pgs=1296 cs=0 l=1 rev1=1 crypto rx=0x7f938000d800 tx=0x7f938000dcc0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 shutdown_connections 2024-03-04T07:32:10.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f937806b4d0 0x7f937806d990 unknown :-1 s=CLOSED pgs=585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f938c15bda0 0x7f938c15cd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f938c0ce8a0 0x7f938c0c42d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 --2- 172.21.15.64:0/3032963095 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f938c0b0ed0 0x7f938c157c60 unknown :-1 s=CLOSED pgs=1296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 >> 172.21.15.64:0/3032963095 conn(0x7f938c0d1a90 msgr2=0x7f938c0b3e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:10.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 shutdown_connections 2024-03-04T07:32:10.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.113+0000 7f9392735640 1 -- 172.21.15.64:0/3032963095 wait complete. 2024-03-04T07:32:10.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:32:10.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.230+0000 7f7ea5a47640 1 Processor -- start 2024-03-04T07:32:10.230 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.230+0000 7f7ea5a47640 1 -- start start 2024-03-04T07:32:10.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7ea5a47640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7ea00cd7d0 0x7f7ea015f900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7ea5a47640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ea015fe40 0x7f7ea00c7ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7ea5a47640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ea015c1d0 con 0x7f7ea015bdf0 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7ea5a47640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7ea00b4590 con 0x7f7ea015fe40 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7ea5a47640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7ea00c66d0 con 0x7f7ea00cd7d0 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7e9f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ea015fe40 0x7f7ea00c7ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7e9f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ea015fe40 0x7f7ea00c7ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52846/0 (socket says 172.21.15.64:52846) 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/2753350206 learned_addr learned my addr 172.21.15.64:0/2753350206 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7e9e7fc640 1 --2- 172.21.15.64:0/2753350206 >> v2:172.21.15.66:3300/0 conn(0x7f7ea00cd7d0 0x7f7ea015f900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/2753350206 >> v2:172.21.15.66:3300/0 conn(0x7f7ea00cd7d0 msgr2=0x7f7ea015f900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7e9f7fe640 1 --2- 172.21.15.64:0/2753350206 >> v2:172.21.15.66:3300/0 conn(0x7f7ea00cd7d0 0x7f7ea015f900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.231+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/2753350206 >> v1:172.21.15.66:6789/0 conn(0x7f7ea015bdf0 legacy=0x7f7ea00cd0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:10.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.232+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/2753350206 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ea00cf870 con 0x7f7ea015fe40 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.232+0000 7f7e9f7fe640 1 --2- 172.21.15.64:0/2753350206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea00c7ac0 secure :-1 s=READY pgs=1297 cs=0 l=1 rev1=1 crypto rx=0x7f7ea00d17b0 tx=0x7f7e9002be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=482abfe53c8ef7d1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.232+0000 7f7e9dffb640 1 -- 172.21.15.64:0/2753350206 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e9002c9f0 con 0x7f7ea015fe40 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.232+0000 7f7e9dffb640 1 -- 172.21.15.64:0/2753350206 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e9002cb90 con 0x7f7ea015fe40 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.232+0000 7f7e9dffb640 1 -- 172.21.15.64:0/2753350206 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e900343f0 con 0x7f7ea015fe40 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 -- 172.21.15.64:0/2753350206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 msgr2=0x7f7ea00c7ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/2753350206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea00c7ac0 secure :-1 s=READY pgs=1297 cs=0 l=1 rev1=1 crypto rx=0x7f7ea00d17b0 tx=0x7f7e9002be50 comp rx=0 tx=0).stop 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 -- 172.21.15.64:0/2753350206 shutdown_connections 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/2753350206 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea00c7ac0 unknown :-1 s=CLOSED pgs=1297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/2753350206 >> v2:172.21.15.66:3300/0 conn(0x7f7ea00cd7d0 0x7f7ea015f900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 -- 172.21.15.64:0/2753350206 >> 172.21.15.64:0/2753350206 conn(0x7f7ea00b3700 msgr2=0x7f7ea00b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 -- 172.21.15.64:0/2753350206 shutdown_connections 2024-03-04T07:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.233+0000 7f7ea5a47640 1 -- 172.21.15.64:0/2753350206 wait complete. 2024-03-04T07:32:10.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.234+0000 7f7ea5a47640 1 Processor -- start 2024-03-04T07:32:10.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.234+0000 7f7ea5a47640 1 -- start start 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7ea5a47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea00cd7d0 0x7f7ea0150310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7ea5a47640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea015bdf0 0x7f7ea00b51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7ea5a47640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea006c0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7ea5a47640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea00d0110 con 0x7f7ea015fe40 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7ea5a47640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea00cfe30 con 0x7f7ea015bdf0 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7ea5a47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea00d03f0 con 0x7f7ea00cd7d0 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea006c0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea006c0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52858/0 (socket says 172.21.15.64:52858) 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/973083344 learned_addr learned my addr 172.21.15.64:0/973083344 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9e7fc640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea015bdf0 0x7f7ea00b51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9effd640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea00cd7d0 0x7f7ea0150310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea00cd7d0 msgr2=0x7f7ea0150310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea00cd7d0 0x7f7ea0150310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea015bdf0 msgr2=0x7f7ea00b51b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea015bdf0 0x7f7ea00b51b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9f7fe640 1 -- 172.21.15.64:0/973083344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ea01f01d0 con 0x7f7ea015fe40 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9e7fc640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea015bdf0 0x7f7ea00b51b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.235+0000 7f7e9effd640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea00cd7d0 0x7f7ea0150310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.236+0000 7f7e9f7fe640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea006c0a0 secure :-1 s=READY pgs=1298 cs=0 l=1 rev1=1 crypto rx=0x7f7e9002c460 tx=0x7f7e90004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.236+0000 7f7e97fff640 1 -- 172.21.15.64:0/973083344 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e90043020 con 0x7f7ea015fe40 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.236+0000 7f7e97fff640 1 -- 172.21.15.64:0/973083344 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e9003e450 con 0x7f7ea015fe40 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.236+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7ea0150850 con 0x7f7ea015fe40 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.236+0000 7f7e97fff640 1 -- 172.21.15.64:0/973083344 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e90009510 con 0x7f7ea015fe40 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.236+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ea00b5d10 con 0x7f7ea015fe40 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.237+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7ea00b2cc0 con 0x7f7ea015fe40 2024-03-04T07:32:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.237+0000 7f7e97fff640 1 -- 172.21.15.64:0/973083344 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7e90046020 con 0x7f7ea015fe40 2024-03-04T07:32:10.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.238+0000 7f7e97fff640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e8006b4d0 0x7f7e8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.238+0000 7f7e97fff640 1 -- 172.21.15.64:0/973083344 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7e900acd80 con 0x7f7ea015fe40 2024-03-04T07:32:10.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.238+0000 7f7e9effd640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e8006b4d0 0x7f7e8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.238+0000 7f7e9effd640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e8006b4d0 0x7f7e8006d990 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7f7e98003eb0 tx=0x7f7e98009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.242+0000 7f7e97fff640 1 -- 172.21.15.64:0/973083344 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7ea00b2cc0 con 0x7f7ea015fe40 2024-03-04T07:32:10.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.460+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f7ea009fcf0 con 0x7f7ea015fe40 2024-03-04T07:32:10.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.470+0000 7f7e97fff640 1 -- 172.21.15.64:0/973083344 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v65) v1 ==== 118+0+0 (secure 0 0 0) 0x7f7e9004d020 con 0x7f7ea015fe40 2024-03-04T07:32:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e8006b4d0 msgr2=0x7f7e8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e8006b4d0 0x7f7e8006d990 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7f7e98003eb0 tx=0x7f7e98009040 comp rx=0 tx=0).stop 2024-03-04T07:32:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 msgr2=0x7f7ea006c0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea006c0a0 secure :-1 s=READY pgs=1298 cs=0 l=1 rev1=1 crypto rx=0x7f7e9002c460 tx=0x7f7e90004a40 comp rx=0 tx=0).stop 2024-03-04T07:32:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 shutdown_connections 2024-03-04T07:32:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e8006b4d0 0x7f7e8006d990 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea015fe40 0x7f7ea006c0a0 unknown :-1 s=CLOSED pgs=1298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea015bdf0 0x7f7ea00b51b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 --2- 172.21.15.64:0/973083344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea00cd7d0 0x7f7ea0150310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 >> 172.21.15.64:0/973083344 conn(0x7f7ea00b3700 msgr2=0x7f7ea00c6850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 shutdown_connections 2024-03-04T07:32:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.473+0000 7f7ea5a47640 1 -- 172.21.15.64:0/973083344 wait complete. 2024-03-04T07:32:10.485 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:32:10.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:32:10.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.590+0000 7f916b2c0640 1 Processor -- start 2024-03-04T07:32:10.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.590+0000 7f916b2c0640 1 -- start start 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916b2c0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f91640cead0 0x7f9164168e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916b2c0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f916415e6d0 con 0x7f916415e2f0 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916b2c0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f91640ccd00 con 0x7f91640b41f0 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916b2c0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f91640d0a10 con 0x7f91640cead0 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916985d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f91640cead0 0x7f9164168e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916905c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f916415e2f0 0x7f916406b4e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44688/0 (socket says 172.21.15.64:44688) 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916905c640 1 -- 172.21.15.64:0/2004283790 learned_addr learned my addr 172.21.15.64:0/2004283790 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:10.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916985d640 1 -- 172.21.15.64:0/2004283790 >> v1:172.21.15.66:6789/0 conn(0x7f91640b41f0 legacy=0x7f916406bbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916985d640 1 -- 172.21.15.64:0/2004283790 >> v1:172.21.15.64:6789/0 conn(0x7f916415e2f0 legacy=0x7f916406b4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f916985d640 1 -- 172.21.15.64:0/2004283790 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f91640ccfe0 con 0x7f91640cead0 2024-03-04T07:32:10.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.591+0000 7f915bfff640 1 -- 172.21.15.64:0/2004283790 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1542998332 0 0) 0x7f916415e6d0 con 0x7f916415e2f0 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.592+0000 7f916985d640 1 --2- 172.21.15.64:0/2004283790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f91640cead0 0x7f9164168e30 secure :-1 s=READY pgs=1300 cs=0 l=1 rev1=1 crypto rx=0x7f91600097a0 tx=0x7f916002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e93469f93b64ce52 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.592+0000 7f915bfff640 1 -- 172.21.15.64:0/2004283790 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9160035020 con 0x7f91640cead0 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.592+0000 7f915bfff640 1 -- 172.21.15.64:0/2004283790 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9160031830 con 0x7f91640cead0 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.592+0000 7f915bfff640 1 -- 172.21.15.64:0/2004283790 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9160031c00 con 0x7f91640cead0 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.593+0000 7f916b2c0640 1 -- 172.21.15.64:0/2004283790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f91640cead0 msgr2=0x7f9164168e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.593+0000 7f916b2c0640 1 --2- 172.21.15.64:0/2004283790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f91640cead0 0x7f9164168e30 secure :-1 s=READY pgs=1300 cs=0 l=1 rev1=1 crypto rx=0x7f91600097a0 tx=0x7f916002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.593+0000 7f916b2c0640 1 -- 172.21.15.64:0/2004283790 shutdown_connections 2024-03-04T07:32:10.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.593+0000 7f916b2c0640 1 --2- 172.21.15.64:0/2004283790 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f91640cead0 0x7f9164168e30 unknown :-1 s=CLOSED pgs=1300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.593+0000 7f916b2c0640 1 -- 172.21.15.64:0/2004283790 >> 172.21.15.64:0/2004283790 conn(0x7f91640b36d0 msgr2=0x7f91640b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:10.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.593+0000 7f916b2c0640 1 -- 172.21.15.64:0/2004283790 shutdown_connections 2024-03-04T07:32:10.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.593+0000 7f916b2c0640 1 -- 172.21.15.64:0/2004283790 wait complete. 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.594+0000 7f916b2c0640 1 Processor -- start 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.594+0000 7f916b2c0640 1 -- start start 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916b2c0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f91640b41f0 0x7f916414e9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916b2c0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91640cead0 0x7f9164149df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916b2c0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f916415e2f0 0x7f916414f3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916b2c0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f916406c300 con 0x7f916415e2f0 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916b2c0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91640cd5a0 con 0x7f91640cead0 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916b2c0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9164169370 con 0x7f91640b41f0 2024-03-04T07:32:10.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f916415e2f0 0x7f916414f3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f916415e2f0 0x7f916414f3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52868/0 (socket says 172.21.15.64:52868) 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 -- 172.21.15.64:0/1388168553 learned_addr learned my addr 172.21.15.64:0/1388168553 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916885b640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91640cead0 0x7f9164149df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916905c640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f91640b41f0 0x7f916414e9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 -- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f91640b41f0 msgr2=0x7f916414e9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f91640b41f0 0x7f916414e9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 -- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91640cead0 msgr2=0x7f9164149df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91640cead0 0x7f9164149df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 -- 172.21.15.64:0/1388168553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f916414d910 con 0x7f916415e2f0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916985d640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f916415e2f0 0x7f916414f3b0 secure :-1 s=READY pgs=1301 cs=0 l=1 rev1=1 crypto rx=0x7f91600098c0 tx=0x7f91600046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f9159ffb640 1 -- 172.21.15.64:0/1388168553 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9160041070 con 0x7f916415e2f0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f9159ffb640 1 -- 172.21.15.64:0/1388168553 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f916003a8d0 con 0x7f916415e2f0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f916414dbf0 con 0x7f916415e2f0 2024-03-04T07:32:10.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.595+0000 7f9159ffb640 1 -- 172.21.15.64:0/1388168553 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9160035040 con 0x7f916415e2f0 2024-03-04T07:32:10.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.596+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f916414e1b0 con 0x7f916415e2f0 2024-03-04T07:32:10.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.597+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f916414eef0 con 0x7f916415e2f0 2024-03-04T07:32:10.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.597+0000 7f9159ffb640 1 -- 172.21.15.64:0/1388168553 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9160038020 con 0x7f916415e2f0 2024-03-04T07:32:10.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.597+0000 7f9159ffb640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f914806b4d0 0x7f914806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:10.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.597+0000 7f9159ffb640 1 -- 172.21.15.64:0/1388168553 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f91600abae0 con 0x7f916415e2f0 2024-03-04T07:32:10.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.597+0000 7f916905c640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f914806b4d0 0x7f914806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:10.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.598+0000 7f916905c640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f914806b4d0 0x7f914806d990 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f915c0012e0 tx=0x7f915c004d40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:10.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.602+0000 7f9159ffb640 1 -- 172.21.15.64:0/1388168553 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f916414eef0 con 0x7f916415e2f0 2024-03-04T07:32:10.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:10.822+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f916406c0a0 con 0x7f916415e2f0 2024-03-04T07:32:11.443 INFO:tasks.ceph.mds.d.smithi066.stderr:2024-03-04T07:32:11.443+0000 7faf468ff640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:32:11.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.479+0000 7f9159ffb640 1 -- 172.21.15.64:0/1388168553 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v66) v1 ==== 122+0+0 (secure 0 0 0) 0x7f916007a3a0 con 0x7f916415e2f0 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f914806b4d0 msgr2=0x7f914806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f914806b4d0 0x7f914806d990 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f915c0012e0 tx=0x7f915c004d40 comp rx=0 tx=0).stop 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f916415e2f0 msgr2=0x7f916414f3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f916415e2f0 0x7f916414f3b0 secure :-1 s=READY pgs=1301 cs=0 l=1 rev1=1 crypto rx=0x7f91600098c0 tx=0x7f91600046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 shutdown_connections 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f914806b4d0 0x7f914806d990 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f916415e2f0 0x7f916414f3b0 unknown :-1 s=CLOSED pgs=1301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91640cead0 0x7f9164149df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 --2- 172.21.15.64:0/1388168553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f91640b41f0 0x7f916414e9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.481+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 >> 172.21.15.64:0/1388168553 conn(0x7f91640b36d0 msgr2=0x7f91640cef20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.482+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 shutdown_connections 2024-03-04T07:32:11.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.482+0000 7f916b2c0640 1 -- 172.21.15.64:0/1388168553 wait complete. 2024-03-04T07:32:11.494 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:32:11.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:11.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.602+0000 7fb88cdf6640 1 Processor -- start 2024-03-04T07:32:11.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.602+0000 7fb88cdf6640 1 -- start start 2024-03-04T07:32:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb88cdf6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb88815df00 0x7fb8880d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb88cdf6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b9a30 0x7fb8880d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb88cdf6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb8880cea30 con 0x7fb8880b95a0 2024-03-04T07:32:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb88cdf6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb8880d2af0 con 0x7fb88815df00 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb88cdf6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb8880cf3c0 con 0x7fb8880b9a30 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb886d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b9a30 0x7fb8880d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb886d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b9a30 0x7fb8880d3220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:34660/0 (socket says 172.21.15.64:34660) 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb886d9d640 1 -- 172.21.15.64:0/3134185682 learned_addr learned my addr 172.21.15.64:0/3134185682 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb88659c640 1 --2- 172.21.15.64:0/3134185682 >> v2:172.21.15.66:3300/0 conn(0x7fb88815df00 0x7fb8880d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb886d9d640 1 -- 172.21.15.64:0/3134185682 >> v2:172.21.15.66:3300/0 conn(0x7fb88815df00 msgr2=0x7fb8880d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb886d9d640 1 --2- 172.21.15.64:0/3134185682 >> v2:172.21.15.66:3300/0 conn(0x7fb88815df00 0x7fb8880d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb886d9d640 1 -- 172.21.15.64:0/3134185682 >> v1:172.21.15.66:6789/0 conn(0x7fb8880b95a0 legacy=0x7fb8880b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb886d9d640 1 -- 172.21.15.64:0/3134185682 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8880cf0e0 con 0x7fb8880b9a30 2024-03-04T07:32:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.603+0000 7fb88659c640 1 --2- 172.21.15.64:0/3134185682 >> v2:172.21.15.66:3300/0 conn(0x7fb88815df00 0x7fb8880d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.604+0000 7fb886d9d640 1 --2- 172.21.15.64:0/3134185682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b9a30 0x7fb8880d3220 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fb880009e50 tx=0x7fb88002bf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=72a999066e564d3d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.604+0000 7fb885d9b640 1 -- 172.21.15.64:0/3134185682 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb88002cb00 con 0x7fb8880b9a30 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.604+0000 7fb885d9b640 1 -- 172.21.15.64:0/3134185682 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb88002cca0 con 0x7fb8880b9a30 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.604+0000 7fb885d9b640 1 -- 172.21.15.64:0/3134185682 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb880034610 con 0x7fb8880b9a30 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3134185682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b9a30 msgr2=0x7fb8880d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3134185682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b9a30 0x7fb8880d3220 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fb880009e50 tx=0x7fb88002bf60 comp rx=0 tx=0).stop 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3134185682 shutdown_connections 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3134185682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b9a30 0x7fb8880d3220 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3134185682 >> v2:172.21.15.66:3300/0 conn(0x7fb88815df00 0x7fb8880d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3134185682 >> 172.21.15.64:0/3134185682 conn(0x7fb888071960 msgr2=0x7fb888071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3134185682 shutdown_connections 2024-03-04T07:32:11.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.605+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3134185682 wait complete. 2024-03-04T07:32:11.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.606+0000 7fb88cdf6640 1 Processor -- start 2024-03-04T07:32:11.607 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.606+0000 7fb88cdf6640 1 -- start start 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88cdf6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8880b95a0 0x7fb8881f2f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88cdf6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb88815df00 0x7fb88815ec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88cdf6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b1df0 0x7fb8880b2280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88cdf6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8880cec70 con 0x7fb8880b95a0 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88cdf6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8880ce750 con 0x7fb88815df00 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88cdf6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb8880cedf0 con 0x7fb8880b1df0 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8880b95a0 0x7fb8881f2f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8880b95a0 0x7fb8881f2f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52878/0 (socket says 172.21.15.64:52878) 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 -- 172.21.15.64:0/3340323439 learned_addr learned my addr 172.21.15.64:0/3340323439 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb886d9d640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b1df0 0x7fb8880b2280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb87ffff640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb88815df00 0x7fb88815ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 -- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b1df0 msgr2=0x7fb8880b2280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b1df0 0x7fb8880b2280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 -- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb88815df00 msgr2=0x7fb88815ec00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb88815df00 0x7fb88815ec00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.607+0000 7fb88659c640 1 -- 172.21.15.64:0/3340323439 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8880cbf20 con 0x7fb8880b95a0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.608+0000 7fb88659c640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8880b95a0 0x7fb8881f2f60 secure :-1 s=READY pgs=1302 cs=0 l=1 rev1=1 crypto rx=0x7fb87800e960 tx=0x7fb87800ee20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.608+0000 7fb87effd640 1 -- 172.21.15.64:0/3340323439 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb878017070 con 0x7fb8880b95a0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.608+0000 7fb87effd640 1 -- 172.21.15.64:0/3340323439 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb87800ca00 con 0x7fb8880b95a0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.608+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb8880cc200 con 0x7fb8880b95a0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.608+0000 7fb87effd640 1 -- 172.21.15.64:0/3340323439 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb87800cdd0 con 0x7fb8880b95a0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.608+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb8881f34a0 con 0x7fb8880b95a0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.609+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb888067ff0 con 0x7fb8880b95a0 2024-03-04T07:32:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.609+0000 7fb87effd640 1 -- 172.21.15.64:0/3340323439 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb87801a070 con 0x7fb8880b95a0 2024-03-04T07:32:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.610+0000 7fb87effd640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb86806b4d0 0x7fb86806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.610+0000 7fb87effd640 1 -- 172.21.15.64:0/3340323439 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb87808a320 con 0x7fb8880b95a0 2024-03-04T07:32:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.610+0000 7fb87ffff640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb86806b4d0 0x7fb86806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.610+0000 7fb87ffff640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb86806b4d0 0x7fb86806d990 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7fb88815eb50 tx=0x7fb870009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:11.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.614+0000 7fb87effd640 1 -- 172.21.15.64:0/3340323439 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb878058be0 con 0x7fb8880b95a0 2024-03-04T07:32:11.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.834+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb88815b820 con 0x7fb8880b95a0 2024-03-04T07:32:11.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.835+0000 7fb87effd640 1 -- 172.21.15.64:0/3340323439 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 66 v66) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fb87805cc80 con 0x7fb8880b95a0 2024-03-04T07:32:11.835 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:11.835 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":66,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6281,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/2549379142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2549379142},{"type":"v1","addr":"172.21.15.66:6834","nonce":2549379142}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7854,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3068683808","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3068683808},{"type":"v1","addr":"172.21.15.64:6836","nonce":3068683808}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":62},{"gid":7857,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/209754994","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":209754994},{"type":"v1","addr":"172.21.15.64:6837","nonce":209754994}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63}],"filesystems":[{"mdsmap":{"epoch":66,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:10.823856+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7872},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7872":{"gid":7872,"name":"d","rank":0,"incarnation":64,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/4084900379","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":4084900379},{"type":"v1","addr":"172.21.15.66:6835","nonce":4084900379}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7872,"qdb_cluster":[7872]},"id":8}]} 2024-03-04T07:32:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 66 2024-03-04T07:32:11.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb86806b4d0 msgr2=0x7fb86806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb86806b4d0 0x7fb86806d990 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7fb88815eb50 tx=0x7fb870009040 comp rx=0 tx=0).stop 2024-03-04T07:32:11.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8880b95a0 msgr2=0x7fb8881f2f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8880b95a0 0x7fb8881f2f60 secure :-1 s=READY pgs=1302 cs=0 l=1 rev1=1 crypto rx=0x7fb87800e960 tx=0x7fb87800ee20 comp rx=0 tx=0).stop 2024-03-04T07:32:11.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 shutdown_connections 2024-03-04T07:32:11.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb86806b4d0 0x7fb86806d990 unknown :-1 s=CLOSED pgs=588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb8880b1df0 0x7fb8880b2280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb88815df00 0x7fb88815ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 --2- 172.21.15.64:0/3340323439 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb8880b95a0 0x7fb8881f2f60 unknown :-1 s=CLOSED pgs=1302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 >> 172.21.15.64:0/3340323439 conn(0x7fb888071960 msgr2=0x7fb8880b6e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:11.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 shutdown_connections 2024-03-04T07:32:11.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.837+0000 7fb88cdf6640 1 -- 172.21.15.64:0/3340323439 wait complete. 2024-03-04T07:32:11.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:32:11.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.950+0000 7f7d0f064640 1 Processor -- start 2024-03-04T07:32:11.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.950+0000 7f7d0f064640 1 -- start start 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0f064640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7d0806af20 0x7f7d0815d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0f064640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d080cc0e0 con 0x7f7d080b1020 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0f064640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d080cc3c0 con 0x7f7d0806ed10 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0f064640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7d080d28b0 con 0x7f7d0806af20 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0ce00640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7d080b1020 0x7f7d080cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44702/0 (socket says 172.21.15.64:44702) 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0ce00640 1 -- 172.21.15.64:0/3633358208 learned_addr learned my addr 172.21.15.64:0/3633358208 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0d601640 1 --2- 172.21.15.64:0/3633358208 >> v2:172.21.15.66:3300/0 conn(0x7f7d0806af20 0x7f7d0815d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3916403225 0 0) 0x7f7d080cc0e0 con 0x7f7d080b1020 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7cf8003520 con 0x7f7d080b1020 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3939652759 0 0) 0x7f7d080cc3c0 con 0x7f7d0806ed10 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7d080cc0e0 con 0x7f7d0806ed10 2024-03-04T07:32:11.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.951+0000 7f7d0d601640 1 -- 172.21.15.64:0/3633358208 >> v1:172.21.15.66:6789/0 conn(0x7f7d0806ed10 legacy=0x7f7d080cd890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.952+0000 7f7d0d601640 1 -- 172.21.15.64:0/3633358208 >> v1:172.21.15.64:6789/0 conn(0x7f7d080b1020 legacy=0x7f7d080cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.952+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 113559450 0 0) 0x7f7cf8003520 con 0x7f7d080b1020 2024-03-04T07:32:11.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.952+0000 7f7d0d601640 1 -- 172.21.15.64:0/3633358208 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d080cc6a0 con 0x7f7d0806af20 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.952+0000 7f7d0d601640 1 --2- 172.21.15.64:0/3633358208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d0815d700 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f7d00009d90 tx=0x7f7d0002be10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c3ad602c4857a9ec server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d00039070 con 0x7f7d0806af20 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d0002c940 con 0x7f7d0806af20 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d077fe640 1 -- 172.21.15.64:0/3633358208 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7cfc001670 con 0x7f7d0806af20 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d0f064640 1 -- 172.21.15.64:0/3633358208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 msgr2=0x7f7d0815d700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d0f064640 1 --2- 172.21.15.64:0/3633358208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d0815d700 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f7d00009d90 tx=0x7f7d0002be10 comp rx=0 tx=0).stop 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d0f064640 1 -- 172.21.15.64:0/3633358208 shutdown_connections 2024-03-04T07:32:11.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d0f064640 1 --2- 172.21.15.64:0/3633358208 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d0815d700 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d0f064640 1 -- 172.21.15.64:0/3633358208 >> 172.21.15.64:0/3633358208 conn(0x7f7d080d1a90 msgr2=0x7f7d080d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d0f064640 1 -- 172.21.15.64:0/3633358208 shutdown_connections 2024-03-04T07:32:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.953+0000 7f7d0f064640 1 -- 172.21.15.64:0/3633358208 wait complete. 2024-03-04T07:32:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.954+0000 7f7d0f064640 1 Processor -- start 2024-03-04T07:32:11.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.954+0000 7f7d0f064640 1 -- start start 2024-03-04T07:32:11.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0f064640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d080b5270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0f064640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0806ed10 0x7f7d081f45e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0f064640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d080b1020 0x7f7d081f4d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0f064640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d080ccc60 con 0x7f7d080b1020 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0f064640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d080cf320 con 0x7f7d0806af20 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0f064640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0815dc40 con 0x7f7d0806ed10 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0d601640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d080b1020 0x7f7d081f4d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0d601640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d080b1020 0x7f7d081f4d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52888/0 (socket says 172.21.15.64:52888) 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0d601640 1 -- 172.21.15.64:0/2922652183 learned_addr learned my addr 172.21.15.64:0/2922652183 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d0ce00640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d080b5270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.955+0000 7f7d07fff640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0806ed10 0x7f7d081f45e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0d601640 1 -- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0806ed10 msgr2=0x7f7d081f45e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0d601640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0806ed10 0x7f7d081f45e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0d601640 1 -- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 msgr2=0x7f7d080b5270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0d601640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d080b5270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0d601640 1 -- 172.21.15.64:0/2922652183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d081f5260 con 0x7f7d080b1020 2024-03-04T07:32:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0ce00640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d080b5270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0d601640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d080b1020 0x7f7d081f4d20 secure :-1 s=READY pgs=1304 cs=0 l=1 rev1=1 crypto rx=0x7f7d00009eb0 tx=0x7f7d00004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d057fa640 1 -- 172.21.15.64:0/2922652183 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d00039050 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d057fa640 1 -- 172.21.15.64:0/2922652183 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d00045070 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d080b5850 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d057fa640 1 -- 172.21.15.64:0/2922652183 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d00009450 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.956+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d080ca0e0 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.957+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d08067ff0 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.957+0000 7f7d057fa640 1 -- 172.21.15.64:0/2922652183 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7d00040690 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.958+0000 7f7d057fa640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cec06b4d0 0x7f7cec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.958+0000 7f7d057fa640 1 -- 172.21.15.64:0/2922652183 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7d000ac990 con 0x7f7d080b1020 2024-03-04T07:32:11.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.958+0000 7f7d0ce00640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cec06b4d0 0x7f7cec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:11.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.958+0000 7f7d0ce00640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cec06b4d0 0x7f7cec06d990 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc000fd0 tx=0x7f7cfc001150 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:11.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:11.962+0000 7f7d057fa640 1 -- 172.21.15.64:0/2922652183 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7d0007b1c0 con 0x7f7d080b1020 2024-03-04T07:32:12.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.144+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7d0806f2f0 con 0x7f7d080b1020 2024-03-04T07:32:12.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.145+0000 7f7d057fa640 1 -- 172.21.15.64:0/2922652183 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v105) v1 ==== 74+0+16579 (secure 0 0 0) 0x7f7d0007f260 con 0x7f7d080b1020 2024-03-04T07:32:12.145 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:12.146 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":105,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:10.470294+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"105","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cec06b4d0 msgr2=0x7f7cec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cec06b4d0 0x7f7cec06d990 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc000fd0 tx=0x7f7cfc001150 comp rx=0 tx=0).stop 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d080b1020 msgr2=0x7f7d081f4d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d080b1020 0x7f7d081f4d20 secure :-1 s=READY pgs=1304 cs=0 l=1 rev1=1 crypto rx=0x7f7d00009eb0 tx=0x7f7d00004a40 comp rx=0 tx=0).stop 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 shutdown_connections 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cec06b4d0 0x7f7cec06d990 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d080b1020 0x7f7d081f4d20 unknown :-1 s=CLOSED pgs=1304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0806ed10 0x7f7d081f45e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 --2- 172.21.15.64:0/2922652183 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0806af20 0x7f7d080b5270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 >> 172.21.15.64:0/2922652183 conn(0x7f7d080d1a90 msgr2=0x7f7d080d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 shutdown_connections 2024-03-04T07:32:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.148+0000 7f7d0f064640 1 -- 172.21.15.64:0/2922652183 wait complete. 2024-03-04T07:32:12.159 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:32:12.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:32:12.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.261+0000 7fb525833640 1 Processor -- start 2024-03-04T07:32:12.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.261+0000 7fb525833640 1 -- start start 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb525833640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb5200b0e90 0x7fb52006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb525833640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5200cee90 con 0x7fb52006b5c0 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb525833640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5200cfc50 con 0x7fb5200ae9b0 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb525833640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5200cfa00 con 0x7fb5200b0e90 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb51effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb52006b5c0 0x7fb5200b2e00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44708/0 (socket says 172.21.15.64:44708) 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb51effd640 1 -- 172.21.15.64:0/1578982861 learned_addr learned my addr 172.21.15.64:0/1578982861 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb51e7fc640 1 --2- 172.21.15.64:0/1578982861 >> v2:172.21.15.66:3300/0 conn(0x7fb5200b0e90 0x7fb52006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb51dffb640 1 -- 172.21.15.64:0/1578982861 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3590414113 0 0) 0x7fb5200cee90 con 0x7fb52006b5c0 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.262+0000 7fb51dffb640 1 -- 172.21.15.64:0/1578982861 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb508003520 con 0x7fb52006b5c0 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.263+0000 7fb51dffb640 1 -- 172.21.15.64:0/1578982861 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2388198504 0 0) 0x7fb5200cfc50 con 0x7fb5200ae9b0 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.263+0000 7fb51dffb640 1 -- 172.21.15.64:0/1578982861 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb5200cee90 con 0x7fb5200ae9b0 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.263+0000 7fb51e7fc640 1 -- 172.21.15.64:0/1578982861 >> v1:172.21.15.66:6789/0 conn(0x7fb5200ae9b0 legacy=0x7fb5200d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.263+0000 7fb51e7fc640 1 -- 172.21.15.64:0/1578982861 >> v1:172.21.15.64:6789/0 conn(0x7fb52006b5c0 legacy=0x7fb5200b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.263+0000 7fb51e7fc640 1 -- 172.21.15.64:0/1578982861 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5200cff30 con 0x7fb5200b0e90 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.263+0000 7fb51e7fc640 1 --2- 172.21.15.64:0/1578982861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 0x7fb52006b9a0 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fb5140097a0 tx=0x7fb51402b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=eaeccb129460edf5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb51dffb640 1 -- 172.21.15.64:0/1578982861 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb514035020 con 0x7fb5200b0e90 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb51dffb640 1 -- 172.21.15.64:0/1578982861 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb514031830 con 0x7fb5200b0e90 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb51dffb640 1 -- 172.21.15.64:0/1578982861 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb514031c00 con 0x7fb5200b0e90 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb525833640 1 -- 172.21.15.64:0/1578982861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 msgr2=0x7fb52006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb525833640 1 --2- 172.21.15.64:0/1578982861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 0x7fb52006b9a0 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fb5140097a0 tx=0x7fb51402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb525833640 1 -- 172.21.15.64:0/1578982861 shutdown_connections 2024-03-04T07:32:12.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb525833640 1 --2- 172.21.15.64:0/1578982861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 0x7fb52006b9a0 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb525833640 1 -- 172.21.15.64:0/1578982861 >> 172.21.15.64:0/1578982861 conn(0x7fb5200b36d0 msgr2=0x7fb5200b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:12.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb525833640 1 -- 172.21.15.64:0/1578982861 shutdown_connections 2024-03-04T07:32:12.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.264+0000 7fb525833640 1 -- 172.21.15.64:0/1578982861 wait complete. 2024-03-04T07:32:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.265+0000 7fb525833640 1 Processor -- start 2024-03-04T07:32:12.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.265+0000 7fb525833640 1 -- start start 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb525833640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb52006b5c0 0x7fb52014dfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb525833640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5200ae9b0 0x7fb5200b46c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb525833640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 0x7fb5200b6220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb525833640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb52006c910 con 0x7fb52006b5c0 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb525833640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb52006c790 con 0x7fb5200b0e90 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb525833640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5200d0ab0 con 0x7fb5200ae9b0 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb51effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb52006b5c0 0x7fb52014dfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb51effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb52006b5c0 0x7fb52014dfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52892/0 (socket says 172.21.15.64:52892) 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb51effd640 1 -- 172.21.15.64:0/4064560422 learned_addr learned my addr 172.21.15.64:0/4064560422 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb51e7fc640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5200ae9b0 0x7fb5200b46c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.266+0000 7fb51f7fe640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 0x7fb5200b6220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb51effd640 1 -- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5200ae9b0 msgr2=0x7fb5200b46c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb51effd640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5200ae9b0 0x7fb5200b46c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb51effd640 1 -- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 msgr2=0x7fb5200b6220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb51effd640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 0x7fb5200b6220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb51effd640 1 -- 172.21.15.64:0/4064560422 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5200b6ef0 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb51effd640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb52006b5c0 0x7fb52014dfd0 secure :-1 s=READY pgs=1306 cs=0 l=1 rev1=1 crypto rx=0x7fb51000cb50 tx=0x7fb51000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb507fff640 1 -- 172.21.15.64:0/4064560422 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb510017070 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb507fff640 1 -- 172.21.15.64:0/4064560422 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb51000ad40 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5200b71d0 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb507fff640 1 -- 172.21.15.64:0/4064560422 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb510012620 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.267+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5200b7790 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.268+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb520068700 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.268+0000 7fb507fff640 1 -- 172.21.15.64:0/4064560422 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb51001a070 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.269+0000 7fb507fff640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb50c06b4d0 0x7fb50c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.269+0000 7fb507fff640 1 -- 172.21.15.64:0/4064560422 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb510089fd0 con 0x7fb52006b5c0 2024-03-04T07:32:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.269+0000 7fb51e7fc640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb50c06b4d0 0x7fb50c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.269+0000 7fb51e7fc640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb50c06b4d0 0x7fb50c06d990 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7fb514036000 tx=0x7fb514008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.273+0000 7fb507fff640 1 -- 172.21.15.64:0/4064560422 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb510058780 con 0x7fb52006b5c0 2024-03-04T07:32:12.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.450+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fb52006fd10 con 0x7fb50c06b4d0 2024-03-04T07:32:12.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.457+0000 7fb507fff640 1 -- 172.21.15.64:0/4064560422 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252463 (secure 0 0 0) 0x7fb52006fd10 con 0x7fb50c06b4d0 2024-03-04T07:32:12.459 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:12.461 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:32:12.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.459+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb50c06b4d0 msgr2=0x7fb50c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.459+0000 7fb525833640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb50c06b4d0 0x7fb50c06d990 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7fb514036000 tx=0x7fb514008030 comp rx=0 tx=0).stop 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.459+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb52006b5c0 msgr2=0x7fb52014dfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.459+0000 7fb525833640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb52006b5c0 0x7fb52014dfd0 secure :-1 s=READY pgs=1306 cs=0 l=1 rev1=1 crypto rx=0x7fb51000cb50 tx=0x7fb51000a430 comp rx=0 tx=0).stop 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 shutdown_connections 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb50c06b4d0 0x7fb50c06d990 unknown :-1 s=CLOSED pgs=590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5200b0e90 0x7fb5200b6220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5200ae9b0 0x7fb5200b46c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 --2- 172.21.15.64:0/4064560422 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb52006b5c0 0x7fb52014dfd0 unknown :-1 s=CLOSED pgs=1306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 >> 172.21.15.64:0/4064560422 conn(0x7fb5200b36d0 msgr2=0x7fb52015e230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 shutdown_connections 2024-03-04T07:32:12.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.460+0000 7fb525833640 1 -- 172.21.15.64:0/4064560422 wait complete. 2024-03-04T07:32:12.471 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2024-03-04T07:32:11.199472+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":121,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":323236,"kb_used_data":11372,"kb_used_omap":12,"kb_used_meta":311731,"kb_avail":754651484,"statfs":{"total":773094113280,"available":772763119616,"internally_reserved":0,"allocated":11644928,"data_stored":5657496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":319213104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000841"},"pg_stats":[{"pgid":"17.3f","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478377+0000","last_change":"2024-03-04T07:32:09.470345+0000","last_active":"2024-03-04T07:32:10.478377+0000","last_peered":"2024-03-04T07:32:10.478377+0000","last_clean":"2024-03-04T07:32:10.478377+0000","last_became_active":"2024-03-04T07:32:09.470078+0000","last_became_peered":"2024-03-04T07:32:09.470078+0000","last_unstale":"2024-03-04T07:32:10.478377+0000","last_undegraded":"2024-03-04T07:32:10.478377+0000","last_fullsized":"2024-03-04T07:32:10.478377+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:02:52.613754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.3e","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479029+0000","last_change":"2024-03-04T07:32:09.479732+0000","last_active":"2024-03-04T07:32:10.479029+0000","last_peered":"2024-03-04T07:32:10.479029+0000","last_clean":"2024-03-04T07:32:10.479029+0000","last_became_active":"2024-03-04T07:32:09.479417+0000","last_became_peered":"2024-03-04T07:32:09.479417+0000","last_unstale":"2024-03-04T07:32:10.479029+0000","last_undegraded":"2024-03-04T07:32:10.479029+0000","last_fullsized":"2024-03-04T07:32:10.479029+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:28.363398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.3d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816900+0000","last_change":"2024-03-04T07:32:09.488729+0000","last_active":"2024-03-04T07:32:10.816900+0000","last_peered":"2024-03-04T07:32:10.816900+0000","last_clean":"2024-03-04T07:32:10.816900+0000","last_became_active":"2024-03-04T07:32:09.487826+0000","last_became_peered":"2024-03-04T07:32:09.487826+0000","last_unstale":"2024-03-04T07:32:10.816900+0000","last_undegraded":"2024-03-04T07:32:10.816900+0000","last_fullsized":"2024-03-04T07:32:10.816900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:44:21.383631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.3c","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477938+0000","last_change":"2024-03-04T07:32:09.486806+0000","last_active":"2024-03-04T07:32:10.477938+0000","last_peered":"2024-03-04T07:32:10.477938+0000","last_clean":"2024-03-04T07:32:10.477938+0000","last_became_active":"2024-03-04T07:32:09.486544+0000","last_became_peered":"2024-03-04T07:32:09.486544+0000","last_unstale":"2024-03-04T07:32:10.477938+0000","last_undegraded":"2024-03-04T07:32:10.477938+0000","last_fullsized":"2024-03-04T07:32:10.477938+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:08:57.046464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.3b","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.482939+0000","last_change":"2024-03-04T07:32:09.479104+0000","last_active":"2024-03-04T07:32:09.482939+0000","last_peered":"2024-03-04T07:32:09.482939+0000","last_clean":"2024-03-04T07:32:09.482939+0000","last_became_active":"2024-03-04T07:32:09.477615+0000","last_became_peered":"2024-03-04T07:32:09.477615+0000","last_unstale":"2024-03-04T07:32:09.482939+0000","last_undegraded":"2024-03-04T07:32:09.482939+0000","last_fullsized":"2024-03-04T07:32:09.482939+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:57:11.655324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.3a","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.481184+0000","last_change":"2024-03-04T07:32:09.480083+0000","last_active":"2024-03-04T07:32:09.481184+0000","last_peered":"2024-03-04T07:32:09.481184+0000","last_clean":"2024-03-04T07:32:09.481184+0000","last_became_active":"2024-03-04T07:32:09.479775+0000","last_became_peered":"2024-03-04T07:32:09.479775+0000","last_unstale":"2024-03-04T07:32:09.481184+0000","last_undegraded":"2024-03-04T07:32:09.481184+0000","last_fullsized":"2024-03-04T07:32:09.481184+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:45:40.392705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.39","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478058+0000","last_change":"2024-03-04T07:32:09.469706+0000","last_active":"2024-03-04T07:32:10.478058+0000","last_peered":"2024-03-04T07:32:10.478058+0000","last_clean":"2024-03-04T07:32:10.478058+0000","last_became_active":"2024-03-04T07:32:09.468580+0000","last_became_peered":"2024-03-04T07:32:09.468580+0000","last_unstale":"2024-03-04T07:32:10.478058+0000","last_undegraded":"2024-03-04T07:32:10.478058+0000","last_fullsized":"2024-03-04T07:32:10.478058+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:46:13.668525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.38","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816589+0000","last_change":"2024-03-04T07:32:09.477916+0000","last_active":"2024-03-04T07:32:10.816589+0000","last_peered":"2024-03-04T07:32:10.816589+0000","last_clean":"2024-03-04T07:32:10.816589+0000","last_became_active":"2024-03-04T07:32:09.477639+0000","last_became_peered":"2024-03-04T07:32:09.477639+0000","last_unstale":"2024-03-04T07:32:10.816589+0000","last_undegraded":"2024-03-04T07:32:10.816589+0000","last_fullsized":"2024-03-04T07:32:10.816589+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:40.280836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.37","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478494+0000","last_change":"2024-03-04T07:32:09.479790+0000","last_active":"2024-03-04T07:32:10.478494+0000","last_peered":"2024-03-04T07:32:10.478494+0000","last_clean":"2024-03-04T07:32:10.478494+0000","last_became_active":"2024-03-04T07:32:09.479485+0000","last_became_peered":"2024-03-04T07:32:09.479485+0000","last_unstale":"2024-03-04T07:32:10.478494+0000","last_undegraded":"2024-03-04T07:32:10.478494+0000","last_fullsized":"2024-03-04T07:32:10.478494+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:40:53.021216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.36","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477962+0000","last_change":"2024-03-04T07:32:09.478798+0000","last_active":"2024-03-04T07:32:10.477962+0000","last_peered":"2024-03-04T07:32:10.477962+0000","last_clean":"2024-03-04T07:32:10.477962+0000","last_became_active":"2024-03-04T07:32:09.478508+0000","last_became_peered":"2024-03-04T07:32:09.478508+0000","last_unstale":"2024-03-04T07:32:10.477962+0000","last_undegraded":"2024-03-04T07:32:10.477962+0000","last_fullsized":"2024-03-04T07:32:10.477962+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:11:54.852936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.35","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478991+0000","last_change":"2024-03-04T07:32:09.484416+0000","last_active":"2024-03-04T07:32:10.478991+0000","last_peered":"2024-03-04T07:32:10.478991+0000","last_clean":"2024-03-04T07:32:10.478991+0000","last_became_active":"2024-03-04T07:32:09.482608+0000","last_became_peered":"2024-03-04T07:32:09.482608+0000","last_unstale":"2024-03-04T07:32:10.478991+0000","last_undegraded":"2024-03-04T07:32:10.478991+0000","last_fullsized":"2024-03-04T07:32:10.478991+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:34.767357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.34","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478078+0000","last_change":"2024-03-04T07:32:09.485056+0000","last_active":"2024-03-04T07:32:10.478078+0000","last_peered":"2024-03-04T07:32:10.478078+0000","last_clean":"2024-03-04T07:32:10.478078+0000","last_became_active":"2024-03-04T07:32:09.484210+0000","last_became_peered":"2024-03-04T07:32:09.484210+0000","last_unstale":"2024-03-04T07:32:10.478078+0000","last_undegraded":"2024-03-04T07:32:10.478078+0000","last_fullsized":"2024-03-04T07:32:10.478078+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:11:08.955473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.33","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816900+0000","last_change":"2024-03-04T07:32:09.480038+0000","last_active":"2024-03-04T07:32:10.816900+0000","last_peered":"2024-03-04T07:32:10.816900+0000","last_clean":"2024-03-04T07:32:10.816900+0000","last_became_active":"2024-03-04T07:32:09.479438+0000","last_became_peered":"2024-03-04T07:32:09.479438+0000","last_unstale":"2024-03-04T07:32:10.816900+0000","last_undegraded":"2024-03-04T07:32:10.816900+0000","last_fullsized":"2024-03-04T07:32:10.816900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:09:39.427339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.32","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816577+0000","last_change":"2024-03-04T07:32:09.477524+0000","last_active":"2024-03-04T07:32:10.816577+0000","last_peered":"2024-03-04T07:32:10.816577+0000","last_clean":"2024-03-04T07:32:10.816577+0000","last_became_active":"2024-03-04T07:32:09.477240+0000","last_became_peered":"2024-03-04T07:32:09.477240+0000","last_unstale":"2024-03-04T07:32:10.816577+0000","last_undegraded":"2024-03-04T07:32:10.816577+0000","last_fullsized":"2024-03-04T07:32:10.816577+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:34.767357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.31","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.481212+0000","last_change":"2024-03-04T07:32:09.480382+0000","last_active":"2024-03-04T07:32:09.481212+0000","last_peered":"2024-03-04T07:32:09.481212+0000","last_clean":"2024-03-04T07:32:09.481212+0000","last_became_active":"2024-03-04T07:32:09.479792+0000","last_became_peered":"2024-03-04T07:32:09.479792+0000","last_unstale":"2024-03-04T07:32:09.481212+0000","last_undegraded":"2024-03-04T07:32:09.481212+0000","last_fullsized":"2024-03-04T07:32:09.481212+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:14:00.917239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.30","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477552+0000","last_change":"2024-03-04T07:32:09.485142+0000","last_active":"2024-03-04T07:32:10.477552+0000","last_peered":"2024-03-04T07:32:10.477552+0000","last_clean":"2024-03-04T07:32:10.477552+0000","last_became_active":"2024-03-04T07:32:09.484852+0000","last_became_peered":"2024-03-04T07:32:09.484852+0000","last_unstale":"2024-03-04T07:32:10.477552+0000","last_undegraded":"2024-03-04T07:32:10.477552+0000","last_fullsized":"2024-03-04T07:32:10.477552+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:46:55.124028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.2f","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479382+0000","last_change":"2024-03-04T07:32:09.486859+0000","last_active":"2024-03-04T07:32:10.479382+0000","last_peered":"2024-03-04T07:32:10.479382+0000","last_clean":"2024-03-04T07:32:10.479382+0000","last_became_active":"2024-03-04T07:32:09.486420+0000","last_became_peered":"2024-03-04T07:32:09.486420+0000","last_unstale":"2024-03-04T07:32:10.479382+0000","last_undegraded":"2024-03-04T07:32:10.479382+0000","last_fullsized":"2024-03-04T07:32:10.479382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:40.280836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2e","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487416+0000","last_change":"2024-03-04T07:32:09.481216+0000","last_active":"2024-03-04T07:32:10.487416+0000","last_peered":"2024-03-04T07:32:10.487416+0000","last_clean":"2024-03-04T07:32:10.487416+0000","last_became_active":"2024-03-04T07:32:09.480891+0000","last_became_peered":"2024-03-04T07:32:09.480891+0000","last_unstale":"2024-03-04T07:32:10.487416+0000","last_undegraded":"2024-03-04T07:32:10.487416+0000","last_fullsized":"2024-03-04T07:32:10.487416+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:26:43.616896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.2d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816922+0000","last_change":"2024-03-04T07:32:09.478968+0000","last_active":"2024-03-04T07:32:10.816922+0000","last_peered":"2024-03-04T07:32:10.816922+0000","last_clean":"2024-03-04T07:32:10.816922+0000","last_became_active":"2024-03-04T07:32:09.478586+0000","last_became_peered":"2024-03-04T07:32:09.478586+0000","last_unstale":"2024-03-04T07:32:10.816922+0000","last_undegraded":"2024-03-04T07:32:10.816922+0000","last_fullsized":"2024-03-04T07:32:10.816922+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:14:14.305392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.2c","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479440+0000","last_change":"2024-03-04T07:32:09.484194+0000","last_active":"2024-03-04T07:32:10.479440+0000","last_peered":"2024-03-04T07:32:10.479440+0000","last_clean":"2024-03-04T07:32:10.479440+0000","last_became_active":"2024-03-04T07:32:09.483823+0000","last_became_peered":"2024-03-04T07:32:09.483823+0000","last_unstale":"2024-03-04T07:32:10.479440+0000","last_undegraded":"2024-03-04T07:32:10.479440+0000","last_fullsized":"2024-03-04T07:32:10.479440+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:16.159999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2b","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478862+0000","last_change":"2024-03-04T07:32:09.481973+0000","last_active":"2024-03-04T07:32:10.478862+0000","last_peered":"2024-03-04T07:32:10.478862+0000","last_clean":"2024-03-04T07:32:10.478862+0000","last_became_active":"2024-03-04T07:32:09.480192+0000","last_became_peered":"2024-03-04T07:32:09.480192+0000","last_unstale":"2024-03-04T07:32:10.478862+0000","last_undegraded":"2024-03-04T07:32:10.478862+0000","last_fullsized":"2024-03-04T07:32:10.478862+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:52.510635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2a","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483387+0000","last_change":"2024-03-04T07:32:09.482308+0000","last_active":"2024-03-04T07:32:09.483387+0000","last_peered":"2024-03-04T07:32:09.483387+0000","last_clean":"2024-03-04T07:32:09.483387+0000","last_became_active":"2024-03-04T07:32:09.482066+0000","last_became_peered":"2024-03-04T07:32:09.482066+0000","last_unstale":"2024-03-04T07:32:09.483387+0000","last_undegraded":"2024-03-04T07:32:09.483387+0000","last_fullsized":"2024-03-04T07:32:09.483387+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:26:43.616896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.29","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478292+0000","last_change":"2024-03-04T07:32:09.484033+0000","last_active":"2024-03-04T07:32:10.478292+0000","last_peered":"2024-03-04T07:32:10.478292+0000","last_clean":"2024-03-04T07:32:10.478292+0000","last_became_active":"2024-03-04T07:32:09.483135+0000","last_became_peered":"2024-03-04T07:32:09.483135+0000","last_unstale":"2024-03-04T07:32:10.478292+0000","last_undegraded":"2024-03-04T07:32:10.478292+0000","last_fullsized":"2024-03-04T07:32:10.478292+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:26:43.616896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.28","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487752+0000","last_change":"2024-03-04T07:32:09.483971+0000","last_active":"2024-03-04T07:32:09.487752+0000","last_peered":"2024-03-04T07:32:09.487752+0000","last_clean":"2024-03-04T07:32:09.487752+0000","last_became_active":"2024-03-04T07:32:09.483659+0000","last_became_peered":"2024-03-04T07:32:09.483659+0000","last_unstale":"2024-03-04T07:32:09.487752+0000","last_undegraded":"2024-03-04T07:32:09.487752+0000","last_fullsized":"2024-03-04T07:32:09.487752+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:52.510635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.27","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479622+0000","last_change":"2024-03-04T07:32:09.481809+0000","last_active":"2024-03-04T07:32:10.479622+0000","last_peered":"2024-03-04T07:32:10.479622+0000","last_clean":"2024-03-04T07:32:10.479622+0000","last_became_active":"2024-03-04T07:32:09.481438+0000","last_became_peered":"2024-03-04T07:32:09.481438+0000","last_unstale":"2024-03-04T07:32:10.479622+0000","last_undegraded":"2024-03-04T07:32:10.479622+0000","last_fullsized":"2024-03-04T07:32:10.479622+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:21:33.697195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.26","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477981+0000","last_change":"2024-03-04T07:32:09.486669+0000","last_active":"2024-03-04T07:32:10.477981+0000","last_peered":"2024-03-04T07:32:10.477981+0000","last_clean":"2024-03-04T07:32:10.477981+0000","last_became_active":"2024-03-04T07:32:09.486394+0000","last_became_peered":"2024-03-04T07:32:09.486394+0000","last_unstale":"2024-03-04T07:32:10.477981+0000","last_undegraded":"2024-03-04T07:32:10.477981+0000","last_fullsized":"2024-03-04T07:32:10.477981+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:48:25.167596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.25","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483038+0000","last_change":"2024-03-04T07:32:09.480769+0000","last_active":"2024-03-04T07:32:09.483038+0000","last_peered":"2024-03-04T07:32:09.483038+0000","last_clean":"2024-03-04T07:32:09.483038+0000","last_became_active":"2024-03-04T07:32:09.480486+0000","last_became_peered":"2024-03-04T07:32:09.480486+0000","last_unstale":"2024-03-04T07:32:09.483038+0000","last_undegraded":"2024-03-04T07:32:09.483038+0000","last_fullsized":"2024-03-04T07:32:09.483038+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:56.215072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.24","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.488272+0000","last_change":"2024-03-04T07:32:09.486862+0000","last_active":"2024-03-04T07:32:09.488272+0000","last_peered":"2024-03-04T07:32:09.488272+0000","last_clean":"2024-03-04T07:32:09.488272+0000","last_became_active":"2024-03-04T07:32:09.486596+0000","last_became_peered":"2024-03-04T07:32:09.486596+0000","last_unstale":"2024-03-04T07:32:09.488272+0000","last_undegraded":"2024-03-04T07:32:09.488272+0000","last_fullsized":"2024-03-04T07:32:09.488272+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:16.159999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.23","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478261+0000","last_change":"2024-03-04T07:32:09.483856+0000","last_active":"2024-03-04T07:32:10.478261+0000","last_peered":"2024-03-04T07:32:10.478261+0000","last_clean":"2024-03-04T07:32:10.478261+0000","last_became_active":"2024-03-04T07:32:09.483548+0000","last_became_peered":"2024-03-04T07:32:09.483548+0000","last_unstale":"2024-03-04T07:32:10.478261+0000","last_undegraded":"2024-03-04T07:32:10.478261+0000","last_fullsized":"2024-03-04T07:32:10.478261+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:36.566044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.22","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478703+0000","last_change":"2024-03-04T07:32:09.483147+0000","last_active":"2024-03-04T07:32:10.478703+0000","last_peered":"2024-03-04T07:32:10.478703+0000","last_clean":"2024-03-04T07:32:10.478703+0000","last_became_active":"2024-03-04T07:32:09.482840+0000","last_became_peered":"2024-03-04T07:32:09.482840+0000","last_unstale":"2024-03-04T07:32:10.478703+0000","last_undegraded":"2024-03-04T07:32:10.478703+0000","last_fullsized":"2024-03-04T07:32:10.478703+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:47:59.291464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.21","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477686+0000","last_change":"2024-03-04T07:32:09.486523+0000","last_active":"2024-03-04T07:32:10.477686+0000","last_peered":"2024-03-04T07:32:10.477686+0000","last_clean":"2024-03-04T07:32:10.477686+0000","last_became_active":"2024-03-04T07:32:09.486243+0000","last_became_peered":"2024-03-04T07:32:09.486243+0000","last_unstale":"2024-03-04T07:32:10.477686+0000","last_undegraded":"2024-03-04T07:32:10.477686+0000","last_fullsized":"2024-03-04T07:32:10.477686+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:13.248314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.20","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.481526+0000","last_change":"2024-03-04T07:32:09.479221+0000","last_active":"2024-03-04T07:32:09.481526+0000","last_peered":"2024-03-04T07:32:09.481526+0000","last_clean":"2024-03-04T07:32:09.481526+0000","last_became_active":"2024-03-04T07:32:09.478921+0000","last_became_peered":"2024-03-04T07:32:09.478921+0000","last_unstale":"2024-03-04T07:32:09.481526+0000","last_undegraded":"2024-03-04T07:32:09.481526+0000","last_fullsized":"2024-03-04T07:32:09.481526+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:17.338651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.1e","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.478855+0000","last_change":"2024-03-04T07:32:09.477807+0000","last_active":"2024-03-04T07:32:09.478855+0000","last_peered":"2024-03-04T07:32:09.478855+0000","last_clean":"2024-03-04T07:32:09.478855+0000","last_became_active":"2024-03-04T07:32:09.477524+0000","last_became_peered":"2024-03-04T07:32:09.477524+0000","last_unstale":"2024-03-04T07:32:09.478855+0000","last_undegraded":"2024-03-04T07:32:09.478855+0000","last_fullsized":"2024-03-04T07:32:09.478855+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:54.885673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.1f","version":"104'4","reported_seq":18,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478685+0000","last_change":"2024-03-04T07:32:08.466784+0000","last_active":"2024-03-04T07:32:10.478685+0000","last_peered":"2024-03-04T07:32:10.478685+0000","last_clean":"2024-03-04T07:32:10.478685+0000","last_became_active":"2024-03-04T07:32:08.466466+0000","last_became_peered":"2024-03-04T07:32:08.466466+0000","last_unstale":"2024-03-04T07:32:10.478685+0000","last_undegraded":"2024-03-04T07:32:10.478685+0000","last_fullsized":"2024-03-04T07:32:10.478685+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:12:37.166337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1f","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487677+0000","last_change":"2024-03-04T07:32:09.476421+0000","last_active":"2024-03-04T07:32:10.487677+0000","last_peered":"2024-03-04T07:32:10.487677+0000","last_clean":"2024-03-04T07:32:10.487677+0000","last_became_active":"2024-03-04T07:32:09.476142+0000","last_became_peered":"2024-03-04T07:32:09.476142+0000","last_unstale":"2024-03-04T07:32:10.487677+0000","last_undegraded":"2024-03-04T07:32:10.487677+0000","last_fullsized":"2024-03-04T07:32:10.487677+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:35.891031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.1e","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816856+0000","last_change":"2024-03-04T07:32:08.470198+0000","last_active":"2024-03-04T07:32:10.816856+0000","last_peered":"2024-03-04T07:32:10.816856+0000","last_clean":"2024-03-04T07:32:10.816856+0000","last_became_active":"2024-03-04T07:32:08.469687+0000","last_became_peered":"2024-03-04T07:32:08.469687+0000","last_unstale":"2024-03-04T07:32:10.816856+0000","last_undegraded":"2024-03-04T07:32:10.816856+0000","last_fullsized":"2024-03-04T07:32:10.816856+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:15.909283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.1c","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.480680+0000","last_change":"2024-03-04T07:32:09.479162+0000","last_active":"2024-03-04T07:32:09.480680+0000","last_peered":"2024-03-04T07:32:09.480680+0000","last_clean":"2024-03-04T07:32:09.480680+0000","last_became_active":"2024-03-04T07:32:09.478879+0000","last_became_peered":"2024-03-04T07:32:09.478879+0000","last_unstale":"2024-03-04T07:32:09.480680+0000","last_undegraded":"2024-03-04T07:32:09.480680+0000","last_fullsized":"2024-03-04T07:32:09.480680+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:07:07.769899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.1d","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496495+0000","last_change":"2024-03-04T07:32:08.468670+0000","last_active":"2024-03-04T07:32:09.496495+0000","last_peered":"2024-03-04T07:32:09.496495+0000","last_clean":"2024-03-04T07:32:09.496495+0000","last_became_active":"2024-03-04T07:32:08.468381+0000","last_became_peered":"2024-03-04T07:32:08.468381+0000","last_unstale":"2024-03-04T07:32:09.496495+0000","last_undegraded":"2024-03-04T07:32:09.496495+0000","last_fullsized":"2024-03-04T07:32:09.496495+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:50.857896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.817015+0000","last_change":"2024-03-04T07:32:09.478218+0000","last_active":"2024-03-04T07:32:10.817015+0000","last_peered":"2024-03-04T07:32:10.817015+0000","last_clean":"2024-03-04T07:32:10.817015+0000","last_became_active":"2024-03-04T07:32:09.477770+0000","last_became_peered":"2024-03-04T07:32:09.477770+0000","last_unstale":"2024-03-04T07:32:10.817015+0000","last_undegraded":"2024-03-04T07:32:10.817015+0000","last_fullsized":"2024-03-04T07:32:10.817015+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:00:12.230203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.1c","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496625+0000","last_change":"2024-03-04T07:32:08.467300+0000","last_active":"2024-03-04T07:32:09.496625+0000","last_peered":"2024-03-04T07:32:09.496625+0000","last_clean":"2024-03-04T07:32:09.496625+0000","last_became_active":"2024-03-04T07:32:08.466925+0000","last_became_peered":"2024-03-04T07:32:08.466925+0000","last_unstale":"2024-03-04T07:32:09.496625+0000","last_undegraded":"2024-03-04T07:32:09.496625+0000","last_fullsized":"2024-03-04T07:32:09.496625+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:42:14.289123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1a","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477884+0000","last_change":"2024-03-04T07:32:09.479579+0000","last_active":"2024-03-04T07:32:10.477884+0000","last_peered":"2024-03-04T07:32:10.477884+0000","last_clean":"2024-03-04T07:32:10.477884+0000","last_became_active":"2024-03-04T07:32:09.479313+0000","last_became_peered":"2024-03-04T07:32:09.479313+0000","last_unstale":"2024-03-04T07:32:10.477884+0000","last_undegraded":"2024-03-04T07:32:10.477884+0000","last_fullsized":"2024-03-04T07:32:10.477884+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:02:28.339354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.1b","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478316+0000","last_change":"2024-03-04T07:32:08.467507+0000","last_active":"2024-03-04T07:32:10.478316+0000","last_peered":"2024-03-04T07:32:10.478316+0000","last_clean":"2024-03-04T07:32:10.478316+0000","last_became_active":"2024-03-04T07:32:08.467200+0000","last_became_peered":"2024-03-04T07:32:08.467200+0000","last_unstale":"2024-03-04T07:32:10.478316+0000","last_undegraded":"2024-03-04T07:32:10.478316+0000","last_fullsized":"2024-03-04T07:32:10.478316+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:53.879250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1b","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478436+0000","last_change":"2024-03-04T07:32:09.482884+0000","last_active":"2024-03-04T07:32:10.478436+0000","last_peered":"2024-03-04T07:32:10.478436+0000","last_clean":"2024-03-04T07:32:10.478436+0000","last_became_active":"2024-03-04T07:32:09.482278+0000","last_became_peered":"2024-03-04T07:32:09.482278+0000","last_unstale":"2024-03-04T07:32:10.478436+0000","last_undegraded":"2024-03-04T07:32:10.478436+0000","last_fullsized":"2024-03-04T07:32:10.478436+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:02.819545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.1a","version":"104'1","reported_seq":13,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.494698+0000","last_change":"2024-03-04T07:32:08.469580+0000","last_active":"2024-03-04T07:32:09.494698+0000","last_peered":"2024-03-04T07:32:09.494698+0000","last_clean":"2024-03-04T07:32:09.494698+0000","last_became_active":"2024-03-04T07:32:08.469315+0000","last_became_peered":"2024-03-04T07:32:08.469315+0000","last_unstale":"2024-03-04T07:32:09.494698+0000","last_undegraded":"2024-03-04T07:32:09.494698+0000","last_fullsized":"2024-03-04T07:32:09.494698+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:26:15.372360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.18","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477960+0000","last_change":"2024-03-04T07:32:09.480285+0000","last_active":"2024-03-04T07:32:10.477960+0000","last_peered":"2024-03-04T07:32:10.477960+0000","last_clean":"2024-03-04T07:32:10.477960+0000","last_became_active":"2024-03-04T07:32:09.479978+0000","last_became_peered":"2024-03-04T07:32:09.479978+0000","last_unstale":"2024-03-04T07:32:10.477960+0000","last_undegraded":"2024-03-04T07:32:10.477960+0000","last_fullsized":"2024-03-04T07:32:10.477960+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:16:33.783469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.19","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478659+0000","last_change":"2024-03-04T07:32:08.467273+0000","last_active":"2024-03-04T07:32:10.478659+0000","last_peered":"2024-03-04T07:32:10.478659+0000","last_clean":"2024-03-04T07:32:10.478659+0000","last_became_active":"2024-03-04T07:32:08.466986+0000","last_became_peered":"2024-03-04T07:32:08.466986+0000","last_unstale":"2024-03-04T07:32:10.478659+0000","last_undegraded":"2024-03-04T07:32:10.478659+0000","last_fullsized":"2024-03-04T07:32:10.478659+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:53:40.996672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.19","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478589+0000","last_change":"2024-03-04T07:32:09.469423+0000","last_active":"2024-03-04T07:32:10.478589+0000","last_peered":"2024-03-04T07:32:10.478589+0000","last_clean":"2024-03-04T07:32:10.478589+0000","last_became_active":"2024-03-04T07:32:09.468524+0000","last_became_peered":"2024-03-04T07:32:09.468524+0000","last_unstale":"2024-03-04T07:32:10.478589+0000","last_undegraded":"2024-03-04T07:32:10.478589+0000","last_fullsized":"2024-03-04T07:32:10.478589+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:31:57.163635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.18","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477648+0000","last_change":"2024-03-04T07:32:08.460895+0000","last_active":"2024-03-04T07:32:10.477648+0000","last_peered":"2024-03-04T07:32:10.477648+0000","last_clean":"2024-03-04T07:32:10.477648+0000","last_became_active":"2024-03-04T07:32:08.460607+0000","last_became_peered":"2024-03-04T07:32:08.460607+0000","last_unstale":"2024-03-04T07:32:10.477648+0000","last_undegraded":"2024-03-04T07:32:10.477648+0000","last_fullsized":"2024-03-04T07:32:10.477648+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:34.884607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.16","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478849+0000","last_change":"2024-03-04T07:32:09.479090+0000","last_active":"2024-03-04T07:32:10.478849+0000","last_peered":"2024-03-04T07:32:10.478849+0000","last_clean":"2024-03-04T07:32:10.478849+0000","last_became_active":"2024-03-04T07:32:09.478810+0000","last_became_peered":"2024-03-04T07:32:09.478810+0000","last_unstale":"2024-03-04T07:32:10.478849+0000","last_undegraded":"2024-03-04T07:32:10.478849+0000","last_fullsized":"2024-03-04T07:32:10.478849+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:29:02.463565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.17","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472287+0000","last_change":"2024-03-04T07:32:08.468765+0000","last_active":"2024-03-04T07:32:09.472287+0000","last_peered":"2024-03-04T07:32:09.472287+0000","last_clean":"2024-03-04T07:32:09.472287+0000","last_became_active":"2024-03-04T07:32:08.467960+0000","last_became_peered":"2024-03-04T07:32:08.467960+0000","last_unstale":"2024-03-04T07:32:09.472287+0000","last_undegraded":"2024-03-04T07:32:09.472287+0000","last_fullsized":"2024-03-04T07:32:09.472287+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:05:56.623034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.17","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487692+0000","last_change":"2024-03-04T07:32:09.474815+0000","last_active":"2024-03-04T07:32:10.487692+0000","last_peered":"2024-03-04T07:32:10.487692+0000","last_clean":"2024-03-04T07:32:10.487692+0000","last_became_active":"2024-03-04T07:32:09.474512+0000","last_became_peered":"2024-03-04T07:32:09.474512+0000","last_unstale":"2024-03-04T07:32:10.487692+0000","last_undegraded":"2024-03-04T07:32:10.487692+0000","last_fullsized":"2024-03-04T07:32:10.487692+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:52.510635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.16","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472660+0000","last_change":"2024-03-04T07:32:08.470330+0000","last_active":"2024-03-04T07:32:09.472660+0000","last_peered":"2024-03-04T07:32:09.472660+0000","last_clean":"2024-03-04T07:32:09.472660+0000","last_became_active":"2024-03-04T07:32:08.470019+0000","last_became_peered":"2024-03-04T07:32:08.470019+0000","last_unstale":"2024-03-04T07:32:09.472660+0000","last_undegraded":"2024-03-04T07:32:09.472660+0000","last_fullsized":"2024-03-04T07:32:09.472660+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:17:44.711098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.14","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483393+0000","last_change":"2024-03-04T07:32:09.481767+0000","last_active":"2024-03-04T07:32:09.483393+0000","last_peered":"2024-03-04T07:32:09.483393+0000","last_clean":"2024-03-04T07:32:09.483393+0000","last_became_active":"2024-03-04T07:32:09.481480+0000","last_became_peered":"2024-03-04T07:32:09.481480+0000","last_unstale":"2024-03-04T07:32:09.483393+0000","last_undegraded":"2024-03-04T07:32:09.483393+0000","last_fullsized":"2024-03-04T07:32:09.483393+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:36.566044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.15","version":"104'3","reported_seq":17,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478157+0000","last_change":"2024-03-04T07:32:08.467932+0000","last_active":"2024-03-04T07:32:10.478157+0000","last_peered":"2024-03-04T07:32:10.478157+0000","last_clean":"2024-03-04T07:32:10.478157+0000","last_became_active":"2024-03-04T07:32:08.467644+0000","last_became_peered":"2024-03-04T07:32:08.467644+0000","last_unstale":"2024-03-04T07:32:10.478157+0000","last_undegraded":"2024-03-04T07:32:10.478157+0000","last_fullsized":"2024-03-04T07:32:10.478157+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:27.356974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.15","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478559+0000","last_change":"2024-03-04T07:32:09.484013+0000","last_active":"2024-03-04T07:32:10.478559+0000","last_peered":"2024-03-04T07:32:10.478559+0000","last_clean":"2024-03-04T07:32:10.478559+0000","last_became_active":"2024-03-04T07:32:09.483664+0000","last_became_peered":"2024-03-04T07:32:09.483664+0000","last_unstale":"2024-03-04T07:32:10.478559+0000","last_undegraded":"2024-03-04T07:32:10.478559+0000","last_fullsized":"2024-03-04T07:32:10.478559+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:20:14.175037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.14","version":"104'6","reported_seq":20,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478146+0000","last_change":"2024-03-04T07:32:08.460224+0000","last_active":"2024-03-04T07:32:10.478146+0000","last_peered":"2024-03-04T07:32:10.478146+0000","last_clean":"2024-03-04T07:32:10.478146+0000","last_became_active":"2024-03-04T07:32:08.459945+0000","last_became_peered":"2024-03-04T07:32:08.459945+0000","last_unstale":"2024-03-04T07:32:10.478146+0000","last_undegraded":"2024-03-04T07:32:10.478146+0000","last_fullsized":"2024-03-04T07:32:10.478146+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:33.760934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.12","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487820+0000","last_change":"2024-03-04T07:32:09.485082+0000","last_active":"2024-03-04T07:32:09.487820+0000","last_peered":"2024-03-04T07:32:09.487820+0000","last_clean":"2024-03-04T07:32:09.487820+0000","last_became_active":"2024-03-04T07:32:09.484799+0000","last_became_peered":"2024-03-04T07:32:09.484799+0000","last_unstale":"2024-03-04T07:32:09.487820+0000","last_undegraded":"2024-03-04T07:32:09.487820+0000","last_fullsized":"2024-03-04T07:32:09.487820+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:28.363398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.13","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.490970+0000","last_change":"2024-03-04T07:32:08.466321+0000","last_active":"2024-03-04T07:32:09.490970+0000","last_peered":"2024-03-04T07:32:09.490970+0000","last_clean":"2024-03-04T07:32:09.490970+0000","last_became_active":"2024-03-04T07:32:08.466057+0000","last_became_peered":"2024-03-04T07:32:08.466057+0000","last_unstale":"2024-03-04T07:32:09.490970+0000","last_undegraded":"2024-03-04T07:32:09.490970+0000","last_fullsized":"2024-03-04T07:32:09.490970+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:41.701610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.13","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487848+0000","last_change":"2024-03-04T07:32:09.483893+0000","last_active":"2024-03-04T07:32:09.487848+0000","last_peered":"2024-03-04T07:32:09.487848+0000","last_clean":"2024-03-04T07:32:09.487848+0000","last_became_active":"2024-03-04T07:32:09.483546+0000","last_became_peered":"2024-03-04T07:32:09.483546+0000","last_unstale":"2024-03-04T07:32:09.487848+0000","last_undegraded":"2024-03-04T07:32:09.487848+0000","last_fullsized":"2024-03-04T07:32:09.487848+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:35.891031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.12","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487226+0000","last_change":"2024-03-04T07:32:08.471165+0000","last_active":"2024-03-04T07:32:10.487226+0000","last_peered":"2024-03-04T07:32:10.487226+0000","last_clean":"2024-03-04T07:32:10.487226+0000","last_became_active":"2024-03-04T07:32:08.470903+0000","last_became_peered":"2024-03-04T07:32:08.470903+0000","last_unstale":"2024-03-04T07:32:10.487226+0000","last_undegraded":"2024-03-04T07:32:10.487226+0000","last_fullsized":"2024-03-04T07:32:10.487226+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:15.909283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.11","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.484100+0000","last_change":"2024-03-04T07:32:09.483142+0000","last_active":"2024-03-04T07:32:09.484100+0000","last_peered":"2024-03-04T07:32:09.484100+0000","last_clean":"2024-03-04T07:32:09.484100+0000","last_became_active":"2024-03-04T07:32:09.482857+0000","last_became_peered":"2024-03-04T07:32:09.482857+0000","last_unstale":"2024-03-04T07:32:09.484100+0000","last_undegraded":"2024-03-04T07:32:09.484100+0000","last_fullsized":"2024-03-04T07:32:09.484100+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:02.819545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.10","version":"104'8","reported_seq":20,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.498060+0000","last_change":"2024-03-04T07:32:08.463643+0000","last_active":"2024-03-04T07:32:09.498060+0000","last_peered":"2024-03-04T07:32:09.498060+0000","last_clean":"2024-03-04T07:32:09.498060+0000","last_became_active":"2024-03-04T07:32:08.463174+0000","last_became_peered":"2024-03-04T07:32:08.463174+0000","last_unstale":"2024-03-04T07:32:09.498060+0000","last_undegraded":"2024-03-04T07:32:09.498060+0000","last_fullsized":"2024-03-04T07:32:09.498060+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:56:46.978000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.e","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.488894+0000","last_change":"2024-03-04T07:32:09.488233+0000","last_active":"2024-03-04T07:32:09.488894+0000","last_peered":"2024-03-04T07:32:09.488894+0000","last_clean":"2024-03-04T07:32:09.488894+0000","last_became_active":"2024-03-04T07:32:09.487896+0000","last_became_peered":"2024-03-04T07:32:09.487896+0000","last_unstale":"2024-03-04T07:32:09.488894+0000","last_undegraded":"2024-03-04T07:32:09.488894+0000","last_fullsized":"2024-03-04T07:32:09.488894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:45:33.189423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"16.f","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478587+0000","last_change":"2024-03-04T07:32:08.468285+0000","last_active":"2024-03-04T07:32:10.478587+0000","last_peered":"2024-03-04T07:32:10.478587+0000","last_clean":"2024-03-04T07:32:10.478587+0000","last_became_active":"2024-03-04T07:32:08.468026+0000","last_became_peered":"2024-03-04T07:32:08.468026+0000","last_unstale":"2024-03-04T07:32:10.478587+0000","last_undegraded":"2024-03-04T07:32:10.478587+0000","last_fullsized":"2024-03-04T07:32:10.478587+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:15.153575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.f","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483843+0000","last_change":"2024-03-04T07:32:09.481192+0000","last_active":"2024-03-04T07:32:09.483843+0000","last_peered":"2024-03-04T07:32:09.483843+0000","last_clean":"2024-03-04T07:32:09.483843+0000","last_became_active":"2024-03-04T07:32:09.480038+0000","last_became_peered":"2024-03-04T07:32:09.480038+0000","last_unstale":"2024-03-04T07:32:09.483843+0000","last_undegraded":"2024-03-04T07:32:09.483843+0000","last_fullsized":"2024-03-04T07:32:09.483843+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:56:49.837533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.e","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496586+0000","last_change":"2024-03-04T07:32:08.463461+0000","last_active":"2024-03-04T07:32:09.496586+0000","last_peered":"2024-03-04T07:32:09.496586+0000","last_clean":"2024-03-04T07:32:09.496586+0000","last_became_active":"2024-03-04T07:32:08.463191+0000","last_became_peered":"2024-03-04T07:32:08.463191+0000","last_unstale":"2024-03-04T07:32:09.496586+0000","last_undegraded":"2024-03-04T07:32:09.496586+0000","last_fullsized":"2024-03-04T07:32:09.496586+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:31:36.255777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.c","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816821+0000","last_change":"2024-03-04T07:32:09.491556+0000","last_active":"2024-03-04T07:32:10.816821+0000","last_peered":"2024-03-04T07:32:10.816821+0000","last_clean":"2024-03-04T07:32:10.816821+0000","last_became_active":"2024-03-04T07:32:09.491077+0000","last_became_peered":"2024-03-04T07:32:09.491077+0000","last_unstale":"2024-03-04T07:32:10.816821+0000","last_undegraded":"2024-03-04T07:32:10.816821+0000","last_fullsized":"2024-03-04T07:32:10.816821+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:14:25.586669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.d","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487411+0000","last_change":"2024-03-04T07:32:08.471936+0000","last_active":"2024-03-04T07:32:10.487411+0000","last_peered":"2024-03-04T07:32:10.487411+0000","last_clean":"2024-03-04T07:32:10.487411+0000","last_became_active":"2024-03-04T07:32:08.471591+0000","last_became_peered":"2024-03-04T07:32:08.471591+0000","last_unstale":"2024-03-04T07:32:10.487411+0000","last_undegraded":"2024-03-04T07:32:10.487411+0000","last_fullsized":"2024-03-04T07:32:10.487411+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:03:12.593774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.817052+0000","last_change":"2024-03-04T07:32:09.488979+0000","last_active":"2024-03-04T07:32:10.817052+0000","last_peered":"2024-03-04T07:32:10.817052+0000","last_clean":"2024-03-04T07:32:10.817052+0000","last_became_active":"2024-03-04T07:32:09.488455+0000","last_became_peered":"2024-03-04T07:32:09.488455+0000","last_unstale":"2024-03-04T07:32:10.817052+0000","last_undegraded":"2024-03-04T07:32:10.817052+0000","last_fullsized":"2024-03-04T07:32:10.817052+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:06:33.445992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.c","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478409+0000","last_change":"2024-03-04T07:32:08.469020+0000","last_active":"2024-03-04T07:32:10.478409+0000","last_peered":"2024-03-04T07:32:10.478409+0000","last_clean":"2024-03-04T07:32:10.478409+0000","last_became_active":"2024-03-04T07:32:08.468735+0000","last_became_peered":"2024-03-04T07:32:08.468735+0000","last_unstale":"2024-03-04T07:32:10.478409+0000","last_undegraded":"2024-03-04T07:32:10.478409+0000","last_fullsized":"2024-03-04T07:32:10.478409+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:58.286990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.a","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816581+0000","last_change":"2024-03-04T07:32:09.486195+0000","last_active":"2024-03-04T07:32:10.816581+0000","last_peered":"2024-03-04T07:32:10.816581+0000","last_clean":"2024-03-04T07:32:10.816581+0000","last_became_active":"2024-03-04T07:32:09.485790+0000","last_became_peered":"2024-03-04T07:32:09.485790+0000","last_unstale":"2024-03-04T07:32:10.816581+0000","last_undegraded":"2024-03-04T07:32:10.816581+0000","last_fullsized":"2024-03-04T07:32:10.816581+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:28.363398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.b","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496844+0000","last_change":"2024-03-04T07:32:08.471852+0000","last_active":"2024-03-04T07:32:09.496844+0000","last_peered":"2024-03-04T07:32:09.496844+0000","last_clean":"2024-03-04T07:32:09.496844+0000","last_became_active":"2024-03-04T07:32:08.471494+0000","last_became_peered":"2024-03-04T07:32:08.471494+0000","last_unstale":"2024-03-04T07:32:09.496844+0000","last_undegraded":"2024-03-04T07:32:09.496844+0000","last_fullsized":"2024-03-04T07:32:09.496844+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:58.286990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.b","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477900+0000","last_change":"2024-03-04T07:32:09.485410+0000","last_active":"2024-03-04T07:32:10.477900+0000","last_peered":"2024-03-04T07:32:10.477900+0000","last_clean":"2024-03-04T07:32:10.477900+0000","last_became_active":"2024-03-04T07:32:09.484796+0000","last_became_peered":"2024-03-04T07:32:09.484796+0000","last_unstale":"2024-03-04T07:32:10.477900+0000","last_undegraded":"2024-03-04T07:32:10.477900+0000","last_fullsized":"2024-03-04T07:32:10.477900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:17:46.919044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.a","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.473013+0000","last_change":"2024-03-04T07:32:08.468140+0000","last_active":"2024-03-04T07:32:09.473013+0000","last_peered":"2024-03-04T07:32:09.473013+0000","last_clean":"2024-03-04T07:32:09.473013+0000","last_became_active":"2024-03-04T07:32:08.467803+0000","last_became_peered":"2024-03-04T07:32:08.467803+0000","last_unstale":"2024-03-04T07:32:09.473013+0000","last_undegraded":"2024-03-04T07:32:09.473013+0000","last_fullsized":"2024-03-04T07:32:09.473013+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:46:11.504981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.8","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816610+0000","last_change":"2024-03-04T07:32:09.480856+0000","last_active":"2024-03-04T07:32:10.816610+0000","last_peered":"2024-03-04T07:32:10.816610+0000","last_clean":"2024-03-04T07:32:10.816610+0000","last_became_active":"2024-03-04T07:32:09.480614+0000","last_became_peered":"2024-03-04T07:32:09.480614+0000","last_unstale":"2024-03-04T07:32:10.816610+0000","last_undegraded":"2024-03-04T07:32:10.816610+0000","last_fullsized":"2024-03-04T07:32:10.816610+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:16.159999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.9","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472622+0000","last_change":"2024-03-04T07:32:08.465812+0000","last_active":"2024-03-04T07:32:09.472622+0000","last_peered":"2024-03-04T07:32:09.472622+0000","last_clean":"2024-03-04T07:32:09.472622+0000","last_became_active":"2024-03-04T07:32:08.465479+0000","last_became_peered":"2024-03-04T07:32:08.465479+0000","last_unstale":"2024-03-04T07:32:09.472622+0000","last_undegraded":"2024-03-04T07:32:09.472622+0000","last_fullsized":"2024-03-04T07:32:09.472622+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:12:37.166337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.9","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487924+0000","last_change":"2024-03-04T07:32:09.485154+0000","last_active":"2024-03-04T07:32:09.487924+0000","last_peered":"2024-03-04T07:32:09.487924+0000","last_clean":"2024-03-04T07:32:09.487924+0000","last_became_active":"2024-03-04T07:32:09.484906+0000","last_became_peered":"2024-03-04T07:32:09.484906+0000","last_unstale":"2024-03-04T07:32:09.487924+0000","last_undegraded":"2024-03-04T07:32:09.487924+0000","last_fullsized":"2024-03-04T07:32:09.487924+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:40.280836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.8","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477740+0000","last_change":"2024-03-04T07:32:08.471088+0000","last_active":"2024-03-04T07:32:10.477740+0000","last_peered":"2024-03-04T07:32:10.477740+0000","last_clean":"2024-03-04T07:32:10.477740+0000","last_became_active":"2024-03-04T07:32:08.470466+0000","last_became_peered":"2024-03-04T07:32:08.470466+0000","last_unstale":"2024-03-04T07:32:10.477740+0000","last_undegraded":"2024-03-04T07:32:10.477740+0000","last_fullsized":"2024-03-04T07:32:10.477740+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:01.813121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.6","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487489+0000","last_change":"2024-03-04T07:32:09.472612+0000","last_active":"2024-03-04T07:32:10.487489+0000","last_peered":"2024-03-04T07:32:10.487489+0000","last_clean":"2024-03-04T07:32:10.487489+0000","last_became_active":"2024-03-04T07:32:09.472330+0000","last_became_peered":"2024-03-04T07:32:09.472330+0000","last_unstale":"2024-03-04T07:32:10.487489+0000","last_undegraded":"2024-03-04T07:32:10.487489+0000","last_fullsized":"2024-03-04T07:32:10.487489+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:02.819545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.7","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.497370+0000","last_change":"2024-03-04T07:32:08.465896+0000","last_active":"2024-03-04T07:32:09.497370+0000","last_peered":"2024-03-04T07:32:09.497370+0000","last_clean":"2024-03-04T07:32:09.497370+0000","last_became_active":"2024-03-04T07:32:08.465603+0000","last_became_peered":"2024-03-04T07:32:08.465603+0000","last_unstale":"2024-03-04T07:32:09.497370+0000","last_undegraded":"2024-03-04T07:32:09.497370+0000","last_fullsized":"2024-03-04T07:32:09.497370+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:33:41.109219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.7","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478510+0000","last_change":"2024-03-04T07:32:09.487556+0000","last_active":"2024-03-04T07:32:10.478510+0000","last_peered":"2024-03-04T07:32:10.478510+0000","last_clean":"2024-03-04T07:32:10.478510+0000","last_became_active":"2024-03-04T07:32:09.487179+0000","last_became_peered":"2024-03-04T07:32:09.487179+0000","last_unstale":"2024-03-04T07:32:10.478510+0000","last_undegraded":"2024-03-04T07:32:10.478510+0000","last_fullsized":"2024-03-04T07:32:10.478510+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:05:15.550074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.6","version":"105'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.489033+0000","last_change":"2024-03-04T07:32:08.470418+0000","last_active":"2024-03-04T07:32:10.489033+0000","last_peered":"2024-03-04T07:32:10.489033+0000","last_clean":"2024-03-04T07:32:10.489033+0000","last_became_active":"2024-03-04T07:32:08.470071+0000","last_became_peered":"2024-03-04T07:32:08.470071+0000","last_unstale":"2024-03-04T07:32:10.489033+0000","last_undegraded":"2024-03-04T07:32:10.489033+0000","last_fullsized":"2024-03-04T07:32:10.489033+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:10:46.660537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.4","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478120+0000","last_change":"2024-03-04T07:32:09.485079+0000","last_active":"2024-03-04T07:32:10.478120+0000","last_peered":"2024-03-04T07:32:10.478120+0000","last_clean":"2024-03-04T07:32:10.478120+0000","last_became_active":"2024-03-04T07:32:09.484278+0000","last_became_peered":"2024-03-04T07:32:09.484278+0000","last_unstale":"2024-03-04T07:32:10.478120+0000","last_undegraded":"2024-03-04T07:32:10.478120+0000","last_fullsized":"2024-03-04T07:32:10.478120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:31:49.227609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.5","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478213+0000","last_change":"2024-03-04T07:32:08.465563+0000","last_active":"2024-03-04T07:32:10.478213+0000","last_peered":"2024-03-04T07:32:10.478213+0000","last_clean":"2024-03-04T07:32:10.478213+0000","last_became_active":"2024-03-04T07:32:08.465302+0000","last_became_peered":"2024-03-04T07:32:08.465302+0000","last_unstale":"2024-03-04T07:32:10.478213+0000","last_undegraded":"2024-03-04T07:32:10.478213+0000","last_fullsized":"2024-03-04T07:32:10.478213+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:39.274412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.5","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487396+0000","last_change":"2024-03-04T07:32:09.471907+0000","last_active":"2024-03-04T07:32:10.487396+0000","last_peered":"2024-03-04T07:32:10.487396+0000","last_clean":"2024-03-04T07:32:10.487396+0000","last_became_active":"2024-03-04T07:32:09.471627+0000","last_became_peered":"2024-03-04T07:32:09.471627+0000","last_unstale":"2024-03-04T07:32:10.487396+0000","last_undegraded":"2024-03-04T07:32:10.487396+0000","last_fullsized":"2024-03-04T07:32:10.487396+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:36.566044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.4","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.473567+0000","last_change":"2024-03-04T07:32:08.469166+0000","last_active":"2024-03-04T07:32:09.473567+0000","last_peered":"2024-03-04T07:32:09.473567+0000","last_clean":"2024-03-04T07:32:09.473567+0000","last_became_active":"2024-03-04T07:32:08.468891+0000","last_became_peered":"2024-03-04T07:32:08.468891+0000","last_unstale":"2024-03-04T07:32:09.473567+0000","last_undegraded":"2024-03-04T07:32:09.473567+0000","last_fullsized":"2024-03-04T07:32:09.473567+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:46:12.882604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.2","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477995+0000","last_change":"2024-03-04T07:32:09.478932+0000","last_active":"2024-03-04T07:32:10.477995+0000","last_peered":"2024-03-04T07:32:10.477995+0000","last_clean":"2024-03-04T07:32:10.477995+0000","last_became_active":"2024-03-04T07:32:09.478632+0000","last_became_peered":"2024-03-04T07:32:09.478632+0000","last_unstale":"2024-03-04T07:32:10.477995+0000","last_undegraded":"2024-03-04T07:32:10.477995+0000","last_fullsized":"2024-03-04T07:32:10.477995+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:26:16.378784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.3","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478449+0000","last_change":"2024-03-04T07:32:08.463277+0000","last_active":"2024-03-04T07:32:10.478449+0000","last_peered":"2024-03-04T07:32:10.478449+0000","last_clean":"2024-03-04T07:32:10.478449+0000","last_became_active":"2024-03-04T07:32:08.462999+0000","last_became_peered":"2024-03-04T07:32:08.462999+0000","last_unstale":"2024-03-04T07:32:10.478449+0000","last_undegraded":"2024-03-04T07:32:10.478449+0000","last_fullsized":"2024-03-04T07:32:10.478449+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:16.332227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.3","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478609+0000","last_change":"2024-03-04T07:32:09.483140+0000","last_active":"2024-03-04T07:32:10.478609+0000","last_peered":"2024-03-04T07:32:10.478609+0000","last_clean":"2024-03-04T07:32:10.478609+0000","last_became_active":"2024-03-04T07:32:09.482602+0000","last_became_peered":"2024-03-04T07:32:09.482602+0000","last_unstale":"2024-03-04T07:32:10.478609+0000","last_undegraded":"2024-03-04T07:32:10.478609+0000","last_fullsized":"2024-03-04T07:32:10.478609+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:35.891031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.2","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479107+0000","last_change":"2024-03-04T07:32:08.468107+0000","last_active":"2024-03-04T07:32:10.479107+0000","last_peered":"2024-03-04T07:32:10.479107+0000","last_clean":"2024-03-04T07:32:10.479107+0000","last_became_active":"2024-03-04T07:32:08.467821+0000","last_became_peered":"2024-03-04T07:32:08.467821+0000","last_unstale":"2024-03-04T07:32:10.479107+0000","last_undegraded":"2024-03-04T07:32:10.479107+0000","last_fullsized":"2024-03-04T07:32:10.479107+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:24:40.241340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.0","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816711+0000","last_change":"2024-03-04T07:32:09.482188+0000","last_active":"2024-03-04T07:32:10.816711+0000","last_peered":"2024-03-04T07:32:10.816711+0000","last_clean":"2024-03-04T07:32:10.816711+0000","last_became_active":"2024-03-04T07:32:09.481703+0000","last_became_peered":"2024-03-04T07:32:09.481703+0000","last_unstale":"2024-03-04T07:32:10.816711+0000","last_undegraded":"2024-03-04T07:32:10.816711+0000","last_fullsized":"2024-03-04T07:32:10.816711+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:21:33.697195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.1","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472131+0000","last_change":"2024-03-04T07:32:08.467470+0000","last_active":"2024-03-04T07:32:09.472131+0000","last_peered":"2024-03-04T07:32:09.472131+0000","last_clean":"2024-03-04T07:32:09.472131+0000","last_became_active":"2024-03-04T07:32:08.467160+0000","last_became_peered":"2024-03-04T07:32:08.467160+0000","last_unstale":"2024-03-04T07:32:09.472131+0000","last_undegraded":"2024-03-04T07:32:09.472131+0000","last_fullsized":"2024-03-04T07:32:09.472131+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:27:34.969057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477738+0000","last_change":"2024-03-04T07:32:09.475607+0000","last_active":"2024-03-04T07:32:10.477738+0000","last_peered":"2024-03-04T07:32:10.477738+0000","last_clean":"2024-03-04T07:32:10.477738+0000","last_became_active":"2024-03-04T07:32:09.475350+0000","last_became_peered":"2024-03-04T07:32:09.475350+0000","last_unstale":"2024-03-04T07:32:10.477738+0000","last_undegraded":"2024-03-04T07:32:10.477738+0000","last_fullsized":"2024-03-04T07:32:10.477738+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:19.773814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.0","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477825+0000","last_change":"2024-03-04T07:32:08.469146+0000","last_active":"2024-03-04T07:32:10.477825+0000","last_peered":"2024-03-04T07:32:10.477825+0000","last_clean":"2024-03-04T07:32:10.477825+0000","last_became_active":"2024-03-04T07:32:08.468450+0000","last_became_peered":"2024-03-04T07:32:08.468450+0000","last_unstale":"2024-03-04T07:32:10.477825+0000","last_undegraded":"2024-03-04T07:32:10.477825+0000","last_fullsized":"2024-03-04T07:32:10.477825+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:51.504211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.10","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487773+0000","last_change":"2024-03-04T07:32:09.484383+0000","last_active":"2024-03-04T07:32:09.487773+0000","last_peered":"2024-03-04T07:32:09.487773+0000","last_clean":"2024-03-04T07:32:09.487773+0000","last_became_active":"2024-03-04T07:32:09.484095+0000","last_became_peered":"2024-03-04T07:32:09.484095+0000","last_unstale":"2024-03-04T07:32:09.487773+0000","last_undegraded":"2024-03-04T07:32:09.487773+0000","last_fullsized":"2024-03-04T07:32:09.487773+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:34.767357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.11","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478163+0000","last_change":"2024-03-04T07:32:08.468822+0000","last_active":"2024-03-04T07:32:10.478163+0000","last_peered":"2024-03-04T07:32:10.478163+0000","last_clean":"2024-03-04T07:32:10.478163+0000","last_became_active":"2024-03-04T07:32:08.468247+0000","last_became_peered":"2024-03-04T07:32:08.468247+0000","last_unstale":"2024-03-04T07:32:10.478163+0000","last_undegraded":"2024-03-04T07:32:10.478163+0000","last_fullsized":"2024-03-04T07:32:10.478163+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:07:06.763475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":325,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816750+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:32:10.816750+0000","last_peered":"2024-03-04T07:32:10.816750+0000","last_clean":"2024-03-04T07:32:10.816750+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:32:10.816750+0000","last_undegraded":"2024-03-04T07:32:10.816750+0000","last_fullsized":"2024-03-04T07:32:10.816750+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":17,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":16,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574919,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37208,"kb_used_data":1864,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334632,"statfs":{"total":96636764160,"available":96598663168,"internally_reserved":0,"allocated":1908736,"data_stored":1153914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574918,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40768,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331072,"statfs":{"total":96636764160,"available":96595017728,"internally_reserved":0,"allocated":1298432,"data_stored":556700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574918,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40788,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331052,"statfs":{"total":96636764160,"available":96594997248,"internally_reserved":0,"allocated":1314816,"data_stored":563534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574919,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40772,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331068,"statfs":{"total":96636764160,"available":96595013632,"internally_reserved":0,"allocated":1294336,"data_stored":556630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574918,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40780,"kb_used_data":1280,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331060,"statfs":{"total":96636764160,"available":96595005440,"internally_reserved":0,"allocated":1310720,"data_stored":558312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574918,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40764,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331076,"statfs":{"total":96636764160,"available":96595021824,"internally_reserved":0,"allocated":1294336,"data_stored":556630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574918,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40772,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331068,"statfs":{"total":96636764160,"available":96595013632,"internally_reserved":0,"allocated":1302528,"data_stored":556754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574918,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41384,"kb_used_data":1876,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330456,"statfs":{"total":96636764160,"available":96594386944,"internally_reserved":0,"allocated":1921024,"data_stored":1155022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:32:12.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:32:12.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.612+0000 7f11ef33e640 1 Processor -- start 2024-03-04T07:32:12.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.613+0000 7f11ef33e640 1 -- start start 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.613+0000 7f11ef33e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f11e80b1020 0x7f11e806af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.613+0000 7f11ef33e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e806ed10 0x7f11e80cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.613+0000 7f11ef33e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f11e80cd6c0 0x7f11e80cdb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.613+0000 7f11ef33e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f11e80d28b0 con 0x7f11e80cd6c0 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.613+0000 7f11ef33e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f11e80cc0e0 con 0x7f11e80b1020 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.613+0000 7f11ef33e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11e80cc3c0 con 0x7f11e806ed10 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f11e80cd6c0 0x7f11e80cdb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed0da640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f11e80b1020 0x7f11e806af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f11e80cd6c0 0x7f11e80cdb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52894/0 (socket says 172.21.15.64:52894) 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 -- 172.21.15.64:0/3095191024 learned_addr learned my addr 172.21.15.64:0/3095191024 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ec8d9640 1 --2- 172.21.15.64:0/3095191024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e806ed10 0x7f11e80cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 -- 172.21.15.64:0/3095191024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e806ed10 msgr2=0x7f11e80cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 --2- 172.21.15.64:0/3095191024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e806ed10 0x7f11e80cd180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 -- 172.21.15.64:0/3095191024 >> v2:172.21.15.66:3300/0 conn(0x7f11e80b1020 msgr2=0x7f11e806af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 --2- 172.21.15.64:0/3095191024 >> v2:172.21.15.66:3300/0 conn(0x7f11e80b1020 0x7f11e806af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed8db640 1 -- 172.21.15.64:0/3095191024 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11e80cc6a0 con 0x7f11e80cd6c0 2024-03-04T07:32:12.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.614+0000 7f11ed0da640 1 --2- 172.21.15.64:0/3095191024 >> v2:172.21.15.66:3300/0 conn(0x7f11e80b1020 0x7f11e806af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:12.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ed8db640 1 --2- 172.21.15.64:0/3095191024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80cd6c0 0x7f11e80cdb10 secure :-1 s=READY pgs=1307 cs=0 l=1 rev1=1 crypto rx=0x7f11e8070d50 tx=0x7f11e002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf02ca52445a182e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11dffff640 1 -- 172.21.15.64:0/3095191024 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11e002ca70 con 0x7f11e80cd6c0 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11dffff640 1 -- 172.21.15.64:0/3095191024 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11e002cc10 con 0x7f11e80cd6c0 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11dffff640 1 -- 172.21.15.64:0/3095191024 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11e00343f0 con 0x7f11e80cd6c0 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 -- 172.21.15.64:0/3095191024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80cd6c0 msgr2=0x7f11e80cdb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 --2- 172.21.15.64:0/3095191024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80cd6c0 0x7f11e80cdb10 secure :-1 s=READY pgs=1307 cs=0 l=1 rev1=1 crypto rx=0x7f11e8070d50 tx=0x7f11e002bed0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 -- 172.21.15.64:0/3095191024 shutdown_connections 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 --2- 172.21.15.64:0/3095191024 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80cd6c0 0x7f11e80cdb10 unknown :-1 s=CLOSED pgs=1307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 --2- 172.21.15.64:0/3095191024 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e806ed10 0x7f11e80cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 --2- 172.21.15.64:0/3095191024 >> v2:172.21.15.66:3300/0 conn(0x7f11e80b1020 0x7f11e806af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 -- 172.21.15.64:0/3095191024 >> 172.21.15.64:0/3095191024 conn(0x7f11e80d1a90 msgr2=0x7f11e80d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 -- 172.21.15.64:0/3095191024 shutdown_connections 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.615+0000 7f11ef33e640 1 -- 172.21.15.64:0/3095191024 wait complete. 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 Processor -- start 2024-03-04T07:32:12.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 -- start start 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11e806ed10 0x7f11e80b75c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80b1020 0x7f11e8159910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e80cd6c0 0x7f11e80b3ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11e815d700 con 0x7f11e80b1020 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11e80ccc60 con 0x7f11e806ed10 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ef33e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f11e815d880 con 0x7f11e80cd6c0 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ec8d9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80b1020 0x7f11e8159910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ec8d9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80b1020 0x7f11e8159910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52900/0 (socket says 172.21.15.64:52900) 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ec8d9640 1 -- 172.21.15.64:0/276881676 learned_addr learned my addr 172.21.15.64:0/276881676 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.617+0000 7f11ed0da640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11e806ed10 0x7f11e80b75c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ed8db640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e80cd6c0 0x7f11e80b3ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ec8d9640 1 -- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e80cd6c0 msgr2=0x7f11e80b3ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ec8d9640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e80cd6c0 0x7f11e80b3ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ec8d9640 1 -- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11e806ed10 msgr2=0x7f11e80b75c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ec8d9640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11e806ed10 0x7f11e80b75c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ec8d9640 1 -- 172.21.15.64:0/276881676 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11e80b2c40 con 0x7f11e80b1020 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ec8d9640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80b1020 0x7f11e8159910 secure :-1 s=READY pgs=1308 cs=0 l=1 rev1=1 crypto rx=0x7f11d800e9e0 tx=0x7f11d800eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ddffb640 1 -- 172.21.15.64:0/276881676 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11d8017070 con 0x7f11e80b1020 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ddffb640 1 -- 172.21.15.64:0/276881676 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f11d800c9d0 con 0x7f11e80b1020 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f11e80b2f20 con 0x7f11e80b1020 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ddffb640 1 -- 172.21.15.64:0/276881676 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11d800cdc0 con 0x7f11e80b1020 2024-03-04T07:32:12.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.618+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f11e80b34e0 con 0x7f11e80b1020 2024-03-04T07:32:12.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.619+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f11e8074980 con 0x7f11e80b1020 2024-03-04T07:32:12.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.620+0000 7f11ddffb640 1 -- 172.21.15.64:0/276881676 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f11d801a070 con 0x7f11e80b1020 2024-03-04T07:32:12.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.620+0000 7f11ddffb640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11cc06b4d0 0x7f11cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.620+0000 7f11ddffb640 1 -- 172.21.15.64:0/276881676 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f11d808a590 con 0x7f11e80b1020 2024-03-04T07:32:12.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.620+0000 7f11ed0da640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11cc06b4d0 0x7f11cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.621+0000 7f11ed0da640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11cc06b4d0 0x7f11cc06d990 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f11e4009560 tx=0x7f11e4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.625+0000 7f11ddffb640 1 -- 172.21.15.64:0/276881676 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f11e8074980 con 0x7f11e80b1020 2024-03-04T07:32:12.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.801+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f11e80cf800 con 0x7f11cc06b4d0 2024-03-04T07:32:12.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.807+0000 7f11ddffb640 1 -- 172.21.15.64:0/276881676 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252463 (secure 0 0 0) 0x7f11e80cf800 con 0x7f11cc06b4d0 2024-03-04T07:32:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:12.811 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11cc06b4d0 msgr2=0x7f11cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11cc06b4d0 0x7f11cc06d990 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f11e4009560 tx=0x7f11e4009190 comp rx=0 tx=0).stop 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80b1020 msgr2=0x7f11e8159910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80b1020 0x7f11e8159910 secure :-1 s=READY pgs=1308 cs=0 l=1 rev1=1 crypto rx=0x7f11d800e9e0 tx=0x7f11d800eea0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 shutdown_connections 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f11cc06b4d0 0x7f11cc06d990 unknown :-1 s=CLOSED pgs=591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f11e80cd6c0 0x7f11e80b3ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f11e80b1020 0x7f11e8159910 unknown :-1 s=CLOSED pgs=1308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 --2- 172.21.15.64:0/276881676 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f11e806ed10 0x7f11e80b75c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 >> 172.21.15.64:0/276881676 conn(0x7f11e80d1a90 msgr2=0x7f11e806af20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:12.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 shutdown_connections 2024-03-04T07:32:12.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.811+0000 7f11ef33e640 1 -- 172.21.15.64:0/276881676 wait complete. 2024-03-04T07:32:12.822 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2024-03-04T07:32:11.199472+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":121,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":323236,"kb_used_data":11372,"kb_used_omap":12,"kb_used_meta":311731,"kb_avail":754651484,"statfs":{"total":773094113280,"available":772763119616,"internally_reserved":0,"allocated":11644928,"data_stored":5657496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":319213104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000841"},"pg_stats":[{"pgid":"17.3f","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478377+0000","last_change":"2024-03-04T07:32:09.470345+0000","last_active":"2024-03-04T07:32:10.478377+0000","last_peered":"2024-03-04T07:32:10.478377+0000","last_clean":"2024-03-04T07:32:10.478377+0000","last_became_active":"2024-03-04T07:32:09.470078+0000","last_became_peered":"2024-03-04T07:32:09.470078+0000","last_unstale":"2024-03-04T07:32:10.478377+0000","last_undegraded":"2024-03-04T07:32:10.478377+0000","last_fullsized":"2024-03-04T07:32:10.478377+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:02:52.613754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.3e","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479029+0000","last_change":"2024-03-04T07:32:09.479732+0000","last_active":"2024-03-04T07:32:10.479029+0000","last_peered":"2024-03-04T07:32:10.479029+0000","last_clean":"2024-03-04T07:32:10.479029+0000","last_became_active":"2024-03-04T07:32:09.479417+0000","last_became_peered":"2024-03-04T07:32:09.479417+0000","last_unstale":"2024-03-04T07:32:10.479029+0000","last_undegraded":"2024-03-04T07:32:10.479029+0000","last_fullsized":"2024-03-04T07:32:10.479029+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:28.363398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.3d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816900+0000","last_change":"2024-03-04T07:32:09.488729+0000","last_active":"2024-03-04T07:32:10.816900+0000","last_peered":"2024-03-04T07:32:10.816900+0000","last_clean":"2024-03-04T07:32:10.816900+0000","last_became_active":"2024-03-04T07:32:09.487826+0000","last_became_peered":"2024-03-04T07:32:09.487826+0000","last_unstale":"2024-03-04T07:32:10.816900+0000","last_undegraded":"2024-03-04T07:32:10.816900+0000","last_fullsized":"2024-03-04T07:32:10.816900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:44:21.383631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.3c","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477938+0000","last_change":"2024-03-04T07:32:09.486806+0000","last_active":"2024-03-04T07:32:10.477938+0000","last_peered":"2024-03-04T07:32:10.477938+0000","last_clean":"2024-03-04T07:32:10.477938+0000","last_became_active":"2024-03-04T07:32:09.486544+0000","last_became_peered":"2024-03-04T07:32:09.486544+0000","last_unstale":"2024-03-04T07:32:10.477938+0000","last_undegraded":"2024-03-04T07:32:10.477938+0000","last_fullsized":"2024-03-04T07:32:10.477938+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:08:57.046464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.3b","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.482939+0000","last_change":"2024-03-04T07:32:09.479104+0000","last_active":"2024-03-04T07:32:09.482939+0000","last_peered":"2024-03-04T07:32:09.482939+0000","last_clean":"2024-03-04T07:32:09.482939+0000","last_became_active":"2024-03-04T07:32:09.477615+0000","last_became_peered":"2024-03-04T07:32:09.477615+0000","last_unstale":"2024-03-04T07:32:09.482939+0000","last_undegraded":"2024-03-04T07:32:09.482939+0000","last_fullsized":"2024-03-04T07:32:09.482939+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:57:11.655324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.3a","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.481184+0000","last_change":"2024-03-04T07:32:09.480083+0000","last_active":"2024-03-04T07:32:09.481184+0000","last_peered":"2024-03-04T07:32:09.481184+0000","last_clean":"2024-03-04T07:32:09.481184+0000","last_became_active":"2024-03-04T07:32:09.479775+0000","last_became_peered":"2024-03-04T07:32:09.479775+0000","last_unstale":"2024-03-04T07:32:09.481184+0000","last_undegraded":"2024-03-04T07:32:09.481184+0000","last_fullsized":"2024-03-04T07:32:09.481184+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:45:40.392705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.39","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478058+0000","last_change":"2024-03-04T07:32:09.469706+0000","last_active":"2024-03-04T07:32:10.478058+0000","last_peered":"2024-03-04T07:32:10.478058+0000","last_clean":"2024-03-04T07:32:10.478058+0000","last_became_active":"2024-03-04T07:32:09.468580+0000","last_became_peered":"2024-03-04T07:32:09.468580+0000","last_unstale":"2024-03-04T07:32:10.478058+0000","last_undegraded":"2024-03-04T07:32:10.478058+0000","last_fullsized":"2024-03-04T07:32:10.478058+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:46:13.668525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.38","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816589+0000","last_change":"2024-03-04T07:32:09.477916+0000","last_active":"2024-03-04T07:32:10.816589+0000","last_peered":"2024-03-04T07:32:10.816589+0000","last_clean":"2024-03-04T07:32:10.816589+0000","last_became_active":"2024-03-04T07:32:09.477639+0000","last_became_peered":"2024-03-04T07:32:09.477639+0000","last_unstale":"2024-03-04T07:32:10.816589+0000","last_undegraded":"2024-03-04T07:32:10.816589+0000","last_fullsized":"2024-03-04T07:32:10.816589+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:40.280836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.37","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478494+0000","last_change":"2024-03-04T07:32:09.479790+0000","last_active":"2024-03-04T07:32:10.478494+0000","last_peered":"2024-03-04T07:32:10.478494+0000","last_clean":"2024-03-04T07:32:10.478494+0000","last_became_active":"2024-03-04T07:32:09.479485+0000","last_became_peered":"2024-03-04T07:32:09.479485+0000","last_unstale":"2024-03-04T07:32:10.478494+0000","last_undegraded":"2024-03-04T07:32:10.478494+0000","last_fullsized":"2024-03-04T07:32:10.478494+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:40:53.021216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.36","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477962+0000","last_change":"2024-03-04T07:32:09.478798+0000","last_active":"2024-03-04T07:32:10.477962+0000","last_peered":"2024-03-04T07:32:10.477962+0000","last_clean":"2024-03-04T07:32:10.477962+0000","last_became_active":"2024-03-04T07:32:09.478508+0000","last_became_peered":"2024-03-04T07:32:09.478508+0000","last_unstale":"2024-03-04T07:32:10.477962+0000","last_undegraded":"2024-03-04T07:32:10.477962+0000","last_fullsized":"2024-03-04T07:32:10.477962+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:11:54.852936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.35","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478991+0000","last_change":"2024-03-04T07:32:09.484416+0000","last_active":"2024-03-04T07:32:10.478991+0000","last_peered":"2024-03-04T07:32:10.478991+0000","last_clean":"2024-03-04T07:32:10.478991+0000","last_became_active":"2024-03-04T07:32:09.482608+0000","last_became_peered":"2024-03-04T07:32:09.482608+0000","last_unstale":"2024-03-04T07:32:10.478991+0000","last_undegraded":"2024-03-04T07:32:10.478991+0000","last_fullsized":"2024-03-04T07:32:10.478991+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:34.767357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.34","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478078+0000","last_change":"2024-03-04T07:32:09.485056+0000","last_active":"2024-03-04T07:32:10.478078+0000","last_peered":"2024-03-04T07:32:10.478078+0000","last_clean":"2024-03-04T07:32:10.478078+0000","last_became_active":"2024-03-04T07:32:09.484210+0000","last_became_peered":"2024-03-04T07:32:09.484210+0000","last_unstale":"2024-03-04T07:32:10.478078+0000","last_undegraded":"2024-03-04T07:32:10.478078+0000","last_fullsized":"2024-03-04T07:32:10.478078+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:11:08.955473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.33","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816900+0000","last_change":"2024-03-04T07:32:09.480038+0000","last_active":"2024-03-04T07:32:10.816900+0000","last_peered":"2024-03-04T07:32:10.816900+0000","last_clean":"2024-03-04T07:32:10.816900+0000","last_became_active":"2024-03-04T07:32:09.479438+0000","last_became_peered":"2024-03-04T07:32:09.479438+0000","last_unstale":"2024-03-04T07:32:10.816900+0000","last_undegraded":"2024-03-04T07:32:10.816900+0000","last_fullsized":"2024-03-04T07:32:10.816900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:09:39.427339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.32","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816577+0000","last_change":"2024-03-04T07:32:09.477524+0000","last_active":"2024-03-04T07:32:10.816577+0000","last_peered":"2024-03-04T07:32:10.816577+0000","last_clean":"2024-03-04T07:32:10.816577+0000","last_became_active":"2024-03-04T07:32:09.477240+0000","last_became_peered":"2024-03-04T07:32:09.477240+0000","last_unstale":"2024-03-04T07:32:10.816577+0000","last_undegraded":"2024-03-04T07:32:10.816577+0000","last_fullsized":"2024-03-04T07:32:10.816577+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:34.767357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.31","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.481212+0000","last_change":"2024-03-04T07:32:09.480382+0000","last_active":"2024-03-04T07:32:09.481212+0000","last_peered":"2024-03-04T07:32:09.481212+0000","last_clean":"2024-03-04T07:32:09.481212+0000","last_became_active":"2024-03-04T07:32:09.479792+0000","last_became_peered":"2024-03-04T07:32:09.479792+0000","last_unstale":"2024-03-04T07:32:09.481212+0000","last_undegraded":"2024-03-04T07:32:09.481212+0000","last_fullsized":"2024-03-04T07:32:09.481212+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:14:00.917239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.30","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477552+0000","last_change":"2024-03-04T07:32:09.485142+0000","last_active":"2024-03-04T07:32:10.477552+0000","last_peered":"2024-03-04T07:32:10.477552+0000","last_clean":"2024-03-04T07:32:10.477552+0000","last_became_active":"2024-03-04T07:32:09.484852+0000","last_became_peered":"2024-03-04T07:32:09.484852+0000","last_unstale":"2024-03-04T07:32:10.477552+0000","last_undegraded":"2024-03-04T07:32:10.477552+0000","last_fullsized":"2024-03-04T07:32:10.477552+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:46:55.124028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.2f","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479382+0000","last_change":"2024-03-04T07:32:09.486859+0000","last_active":"2024-03-04T07:32:10.479382+0000","last_peered":"2024-03-04T07:32:10.479382+0000","last_clean":"2024-03-04T07:32:10.479382+0000","last_became_active":"2024-03-04T07:32:09.486420+0000","last_became_peered":"2024-03-04T07:32:09.486420+0000","last_unstale":"2024-03-04T07:32:10.479382+0000","last_undegraded":"2024-03-04T07:32:10.479382+0000","last_fullsized":"2024-03-04T07:32:10.479382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:40.280836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2e","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487416+0000","last_change":"2024-03-04T07:32:09.481216+0000","last_active":"2024-03-04T07:32:10.487416+0000","last_peered":"2024-03-04T07:32:10.487416+0000","last_clean":"2024-03-04T07:32:10.487416+0000","last_became_active":"2024-03-04T07:32:09.480891+0000","last_became_peered":"2024-03-04T07:32:09.480891+0000","last_unstale":"2024-03-04T07:32:10.487416+0000","last_undegraded":"2024-03-04T07:32:10.487416+0000","last_fullsized":"2024-03-04T07:32:10.487416+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:26:43.616896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.2d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816922+0000","last_change":"2024-03-04T07:32:09.478968+0000","last_active":"2024-03-04T07:32:10.816922+0000","last_peered":"2024-03-04T07:32:10.816922+0000","last_clean":"2024-03-04T07:32:10.816922+0000","last_became_active":"2024-03-04T07:32:09.478586+0000","last_became_peered":"2024-03-04T07:32:09.478586+0000","last_unstale":"2024-03-04T07:32:10.816922+0000","last_undegraded":"2024-03-04T07:32:10.816922+0000","last_fullsized":"2024-03-04T07:32:10.816922+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:14:14.305392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.2c","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479440+0000","last_change":"2024-03-04T07:32:09.484194+0000","last_active":"2024-03-04T07:32:10.479440+0000","last_peered":"2024-03-04T07:32:10.479440+0000","last_clean":"2024-03-04T07:32:10.479440+0000","last_became_active":"2024-03-04T07:32:09.483823+0000","last_became_peered":"2024-03-04T07:32:09.483823+0000","last_unstale":"2024-03-04T07:32:10.479440+0000","last_undegraded":"2024-03-04T07:32:10.479440+0000","last_fullsized":"2024-03-04T07:32:10.479440+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:16.159999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2b","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478862+0000","last_change":"2024-03-04T07:32:09.481973+0000","last_active":"2024-03-04T07:32:10.478862+0000","last_peered":"2024-03-04T07:32:10.478862+0000","last_clean":"2024-03-04T07:32:10.478862+0000","last_became_active":"2024-03-04T07:32:09.480192+0000","last_became_peered":"2024-03-04T07:32:09.480192+0000","last_unstale":"2024-03-04T07:32:10.478862+0000","last_undegraded":"2024-03-04T07:32:10.478862+0000","last_fullsized":"2024-03-04T07:32:10.478862+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:52.510635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.2a","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483387+0000","last_change":"2024-03-04T07:32:09.482308+0000","last_active":"2024-03-04T07:32:09.483387+0000","last_peered":"2024-03-04T07:32:09.483387+0000","last_clean":"2024-03-04T07:32:09.483387+0000","last_became_active":"2024-03-04T07:32:09.482066+0000","last_became_peered":"2024-03-04T07:32:09.482066+0000","last_unstale":"2024-03-04T07:32:09.483387+0000","last_undegraded":"2024-03-04T07:32:09.483387+0000","last_fullsized":"2024-03-04T07:32:09.483387+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:26:43.616896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.29","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478292+0000","last_change":"2024-03-04T07:32:09.484033+0000","last_active":"2024-03-04T07:32:10.478292+0000","last_peered":"2024-03-04T07:32:10.478292+0000","last_clean":"2024-03-04T07:32:10.478292+0000","last_became_active":"2024-03-04T07:32:09.483135+0000","last_became_peered":"2024-03-04T07:32:09.483135+0000","last_unstale":"2024-03-04T07:32:10.478292+0000","last_undegraded":"2024-03-04T07:32:10.478292+0000","last_fullsized":"2024-03-04T07:32:10.478292+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:26:43.616896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.28","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487752+0000","last_change":"2024-03-04T07:32:09.483971+0000","last_active":"2024-03-04T07:32:09.487752+0000","last_peered":"2024-03-04T07:32:09.487752+0000","last_clean":"2024-03-04T07:32:09.487752+0000","last_became_active":"2024-03-04T07:32:09.483659+0000","last_became_peered":"2024-03-04T07:32:09.483659+0000","last_unstale":"2024-03-04T07:32:09.487752+0000","last_undegraded":"2024-03-04T07:32:09.487752+0000","last_fullsized":"2024-03-04T07:32:09.487752+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:52.510635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.27","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479622+0000","last_change":"2024-03-04T07:32:09.481809+0000","last_active":"2024-03-04T07:32:10.479622+0000","last_peered":"2024-03-04T07:32:10.479622+0000","last_clean":"2024-03-04T07:32:10.479622+0000","last_became_active":"2024-03-04T07:32:09.481438+0000","last_became_peered":"2024-03-04T07:32:09.481438+0000","last_unstale":"2024-03-04T07:32:10.479622+0000","last_undegraded":"2024-03-04T07:32:10.479622+0000","last_fullsized":"2024-03-04T07:32:10.479622+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:21:33.697195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.26","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477981+0000","last_change":"2024-03-04T07:32:09.486669+0000","last_active":"2024-03-04T07:32:10.477981+0000","last_peered":"2024-03-04T07:32:10.477981+0000","last_clean":"2024-03-04T07:32:10.477981+0000","last_became_active":"2024-03-04T07:32:09.486394+0000","last_became_peered":"2024-03-04T07:32:09.486394+0000","last_unstale":"2024-03-04T07:32:10.477981+0000","last_undegraded":"2024-03-04T07:32:10.477981+0000","last_fullsized":"2024-03-04T07:32:10.477981+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:48:25.167596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.25","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483038+0000","last_change":"2024-03-04T07:32:09.480769+0000","last_active":"2024-03-04T07:32:09.483038+0000","last_peered":"2024-03-04T07:32:09.483038+0000","last_clean":"2024-03-04T07:32:09.483038+0000","last_became_active":"2024-03-04T07:32:09.480486+0000","last_became_peered":"2024-03-04T07:32:09.480486+0000","last_unstale":"2024-03-04T07:32:09.483038+0000","last_undegraded":"2024-03-04T07:32:09.483038+0000","last_fullsized":"2024-03-04T07:32:09.483038+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:54:56.215072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.24","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.488272+0000","last_change":"2024-03-04T07:32:09.486862+0000","last_active":"2024-03-04T07:32:09.488272+0000","last_peered":"2024-03-04T07:32:09.488272+0000","last_clean":"2024-03-04T07:32:09.488272+0000","last_became_active":"2024-03-04T07:32:09.486596+0000","last_became_peered":"2024-03-04T07:32:09.486596+0000","last_unstale":"2024-03-04T07:32:09.488272+0000","last_undegraded":"2024-03-04T07:32:09.488272+0000","last_fullsized":"2024-03-04T07:32:09.488272+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:16.159999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.23","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478261+0000","last_change":"2024-03-04T07:32:09.483856+0000","last_active":"2024-03-04T07:32:10.478261+0000","last_peered":"2024-03-04T07:32:10.478261+0000","last_clean":"2024-03-04T07:32:10.478261+0000","last_became_active":"2024-03-04T07:32:09.483548+0000","last_became_peered":"2024-03-04T07:32:09.483548+0000","last_unstale":"2024-03-04T07:32:10.478261+0000","last_undegraded":"2024-03-04T07:32:10.478261+0000","last_fullsized":"2024-03-04T07:32:10.478261+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:36.566044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.22","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478703+0000","last_change":"2024-03-04T07:32:09.483147+0000","last_active":"2024-03-04T07:32:10.478703+0000","last_peered":"2024-03-04T07:32:10.478703+0000","last_clean":"2024-03-04T07:32:10.478703+0000","last_became_active":"2024-03-04T07:32:09.482840+0000","last_became_peered":"2024-03-04T07:32:09.482840+0000","last_unstale":"2024-03-04T07:32:10.478703+0000","last_undegraded":"2024-03-04T07:32:10.478703+0000","last_fullsized":"2024-03-04T07:32:10.478703+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:47:59.291464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.21","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477686+0000","last_change":"2024-03-04T07:32:09.486523+0000","last_active":"2024-03-04T07:32:10.477686+0000","last_peered":"2024-03-04T07:32:10.477686+0000","last_clean":"2024-03-04T07:32:10.477686+0000","last_became_active":"2024-03-04T07:32:09.486243+0000","last_became_peered":"2024-03-04T07:32:09.486243+0000","last_unstale":"2024-03-04T07:32:10.477686+0000","last_undegraded":"2024-03-04T07:32:10.477686+0000","last_fullsized":"2024-03-04T07:32:10.477686+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:16:13.248314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.20","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.481526+0000","last_change":"2024-03-04T07:32:09.479221+0000","last_active":"2024-03-04T07:32:09.481526+0000","last_peered":"2024-03-04T07:32:09.481526+0000","last_clean":"2024-03-04T07:32:09.481526+0000","last_became_active":"2024-03-04T07:32:09.478921+0000","last_became_peered":"2024-03-04T07:32:09.478921+0000","last_unstale":"2024-03-04T07:32:09.481526+0000","last_undegraded":"2024-03-04T07:32:09.481526+0000","last_fullsized":"2024-03-04T07:32:09.481526+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:17.338651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.1e","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.478855+0000","last_change":"2024-03-04T07:32:09.477807+0000","last_active":"2024-03-04T07:32:09.478855+0000","last_peered":"2024-03-04T07:32:09.478855+0000","last_clean":"2024-03-04T07:32:09.478855+0000","last_became_active":"2024-03-04T07:32:09.477524+0000","last_became_peered":"2024-03-04T07:32:09.477524+0000","last_unstale":"2024-03-04T07:32:09.478855+0000","last_undegraded":"2024-03-04T07:32:09.478855+0000","last_fullsized":"2024-03-04T07:32:09.478855+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:54.885673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.1f","version":"104'4","reported_seq":18,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478685+0000","last_change":"2024-03-04T07:32:08.466784+0000","last_active":"2024-03-04T07:32:10.478685+0000","last_peered":"2024-03-04T07:32:10.478685+0000","last_clean":"2024-03-04T07:32:10.478685+0000","last_became_active":"2024-03-04T07:32:08.466466+0000","last_became_peered":"2024-03-04T07:32:08.466466+0000","last_unstale":"2024-03-04T07:32:10.478685+0000","last_undegraded":"2024-03-04T07:32:10.478685+0000","last_fullsized":"2024-03-04T07:32:10.478685+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:12:37.166337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1f","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487677+0000","last_change":"2024-03-04T07:32:09.476421+0000","last_active":"2024-03-04T07:32:10.487677+0000","last_peered":"2024-03-04T07:32:10.487677+0000","last_clean":"2024-03-04T07:32:10.487677+0000","last_became_active":"2024-03-04T07:32:09.476142+0000","last_became_peered":"2024-03-04T07:32:09.476142+0000","last_unstale":"2024-03-04T07:32:10.487677+0000","last_undegraded":"2024-03-04T07:32:10.487677+0000","last_fullsized":"2024-03-04T07:32:10.487677+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:35.891031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.1e","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816856+0000","last_change":"2024-03-04T07:32:08.470198+0000","last_active":"2024-03-04T07:32:10.816856+0000","last_peered":"2024-03-04T07:32:10.816856+0000","last_clean":"2024-03-04T07:32:10.816856+0000","last_became_active":"2024-03-04T07:32:08.469687+0000","last_became_peered":"2024-03-04T07:32:08.469687+0000","last_unstale":"2024-03-04T07:32:10.816856+0000","last_undegraded":"2024-03-04T07:32:10.816856+0000","last_fullsized":"2024-03-04T07:32:10.816856+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:15.909283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"17.1c","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.480680+0000","last_change":"2024-03-04T07:32:09.479162+0000","last_active":"2024-03-04T07:32:09.480680+0000","last_peered":"2024-03-04T07:32:09.480680+0000","last_clean":"2024-03-04T07:32:09.480680+0000","last_became_active":"2024-03-04T07:32:09.478879+0000","last_became_peered":"2024-03-04T07:32:09.478879+0000","last_unstale":"2024-03-04T07:32:09.480680+0000","last_undegraded":"2024-03-04T07:32:09.480680+0000","last_fullsized":"2024-03-04T07:32:09.480680+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:07:07.769899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.1d","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496495+0000","last_change":"2024-03-04T07:32:08.468670+0000","last_active":"2024-03-04T07:32:09.496495+0000","last_peered":"2024-03-04T07:32:09.496495+0000","last_clean":"2024-03-04T07:32:09.496495+0000","last_became_active":"2024-03-04T07:32:08.468381+0000","last_became_peered":"2024-03-04T07:32:08.468381+0000","last_unstale":"2024-03-04T07:32:09.496495+0000","last_undegraded":"2024-03-04T07:32:09.496495+0000","last_fullsized":"2024-03-04T07:32:09.496495+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:50.857896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.817015+0000","last_change":"2024-03-04T07:32:09.478218+0000","last_active":"2024-03-04T07:32:10.817015+0000","last_peered":"2024-03-04T07:32:10.817015+0000","last_clean":"2024-03-04T07:32:10.817015+0000","last_became_active":"2024-03-04T07:32:09.477770+0000","last_became_peered":"2024-03-04T07:32:09.477770+0000","last_unstale":"2024-03-04T07:32:10.817015+0000","last_undegraded":"2024-03-04T07:32:10.817015+0000","last_fullsized":"2024-03-04T07:32:10.817015+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:00:12.230203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.1c","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496625+0000","last_change":"2024-03-04T07:32:08.467300+0000","last_active":"2024-03-04T07:32:09.496625+0000","last_peered":"2024-03-04T07:32:09.496625+0000","last_clean":"2024-03-04T07:32:09.496625+0000","last_became_active":"2024-03-04T07:32:08.466925+0000","last_became_peered":"2024-03-04T07:32:08.466925+0000","last_unstale":"2024-03-04T07:32:09.496625+0000","last_undegraded":"2024-03-04T07:32:09.496625+0000","last_fullsized":"2024-03-04T07:32:09.496625+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:42:14.289123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1a","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477884+0000","last_change":"2024-03-04T07:32:09.479579+0000","last_active":"2024-03-04T07:32:10.477884+0000","last_peered":"2024-03-04T07:32:10.477884+0000","last_clean":"2024-03-04T07:32:10.477884+0000","last_became_active":"2024-03-04T07:32:09.479313+0000","last_became_peered":"2024-03-04T07:32:09.479313+0000","last_unstale":"2024-03-04T07:32:10.477884+0000","last_undegraded":"2024-03-04T07:32:10.477884+0000","last_fullsized":"2024-03-04T07:32:10.477884+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:02:28.339354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.1b","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478316+0000","last_change":"2024-03-04T07:32:08.467507+0000","last_active":"2024-03-04T07:32:10.478316+0000","last_peered":"2024-03-04T07:32:10.478316+0000","last_clean":"2024-03-04T07:32:10.478316+0000","last_became_active":"2024-03-04T07:32:08.467200+0000","last_became_peered":"2024-03-04T07:32:08.467200+0000","last_unstale":"2024-03-04T07:32:10.478316+0000","last_undegraded":"2024-03-04T07:32:10.478316+0000","last_fullsized":"2024-03-04T07:32:10.478316+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:57:53.879250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.1b","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478436+0000","last_change":"2024-03-04T07:32:09.482884+0000","last_active":"2024-03-04T07:32:10.478436+0000","last_peered":"2024-03-04T07:32:10.478436+0000","last_clean":"2024-03-04T07:32:10.478436+0000","last_became_active":"2024-03-04T07:32:09.482278+0000","last_became_peered":"2024-03-04T07:32:09.482278+0000","last_unstale":"2024-03-04T07:32:10.478436+0000","last_undegraded":"2024-03-04T07:32:10.478436+0000","last_fullsized":"2024-03-04T07:32:10.478436+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:02.819545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.1a","version":"104'1","reported_seq":13,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.494698+0000","last_change":"2024-03-04T07:32:08.469580+0000","last_active":"2024-03-04T07:32:09.494698+0000","last_peered":"2024-03-04T07:32:09.494698+0000","last_clean":"2024-03-04T07:32:09.494698+0000","last_became_active":"2024-03-04T07:32:08.469315+0000","last_became_peered":"2024-03-04T07:32:08.469315+0000","last_unstale":"2024-03-04T07:32:09.494698+0000","last_undegraded":"2024-03-04T07:32:09.494698+0000","last_fullsized":"2024-03-04T07:32:09.494698+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:26:15.372360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.18","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477960+0000","last_change":"2024-03-04T07:32:09.480285+0000","last_active":"2024-03-04T07:32:10.477960+0000","last_peered":"2024-03-04T07:32:10.477960+0000","last_clean":"2024-03-04T07:32:10.477960+0000","last_became_active":"2024-03-04T07:32:09.479978+0000","last_became_peered":"2024-03-04T07:32:09.479978+0000","last_unstale":"2024-03-04T07:32:10.477960+0000","last_undegraded":"2024-03-04T07:32:10.477960+0000","last_fullsized":"2024-03-04T07:32:10.477960+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:16:33.783469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.19","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478659+0000","last_change":"2024-03-04T07:32:08.467273+0000","last_active":"2024-03-04T07:32:10.478659+0000","last_peered":"2024-03-04T07:32:10.478659+0000","last_clean":"2024-03-04T07:32:10.478659+0000","last_became_active":"2024-03-04T07:32:08.466986+0000","last_became_peered":"2024-03-04T07:32:08.466986+0000","last_unstale":"2024-03-04T07:32:10.478659+0000","last_undegraded":"2024-03-04T07:32:10.478659+0000","last_fullsized":"2024-03-04T07:32:10.478659+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:53:40.996672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.19","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478589+0000","last_change":"2024-03-04T07:32:09.469423+0000","last_active":"2024-03-04T07:32:10.478589+0000","last_peered":"2024-03-04T07:32:10.478589+0000","last_clean":"2024-03-04T07:32:10.478589+0000","last_became_active":"2024-03-04T07:32:09.468524+0000","last_became_peered":"2024-03-04T07:32:09.468524+0000","last_unstale":"2024-03-04T07:32:10.478589+0000","last_undegraded":"2024-03-04T07:32:10.478589+0000","last_fullsized":"2024-03-04T07:32:10.478589+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:31:57.163635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.18","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477648+0000","last_change":"2024-03-04T07:32:08.460895+0000","last_active":"2024-03-04T07:32:10.477648+0000","last_peered":"2024-03-04T07:32:10.477648+0000","last_clean":"2024-03-04T07:32:10.477648+0000","last_became_active":"2024-03-04T07:32:08.460607+0000","last_became_peered":"2024-03-04T07:32:08.460607+0000","last_unstale":"2024-03-04T07:32:10.477648+0000","last_undegraded":"2024-03-04T07:32:10.477648+0000","last_fullsized":"2024-03-04T07:32:10.477648+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:34.884607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.16","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478849+0000","last_change":"2024-03-04T07:32:09.479090+0000","last_active":"2024-03-04T07:32:10.478849+0000","last_peered":"2024-03-04T07:32:10.478849+0000","last_clean":"2024-03-04T07:32:10.478849+0000","last_became_active":"2024-03-04T07:32:09.478810+0000","last_became_peered":"2024-03-04T07:32:09.478810+0000","last_unstale":"2024-03-04T07:32:10.478849+0000","last_undegraded":"2024-03-04T07:32:10.478849+0000","last_fullsized":"2024-03-04T07:32:10.478849+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:29:02.463565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.17","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472287+0000","last_change":"2024-03-04T07:32:08.468765+0000","last_active":"2024-03-04T07:32:09.472287+0000","last_peered":"2024-03-04T07:32:09.472287+0000","last_clean":"2024-03-04T07:32:09.472287+0000","last_became_active":"2024-03-04T07:32:08.467960+0000","last_became_peered":"2024-03-04T07:32:08.467960+0000","last_unstale":"2024-03-04T07:32:09.472287+0000","last_undegraded":"2024-03-04T07:32:09.472287+0000","last_fullsized":"2024-03-04T07:32:09.472287+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:05:56.623034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.17","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487692+0000","last_change":"2024-03-04T07:32:09.474815+0000","last_active":"2024-03-04T07:32:10.487692+0000","last_peered":"2024-03-04T07:32:10.487692+0000","last_clean":"2024-03-04T07:32:10.487692+0000","last_became_active":"2024-03-04T07:32:09.474512+0000","last_became_peered":"2024-03-04T07:32:09.474512+0000","last_unstale":"2024-03-04T07:32:10.487692+0000","last_undegraded":"2024-03-04T07:32:10.487692+0000","last_fullsized":"2024-03-04T07:32:10.487692+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:52.510635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.16","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472660+0000","last_change":"2024-03-04T07:32:08.470330+0000","last_active":"2024-03-04T07:32:09.472660+0000","last_peered":"2024-03-04T07:32:09.472660+0000","last_clean":"2024-03-04T07:32:09.472660+0000","last_became_active":"2024-03-04T07:32:08.470019+0000","last_became_peered":"2024-03-04T07:32:08.470019+0000","last_unstale":"2024-03-04T07:32:09.472660+0000","last_undegraded":"2024-03-04T07:32:09.472660+0000","last_fullsized":"2024-03-04T07:32:09.472660+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:17:44.711098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.14","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483393+0000","last_change":"2024-03-04T07:32:09.481767+0000","last_active":"2024-03-04T07:32:09.483393+0000","last_peered":"2024-03-04T07:32:09.483393+0000","last_clean":"2024-03-04T07:32:09.483393+0000","last_became_active":"2024-03-04T07:32:09.481480+0000","last_became_peered":"2024-03-04T07:32:09.481480+0000","last_unstale":"2024-03-04T07:32:09.483393+0000","last_undegraded":"2024-03-04T07:32:09.483393+0000","last_fullsized":"2024-03-04T07:32:09.483393+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:36.566044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.15","version":"104'3","reported_seq":17,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478157+0000","last_change":"2024-03-04T07:32:08.467932+0000","last_active":"2024-03-04T07:32:10.478157+0000","last_peered":"2024-03-04T07:32:10.478157+0000","last_clean":"2024-03-04T07:32:10.478157+0000","last_became_active":"2024-03-04T07:32:08.467644+0000","last_became_peered":"2024-03-04T07:32:08.467644+0000","last_unstale":"2024-03-04T07:32:10.478157+0000","last_undegraded":"2024-03-04T07:32:10.478157+0000","last_fullsized":"2024-03-04T07:32:10.478157+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:27.356974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.15","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478559+0000","last_change":"2024-03-04T07:32:09.484013+0000","last_active":"2024-03-04T07:32:10.478559+0000","last_peered":"2024-03-04T07:32:10.478559+0000","last_clean":"2024-03-04T07:32:10.478559+0000","last_became_active":"2024-03-04T07:32:09.483664+0000","last_became_peered":"2024-03-04T07:32:09.483664+0000","last_unstale":"2024-03-04T07:32:10.478559+0000","last_undegraded":"2024-03-04T07:32:10.478559+0000","last_fullsized":"2024-03-04T07:32:10.478559+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:20:14.175037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"16.14","version":"104'6","reported_seq":20,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478146+0000","last_change":"2024-03-04T07:32:08.460224+0000","last_active":"2024-03-04T07:32:10.478146+0000","last_peered":"2024-03-04T07:32:10.478146+0000","last_clean":"2024-03-04T07:32:10.478146+0000","last_became_active":"2024-03-04T07:32:08.459945+0000","last_became_peered":"2024-03-04T07:32:08.459945+0000","last_unstale":"2024-03-04T07:32:10.478146+0000","last_undegraded":"2024-03-04T07:32:10.478146+0000","last_fullsized":"2024-03-04T07:32:10.478146+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:33.760934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.12","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487820+0000","last_change":"2024-03-04T07:32:09.485082+0000","last_active":"2024-03-04T07:32:09.487820+0000","last_peered":"2024-03-04T07:32:09.487820+0000","last_clean":"2024-03-04T07:32:09.487820+0000","last_became_active":"2024-03-04T07:32:09.484799+0000","last_became_peered":"2024-03-04T07:32:09.484799+0000","last_unstale":"2024-03-04T07:32:09.487820+0000","last_undegraded":"2024-03-04T07:32:09.487820+0000","last_fullsized":"2024-03-04T07:32:09.487820+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:28.363398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.13","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.490970+0000","last_change":"2024-03-04T07:32:08.466321+0000","last_active":"2024-03-04T07:32:09.490970+0000","last_peered":"2024-03-04T07:32:09.490970+0000","last_clean":"2024-03-04T07:32:09.490970+0000","last_became_active":"2024-03-04T07:32:08.466057+0000","last_became_peered":"2024-03-04T07:32:08.466057+0000","last_unstale":"2024-03-04T07:32:09.490970+0000","last_undegraded":"2024-03-04T07:32:09.490970+0000","last_fullsized":"2024-03-04T07:32:09.490970+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:24:41.701610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.13","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487848+0000","last_change":"2024-03-04T07:32:09.483893+0000","last_active":"2024-03-04T07:32:09.487848+0000","last_peered":"2024-03-04T07:32:09.487848+0000","last_clean":"2024-03-04T07:32:09.487848+0000","last_became_active":"2024-03-04T07:32:09.483546+0000","last_became_peered":"2024-03-04T07:32:09.483546+0000","last_unstale":"2024-03-04T07:32:09.487848+0000","last_undegraded":"2024-03-04T07:32:09.487848+0000","last_fullsized":"2024-03-04T07:32:09.487848+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:35.891031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.12","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487226+0000","last_change":"2024-03-04T07:32:08.471165+0000","last_active":"2024-03-04T07:32:10.487226+0000","last_peered":"2024-03-04T07:32:10.487226+0000","last_clean":"2024-03-04T07:32:10.487226+0000","last_became_active":"2024-03-04T07:32:08.470903+0000","last_became_peered":"2024-03-04T07:32:08.470903+0000","last_unstale":"2024-03-04T07:32:10.487226+0000","last_undegraded":"2024-03-04T07:32:10.487226+0000","last_fullsized":"2024-03-04T07:32:10.487226+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:34:15.909283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.11","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.484100+0000","last_change":"2024-03-04T07:32:09.483142+0000","last_active":"2024-03-04T07:32:09.484100+0000","last_peered":"2024-03-04T07:32:09.484100+0000","last_clean":"2024-03-04T07:32:09.484100+0000","last_became_active":"2024-03-04T07:32:09.482857+0000","last_became_peered":"2024-03-04T07:32:09.482857+0000","last_unstale":"2024-03-04T07:32:09.484100+0000","last_undegraded":"2024-03-04T07:32:09.484100+0000","last_fullsized":"2024-03-04T07:32:09.484100+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:02.819545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.10","version":"104'8","reported_seq":20,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.498060+0000","last_change":"2024-03-04T07:32:08.463643+0000","last_active":"2024-03-04T07:32:09.498060+0000","last_peered":"2024-03-04T07:32:09.498060+0000","last_clean":"2024-03-04T07:32:09.498060+0000","last_became_active":"2024-03-04T07:32:08.463174+0000","last_became_peered":"2024-03-04T07:32:08.463174+0000","last_unstale":"2024-03-04T07:32:09.498060+0000","last_undegraded":"2024-03-04T07:32:09.498060+0000","last_fullsized":"2024-03-04T07:32:09.498060+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:56:46.978000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.e","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.488894+0000","last_change":"2024-03-04T07:32:09.488233+0000","last_active":"2024-03-04T07:32:09.488894+0000","last_peered":"2024-03-04T07:32:09.488894+0000","last_clean":"2024-03-04T07:32:09.488894+0000","last_became_active":"2024-03-04T07:32:09.487896+0000","last_became_peered":"2024-03-04T07:32:09.487896+0000","last_unstale":"2024-03-04T07:32:09.488894+0000","last_undegraded":"2024-03-04T07:32:09.488894+0000","last_fullsized":"2024-03-04T07:32:09.488894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:45:33.189423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"16.f","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478587+0000","last_change":"2024-03-04T07:32:08.468285+0000","last_active":"2024-03-04T07:32:10.478587+0000","last_peered":"2024-03-04T07:32:10.478587+0000","last_clean":"2024-03-04T07:32:10.478587+0000","last_became_active":"2024-03-04T07:32:08.468026+0000","last_became_peered":"2024-03-04T07:32:08.468026+0000","last_unstale":"2024-03-04T07:32:10.478587+0000","last_undegraded":"2024-03-04T07:32:10.478587+0000","last_fullsized":"2024-03-04T07:32:10.478587+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:15.153575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.f","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.483843+0000","last_change":"2024-03-04T07:32:09.481192+0000","last_active":"2024-03-04T07:32:09.483843+0000","last_peered":"2024-03-04T07:32:09.483843+0000","last_clean":"2024-03-04T07:32:09.483843+0000","last_became_active":"2024-03-04T07:32:09.480038+0000","last_became_peered":"2024-03-04T07:32:09.480038+0000","last_unstale":"2024-03-04T07:32:09.483843+0000","last_undegraded":"2024-03-04T07:32:09.483843+0000","last_fullsized":"2024-03-04T07:32:09.483843+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:56:49.837533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"16.e","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496586+0000","last_change":"2024-03-04T07:32:08.463461+0000","last_active":"2024-03-04T07:32:09.496586+0000","last_peered":"2024-03-04T07:32:09.496586+0000","last_clean":"2024-03-04T07:32:09.496586+0000","last_became_active":"2024-03-04T07:32:08.463191+0000","last_became_peered":"2024-03-04T07:32:08.463191+0000","last_unstale":"2024-03-04T07:32:09.496586+0000","last_undegraded":"2024-03-04T07:32:09.496586+0000","last_fullsized":"2024-03-04T07:32:09.496586+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:31:36.255777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.c","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816821+0000","last_change":"2024-03-04T07:32:09.491556+0000","last_active":"2024-03-04T07:32:10.816821+0000","last_peered":"2024-03-04T07:32:10.816821+0000","last_clean":"2024-03-04T07:32:10.816821+0000","last_became_active":"2024-03-04T07:32:09.491077+0000","last_became_peered":"2024-03-04T07:32:09.491077+0000","last_unstale":"2024-03-04T07:32:10.816821+0000","last_undegraded":"2024-03-04T07:32:10.816821+0000","last_fullsized":"2024-03-04T07:32:10.816821+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:14:25.586669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.d","version":"104'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487411+0000","last_change":"2024-03-04T07:32:08.471936+0000","last_active":"2024-03-04T07:32:10.487411+0000","last_peered":"2024-03-04T07:32:10.487411+0000","last_clean":"2024-03-04T07:32:10.487411+0000","last_became_active":"2024-03-04T07:32:08.471591+0000","last_became_peered":"2024-03-04T07:32:08.471591+0000","last_unstale":"2024-03-04T07:32:10.487411+0000","last_undegraded":"2024-03-04T07:32:10.487411+0000","last_fullsized":"2024-03-04T07:32:10.487411+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:03:12.593774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.d","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.817052+0000","last_change":"2024-03-04T07:32:09.488979+0000","last_active":"2024-03-04T07:32:10.817052+0000","last_peered":"2024-03-04T07:32:10.817052+0000","last_clean":"2024-03-04T07:32:10.817052+0000","last_became_active":"2024-03-04T07:32:09.488455+0000","last_became_peered":"2024-03-04T07:32:09.488455+0000","last_unstale":"2024-03-04T07:32:10.817052+0000","last_undegraded":"2024-03-04T07:32:10.817052+0000","last_fullsized":"2024-03-04T07:32:10.817052+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:06:33.445992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.c","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478409+0000","last_change":"2024-03-04T07:32:08.469020+0000","last_active":"2024-03-04T07:32:10.478409+0000","last_peered":"2024-03-04T07:32:10.478409+0000","last_clean":"2024-03-04T07:32:10.478409+0000","last_became_active":"2024-03-04T07:32:08.468735+0000","last_became_peered":"2024-03-04T07:32:08.468735+0000","last_unstale":"2024-03-04T07:32:10.478409+0000","last_undegraded":"2024-03-04T07:32:10.478409+0000","last_fullsized":"2024-03-04T07:32:10.478409+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:58.286990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.a","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816581+0000","last_change":"2024-03-04T07:32:09.486195+0000","last_active":"2024-03-04T07:32:10.816581+0000","last_peered":"2024-03-04T07:32:10.816581+0000","last_clean":"2024-03-04T07:32:10.816581+0000","last_became_active":"2024-03-04T07:32:09.485790+0000","last_became_peered":"2024-03-04T07:32:09.485790+0000","last_unstale":"2024-03-04T07:32:10.816581+0000","last_undegraded":"2024-03-04T07:32:10.816581+0000","last_fullsized":"2024-03-04T07:32:10.816581+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:28:28.363398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.b","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.496844+0000","last_change":"2024-03-04T07:32:08.471852+0000","last_active":"2024-03-04T07:32:09.496844+0000","last_peered":"2024-03-04T07:32:09.496844+0000","last_clean":"2024-03-04T07:32:09.496844+0000","last_became_active":"2024-03-04T07:32:08.471494+0000","last_became_peered":"2024-03-04T07:32:08.471494+0000","last_unstale":"2024-03-04T07:32:09.496844+0000","last_undegraded":"2024-03-04T07:32:09.496844+0000","last_fullsized":"2024-03-04T07:32:09.496844+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:25:58.286990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.b","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477900+0000","last_change":"2024-03-04T07:32:09.485410+0000","last_active":"2024-03-04T07:32:10.477900+0000","last_peered":"2024-03-04T07:32:10.477900+0000","last_clean":"2024-03-04T07:32:10.477900+0000","last_became_active":"2024-03-04T07:32:09.484796+0000","last_became_peered":"2024-03-04T07:32:09.484796+0000","last_unstale":"2024-03-04T07:32:10.477900+0000","last_undegraded":"2024-03-04T07:32:10.477900+0000","last_fullsized":"2024-03-04T07:32:10.477900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:17:46.919044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.a","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.473013+0000","last_change":"2024-03-04T07:32:08.468140+0000","last_active":"2024-03-04T07:32:09.473013+0000","last_peered":"2024-03-04T07:32:09.473013+0000","last_clean":"2024-03-04T07:32:09.473013+0000","last_became_active":"2024-03-04T07:32:08.467803+0000","last_became_peered":"2024-03-04T07:32:08.467803+0000","last_unstale":"2024-03-04T07:32:09.473013+0000","last_undegraded":"2024-03-04T07:32:09.473013+0000","last_fullsized":"2024-03-04T07:32:09.473013+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:46:11.504981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.8","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816610+0000","last_change":"2024-03-04T07:32:09.480856+0000","last_active":"2024-03-04T07:32:10.816610+0000","last_peered":"2024-03-04T07:32:10.816610+0000","last_clean":"2024-03-04T07:32:10.816610+0000","last_became_active":"2024-03-04T07:32:09.480614+0000","last_became_peered":"2024-03-04T07:32:09.480614+0000","last_unstale":"2024-03-04T07:32:10.816610+0000","last_undegraded":"2024-03-04T07:32:10.816610+0000","last_fullsized":"2024-03-04T07:32:10.816610+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:30:16.159999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.9","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472622+0000","last_change":"2024-03-04T07:32:08.465812+0000","last_active":"2024-03-04T07:32:09.472622+0000","last_peered":"2024-03-04T07:32:09.472622+0000","last_clean":"2024-03-04T07:32:09.472622+0000","last_became_active":"2024-03-04T07:32:08.465479+0000","last_became_peered":"2024-03-04T07:32:08.465479+0000","last_unstale":"2024-03-04T07:32:09.472622+0000","last_undegraded":"2024-03-04T07:32:09.472622+0000","last_fullsized":"2024-03-04T07:32:09.472622+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:12:37.166337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"17.9","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487924+0000","last_change":"2024-03-04T07:32:09.485154+0000","last_active":"2024-03-04T07:32:09.487924+0000","last_peered":"2024-03-04T07:32:09.487924+0000","last_clean":"2024-03-04T07:32:09.487924+0000","last_became_active":"2024-03-04T07:32:09.484906+0000","last_became_peered":"2024-03-04T07:32:09.484906+0000","last_unstale":"2024-03-04T07:32:09.487924+0000","last_undegraded":"2024-03-04T07:32:09.487924+0000","last_fullsized":"2024-03-04T07:32:09.487924+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:40.280836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.8","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477740+0000","last_change":"2024-03-04T07:32:08.471088+0000","last_active":"2024-03-04T07:32:10.477740+0000","last_peered":"2024-03-04T07:32:10.477740+0000","last_clean":"2024-03-04T07:32:10.477740+0000","last_became_active":"2024-03-04T07:32:08.470466+0000","last_became_peered":"2024-03-04T07:32:08.470466+0000","last_unstale":"2024-03-04T07:32:10.477740+0000","last_undegraded":"2024-03-04T07:32:10.477740+0000","last_fullsized":"2024-03-04T07:32:10.477740+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:01.813121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.6","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487489+0000","last_change":"2024-03-04T07:32:09.472612+0000","last_active":"2024-03-04T07:32:10.487489+0000","last_peered":"2024-03-04T07:32:10.487489+0000","last_clean":"2024-03-04T07:32:10.487489+0000","last_became_active":"2024-03-04T07:32:09.472330+0000","last_became_peered":"2024-03-04T07:32:09.472330+0000","last_unstale":"2024-03-04T07:32:10.487489+0000","last_undegraded":"2024-03-04T07:32:10.487489+0000","last_fullsized":"2024-03-04T07:32:10.487489+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:52:02.819545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.7","version":"104'2","reported_seq":14,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.497370+0000","last_change":"2024-03-04T07:32:08.465896+0000","last_active":"2024-03-04T07:32:09.497370+0000","last_peered":"2024-03-04T07:32:09.497370+0000","last_clean":"2024-03-04T07:32:09.497370+0000","last_became_active":"2024-03-04T07:32:08.465603+0000","last_became_peered":"2024-03-04T07:32:08.465603+0000","last_unstale":"2024-03-04T07:32:09.497370+0000","last_undegraded":"2024-03-04T07:32:09.497370+0000","last_fullsized":"2024-03-04T07:32:09.497370+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:33:41.109219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.7","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478510+0000","last_change":"2024-03-04T07:32:09.487556+0000","last_active":"2024-03-04T07:32:10.478510+0000","last_peered":"2024-03-04T07:32:10.478510+0000","last_clean":"2024-03-04T07:32:10.478510+0000","last_became_active":"2024-03-04T07:32:09.487179+0000","last_became_peered":"2024-03-04T07:32:09.487179+0000","last_unstale":"2024-03-04T07:32:10.478510+0000","last_undegraded":"2024-03-04T07:32:10.478510+0000","last_fullsized":"2024-03-04T07:32:10.478510+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:05:15.550074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.6","version":"105'1","reported_seq":15,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.489033+0000","last_change":"2024-03-04T07:32:08.470418+0000","last_active":"2024-03-04T07:32:10.489033+0000","last_peered":"2024-03-04T07:32:10.489033+0000","last_clean":"2024-03-04T07:32:10.489033+0000","last_became_active":"2024-03-04T07:32:08.470071+0000","last_became_peered":"2024-03-04T07:32:08.470071+0000","last_unstale":"2024-03-04T07:32:10.489033+0000","last_undegraded":"2024-03-04T07:32:10.489033+0000","last_fullsized":"2024-03-04T07:32:10.489033+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:10:46.660537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"17.4","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478120+0000","last_change":"2024-03-04T07:32:09.485079+0000","last_active":"2024-03-04T07:32:10.478120+0000","last_peered":"2024-03-04T07:32:10.478120+0000","last_clean":"2024-03-04T07:32:10.478120+0000","last_became_active":"2024-03-04T07:32:09.484278+0000","last_became_peered":"2024-03-04T07:32:09.484278+0000","last_unstale":"2024-03-04T07:32:10.478120+0000","last_undegraded":"2024-03-04T07:32:10.478120+0000","last_fullsized":"2024-03-04T07:32:10.478120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:31:49.227609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.5","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478213+0000","last_change":"2024-03-04T07:32:08.465563+0000","last_active":"2024-03-04T07:32:10.478213+0000","last_peered":"2024-03-04T07:32:10.478213+0000","last_clean":"2024-03-04T07:32:10.478213+0000","last_became_active":"2024-03-04T07:32:08.465302+0000","last_became_peered":"2024-03-04T07:32:08.465302+0000","last_unstale":"2024-03-04T07:32:10.478213+0000","last_undegraded":"2024-03-04T07:32:10.478213+0000","last_fullsized":"2024-03-04T07:32:10.478213+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:24:39.274412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.5","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.487396+0000","last_change":"2024-03-04T07:32:09.471907+0000","last_active":"2024-03-04T07:32:10.487396+0000","last_peered":"2024-03-04T07:32:10.487396+0000","last_clean":"2024-03-04T07:32:10.487396+0000","last_became_active":"2024-03-04T07:32:09.471627+0000","last_became_peered":"2024-03-04T07:32:09.471627+0000","last_unstale":"2024-03-04T07:32:10.487396+0000","last_undegraded":"2024-03-04T07:32:10.487396+0000","last_fullsized":"2024-03-04T07:32:10.487396+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:36.566044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"16.4","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.473567+0000","last_change":"2024-03-04T07:32:08.469166+0000","last_active":"2024-03-04T07:32:09.473567+0000","last_peered":"2024-03-04T07:32:09.473567+0000","last_clean":"2024-03-04T07:32:09.473567+0000","last_became_active":"2024-03-04T07:32:08.468891+0000","last_became_peered":"2024-03-04T07:32:08.468891+0000","last_unstale":"2024-03-04T07:32:09.473567+0000","last_undegraded":"2024-03-04T07:32:09.473567+0000","last_fullsized":"2024-03-04T07:32:09.473567+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:46:12.882604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"17.2","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477995+0000","last_change":"2024-03-04T07:32:09.478932+0000","last_active":"2024-03-04T07:32:10.477995+0000","last_peered":"2024-03-04T07:32:10.477995+0000","last_clean":"2024-03-04T07:32:10.477995+0000","last_became_active":"2024-03-04T07:32:09.478632+0000","last_became_peered":"2024-03-04T07:32:09.478632+0000","last_unstale":"2024-03-04T07:32:10.477995+0000","last_undegraded":"2024-03-04T07:32:10.477995+0000","last_fullsized":"2024-03-04T07:32:10.477995+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:26:16.378784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.3","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478449+0000","last_change":"2024-03-04T07:32:08.463277+0000","last_active":"2024-03-04T07:32:10.478449+0000","last_peered":"2024-03-04T07:32:10.478449+0000","last_clean":"2024-03-04T07:32:10.478449+0000","last_became_active":"2024-03-04T07:32:08.462999+0000","last_became_peered":"2024-03-04T07:32:08.462999+0000","last_unstale":"2024-03-04T07:32:10.478449+0000","last_undegraded":"2024-03-04T07:32:10.478449+0000","last_fullsized":"2024-03-04T07:32:10.478449+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:07:16.332227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"17.3","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478609+0000","last_change":"2024-03-04T07:32:09.483140+0000","last_active":"2024-03-04T07:32:10.478609+0000","last_peered":"2024-03-04T07:32:10.478609+0000","last_clean":"2024-03-04T07:32:10.478609+0000","last_became_active":"2024-03-04T07:32:09.482602+0000","last_became_peered":"2024-03-04T07:32:09.482602+0000","last_unstale":"2024-03-04T07:32:10.478609+0000","last_undegraded":"2024-03-04T07:32:10.478609+0000","last_fullsized":"2024-03-04T07:32:10.478609+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:35.891031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"16.2","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.479107+0000","last_change":"2024-03-04T07:32:08.468107+0000","last_active":"2024-03-04T07:32:10.479107+0000","last_peered":"2024-03-04T07:32:10.479107+0000","last_clean":"2024-03-04T07:32:10.479107+0000","last_became_active":"2024-03-04T07:32:08.467821+0000","last_became_peered":"2024-03-04T07:32:08.467821+0000","last_unstale":"2024-03-04T07:32:10.479107+0000","last_undegraded":"2024-03-04T07:32:10.479107+0000","last_fullsized":"2024-03-04T07:32:10.479107+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:24:40.241340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"17.0","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816711+0000","last_change":"2024-03-04T07:32:09.482188+0000","last_active":"2024-03-04T07:32:10.816711+0000","last_peered":"2024-03-04T07:32:10.816711+0000","last_clean":"2024-03-04T07:32:10.816711+0000","last_became_active":"2024-03-04T07:32:09.481703+0000","last_became_peered":"2024-03-04T07:32:09.481703+0000","last_unstale":"2024-03-04T07:32:10.816711+0000","last_undegraded":"2024-03-04T07:32:10.816711+0000","last_fullsized":"2024-03-04T07:32:10.816711+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:21:33.697195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"16.1","version":"0'0","reported_seq":12,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.472131+0000","last_change":"2024-03-04T07:32:08.467470+0000","last_active":"2024-03-04T07:32:09.472131+0000","last_peered":"2024-03-04T07:32:09.472131+0000","last_clean":"2024-03-04T07:32:09.472131+0000","last_became_active":"2024-03-04T07:32:08.467160+0000","last_became_peered":"2024-03-04T07:32:08.467160+0000","last_unstale":"2024-03-04T07:32:09.472131+0000","last_undegraded":"2024-03-04T07:32:09.472131+0000","last_fullsized":"2024-03-04T07:32:09.472131+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:27:34.969057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"17.1","version":"0'0","reported_seq":12,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477738+0000","last_change":"2024-03-04T07:32:09.475607+0000","last_active":"2024-03-04T07:32:10.477738+0000","last_peered":"2024-03-04T07:32:10.477738+0000","last_clean":"2024-03-04T07:32:10.477738+0000","last_became_active":"2024-03-04T07:32:09.475350+0000","last_became_peered":"2024-03-04T07:32:09.475350+0000","last_unstale":"2024-03-04T07:32:10.477738+0000","last_undegraded":"2024-03-04T07:32:10.477738+0000","last_fullsized":"2024-03-04T07:32:10.477738+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:23:19.773814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"16.0","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.477825+0000","last_change":"2024-03-04T07:32:08.469146+0000","last_active":"2024-03-04T07:32:10.477825+0000","last_peered":"2024-03-04T07:32:10.477825+0000","last_clean":"2024-03-04T07:32:10.477825+0000","last_became_active":"2024-03-04T07:32:08.468450+0000","last_became_peered":"2024-03-04T07:32:08.468450+0000","last_unstale":"2024-03-04T07:32:10.477825+0000","last_undegraded":"2024-03-04T07:32:10.477825+0000","last_fullsized":"2024-03-04T07:32:10.477825+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:01:51.504211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"17.10","version":"0'0","reported_seq":9,"reported_epoch":104,"state":"active+clean","last_fresh":"2024-03-04T07:32:09.487773+0000","last_change":"2024-03-04T07:32:09.484383+0000","last_active":"2024-03-04T07:32:09.487773+0000","last_peered":"2024-03-04T07:32:09.487773+0000","last_clean":"2024-03-04T07:32:09.487773+0000","last_became_active":"2024-03-04T07:32:09.484095+0000","last_became_peered":"2024-03-04T07:32:09.484095+0000","last_unstale":"2024-03-04T07:32:09.487773+0000","last_undegraded":"2024-03-04T07:32:09.487773+0000","last_fullsized":"2024-03-04T07:32:09.487773+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":103,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:08.444892+0000","last_clean_scrub_stamp":"2024-03-04T07:32:08.444892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:05:34.767357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"16.11","version":"0'0","reported_seq":14,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.478163+0000","last_change":"2024-03-04T07:32:08.468822+0000","last_active":"2024-03-04T07:32:10.478163+0000","last_peered":"2024-03-04T07:32:10.478163+0000","last_clean":"2024-03-04T07:32:10.478163+0000","last_became_active":"2024-03-04T07:32:08.468247+0000","last_became_peered":"2024-03-04T07:32:08.468247+0000","last_unstale":"2024-03-04T07:32:10.478163+0000","last_undegraded":"2024-03-04T07:32:10.478163+0000","last_fullsized":"2024-03-04T07:32:10.478163+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:32:07.438468+0000","last_clean_scrub_stamp":"2024-03-04T07:32:07.438468+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:07:06.763475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":325,"reported_epoch":105,"state":"active+clean","last_fresh":"2024-03-04T07:32:10.816750+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:32:10.816750+0000","last_peered":"2024-03-04T07:32:10.816750+0000","last_clean":"2024-03-04T07:32:10.816750+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:32:10.816750+0000","last_undegraded":"2024-03-04T07:32:10.816750+0000","last_fullsized":"2024-03-04T07:32:10.816750+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":17,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":16,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574919,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37208,"kb_used_data":1864,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334632,"statfs":{"total":96636764160,"available":96598663168,"internally_reserved":0,"allocated":1908736,"data_stored":1153914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574918,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40768,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331072,"statfs":{"total":96636764160,"available":96595017728,"internally_reserved":0,"allocated":1298432,"data_stored":556700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574918,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40788,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331052,"statfs":{"total":96636764160,"available":96594997248,"internally_reserved":0,"allocated":1314816,"data_stored":563534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574919,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40772,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331068,"statfs":{"total":96636764160,"available":96595013632,"internally_reserved":0,"allocated":1294336,"data_stored":556630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574918,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40780,"kb_used_data":1280,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331060,"statfs":{"total":96636764160,"available":96595005440,"internally_reserved":0,"allocated":1310720,"data_stored":558312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574918,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40764,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331076,"statfs":{"total":96636764160,"available":96595021824,"internally_reserved":0,"allocated":1294336,"data_stored":556630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574918,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40772,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94331068,"statfs":{"total":96636764160,"available":96595013632,"internally_reserved":0,"allocated":1302528,"data_stored":556754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574918,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41384,"kb_used_data":1876,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330456,"statfs":{"total":96636764160,"available":96594386944,"internally_reserved":0,"allocated":1921024,"data_stored":1155022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":16,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":17,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:32:12.826 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:32:12.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:32:12.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.970+0000 7f7e5aa4e640 1 Processor -- start 2024-03-04T07:32:12.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.970+0000 7f7e5aa4e640 1 -- start start 2024-03-04T07:32:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e5aa4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540b1c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e5aa4e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7e5406b2c0 0x7f7e5406b750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e5aa4e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e54161a40 con 0x7f7e540cdcd0 2024-03-04T07:32:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e5aa4e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7e540d11e0 con 0x7f7e5406b2c0 2024-03-04T07:32:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e5aa4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e540b2a20 con 0x7f7e540b2cd0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e53fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7e540cdcd0 0x7f7e540b1550 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:44724/0 (socket says 172.21.15.64:44724) 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e53fff640 1 -- 172.21.15.64:0/3215720973 learned_addr learned my addr 172.21.15.64:0/3215720973 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e58feb640 1 --2- 172.21.15.64:0/3215720973 >> v2:172.21.15.66:3300/0 conn(0x7f7e5406b2c0 0x7f7e5406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.971+0000 7f7e537fe640 1 --2- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540b1c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e52ffd640 1 -- 172.21.15.64:0/3215720973 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1412074362 0 0) 0x7f7e54161a40 con 0x7f7e540cdcd0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e52ffd640 1 -- 172.21.15.64:0/3215720973 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7e3c003520 con 0x7f7e540cdcd0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e58feb640 1 -- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 msgr2=0x7f7e540b1c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e58feb640 1 --2- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540b1c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e58feb640 1 -- 172.21.15.64:0/3215720973 >> v1:172.21.15.64:6789/0 conn(0x7f7e540cdcd0 legacy=0x7f7e540b1550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e58feb640 1 -- 172.21.15.64:0/3215720973 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e540d0940 con 0x7f7e5406b2c0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e537fe640 1 --2- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540b1c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.972+0000 7f7e58feb640 1 --2- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5406b750 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f7e4c009d90 tx=0x7f7e4c02be10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=321e2af16cb2b504 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e52ffd640 1 -- 172.21.15.64:0/3215720973 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e4c039070 con 0x7f7e5406b2c0 2024-03-04T07:32:12.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e52ffd640 1 -- 172.21.15.64:0/3215720973 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e4c02cbc0 con 0x7f7e5406b2c0 2024-03-04T07:32:12.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e52ffd640 1 -- 172.21.15.64:0/3215720973 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e4c034440 con 0x7f7e5406b2c0 2024-03-04T07:32:12.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 msgr2=0x7f7e5406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5406b750 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f7e4c009d90 tx=0x7f7e4c02be10 comp rx=0 tx=0).stop 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3215720973 shutdown_connections 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5406b750 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3215720973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540b1c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3215720973 >> 172.21.15.64:0/3215720973 conn(0x7f7e540b3700 msgr2=0x7f7e540b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3215720973 shutdown_connections 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.973+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3215720973 wait complete. 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.974+0000 7f7e5aa4e640 1 Processor -- start 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e5aa4e640 1 -- start start 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e5aa4e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5414fe30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e5aa4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540af380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e5aa4e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e540cdcd0 0x7f7e540b6e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e5aa4e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e540cc1c0 con 0x7f7e540cdcd0 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e5aa4e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e540d0f00 con 0x7f7e5406b2c0 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e5aa4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e540cc340 con 0x7f7e540b2cd0 2024-03-04T07:32:12.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e58feb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e540cdcd0 0x7f7e540b6e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e58feb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e540cdcd0 0x7f7e540b6e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52912/0 (socket says 172.21.15.64:52912) 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e58feb640 1 -- 172.21.15.64:0/3347735693 learned_addr learned my addr 172.21.15.64:0/3347735693 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e53fff640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5414fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.975+0000 7f7e537fe640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540af380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e58feb640 1 -- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 msgr2=0x7f7e540af380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e58feb640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540af380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e58feb640 1 -- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 msgr2=0x7f7e5414fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e58feb640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5414fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e58feb640 1 -- 172.21.15.64:0/3347735693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e540cb9c0 con 0x7f7e540cdcd0 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e537fe640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540af380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e53fff640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5414fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e58feb640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e540cdcd0 0x7f7e540b6e70 secure :-1 s=READY pgs=1310 cs=0 l=1 rev1=1 crypto rx=0x7f7e4c009d90 tx=0x7f7e4c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e50ff9640 1 -- 172.21.15.64:0/3347735693 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e4c039050 con 0x7f7e540cdcd0 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e50ff9640 1 -- 172.21.15.64:0/3347735693 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e4c045070 con 0x7f7e540cdcd0 2024-03-04T07:32:12.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7e540b6390 con 0x7f7e540cdcd0 2024-03-04T07:32:12.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e50ff9640 1 -- 172.21.15.64:0/3347735693 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e4c009450 con 0x7f7e540cdcd0 2024-03-04T07:32:12.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.976+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e540b6950 con 0x7f7e540cdcd0 2024-03-04T07:32:12.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.977+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7e5409fcf0 con 0x7f7e540cdcd0 2024-03-04T07:32:12.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.978+0000 7f7e50ff9640 1 -- 172.21.15.64:0/3347735693 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7e4c03e850 con 0x7f7e540cdcd0 2024-03-04T07:32:12.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.978+0000 7f7e50ff9640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e4006b4d0 0x7f7e4006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:12.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.978+0000 7f7e50ff9640 1 -- 172.21.15.64:0/3347735693 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7e4c0acf00 con 0x7f7e540cdcd0 2024-03-04T07:32:12.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.978+0000 7f7e53fff640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e4006b4d0 0x7f7e4006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:12.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.978+0000 7f7e53fff640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e4006b4d0 0x7f7e4006d990 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f7e44005880 tx=0x7f7e44009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:12.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:12.982+0000 7f7e50ff9640 1 -- 172.21.15.64:0/3347735693 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7e4c07b7d0 con 0x7f7e540cdcd0 2024-03-04T07:32:13.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.232+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f7e540d1690 con 0x7f7e540cdcd0 2024-03-04T07:32:13.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.233+0000 7f7e50ff9640 1 -- 172.21.15.64:0/3347735693 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v21) v1 ==== 190+0+147 (secure 0 0 0) 0x7f7e4c07f870 con 0x7f7e540cdcd0 2024-03-04T07:32:13.233 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:32:13.233 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:32:13.233 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:32:13.234 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:32:13.234 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:32:13.234 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:32:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e4006b4d0 msgr2=0x7f7e4006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e4006b4d0 0x7f7e4006d990 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f7e44005880 tx=0x7f7e44009040 comp rx=0 tx=0).stop 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e540cdcd0 msgr2=0x7f7e540b6e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e540cdcd0 0x7f7e540b6e70 secure :-1 s=READY pgs=1310 cs=0 l=1 rev1=1 crypto rx=0x7f7e4c009d90 tx=0x7f7e4c004a40 comp rx=0 tx=0).stop 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 shutdown_connections 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e4006b4d0 0x7f7e4006d990 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e540cdcd0 0x7f7e540b6e70 unknown :-1 s=CLOSED pgs=1310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e540b2cd0 0x7f7e540af380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 --2- 172.21.15.64:0/3347735693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e5406b2c0 0x7f7e5414fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 >> 172.21.15.64:0/3347735693 conn(0x7f7e540b3700 msgr2=0x7f7e540cd170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 shutdown_connections 2024-03-04T07:32:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.236+0000 7f7e5aa4e640 1 -- 172.21.15.64:0/3347735693 wait complete. 2024-03-04T07:32:13.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:13.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.349+0000 7f333e5f7640 1 Processor -- start 2024-03-04T07:32:13.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.349+0000 7f333e5f7640 1 -- start start 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333e5f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3338071c50 0x7f33380d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333e5f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f33380b2c50 0x7f33380d2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333e5f7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f33380ce970 con 0x7f333815e000 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333e5f7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f33380cece0 con 0x7f3338071c50 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333e5f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f33380cef30 con 0x7f33380b2c50 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333cb94640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f33380b2c50 0x7f33380d2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f33377fe640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3338071c50 0x7f33380d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333cb94640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f33380b2c50 0x7f33380d2b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:34744/0 (socket says 172.21.15.64:34744) 2024-03-04T07:32:13.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333cb94640 1 -- 172.21.15.64:0/3705239234 learned_addr learned my addr 172.21.15.64:0/3705239234 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:13.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f3337fff640 1 --1- 172.21.15.64:0/3705239234 >> v1:172.21.15.66:6789/0 conn(0x7f333815e000 0x7f3338071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:56916/0 (socket says 172.21.15.64:56916) 2024-03-04T07:32:13.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333cb94640 1 -- 172.21.15.64:0/3705239234 >> v2:172.21.15.66:3300/0 conn(0x7f3338071c50 msgr2=0x7f33380d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:13.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333cb94640 1 --2- 172.21.15.64:0/3705239234 >> v2:172.21.15.66:3300/0 conn(0x7f3338071c50 0x7f33380d25d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333cb94640 1 -- 172.21.15.64:0/3705239234 >> v1:172.21.15.66:6789/0 conn(0x7f333815e000 legacy=0x7f3338071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.350+0000 7f333cb94640 1 -- 172.21.15.64:0/3705239234 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33380cf210 con 0x7f33380b2c50 2024-03-04T07:32:13.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.351+0000 7f333cb94640 1 --2- 172.21.15.64:0/3705239234 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f33380b2c50 0x7f33380d2b10 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f3330009e50 tx=0x7f333002bed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=415e433621a5595 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:13.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.351+0000 7f3336ffd640 1 -- 172.21.15.64:0/3705239234 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f333002ca70 con 0x7f33380b2c50 2024-03-04T07:32:13.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.351+0000 7f3336ffd640 1 -- 172.21.15.64:0/3705239234 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f333002cc10 con 0x7f33380b2c50 2024-03-04T07:32:13.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.351+0000 7f3336ffd640 1 -- 172.21.15.64:0/3705239234 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f33300343f0 con 0x7f33380b2c50 2024-03-04T07:32:13.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 -- 172.21.15.64:0/3705239234 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f33380b2c50 msgr2=0x7f33380d2b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 --2- 172.21.15.64:0/3705239234 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f33380b2c50 0x7f33380d2b10 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f3330009e50 tx=0x7f333002bed0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 -- 172.21.15.64:0/3705239234 shutdown_connections 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 --2- 172.21.15.64:0/3705239234 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f33380b2c50 0x7f33380d2b10 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 --2- 172.21.15.64:0/3705239234 >> v2:172.21.15.66:3300/0 conn(0x7f3338071c50 0x7f33380d25d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 -- 172.21.15.64:0/3705239234 >> 172.21.15.64:0/3705239234 conn(0x7f33380d3c80 msgr2=0x7f33380d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 -- 172.21.15.64:0/3705239234 shutdown_connections 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.352+0000 7f333e5f7640 1 -- 172.21.15.64:0/3705239234 wait complete. 2024-03-04T07:32:13.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.353+0000 7f333e5f7640 1 Processor -- start 2024-03-04T07:32:13.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.353+0000 7f333e5f7640 1 -- start start 2024-03-04T07:32:13.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333e5f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3338071c50 0x7f33380cbf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:13.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333e5f7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33380b2c50 0x7f33380c9b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:13.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333e5f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f333815e000 0x7f33380c8a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333e5f7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33380b3320 con 0x7f3338071c50 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333e5f7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33380b31a0 con 0x7f33380b2c50 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333e5f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f333815e3e0 con 0x7f333815e000 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f3337fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3338071c50 0x7f33380cbf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f3337fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3338071c50 0x7f33380cbf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:52928/0 (socket says 172.21.15.64:52928) 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f3337fff640 1 -- 172.21.15.64:0/1990740664 learned_addr learned my addr 172.21.15.64:0/1990740664 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333cb94640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f333815e000 0x7f33380c8a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f33377fe640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33380b2c50 0x7f33380c9b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333cb94640 1 -- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33380b2c50 msgr2=0x7f33380c9b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333cb94640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33380b2c50 0x7f33380c9b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333cb94640 1 -- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3338071c50 msgr2=0x7f33380cbf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333cb94640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3338071c50 0x7f33380cbf60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333cb94640 1 -- 172.21.15.64:0/1990740664 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33380c79a0 con 0x7f333815e000 2024-03-04T07:32:13.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f3337fff640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3338071c50 0x7f33380cbf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.354+0000 7f333cb94640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f333815e000 0x7f33380c8a40 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f3330032000 tx=0x7f3330004a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.355+0000 7f3334ff9640 1 -- 172.21.15.64:0/1990740664 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3330043020 con 0x7f333815e000 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.355+0000 7f3334ff9640 1 -- 172.21.15.64:0/1990740664 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f333003e450 con 0x7f333815e000 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.355+0000 7f3334ff9640 1 -- 172.21.15.64:0/1990740664 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3330009510 con 0x7f333815e000 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.355+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f33380c7c80 con 0x7f333815e000 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.355+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f33380c8240 con 0x7f333815e000 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.356+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3338067ff0 con 0x7f333815e000 2024-03-04T07:32:13.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.356+0000 7f3334ff9640 1 -- 172.21.15.64:0/1990740664 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3330046020 con 0x7f333815e000 2024-03-04T07:32:13.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.356+0000 7f3334ff9640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f332406b4d0 0x7f332406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:13.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.356+0000 7f3334ff9640 1 -- 172.21.15.64:0/1990740664 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f33300ac710 con 0x7f333815e000 2024-03-04T07:32:13.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.356+0000 7f3337fff640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f332406b4d0 0x7f332406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:13.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.357+0000 7f3337fff640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f332406b4d0 0x7f332406d990 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f332c004e00 tx=0x7f332c007f10 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:13.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.361+0000 7f3334ff9640 1 -- 172.21.15.64:0/1990740664 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f333004d030 con 0x7f333815e000 2024-03-04T07:32:13.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.580+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f33380d2080 con 0x7f333815e000 2024-03-04T07:32:13.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.580+0000 7f3334ff9640 1 -- 172.21.15.64:0/1990740664 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 66 v66) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f333007afd0 con 0x7f333815e000 2024-03-04T07:32:13.580 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:13.580 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":66,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6281,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/2549379142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2549379142},{"type":"v1","addr":"172.21.15.66:6834","nonce":2549379142}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7854,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3068683808","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3068683808},{"type":"v1","addr":"172.21.15.64:6836","nonce":3068683808}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":62},{"gid":7857,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/209754994","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":209754994},{"type":"v1","addr":"172.21.15.64:6837","nonce":209754994}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63}],"filesystems":[{"mdsmap":{"epoch":66,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:10.823856+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7872},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7872":{"gid":7872,"name":"d","rank":0,"incarnation":64,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/4084900379","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":4084900379},{"type":"v1","addr":"172.21.15.66:6835","nonce":4084900379}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7872,"qdb_cluster":[7872]},"id":8}]} 2024-03-04T07:32:13.581 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 66 2024-03-04T07:32:13.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f332406b4d0 msgr2=0x7f332406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f332406b4d0 0x7f332406d990 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f332c004e00 tx=0x7f332c007f10 comp rx=0 tx=0).stop 2024-03-04T07:32:13.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f333815e000 msgr2=0x7f33380c8a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f333815e000 0x7f33380c8a40 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f3330032000 tx=0x7f3330004a40 comp rx=0 tx=0).stop 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 shutdown_connections 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f332406b4d0 0x7f332406d990 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f333815e000 0x7f33380c8a40 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f33380b2c50 0x7f33380c9b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 --2- 172.21.15.64:0/1990740664 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3338071c50 0x7f33380cbf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 >> 172.21.15.64:0/1990740664 conn(0x7f33380d3c80 msgr2=0x7f3338071540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 shutdown_connections 2024-03-04T07:32:13.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:13.582+0000 7f333e5f7640 1 -- 172.21.15.64:0/1990740664 wait complete. 2024-03-04T07:32:13.996 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 66, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:32:08.826165+0000', 'modified': '2024-03-04T07:32:10.823856+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 7872}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_7872': {'gid': 7872, 'name': 'd', 'rank': 0, 'incarnation': 64, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6835/4084900379', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 4084900379}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 4084900379}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 7872, 'qdb_cluster': [7872]} 2024-03-04T07:32:13.996 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:32:13.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.d status 2024-03-04T07:32:14.103 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.104+0000 7fc9a4d3e640 1 Processor -- start 2024-03-04T07:32:14.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.104+0000 7fc9a4d3e640 1 -- start start 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc9a4d3e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc9a006d400 0x7fc9a015cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc9a4d3e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9a00729b0 con 0x7fc9a00aee10 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc9a4d3e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9a006c920 con 0x7fc9a015bac0 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc9a4d3e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc9a006cf70 con 0x7fc9a006d400 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc99ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc9a006d400 0x7fc9a015cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc997fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc9a00aee10 0x7fc9a0070870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39314/0 (socket says 172.21.15.64:39314) 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc99ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc9a006d400 0x7fc9a015cbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43446/0 (socket says 172.21.15.64:43446) 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc997fff640 1 -- 172.21.15.64:0/2407536388 learned_addr learned my addr 172.21.15.64:0/2407536388 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc99ed9d640 1 -- 172.21.15.64:0/2407536388 >> v1:172.21.15.66:6789/0 conn(0x7fc9a015bac0 legacy=0x7fc9a00ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc99ed9d640 1 -- 172.21.15.64:0/2407536388 >> v1:172.21.15.64:6789/0 conn(0x7fc9a00aee10 legacy=0x7fc9a0070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc99dd9b640 1 -- 172.21.15.64:0/2407536388 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2201158939 0 0) 0x7fc9a00729b0 con 0x7fc9a00aee10 2024-03-04T07:32:14.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.105+0000 7fc99ed9d640 1 -- 172.21.15.64:0/2407536388 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9a006cc00 con 0x7fc9a006d400 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.106+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/2407536388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a006d400 0x7fc9a015cbe0 secure :-1 s=READY pgs=1312 cs=0 l=1 rev1=1 crypto rx=0x7fc998009d90 tx=0x7fc99802bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b00630710c969569 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.106+0000 7fc99dd9b640 1 -- 172.21.15.64:0/2407536388 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc998039070 con 0x7fc9a006d400 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.106+0000 7fc99dd9b640 1 -- 172.21.15.64:0/2407536388 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc99802cd00 con 0x7fc9a006d400 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.106+0000 7fc99dd9b640 1 -- 172.21.15.64:0/2407536388 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc998034660 con 0x7fc9a006d400 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.107+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/2407536388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a006d400 msgr2=0x7fc9a015cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.107+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/2407536388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a006d400 0x7fc9a015cbe0 secure :-1 s=READY pgs=1312 cs=0 l=1 rev1=1 crypto rx=0x7fc998009d90 tx=0x7fc99802bf50 comp rx=0 tx=0).stop 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.107+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/2407536388 shutdown_connections 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.107+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/2407536388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a006d400 0x7fc9a015cbe0 unknown :-1 s=CLOSED pgs=1312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.107+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/2407536388 >> 172.21.15.64:0/2407536388 conn(0x7fc9a0066d00 msgr2=0x7fc9a00670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.107+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/2407536388 shutdown_connections 2024-03-04T07:32:14.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.107+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/2407536388 wait complete. 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.108+0000 7fc9a4d3e640 1 Processor -- start 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.108+0000 7fc9a4d3e640 1 -- start start 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc9a4d3e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9a006d400 0x7fc9a014bfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc9a4d3e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9a00aee10 0x7fc9a01e51e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc9a4d3e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a015bac0 0x7fc9a00ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc9a4d3e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9a015c020 con 0x7fc9a015bac0 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc9a4d3e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9a015bea0 con 0x7fc9a00aee10 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc9a4d3e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9a0072580 con 0x7fc9a006d400 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a015bac0 0x7fc9a00ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a015bac0 0x7fc9a00ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43462/0 (socket says 172.21.15.64:43462) 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 -- 172.21.15.64:0/4046268432 learned_addr learned my addr 172.21.15.64:0/4046268432 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:14.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99e59c640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9a006d400 0x7fc9a014bfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc997fff640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9a00aee10 0x7fc9a01e51e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 -- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9a006d400 msgr2=0x7fc9a014bfc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9a006d400 0x7fc9a014bfc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 -- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9a00aee10 msgr2=0x7fc9a01e51e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9a00aee10 0x7fc9a01e51e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 -- 172.21.15.64:0/4046268432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9a014c950 con 0x7fc9a015bac0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.109+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a015bac0 0x7fc9a00ad8b0 secure :-1 s=READY pgs=1313 cs=0 l=1 rev1=1 crypto rx=0x7fc998009d90 tx=0x7fc998004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.110+0000 7fc996ffd640 1 -- 172.21.15.64:0/4046268432 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc998044020 con 0x7fc9a015bac0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.110+0000 7fc996ffd640 1 -- 172.21.15.64:0/4046268432 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc99803a040 con 0x7fc9a015bac0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.110+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc9a01e4420 con 0x7fc9a015bac0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.110+0000 7fc996ffd640 1 -- 172.21.15.64:0/4046268432 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc998039050 con 0x7fc9a015bac0 2024-03-04T07:32:14.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.110+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc9a01e49e0 con 0x7fc9a015bac0 2024-03-04T07:32:14.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.111+0000 7fc996ffd640 1 -- 172.21.15.64:0/4046268432 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc99802cb80 con 0x7fc9a015bac0 2024-03-04T07:32:14.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.111+0000 7fc996ffd640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc98006b4d0 0x7fc98006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.111+0000 7fc996ffd640 1 -- 172.21.15.64:0/4046268432 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc9980ac750 con 0x7fc9a015bac0 2024-03-04T07:32:14.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.111+0000 7fc99e59c640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc98006b4d0 0x7fc98006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.112+0000 7fc99e59c640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc98006b4d0 0x7fc98006d990 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7fc990000f60 tx=0x7fc990009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.112+0000 7fc9a7a07740 1 Processor -- start 2024-03-04T07:32:14.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.112+0000 7fc9a7a07740 1 -- start start 2024-03-04T07:32:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba471870 0x556bba471c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba4758c0 0x556bba477360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4a1a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x556bba3db6b0 con 0x556bba4758c0 2024-03-04T07:32:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x556bba384a60 con 0x556bba471870 2024-03-04T07:32:14.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x556bba3d7750 con 0x556bba4a1600 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc997fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba4758c0 0x556bba477360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc997fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba4758c0 0x556bba477360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43470/0 (socket says 172.21.15.64:43470) 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc997fff640 1 -- 172.21.15.64:0/61867707 learned_addr learned my addr 172.21.15.64:0/61867707 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc99e59c640 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba471870 0x556bba471c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.113+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4a1a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc997fff640 1 -- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 msgr2=0x556bba4a1a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc997fff640 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4a1a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc997fff640 1 -- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba471870 msgr2=0x556bba471c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc997fff640 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba471870 0x556bba471c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc997fff640 1 -- 172.21.15.64:0/61867707 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556bba3a2f30 con 0x556bba4758c0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc997fff640 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba4758c0 0x556bba477360 secure :-1 s=READY pgs=1314 cs=0 l=1 rev1=1 crypto rx=0x7fc988007650 tx=0x7fc988004bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f3a8d675d4902c2e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc97ffff640 1 -- 172.21.15.64:0/61867707 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc988001890 con 0x556bba4758c0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc97ffff640 1 -- 172.21.15.64:0/61867707 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc988018070 con 0x556bba4758c0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc97ffff640 1 -- 172.21.15.64:0/61867707 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc988001c50 con 0x556bba4758c0 2024-03-04T07:32:14.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc9a7a07740 1 -- 172.21.15.64:0/61867707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba4758c0 msgr2=0x556bba477360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba4758c0 0x556bba477360 secure :-1 s=READY pgs=1314 cs=0 l=1 rev1=1 crypto rx=0x7fc988007650 tx=0x7fc988004bb0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc9a7a07740 1 -- 172.21.15.64:0/61867707 shutdown_connections 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4a1a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.114+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba4758c0 0x556bba477360 unknown :-1 s=CLOSED pgs=1314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.115+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/61867707 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba471870 0x556bba471c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.115+0000 7fc9a7a07740 1 -- 172.21.15.64:0/61867707 >> 172.21.15.64:0/61867707 conn(0x556bba4738f0 msgr2=0x556bba4751b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.115+0000 7fc9a7a07740 1 -- 172.21.15.64:0/61867707 shutdown_connections 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.115+0000 7fc9a7a07740 1 -- 172.21.15.64:0/61867707 wait complete. 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.115+0000 7fc9a7a07740 1 -- start start 2024-03-04T07:32:14.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.115+0000 7fc9a7a07740 1 Processor -- start 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4bd220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba49caf0 0x556bba4ba160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4a2a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x556bba475d50 con 0x556bba49c020 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x556bba397ea0 con 0x556bba49caf0 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x556bba476030 con 0x556bba4a1600 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc9a7a07740 10 client.0 _ll_register_callbacks cb 0x556bba397870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc99e59c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4bd220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc99ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4a2a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc997fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba49caf0 0x556bba4ba160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc99e59c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4bd220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:60844/0 (socket says 172.21.15.64:60844) 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc99e59c640 1 -- 172.21.15.64:0/1843257642 learned_addr learned my addr 172.21.15.64:0/1843257642 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.116+0000 7fc997fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba49caf0 0x556bba4ba160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:37954/0 (socket says 172.21.15.64:37954) 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc99e59c640 1 -- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba49caf0 msgr2=0x556bba4ba160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc99e59c640 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba49caf0 0x556bba4ba160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc99e59c640 1 -- 172.21.15.64:0/1843257642 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 msgr2=0x556bba4a2a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc99e59c640 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4a2a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc99e59c640 1 -- 172.21.15.64:0/1843257642 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x556bba398270 con 0x556bba4a1600 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4a2a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc99e59c640 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4bd220 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fc990014730 tx=0x7fc990001080 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3c1f8dbd86a7b23f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc97d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.66:3301/0 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc97d7fa640 1 -- 172.21.15.64:0/1843257642 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc990025020 con 0x556bba4a1600 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc97d7fa640 1 -- 172.21.15.64:0/1843257642 <== mon.2 v2:172.21.15.66:3301/0 2 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc990013740 con 0x556bba4a1600 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc988018070 con 0x556bba4a1600 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc97d7fa640 1 -- 172.21.15.64:0/1843257642 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_get_version_reply(handle=1 version=66) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc99002f050 con 0x556bba4a1600 2024-03-04T07:32:14.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 10 client.6356 fetch_fsmap learned FSMap version 66 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,fsmap=66,monmap=2+}) v3 -- 0x556bba0f02d0 con 0x556bba4a1600 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc97d7fa640 1 -- 172.21.15.64:0/1843257642 <== mon.2 v2:172.21.15.66:3301/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc988018070 con 0x556bba4a1600 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc97d7fa640 1 -- 172.21.15.64:0/1843257642 <== mon.2 v2:172.21.15.66:3301/0 5 ==== fsmap(e 66) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fc990032020 con 0x556bba4a1600 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 10 client.6356 fetch_fsmap finished waiting for FSMap version 66 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 10 client.6356 resolve_mds: resolved name 'd' to daemon mds.d 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 20 client.6356 populate_metadata read hostname 'smithi064' 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4b7ba0 0x556bba4dbbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 4 client.6356 mds_command: new command op to 7872 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.117+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 --> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x556bba3e8bf0 con 0x556bba4b7ba0 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.118+0000 7fc997fff640 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4b7ba0 0x556bba4dbbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.118+0000 7fc997fff640 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4b7ba0 0x556bba4dbbe0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=20cbb6c8b96c1465 server_cookie=663060c54b827902 in_seq=0 out_seq=0 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.118+0000 7fc97d7fa640 10 client.6356 ms_handle_connect on v2:172.21.15.66:6833/4084900379 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc97d7fa640 1 -- 172.21.15.64:0/1843257642 <== mds.0 v2:172.21.15.66:6833/4084900379 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x556bba3e8bf0 con 0x556bba4b7ba0 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc97d7fa640 10 client.6356 handle_command_reply: tid=0 2024-03-04T07:32:14.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 client.6356 shutdown 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 msgr2=0x556bba4bd220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4bd220 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fc990014730 tx=0x7fc990001080 comp rx=0 tx=0).stop 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 shutdown_connections 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4b7ba0 0x556bba4dbbe0 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4a2a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba49caf0 0x556bba4ba160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc97d7fa640 0 client.6356 ms_handle_reset on v2:172.21.15.66:6833/4084900379 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/1843257642 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba4a1600 0x556bba4bd220 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 >> 172.21.15.64:0/1843257642 conn(0x556bba471870 msgr2=0x556bba470510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 shutdown_connections 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 1 -- 172.21.15.64:0/1843257642 wait complete. 2024-03-04T07:32:14.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.119+0000 7fc9a7a07740 20 client.6356 trim_cache size 0 max 16384 2024-03-04T07:32:14.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 Processor -- start 2024-03-04T07:32:14.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 -- start start 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49c020 0x556bba49afd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49caf0 0x556bba49b510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba477040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9a00b3780 con 0x556bba49caf0 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9a00b2b80 con 0x556bba4a1600 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.136+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9a00b3180 con 0x556bba49c020 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49caf0 0x556bba49b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc99e59c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49c020 0x556bba49afd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49caf0 0x556bba49b510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43480/0 (socket says 172.21.15.64:43480) 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 -- 172.21.15.64:0/2014844917 learned_addr learned my addr 172.21.15.64:0/2014844917 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc99ed9d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba477040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 -- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49c020 msgr2=0x556bba49afd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49c020 0x556bba49afd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 -- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 msgr2=0x556bba477040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba477040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 -- 172.21.15.64:0/2014844917 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556bba1ba6b0 con 0x556bba49caf0 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc99e59c640 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49c020 0x556bba49afd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba477040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc997fff640 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49caf0 0x556bba49b510 secure :-1 s=READY pgs=1315 cs=0 l=1 rev1=1 crypto rx=0x556bba4a5cf0 tx=0x7fc988009030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2b5b5fda4aa1847 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc97f7fe640 1 -- 172.21.15.64:0/2014844917 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc988001d40 con 0x556bba49caf0 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc97f7fe640 1 -- 172.21.15.64:0/2014844917 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc988018070 con 0x556bba49caf0 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.137+0000 7fc97f7fe640 1 -- 172.21.15.64:0/2014844917 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc988029270 con 0x556bba49caf0 2024-03-04T07:32:14.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2014844917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49caf0 msgr2=0x556bba49b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49caf0 0x556bba49b510 secure :-1 s=READY pgs=1315 cs=0 l=1 rev1=1 crypto rx=0x556bba4a5cf0 tx=0x7fc988009030 comp rx=0 tx=0).stop 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2014844917 shutdown_connections 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba477040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49caf0 0x556bba49b510 unknown :-1 s=CLOSED pgs=1315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2014844917 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49c020 0x556bba49afd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2014844917 >> 172.21.15.64:0/2014844917 conn(0x556bba471870 msgr2=0x556bba475080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2014844917 shutdown_connections 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2014844917 wait complete. 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 -- start start 2024-03-04T07:32:14.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.138+0000 7fc9a7a07740 1 Processor -- start 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4dd6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49caf0 0x556bba4b7f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc9a7a07740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba4b84c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x556bba4a3530 con 0x556bba49c020 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x556bba4bca80 con 0x556bba4a1600 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc9a7a07740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x556bba4bc5d0 con 0x556bba49caf0 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc9a7a07740 10 client.0 _ll_register_callbacks cb 0x7fc99000fed0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:32:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc99e59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4dd6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc99e59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4dd6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43494/0 (socket says 172.21.15.64:43494) 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc99e59c640 1 -- 172.21.15.64:0/2515990429 learned_addr learned my addr 172.21.15.64:0/2515990429 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc99ed9d640 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba4b84c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.139+0000 7fc997fff640 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49caf0 0x556bba4b7f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc99e59c640 1 -- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49caf0 msgr2=0x556bba4b7f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc99e59c640 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49caf0 0x556bba4b7f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc99e59c640 1 -- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 msgr2=0x556bba4b84c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc99e59c640 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba4b84c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc99e59c640 1 -- 172.21.15.64:0/2515990429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x556bba478600 con 0x556bba49c020 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc99e59c640 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4dd6c0 secure :-1 s=READY pgs=1316 cs=0 l=1 rev1=1 crypto rx=0x7fc990014730 tx=0x7fc990014000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ead2f670d51ad2ac server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc97d7fa640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc97d7fa640 1 -- 172.21.15.64:0/2515990429 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc990025070 con 0x556bba49c020 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc97d7fa640 1 -- 172.21.15.64:0/2515990429 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fc990021600 con 0x556bba49c020 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc988018070 con 0x556bba49c020 2024-03-04T07:32:14.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc97d7fa640 1 -- 172.21.15.64:0/2515990429 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=66) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc990040050 con 0x556bba49c020 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc9a7a07740 10 client.8004 fetch_fsmap learned FSMap version 66 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=66,monmap=2+}) v3 -- 0x556bba4b6f00 con 0x556bba49c020 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc97d7fa640 1 -- 172.21.15.64:0/2515990429 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc988018070 con 0x556bba49c020 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc97d7fa640 1 -- 172.21.15.64:0/2515990429 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 66) v1 ==== 3314+0+0 (secure 0 0 0) 0x7fc990043020 con 0x556bba49c020 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc9a7a07740 10 client.8004 fetch_fsmap finished waiting for FSMap version 66 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc9a7a07740 10 client.8004 resolve_mds: resolved name 'd' to daemon mds.d 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.140+0000 7fc9a7a07740 20 client.8004 populate_metadata read hostname 'smithi064' 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4dc010 0x556bba476030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 4 client.8004 mds_command: new command op to 7872 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 --> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] -- command(tid 0: {"prefix": "status"}) v1 -- 0x556bba474050 con 0x556bba4dc010 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc997fff640 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4dc010 0x556bba476030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc97d7fa640 10 client.8004 ms_handle_connect on v2:172.21.15.66:6833/4084900379 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc997fff640 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4dc010 0x556bba476030 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=89ffb2783c3f57c8 server_cookie=6bdb7ed6ff5c6607 in_seq=0 out_seq=0 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc97d7fa640 1 -- 172.21.15.64:0/2515990429 <== mds.0 v2:172.21.15.66:6833/4084900379 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+333 (crc 0 0 0) 0x556bba474050 con 0x556bba4dc010 2024-03-04T07:32:14.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc97d7fa640 10 client.8004 handle_command_reply: tid=0 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 client.8004 shutdown 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 msgr2=0x556bba4dd6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4dd6c0 secure :-1 s=READY pgs=1316 cs=0 l=1 rev1=1 crypto rx=0x7fc990014730 tx=0x7fc990014000 comp rx=0 tx=0).stop 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 shutdown_connections 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:6833/4084900379,v1:172.21.15.66:6835/4084900379] conn(0x556bba4dc010 0x556bba476030 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x556bba4a1600 0x556bba4b84c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x556bba49caf0 0x556bba4b7f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc97d7fa640 0 client.8004 ms_handle_reset on v2:172.21.15.66:6833/4084900379 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 --2- 172.21.15.64:0/2515990429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x556bba49c020 0x556bba4dd6c0 unknown :-1 s=CLOSED pgs=1316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.141+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 >> 172.21.15.64:0/2515990429 conn(0x556bba471870 msgr2=0x556bba49afd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 shutdown_connections 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a7a07740 1 -- 172.21.15.64:0/2515990429 wait complete. 2024-03-04T07:32:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a7a07740 20 client.8004 trim_cache size 0 max 16384 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 7872, 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 4.6696847420000003, 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 66, 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 105, 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 105, 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 9.6846455369999997 2024-03-04T07:32:14.148 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:32:14.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc98006b4d0 msgr2=0x7fc98006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc98006b4d0 0x7fc98006d990 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7fc990000f60 tx=0x7fc990009040 comp rx=0 tx=0).stop 2024-03-04T07:32:14.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a015bac0 msgr2=0x7fc9a00ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a015bac0 0x7fc9a00ad8b0 secure :-1 s=READY pgs=1313 cs=0 l=1 rev1=1 crypto rx=0x7fc998009d90 tx=0x7fc998004a40 comp rx=0 tx=0).stop 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 shutdown_connections 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc98006b4d0 0x7fc98006d990 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9a015bac0 0x7fc9a00ad8b0 unknown :-1 s=CLOSED pgs=1313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9a00aee10 0x7fc9a01e51e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 --2- 172.21.15.64:0/4046268432 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9a006d400 0x7fc9a014bfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 >> 172.21.15.64:0/4046268432 conn(0x7fc9a0066d00 msgr2=0x7fc9a015e5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 shutdown_connections 2024-03-04T07:32:14.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.142+0000 7fc9a4d3e640 1 -- 172.21.15.64:0/4046268432 wait complete. 2024-03-04T07:32:14.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:32:14.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.254+0000 7fd44d93e640 1 Processor -- start 2024-03-04T07:32:14.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd44d93e640 1 -- start start 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd44d93e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd44806ed10 0x7fd44806af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd44d93e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd4480cc0e0 con 0x7fd4480b1020 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd44d93e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd4480cc3c0 con 0x7fd4480cd890 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd44d93e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd4480d28b0 con 0x7fd44806ed10 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd446ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd4480b1020 0x7fd4480cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39322/0 (socket says 172.21.15.64:39322) 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd446ffd640 1 -- 172.21.15.64:0/3266630734 learned_addr learned my addr 172.21.15.64:0/3266630734 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.255+0000 7fd4467fc640 1 --2- 172.21.15.64:0/3266630734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd44806ed10 0x7fd44806af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.256+0000 7fd445ffb640 1 -- 172.21.15.64:0/3266630734 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 348557263 0 0) 0x7fd4480cc0e0 con 0x7fd4480b1020 2024-03-04T07:32:14.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.256+0000 7fd445ffb640 1 -- 172.21.15.64:0/3266630734 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd434003520 con 0x7fd4480b1020 2024-03-04T07:32:14.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.256+0000 7fd445ffb640 1 -- 172.21.15.64:0/3266630734 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3760944580 0 0) 0x7fd4480cc3c0 con 0x7fd4480cd890 2024-03-04T07:32:14.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.256+0000 7fd445ffb640 1 -- 172.21.15.64:0/3266630734 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd4480cc0e0 con 0x7fd4480cd890 2024-03-04T07:32:14.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.256+0000 7fd4467fc640 1 -- 172.21.15.64:0/3266630734 >> v1:172.21.15.66:6789/0 conn(0x7fd4480cd890 legacy=0x7fd44815d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.256+0000 7fd4467fc640 1 -- 172.21.15.64:0/3266630734 >> v1:172.21.15.64:6789/0 conn(0x7fd4480b1020 legacy=0x7fd4480cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.256+0000 7fd4467fc640 1 -- 172.21.15.64:0/3266630734 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4480cc6a0 con 0x7fd44806ed10 2024-03-04T07:32:14.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd4467fc640 1 --2- 172.21.15.64:0/3266630734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd44806ed10 0x7fd44806af20 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fd4300097d0 tx=0x7fd43002b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dd10d86d85a20b66 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd445ffb640 1 -- 172.21.15.64:0/3266630734 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd430035020 con 0x7fd44806ed10 2024-03-04T07:32:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd445ffb640 1 -- 172.21.15.64:0/3266630734 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd430031890 con 0x7fd44806ed10 2024-03-04T07:32:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd445ffb640 1 -- 172.21.15.64:0/3266630734 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd430031c40 con 0x7fd44806ed10 2024-03-04T07:32:14.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd44d93e640 1 -- 172.21.15.64:0/3266630734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd44806ed10 msgr2=0x7fd44806af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd44d93e640 1 --2- 172.21.15.64:0/3266630734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd44806ed10 0x7fd44806af20 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fd4300097d0 tx=0x7fd43002b930 comp rx=0 tx=0).stop 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd44d93e640 1 -- 172.21.15.64:0/3266630734 shutdown_connections 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd44d93e640 1 --2- 172.21.15.64:0/3266630734 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd44806ed10 0x7fd44806af20 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd44d93e640 1 -- 172.21.15.64:0/3266630734 >> 172.21.15.64:0/3266630734 conn(0x7fd4480d1a90 msgr2=0x7fd4480d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd44d93e640 1 -- 172.21.15.64:0/3266630734 shutdown_connections 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.257+0000 7fd44d93e640 1 -- 172.21.15.64:0/3266630734 wait complete. 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.258+0000 7fd44d93e640 1 Processor -- start 2024-03-04T07:32:14.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd44d93e640 1 -- start start 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd44d93e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd44806ed10 0x7fd448157920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd44d93e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4480b1020 0x7fd4480b68b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd44d93e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4480cd890 0x7fd44815fae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd44d93e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4480ccc60 con 0x7fd44806ed10 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd44d93e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4480cf320 con 0x7fd4480b1020 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd44d93e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd4480cdce0 con 0x7fd4480cd890 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd446ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd44806ed10 0x7fd448157920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd446ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd44806ed10 0x7fd448157920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43500/0 (socket says 172.21.15.64:43500) 2024-03-04T07:32:14.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd446ffd640 1 -- 172.21.15.64:0/2578193483 learned_addr learned my addr 172.21.15.64:0/2578193483 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd4477fe640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4480cd890 0x7fd44815fae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.259+0000 7fd4467fc640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4480b1020 0x7fd4480b68b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd446ffd640 1 -- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4480cd890 msgr2=0x7fd44815fae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd446ffd640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4480cd890 0x7fd44815fae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd446ffd640 1 -- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4480b1020 msgr2=0x7fd4480b68b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd446ffd640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4480b1020 0x7fd4480b68b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd446ffd640 1 -- 172.21.15.64:0/2578193483 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4480b6f40 con 0x7fd44806ed10 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd4467fc640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4480b1020 0x7fd4480b68b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd4477fe640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4480cd890 0x7fd44815fae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd446ffd640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd44806ed10 0x7fd448157920 secure :-1 s=READY pgs=1318 cs=0 l=1 rev1=1 crypto rx=0x7fd44000c530 tx=0x7fd44000c9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd43ffff640 1 -- 172.21.15.64:0/2578193483 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd440017070 con 0x7fd44806ed10 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4480c56f0 con 0x7fd44806ed10 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd43ffff640 1 -- 172.21.15.64:0/2578193483 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd440005d00 con 0x7fd44806ed10 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd43ffff640 1 -- 172.21.15.64:0/2578193483 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4400072b0 con 0x7fd44806ed10 2024-03-04T07:32:14.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.260+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd4480c5c70 con 0x7fd44806ed10 2024-03-04T07:32:14.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.261+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd448067ff0 con 0x7fd44806ed10 2024-03-04T07:32:14.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.262+0000 7fd43ffff640 1 -- 172.21.15.64:0/2578193483 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd44001a070 con 0x7fd44806ed10 2024-03-04T07:32:14.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.262+0000 7fd43ffff640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd42806b4d0 0x7fd42806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:14.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.262+0000 7fd4467fc640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd42806b4d0 0x7fd42806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:14.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.262+0000 7fd43ffff640 1 -- 172.21.15.64:0/2578193483 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd440089530 con 0x7fd44806ed10 2024-03-04T07:32:14.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.263+0000 7fd4467fc640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd42806b4d0 0x7fd42806d990 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7fd430036000 tx=0x7fd430008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:14.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.267+0000 7fd43ffff640 1 -- 172.21.15.64:0/2578193483 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd440057e30 con 0x7fd44806ed10 2024-03-04T07:32:14.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.486+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fd4480cf0d0 con 0x7fd44806ed10 2024-03-04T07:32:14.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.487+0000 7fd43ffff640 1 -- 172.21.15.64:0/2578193483 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v66) v1 ==== 53+0+73 (secure 0 0 0) 0x7fd44005bed0 con 0x7fd44806ed10 2024-03-04T07:32:14.486 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd42806b4d0 msgr2=0x7fd42806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd42806b4d0 0x7fd42806d990 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7fd430036000 tx=0x7fd430008030 comp rx=0 tx=0).stop 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd44806ed10 msgr2=0x7fd448157920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd44806ed10 0x7fd448157920 secure :-1 s=READY pgs=1318 cs=0 l=1 rev1=1 crypto rx=0x7fd44000c530 tx=0x7fd44000c9f0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 shutdown_connections 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd42806b4d0 0x7fd42806d990 unknown :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4480cd890 0x7fd44815fae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4480b1020 0x7fd4480b68b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 --2- 172.21.15.64:0/2578193483 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd44806ed10 0x7fd448157920 unknown :-1 s=CLOSED pgs=1318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 >> 172.21.15.64:0/2578193483 conn(0x7fd4480d1a90 msgr2=0x7fd44806bff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:14.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 shutdown_connections 2024-03-04T07:32:14.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:14.489+0000 7fd44d93e640 1 -- 172.21.15.64:0/2578193483 wait complete. 2024-03-04T07:32:14.499 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:32:14.500 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:32:14.500 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:14.500 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:32:14.515 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:32:14.515 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:32:14.515 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:32:14.515 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:32:14.516 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42378sec preferred_lft 42378sec 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:32:14.517 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:14.518 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:32:14.518 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:14.518 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:14.518 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:32:14.580 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:32:14.582 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:14.582 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:32:14.637 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:32:14.637 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:32:14.637 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:32:14.637 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:32:14.638 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:32:14.638 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:14.638 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:32:14.638 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:32:14.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:32:14.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:32:14.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:32:14.638 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:32:14.732 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:14.732 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:32:14.746 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:32:14.746 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:32:14.747 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:32:14.747 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:14.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:32:14.813 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:32:14.813 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:32:14.813 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:14.867 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:32:14.867 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:32:14.867 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:14.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:32:14.988 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:32:15.043 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:32:15.044 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:15.045 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:15.046 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.046 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:32:15.100 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:32:15.101 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:32:15.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:32:15.170 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:32:15.226 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:32:15.227 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:32:15.228 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:32:15.228 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:15.228 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:15.228 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:15.228 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:15.228 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.228 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:32:15.230 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:32:15.230+0000 7f828683c480 -1 init, newargv = 0x55febd5cb5c0 newargc=15 2024-03-04T07:32:15.230 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[72745]: starting ceph client 2024-03-04T07:32:15.234 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[72745]: starting fuse 2024-03-04T07:32:15.244 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:32:15.244 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:32:15.245 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:32:15.246 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:32:15.365 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.72745.asok 2024-03-04T07:32:15.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:32:15.375 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.72745.asok 2024-03-04T07:32:15.375 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.72745.asok status 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:32:15.513 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "72745", 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 6374, 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 6374 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "172.21.15.64:0", 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 92723010 2024-03-04T07:32:15.514 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "172.21.15.64:0", 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 92723010 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.6374 172.21.15.64:0/92723010", 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "172.21.15.64:0/92723010", 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 66, 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 105, 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:32:15.515 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:32:15.521 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.522 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:32:15.522 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:15.536 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:32:15.536 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:15.537 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:15.603 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.604 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec mkdir dir_1) 2024-03-04T07:32:15.663 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:32:15.663 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:32:15.717 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.717 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec dd if=/dev/urandom of=dir_1/file_a bs=1M conv=fdatasync count=8 seek=0) 2024-03-04T07:32:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:8+0 records in 2024-03-04T07:32:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:8+0 records out 2024-03-04T07:32:15.834 INFO:teuthology.orchestra.run.smithi064.stderr:8388608 bytes (8.4 MB, 8.0 MiB) copied, 0.059603 s, 141 MB/s 2024-03-04T07:32:15.835 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:32:15.835 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:32:15.835 DEBUG:teuthology.orchestra.run.smithi064:> import stat 2024-03-04T07:32:15.835 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:15.835 DEBUG:teuthology.orchestra.run.smithi064:> print(os.stat("/home/ubuntu/cephtest/mnt.0/dir_1/file_a").st_ino) 2024-03-04T07:32:15.835 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:32:15.900 INFO:teuthology.orchestra.run.smithi064.stdout:1099511627777 2024-03-04T07:32:15.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:15 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:32:15.909 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.909 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec mkdir dir_2) 2024-03-04T07:32:15.965 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:15.965 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec ln dir_1/file_a dir_2/file_b) 2024-03-04T07:32:16.024 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:32:16.024 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:32:16.024 DEBUG:teuthology.orchestra.run.smithi064:> import stat 2024-03-04T07:32:16.024 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:16.025 DEBUG:teuthology.orchestra.run.smithi064:> print(os.stat("/home/ubuntu/cephtest/mnt.0/dir_2/file_b").st_ino) 2024-03-04T07:32:16.025 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:32:16.129 INFO:teuthology.orchestra.run.smithi064.stdout:1099511627777 2024-03-04T07:32:16.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:32:16.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:16.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.280+0000 7fb61aff4640 1 Processor -- start 2024-03-04T07:32:16.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.280+0000 7fb61aff4640 1 -- start start 2024-03-04T07:32:16.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb61aff4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61415bac0 0x7fb61415cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb61aff4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb61406d400 0x7fb6140ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb61aff4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6140729b0 con 0x7fb6140aec40 2024-03-04T07:32:16.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb61aff4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb6140af0b0 con 0x7fb61406d400 2024-03-04T07:32:16.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb61aff4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb61406cf70 con 0x7fb61415bac0 2024-03-04T07:32:16.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb618d90640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61415bac0 0x7fb61415cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb613fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb61406d400 0x7fb6140ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb618d90640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61415bac0 0x7fb61415cbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:60874/0 (socket says 172.21.15.64:60874) 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb618d90640 1 -- 172.21.15.64:0/1060741251 learned_addr learned my addr 172.21.15.64:0/1060741251 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb619591640 1 --1- 172.21.15.64:0/1060741251 >> v1:172.21.15.66:6789/0 conn(0x7fb6140aec40 0x7fb614070870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:33428/0 (socket says 172.21.15.64:33428) 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb618d90640 1 -- 172.21.15.64:0/1060741251 >> v2:172.21.15.66:3300/0 conn(0x7fb61406d400 msgr2=0x7fb6140ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb618d90640 1 --2- 172.21.15.64:0/1060741251 >> v2:172.21.15.66:3300/0 conn(0x7fb61406d400 0x7fb6140ae700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb618d90640 1 -- 172.21.15.64:0/1060741251 >> v1:172.21.15.66:6789/0 conn(0x7fb6140aec40 legacy=0x7fb614070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.281+0000 7fb618d90640 1 -- 172.21.15.64:0/1060741251 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb61406c920 con 0x7fb61415bac0 2024-03-04T07:32:16.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.282+0000 7fb618d90640 1 --2- 172.21.15.64:0/1060741251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61415bac0 0x7fb61415cbe0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fb60c009e50 tx=0x7fb60c02bf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f10e2b27ee31baf6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.282+0000 7fb6137fe640 1 -- 172.21.15.64:0/1060741251 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb60c02cb00 con 0x7fb61415bac0 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.282+0000 7fb6137fe640 1 -- 172.21.15.64:0/1060741251 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb60c02cca0 con 0x7fb61415bac0 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.282+0000 7fb6137fe640 1 -- 172.21.15.64:0/1060741251 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb60c034610 con 0x7fb61415bac0 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 -- 172.21.15.64:0/1060741251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61415bac0 msgr2=0x7fb61415cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 --2- 172.21.15.64:0/1060741251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61415bac0 0x7fb61415cbe0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fb60c009e50 tx=0x7fb60c02bf60 comp rx=0 tx=0).stop 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 -- 172.21.15.64:0/1060741251 shutdown_connections 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 --2- 172.21.15.64:0/1060741251 >> v2:172.21.15.66:3300/0 conn(0x7fb61406d400 0x7fb6140ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 --2- 172.21.15.64:0/1060741251 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61415bac0 0x7fb61415cbe0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 -- 172.21.15.64:0/1060741251 >> 172.21.15.64:0/1060741251 conn(0x7fb614066d00 msgr2=0x7fb6140670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 -- 172.21.15.64:0/1060741251 shutdown_connections 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.283+0000 7fb61aff4640 1 -- 172.21.15.64:0/1060741251 wait complete. 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.284+0000 7fb61aff4640 1 Processor -- start 2024-03-04T07:32:16.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.284+0000 7fb61aff4640 1 -- start start 2024-03-04T07:32:16.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb61aff4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61406d400 0x7fb614151a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb61aff4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6140aec40 0x7fb614153820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb61aff4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb61415bac0 0x7fb61414fc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb61aff4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb61415bea0 con 0x7fb6140aec40 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb61aff4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6140cc490 con 0x7fb61415bac0 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb61aff4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb61415c020 con 0x7fb61406d400 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6140aec40 0x7fb614153820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6140aec40 0x7fb614153820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43512/0 (socket says 172.21.15.64:43512) 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 -- 172.21.15.64:0/368002237 learned_addr learned my addr 172.21.15.64:0/368002237 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb619591640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb61415bac0 0x7fb61414fc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb618d90640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61406d400 0x7fb614151a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 -- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61406d400 msgr2=0x7fb614151a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61406d400 0x7fb614151a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 -- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb61415bac0 msgr2=0x7fb61414fc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb61415bac0 0x7fb61414fc50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.285+0000 7fb613fff640 1 -- 172.21.15.64:0/368002237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6141506d0 con 0x7fb6140aec40 2024-03-04T07:32:16.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.286+0000 7fb613fff640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6140aec40 0x7fb614153820 secure :-1 s=READY pgs=1319 cs=0 l=1 rev1=1 crypto rx=0x7fb60400e9b0 tx=0x7fb60400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:16.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.286+0000 7fb6117fa640 1 -- 172.21.15.64:0/368002237 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb604017070 con 0x7fb6140aec40 2024-03-04T07:32:16.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.286+0000 7fb6117fa640 1 -- 172.21.15.64:0/368002237 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb60400c9a0 con 0x7fb6140aec40 2024-03-04T07:32:16.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.286+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6141509b0 con 0x7fb6140aec40 2024-03-04T07:32:16.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.286+0000 7fb6117fa640 1 -- 172.21.15.64:0/368002237 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb60400cd90 con 0x7fb6140aec40 2024-03-04T07:32:16.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.286+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb61414d660 con 0x7fb6140aec40 2024-03-04T07:32:16.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.287+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb614150ed0 con 0x7fb6140aec40 2024-03-04T07:32:16.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.287+0000 7fb6117fa640 1 -- 172.21.15.64:0/368002237 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb60401a070 con 0x7fb6140aec40 2024-03-04T07:32:16.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.288+0000 7fb6117fa640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5f406b4d0 0x7fb5f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.288+0000 7fb6117fa640 1 -- 172.21.15.64:0/368002237 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb60408a540 con 0x7fb6140aec40 2024-03-04T07:32:16.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.288+0000 7fb618d90640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5f406b4d0 0x7fb5f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.288+0000 7fb618d90640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5f406b4d0 0x7fb5f406d990 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7fb60c035000 tx=0x7fb60c030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:16.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.292+0000 7fb6117fa640 1 -- 172.21.15.64:0/368002237 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb614150ed0 con 0x7fb6140aec40 2024-03-04T07:32:16.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.515+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb614067ff0 con 0x7fb6140aec40 2024-03-04T07:32:16.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.515+0000 7fb6117fa640 1 -- 172.21.15.64:0/368002237 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 66 v66) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fb604058d60 con 0x7fb6140aec40 2024-03-04T07:32:16.516 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:16.516 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":66,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6281,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/2549379142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2549379142},{"type":"v1","addr":"172.21.15.66:6834","nonce":2549379142}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7854,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3068683808","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3068683808},{"type":"v1","addr":"172.21.15.64:6836","nonce":3068683808}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":62},{"gid":7857,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/209754994","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":209754994},{"type":"v1","addr":"172.21.15.64:6837","nonce":209754994}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63}],"filesystems":[{"mdsmap":{"epoch":66,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:10.823856+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7872},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7872":{"gid":7872,"name":"d","rank":0,"incarnation":64,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/4084900379","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":4084900379},{"type":"v1","addr":"172.21.15.66:6835","nonce":4084900379}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7872,"qdb_cluster":[7872]},"id":8}]} 2024-03-04T07:32:16.516 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 66 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5f406b4d0 msgr2=0x7fb5f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5f406b4d0 0x7fb5f406d990 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7fb60c035000 tx=0x7fb60c030000 comp rx=0 tx=0).stop 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6140aec40 msgr2=0x7fb614153820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6140aec40 0x7fb614153820 secure :-1 s=READY pgs=1319 cs=0 l=1 rev1=1 crypto rx=0x7fb60400e9b0 tx=0x7fb60400ee70 comp rx=0 tx=0).stop 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 shutdown_connections 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5f406b4d0 0x7fb5f406d990 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb61415bac0 0x7fb61414fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6140aec40 0x7fb614153820 unknown :-1 s=CLOSED pgs=1319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 --2- 172.21.15.64:0/368002237 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb61406d400 0x7fb614151a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 >> 172.21.15.64:0/368002237 conn(0x7fb614066d00 msgr2=0x7fb6140670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:16.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 shutdown_connections 2024-03-04T07:32:16.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.518+0000 7fb61aff4640 1 -- 172.21.15.64:0/368002237 wait complete. 2024-03-04T07:32:16.529 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json flush journal 2024-03-04T07:32:16.659 INFO:teuthology.orchestra.run.smithi066.stdout:{"message":"","return_code":0} 2024-03-04T07:32:16.660 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:32:16.660 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-04T07:32:16.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.702+0000 7f2774c37f40 1 Processor -- start 2024-03-04T07:32:16.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.702+0000 7f2774c37f40 1 -- start start 2024-03-04T07:32:16.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2774c37f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce06ba10 0x5575ce093160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2774c37f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce13dd30 0x5575ce06acf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2774c37f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13e650 0x5575ce13ca60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2774c37f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575ce067740 con 0x5575ce13e650 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2774c37f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575ce0675c0 con 0x5575ce13dd30 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2774c37f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5575ce0afdd0 con 0x5575ce06ba10 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f277279b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13e650 0x5575ce13ca60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f277279b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13e650 0x5575ce13ca60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43536/0 (socket says 172.21.15.64:43536) 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f277279b640 1 -- 172.21.15.64:0/2567422869 learned_addr learned my addr 172.21.15.64:0/2567422869 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2771f9a640 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce06ba10 0x5575ce093160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f2771799640 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce13dd30 0x5575ce06acf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f277279b640 1 -- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce06ba10 msgr2=0x5575ce093160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f277279b640 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce06ba10 0x5575ce093160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f277279b640 1 -- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce13dd30 msgr2=0x5575ce06acf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.703+0000 7f277279b640 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce13dd30 0x5575ce06acf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f277279b640 1 -- 172.21.15.64:0/2567422869 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5575cdf95c50 con 0x5575ce13e650 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f277279b640 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13e650 0x5575ce13ca60 secure :-1 s=READY pgs=1321 cs=0 l=1 rev1=1 crypto rx=0x7f276400b6f0 tx=0x7f276400bbb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a784c3fb5cd347e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f276bfff640 1 -- 172.21.15.64:0/2567422869 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2764016020 con 0x5575ce13e650 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f276bfff640 1 -- 172.21.15.64:0/2567422869 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f276400be00 con 0x5575ce13e650 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f276bfff640 1 -- 172.21.15.64:0/2567422869 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f276400dba0 con 0x5575ce13e650 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 -- 172.21.15.64:0/2567422869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13e650 msgr2=0x5575ce13ca60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13e650 0x5575ce13ca60 secure :-1 s=READY pgs=1321 cs=0 l=1 rev1=1 crypto rx=0x7f276400b6f0 tx=0x7f276400bbb0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 -- 172.21.15.64:0/2567422869 shutdown_connections 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13e650 0x5575ce13ca60 unknown :-1 s=CLOSED pgs=1321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce13dd30 0x5575ce06acf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 --2- 172.21.15.64:0/2567422869 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce06ba10 0x5575ce093160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 -- 172.21.15.64:0/2567422869 >> 172.21.15.64:0/2567422869 conn(0x5575ce0685e0 msgr2=0x5575ce133ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 -- 172.21.15.64:0/2567422869 shutdown_connections 2024-03-04T07:32:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.704+0000 7f2774c37f40 1 -- 172.21.15.64:0/2567422869 wait complete. 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.705+0000 7f2774c37f40 1 Processor -- start 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.705+0000 7f2774c37f40 1 -- start start 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce06ba10 0x5575ce13bbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13dd30 0x5575ce127510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce13e650 0x5575ce12b810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575ce1481c0 con 0x5575ce13dd30 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575ce092c00 con 0x5575ce06ba10 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5575ce147f20 con 0x5575ce13e650 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13dd30 0x5575ce127510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13dd30 0x5575ce127510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43540/0 (socket says 172.21.15.64:43540) 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 -- 172.21.15.64:0/4026157361 learned_addr learned my addr 172.21.15.64:0/4026157361 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771f9a640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce06ba10 0x5575ce13bbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f277279b640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce13e650 0x5575ce12b810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 -- 172.21.15.64:0/4026157361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce13e650 msgr2=0x5575ce12b810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5575ce13e650 0x5575ce12b810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 -- 172.21.15.64:0/4026157361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce06ba10 msgr2=0x5575ce13bbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5575ce06ba10 0x5575ce13bbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 -- 172.21.15.64:0/4026157361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5575ce128ed0 con 0x5575ce13dd30 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2771799640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13dd30 0x5575ce127510 secure :-1 s=READY pgs=1322 cs=0 l=1 rev1=1 crypto rx=0x7f275c0087c0 tx=0x7f275c0088b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f276affd640 1 -- 172.21.15.64:0/4026157361 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f275c019030 con 0x5575ce13dd30 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f276affd640 1 -- 172.21.15.64:0/4026157361 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f275c015680 con 0x5575ce13dd30 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x5575ce128910 con 0x5575ce13dd30 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f276affd640 1 -- 172.21.15.64:0/4026157361 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f275c015a30 con 0x5575ce13dd30 2024-03-04T07:32:16.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.706+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5575ce130650 con 0x5575ce13dd30 2024-03-04T07:32:16.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.708+0000 7f276affd640 1 -- 172.21.15.64:0/4026157361 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f275c01c070 con 0x5575ce13dd30 2024-03-04T07:32:16.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.708+0000 7f276affd640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2754069ee0 0x7f275406c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.708+0000 7f276affd640 1 -- 172.21.15.64:0/4026157361 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f275c089530 con 0x5575ce13dd30 2024-03-04T07:32:16.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.708+0000 7f2771f9a640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2754069ee0 0x7f275406c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.708+0000 7f2774c37f40 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x5575ce12f620 0x5575ce131360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.708+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e105) v9 -- 0x5575ce1318a0 con 0x5575ce12f620 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f277279b640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x5575ce12f620 0x5575ce131360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2771f9a640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2754069ee0 0x7f275406c3a0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7f276c054000 tx=0x7f276c04e000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f277279b640 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x5575ce12f620 0x5575ce131360 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f277279b640 1 -- 172.21.15.64:0/4026157361 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (93) out=93b] v0'0 uv3 ondisk = 0) v8 ==== 164+0+93 (crc 0 0 0) 0x7f276400dbe0 con 0x5575ce12f620 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x5575ce12f620 msgr2=0x5575ce131360 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x5575ce12f620 0x5575ce131360 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2754069ee0 msgr2=0x7f275406c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2754069ee0 0x7f275406c3a0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7f276c054000 tx=0x7f276c04e000 comp rx=0 tx=0).stop 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13dd30 msgr2=0x5575ce127510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 --2- 172.21.15.64:0/4026157361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5575ce13dd30 0x5575ce127510 secure :-1 s=READY pgs=1322 cs=0 l=1 rev1=1 crypto rx=0x7f275c0087c0 tx=0x7f275c0088b0 comp rx=0 tx=0).stop 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f277279b640 1 -- 172.21.15.64:0/4026157361 reap_dead start 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 shutdown_connections 2024-03-04T07:32:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 >> 172.21.15.64:0/4026157361 conn(0x5575ce0685e0 msgr2=0x5575ce0b1ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:16.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 shutdown_connections 2024-03-04T07:32:16.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:16.709+0000 7f2774c37f40 1 -- 172.21.15.64:0/4026157361 wait complete. 2024-03-04T07:32:16.714 INFO:teuthology.orchestra.run.smithi064.stdout:Wfile_a 2024-03-04T07:32:16.714 INFO:teuthology.orchestra.run.smithi064.stdout:dir_1 2024-03-04T07:32:16.715 DEBUG:teuthology.orchestra.run.smithi064:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-04T07:32:16.857 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:32:16.857 INFO:teuthology.orchestra.run.smithi064.stdout: "ino": 1099511627777, 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "ancestors": [ 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 1099511627776, 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "file_a", 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 10 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 1, 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "dir_1", 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 17 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "pool": 17, 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout: "old_pools": [] 2024-03-04T07:32:16.858 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:32:16.859 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:16.866 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:32:16.866 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:32:16.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:32:17.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.023+0000 7fe4364b8640 1 Processor -- start 2024-03-04T07:32:17.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.023+0000 7fe4364b8640 1 -- start start 2024-03-04T07:32:17.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.023+0000 7fe4364b8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe43006ed10 0x7fe43006af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.023+0000 7fe4364b8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe4300cd890 0x7fe43015d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe4364b8640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4300cc0e0 con 0x7fe4300b1020 2024-03-04T07:32:17.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe4364b8640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe4300d28b0 con 0x7fe4300cd890 2024-03-04T07:32:17.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe4364b8640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4300cf320 con 0x7fe43006ed10 2024-03-04T07:32:17.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe434a55640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe4300cd890 0x7fe43015d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe42ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe4300b1020 0x7fe4300cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39324/0 (socket says 172.21.15.64:39324) 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe42ffff640 1 -- 172.21.15.64:0/4272633340 learned_addr learned my addr 172.21.15.64:0/4272633340 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe434a55640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe4300cd890 0x7fe43015d700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43546/0 (socket says 172.21.15.64:43546) 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe42f7fe640 1 --2- 172.21.15.64:0/4272633340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe43006ed10 0x7fe43006af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe434a55640 1 -- 172.21.15.64:0/4272633340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe43006ed10 msgr2=0x7fe43006af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe434a55640 1 --2- 172.21.15.64:0/4272633340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe43006ed10 0x7fe43006af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe434a55640 1 -- 172.21.15.64:0/4272633340 >> v1:172.21.15.64:6789/0 conn(0x7fe4300b1020 legacy=0x7fe4300cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe434a55640 1 -- 172.21.15.64:0/4272633340 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4300cc3c0 con 0x7fe4300cd890 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe42effd640 1 -- 172.21.15.64:0/4272633340 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 580537543 0 0) 0x7fe4300cc0e0 con 0x7fe4300b1020 2024-03-04T07:32:17.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.024+0000 7fe42f7fe640 1 --2- 172.21.15.64:0/4272633340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe43006ed10 0x7fe43006af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:17.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe434a55640 1 --2- 172.21.15.64:0/4272633340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4300cd890 0x7fe43015d700 secure :-1 s=READY pgs=1324 cs=0 l=1 rev1=1 crypto rx=0x7fe428009d90 tx=0x7fe42802bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f95b2102b63fe5c8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe42effd640 1 -- 172.21.15.64:0/4272633340 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe428039070 con 0x7fe4300cd890 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe42effd640 1 -- 172.21.15.64:0/4272633340 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe42802cc50 con 0x7fe4300cd890 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe42effd640 1 -- 172.21.15.64:0/4272633340 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe428034440 con 0x7fe4300cd890 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe4364b8640 1 -- 172.21.15.64:0/4272633340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4300cd890 msgr2=0x7fe43015d700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe4364b8640 1 --2- 172.21.15.64:0/4272633340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4300cd890 0x7fe43015d700 secure :-1 s=READY pgs=1324 cs=0 l=1 rev1=1 crypto rx=0x7fe428009d90 tx=0x7fe42802bea0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe4364b8640 1 -- 172.21.15.64:0/4272633340 shutdown_connections 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe4364b8640 1 --2- 172.21.15.64:0/4272633340 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe4300cd890 0x7fe43015d700 unknown :-1 s=CLOSED pgs=1324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe4364b8640 1 --2- 172.21.15.64:0/4272633340 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe43006ed10 0x7fe43006af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.025+0000 7fe4364b8640 1 -- 172.21.15.64:0/4272633340 >> 172.21.15.64:0/4272633340 conn(0x7fe4300d1a90 msgr2=0x7fe4300d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.026+0000 7fe4364b8640 1 -- 172.21.15.64:0/4272633340 shutdown_connections 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.026+0000 7fe4364b8640 1 -- 172.21.15.64:0/4272633340 wait complete. 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.027+0000 7fe4364b8640 1 Processor -- start 2024-03-04T07:32:17.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.027+0000 7fe4364b8640 1 -- start start 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe43006ed10 0x7fe4301f7a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4300b1020 0x7fe43014fde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4300cd890 0x7fe4300b69b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4300ccc60 con 0x7fe43006ed10 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4300cc980 con 0x7fe4300cd890 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4300cdce0 con 0x7fe4300b1020 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe43006ed10 0x7fe4301f7a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe43006ed10 0x7fe4301f7a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43558/0 (socket says 172.21.15.64:43558) 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 -- 172.21.15.64:0/863837753 learned_addr learned my addr 172.21.15.64:0/863837753 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42f7fe640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4300b1020 0x7fe43014fde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe434a55640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4300cd890 0x7fe4300b69b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 -- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4300b1020 msgr2=0x7fe43014fde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4300b1020 0x7fe43014fde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 -- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4300cd890 msgr2=0x7fe4300b69b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4300cd890 0x7fe4300b69b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 -- 172.21.15.64:0/863837753 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4301f7fa0 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42ffff640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe43006ed10 0x7fe4301f7a60 secure :-1 s=READY pgs=1325 cs=0 l=1 rev1=1 crypto rx=0x7fe42000c4d0 tx=0x7fe42000c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42cff9640 1 -- 172.21.15.64:0/863837753 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe420018070 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42cff9640 1 -- 172.21.15.64:0/863837753 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe420007500 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe42cff9640 1 -- 172.21.15.64:0/863837753 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe420004a20 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4300b7470 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.028+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4301f7260 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.029+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe4300b4680 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.030+0000 7fe42cff9640 1 -- 172.21.15.64:0/863837753 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe42001b070 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.030+0000 7fe42cff9640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe41c06b4d0 0x7fe41c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.030+0000 7fe42cff9640 1 -- 172.21.15.64:0/863837753 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fe420089460 con 0x7fe43006ed10 2024-03-04T07:32:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.030+0000 7fe42f7fe640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe41c06b4d0 0x7fe41c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.031+0000 7fe42f7fe640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe41c06b4d0 0x7fe41c06d990 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7fe424009560 tx=0x7fe424009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.034+0000 7fe42cff9640 1 -- 172.21.15.64:0/863837753 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe4300b4680 con 0x7fe43006ed10 2024-03-04T07:32:17.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.215+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe43006f2f0 con 0x7fe43006ed10 2024-03-04T07:32:17.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.216+0000 7fe42cff9640 1 -- 172.21.15.64:0/863837753 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v105) v1 ==== 74+0+16579 (secure 0 0 0) 0x7fe420057c90 con 0x7fe43006ed10 2024-03-04T07:32:17.216 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:17.216 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":105,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:10.470294+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"105","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:32:17.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe41c06b4d0 msgr2=0x7fe41c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe41c06b4d0 0x7fe41c06d990 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7fe424009560 tx=0x7fe424009190 comp rx=0 tx=0).stop 2024-03-04T07:32:17.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe43006ed10 msgr2=0x7fe4301f7a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe43006ed10 0x7fe4301f7a60 secure :-1 s=READY pgs=1325 cs=0 l=1 rev1=1 crypto rx=0x7fe42000c4d0 tx=0x7fe42000c990 comp rx=0 tx=0).stop 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 shutdown_connections 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe41c06b4d0 0x7fe41c06d990 unknown :-1 s=CLOSED pgs=598 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe4300cd890 0x7fe4300b69b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4300b1020 0x7fe43014fde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 --2- 172.21.15.64:0/863837753 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe43006ed10 0x7fe4301f7a60 unknown :-1 s=CLOSED pgs=1325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 >> 172.21.15.64:0/863837753 conn(0x7fe4300d1a90 msgr2=0x7fe4300d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 shutdown_connections 2024-03-04T07:32:17.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.218+0000 7fe4364b8640 1 -- 172.21.15.64:0/863837753 wait complete. 2024-03-04T07:32:17.229 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 16, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:32:07.087121+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '104', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 17, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:32:07.785433+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '105', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:32:17.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:32:17.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.334+0000 7fef2fcc9640 1 Processor -- start 2024-03-04T07:32:17.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.334+0000 7fef2fcc9640 1 -- start start 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2fcc9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef2815dbd0 0x7fef2815ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2fcc9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fef280b5840 0x7fef280b39b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2fcc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef2815dfb0 0x7fef280b3ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2fcc9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fef2815f3b0 con 0x7fef2815dbd0 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2fcc9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fef2806f070 con 0x7fef280b5840 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2fcc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fef28071940 con 0x7fef2815dfb0 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef2815dbd0 0x7fef2815ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2d264640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fef280b5840 0x7fef280b39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fef2815dbd0 0x7fef2815ee70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43570/0 (socket says 172.21.15.64:43570) 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 -- 172.21.15.64:0/1099807780 learned_addr learned my addr 172.21.15.64:0/1099807780 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2e266640 1 --2- 172.21.15.64:0/1099807780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef2815dfb0 0x7fef280b3ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 -- 172.21.15.64:0/1099807780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef2815dfb0 msgr2=0x7fef280b3ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 --2- 172.21.15.64:0/1099807780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef2815dfb0 0x7fef280b3ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 -- 172.21.15.64:0/1099807780 >> v2:172.21.15.66:3300/0 conn(0x7fef280b5840 msgr2=0x7fef280b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 --2- 172.21.15.64:0/1099807780 >> v2:172.21.15.66:3300/0 conn(0x7fef280b5840 0x7fef280b39b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2da65640 1 -- 172.21.15.64:0/1099807780 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef280b5db0 con 0x7fef2815dbd0 2024-03-04T07:32:17.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.335+0000 7fef2d264640 1 --2- 172.21.15.64:0/1099807780 >> v2:172.21.15.66:3300/0 conn(0x7fef280b5840 0x7fef280b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:17.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.336+0000 7fef2da65640 1 --2- 172.21.15.64:0/1099807780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef2815ee70 secure :-1 s=READY pgs=1326 cs=0 l=1 rev1=1 crypto rx=0x7fef28073350 tx=0x7fef2402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d18ee4d7326b9297 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.336+0000 7fef2ca63640 1 -- 172.21.15.64:0/1099807780 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef24035020 con 0x7fef2815dbd0 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.336+0000 7fef2ca63640 1 -- 172.21.15.64:0/1099807780 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fef24031870 con 0x7fef2815dbd0 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.336+0000 7fef2ca63640 1 -- 172.21.15.64:0/1099807780 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef24031c40 con 0x7fef2815dbd0 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 -- 172.21.15.64:0/1099807780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 msgr2=0x7fef2815ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/1099807780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef2815ee70 secure :-1 s=READY pgs=1326 cs=0 l=1 rev1=1 crypto rx=0x7fef28073350 tx=0x7fef2402b910 comp rx=0 tx=0).stop 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 -- 172.21.15.64:0/1099807780 shutdown_connections 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/1099807780 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef2815dfb0 0x7fef280b3ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/1099807780 >> v2:172.21.15.66:3300/0 conn(0x7fef280b5840 0x7fef280b39b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/1099807780 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef2815ee70 unknown :-1 s=CLOSED pgs=1326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 -- 172.21.15.64:0/1099807780 >> 172.21.15.64:0/1099807780 conn(0x7fef280b26b0 msgr2=0x7fef280b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 -- 172.21.15.64:0/1099807780 shutdown_connections 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.337+0000 7fef2fcc9640 1 -- 172.21.15.64:0/1099807780 wait complete. 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.338+0000 7fef2fcc9640 1 Processor -- start 2024-03-04T07:32:17.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.338+0000 7fef2fcc9640 1 -- start start 2024-03-04T07:32:17.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2fcc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef280b5840 0x7fef28157450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2fcc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef28158070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2fcc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef2815dfb0 0x7fef280aedd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2fcc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef280721c0 con 0x7fef2815dbd0 2024-03-04T07:32:17.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2fcc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef28071ee0 con 0x7fef2815dfb0 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2fcc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fef280726e0 con 0x7fef280b5840 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef28158070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef28158070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43578/0 (socket says 172.21.15.64:43578) 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 -- 172.21.15.64:0/2807487852 learned_addr learned my addr 172.21.15.64:0/2807487852 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2e266640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef2815dfb0 0x7fef280aedd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2da65640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef280b5840 0x7fef28157450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 -- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef280b5840 msgr2=0x7fef28157450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef280b5840 0x7fef28157450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 -- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef2815dfb0 msgr2=0x7fef280aedd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef2815dfb0 0x7fef280aedd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 -- 172.21.15.64:0/2807487852 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef281563b0 con 0x7fef2815dbd0 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2d264640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef28158070 secure :-1 s=READY pgs=1327 cs=0 l=1 rev1=1 crypto rx=0x7fef1800e960 tx=0x7fef1800ee20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef1e7fc640 1 -- 172.21.15.64:0/2807487852 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef18017070 con 0x7fef2815dbd0 2024-03-04T07:32:17.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef1e7fc640 1 -- 172.21.15.64:0/2807487852 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fef1800ca00 con 0x7fef2815dbd0 2024-03-04T07:32:17.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fef28156690 con 0x7fef2815dbd0 2024-03-04T07:32:17.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.339+0000 7fef1e7fc640 1 -- 172.21.15.64:0/2807487852 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef1800cdd0 con 0x7fef2815dbd0 2024-03-04T07:32:17.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.340+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef28156c50 con 0x7fef2815dbd0 2024-03-04T07:32:17.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.340+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fef28073380 con 0x7fef2815dbd0 2024-03-04T07:32:17.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.341+0000 7fef1e7fc640 1 -- 172.21.15.64:0/2807487852 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fef1801a070 con 0x7fef2815dbd0 2024-03-04T07:32:17.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.341+0000 7fef1e7fc640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef0c06b4d0 0x7fef0c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.341+0000 7fef1e7fc640 1 -- 172.21.15.64:0/2807487852 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fef1808a3b0 con 0x7fef2815dbd0 2024-03-04T07:32:17.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.341+0000 7fef2da65640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef0c06b4d0 0x7fef0c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.342+0000 7fef2da65640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef0c06b4d0 0x7fef0c06d990 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7fef24036000 tx=0x7fef2403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.346+0000 7fef1e7fc640 1 -- 172.21.15.64:0/2807487852 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fef18058be0 con 0x7fef2815dbd0 2024-03-04T07:32:17.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.528+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fef28159c10 con 0x7fef2815dbd0 2024-03-04T07:32:17.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.529+0000 7fef1e7fc640 1 -- 172.21.15.64:0/2807487852 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v105) v1 ==== 93+0+10 (secure 0 0 0) 0x7fef1805cc80 con 0x7fef2815dbd0 2024-03-04T07:32:17.529 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef0c06b4d0 msgr2=0x7fef0c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef0c06b4d0 0x7fef0c06d990 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7fef24036000 tx=0x7fef2403c040 comp rx=0 tx=0).stop 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 msgr2=0x7fef28158070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef28158070 secure :-1 s=READY pgs=1327 cs=0 l=1 rev1=1 crypto rx=0x7fef1800e960 tx=0x7fef1800ee20 comp rx=0 tx=0).stop 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 shutdown_connections 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef0c06b4d0 0x7fef0c06d990 unknown :-1 s=CLOSED pgs=599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef2815dfb0 0x7fef280aedd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef2815dbd0 0x7fef28158070 unknown :-1 s=CLOSED pgs=1327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 --2- 172.21.15.64:0/2807487852 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef280b5840 0x7fef28157450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 >> 172.21.15.64:0/2807487852 conn(0x7fef280b26b0 msgr2=0x7fef2806dc40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 shutdown_connections 2024-03-04T07:32:17.532 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.531+0000 7fef2fcc9640 1 -- 172.21.15.64:0/2807487852 wait complete. 2024-03-04T07:32:17.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:32:17.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.646+0000 7f5a99fc9640 1 Processor -- start 2024-03-04T07:32:17.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.646+0000 7f5a99fc9640 1 -- start start 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.646+0000 7f5a99fc9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5a940ce4f0 0x7f5a940ce960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.646+0000 7f5a99fc9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5a940d0150 0x7f5a940d05a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.646+0000 7f5a99fc9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5a9415f730 con 0x7f5a941618e0 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.646+0000 7f5a99fc9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5a940cfce0 con 0x7f5a940d0150 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.646+0000 7f5a99fc9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5a940ceea0 con 0x7f5a940ce4f0 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a93fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5a940d0150 0x7f5a940d05a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a937fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5a941618e0 0x7f5a9406b130 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39332/0 (socket says 172.21.15.64:39332) 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a937fe640 1 -- 172.21.15.64:0/1743491486 learned_addr learned my addr 172.21.15.64:0/1743491486 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a93fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5a940d0150 0x7f5a940d05a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43588/0 (socket says 172.21.15.64:43588) 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a92ffd640 1 --2- 172.21.15.64:0/1743491486 >> v2:172.21.15.66:3300/0 conn(0x7f5a940ce4f0 0x7f5a940ce960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a927fc640 1 -- 172.21.15.64:0/1743491486 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2547162015 0 0) 0x7f5a9415f730 con 0x7f5a941618e0 2024-03-04T07:32:17.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a927fc640 1 -- 172.21.15.64:0/1743491486 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5a7c003520 con 0x7f5a941618e0 2024-03-04T07:32:17.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a93fff640 1 -- 172.21.15.64:0/1743491486 >> v2:172.21.15.66:3300/0 conn(0x7f5a940ce4f0 msgr2=0x7f5a940ce960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:17.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a93fff640 1 --2- 172.21.15.64:0/1743491486 >> v2:172.21.15.66:3300/0 conn(0x7f5a940ce4f0 0x7f5a940ce960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a93fff640 1 -- 172.21.15.64:0/1743491486 >> v1:172.21.15.64:6789/0 conn(0x7f5a941618e0 legacy=0x7f5a9406b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.647+0000 7f5a93fff640 1 -- 172.21.15.64:0/1743491486 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a9415ebb0 con 0x7f5a940d0150 2024-03-04T07:32:17.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a93fff640 1 --2- 172.21.15.64:0/1743491486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940d0150 0x7f5a940d05a0 secure :-1 s=READY pgs=1329 cs=0 l=1 rev1=1 crypto rx=0x7f5a94071ab0 tx=0x7f5a8c02bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=95f61058c380aa18 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a927fc640 1 -- 172.21.15.64:0/1743491486 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a8c039070 con 0x7f5a940d0150 2024-03-04T07:32:17.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a927fc640 1 -- 172.21.15.64:0/1743491486 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a8c02cd00 con 0x7f5a940d0150 2024-03-04T07:32:17.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a927fc640 1 -- 172.21.15.64:0/1743491486 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a8c034640 con 0x7f5a940d0150 2024-03-04T07:32:17.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1743491486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940d0150 msgr2=0x7f5a940d05a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1743491486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940d0150 0x7f5a940d05a0 secure :-1 s=READY pgs=1329 cs=0 l=1 rev1=1 crypto rx=0x7f5a94071ab0 tx=0x7f5a8c02bf50 comp rx=0 tx=0).stop 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1743491486 shutdown_connections 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1743491486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940d0150 0x7f5a940d05a0 unknown :-1 s=CLOSED pgs=1329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1743491486 >> v2:172.21.15.66:3300/0 conn(0x7f5a940ce4f0 0x7f5a940ce960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1743491486 >> 172.21.15.64:0/1743491486 conn(0x7f5a940b3660 msgr2=0x7f5a940b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1743491486 shutdown_connections 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.648+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1743491486 wait complete. 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.649+0000 7f5a99fc9640 1 Processor -- start 2024-03-04T07:32:17.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a99fc9640 1 -- start start 2024-03-04T07:32:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a99fc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940ce4f0 0x7f5a940ca6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a99fc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a940d0150 0x7f5a941606e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a99fc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a941563e0 0x7f5a94160c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a99fc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a9415f450 con 0x7f5a940ce4f0 2024-03-04T07:32:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a99fc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a9415f170 con 0x7f5a940d0150 2024-03-04T07:32:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a99fc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a9415e7d0 con 0x7f5a941563e0 2024-03-04T07:32:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940ce4f0 0x7f5a940ca6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a937fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940ce4f0 0x7f5a940ca6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43592/0 (socket says 172.21.15.64:43592) 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.650+0000 7f5a937fe640 1 -- 172.21.15.64:0/1528296950 learned_addr learned my addr 172.21.15.64:0/1528296950 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a92ffd640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a940d0150 0x7f5a941606e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a93fff640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a941563e0 0x7f5a94160c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a937fe640 1 -- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a941563e0 msgr2=0x7f5a94160c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a937fe640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a941563e0 0x7f5a94160c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a937fe640 1 -- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a940d0150 msgr2=0x7f5a941606e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a937fe640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a940d0150 0x7f5a941606e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a937fe640 1 -- 172.21.15.64:0/1528296950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a940c67a0 con 0x7f5a940ce4f0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a937fe640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940ce4f0 0x7f5a940ca6a0 secure :-1 s=READY pgs=1330 cs=0 l=1 rev1=1 crypto rx=0x7f5a8400c390 tx=0x7f5a8400c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a7bfff640 1 -- 172.21.15.64:0/1528296950 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a84017070 con 0x7f5a940ce4f0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a7bfff640 1 -- 172.21.15.64:0/1528296950 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a84005ce0 con 0x7f5a940ce4f0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5a940c5c20 con 0x7f5a940ce4f0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a7bfff640 1 -- 172.21.15.64:0/1528296950 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a840072f0 con 0x7f5a940ce4f0 2024-03-04T07:32:17.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.651+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5a940c61e0 con 0x7f5a940ce4f0 2024-03-04T07:32:17.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.652+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5a9409fc50 con 0x7f5a940ce4f0 2024-03-04T07:32:17.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.653+0000 7f5a7bfff640 1 -- 172.21.15.64:0/1528296950 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5a8401a070 con 0x7f5a940ce4f0 2024-03-04T07:32:17.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.653+0000 7f5a7bfff640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a8006b4d0 0x7f5a8006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.653+0000 7f5a92ffd640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a8006b4d0 0x7f5a8006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.653+0000 7f5a7bfff640 1 -- 172.21.15.64:0/1528296950 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5a840894c0 con 0x7f5a940ce4f0 2024-03-04T07:32:17.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.654+0000 7f5a92ffd640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a8006b4d0 0x7f5a8006d990 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f5a940cbc20 tx=0x7f5a88009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.657+0000 7f5a7bfff640 1 -- 172.21.15.64:0/1528296950 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5a84057d80 con 0x7f5a940ce4f0 2024-03-04T07:32:17.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.837+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f5a9404f410 con 0x7f5a940ce4f0 2024-03-04T07:32:17.837 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.837+0000 7f5a7bfff640 1 -- 172.21.15.64:0/1528296950 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v105) v1 ==== 104+0+11 (secure 0 0 0) 0x7f5a8405be20 con 0x7f5a940ce4f0 2024-03-04T07:32:17.837 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:32:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a8006b4d0 msgr2=0x7f5a8006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a8006b4d0 0x7f5a8006d990 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f5a940cbc20 tx=0x7f5a88009190 comp rx=0 tx=0).stop 2024-03-04T07:32:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940ce4f0 msgr2=0x7f5a940ca6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940ce4f0 0x7f5a940ca6a0 secure :-1 s=READY pgs=1330 cs=0 l=1 rev1=1 crypto rx=0x7f5a8400c390 tx=0x7f5a8400c850 comp rx=0 tx=0).stop 2024-03-04T07:32:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 shutdown_connections 2024-03-04T07:32:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a8006b4d0 0x7f5a8006d990 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a941563e0 0x7f5a94160c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a940d0150 0x7f5a941606e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 --2- 172.21.15.64:0/1528296950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a940ce4f0 0x7f5a940ca6a0 unknown :-1 s=CLOSED pgs=1330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 >> 172.21.15.64:0/1528296950 conn(0x7f5a940b3660 msgr2=0x7f5a940d0530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 shutdown_connections 2024-03-04T07:32:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.839+0000 7f5a99fc9640 1 -- 172.21.15.64:0/1528296950 wait complete. 2024-03-04T07:32:17.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:32:17.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:32:17.910 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:32:17.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.955+0000 7fcc51a2a640 1 Processor -- start 2024-03-04T07:32:17.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.955+0000 7fcc51a2a640 1 -- start start 2024-03-04T07:32:17.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.956+0000 7fcc51a2a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc4c0717b0 0x7fcc4c071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.956+0000 7fcc51a2a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc4c15e4b0 con 0x7fcc4c0b2b10 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.956+0000 7fcc51a2a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc4c0b97e0 con 0x7fcc4c0b3060 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.956+0000 7fcc51a2a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc4c0b7c70 con 0x7fcc4c0717b0 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.956+0000 7fcc4affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc4c0717b0 0x7fcc4c071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.956+0000 7fcc4b7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcc4c0b2b10 0x7fcc4c0b7560 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39340/0 (socket says 172.21.15.64:39340) 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.956+0000 7fcc4b7fe640 1 -- 172.21.15.64:0/1648933080 learned_addr learned my addr 172.21.15.64:0/1648933080 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.957+0000 7fcc4affd640 1 -- 172.21.15.64:0/1648933080 >> v1:172.21.15.66:6789/0 conn(0x7fcc4c0b3060 legacy=0x7fcc4c0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.957+0000 7fcc4affd640 1 -- 172.21.15.64:0/1648933080 >> v1:172.21.15.64:6789/0 conn(0x7fcc4c0b2b10 legacy=0x7fcc4c0b7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.957+0000 7fcc4affd640 1 -- 172.21.15.64:0/1648933080 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc4c0b9a80 con 0x7fcc4c0717b0 2024-03-04T07:32:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.957+0000 7fcc4a7fc640 1 -- 172.21.15.64:0/1648933080 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2891305060 0 0) 0x7fcc44001490 con 0x7fcc4c0b2b10 2024-03-04T07:32:17.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.957+0000 7fcc4affd640 1 --2- 172.21.15.64:0/1648933080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c071b90 secure :-1 s=READY pgs=1332 cs=0 l=1 rev1=1 crypto rx=0x7fcc3c0097a0 tx=0x7fcc3c02b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d72a010c30b1204f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.957+0000 7fcc4a7fc640 1 -- 172.21.15.64:0/1648933080 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc3c034070 con 0x7fcc4c0717b0 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc4a7fc640 1 -- 172.21.15.64:0/1648933080 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc3c030610 con 0x7fcc4c0717b0 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc4a7fc640 1 -- 172.21.15.64:0/1648933080 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc3c0309e0 con 0x7fcc4c0717b0 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1648933080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 msgr2=0x7fcc4c071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1648933080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c071b90 secure :-1 s=READY pgs=1332 cs=0 l=1 rev1=1 crypto rx=0x7fcc3c0097a0 tx=0x7fcc3c02b960 comp rx=0 tx=0).stop 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1648933080 shutdown_connections 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1648933080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c071b90 unknown :-1 s=CLOSED pgs=1332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1648933080 >> 172.21.15.64:0/1648933080 conn(0x7fcc4c0d3d20 msgr2=0x7fcc4c0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1648933080 shutdown_connections 2024-03-04T07:32:17.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.958+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1648933080 wait complete. 2024-03-04T07:32:17.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.959+0000 7fcc51a2a640 1 Processor -- start 2024-03-04T07:32:17.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.959+0000 7fcc51a2a640 1 -- start start 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc51a2a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c0c9730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc51a2a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c0b2b10 0x7fcc4c149940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc51a2a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0b3060 0x7fcc4c149e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc51a2a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc4c0d4bf0 con 0x7fcc4c0717b0 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc51a2a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc4c0d2e10 con 0x7fcc4c0b2b10 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc51a2a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc4c0d32a0 con 0x7fcc4c0b3060 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c0c9730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c0c9730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43604/0 (socket says 172.21.15.64:43604) 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 -- 172.21.15.64:0/1681834763 learned_addr learned my addr 172.21.15.64:0/1681834763 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4b7fe640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0b3060 0x7fcc4c149e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc43fff640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c0b2b10 0x7fcc4c149940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 -- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0b3060 msgr2=0x7fcc4c149e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0b3060 0x7fcc4c149e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 -- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c0b2b10 msgr2=0x7fcc4c149940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c0b2b10 0x7fcc4c149940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.960+0000 7fcc4affd640 1 -- 172.21.15.64:0/1681834763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc4c0cd4f0 con 0x7fcc4c0717b0 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.961+0000 7fcc4affd640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c0c9730 secure :-1 s=READY pgs=1333 cs=0 l=1 rev1=1 crypto rx=0x7fcc3c02bf70 tx=0x7fcc3c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.961+0000 7fcc48ff9640 1 -- 172.21.15.64:0/1681834763 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc3c034040 con 0x7fcc4c0717b0 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.961+0000 7fcc48ff9640 1 -- 172.21.15.64:0/1681834763 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc3c039a10 con 0x7fcc4c0717b0 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.961+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc4c147a40 con 0x7fcc4c0717b0 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.961+0000 7fcc48ff9640 1 -- 172.21.15.64:0/1681834763 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc3c039e20 con 0x7fcc4c0717b0 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.961+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc4c0c9c70 con 0x7fcc4c0717b0 2024-03-04T07:32:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.962+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc4c06ce40 con 0x7fcc4c0717b0 2024-03-04T07:32:17.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.962+0000 7fcc48ff9640 1 -- 172.21.15.64:0/1681834763 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcc3c037070 con 0x7fcc4c0717b0 2024-03-04T07:32:17.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.963+0000 7fcc48ff9640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc2c06b4d0 0x7fcc2c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:17.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.963+0000 7fcc43fff640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc2c06b4d0 0x7fcc2c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:17.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.963+0000 7fcc48ff9640 1 -- 172.21.15.64:0/1681834763 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcc3c0ac170 con 0x7fcc4c0717b0 2024-03-04T07:32:17.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.963+0000 7fcc43fff640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc2c06b4d0 0x7fcc2c06d990 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fcc4c0684e0 tx=0x7fcc34009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:17.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:17.967+0000 7fcc48ff9640 1 -- 172.21.15.64:0/1681834763 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcc3c07aa30 con 0x7fcc4c0717b0 2024-03-04T07:32:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.147+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fcc4c0cd7d0 con 0x7fcc4c0717b0 2024-03-04T07:32:18.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.148+0000 7fcc48ff9640 1 -- 172.21.15.64:0/1681834763 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v105) v1 ==== 100+0+11 (secure 0 0 0) 0x7fcc3c07ead0 con 0x7fcc4c0717b0 2024-03-04T07:32:18.148 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc2c06b4d0 msgr2=0x7fcc2c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc2c06b4d0 0x7fcc2c06d990 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fcc4c0684e0 tx=0x7fcc34009040 comp rx=0 tx=0).stop 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 msgr2=0x7fcc4c0c9730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c0c9730 secure :-1 s=READY pgs=1333 cs=0 l=1 rev1=1 crypto rx=0x7fcc3c02bf70 tx=0x7fcc3c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 shutdown_connections 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc2c06b4d0 0x7fcc2c06d990 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc4c0b3060 0x7fcc4c149e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc4c0b2b10 0x7fcc4c149940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 --2- 172.21.15.64:0/1681834763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc4c0717b0 0x7fcc4c0c9730 unknown :-1 s=CLOSED pgs=1333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 >> 172.21.15.64:0/1681834763 conn(0x7fcc4c0d3d20 msgr2=0x7fcc4c0b9010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 shutdown_connections 2024-03-04T07:32:18.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.150+0000 7fcc51a2a640 1 -- 172.21.15.64:0/1681834763 wait complete. 2024-03-04T07:32:18.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:32:18.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.263+0000 7f781757a640 1 Processor -- start 2024-03-04T07:32:18.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f781757a640 1 -- start start 2024-03-04T07:32:18.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f781757a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b95a0 0x7f78100b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f781757a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f78100b9a10 0x7f78100b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f781757a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f78100cea30 con 0x7f781015df00 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f781757a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f78100cf3c0 con 0x7f78100b9a10 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f781757a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f78100cf0e0 con 0x7f78100b95a0 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f7815316640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f781015df00 0x7f78100d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:33442/0 (socket says 172.21.15.64:33442) 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f7815316640 1 -- 172.21.15.64:0/1217692446 learned_addr learned my addr 172.21.15.64:0/1217692446 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f7815b17640 1 --2- 172.21.15.64:0/1217692446 >> v2:172.21.15.66:3300/0 conn(0x7f78100b9a10 0x7f78100b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.264+0000 7f7814b15640 1 --2- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b95a0 0x7f78100b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.265+0000 7f7803fff640 1 -- 172.21.15.64:0/1217692446 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2073229564 0 0) 0x7f78100cea30 con 0x7f781015df00 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.265+0000 7f7803fff640 1 -- 172.21.15.64:0/1217692446 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f77fc003520 con 0x7f781015df00 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.265+0000 7f7815b17640 1 -- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b95a0 msgr2=0x7f78100b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:18.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.265+0000 7f7815b17640 1 --2- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b95a0 0x7f78100b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.265+0000 7f7815b17640 1 -- 172.21.15.64:0/1217692446 >> v1:172.21.15.66:6789/0 conn(0x7f781015df00 legacy=0x7f78100d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.265+0000 7f7815b17640 1 -- 172.21.15.64:0/1217692446 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78100ce470 con 0x7f78100b9a10 2024-03-04T07:32:18.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f7815b17640 1 --2- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f78100b9a10 0x7f78100b5c40 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f7808009d90 tx=0x7f780802bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3d52dcf72107483f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f7803fff640 1 -- 172.21.15.64:0/1217692446 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7808039070 con 0x7f78100b9a10 2024-03-04T07:32:18.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f7803fff640 1 -- 172.21.15.64:0/1217692446 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f780802cc50 con 0x7f78100b9a10 2024-03-04T07:32:18.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f7803fff640 1 -- 172.21.15.64:0/1217692446 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7808034440 con 0x7f78100b9a10 2024-03-04T07:32:18.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 -- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f78100b9a10 msgr2=0x7f78100b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 --2- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f78100b9a10 0x7f78100b5c40 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f7808009d90 tx=0x7f780802bea0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 -- 172.21.15.64:0/1217692446 shutdown_connections 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 --2- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f78100b9a10 0x7f78100b5c40 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 --2- 172.21.15.64:0/1217692446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b95a0 0x7f78100b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 -- 172.21.15.64:0/1217692446 >> 172.21.15.64:0/1217692446 conn(0x7f7810071960 msgr2=0x7f7810071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 -- 172.21.15.64:0/1217692446 shutdown_connections 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.266+0000 7f781757a640 1 -- 172.21.15.64:0/1217692446 wait complete. 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.267+0000 7f781757a640 1 Processor -- start 2024-03-04T07:32:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f781757a640 1 -- start start 2024-03-04T07:32:18.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f781757a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78100b95a0 0x7f7810155df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f781757a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b9a10 0x7f78100ccf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f781757a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f781015df00 0x7f7810160860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f781757a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78100cedf0 con 0x7f78100b95a0 2024-03-04T07:32:18.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f781757a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78100cec70 con 0x7f781015df00 2024-03-04T07:32:18.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f781757a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f78100d3220 con 0x7f78100b9a10 2024-03-04T07:32:18.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f7815316640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78100b95a0 0x7f7810155df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f7815316640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78100b95a0 0x7f7810155df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43610/0 (socket says 172.21.15.64:43610) 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f7815316640 1 -- 172.21.15.64:0/2611901663 learned_addr learned my addr 172.21.15.64:0/2611901663 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f7814b15640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b9a10 0x7f78100ccf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.268+0000 7f7815b17640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f781015df00 0x7f7810160860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7815316640 1 -- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b9a10 msgr2=0x7f78100ccf90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7815316640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b9a10 0x7f78100ccf90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7815316640 1 -- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f781015df00 msgr2=0x7f7810160860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7815316640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f781015df00 0x7f7810160860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7815316640 1 -- 172.21.15.64:0/2611901663 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78100c7b90 con 0x7f78100b95a0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7815316640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78100b95a0 0x7f7810155df0 secure :-1 s=READY pgs=1334 cs=0 l=1 rev1=1 crypto rx=0x7f780400c390 tx=0x7f780400c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7801ffb640 1 -- 172.21.15.64:0/2611901663 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7804017070 con 0x7f78100b95a0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7801ffb640 1 -- 172.21.15.64:0/2611901663 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7804005ce0 con 0x7f78100b95a0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78100c7e70 con 0x7f78100b95a0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f7801ffb640 1 -- 172.21.15.64:0/2611901663 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78040072f0 con 0x7f78100b95a0 2024-03-04T07:32:18.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.269+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78100c8430 con 0x7f78100b95a0 2024-03-04T07:32:18.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.270+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f781015d730 con 0x7f78100b95a0 2024-03-04T07:32:18.272 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.270+0000 7f7801ffb640 1 -- 172.21.15.64:0/2611901663 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f780401a070 con 0x7f78100b95a0 2024-03-04T07:32:18.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.271+0000 7f7801ffb640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77f406b4d0 0x7f77f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.271+0000 7f7801ffb640 1 -- 172.21.15.64:0/2611901663 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7804089570 con 0x7f78100b95a0 2024-03-04T07:32:18.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.271+0000 7f7814b15640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77f406b4d0 0x7f77f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.273 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.271+0000 7f7814b15640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77f406b4d0 0x7f77f406d990 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f781015a100 tx=0x7f780c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.275 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.275+0000 7f7801ffb640 1 -- 172.21.15.64:0/2611901663 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f781015d730 con 0x7f78100b95a0 2024-03-04T07:32:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.468+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f78100b60e0 con 0x7f78100b95a0 2024-03-04T07:32:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.468+0000 7f7801ffb640 1 -- 172.21.15.64:0/2611901663 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v105) v1 ==== 80+0+0 (secure 0 0 0) 0x7f78100b60e0 con 0x7f78100b95a0 2024-03-04T07:32:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:32:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77f406b4d0 msgr2=0x7f77f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77f406b4d0 0x7f77f406d990 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f781015a100 tx=0x7f780c009190 comp rx=0 tx=0).stop 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78100b95a0 msgr2=0x7f7810155df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78100b95a0 0x7f7810155df0 secure :-1 s=READY pgs=1334 cs=0 l=1 rev1=1 crypto rx=0x7f780400c390 tx=0x7f780400c850 comp rx=0 tx=0).stop 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 shutdown_connections 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f77f406b4d0 0x7f77f406d990 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f781015df00 0x7f7810160860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f78100b9a10 0x7f78100ccf90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 --2- 172.21.15.64:0/2611901663 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f78100b95a0 0x7f7810155df0 unknown :-1 s=CLOSED pgs=1334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 >> 172.21.15.64:0/2611901663 conn(0x7f7810071960 msgr2=0x7f7810071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.470+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 shutdown_connections 2024-03-04T07:32:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.471+0000 7f781757a640 1 -- 172.21.15.64:0/2611901663 wait complete. 2024-03-04T07:32:18.481 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:32:18.481 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:32:18.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:32:18.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.638+0000 7efc1bac9640 1 Processor -- start 2024-03-04T07:32:18.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.638+0000 7efc1bac9640 1 -- start start 2024-03-04T07:32:18.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc1bac9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efc1415bda0 0x7efc140d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc1bac9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc140ce360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc1bac9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efc140cd350 con 0x7efc140ce8a0 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc1bac9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efc1406eb20 con 0x7efc1415bda0 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc1bac9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efc140b1330 con 0x7efc140b0ed0 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19865640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efc1415bda0 0x7efc140d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19865640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efc1415bda0 0x7efc140d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43620/0 (socket says 172.21.15.64:43620) 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19865640 1 -- 172.21.15.64:0/3458838000 learned_addr learned my addr 172.21.15.64:0/3458838000 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19064640 1 --2- 172.21.15.64:0/3458838000 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc140ce360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19865640 1 -- 172.21.15.64:0/3458838000 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 msgr2=0x7efc140ce360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19865640 1 --2- 172.21.15.64:0/3458838000 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc140ce360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19865640 1 -- 172.21.15.64:0/3458838000 >> v1:172.21.15.66:6789/0 conn(0x7efc140ce8a0 legacy=0x7efc140cfc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.639+0000 7efc19865640 1 -- 172.21.15.64:0/3458838000 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc140cd630 con 0x7efc1415bda0 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.640+0000 7efc19865640 1 --2- 172.21.15.64:0/3458838000 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140d25b0 secure :-1 s=READY pgs=1335 cs=0 l=1 rev1=1 crypto rx=0x7efc0c009870 tx=0x7efc0c02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=611fbbd2504a6c92 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.640+0000 7efc18863640 1 -- 172.21.15.64:0/3458838000 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc0c034070 con 0x7efc1415bda0 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.640+0000 7efc18863640 1 -- 172.21.15.64:0/3458838000 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efc0c030840 con 0x7efc1415bda0 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.640+0000 7efc18863640 1 -- 172.21.15.64:0/3458838000 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc0c030c10 con 0x7efc1415bda0 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 -- 172.21.15.64:0/3458838000 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 msgr2=0x7efc140d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 --2- 172.21.15.64:0/3458838000 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140d25b0 secure :-1 s=READY pgs=1335 cs=0 l=1 rev1=1 crypto rx=0x7efc0c009870 tx=0x7efc0c02b980 comp rx=0 tx=0).stop 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 -- 172.21.15.64:0/3458838000 shutdown_connections 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 --2- 172.21.15.64:0/3458838000 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc140ce360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 --2- 172.21.15.64:0/3458838000 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140d25b0 unknown :-1 s=CLOSED pgs=1335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 -- 172.21.15.64:0/3458838000 >> 172.21.15.64:0/3458838000 conn(0x7efc140d1a90 msgr2=0x7efc140d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 -- 172.21.15.64:0/3458838000 shutdown_connections 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.641+0000 7efc1bac9640 1 -- 172.21.15.64:0/3458838000 wait complete. 2024-03-04T07:32:18.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.642+0000 7efc1bac9640 1 Processor -- start 2024-03-04T07:32:18.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.642+0000 7efc1bac9640 1 -- start start 2024-03-04T07:32:18.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1bac9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc1415a240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1bac9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc140ce8a0 0x7efc1406c1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1bac9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140b5be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1bac9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc1415c180 con 0x7efc1415bda0 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1bac9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc1406ed70 con 0x7efc140ce8a0 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1bac9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efc1415c300 con 0x7efc140b0ed0 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140b5be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140b5be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43636/0 (socket says 172.21.15.64:43636) 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 -- 172.21.15.64:0/1834748013 learned_addr learned my addr 172.21.15.64:0/1834748013 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc19865640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc1415a240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc19064640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc140ce8a0 0x7efc1406c1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 -- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 msgr2=0x7efc1415a240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc1415a240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 -- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc140ce8a0 msgr2=0x7efc1406c1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc140ce8a0 0x7efc1406c1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 -- 172.21.15.64:0/1834748013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc1415a780 con 0x7efc1415bda0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc1a066640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140b5be0 secure :-1 s=READY pgs=1336 cs=0 l=1 rev1=1 crypto rx=0x7efc1000c6b0 tx=0x7efc1000cb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.643+0000 7efc0a7fc640 1 -- 172.21.15.64:0/1834748013 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc10003b70 con 0x7efc1415bda0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.644+0000 7efc0a7fc640 1 -- 172.21.15.64:0/1834748013 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efc10003d30 con 0x7efc1415bda0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.644+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc140c8c90 con 0x7efc1415bda0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.644+0000 7efc0a7fc640 1 -- 172.21.15.64:0/1834748013 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc10005310 con 0x7efc1415bda0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.644+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc1406b9d0 con 0x7efc1415bda0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.645+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc14067ff0 con 0x7efc1415bda0 2024-03-04T07:32:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.645+0000 7efc0a7fc640 1 -- 172.21.15.64:0/1834748013 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efc100054b0 con 0x7efc1415bda0 2024-03-04T07:32:18.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.646+0000 7efc0a7fc640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efbf806b4d0 0x7efbf806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.646+0000 7efc0a7fc640 1 -- 172.21.15.64:0/1834748013 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7efc10089e90 con 0x7efc1415bda0 2024-03-04T07:32:18.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.646+0000 7efc19865640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efbf806b4d0 0x7efbf806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.646+0000 7efc19865640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efbf806b4d0 0x7efbf806d990 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7efc0c036000 tx=0x7efc0c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.650+0000 7efc0a7fc640 1 -- 172.21.15.64:0/1834748013 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efc1415a780 con 0x7efc1415bda0 2024-03-04T07:32:18.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.832+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7efc140b3910 con 0x7efc1415bda0 2024-03-04T07:32:18.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.833+0000 7efc0a7fc640 1 -- 172.21.15.64:0/1834748013 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v105) v1 ==== 74+0+16579 (secure 0 0 0) 0x7efc10058750 con 0x7efc1415bda0 2024-03-04T07:32:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:18.833 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":105,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:10.470294+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"105","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efbf806b4d0 msgr2=0x7efbf806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efbf806b4d0 0x7efbf806d990 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7efc0c036000 tx=0x7efc0c008030 comp rx=0 tx=0).stop 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 msgr2=0x7efc140b5be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140b5be0 secure :-1 s=READY pgs=1336 cs=0 l=1 rev1=1 crypto rx=0x7efc1000c6b0 tx=0x7efc1000cb70 comp rx=0 tx=0).stop 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 shutdown_connections 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efbf806b4d0 0x7efbf806d990 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc1415bda0 0x7efc140b5be0 unknown :-1 s=CLOSED pgs=1336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc140ce8a0 0x7efc1406c1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 --2- 172.21.15.64:0/1834748013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc140b0ed0 0x7efc1415a240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 >> 172.21.15.64:0/1834748013 conn(0x7efc140d1a90 msgr2=0x7efc140b14b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 shutdown_connections 2024-03-04T07:32:18.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.835+0000 7efc1bac9640 1 -- 172.21.15.64:0/1834748013 wait complete. 2024-03-04T07:32:18.846 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 16, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:32:07.087121+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '104', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 17, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:32:07.785433+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '105', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:32:18.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:32:18.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.950+0000 7f587b181640 1 Processor -- start 2024-03-04T07:32:18.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.950+0000 7f587b181640 1 -- start start 2024-03-04T07:32:18.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587b181640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f58740ce4f0 0x7f58740ce960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587b181640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f58740d0150 0x7f58740d05a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587b181640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f587415f730 con 0x7f58741618e0 2024-03-04T07:32:18.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587b181640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f58740cfce0 con 0x7f58740d0150 2024-03-04T07:32:18.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587b181640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f58740ceea0 con 0x7f58740ce4f0 2024-03-04T07:32:18.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587971e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f58740d0150 0x7f58740d05a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587971e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f58740d0150 0x7f58740d05a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43650/0 (socket says 172.21.15.64:43650) 2024-03-04T07:32:18.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587971e640 1 -- 172.21.15.64:0/2312745335 learned_addr learned my addr 172.21.15.64:0/2312745335 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:18.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f5873fff640 1 --2- 172.21.15.64:0/2312745335 >> v2:172.21.15.66:3300/0 conn(0x7f58740ce4f0 0x7f58740ce960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587971e640 1 -- 172.21.15.64:0/2312745335 >> v2:172.21.15.66:3300/0 conn(0x7f58740ce4f0 msgr2=0x7f58740ce960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:18.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587971e640 1 --2- 172.21.15.64:0/2312745335 >> v2:172.21.15.66:3300/0 conn(0x7f58740ce4f0 0x7f58740ce960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587971e640 1 -- 172.21.15.64:0/2312745335 >> v1:172.21.15.66:6789/0 conn(0x7f58741618e0 legacy=0x7f587406b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.951+0000 7f587971e640 1 -- 172.21.15.64:0/2312745335 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f587415ebb0 con 0x7f58740d0150 2024-03-04T07:32:18.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.952+0000 7f587971e640 1 --2- 172.21.15.64:0/2312745335 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740d05a0 secure :-1 s=READY pgs=1337 cs=0 l=1 rev1=1 crypto rx=0x7f5874071ab0 tx=0x7f585c02ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=df8dab411d997b3c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.952+0000 7f58737fe640 1 -- 172.21.15.64:0/2312745335 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f585c02f850 con 0x7f58740d0150 2024-03-04T07:32:18.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.952+0000 7f58737fe640 1 -- 172.21.15.64:0/2312745335 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f585c02f9f0 con 0x7f58740d0150 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.952+0000 7f58737fe640 1 -- 172.21.15.64:0/2312745335 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f585c02fdc0 con 0x7f58740d0150 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 -- 172.21.15.64:0/2312745335 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 msgr2=0x7f58740d05a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 --2- 172.21.15.64:0/2312745335 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740d05a0 secure :-1 s=READY pgs=1337 cs=0 l=1 rev1=1 crypto rx=0x7f5874071ab0 tx=0x7f585c02ba30 comp rx=0 tx=0).stop 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 -- 172.21.15.64:0/2312745335 shutdown_connections 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 --2- 172.21.15.64:0/2312745335 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740d05a0 unknown :-1 s=CLOSED pgs=1337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 --2- 172.21.15.64:0/2312745335 >> v2:172.21.15.66:3300/0 conn(0x7f58740ce4f0 0x7f58740ce960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 -- 172.21.15.64:0/2312745335 >> 172.21.15.64:0/2312745335 conn(0x7f58740b3660 msgr2=0x7f58740b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 -- 172.21.15.64:0/2312745335 shutdown_connections 2024-03-04T07:32:18.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.953+0000 7f587b181640 1 -- 172.21.15.64:0/2312745335 wait complete. 2024-03-04T07:32:18.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.954+0000 7f587b181640 1 Processor -- start 2024-03-04T07:32:18.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.954+0000 7f587b181640 1 -- start start 2024-03-04T07:32:18.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587b181640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f58740ce4f0 0x7f58740cb700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587b181640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740b4770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587b181640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f58740b72b0 0x7f58741600e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587b181640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f587415f450 con 0x7f58740d0150 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587b181640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f587415f170 con 0x7f58740ce4f0 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587b181640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f587415e7d0 con 0x7f58740b72b0 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740b4770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740b4770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43664/0 (socket says 172.21.15.64:43664) 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5878f1d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f58740ce4f0 0x7f58740cb700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 -- 172.21.15.64:0/1591111636 learned_addr learned my addr 172.21.15.64:0/1591111636 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587971e640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f58740b72b0 0x7f58741600e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 -- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f58740b72b0 msgr2=0x7f58741600e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f58740b72b0 0x7f58741600e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 -- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f58740ce4f0 msgr2=0x7f58740cb700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f58740ce4f0 0x7f58740cb700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5873fff640 1 -- 172.21.15.64:0/1591111636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58740b64f0 con 0x7f58740d0150 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f587971e640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f58740b72b0 0x7f58741600e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.955+0000 7f5878f1d640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f58740ce4f0 0x7f58740cb700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:18.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.956+0000 7f5873fff640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740b4770 secure :-1 s=READY pgs=1338 cs=0 l=1 rev1=1 crypto rx=0x7f586400e9b0 tx=0x7f586400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.956+0000 7f58717fa640 1 -- 172.21.15.64:0/1591111636 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5864017070 con 0x7f58740d0150 2024-03-04T07:32:18.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.956+0000 7f58717fa640 1 -- 172.21.15.64:0/1591111636 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f586400c9a0 con 0x7f58740d0150 2024-03-04T07:32:18.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.956+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f58740b67d0 con 0x7f58740d0150 2024-03-04T07:32:18.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.956+0000 7f58717fa640 1 -- 172.21.15.64:0/1591111636 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f586400cd90 con 0x7f58740d0150 2024-03-04T07:32:18.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.956+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f58740b4e00 con 0x7f58740d0150 2024-03-04T07:32:18.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.957+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f587409fc50 con 0x7f58740d0150 2024-03-04T07:32:18.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.957+0000 7f58717fa640 1 -- 172.21.15.64:0/1591111636 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f586401a070 con 0x7f58740d0150 2024-03-04T07:32:18.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.958+0000 7f58717fa640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f585406b4d0 0x7f585406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:18.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.958+0000 7f58717fa640 1 -- 172.21.15.64:0/1591111636 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f586408a500 con 0x7f58740d0150 2024-03-04T07:32:18.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.958+0000 7f5878f1d640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f585406b4d0 0x7f585406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:18.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.958+0000 7f5878f1d640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f585406b4d0 0x7f585406d990 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7f586c00f1d0 tx=0x7f586c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:18.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:18.962+0000 7f58717fa640 1 -- 172.21.15.64:0/1591111636 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5864058d30 con 0x7f58740d0150 2024-03-04T07:32:19.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.141+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f587404f410 con 0x7f58740d0150 2024-03-04T07:32:19.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.142+0000 7f58717fa640 1 -- 172.21.15.64:0/1591111636 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v105) v1 ==== 93+0+10 (secure 0 0 0) 0x7f586405cdd0 con 0x7f58740d0150 2024-03-04T07:32:19.142 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f585406b4d0 msgr2=0x7f585406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f585406b4d0 0x7f585406d990 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7f586c00f1d0 tx=0x7f586c009040 comp rx=0 tx=0).stop 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 msgr2=0x7f58740b4770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740b4770 secure :-1 s=READY pgs=1338 cs=0 l=1 rev1=1 crypto rx=0x7f586400e9b0 tx=0x7f586400ee70 comp rx=0 tx=0).stop 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 shutdown_connections 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f585406b4d0 0x7f585406d990 unknown :-1 s=CLOSED pgs=604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f58740b72b0 0x7f58741600e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f58740d0150 0x7f58740b4770 unknown :-1 s=CLOSED pgs=1338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 --2- 172.21.15.64:0/1591111636 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f58740ce4f0 0x7f58740cb700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 >> 172.21.15.64:0/1591111636 conn(0x7f58740b3660 msgr2=0x7f58740cd5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:19.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 shutdown_connections 2024-03-04T07:32:19.145 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.144+0000 7f587b181640 1 -- 172.21.15.64:0/1591111636 wait complete. 2024-03-04T07:32:19.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:32:19.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.259+0000 7fd782688640 1 Processor -- start 2024-03-04T07:32:19.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.259+0000 7fd782688640 1 -- start start 2024-03-04T07:32:19.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd782688640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd77c15ae50 0x7fd77c0708e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd782688640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd77c15b230 0x7fd77c071160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd782688640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd77c070e20 con 0x7fd77c0716a0 2024-03-04T07:32:19.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd782688640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd77c071b10 con 0x7fd77c15ae50 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd782688640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd77c0b2e30 con 0x7fd77c15b230 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd77c15ae50 0x7fd77c0708e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd780c25640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd77c0716a0 0x7fd77c150a00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39354/0 (socket says 172.21.15.64:39354) 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd780c25640 1 -- 172.21.15.64:0/1524216861 learned_addr learned my addr 172.21.15.64:0/1524216861 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77bfff640 1 --2- 172.21.15.64:0/1524216861 >> v2:172.21.15.64:3300/0 conn(0x7fd77c15ae50 0x7fd77c0708e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43670/0 (socket says 172.21.15.64:43670) 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77b7fe640 1 --2- 172.21.15.64:0/1524216861 >> v2:172.21.15.66:3300/0 conn(0x7fd77c15b230 0x7fd77c071160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77bfff640 1 -- 172.21.15.64:0/1524216861 >> v2:172.21.15.66:3300/0 conn(0x7fd77c15b230 msgr2=0x7fd77c071160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77bfff640 1 --2- 172.21.15.64:0/1524216861 >> v2:172.21.15.66:3300/0 conn(0x7fd77c15b230 0x7fd77c071160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77affd640 1 -- 172.21.15.64:0/1524216861 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3926719069 0 0) 0x7fd77c070e20 con 0x7fd77c0716a0 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77bfff640 1 -- 172.21.15.64:0/1524216861 >> v1:172.21.15.64:6789/0 conn(0x7fd77c0716a0 legacy=0x7fd77c150a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77bfff640 1 -- 172.21.15.64:0/1524216861 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd77c06dfe0 con 0x7fd77c15ae50 2024-03-04T07:32:19.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.260+0000 7fd77b7fe640 1 --2- 172.21.15.64:0/1524216861 >> v2:172.21.15.66:3300/0 conn(0x7fd77c15b230 0x7fd77c071160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.261+0000 7fd77bfff640 1 --2- 172.21.15.64:0/1524216861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c0708e0 secure :-1 s=READY pgs=1340 cs=0 l=1 rev1=1 crypto rx=0x7fd77c06ed40 tx=0x7fd77002c950 comp rx=0 tx=0).ready entity=mon.0 client_cookie=95b654a114053ea6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.261+0000 7fd77affd640 1 -- 172.21.15.64:0/1524216861 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd770035070 con 0x7fd77c15ae50 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.261+0000 7fd77affd640 1 -- 172.21.15.64:0/1524216861 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd770031610 con 0x7fd77c15ae50 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.261+0000 7fd77affd640 1 -- 172.21.15.64:0/1524216861 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd7700319e0 con 0x7fd77c15ae50 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 -- 172.21.15.64:0/1524216861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 msgr2=0x7fd77c0708e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 --2- 172.21.15.64:0/1524216861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c0708e0 secure :-1 s=READY pgs=1340 cs=0 l=1 rev1=1 crypto rx=0x7fd77c06ed40 tx=0x7fd77002c950 comp rx=0 tx=0).stop 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 -- 172.21.15.64:0/1524216861 shutdown_connections 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 --2- 172.21.15.64:0/1524216861 >> v2:172.21.15.66:3300/0 conn(0x7fd77c15b230 0x7fd77c071160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 --2- 172.21.15.64:0/1524216861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c0708e0 unknown :-1 s=CLOSED pgs=1340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 -- 172.21.15.64:0/1524216861 >> 172.21.15.64:0/1524216861 conn(0x7fd77c066d00 msgr2=0x7fd77c067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 -- 172.21.15.64:0/1524216861 shutdown_connections 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.262+0000 7fd782688640 1 -- 172.21.15.64:0/1524216861 wait complete. 2024-03-04T07:32:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.263+0000 7fd782688640 1 Processor -- start 2024-03-04T07:32:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.263+0000 7fd782688640 1 -- start start 2024-03-04T07:32:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd782688640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd77c0716a0 0x7fd77c13c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd782688640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c1541d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd782688640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd77c15b230 0x7fd77c154710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd782688640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd77c0b3670 con 0x7fd77c15ae50 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd782688640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd77c0b3390 con 0x7fd77c15b230 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd782688640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd77c0b3950 con 0x7fd77c0716a0 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c1541d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c1541d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43676/0 (socket says 172.21.15.64:43676) 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 -- 172.21.15.64:0/180799250 learned_addr learned my addr 172.21.15.64:0/180799250 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd780c25640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd77c15b230 0x7fd77c154710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77bfff640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd77c0716a0 0x7fd77c13c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 -- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd77c0716a0 msgr2=0x7fd77c13c8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd77c0716a0 0x7fd77c13c8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 -- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd77c15b230 msgr2=0x7fd77c154710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd77c15b230 0x7fd77c154710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 -- 172.21.15.64:0/180799250 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd77c150460 con 0x7fd77c15ae50 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.264+0000 7fd77b7fe640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c1541d0 secure :-1 s=READY pgs=1341 cs=0 l=1 rev1=1 crypto rx=0x7fd76c00ebe0 tx=0x7fd76c00c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.265+0000 7fd778ff9640 1 -- 172.21.15.64:0/180799250 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd76c017070 con 0x7fd77c15ae50 2024-03-04T07:32:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.265+0000 7fd778ff9640 1 -- 172.21.15.64:0/180799250 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd76c00cc30 con 0x7fd77c15ae50 2024-03-04T07:32:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.265+0000 7fd778ff9640 1 -- 172.21.15.64:0/180799250 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd76c013470 con 0x7fd77c15ae50 2024-03-04T07:32:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.265+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd77c13ce00 con 0x7fd77c15ae50 2024-03-04T07:32:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.265+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd77c14fb30 con 0x7fd77c15ae50 2024-03-04T07:32:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.266+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd77c0733c0 con 0x7fd77c15ae50 2024-03-04T07:32:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.266+0000 7fd778ff9640 1 -- 172.21.15.64:0/180799250 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd76c01a070 con 0x7fd77c15ae50 2024-03-04T07:32:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.266+0000 7fd778ff9640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76806b4d0 0x7fd76806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.267+0000 7fd77bfff640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76806b4d0 0x7fd76806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.267+0000 7fd778ff9640 1 -- 172.21.15.64:0/180799250 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fd76c089fe0 con 0x7fd77c15ae50 2024-03-04T07:32:19.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.267+0000 7fd77bfff640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76806b4d0 0x7fd76806d990 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7fd770037000 tx=0x7fd77003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.271+0000 7fd778ff9640 1 -- 172.21.15.64:0/180799250 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd76c0588a0 con 0x7fd77c15ae50 2024-03-04T07:32:19.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.451+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fd77c04f410 con 0x7fd77c15ae50 2024-03-04T07:32:19.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.452+0000 7fd778ff9640 1 -- 172.21.15.64:0/180799250 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v105) v1 ==== 104+0+11 (secure 0 0 0) 0x7fd76c05c940 con 0x7fd77c15ae50 2024-03-04T07:32:19.451 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76806b4d0 msgr2=0x7fd76806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76806b4d0 0x7fd76806d990 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7fd770037000 tx=0x7fd77003c040 comp rx=0 tx=0).stop 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 msgr2=0x7fd77c1541d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c1541d0 secure :-1 s=READY pgs=1341 cs=0 l=1 rev1=1 crypto rx=0x7fd76c00ebe0 tx=0x7fd76c00c520 comp rx=0 tx=0).stop 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 shutdown_connections 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd76806b4d0 0x7fd76806d990 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd77c15b230 0x7fd77c154710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd77c15ae50 0x7fd77c1541d0 unknown :-1 s=CLOSED pgs=1341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 --2- 172.21.15.64:0/180799250 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd77c0716a0 0x7fd77c13c8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 >> 172.21.15.64:0/180799250 conn(0x7fd77c066d00 msgr2=0x7fd77c15ceb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:19.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 shutdown_connections 2024-03-04T07:32:19.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.454+0000 7fd782688640 1 -- 172.21.15.64:0/180799250 wait complete. 2024-03-04T07:32:19.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:32:19.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.569+0000 7f89bfc81640 1 Processor -- start 2024-03-04T07:32:19.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.569+0000 7f89bfc81640 1 -- start start 2024-03-04T07:32:19.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bfc81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b8071540 0x7f89b8071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bfc81640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f89b80b2d30 0x7f89b80ce770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bfc81640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f89b8071e60 con 0x7f89b80cecb0 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bfc81640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f89b80d3230 con 0x7f89b80b2d30 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bfc81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89b80d4960 con 0x7f89b8071540 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bd21c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f89b80b2d30 0x7f89b80ce770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bd21c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f89b80b2d30 0x7f89b80ce770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43678/0 (socket says 172.21.15.64:43678) 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bd21c640 1 -- 172.21.15.64:0/698206115 learned_addr learned my addr 172.21.15.64:0/698206115 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.570+0000 7f89bda1d640 1 --2- 172.21.15.64:0/698206115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b8071540 0x7f89b8071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.571+0000 7f89bd21c640 1 -- 172.21.15.64:0/698206115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b8071540 msgr2=0x7f89b8071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.571+0000 7f89bd21c640 1 --2- 172.21.15.64:0/698206115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b8071540 0x7f89b8071920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.571+0000 7f89bd21c640 1 -- 172.21.15.64:0/698206115 >> v1:172.21.15.66:6789/0 conn(0x7f89b80cecb0 legacy=0x7f89b80b72e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.571+0000 7f89bca1b640 1 -- 172.21.15.64:0/698206115 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2955610620 0 0) 0x7f89b8071e60 con 0x7f89b80cecb0 2024-03-04T07:32:19.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.571+0000 7f89bd21c640 1 -- 172.21.15.64:0/698206115 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89b80d36c0 con 0x7f89b80b2d30 2024-03-04T07:32:19.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.571+0000 7f89bd21c640 1 --2- 172.21.15.64:0/698206115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b80b2d30 0x7f89b80ce770 secure :-1 s=READY pgs=1342 cs=0 l=1 rev1=1 crypto rx=0x7f89ac009870 tx=0x7f89ac02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e5cf3f1b7b575e1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bca1b640 1 -- 172.21.15.64:0/698206115 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89ac034070 con 0x7f89b80b2d30 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bca1b640 1 -- 172.21.15.64:0/698206115 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89ac030840 con 0x7f89b80b2d30 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bca1b640 1 -- 172.21.15.64:0/698206115 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89ac030c10 con 0x7f89b80b2d30 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 -- 172.21.15.64:0/698206115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b80b2d30 msgr2=0x7f89b80ce770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 --2- 172.21.15.64:0/698206115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b80b2d30 0x7f89b80ce770 secure :-1 s=READY pgs=1342 cs=0 l=1 rev1=1 crypto rx=0x7f89ac009870 tx=0x7f89ac02b980 comp rx=0 tx=0).stop 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 -- 172.21.15.64:0/698206115 shutdown_connections 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 --2- 172.21.15.64:0/698206115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b80b2d30 0x7f89b80ce770 unknown :-1 s=CLOSED pgs=1342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 --2- 172.21.15.64:0/698206115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b8071540 0x7f89b8071920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 -- 172.21.15.64:0/698206115 >> 172.21.15.64:0/698206115 conn(0x7f89b80d3d60 msgr2=0x7f89b80d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 -- 172.21.15.64:0/698206115 shutdown_connections 2024-03-04T07:32:19.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.572+0000 7f89bfc81640 1 -- 172.21.15.64:0/698206115 wait complete. 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.573+0000 7f89bfc81640 1 Processor -- start 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.573+0000 7f89bfc81640 1 -- start start 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bfc81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b8071540 0x7f89b81e3f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bfc81640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89b80b2d30 0x7f89b815c030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bfc81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b80cecb0 0x7f89b81e5240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bfc81640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89b80d2be0 con 0x7f89b8071540 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bfc81640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89b80d2900 con 0x7f89b80b2d30 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bfc81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f89b80d2ec0 con 0x7f89b80cecb0 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bda1d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b8071540 0x7f89b81e3f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bda1d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b8071540 0x7f89b81e3f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43694/0 (socket says 172.21.15.64:43694) 2024-03-04T07:32:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bda1d640 1 -- 172.21.15.64:0/4138338871 learned_addr learned my addr 172.21.15.64:0/4138338871 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bd21c640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89b80b2d30 0x7f89b815c030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89be21e640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b80cecb0 0x7f89b81e5240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bda1d640 1 -- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b80cecb0 msgr2=0x7f89b81e5240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.574+0000 7f89bda1d640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b80cecb0 0x7f89b81e5240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89bda1d640 1 -- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89b80b2d30 msgr2=0x7f89b815c030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89bda1d640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89b80b2d30 0x7f89b815c030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89bda1d640 1 -- 172.21.15.64:0/4138338871 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89b81e44c0 con 0x7f89b8071540 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89bda1d640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b8071540 0x7f89b81e3f80 secure :-1 s=READY pgs=1343 cs=0 l=1 rev1=1 crypto rx=0x7f89a800efc0 tx=0x7f89a800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89b67fc640 1 -- 172.21.15.64:0/4138338871 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89a8018020 con 0x7f89b8071540 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89b67fc640 1 -- 172.21.15.64:0/4138338871 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f89a8014480 con 0x7f89b8071540 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f89b81e4760 con 0x7f89b8071540 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89b67fc640 1 -- 172.21.15.64:0/4138338871 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89a8014850 con 0x7f89b8071540 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.575+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89b81e7780 con 0x7f89b8071540 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.576+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f89b8072070 con 0x7f89b8071540 2024-03-04T07:32:19.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.576+0000 7f89b67fc640 1 -- 172.21.15.64:0/4138338871 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f89a8005d10 con 0x7f89b8071540 2024-03-04T07:32:19.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.577+0000 7f89b67fc640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89a406b4d0 0x7f89a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.577+0000 7f89b67fc640 1 -- 172.21.15.64:0/4138338871 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f89a808aa90 con 0x7f89b8071540 2024-03-04T07:32:19.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.577+0000 7f89bd21c640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89a406b4d0 0x7f89a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.577+0000 7f89bd21c640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89a406b4d0 0x7f89a406d990 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f89ac036000 tx=0x7f89ac008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.581+0000 7f89b67fc640 1 -- 172.21.15.64:0/4138338871 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f89a8022070 con 0x7f89b8071540 2024-03-04T07:32:19.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.763+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f89b804f410 con 0x7f89b8071540 2024-03-04T07:32:19.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.763+0000 7f89b67fc640 1 -- 172.21.15.64:0/4138338871 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v105) v1 ==== 100+0+11 (secure 0 0 0) 0x7f89a8010040 con 0x7f89b8071540 2024-03-04T07:32:19.763 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:32:19.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89a406b4d0 msgr2=0x7f89a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89a406b4d0 0x7f89a406d990 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f89ac036000 tx=0x7f89ac008030 comp rx=0 tx=0).stop 2024-03-04T07:32:19.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b8071540 msgr2=0x7f89b81e3f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b8071540 0x7f89b81e3f80 secure :-1 s=READY pgs=1343 cs=0 l=1 rev1=1 crypto rx=0x7f89a800efc0 tx=0x7f89a800c490 comp rx=0 tx=0).stop 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 shutdown_connections 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f89a406b4d0 0x7f89a406d990 unknown :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f89b80cecb0 0x7f89b81e5240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f89b80b2d30 0x7f89b815c030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 --2- 172.21.15.64:0/4138338871 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f89b8071540 0x7f89b81e3f80 unknown :-1 s=CLOSED pgs=1343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 >> 172.21.15.64:0/4138338871 conn(0x7f89b80d3d60 msgr2=0x7f89b80c76c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 shutdown_connections 2024-03-04T07:32:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.765+0000 7f89bfc81640 1 -- 172.21.15.64:0/4138338871 wait complete. 2024-03-04T07:32:19.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:32:19.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.878+0000 7f7d146c9640 1 Processor -- start 2024-03-04T07:32:19.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.878+0000 7f7d146c9640 1 -- start start 2024-03-04T07:32:19.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.879+0000 7f7d146c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c158520 0x7f7d0c06d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.879+0000 7f7d146c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d0c071240 con 0x7f7d0c0aee10 2024-03-04T07:32:19.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.879+0000 7f7d146c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d0c071520 con 0x7f7d0c06dd60 2024-03-04T07:32:19.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.879+0000 7f7d146c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c06e1c0 con 0x7f7d0c158520 2024-03-04T07:32:19.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d12c66640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7d0c0aee10 0x7f7d0c0b3640 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39368/0 (socket says 172.21.15.64:39368) 2024-03-04T07:32:19.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d12c66640 1 -- 172.21.15.64:0/1912416722 learned_addr learned my addr 172.21.15.64:0/1912416722 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:19.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d12465640 1 --2- 172.21.15.64:0/1912416722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c158520 0x7f7d0c06d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d11463640 1 -- 172.21.15.64:0/1912416722 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2176887401 0 0) 0x7f7d0c071520 con 0x7f7d0c06dd60 2024-03-04T07:32:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d11463640 1 -- 172.21.15.64:0/1912416722 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7cf8003520 con 0x7f7d0c06dd60 2024-03-04T07:32:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d11463640 1 -- 172.21.15.64:0/1912416722 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4256748360 0 0) 0x7f7d0c071240 con 0x7f7d0c0aee10 2024-03-04T07:32:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d11463640 1 -- 172.21.15.64:0/1912416722 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7d0c071520 con 0x7f7d0c0aee10 2024-03-04T07:32:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d12465640 1 -- 172.21.15.64:0/1912416722 >> v1:172.21.15.66:6789/0 conn(0x7f7d0c06dd60 legacy=0x7f7d0c0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d12465640 1 -- 172.21.15.64:0/1912416722 >> v1:172.21.15.64:6789/0 conn(0x7f7d0c0aee10 legacy=0x7f7d0c0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.880+0000 7f7d12465640 1 -- 172.21.15.64:0/1912416722 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d0c071800 con 0x7f7d0c158520 2024-03-04T07:32:19.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.881+0000 7f7d12465640 1 --2- 172.21.15.64:0/1912416722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c158520 0x7f7d0c06d820 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f7d080097d0 tx=0x7f7d0802b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=29ee14b6f36c5035 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.881+0000 7f7d11463640 1 -- 172.21.15.64:0/1912416722 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d08035020 con 0x7f7d0c158520 2024-03-04T07:32:19.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.881+0000 7f7d11463640 1 -- 172.21.15.64:0/1912416722 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d08031890 con 0x7f7d0c158520 2024-03-04T07:32:19.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.881+0000 7f7d11463640 1 -- 172.21.15.64:0/1912416722 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d08031c60 con 0x7f7d0c158520 2024-03-04T07:32:19.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.881+0000 7f7d146c9640 1 -- 172.21.15.64:0/1912416722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c158520 msgr2=0x7f7d0c06d820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.881+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1912416722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c158520 0x7f7d0c06d820 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f7d080097d0 tx=0x7f7d0802b930 comp rx=0 tx=0).stop 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.882+0000 7f7d146c9640 1 -- 172.21.15.64:0/1912416722 shutdown_connections 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.882+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1912416722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c158520 0x7f7d0c06d820 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.882+0000 7f7d146c9640 1 -- 172.21.15.64:0/1912416722 >> 172.21.15.64:0/1912416722 conn(0x7f7d0c066d00 msgr2=0x7f7d0c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.882+0000 7f7d146c9640 1 -- 172.21.15.64:0/1912416722 shutdown_connections 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.882+0000 7f7d146c9640 1 -- 172.21.15.64:0/1912416722 wait complete. 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 Processor -- start 2024-03-04T07:32:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 -- start start 2024-03-04T07:32:19.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0c06dd60 0x7f7d0c152310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c0aee10 0x7f7d0c15ceb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d0c158520 0x7f7d0c0cdff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c158900 con 0x7f7d0c158520 2024-03-04T07:32:19.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c071b70 con 0x7f7d0c06dd60 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.883+0000 7f7d146c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d0c158a80 con 0x7f7d0c0aee10 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d0c158520 0x7f7d0c0cdff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d0c158520 0x7f7d0c0cdff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43698/0 (socket says 172.21.15.64:43698) 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12465640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0c06dd60 0x7f7d0c152310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 -- 172.21.15.64:0/1808022476 learned_addr learned my addr 172.21.15.64:0/1808022476 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d11c64640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c0aee10 0x7f7d0c15ceb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 -- 172.21.15.64:0/1808022476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c0aee10 msgr2=0x7f7d0c15ceb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c0aee10 0x7f7d0c15ceb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 -- 172.21.15.64:0/1808022476 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0c06dd60 msgr2=0x7f7d0c152310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0c06dd60 0x7f7d0c152310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 -- 172.21.15.64:0/1808022476 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d0c15a780 con 0x7f7d0c158520 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12465640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0c06dd60 0x7f7d0c152310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d12c66640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d0c158520 0x7f7d0c0cdff0 secure :-1 s=READY pgs=1345 cs=0 l=1 rev1=1 crypto rx=0x7f7d0400cb40 tx=0x7f7d0400a060 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d02ffd640 1 -- 172.21.15.64:0/1808022476 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d04017070 con 0x7f7d0c158520 2024-03-04T07:32:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d02ffd640 1 -- 172.21.15.64:0/1808022476 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d0400aca0 con 0x7f7d0c158520 2024-03-04T07:32:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d0c15bca0 con 0x7f7d0c158520 2024-03-04T07:32:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d02ffd640 1 -- 172.21.15.64:0/1808022476 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d04012620 con 0x7f7d0c158520 2024-03-04T07:32:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.884+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d0c15c220 con 0x7f7d0c158520 2024-03-04T07:32:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.885+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d0c067ff0 con 0x7f7d0c158520 2024-03-04T07:32:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.886+0000 7f7d02ffd640 1 -- 172.21.15.64:0/1808022476 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7d0401a070 con 0x7f7d0c158520 2024-03-04T07:32:19.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.886+0000 7f7d02ffd640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cf006b4d0 0x7f7cf006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:19.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.886+0000 7f7d02ffd640 1 -- 172.21.15.64:0/1808022476 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7d0408a040 con 0x7f7d0c158520 2024-03-04T07:32:19.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.886+0000 7f7d12465640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cf006b4d0 0x7f7cf006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:19.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.886+0000 7f7d12465640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cf006b4d0 0x7f7cf006d990 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f7d08036000 tx=0x7f7d08008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:19.890 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:19.890+0000 7f7d02ffd640 1 -- 172.21.15.64:0/1808022476 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7d040587e0 con 0x7f7d0c158520 2024-03-04T07:32:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.083+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f7d0c159540 con 0x7f7d0c158520 2024-03-04T07:32:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.084+0000 7f7d02ffd640 1 -- 172.21.15.64:0/1808022476 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v105) v1 ==== 80+0+0 (secure 0 0 0) 0x7f7d0405c880 con 0x7f7d0c158520 2024-03-04T07:32:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:32:20.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cf006b4d0 msgr2=0x7f7cf006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cf006b4d0 0x7f7cf006d990 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f7d08036000 tx=0x7f7d08008030 comp rx=0 tx=0).stop 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d0c158520 msgr2=0x7f7d0c0cdff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d0c158520 0x7f7d0c0cdff0 secure :-1 s=READY pgs=1345 cs=0 l=1 rev1=1 crypto rx=0x7f7d0400cb40 tx=0x7f7d0400a060 comp rx=0 tx=0).stop 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 shutdown_connections 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7cf006b4d0 0x7f7cf006d990 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d0c158520 0x7f7d0c0cdff0 unknown :-1 s=CLOSED pgs=1345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d0c0aee10 0x7f7d0c15ceb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 --2- 172.21.15.64:0/1808022476 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d0c06dd60 0x7f7d0c152310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 >> 172.21.15.64:0/1808022476 conn(0x7f7d0c066d00 msgr2=0x7f7d0c159a60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.085+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 shutdown_connections 2024-03-04T07:32:20.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.086+0000 7f7d146c9640 1 -- 172.21.15.64:0/1808022476 wait complete. 2024-03-04T07:32:20.096 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:32:20.096 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:20.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:20.121 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[72745]: fuse finished with error 0 and tester_r 0 2024-03-04T07:32:20.124 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:20.225 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:32:20.225 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:20.225 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:32:20.239 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:20.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.381+0000 7fbdd9313640 1 Processor -- start 2024-03-04T07:32:20.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.381+0000 7fbdd9313640 1 -- start start 2024-03-04T07:32:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd9313640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd415e000 0x7fbdd4071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd9313640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbdd40b2c50 0x7fbdd4071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd9313640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fbdd4161cb0 0x7fbdd4162140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd9313640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbdd40cece0 con 0x7fbdd40b2c50 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd9313640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbdd40ce970 con 0x7fbdd4161cb0 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd9313640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbdd40cef30 con 0x7fbdd415e000 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbdd40b2c50 0x7fbdd4071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbdd40b2c50 0x7fbdd4071a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43706/0 (socket says 172.21.15.64:43706) 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 -- 172.21.15.64:0/2066260025 learned_addr learned my addr 172.21.15.64:0/2066260025 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd37fe640 1 --2- 172.21.15.64:0/2066260025 >> v2:172.21.15.66:3300/0 conn(0x7fbdd4161cb0 0x7fbdd4162140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd2ffd640 1 --2- 172.21.15.64:0/2066260025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd415e000 0x7fbdd4071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 -- 172.21.15.64:0/2066260025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd415e000 msgr2=0x7fbdd4071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 --2- 172.21.15.64:0/2066260025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd415e000 0x7fbdd4071540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 -- 172.21.15.64:0/2066260025 >> v2:172.21.15.66:3300/0 conn(0x7fbdd4161cb0 msgr2=0x7fbdd4162140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 --2- 172.21.15.64:0/2066260025 >> v2:172.21.15.66:3300/0 conn(0x7fbdd4161cb0 0x7fbdd4162140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.382+0000 7fbdd27fc640 1 -- 172.21.15.64:0/2066260025 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbdd40cf210 con 0x7fbdd40b2c50 2024-03-04T07:32:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.383+0000 7fbdd27fc640 1 --2- 172.21.15.64:0/2066260025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4071a80 secure :-1 s=READY pgs=1346 cs=0 l=1 rev1=1 crypto rx=0x7fbdd40cad00 tx=0x7fbdc802b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=60752c77d7a4c01 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.383+0000 7fbdd1ffb640 1 -- 172.21.15.64:0/2066260025 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdc802e070 con 0x7fbdd40b2c50 2024-03-04T07:32:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.383+0000 7fbdd1ffb640 1 -- 172.21.15.64:0/2066260025 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbdc8037070 con 0x7fbdd40b2c50 2024-03-04T07:32:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.383+0000 7fbdd1ffb640 1 -- 172.21.15.64:0/2066260025 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdc8031a00 con 0x7fbdd40b2c50 2024-03-04T07:32:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 -- 172.21.15.64:0/2066260025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd40b2c50 msgr2=0x7fbdd4071a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 --2- 172.21.15.64:0/2066260025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4071a80 secure :-1 s=READY pgs=1346 cs=0 l=1 rev1=1 crypto rx=0x7fbdd40cad00 tx=0x7fbdc802b880 comp rx=0 tx=0).stop 2024-03-04T07:32:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 -- 172.21.15.64:0/2066260025 shutdown_connections 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 --2- 172.21.15.64:0/2066260025 >> v2:172.21.15.66:3300/0 conn(0x7fbdd4161cb0 0x7fbdd4162140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 --2- 172.21.15.64:0/2066260025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4071a80 unknown :-1 s=CLOSED pgs=1346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 --2- 172.21.15.64:0/2066260025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd415e000 0x7fbdd4071540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 -- 172.21.15.64:0/2066260025 >> 172.21.15.64:0/2066260025 conn(0x7fbdd40d3c80 msgr2=0x7fbdd40d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 -- 172.21.15.64:0/2066260025 shutdown_connections 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.384+0000 7fbdd9313640 1 -- 172.21.15.64:0/2066260025 wait complete. 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.385+0000 7fbdd9313640 1 Processor -- start 2024-03-04T07:32:20.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.385+0000 7fbdd9313640 1 -- start start 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd9313640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4149de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd9313640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd415e000 0x7fbdd40b5b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd9313640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd4161cb0 0x7fbdd40b6080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd9313640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbdd40b3320 con 0x7fbdd415e000 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd9313640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbdd40b31a0 con 0x7fbdd40b2c50 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd9313640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbdd415e3e0 con 0x7fbdd4161cb0 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd415e000 0x7fbdd40b5b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd415e000 0x7fbdd40b5b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43714/0 (socket says 172.21.15.64:43714) 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 -- 172.21.15.64:0/1277242920 learned_addr learned my addr 172.21.15.64:0/1277242920 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd2ffd640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4149de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd37fe640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd4161cb0 0x7fbdd40b6080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 -- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd4161cb0 msgr2=0x7fbdd40b6080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd4161cb0 0x7fbdd40b6080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 -- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbdd40b2c50 msgr2=0x7fbdd4149de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4149de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 -- 172.21.15.64:0/1277242920 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbdd414a320 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd2ffd640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4149de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.386+0000 7fbdd27fc640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd415e000 0x7fbdd40b5b40 secure :-1 s=READY pgs=1347 cs=0 l=1 rev1=1 crypto rx=0x7fbdc8000c00 tx=0x7fbdc80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.387+0000 7fbdbb7fe640 1 -- 172.21.15.64:0/1277242920 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdc8032670 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.387+0000 7fbdbb7fe640 1 -- 172.21.15.64:0/1277242920 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbdc8032810 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.387+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbdd40cce30 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.387+0000 7fbdbb7fe640 1 -- 172.21.15.64:0/1277242920 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbdc8032c00 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.387+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbdd40cd3f0 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.388+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbdd40d20d0 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.388+0000 7fbdbb7fe640 1 -- 172.21.15.64:0/1277242920 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbdc804c060 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.388+0000 7fbdbb7fe640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbdc006b4d0 0x7fbdc006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.389+0000 7fbdbb7fe640 1 -- 172.21.15.64:0/1277242920 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbdc80ab750 con 0x7fbdd415e000 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.389+0000 7fbdd2ffd640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbdc006b4d0 0x7fbdc006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.389+0000 7fbdd2ffd640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbdc006b4d0 0x7fbdc006d990 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7fbdcc009eb0 tx=0x7fbdcc009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.393+0000 7fbdbb7fe640 1 -- 172.21.15.64:0/1277242920 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbdd40d20d0 con 0x7fbdd415e000 2024-03-04T07:32:20.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.611+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbdd415d5e0 con 0x7fbdd415e000 2024-03-04T07:32:20.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.612+0000 7fbdbb7fe640 1 -- 172.21.15.64:0/1277242920 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 66 v66) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fbdc8040070 con 0x7fbdd415e000 2024-03-04T07:32:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:20.612 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":66,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6281,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/2549379142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2549379142},{"type":"v1","addr":"172.21.15.66:6834","nonce":2549379142}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63},{"gid":7854,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3068683808","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3068683808},{"type":"v1","addr":"172.21.15.64:6836","nonce":3068683808}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":62},{"gid":7857,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/209754994","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":209754994},{"type":"v1","addr":"172.21.15.64:6837","nonce":209754994}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":63}],"filesystems":[{"mdsmap":{"epoch":66,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:10.823856+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":7872},"failed":[],"damaged":[],"stopped":[],"info":{"gid_7872":{"gid":7872,"name":"d","rank":0,"incarnation":64,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6835/4084900379","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":4084900379},{"type":"v1","addr":"172.21.15.66:6835","nonce":4084900379}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":7872,"qdb_cluster":[7872]},"id":8}]} 2024-03-04T07:32:20.612 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 66 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbdc006b4d0 msgr2=0x7fbdc006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbdc006b4d0 0x7fbdc006d990 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7fbdcc009eb0 tx=0x7fbdcc009190 comp rx=0 tx=0).stop 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd415e000 msgr2=0x7fbdd40b5b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd415e000 0x7fbdd40b5b40 secure :-1 s=READY pgs=1347 cs=0 l=1 rev1=1 crypto rx=0x7fbdc8000c00 tx=0x7fbdc80046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 shutdown_connections 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbdc006b4d0 0x7fbdc006d990 unknown :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbdd4161cb0 0x7fbdd40b6080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbdd415e000 0x7fbdd40b5b40 unknown :-1 s=CLOSED pgs=1347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 --2- 172.21.15.64:0/1277242920 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbdd40b2c50 0x7fbdd4149de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 >> 172.21.15.64:0/1277242920 conn(0x7fbdd40d3c80 msgr2=0x7fbdd40b6f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:20.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 shutdown_connections 2024-03-04T07:32:20.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.614+0000 7fbdd9313640 1 -- 172.21.15.64:0/1277242920 wait complete. 2024-03-04T07:32:20.625 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.d.asok --format=json flush journal 2024-03-04T07:32:20.759 INFO:teuthology.orchestra.run.smithi066.stdout:{"message":"","return_code":0} 2024-03-04T07:32:20.759 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:32:20.760 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:32:20.760 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:20.760 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:32:20.760 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:20.761 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:32:20.761 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:20.761 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:32:20.762 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:32:20.770 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:32:20.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail b 2024-03-04T07:32:20.771 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:32:20.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail a 2024-03-04T07:32:20.772 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:32:20.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail d 2024-03-04T07:32:20.773 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:32:20.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail c 2024-03-04T07:32:20.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7f2d6fbc7640 1 Processor -- start 2024-03-04T07:32:20.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7f03f53d7640 1 Processor -- start 2024-03-04T07:32:20.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7fcbe5837640 1 Processor -- start 2024-03-04T07:32:20.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7f03f53d7640 1 -- start start 2024-03-04T07:32:20.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7fcbe5837640 1 -- start start 2024-03-04T07:32:20.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7f64d18d4640 1 Processor -- start 2024-03-04T07:32:20.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7f2d6fbc7640 1 -- start start 2024-03-04T07:32:20.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.919+0000 7f64d18d4640 1 -- start start 2024-03-04T07:32:20.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03f53d7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f03f0158520 0x7f03f006d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbe5837640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe006d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03f53d7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f03f006dd60 0x7f03f00ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03f53d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f015e670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbe5837640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcbe00aee10 0x7fcbe00b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f2d6fbc7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2d6815df00 0x7f2d680d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f2d6fbc7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2d680b9a30 0x7f2d680d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f2d6fbc7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d680cea30 con 0x7f2d680b95a0 2024-03-04T07:32:20.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f2d6fbc7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d680d2af0 con 0x7f2d6815df00 2024-03-04T07:32:20.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f2d6fbc7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d680cf3c0 con 0x7f2d680b9a30 2024-03-04T07:32:20.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbe5837640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcbe0071240 con 0x7fcbe006dd60 2024-03-04T07:32:20.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03f53d7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f03f0071b70 con 0x7f03f006dd60 2024-03-04T07:32:20.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03f53d7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f03f00af080 con 0x7f03f0158520 2024-03-04T07:32:20.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbe5837640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcbe006e1c0 con 0x7fcbe00aee10 2024-03-04T07:32:20.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbe5837640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe0071b70 con 0x7fcbe0158520 2024-03-04T07:32:20.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64d18d4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f64cc15df00 0x7f64cc0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03f53d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03f0071240 con 0x7f03f00aec40 2024-03-04T07:32:20.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64d18d4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f64cc0b95a0 0x7f64cc0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64d18d4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f64cc0cea30 con 0x7f64cc0b9a30 2024-03-04T07:32:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64d18d4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f64cc0d2af0 con 0x7f64cc15df00 2024-03-04T07:32:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64d18d4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f64cc0cf3c0 con 0x7f64cc0b95a0 2024-03-04T07:32:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03ef7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f015e670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03ee7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f03f006dd60 0x7f03f00ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbdeffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe006d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64c3fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f64cc0b95a0 0x7f64cc0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64c3fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f64cc0b95a0 0x7f64cc0b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:38210/0 (socket says 172.21.15.64:38210) 2024-03-04T07:32:20.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03ee7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f03f006dd60 0x7f03f00ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43720/0 (socket says 172.21.15.64:43720) 2024-03-04T07:32:20.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f64c3fff640 1 -- 172.21.15.64:0/383088085 learned_addr learned my addr 172.21.15.64:0/383088085 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03ee7fc640 1 -- 172.21.15.64:0/4078061998 learned_addr learned my addr 172.21.15.64:0/4078061998 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f03ef7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f015e670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:32862/0 (socket says 172.21.15.64:32862) 2024-03-04T07:32:20.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbdeffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe006d820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:32860/0 (socket says 172.21.15.64:32860) 2024-03-04T07:32:20.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7fcbdeffd640 1 -- 172.21.15.64:0/702418303 learned_addr learned my addr 172.21.15.64:0/702418303 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.920+0000 7f2d6d963640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2d6815df00 0x7f2d680d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f2d6d963640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2d6815df00 0x7f2d680d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43728/0 (socket says 172.21.15.64:43728) 2024-03-04T07:32:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f2d6d963640 1 -- 172.21.15.64:0/1000709042 learned_addr learned my addr 172.21.15.64:0/1000709042 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7fcbdf7fe640 1 --2- 172.21.15.64:0/702418303 >> v2:172.21.15.64:3300/0 conn(0x7fcbe00aee10 0x7fcbe00b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f03eeffd640 1 --2- 172.21.15.64:0/4078061998 >> v2:172.21.15.66:3300/0 conn(0x7f03f0158520 0x7f03f006d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f64caffd640 1 --2- 172.21.15.64:0/383088085 >> v2:172.21.15.64:3300/0 conn(0x7f64cc15df00 0x7f64cc0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f2d6e164640 1 --2- 172.21.15.64:0/1000709042 >> v2:172.21.15.66:3300/0 conn(0x7f2d680b9a30 0x7f2d680d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f03ef7fe640 1 -- 172.21.15.64:0/4078061998 >> v2:172.21.15.66:3300/0 conn(0x7f03f0158520 msgr2=0x7f03f006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f64c3fff640 1 -- 172.21.15.64:0/383088085 >> v2:172.21.15.64:3300/0 conn(0x7f64cc15df00 msgr2=0x7f64cc0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f64c3fff640 1 --2- 172.21.15.64:0/383088085 >> v2:172.21.15.64:3300/0 conn(0x7f64cc15df00 0x7f64cc0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f64c3fff640 1 -- 172.21.15.64:0/383088085 >> v1:172.21.15.64:6789/0 conn(0x7f64cc0b9a30 legacy=0x7f64cc0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f64c3fff640 1 -- 172.21.15.64:0/383088085 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64cc0cf0e0 con 0x7f64cc0b95a0 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f03ef7fe640 1 --2- 172.21.15.64:0/4078061998 >> v2:172.21.15.66:3300/0 conn(0x7f03f0158520 0x7f03f006d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f03ef7fe640 1 -- 172.21.15.64:0/4078061998 >> v2:172.21.15.64:3300/0 conn(0x7f03f006dd60 msgr2=0x7f03f00ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f03ef7fe640 1 --2- 172.21.15.64:0/4078061998 >> v2:172.21.15.64:3300/0 conn(0x7f03f006dd60 0x7f03f00ae700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f03ef7fe640 1 -- 172.21.15.64:0/4078061998 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03f0071520 con 0x7f03f00aec40 2024-03-04T07:32:20.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7fcbdeffd640 1 -- 172.21.15.64:0/702418303 >> v2:172.21.15.64:3300/0 conn(0x7fcbe00aee10 msgr2=0x7fcbe00b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:20.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7fcbdeffd640 1 --2- 172.21.15.64:0/702418303 >> v2:172.21.15.64:3300/0 conn(0x7fcbe00aee10 0x7fcbe00b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7fcbddffb640 1 -- 172.21.15.64:0/702418303 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1533112810 0 0) 0x7fcbe0071240 con 0x7fcbe006dd60 2024-03-04T07:32:20.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7fcbdeffd640 1 -- 172.21.15.64:0/702418303 >> v1:172.21.15.66:6789/0 conn(0x7fcbe006dd60 legacy=0x7fcbe00ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7fcbdeffd640 1 -- 172.21.15.64:0/702418303 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbe0071520 con 0x7fcbe0158520 2024-03-04T07:32:20.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f2d6e164640 1 -- 172.21.15.64:0/1000709042 >> v2:172.21.15.64:3300/0 conn(0x7f2d6815df00 msgr2=0x7f2d680d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:20.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f2d6e164640 1 --2- 172.21.15.64:0/1000709042 >> v2:172.21.15.64:3300/0 conn(0x7f2d6815df00 0x7f2d680d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f2d6e164640 1 -- 172.21.15.64:0/1000709042 >> v1:172.21.15.64:6789/0 conn(0x7f2d680b95a0 legacy=0x7f2d680b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.921+0000 7f2d6e164640 1 -- 172.21.15.64:0/1000709042 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d680cf0e0 con 0x7f2d680b9a30 2024-03-04T07:32:20.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7fcbdeffd640 1 --2- 172.21.15.64:0/702418303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe006d820 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7fcbd0009870 tx=0x7fcbd002b980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c39a3ae0f6b9812b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f03ef7fe640 1 --2- 172.21.15.64:0/4078061998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f015e670 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f03f006e900 tx=0x7f03e802e030 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4a64a1b57fd7f1df server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7fcbddffb640 1 -- 172.21.15.64:0/702418303 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd0034070 con 0x7fcbe0158520 2024-03-04T07:32:20.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7fcbddffb640 1 -- 172.21.15.64:0/702418303 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcbd0030840 con 0x7fcbe0158520 2024-03-04T07:32:20.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7fcbddffb640 1 -- 172.21.15.64:0/702418303 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd0030c10 con 0x7fcbe0158520 2024-03-04T07:32:20.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f64c3fff640 1 --2- 172.21.15.64:0/383088085 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc0b95a0 0x7f64cc0b5700 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f64cc0d1760 tx=0x7f64b402b9a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e2e1354ad36241f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f03edffb640 1 -- 172.21.15.64:0/4078061998 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03e803b070 con 0x7f03f00aec40 2024-03-04T07:32:20.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f03edffb640 1 -- 172.21.15.64:0/4078061998 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03e802ede0 con 0x7f03f00aec40 2024-03-04T07:32:20.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f03edffb640 1 -- 172.21.15.64:0/4078061998 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03e8036660 con 0x7f03f00aec40 2024-03-04T07:32:20.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f64ca7fc640 1 -- 172.21.15.64:0/383088085 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64b4034070 con 0x7f64cc0b95a0 2024-03-04T07:32:20.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f64ca7fc640 1 -- 172.21.15.64:0/383088085 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f64b4030860 con 0x7f64cc0b95a0 2024-03-04T07:32:20.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f2d6e164640 1 --2- 172.21.15.64:0/1000709042 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680b9a30 0x7f2d680d3220 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f2d680d4d20 tx=0x7f2d6002bec0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c6e177ff5d978933 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f64ca7fc640 1 -- 172.21.15.64:0/383088085 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64b4030c30 con 0x7f64cc0b95a0 2024-03-04T07:32:20.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.922+0000 7f2d6c961640 1 -- 172.21.15.64:0/1000709042 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d60039070 con 0x7f2d680b9a30 2024-03-04T07:32:20.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6c961640 1 -- 172.21.15.64:0/1000709042 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d6002cc70 con 0x7f2d680b9a30 2024-03-04T07:32:20.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6c961640 1 -- 172.21.15.64:0/1000709042 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d60034660 con 0x7f2d680b9a30 2024-03-04T07:32:20.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 -- 172.21.15.64:0/702418303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 msgr2=0x7fcbe006d820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 --2- 172.21.15.64:0/702418303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe006d820 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7fcbd0009870 tx=0x7fcbd002b980 comp rx=0 tx=0).stop 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 -- 172.21.15.64:0/702418303 shutdown_connections 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 --2- 172.21.15.64:0/702418303 >> v2:172.21.15.64:3300/0 conn(0x7fcbe00aee10 0x7fcbe00b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 --2- 172.21.15.64:0/702418303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe006d820 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 -- 172.21.15.64:0/702418303 >> 172.21.15.64:0/702418303 conn(0x7fcbe0066d00 msgr2=0x7fcbe00670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 -- 172.21.15.64:0/702418303 shutdown_connections 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 -- 172.21.15.64:0/4078061998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 msgr2=0x7f03f015e670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 --2- 172.21.15.64:0/4078061998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f015e670 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f03f006e900 tx=0x7f03e802e030 comp rx=0 tx=0).stop 2024-03-04T07:32:20.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7fcbe5837640 1 -- 172.21.15.64:0/702418303 wait complete. 2024-03-04T07:32:20.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 -- 172.21.15.64:0/4078061998 shutdown_connections 2024-03-04T07:32:20.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 --2- 172.21.15.64:0/4078061998 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f015e670 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 --2- 172.21.15.64:0/4078061998 >> v2:172.21.15.64:3300/0 conn(0x7f03f006dd60 0x7f03f00ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 --2- 172.21.15.64:0/4078061998 >> v2:172.21.15.66:3300/0 conn(0x7f03f0158520 0x7f03f006d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 -- 172.21.15.64:0/4078061998 >> 172.21.15.64:0/4078061998 conn(0x7f03f0066d00 msgr2=0x7f03f00670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:20.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 -- 172.21.15.64:0/383088085 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc0b95a0 msgr2=0x7f64cc0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 --2- 172.21.15.64:0/383088085 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc0b95a0 0x7f64cc0b5700 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f64cc0d1760 tx=0x7f64b402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 -- 172.21.15.64:0/4078061998 shutdown_connections 2024-03-04T07:32:20.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 -- 172.21.15.64:0/383088085 shutdown_connections 2024-03-04T07:32:20.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 --2- 172.21.15.64:0/383088085 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc0b95a0 0x7f64cc0b5700 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 --2- 172.21.15.64:0/383088085 >> v2:172.21.15.64:3300/0 conn(0x7f64cc15df00 0x7f64cc0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 -- 172.21.15.64:0/383088085 >> 172.21.15.64:0/383088085 conn(0x7f64cc071960 msgr2=0x7f64cc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:20.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f03f53d7640 1 -- 172.21.15.64:0/4078061998 wait complete. 2024-03-04T07:32:20.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1000709042 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680b9a30 msgr2=0x7f2d680d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1000709042 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680b9a30 0x7f2d680d3220 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f2d680d4d20 tx=0x7f2d6002bec0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 -- 172.21.15.64:0/383088085 shutdown_connections 2024-03-04T07:32:20.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f64d18d4640 1 -- 172.21.15.64:0/383088085 wait complete. 2024-03-04T07:32:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1000709042 shutdown_connections 2024-03-04T07:32:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1000709042 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680b9a30 0x7f2d680d3220 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1000709042 >> v2:172.21.15.64:3300/0 conn(0x7f2d6815df00 0x7f2d680d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1000709042 >> 172.21.15.64:0/1000709042 conn(0x7f2d68071960 msgr2=0x7f2d68071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1000709042 shutdown_connections 2024-03-04T07:32:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.923+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1000709042 wait complete. 2024-03-04T07:32:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.924+0000 7fcbe5837640 1 Processor -- start 2024-03-04T07:32:20.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.924+0000 7fcbe5837640 1 -- start start 2024-03-04T07:32:20.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.924+0000 7f64d18d4640 1 Processor -- start 2024-03-04T07:32:20.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.924+0000 7f03f53d7640 1 Processor -- start 2024-03-04T07:32:20.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f64d18d4640 1 -- start start 2024-03-04T07:32:20.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f03f53d7640 1 -- start start 2024-03-04T07:32:20.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f2d6fbc7640 1 Processor -- start 2024-03-04T07:32:20.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f2d6fbc7640 1 -- start start 2024-03-04T07:32:20.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbe5837640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbe006dd60 0x7fcbe01f3940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f03f53d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03f006dd60 0x7f03f014c2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbe5837640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbe00aee10 0x7fcbe014d310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f03f53d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f01f49e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f03f53d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03f0158520 0x7f03f014f590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f03f53d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03f0158900 con 0x7f03f0158520 2024-03-04T07:32:20.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f03f53d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03f0072000 con 0x7f03f006dd60 2024-03-04T07:32:20.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f03f53d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f03f0158a80 con 0x7f03f00aec40 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbdeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbe006dd60 0x7fcbe01f3940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbe5837640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe013f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbdeffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbe006dd60 0x7fcbe01f3940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43738/0 (socket says 172.21.15.64:43738) 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbdeffd640 1 -- 172.21.15.64:0/1289457840 learned_addr learned my addr 172.21.15.64:0/1289457840 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbe5837640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe0158900 con 0x7fcbe006dd60 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbe5837640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe0072000 con 0x7fcbe00aee10 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7fcbe5837640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe0158a80 con 0x7fcbe0158520 2024-03-04T07:32:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbde7fc640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbe00aee10 0x7fcbe014d310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f01f49e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03f0158520 0x7f03f014f590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f01f49e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:32878/0 (socket says 172.21.15.64:32878) 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03f0158520 0x7f03f014f590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43754/0 (socket says 172.21.15.64:43754) 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 -- 172.21.15.64:0/1804800658 learned_addr learned my addr 172.21.15.64:0/1804800658 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03eeffd640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03f006dd60 0x7f03f014c2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6fbc7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d680b95a0 0x7f2d68154ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6fbc7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d6815df00 0x7f2d681e8160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6fbc7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680c7940 0x7f2d680c7dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6fbc7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d680cec70 con 0x7f2d680b95a0 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6fbc7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d680ce750 con 0x7f2d680c7940 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6fbc7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d680cedf0 con 0x7f2d6815df00 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f64d18d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64cc0b95a0 0x7f64cc0cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f64d18d4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc15df00 0x7f64cc1552c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.925+0000 7f64d18d4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64cc0cd360 0x7f64cc0cc7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64d18d4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f64cc0cec70 con 0x7f64cc0b95a0 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64d18d4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f64cc0ce750 con 0x7f64cc15df00 2024-03-04T07:32:20.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64d18d4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f64cc0cedf0 con 0x7f64cc0cd360 2024-03-04T07:32:20.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbdf7fe640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe013f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64c3fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc15df00 0x7f64cc1552c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64c3fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc15df00 0x7f64cc1552c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:38232/0 (socket says 172.21.15.64:38232) 2024-03-04T07:32:20.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64c3fff640 1 -- 172.21.15.64:0/2876447093 learned_addr learned my addr 172.21.15.64:0/2876447093 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d6815df00 0x7f2d681e8160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d6815df00 0x7f2d681e8160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:32900/0 (socket says 172.21.15.64:32900) 2024-03-04T07:32:20.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 -- 172.21.15.64:0/1272775761 learned_addr learned my addr 172.21.15.64:0/1272775761 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:20.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 -- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03f006dd60 msgr2=0x7f03f014c2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03f006dd60 0x7f03f014c2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 -- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03f0158520 msgr2=0x7f03f014f590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03f0158520 0x7f03f014f590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 -- 172.21.15.64:0/1804800658 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03f00cde10 con 0x7f03f00aec40 2024-03-04T07:32:20.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64cb7fe640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64cc0cd360 0x7f64cc0cc7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03eeffd640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03f006dd60 0x7f03f014c2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:20.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbdeffd640 1 -- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 msgr2=0x7fcbe013f0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbdeffd640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe013f0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d963640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d680b95a0 0x7f2d68154ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6e164640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680c7940 0x7f2d680c7dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64caffd640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64cc0b95a0 0x7f64cc0cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbdeffd640 1 -- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbe00aee10 msgr2=0x7fcbe014d310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbdeffd640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbe00aee10 0x7fcbe014d310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbdeffd640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbe014dab0 con 0x7fcbe006dd60 2024-03-04T07:32:20.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 -- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680c7940 msgr2=0x7f2d680c7dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680c7940 0x7f2d680c7dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 -- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d680b95a0 msgr2=0x7f2d68154ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d680b95a0 0x7f2d68154ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d162640 1 -- 172.21.15.64:0/1272775761 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d680c9fc0 con 0x7f2d6815df00 2024-03-04T07:32:20.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f2d6d963640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d680b95a0 0x7f2d68154ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:20.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7fcbdeffd640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbe006dd60 0x7fcbe01f3940 secure :-1 s=READY pgs=1350 cs=0 l=1 rev1=1 crypto rx=0x7fcbd0000c00 tx=0x7fcbd00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f03ee7fc640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f01f49e0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f03e4009fd0 tx=0x7f03e400ed40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.926+0000 7f64c3fff640 1 -- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64cc0cd360 msgr2=0x7f64cc0cc7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c3fff640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64cc0cd360 0x7f64cc0cc7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c3fff640 1 -- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64cc0b95a0 msgr2=0x7f64cc0cad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c3fff640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64cc0b95a0 0x7f64cc0cad50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c3fff640 1 -- 172.21.15.64:0/2876447093 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64cc158d30 con 0x7f64cc15df00 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64cb7fe640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64cc0cd360 0x7f64cc0cc7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f2d6d162640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d6815df00 0x7f2d681e8160 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f2d5800c400 tx=0x7f2d5800c8c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f03d77fe640 1 -- 172.21.15.64:0/1804800658 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03e4017070 con 0x7f03f00aec40 2024-03-04T07:32:20.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7fcbc7fff640 1 -- 172.21.15.64:0/1289457840 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd0034040 con 0x7fcbe006dd60 2024-03-04T07:32:20.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7fcbc7fff640 1 -- 172.21.15.64:0/1289457840 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcbd003ab90 con 0x7fcbe006dd60 2024-03-04T07:32:20.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7fcbc7fff640 1 -- 172.21.15.64:0/1289457840 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd0004030 con 0x7fcbe006dd60 2024-03-04T07:32:20.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f03d77fe640 1 -- 172.21.15.64:0/1804800658 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f03e400c940 con 0x7f03f00aec40 2024-03-04T07:32:20.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03f00ce0f0 con 0x7f03f00aec40 2024-03-04T07:32:20.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03f00ce6b0 con 0x7f03f00aec40 2024-03-04T07:32:20.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7fcbe5837640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbe014ef10 con 0x7fcbe006dd60 2024-03-04T07:32:20.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7fcbe5837640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbe014f490 con 0x7fcbe006dd60 2024-03-04T07:32:20.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c3fff640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc15df00 0x7f64cc1552c0 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f64b402b9e0 tx=0x7f64b4004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f2d5e7fc640 1 -- 172.21.15.64:0/1272775761 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d580052d0 con 0x7f2d6815df00 2024-03-04T07:32:20.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f2d5e7fc640 1 -- 172.21.15.64:0/1272775761 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d58005490 con 0x7f2d6815df00 2024-03-04T07:32:20.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f2d5e7fc640 1 -- 172.21.15.64:0/1272775761 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d58007260 con 0x7f2d6815df00 2024-03-04T07:32:20.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d68155220 con 0x7f2d6815df00 2024-03-04T07:32:20.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d6815e930 con 0x7f2d6815df00 2024-03-04T07:32:20.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c8ff9640 1 -- 172.21.15.64:0/2876447093 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64b4048070 con 0x7f64cc15df00 2024-03-04T07:32:20.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c8ff9640 1 -- 172.21.15.64:0/2876447093 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f64b4030660 con 0x7f64cc15df00 2024-03-04T07:32:20.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.927+0000 7f64c8ff9640 1 -- 172.21.15.64:0/2876447093 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64b4034040 con 0x7f64cc15df00 2024-03-04T07:32:20.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.928+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f64cc0cb3e0 con 0x7f64cc15df00 2024-03-04T07:32:20.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.928+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64cc0cb9a0 con 0x7f64cc15df00 2024-03-04T07:32:20.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.928+0000 7f03d77fe640 1 -- 172.21.15.64:0/1804800658 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03e400cd10 con 0x7f03f00aec40 2024-03-04T07:32:20.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.928+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbe0067ff0 con 0x7fcbe006dd60 2024-03-04T07:32:20.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.928+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d680d18d0 con 0x7f2d6815df00 2024-03-04T07:32:20.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.929+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f03d0005290 con 0x7f03f00aec40 2024-03-04T07:32:20.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.929+0000 7fcbc7fff640 1 -- 172.21.15.64:0/1289457840 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcbd004b060 con 0x7fcbe006dd60 2024-03-04T07:32:20.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.929+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f64cc067ff0 con 0x7f64cc15df00 2024-03-04T07:32:20.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.929+0000 7f2d5e7fc640 1 -- 172.21.15.64:0/1272775761 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d58007400 con 0x7f2d6815df00 2024-03-04T07:32:20.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.929+0000 7f03d77fe640 1 -- 172.21.15.64:0/1804800658 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f03e401a070 con 0x7f03f00aec40 2024-03-04T07:32:20.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f2d5e7fc640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d4c06b500 0x7f2d4c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7fcbc7fff640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbcc06b4d0 0x7fcbcc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f2d5e7fc640 1 -- 172.21.15.64:0/1272775761 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2d58089780 con 0x7f2d6815df00 2024-03-04T07:32:20.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f03d77fe640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03dc06b4d0 0x7f03dc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.963 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7fcbc7fff640 1 -- 172.21.15.64:0/1289457840 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcbd00abfd0 con 0x7fcbe006dd60 2024-03-04T07:32:20.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f03d77fe640 1 -- 172.21.15.64:0/1804800658 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f03e408a4f0 con 0x7f03f00aec40 2024-03-04T07:32:20.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f2d6d963640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d4c06b500 0x7f2d4c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.964 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7fcbde7fc640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbcc06b4d0 0x7fcbcc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f03eeffd640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03dc06b4d0 0x7f03dc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f64c8ff9640 1 -- 172.21.15.64:0/2876447093 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f64b404b070 con 0x7f64cc15df00 2024-03-04T07:32:20.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7fcbde7fc640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbcc06b4d0 0x7fcbcc06d990 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7fcbe0149ab0 tx=0x7fcbd4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.930+0000 7f2d6d963640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d4c06b500 0x7f2d4c06d9c0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f2d64009b80 tx=0x7f2d64008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.931+0000 7f03eeffd640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03dc06b4d0 0x7f03dc06d990 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f03e0007d10 tx=0x7f03e0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.931+0000 7f64c8ff9640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f64ac06b520 0x7f64ac06d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:20.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.931+0000 7f64c8ff9640 1 -- 172.21.15.64:0/2876447093 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f64b40ac130 con 0x7f64cc15df00 2024-03-04T07:32:20.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.931+0000 7f64caffd640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f64ac06b520 0x7f64ac06d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:20.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.931+0000 7f64caffd640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f64ac06b520 0x7f64ac06d9e0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f64cc0c7330 tx=0x7f64bc008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:20.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.934+0000 7fcbc7fff640 1 -- 172.21.15.64:0/1289457840 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcbe014dab0 con 0x7fcbe006dd60 2024-03-04T07:32:20.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.938+0000 7f03d77fe640 1 -- 172.21.15.64:0/1804800658 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f03e4058c80 con 0x7f03f00aec40 2024-03-04T07:32:20.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.941+0000 7f64c8ff9640 1 -- 172.21.15.64:0/2876447093 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f64b407abf0 con 0x7f64cc15df00 2024-03-04T07:32:20.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:20.944+0000 7f2d5e7fc640 1 -- 172.21.15.64:0/1272775761 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d680d18d0 con 0x7f2d6815df00 2024-03-04T07:32:20.973 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.b is failed for ~0s 2024-03-04T07:32:20.973 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.d is failed for ~0s 2024-03-04T07:32:21.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.191+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "c"} v 0) v1 -- 0x7fcbe00d0680 con 0x7fcbe006dd60 2024-03-04T07:32:21.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.194+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "d"} v 0) v1 -- 0x7f03d0005580 con 0x7f03f00aec40 2024-03-04T07:32:21.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.199+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "b"} v 0) v1 -- 0x7f64cc0b6e40 con 0x7f64cc15df00 2024-03-04T07:32:21.202 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.202+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "a"} v 0) v1 -- 0x7f2d68067ff0 con 0x7f2d6815df00 2024-03-04T07:32:21.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.505+0000 7fcbc7fff640 1 -- 172.21.15.64:0/1289457840 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "c"}]=0 failed mds gid 7854 v67) v1 ==== 95+0+0 (secure 0 0 0) 0x7fcbd003f070 con 0x7fcbe006dd60 2024-03-04T07:32:21.505 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 7854 2024-03-04T07:32:21.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbcc06b4d0 msgr2=0x7fcbcc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:21.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbcc06b4d0 0x7fcbcc06d990 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7fcbe0149ab0 tx=0x7fcbd4009040 comp rx=0 tx=0).stop 2024-03-04T07:32:21.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbe006dd60 msgr2=0x7fcbe01f3940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbe006dd60 0x7fcbe01f3940 secure :-1 s=READY pgs=1350 cs=0 l=1 rev1=1 crypto rx=0x7fcbd0000c00 tx=0x7fcbd00046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 shutdown_connections 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcbcc06b4d0 0x7fcbcc06d990 unknown :-1 s=CLOSED pgs=609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbe0158520 0x7fcbe013f0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbe00aee10 0x7fcbe014d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 --2- 172.21.15.64:0/1289457840 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbe006dd60 0x7fcbe01f3940 unknown :-1 s=CLOSED pgs=1350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 >> 172.21.15.64:0/1289457840 conn(0x7fcbe0066d00 msgr2=0x7fcbe0158e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 shutdown_connections 2024-03-04T07:32:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.507+0000 7fcbc5ffb640 1 -- 172.21.15.64:0/1289457840 wait complete. 2024-03-04T07:32:21.508 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:32:21.508+0000 7ff841c6b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:32:21.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.511+0000 7f03d77fe640 1 -- 172.21.15.64:0/1804800658 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "d"}]=0 failed mds gid 7872 v68) v1 ==== 95+0+0 (secure 0 0 0) 0x7f03e405cd20 con 0x7f03f00aec40 2024-03-04T07:32:21.511 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 7872 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.513+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03dc06b4d0 msgr2=0x7f03dc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.513+0000 7f03f53d7640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03dc06b4d0 0x7f03dc06d990 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f03e0007d10 tx=0x7f03e0008040 comp rx=0 tx=0).stop 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.513+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 msgr2=0x7f03f01f49e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.513+0000 7f03f53d7640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f01f49e0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f03e4009fd0 tx=0x7f03e400ed40 comp rx=0 tx=0).stop 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 shutdown_connections 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f03dc06b4d0 0x7f03dc06d990 unknown :-1 s=CLOSED pgs=611 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f03f0158520 0x7f03f014f590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f03f00aec40 0x7f03f01f49e0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 --2- 172.21.15.64:0/1804800658 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f03f006dd60 0x7f03f014c2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 >> 172.21.15.64:0/1804800658 conn(0x7f03f0066d00 msgr2=0x7f03f00b3640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:21.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 shutdown_connections 2024-03-04T07:32:21.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.514+0000 7f03f53d7640 1 -- 172.21.15.64:0/1804800658 wait complete. 2024-03-04T07:32:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.520+0000 7f64c8ff9640 1 -- 172.21.15.64:0/2876447093 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "b"}]=0 failed mds gid 6281 v69) v1 ==== 95+0+0 (secure 0 0 0) 0x7f64b407ec90 con 0x7f64cc15df00 2024-03-04T07:32:21.519 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 6281 2024-03-04T07:32:21.522 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:32:21.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:32:21.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f64ac06b520 msgr2=0x7f64ac06d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:21.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f64ac06b520 0x7f64ac06d9e0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f64cc0c7330 tx=0x7f64bc008040 comp rx=0 tx=0).stop 2024-03-04T07:32:21.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc15df00 msgr2=0x7f64cc1552c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:21.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc15df00 0x7f64cc1552c0 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f64b402b9e0 tx=0x7f64b4004730 comp rx=0 tx=0).stop 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 shutdown_connections 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f64ac06b520 0x7f64ac06d9e0 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64cc0cd360 0x7f64cc0cc7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64cc15df00 0x7f64cc1552c0 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 --2- 172.21.15.64:0/2876447093 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64cc0b95a0 0x7f64cc0cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 >> 172.21.15.64:0/2876447093 conn(0x7f64cc071960 msgr2=0x7f64cc0b8c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 shutdown_connections 2024-03-04T07:32:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:21.522+0000 7f64d18d4640 1 -- 172.21.15.64:0/2876447093 wait complete. 2024-03-04T07:32:21.526 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:32:21.526 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:21.530 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:32:21.530 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:32:21.533 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:32:21.533 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:21.536 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:32:21.536 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:32:21.538 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:32:21.539 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:21.601 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:32:21.617 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:32:21.620 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:32:22.509 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:32:22.510+0000 7ff841c6b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:32:22.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.519+0000 7f2d5e7fc640 1 -- 172.21.15.64:0/1272775761 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "a"}]=0 failed mds gid 7857 v70) v1 ==== 95+0+0 (secure 0 0 0) 0x7f2d58058560 con 0x7f2d6815df00 2024-03-04T07:32:22.519 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 7857 2024-03-04T07:32:22.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d4c06b500 msgr2=0x7f2d4c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d4c06b500 0x7f2d4c06d9c0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f2d64009b80 tx=0x7f2d64008040 comp rx=0 tx=0).stop 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d6815df00 msgr2=0x7f2d681e8160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d6815df00 0x7f2d681e8160 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f2d5800c400 tx=0x7f2d5800c8c0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 shutdown_connections 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d4c06b500 0x7f2d4c06d9c0 unknown :-1 s=CLOSED pgs=610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d680c7940 0x7f2d680c7dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d6815df00 0x7f2d681e8160 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 --2- 172.21.15.64:0/1272775761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d680b95a0 0x7f2d68154ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.522+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 >> 172.21.15.64:0/1272775761 conn(0x7f2d68071960 msgr2=0x7f2d680b77e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.523+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 shutdown_connections 2024-03-04T07:32:22.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.523+0000 7f2d6fbc7640 1 -- 172.21.15.64:0/1272775761 wait complete. 2024-03-04T07:32:22.540 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:32:22.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:32:22.542 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:32:22.543 DEBUG:teuthology.parallel:result is None 2024-03-04T07:32:22.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:22.623 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:32:22.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.651+0000 7fe21aba1640 1 Processor -- start 2024-03-04T07:32:22.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.651+0000 7fe21aba1640 1 -- start start 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.651+0000 7fe21aba1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe21406b5c0 0x7fe2140b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.651+0000 7fe21aba1640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe2140b3340 con 0x7fe2140ae9b0 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.651+0000 7fe21aba1640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe2140cee90 con 0x7fe2140b0e90 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.651+0000 7fe21aba1640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe2140cfa00 con 0x7fe21406b5c0 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.652+0000 7fe21893d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe21406b5c0 0x7fe2140b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.652+0000 7fe21913e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fe2140ae9b0 0x7fe2140d03a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:39412/0 (socket says 172.21.15.64:39412) 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.652+0000 7fe21913e640 1 -- 172.21.15.64:0/3347450046 learned_addr learned my addr 172.21.15.64:0/3347450046 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.652+0000 7fe21893d640 1 -- 172.21.15.64:0/3347450046 >> v1:172.21.15.66:6789/0 conn(0x7fe2140b0e90 legacy=0x7fe21406b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.652+0000 7fe21893d640 1 -- 172.21.15.64:0/3347450046 >> v1:172.21.15.64:6789/0 conn(0x7fe2140ae9b0 legacy=0x7fe2140d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.652+0000 7fe21893d640 1 -- 172.21.15.64:0/3347450046 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe2140cfc50 con 0x7fe21406b5c0 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21893d640 1 --2- 172.21.15.64:0/3347450046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe21406b5c0 0x7fe2140b2e00 secure :-1 s=READY pgs=1359 cs=0 l=1 rev1=1 crypto rx=0x7fe2040097a0 tx=0x7fe20402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=260bb517fd0e4b43 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe2137fe640 1 -- 172.21.15.64:0/3347450046 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe204035020 con 0x7fe21406b5c0 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe2137fe640 1 -- 172.21.15.64:0/3347450046 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe204031830 con 0x7fe21406b5c0 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe2137fe640 1 -- 172.21.15.64:0/3347450046 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe204031c00 con 0x7fe21406b5c0 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21aba1640 1 -- 172.21.15.64:0/3347450046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe21406b5c0 msgr2=0x7fe2140b2e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21aba1640 1 --2- 172.21.15.64:0/3347450046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe21406b5c0 0x7fe2140b2e00 secure :-1 s=READY pgs=1359 cs=0 l=1 rev1=1 crypto rx=0x7fe2040097a0 tx=0x7fe20402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21aba1640 1 -- 172.21.15.64:0/3347450046 shutdown_connections 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21aba1640 1 --2- 172.21.15.64:0/3347450046 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe21406b5c0 0x7fe2140b2e00 unknown :-1 s=CLOSED pgs=1359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21aba1640 1 -- 172.21.15.64:0/3347450046 >> 172.21.15.64:0/3347450046 conn(0x7fe2140b36d0 msgr2=0x7fe2140b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21aba1640 1 -- 172.21.15.64:0/3347450046 shutdown_connections 2024-03-04T07:32:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.653+0000 7fe21aba1640 1 -- 172.21.15.64:0/3347450046 wait complete. 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.654+0000 7fe21aba1640 1 Processor -- start 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21aba1640 1 -- start start 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21aba1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe21406b5c0 0x7fe2140c6fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21aba1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe2140ae9b0 0x7fe2140c8dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21aba1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe2140b0e90 0x7fe214147010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21aba1640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe21406c910 con 0x7fe2140b0e90 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21aba1640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe21406c790 con 0x7fe2140ae9b0 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21aba1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe2140d0ab0 con 0x7fe21406b5c0 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.655+0000 7fe21913e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe2140b0e90 0x7fe214147010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe2140b0e90 0x7fe214147010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43824/0 (socket says 172.21.15.64:43824) 2024-03-04T07:32:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 -- 172.21.15.64:0/489519446 learned_addr learned my addr 172.21.15.64:0/489519446 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21893d640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe21406b5c0 0x7fe2140c6fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe213fff640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe2140ae9b0 0x7fe2140c8dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 -- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe21406b5c0 msgr2=0x7fe2140c6fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe21406b5c0 0x7fe2140c6fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 -- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe2140ae9b0 msgr2=0x7fe2140c8dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe2140ae9b0 0x7fe2140c8dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 -- 172.21.15.64:0/489519446 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe214145f70 con 0x7fe2140b0e90 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21913e640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe2140b0e90 0x7fe214147010 secure :-1 s=READY pgs=1360 cs=0 l=1 rev1=1 crypto rx=0x7fe20c00cae0 tx=0x7fe20c00a060 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe2117fa640 1 -- 172.21.15.64:0/489519446 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe20c017070 con 0x7fe2140b0e90 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe2117fa640 1 -- 172.21.15.64:0/489519446 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe20c00ac40 con 0x7fe2140b0e90 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe214146250 con 0x7fe2140b0e90 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe2117fa640 1 -- 172.21.15.64:0/489519446 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe20c012400 con 0x7fe2140b0e90 2024-03-04T07:32:22.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.656+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe214146810 con 0x7fe2140b0e90 2024-03-04T07:32:22.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.657+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe214068700 con 0x7fe2140b0e90 2024-03-04T07:32:22.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.658+0000 7fe2117fa640 1 -- 172.21.15.64:0/489519446 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe20c01a070 con 0x7fe2140b0e90 2024-03-04T07:32:22.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.658+0000 7fe2117fa640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe20006b4d0 0x7fe20006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:22.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.658+0000 7fe2117fa640 1 -- 172.21.15.64:0/489519446 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fe20c08a0a0 con 0x7fe2140b0e90 2024-03-04T07:32:22.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.658+0000 7fe21893d640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe20006b4d0 0x7fe20006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:22.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.659+0000 7fe21893d640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe20006b4d0 0x7fe20006d990 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7fe204036000 tx=0x7fe20403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:22.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.662+0000 7fe2117fa640 1 -- 172.21.15.64:0/489519446 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe20c058710 con 0x7fe2140b0e90 2024-03-04T07:32:22.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.881+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe214069030 con 0x7fe2140b0e90 2024-03-04T07:32:22.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.881+0000 7fe2117fa640 1 -- 172.21.15.64:0/489519446 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 71 v71) v1 ==== 94+0+4183 (secure 0 0 0) 0x7fe20c05c7b0 con 0x7fe2140b0e90 2024-03-04T07:32:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:22.881 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":71,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70}],"filesystems":[{"mdsmap":{"epoch":71,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:22.516178+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:replay","state_seq":1,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":8}]} 2024-03-04T07:32:22.882 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 71 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.883+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe20006b4d0 msgr2=0x7fe20006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.883+0000 7fe21aba1640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe20006b4d0 0x7fe20006d990 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7fe204036000 tx=0x7fe20403c040 comp rx=0 tx=0).stop 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.883+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe2140b0e90 msgr2=0x7fe214147010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.883+0000 7fe21aba1640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe2140b0e90 0x7fe214147010 secure :-1 s=READY pgs=1360 cs=0 l=1 rev1=1 crypto rx=0x7fe20c00cae0 tx=0x7fe20c00a060 comp rx=0 tx=0).stop 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 shutdown_connections 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe20006b4d0 0x7fe20006d990 unknown :-1 s=CLOSED pgs=616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe2140b0e90 0x7fe214147010 unknown :-1 s=CLOSED pgs=1360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe2140ae9b0 0x7fe2140c8dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 --2- 172.21.15.64:0/489519446 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe21406b5c0 0x7fe2140c6fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 >> 172.21.15.64:0/489519446 conn(0x7fe2140b36d0 msgr2=0x7fe21406ebb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 shutdown_connections 2024-03-04T07:32:22.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:22.884+0000 7fe21aba1640 1 -- 172.21.15.64:0/489519446 wait complete. 2024-03-04T07:32:23.297 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 71, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:32:08.826165+0000', 'modified': '2024-03-04T07:32:22.516178+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 109, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 8139}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8139': {'gid': 8139, 'name': 'c', 'rank': 0, 'incarnation': 71, 'state': 'up:replay', 'state_seq': 1, 'addr': '172.21.15.64:6835/1389682232', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 1389682232}, {'type': 'v1', 'addr': '172.21.15.64:6835', 'nonce': 1389682232}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-04T07:32:23.297 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8139:up:replay 2024-03-04T07:32:24.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:24.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.412+0000 7f26b14d4640 1 Processor -- start 2024-03-04T07:32:24.412 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.412+0000 7f26b14d4640 1 -- start start 2024-03-04T07:32:24.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26b14d4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f26ac15df00 0x7f26ac0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:24.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26b14d4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f26ac0b95a0 0x7f26ac0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:24.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26b14d4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f26ac0cea30 con 0x7f26ac0b9a30 2024-03-04T07:32:24.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26b14d4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f26ac0d2af0 con 0x7f26ac0b95a0 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26b14d4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f26ac0cf3c0 con 0x7f26ac15df00 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26aa7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f26ac0b95a0 0x7f26ac0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26ab7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f26ac0b9a30 0x7f26ac0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45038/0 (socket says 172.21.15.64:45038) 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26ab7fe640 1 -- 172.21.15.64:0/1766169369 learned_addr learned my addr 172.21.15.64:0/1766169369 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26aaffd640 1 --2- 172.21.15.64:0/1766169369 >> v2:172.21.15.66:3300/0 conn(0x7f26ac15df00 0x7f26ac0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26aa7fc640 1 -- 172.21.15.64:0/1766169369 >> v2:172.21.15.66:3300/0 conn(0x7f26ac15df00 msgr2=0x7f26ac0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26aa7fc640 1 --2- 172.21.15.64:0/1766169369 >> v2:172.21.15.66:3300/0 conn(0x7f26ac15df00 0x7f26ac0d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26aa7fc640 1 -- 172.21.15.64:0/1766169369 >> v1:172.21.15.64:6789/0 conn(0x7f26ac0b9a30 legacy=0x7f26ac0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26a9ffb640 1 -- 172.21.15.64:0/1766169369 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2580940268 0 0) 0x7f26ac0cea30 con 0x7f26ac0b9a30 2024-03-04T07:32:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.413+0000 7f26aa7fc640 1 -- 172.21.15.64:0/1766169369 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f26ac0cf0e0 con 0x7f26ac0b95a0 2024-03-04T07:32:24.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.414+0000 7f26aa7fc640 1 --2- 172.21.15.64:0/1766169369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac0b95a0 0x7f26ac0b5700 secure :-1 s=READY pgs=1362 cs=0 l=1 rev1=1 crypto rx=0x7f26ac0d11b0 tx=0x7f269402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a5342b857376824 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:24.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.414+0000 7f26a9ffb640 1 -- 172.21.15.64:0/1766169369 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2694035020 con 0x7f26ac0b95a0 2024-03-04T07:32:24.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.414+0000 7f26a9ffb640 1 -- 172.21.15.64:0/1766169369 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2694031830 con 0x7f26ac0b95a0 2024-03-04T07:32:24.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.414+0000 7f26a9ffb640 1 -- 172.21.15.64:0/1766169369 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2694031c00 con 0x7f26ac0b95a0 2024-03-04T07:32:24.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 -- 172.21.15.64:0/1766169369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac0b95a0 msgr2=0x7f26ac0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:24.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1766169369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac0b95a0 0x7f26ac0b5700 secure :-1 s=READY pgs=1362 cs=0 l=1 rev1=1 crypto rx=0x7f26ac0d11b0 tx=0x7f269402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 -- 172.21.15.64:0/1766169369 shutdown_connections 2024-03-04T07:32:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1766169369 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac0b95a0 0x7f26ac0b5700 unknown :-1 s=CLOSED pgs=1362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1766169369 >> v2:172.21.15.66:3300/0 conn(0x7f26ac15df00 0x7f26ac0d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 -- 172.21.15.64:0/1766169369 >> 172.21.15.64:0/1766169369 conn(0x7f26ac071960 msgr2=0x7f26ac071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 -- 172.21.15.64:0/1766169369 shutdown_connections 2024-03-04T07:32:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.415+0000 7f26b14d4640 1 -- 172.21.15.64:0/1766169369 wait complete. 2024-03-04T07:32:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.416+0000 7f26b14d4640 1 Processor -- start 2024-03-04T07:32:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.416+0000 7f26b14d4640 1 -- start start 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26b14d4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26ac0b95a0 0x7f26ac15a860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26b14d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac15df00 0x7f26ac1f78d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26b14d4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26ac15ae40 0x7f26ac161060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26b14d4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f26ac0cec70 con 0x7f26ac15df00 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26b14d4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f26ac0ce750 con 0x7f26ac0b95a0 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26b14d4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f26ac0cedf0 con 0x7f26ac15ae40 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26aa7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac15df00 0x7f26ac1f78d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26aa7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac15df00 0x7f26ac1f78d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54132/0 (socket says 172.21.15.64:54132) 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.417+0000 7f26aa7fc640 1 -- 172.21.15.64:0/1425251598 learned_addr learned my addr 172.21.15.64:0/1425251598 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:24.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aaffd640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26ac0b95a0 0x7f26ac15a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26ab7fe640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26ac15ae40 0x7f26ac161060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aa7fc640 1 -- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26ac15ae40 msgr2=0x7f26ac161060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aa7fc640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26ac15ae40 0x7f26ac161060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aa7fc640 1 -- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26ac0b95a0 msgr2=0x7f26ac15a860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aa7fc640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26ac0b95a0 0x7f26ac15a860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aa7fc640 1 -- 172.21.15.64:0/1425251598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f26ac1f6e30 con 0x7f26ac15df00 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aaffd640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26ac0b95a0 0x7f26ac15a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26aa7fc640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac15df00 0x7f26ac1f78d0 secure :-1 s=READY pgs=1363 cs=0 l=1 rev1=1 crypto rx=0x7f2694009890 tx=0x7f26940046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26a37fe640 1 -- 172.21.15.64:0/1425251598 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2694041070 con 0x7f26ac15df00 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26a37fe640 1 -- 172.21.15.64:0/1425251598 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f269403b920 con 0x7f26ac15df00 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f26ac1e8db0 con 0x7f26ac15df00 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26a37fe640 1 -- 172.21.15.64:0/1425251598 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2694035040 con 0x7f26ac15df00 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.418+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f26ac1f73b0 con 0x7f26ac15df00 2024-03-04T07:32:24.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.419+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f26ac067ff0 con 0x7f26ac15df00 2024-03-04T07:32:24.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.420+0000 7f26a37fe640 1 -- 172.21.15.64:0/1425251598 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2694038020 con 0x7f26ac15df00 2024-03-04T07:32:24.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.420+0000 7f26a37fe640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f268c06b4d0 0x7f268c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:24.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.420+0000 7f26a37fe640 1 -- 172.21.15.64:0/1425251598 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f26940abc70 con 0x7f26ac15df00 2024-03-04T07:32:24.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.420+0000 7f26aaffd640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f268c06b4d0 0x7f268c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:24.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.421+0000 7f26aaffd640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f268c06b4d0 0x7f268c06d990 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f26a4007440 tx=0x7f26a400b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:24.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.424+0000 7f26a37fe640 1 -- 172.21.15.64:0/1425251598 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2694031800 con 0x7f26ac15df00 2024-03-04T07:32:24.647 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.648+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f26ac0ce0a0 con 0x7f26ac15df00 2024-03-04T07:32:24.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.648+0000 7f26a37fe640 1 -- 172.21.15.64:0/1425251598 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 73 v73) v1 ==== 94+0+4971 (secure 0 0 0) 0x7f269407aa40 con 0x7f26ac15df00 2024-03-04T07:32:24.648 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:24.648 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":73,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":73,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:23.528614+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:rejoin","state_seq":3,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":0,"qdb_cluster":[]},"id":8}]} 2024-03-04T07:32:24.649 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 73 2024-03-04T07:32:24.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f268c06b4d0 msgr2=0x7f268c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:24.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f268c06b4d0 0x7f268c06d990 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f26a4007440 tx=0x7f26a400b040 comp rx=0 tx=0).stop 2024-03-04T07:32:24.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac15df00 msgr2=0x7f26ac1f78d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:24.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac15df00 0x7f26ac1f78d0 secure :-1 s=READY pgs=1363 cs=0 l=1 rev1=1 crypto rx=0x7f2694009890 tx=0x7f26940046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 shutdown_connections 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f268c06b4d0 0x7f268c06d990 unknown :-1 s=CLOSED pgs=618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f26ac15ae40 0x7f26ac161060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f26ac15df00 0x7f26ac1f78d0 unknown :-1 s=CLOSED pgs=1363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 --2- 172.21.15.64:0/1425251598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f26ac0b95a0 0x7f26ac15a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 >> 172.21.15.64:0/1425251598 conn(0x7f26ac071960 msgr2=0x7f26ac0b8730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 shutdown_connections 2024-03-04T07:32:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:24.650+0000 7f26b14d4640 1 -- 172.21.15.64:0/1425251598 wait complete. 2024-03-04T07:32:25.063 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 73, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:32:08.826165+0000', 'modified': '2024-03-04T07:32:23.528614+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 109, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 8139}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8139': {'gid': 8139, 'name': 'c', 'rank': 0, 'incarnation': 71, 'state': 'up:rejoin', 'state_seq': 3, 'addr': '172.21.15.64:6835/1389682232', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 1389682232}, {'type': 'v1', 'addr': '172.21.15.64:6835', 'nonce': 1389682232}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 0, 'qdb_cluster': []} 2024-03-04T07:32:25.063 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_8139:up:rejoin 2024-03-04T07:32:26.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:26.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.166+0000 7f64241dd640 1 Processor -- start 2024-03-04T07:32:26.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.166+0000 7f64241dd640 1 -- start start 2024-03-04T07:32:26.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f64241dd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f641c0b95a0 0x7f641c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f64241dd640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f641c0cea30 con 0x7f641c15df00 2024-03-04T07:32:26.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f64241dd640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f641c0cf3c0 con 0x7f641c0b9a10 2024-03-04T07:32:26.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f64241dd640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f641c0cf0e0 con 0x7f641c0b95a0 2024-03-04T07:32:26.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6421778640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f641c0b95a0 0x7f641c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6421f79640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f641c15df00 0x7f641c0d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45042/0 (socket says 172.21.15.64:45042) 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6421778640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f641c0b95a0 0x7f641c0b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54136/0 (socket says 172.21.15.64:54136) 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6421f79640 1 -- 172.21.15.64:0/3612308266 learned_addr learned my addr 172.21.15.64:0/3612308266 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6421778640 1 -- 172.21.15.64:0/3612308266 >> v1:172.21.15.66:6789/0 conn(0x7f641c0b9a10 legacy=0x7f641c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6421778640 1 -- 172.21.15.64:0/3612308266 >> v1:172.21.15.64:6789/0 conn(0x7f641c15df00 legacy=0x7f641c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6421778640 1 -- 172.21.15.64:0/3612308266 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f641c0ce470 con 0x7f641c0b95a0 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.167+0000 7f6420f77640 1 -- 172.21.15.64:0/3612308266 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2877756136 0 0) 0x7f641c0cea30 con 0x7f641c15df00 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.168+0000 7f6421778640 1 --2- 172.21.15.64:0/3612308266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c0b5700 secure :-1 s=READY pgs=1365 cs=0 l=1 rev1=1 crypto rx=0x7f641c0cf590 tx=0x7f640c02b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6193193e7a71f3e6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.168+0000 7f6420f77640 1 -- 172.21.15.64:0/3612308266 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f640c034070 con 0x7f641c0b95a0 2024-03-04T07:32:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.168+0000 7f6420f77640 1 -- 172.21.15.64:0/3612308266 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f640c030610 con 0x7f641c0b95a0 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.168+0000 7f6420f77640 1 -- 172.21.15.64:0/3612308266 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f640c0309e0 con 0x7f641c0b95a0 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.169+0000 7f64241dd640 1 -- 172.21.15.64:0/3612308266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 msgr2=0x7f641c0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.169+0000 7f64241dd640 1 --2- 172.21.15.64:0/3612308266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c0b5700 secure :-1 s=READY pgs=1365 cs=0 l=1 rev1=1 crypto rx=0x7f641c0cf590 tx=0x7f640c02b960 comp rx=0 tx=0).stop 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.169+0000 7f64241dd640 1 -- 172.21.15.64:0/3612308266 shutdown_connections 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.169+0000 7f64241dd640 1 --2- 172.21.15.64:0/3612308266 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c0b5700 unknown :-1 s=CLOSED pgs=1365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.169+0000 7f64241dd640 1 -- 172.21.15.64:0/3612308266 >> 172.21.15.64:0/3612308266 conn(0x7f641c071960 msgr2=0x7f641c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.169+0000 7f64241dd640 1 -- 172.21.15.64:0/3612308266 shutdown_connections 2024-03-04T07:32:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.169+0000 7f64241dd640 1 -- 172.21.15.64:0/3612308266 wait complete. 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.170+0000 7f64241dd640 1 Processor -- start 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.170+0000 7f64241dd640 1 -- start start 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f64241dd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c159f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f64241dd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f641c0b9a10 0x7f641c15a460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f64241dd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f641c15df00 0x7f641c1f6c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f64241dd640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f641c0cedf0 con 0x7f641c0b95a0 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f64241dd640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f641c0cec70 con 0x7f641c0b9a10 2024-03-04T07:32:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f64241dd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f641c0b5c40 con 0x7f641c15df00 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c159f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c159f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54142/0 (socket says 172.21.15.64:54142) 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 -- 172.21.15.64:0/4052415995 learned_addr learned my addr 172.21.15.64:0/4052415995 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421778640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f641c0b9a10 0x7f641c15a460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f642277a640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f641c15df00 0x7f641c1f6c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 -- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f641c15df00 msgr2=0x7f641c1f6c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f641c15df00 0x7f641c1f6c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 -- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f641c0b9a10 msgr2=0x7f641c15a460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f641c0b9a10 0x7f641c15a460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 -- 172.21.15.64:0/4052415995 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f641c1f7910 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f642277a640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f641c15df00 0x7f641c1f6c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421778640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f641c0b9a10 0x7f641c15a460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.171+0000 7f6421f79640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c159f20 secure :-1 s=READY pgs=1366 cs=0 l=1 rev1=1 crypto rx=0x7f641800c3f0 tx=0x7f641800c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.172+0000 7f64127fc640 1 -- 172.21.15.64:0/4052415995 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64180074e0 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.172+0000 7f64127fc640 1 -- 172.21.15.64:0/4052415995 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f64180047e0 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.172+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f641c0cc390 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.172+0000 7f64127fc640 1 -- 172.21.15.64:0/4052415995 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6418004c10 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.172+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f641c0cc950 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.172+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f641c067ff0 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.174+0000 7f64127fc640 1 -- 172.21.15.64:0/4052415995 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6418011070 con 0x7f641c0b95a0 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.174+0000 7f64127fc640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f640006b500 0x7f640006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.174+0000 7f64127fc640 1 -- 172.21.15.64:0/4052415995 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f6418089f10 con 0x7f641c0b95a0 2024-03-04T07:32:26.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.174+0000 7f6421778640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f640006b500 0x7f640006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.174+0000 7f6421778640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f640006b500 0x7f640006d9c0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f640c036000 tx=0x7f640c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.178+0000 7f64127fc640 1 -- 172.21.15.64:0/4052415995 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f641801a2b0 con 0x7f641c0b95a0 2024-03-04T07:32:26.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.397+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f641c0c7530 con 0x7f641c0b95a0 2024-03-04T07:32:26.397 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.397+0000 7f64127fc640 1 -- 172.21.15.64:0/4052415995 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f6418058ce0 con 0x7f641c0b95a0 2024-03-04T07:32:26.397 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:26.398 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:32:26.398 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f640006b500 msgr2=0x7f640006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f640006b500 0x7f640006d9c0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f640c036000 tx=0x7f640c008030 comp rx=0 tx=0).stop 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 msgr2=0x7f641c159f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c159f20 secure :-1 s=READY pgs=1366 cs=0 l=1 rev1=1 crypto rx=0x7f641800c3f0 tx=0x7f641800c8b0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 shutdown_connections 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f640006b500 0x7f640006d9c0 unknown :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f641c15df00 0x7f641c1f6c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f641c0b9a10 0x7f641c15a460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 --2- 172.21.15.64:0/4052415995 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f641c0b95a0 0x7f641c159f20 unknown :-1 s=CLOSED pgs=1366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 >> 172.21.15.64:0/4052415995 conn(0x7f641c071960 msgr2=0x7f641c0b6d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 shutdown_connections 2024-03-04T07:32:26.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.400+0000 7f64241dd640 1 -- 172.21.15.64:0/4052415995 wait complete. 2024-03-04T07:32:26.814 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 74, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:32:08.826165+0000', 'modified': '2024-03-04T07:32:25.534407+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 109, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 8139}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8139': {'gid': 8139, 'name': 'c', 'rank': 0, 'incarnation': 71, 'state': 'up:active', 'state_seq': 4, 'addr': '172.21.15.64:6835/1389682232', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.64:6834', 'nonce': 1389682232}, {'type': 'v1', 'addr': '172.21.15.64:6835', 'nonce': 1389682232}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [17], 'metadata_pool': 16, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8139, 'qdb_cluster': [8139]} 2024-03-04T07:32:26.814 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:32:26.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-04T07:32:26.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.919+0000 7f7d31c83640 1 Processor -- start 2024-03-04T07:32:26.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.919+0000 7f7d31c83640 1 -- start start 2024-03-04T07:32:26.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.919+0000 7f7d31c83640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c0cebb0 0x7f7d2c0d0940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d31c83640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7d2c1627f0 0x7f7d2c06ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d31c83640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d2c0d1090 con 0x7f7d2c161c80 2024-03-04T07:32:26.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d31c83640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7d2c06b0a0 con 0x7f7d2c1627f0 2024-03-04T07:32:26.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d31c83640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c06b380 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2b7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c0cebb0 0x7f7d2c0d0940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2bfff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7d2c1627f0 0x7f7d2c06ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d23fff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f7d2c161c80 0x7f7d2c1620e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:47164/0 (socket says 172.21.15.64:47164) 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d23fff640 1 -- 172.21.15.64:0/3854793559 learned_addr learned my addr 172.21.15.64:0/3854793559 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2affd640 1 -- 172.21.15.64:0/3854793559 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1156414018 0 0) 0x7f7d2c0d1090 con 0x7f7d2c161c80 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2affd640 1 -- 172.21.15.64:0/3854793559 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7d18003520 con 0x7f7d2c161c80 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/3854793559 >> v2:172.21.15.66:3300/0 conn(0x7f7d2c1627f0 msgr2=0x7f7d2c06ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/3854793559 >> v2:172.21.15.66:3300/0 conn(0x7f7d2c1627f0 0x7f7d2c06ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/3854793559 >> v1:172.21.15.66:6789/0 conn(0x7f7d2c161c80 legacy=0x7f7d2c1620e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/3854793559 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d2c06b660 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.920+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/3854793559 >> v2:172.21.15.66:3300/0 conn(0x7f7d2c1627f0 0x7f7d2c06ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/3854793559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c0cebb0 0x7f7d2c0d0940 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c0097d0 tx=0x7f7d1c02b8a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7344d18d8921b9f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d2affd640 1 -- 172.21.15.64:0/3854793559 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d1c035020 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d2affd640 1 -- 172.21.15.64:0/3854793559 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d1c031800 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d2affd640 1 -- 172.21.15.64:0/3854793559 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d1c031bb0 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 -- 172.21.15.64:0/3854793559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c0cebb0 msgr2=0x7f7d2c0d0940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 --2- 172.21.15.64:0/3854793559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c0cebb0 0x7f7d2c0d0940 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c0097d0 tx=0x7f7d1c02b8a0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 -- 172.21.15.64:0/3854793559 shutdown_connections 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 --2- 172.21.15.64:0/3854793559 >> v2:172.21.15.66:3300/0 conn(0x7f7d2c1627f0 0x7f7d2c06ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.923 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 --2- 172.21.15.64:0/3854793559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c0cebb0 0x7f7d2c0d0940 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 -- 172.21.15.64:0/3854793559 >> 172.21.15.64:0/3854793559 conn(0x7f7d2c0b35d0 msgr2=0x7f7d2c0b39e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 -- 172.21.15.64:0/3854793559 shutdown_connections 2024-03-04T07:32:26.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.922+0000 7f7d31c83640 1 -- 172.21.15.64:0/3854793559 wait complete. 2024-03-04T07:32:26.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 Processor -- start 2024-03-04T07:32:26.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 -- start start 2024-03-04T07:32:26.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d2c0cebb0 0x7f7d2c15c1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d2c161c80 0x7f7d2c152310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c1627f0 0x7f7d2c0afa30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c06f850 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c06f6d0 con 0x7f7d2c161c80 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d31c83640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c06f9d0 con 0x7f7d2c1627f0 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d2c0cebb0 0x7f7d2c15c1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d2c0cebb0 0x7f7d2c15c1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54154/0 (socket says 172.21.15.64:54154) 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.924+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/55942639 learned_addr learned my addr 172.21.15.64:0/55942639 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d23fff640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d2c161c80 0x7f7d2c152310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c1627f0 0x7f7d2c0afa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c1627f0 msgr2=0x7f7d2c0afa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c1627f0 0x7f7d2c0afa30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d2c161c80 msgr2=0x7f7d2c152310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d2c161c80 0x7f7d2c152310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/55942639 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d2c15c730 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d2c0cebb0 0x7f7d2c15c1f0 secure :-1 s=READY pgs=1367 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c0098f0 tx=0x7f7d1c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d28ff9640 1 -- 172.21.15.64:0/55942639 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d1c041070 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d28ff9640 1 -- 172.21.15.64:0/55942639 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d1c031e30 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d2c0caae0 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d28ff9640 1 -- 172.21.15.64:0/55942639 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d1c035040 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.925+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d2c0cb0a0 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.927+0000 7f7d28ff9640 1 -- 172.21.15.64:0/55942639 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7d1c038020 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.927+0000 7f7d28ff9640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d0c06b4d0 0x7f7d0c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.927+0000 7f7d23fff640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d0c06b4d0 0x7f7d0c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.927+0000 7f7d28ff9640 1 -- 172.21.15.64:0/55942639 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f7d1c0abd20 con 0x7f7d2c0cebb0 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.928+0000 7f7d23fff640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d0c06b4d0 0x7f7d0c06d990 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f7d2c06bb30 tx=0x7f7d14009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.928+0000 7f7d3485f740 1 Processor -- start 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.929+0000 7f7d3485f740 1 -- start start 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.929+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f5126870 0x55c1f5126c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.929+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f512a8c0 0x55c1f512c360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.929+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f5156600 0x55c1f5156a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.929+0000 7f7d3485f740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c1f50906b0 con 0x55c1f5156600 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.929+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c1f5039a60 con 0x55c1f5126870 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.929+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55c1f508c750 con 0x55c1f512a8c0 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f5156600 0x55c1f5156a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f5156600 0x55c1f5156a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54166/0 (socket says 172.21.15.64:54166) 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 -- 172.21.15.64:0/1069311765 learned_addr learned my addr 172.21.15.64:0/1069311765 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:26.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f5126870 0x55c1f5126c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d23fff640 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f512a8c0 0x55c1f512c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 -- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f512a8c0 msgr2=0x55c1f512c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f512a8c0 0x55c1f512c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 -- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f5126870 msgr2=0x55c1f5126c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f5126870 0x55c1f5126c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 -- 172.21.15.64:0/1069311765 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c1f5057f30 con 0x55c1f5156600 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f5156600 0x55c1f5156a80 secure :-1 s=READY pgs=1368 cs=0 l=1 rev1=1 crypto rx=0x7f7d24007490 tx=0x7f7d24009750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2ce41debb56ec10 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d217fa640 1 -- 172.21.15.64:0/1069311765 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d24019020 con 0x55c1f5156600 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d217fa640 1 -- 172.21.15.64:0/1069311765 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d24015850 con 0x55c1f5156600 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.930+0000 7f7d217fa640 1 -- 172.21.15.64:0/1069311765 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d24015c00 con 0x55c1f5156600 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 -- 172.21.15.64:0/1069311765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f5156600 msgr2=0x55c1f5156a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f5156600 0x55c1f5156a80 secure :-1 s=READY pgs=1368 cs=0 l=1 rev1=1 crypto rx=0x7f7d24007490 tx=0x7f7d24009750 comp rx=0 tx=0).stop 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 -- 172.21.15.64:0/1069311765 shutdown_connections 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f5156600 0x55c1f5156a80 unknown :-1 s=CLOSED pgs=1368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f512a8c0 0x55c1f512c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1069311765 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f5126870 0x55c1f5126c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 -- 172.21.15.64:0/1069311765 >> 172.21.15.64:0/1069311765 conn(0x55c1f51288f0 msgr2=0x55c1f512a1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 -- 172.21.15.64:0/1069311765 shutdown_connections 2024-03-04T07:32:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.931+0000 7f7d3485f740 1 -- 172.21.15.64:0/1069311765 wait complete. 2024-03-04T07:32:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 -- start start 2024-03-04T07:32:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 Processor -- start 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f5151230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 0x55c1f514a830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f514ad70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c1f512ad50 con 0x55c1f514b990 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c1f504cea0 con 0x55c1f514b440 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55c1f512b030 con 0x55c1f5156600 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d3485f740 10 client.0 _ll_register_callbacks cb 0x55c1f504c870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f514ad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f514ad70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54174/0 (socket says 172.21.15.64:54174) 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d2bfff640 1 -- 172.21.15.64:0/447866327 learned_addr learned my addr 172.21.15.64:0/447866327 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.932+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f5151230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d23fff640 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 0x55c1f514a830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d2bfff640 1 -- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 msgr2=0x55c1f5151230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f5151230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d2bfff640 1 -- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 msgr2=0x55c1f514a830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 0x55c1f514a830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d2bfff640 1 -- 172.21.15.64:0/447866327 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c1f504d270 con 0x55c1f514b990 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f514ad70 secure :-1 s=READY pgs=1369 cs=0 l=1 rev1=1 crypto rx=0x7f7d2400b010 tx=0x7f7d24004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1bb7790ae5f9fc43 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d0a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/447866327 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d24024070 con 0x55c1f514b990 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/447866327 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f7d24029b60 con 0x55c1f514b990 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7d24015850 con 0x55c1f514b990 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/447866327 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=74) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7d24019040 con 0x55c1f514b990 2024-03-04T07:32:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d3485f740 10 client.8193 fetch_fsmap learned FSMap version 74 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=74,monmap=2+}) v3 -- 0x55c1f4da52d0 con 0x55c1f514b990 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.933+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/447866327 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d24015850 con 0x55c1f514b990 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/447866327 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 74) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f7d2401e630 con 0x55c1f514b990 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d3485f740 10 client.8193 fetch_fsmap finished waiting for FSMap version 74 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d3485f740 10 client.8193 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d3485f740 20 client.8193 populate_metadata read hostname 'smithi064' 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d3485f740 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f5159480 0x55c1f5159860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d3485f740 4 client.8193 mds_command: new command op to 8139 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 --> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c1f509dbf0 con 0x55c1f5159480 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f5159480 0x55c1f5159860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f5159480 0x55c1f5159860 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4ec35fbcbe793e6 server_cookie=1d63742de3337f18 in_seq=0 out_seq=0 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.934+0000 7f7d0a7fc640 10 client.8193 ms_handle_connect on v2:172.21.15.64:6834/1389682232 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/447866327 <== mds.0 v2:172.21.15.64:6834/1389682232 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x55c1f509dbf0 con 0x55c1f5159480 2024-03-04T07:32:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d0a7fc640 10 client.8193 handle_command_reply: tid=0 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 client.8193 shutdown 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 msgr2=0x55c1f514ad70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f514ad70 secure :-1 s=READY pgs=1369 cs=0 l=1 rev1=1 crypto rx=0x7f7d2400b010 tx=0x7f7d24004ab0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 shutdown_connections 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f5159480 0x55c1f5159860 crc :-1 s=READY pgs=12 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f514ad70 unknown :-1 s=CLOSED pgs=1369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 0x55c1f514a830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 --2- 172.21.15.64:0/447866327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f5151230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d0a7fc640 0 client.8193 ms_handle_reset on v2:172.21.15.64:6834/1389682232 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 >> 172.21.15.64:0/447866327 conn(0x55c1f5126870 msgr2=0x55c1f51545c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 shutdown_connections 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 1 -- 172.21.15.64:0/447866327 wait complete. 2024-03-04T07:32:26.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.935+0000 7f7d3485f740 20 client.8193 trim_cache size 0 max 16384 2024-03-04T07:32:26.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.963+0000 7f7d3485f740 1 Processor -- start 2024-03-04T07:32:26.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.963+0000 7f7d3485f740 1 -- start start 2024-03-04T07:32:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b440 0x55c1f51560c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b990 0x55c1f51794e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f51569e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d3485f740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c200310 con 0x55c1f514b440 2024-03-04T07:32:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c0b5ec0 con 0x55c1f514b990 2024-03-04T07:32:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c15b3f0 con 0x55c1f5156600 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b440 0x55c1f51560c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b440 0x55c1f51560c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54188/0 (socket says 172.21.15.64:54188) 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/1197298991 learned_addr learned my addr 172.21.15.64:0/1197298991 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d23fff640 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b990 0x55c1f51794e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2bfff640 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f51569e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 msgr2=0x55c1f51569e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f51569e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b990 msgr2=0x55c1f51794e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b990 0x55c1f51794e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 -- 172.21.15.64:0/1197298991 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c1f4e6f6b0 con 0x55c1f514b440 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.964+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b440 0x55c1f51560c0 secure :-1 s=READY pgs=1370 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c000c00 tx=0x7f7d1c047040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=937eb426f8d26b1c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d20ff9640 1 -- 172.21.15.64:0/1197298991 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d1c07ab00 con 0x55c1f514b440 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d20ff9640 1 -- 172.21.15.64:0/1197298991 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d1c07a400 con 0x55c1f514b440 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d20ff9640 1 -- 172.21.15.64:0/1197298991 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d1c07e550 con 0x55c1f514b440 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 -- 172.21.15.64:0/1197298991 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b440 msgr2=0x55c1f51560c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b440 0x55c1f51560c0 secure :-1 s=READY pgs=1370 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c000c00 tx=0x7f7d1c047040 comp rx=0 tx=0).stop 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 -- 172.21.15.64:0/1197298991 shutdown_connections 2024-03-04T07:32:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f51569e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b990 0x55c1f51794e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 --2- 172.21.15.64:0/1197298991 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b440 0x55c1f51560c0 unknown :-1 s=CLOSED pgs=1370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 -- 172.21.15.64:0/1197298991 >> 172.21.15.64:0/1197298991 conn(0x55c1f5126870 msgr2=0x55c1f51598d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 -- 172.21.15.64:0/1197298991 shutdown_connections 2024-03-04T07:32:26.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.965+0000 7f7d3485f740 1 -- 172.21.15.64:0/1197298991 wait complete. 2024-03-04T07:32:26.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.966+0000 7f7d3485f740 1 -- start start 2024-03-04T07:32:26.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.966+0000 7f7d3485f740 1 Processor -- start 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 0x55c1f51945b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f5194af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f51e4440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c1f514f960 con 0x55c1f514b990 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2c0b58c0 con 0x55c1f514b440 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55c1f5151770 con 0x55c1f5156600 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 10 client.0 _ll_register_callbacks cb 0x7f7d24015d50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f5194af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f5194af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54202/0 (socket says 172.21.15.64:54202) 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 -- 172.21.15.64:0/2844795764 learned_addr learned my addr 172.21.15.64:0/2844795764 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 -- 172.21.15.64:0/2844795764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 msgr2=0x55c1f51e4440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f51e4440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 -- 172.21.15.64:0/2844795764 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 msgr2=0x55c1f51945b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 0x55c1f51945b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 -- 172.21.15.64:0/2844795764 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c1f5157b80 con 0x55c1f514b990 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d23fff640 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f5194af0 secure :-1 s=READY pgs=1371 cs=0 l=1 rev1=1 crypto rx=0x7f7d2c15c180 tx=0x7f7d140010a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6829a52b6b90b42a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d0a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/2844795764 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d14007830 con 0x55c1f514b990 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/2844795764 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f7d1401e040 con 0x55c1f514b990 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7d1c07a400 con 0x55c1f514b990 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.967+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/2844795764 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=74) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7d14013420 con 0x55c1f514b990 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 10 client.8199 fetch_fsmap learned FSMap version 74 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=74,monmap=2+}) v3 -- 0x55c1f5172200 con 0x55c1f514b990 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/2844795764 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d1c07a400 con 0x55c1f514b990 2024-03-04T07:32:26.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/2844795764 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 74) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f7d140135c0 con 0x55c1f514b990 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 10 client.8199 fetch_fsmap finished waiting for FSMap version 74 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 10 client.8199 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 20 client.8199 populate_metadata read hostname 'smithi064' 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f51e4fb0 0x55c1f51711c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 4 client.8199 mds_command: new command op to 8139 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 --> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55c1f5129050 con 0x55c1f51e4fb0 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f51e4fb0 0x55c1f51711c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d0a7fc640 10 client.8199 ms_handle_connect on v2:172.21.15.64:6834/1389682232 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d2b7fe640 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f51e4fb0 0x55c1f51711c0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=914773748ebe5dbd server_cookie=2e761961dd212f49 in_seq=0 out_seq=0 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d0a7fc640 1 -- 172.21.15.64:0/2844795764 <== mds.0 v2:172.21.15.64:6834/1389682232 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+333 (crc 0 0 0) 0x55c1f5129050 con 0x55c1f51e4fb0 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d0a7fc640 10 client.8199 handle_command_reply: tid=0 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.968+0000 7f7d3485f740 1 client.8199 shutdown 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 msgr2=0x55c1f5194af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f5194af0 secure :-1 s=READY pgs=1371 cs=0 l=1 rev1=1 crypto rx=0x7f7d2c15c180 tx=0x7f7d140010a0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 shutdown_connections 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:6834/1389682232,v1:172.21.15.64:6835/1389682232] conn(0x55c1f51e4fb0 0x55c1f51711c0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55c1f5156600 0x55c1f51e4440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55c1f514b990 0x55c1f5194af0 unknown :-1 s=CLOSED pgs=1371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 --2- 172.21.15.64:0/2844795764 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55c1f514b440 0x55c1f51945b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 >> 172.21.15.64:0/2844795764 conn(0x55c1f5126870 msgr2=0x55c1f512a080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d0a7fc640 0 client.8199 ms_handle_reset on v2:172.21.15.64:6834/1389682232 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 shutdown_connections 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 1 -- 172.21.15.64:0/2844795764 wait complete. 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d3485f740 20 client.8199 trim_cache size 0 max 16384 2024-03-04T07:32:26.976 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 8139, 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 4.4483079759999997, 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 74, 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 109, 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 109, 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 5.3659664940000003 2024-03-04T07:32:26.977 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:32:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d0c06b4d0 msgr2=0x7f7d0c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d0c06b4d0 0x7f7d0c06d990 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f7d2c06bb30 tx=0x7f7d14009040 comp rx=0 tx=0).stop 2024-03-04T07:32:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d2c0cebb0 msgr2=0x7f7d2c15c1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d2c0cebb0 0x7f7d2c15c1f0 secure :-1 s=READY pgs=1367 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c0098f0 tx=0x7f7d1c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 shutdown_connections 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d0c06b4d0 0x7f7d0c06d990 unknown :-1 s=CLOSED pgs=620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d2c1627f0 0x7f7d2c0afa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d2c161c80 0x7f7d2c152310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 --2- 172.21.15.64:0/55942639 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d2c0cebb0 0x7f7d2c15c1f0 unknown :-1 s=CLOSED pgs=1367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 >> 172.21.15.64:0/55942639 conn(0x7f7d2c0b35d0 msgr2=0x7f7d2c0b39b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 shutdown_connections 2024-03-04T07:32:26.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:26.969+0000 7f7d31c83640 1 -- 172.21.15.64:0/55942639 wait complete. 2024-03-04T07:32:26.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:32:27.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.082+0000 7fc506be9640 1 Processor -- start 2024-03-04T07:32:27.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.082+0000 7fc506be9640 1 -- start start 2024-03-04T07:32:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc506be9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc50015e060 0x7fc5000ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc506be9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc50006fe20 0x7fc5000ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc506be9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5000d0880 0x7fc5000ced90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc506be9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5000d0dd0 con 0x7fc50006fe20 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc506be9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5000d10b0 con 0x7fc50015e060 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc506be9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5000cc0e0 con 0x7fc5000d0880 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc4fffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc50006fe20 0x7fc5000ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc4fffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc50006fe20 0x7fc5000ce850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54212/0 (socket says 172.21.15.64:54212) 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc4fffff640 1 -- 172.21.15.64:0/3795739531 learned_addr learned my addr 172.21.15.64:0/3795739531 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc505186640 1 --2- 172.21.15.64:0/3795739531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5000d0880 0x7fc5000ced90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.083+0000 7fc504985640 1 --2- 172.21.15.64:0/3795739531 >> v2:172.21.15.66:3300/0 conn(0x7fc50015e060 0x7fc5000ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc4fffff640 1 -- 172.21.15.64:0/3795739531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5000d0880 msgr2=0x7fc5000ced90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc4fffff640 1 --2- 172.21.15.64:0/3795739531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5000d0880 0x7fc5000ced90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc4fffff640 1 -- 172.21.15.64:0/3795739531 >> v2:172.21.15.66:3300/0 conn(0x7fc50015e060 msgr2=0x7fc5000ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc4fffff640 1 --2- 172.21.15.64:0/3795739531 >> v2:172.21.15.66:3300/0 conn(0x7fc50015e060 0x7fc5000ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc4fffff640 1 -- 172.21.15.64:0/3795739531 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5000cc3c0 con 0x7fc50006fe20 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc505186640 1 --2- 172.21.15.64:0/3795739531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5000d0880 0x7fc5000ced90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc504985640 1 --2- 172.21.15.64:0/3795739531 >> v2:172.21.15.66:3300/0 conn(0x7fc50015e060 0x7fc5000ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.084+0000 7fc4fffff640 1 --2- 172.21.15.64:0/3795739531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc5000ce850 secure :-1 s=READY pgs=1372 cs=0 l=1 rev1=1 crypto rx=0x7fc5000722f0 tx=0x7fc4f002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=798130873b2f228c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc4ff7fe640 1 -- 172.21.15.64:0/3795739531 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4f0035020 con 0x7fc50006fe20 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc4ff7fe640 1 -- 172.21.15.64:0/3795739531 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc4f0031870 con 0x7fc50006fe20 2024-03-04T07:32:27.086 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc4ff7fe640 1 -- 172.21.15.64:0/3795739531 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4f0031c40 con 0x7fc50006fe20 2024-03-04T07:32:27.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 -- 172.21.15.64:0/3795739531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 msgr2=0x7fc5000ce850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:27.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 --2- 172.21.15.64:0/3795739531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc5000ce850 secure :-1 s=READY pgs=1372 cs=0 l=1 rev1=1 crypto rx=0x7fc5000722f0 tx=0x7fc4f002b910 comp rx=0 tx=0).stop 2024-03-04T07:32:27.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 -- 172.21.15.64:0/3795739531 shutdown_connections 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 --2- 172.21.15.64:0/3795739531 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5000d0880 0x7fc5000ced90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 --2- 172.21.15.64:0/3795739531 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc5000ce850 unknown :-1 s=CLOSED pgs=1372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 --2- 172.21.15.64:0/3795739531 >> v2:172.21.15.66:3300/0 conn(0x7fc50015e060 0x7fc5000ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 -- 172.21.15.64:0/3795739531 >> 172.21.15.64:0/3795739531 conn(0x7fc5000d1a90 msgr2=0x7fc5000d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 -- 172.21.15.64:0/3795739531 shutdown_connections 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.085+0000 7fc506be9640 1 -- 172.21.15.64:0/3795739531 wait complete. 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.087+0000 7fc506be9640 1 Processor -- start 2024-03-04T07:32:27.088 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.087+0000 7fc506be9640 1 -- start start 2024-03-04T07:32:27.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc500153040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:27.091 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5000d0880 0x7fc50006af20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc50015e060 0x7fc50006b460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc50006ed30 con 0x7fc50006fe20 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc50015e440 con 0x7fc5000d0880 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc50006eeb0 con 0x7fc50015e060 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc500153040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc500153040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54224/0 (socket says 172.21.15.64:54224) 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 -- 172.21.15.64:0/2100283620 learned_addr learned my addr 172.21.15.64:0/2100283620 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 -- 172.21.15.64:0/2100283620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc50015e060 msgr2=0x7fc50006b460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc50015e060 0x7fc50006b460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 -- 172.21.15.64:0/2100283620 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5000d0880 msgr2=0x7fc50006af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc4fffff640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5000d0880 0x7fc50006af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:27.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5000d0880 0x7fc50006af20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 -- 172.21.15.64:0/2100283620 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc50015f7b0 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc504985640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc500153040 secure :-1 s=READY pgs=1373 cs=0 l=1 rev1=1 crypto rx=0x7fc4e800d800 tx=0x7fc4e800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc4fd7fa640 1 -- 172.21.15.64:0/2100283620 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4e8004030 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc4fd7fa640 1 -- 172.21.15.64:0/2100283620 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc4e80041d0 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc50015fa90 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc4fd7fa640 1 -- 172.21.15.64:0/2100283620 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc4e800b6c0 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.089+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc500160050 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.090+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc500067ff0 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.091+0000 7fc4fd7fa640 1 -- 172.21.15.64:0/2100283620 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc4e800b860 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.091+0000 7fc4fd7fa640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc4e006b4d0 0x7fc4e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.091+0000 7fc4fd7fa640 1 -- 172.21.15.64:0/2100283620 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fc4e808a530 con 0x7fc50006fe20 2024-03-04T07:32:27.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.091+0000 7fc4fffff640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc4e006b4d0 0x7fc4e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:27.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.092+0000 7fc4fffff640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc4e006b4d0 0x7fc4e006d990 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7fc4f0036000 tx=0x7fc4f0008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:27.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.096+0000 7fc4fd7fa640 1 -- 172.21.15.64:0/2100283620 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc4e80592f0 con 0x7fc50006fe20 2024-03-04T07:32:27.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.318+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fc5000d1670 con 0x7fc50006fe20 2024-03-04T07:32:27.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.318+0000 7fc4fd7fa640 1 -- 172.21.15.64:0/2100283620 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v74) v1 ==== 53+0+73 (secure 0 0 0) 0x7fc4e8058bf0 con 0x7fc50006fe20 2024-03-04T07:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:32:27.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc4e006b4d0 msgr2=0x7fc4e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:27.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc4e006b4d0 0x7fc4e006d990 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7fc4f0036000 tx=0x7fc4f0008030 comp rx=0 tx=0).stop 2024-03-04T07:32:27.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 msgr2=0x7fc500153040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:27.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc500153040 secure :-1 s=READY pgs=1373 cs=0 l=1 rev1=1 crypto rx=0x7fc4e800d800 tx=0x7fc4e800dcc0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 shutdown_connections 2024-03-04T07:32:27.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc4e006b4d0 0x7fc4e006d990 unknown :-1 s=CLOSED pgs=621 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc50015e060 0x7fc50006b460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5000d0880 0x7fc50006af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 --2- 172.21.15.64:0/2100283620 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc50006fe20 0x7fc500153040 unknown :-1 s=CLOSED pgs=1373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:27.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 >> 172.21.15.64:0/2100283620 conn(0x7fc5000d1a90 msgr2=0x7fc5000cead0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:27.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 shutdown_connections 2024-03-04T07:32:27.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:27.320+0000 7fc506be9640 1 -- 172.21.15.64:0/2100283620 wait complete. 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:32:27.331 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:32:27.332 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:27.332 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:32:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42365sec preferred_lft 42365sec 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:32:27.349 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:27.349 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:32:27.412 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:32:27.414 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:27.414 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:32:27.469 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:32:27.469 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:32:27.469 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:32:27.469 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:32:27.470 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:32:27.470 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:27.470 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:32:27.470 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:32:27.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:32:27.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:32:27.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:32:27.470 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:32:27.519 INFO:tasks.ceph.mds.c.smithi064.stderr:2024-03-04T07:32:27.520+0000 7fedfc44e640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:32:27.566 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:27.566 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:32:27.620 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:32:27.620 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:32:27.621 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:32:27.621 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:27.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:32:27.687 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:32:27.688 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:32:27.688 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:27.690 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.b has been restored 2024-03-04T07:32:27.690 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mds.d has been restored 2024-03-04T07:32:27.743 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:32:27.744 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:32:27.744 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:27.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:32:27.864 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:32:27.920 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:32:27.920 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:32:27.921 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:32:27.922 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:27.923 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:27.924 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:32:27.974 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:32:27.974 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:32:28.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:32:28.043 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:32:28.098 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:32:28.099 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:32:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:32:28.101 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:28.101 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:32:28.107 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:32:28.107+0000 7f0e2c638480 -1 init, newargv = 0x55580c0aa480 newargc=15 2024-03-04T07:32:28.107 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[74376]: starting ceph client 2024-03-04T07:32:28.111 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[74376]: starting fuse 2024-03-04T07:32:28.121 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:32:28.122 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:32:28.122 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:32:28.123 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:32:28.242 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.74376.asok 2024-03-04T07:32:28.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:32:28.252 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.74376.asok 2024-03-04T07:32:28.252 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:28.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.74376.asok status 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:32:28.390 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "74376", 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 8211, 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 8211 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:32:28.391 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 439418770 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 439418770 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.8211 192.168.0.1:0/439418770", 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/439418770", 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 74, 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 109, 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:32:28.392 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:32:28.393 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:32:28.393 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:32:28.398 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:28.398 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:32:28.399 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:28.414 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:32:28.414 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:28.414 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:28.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:32:28.482 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:32:28.482 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f dir_1/file_a) 2024-03-04T07:32:28.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.683+0000 7fc9ddb25640 1 Processor -- start 2024-03-04T07:32:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.683+0000 7fc9ddb25640 1 -- start start 2024-03-04T07:32:28.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.683+0000 7fc9ddb25640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc9d815df00 0x7fc9d80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.683+0000 7fc9ddb25640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 0x7fc9d80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.683+0000 7fc9ddb25640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc9d80cea30 con 0x7fc9d80b9a30 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.683+0000 7fc9ddb25640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc9d80d2af0 con 0x7fc9d815df00 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.683+0000 7fc9ddb25640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9d80cf3c0 con 0x7fc9d80b95a0 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc9d80b9a30 0x7fc9d80d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45054/0 (socket says 172.21.15.64:45054) 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d7fff640 1 -- 172.21.15.64:0/3977884652 learned_addr learned my addr 172.21.15.64:0/3977884652 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d6ffd640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 0x7fc9d80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d77fe640 1 --2- 172.21.15.64:0/3977884652 >> v2:172.21.15.66:3300/0 conn(0x7fc9d815df00 0x7fc9d80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1757724575 0 0) 0x7fc9d80cea30 con 0x7fc9d80b9a30 2024-03-04T07:32:28.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc9c0003520 con 0x7fc9d80b9a30 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2690084784 0 0) 0x7fc9c0003520 con 0x7fc9d80b9a30 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 msgr2=0x7fc9d80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 0x7fc9d80b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 >> v2:172.21.15.66:3300/0 conn(0x7fc9d815df00 msgr2=0x7fc9d80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 --2- 172.21.15.64:0/3977884652 >> v2:172.21.15.66:3300/0 conn(0x7fc9d815df00 0x7fc9d80d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9d80cf0e0 con 0x7fc9d80b9a30 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fc9d00019f0 con 0x7fc9d80b9a30 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d77fe640 1 --2- 172.21.15.64:0/3977884652 >> v2:172.21.15.66:3300/0 conn(0x7fc9d815df00 0x7fc9d80d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d6ffd640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 0x7fc9d80b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 >> v1:172.21.15.64:6789/0 conn(0x7fc9d80b9a30 legacy=0x7fc9d80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9c00039b0 0x7fc9c0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9c0004800 0x7fc9c0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 0x7fc9c00051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9d80d2af0 con 0x7fc9c0004800 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9d80cf3c0 con 0x7fc9c00039b0 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9c000b9c0 con 0x7fc9d80b95a0 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d6ffd640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9c0004800 0x7fc9c0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.684+0000 7fc9d77fe640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9c00039b0 0x7fc9c0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.685+0000 7fc9d7fff640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 0x7fc9c00051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.685+0000 7fc9d6ffd640 1 -- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 msgr2=0x7fc9c00051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:28.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.685+0000 7fc9d6ffd640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9d80b95a0 0x7fc9c00051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.685+0000 7fc9d6ffd640 1 -- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9c00039b0 msgr2=0x7fc9c0003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.685+0000 7fc9d6ffd640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9c00039b0 0x7fc9c0003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.685+0000 7fc9d6ffd640 1 -- 172.21.15.64:0/3977884652 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9c0003520 con 0x7fc9c0004800 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.685+0000 7fc9d6ffd640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9c0004800 0x7fc9c0004c80 secure :-1 s=READY pgs=1377 cs=0 l=1 rev1=1 crypto rx=0x7fc9cc009c90 tx=0x7fc9c0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7111abfcba1fa8b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9cc039070 con 0x7fc9c0004800 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc9cc02c8f0 con 0x7fc9c0004800 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9d67fc640 1 -- 172.21.15.64:0/3977884652 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9cc009860 con 0x7fc9c0004800 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3977884652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9c0004800 msgr2=0x7fc9c0004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9ddb25640 1 --2- 172.21.15.64:0/3977884652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9c0004800 0x7fc9c0004c80 secure :-1 s=READY pgs=1377 cs=0 l=1 rev1=1 crypto rx=0x7fc9cc009c90 tx=0x7fc9c0005830 comp rx=0 tx=0).stop 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9d7fff640 1 -- 172.21.15.64:0/3977884652 reap_dead start 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3977884652 shutdown_connections 2024-03-04T07:32:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3977884652 >> 172.21.15.64:0/3977884652 conn(0x7fc9d8071960 msgr2=0x7fc9d8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:28.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3977884652 shutdown_connections 2024-03-04T07:32:28.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.686+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3977884652 wait complete. 2024-03-04T07:32:28.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 Processor -- start 2024-03-04T07:32:28.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 -- start start 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9c0004800 0x7fc9d81e8390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9d8159b10 0x7fc9d814f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9d80d3340 0x7fc9d80d37c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9d80cec70 con 0x7fc9d80d3340 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9d80ce750 con 0x7fc9d8159b10 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.688+0000 7fc9ddb25640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc9d80cedf0 con 0x7fc9c0004800 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9d80d3340 0x7fc9d80d37c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9d80d3340 0x7fc9d80d37c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54242/0 (socket says 172.21.15.64:54242) 2024-03-04T07:32:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 -- 172.21.15.64:0/3054747758 learned_addr learned my addr 172.21.15.64:0/3054747758 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d77fe640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9c0004800 0x7fc9d81e8390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d6ffd640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9d8159b10 0x7fc9d814f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 -- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9c0004800 msgr2=0x7fc9d81e8390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9c0004800 0x7fc9d81e8390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 -- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9d8159b10 msgr2=0x7fc9d814f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9d8159b10 0x7fc9d814f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 -- 172.21.15.64:0/3054747758 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9d814f610 con 0x7fc9d80d3340 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9d7fff640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9d80d3340 0x7fc9d80d37c0 secure :-1 s=READY pgs=1378 cs=0 l=1 rev1=1 crypto rx=0x7fc9d80d4d20 tx=0x7fc9d000e530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9bffff640 1 -- 172.21.15.64:0/3054747758 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9d000ee20 con 0x7fc9d80d3340 2024-03-04T07:32:28.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9bffff640 1 -- 172.21.15.64:0/3054747758 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc9d0014430 con 0x7fc9d80d3340 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc9d815a290 con 0x7fc9d80d3340 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9bffff640 1 -- 172.21.15.64:0/3054747758 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9d0014820 con 0x7fc9d80d3340 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.689+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc9d815a850 con 0x7fc9d80d3340 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.690+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc9d8067ff0 con 0x7fc9d80d3340 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.691+0000 7fc9bffff640 1 -- 172.21.15.64:0/3054747758 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc9d00149c0 con 0x7fc9d80d3340 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.692+0000 7fc9bffff640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc9c406b4d0 0x7fc9c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.692+0000 7fc9bffff640 1 -- 172.21.15.64:0/3054747758 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7fc9d008a050 con 0x7fc9d80d3340 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.692+0000 7fc9d77fe640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc9c406b4d0 0x7fc9c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.692+0000 7fc9d77fe640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc9c406b4d0 0x7fc9c406d990 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7fc9c8009dc0 tx=0x7fc9c8009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:28.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.696+0000 7fc9bffff640 1 -- 172.21.15.64:0/3054747758 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc9d814f610 con 0x7fc9d80d3340 2024-03-04T07:32:28.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.915+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc9d80d2640 con 0x7fc9d80d3340 2024-03-04T07:32:28.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.915+0000 7fc9bffff640 1 -- 172.21.15.64:0/3054747758 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7fc9d0058860 con 0x7fc9d80d3340 2024-03-04T07:32:28.916 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:28.916 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:32:28.916 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:32:28.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc9c406b4d0 msgr2=0x7fc9c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc9c406b4d0 0x7fc9c406d990 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7fc9c8009dc0 tx=0x7fc9c8009190 comp rx=0 tx=0).stop 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9d80d3340 msgr2=0x7fc9d80d37c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9d80d3340 0x7fc9d80d37c0 secure :-1 s=READY pgs=1378 cs=0 l=1 rev1=1 crypto rx=0x7fc9d80d4d20 tx=0x7fc9d000e530 comp rx=0 tx=0).stop 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 shutdown_connections 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc9c406b4d0 0x7fc9c406d990 unknown :-1 s=CLOSED pgs=622 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc9d80d3340 0x7fc9d80d37c0 unknown :-1 s=CLOSED pgs=1378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc9d8159b10 0x7fc9d814f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 --2- 172.21.15.64:0/3054747758 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc9c0004800 0x7fc9d81e8390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.917+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 >> 172.21.15.64:0/3054747758 conn(0x7fc9d8071960 msgr2=0x7fc9d80b8680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.918+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 shutdown_connections 2024-03-04T07:32:28.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:28.918+0000 7fc9ddb25640 1 -- 172.21.15.64:0/3054747758 wait complete. 2024-03-04T07:32:28.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-04T07:32:29.049 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"num_strays":1}} 2024-03-04T07:32:29.049 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 1 } } 2024-03-04T07:32:29.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:29.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.151+0000 7f8dfae4e640 1 Processor -- start 2024-03-04T07:32:29.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.151+0000 7f8dfae4e640 1 -- start start 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8dfae4e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8df415df00 0x7f8df40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8dfae4e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8df40b95a0 0x7f8df40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8dfae4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df40b9a30 0x7f8df40b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8dfae4e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8df40cea30 con 0x7f8df40b95a0 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8dfae4e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8df40d2af0 con 0x7f8df415df00 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8dfae4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8df40cf3c0 con 0x7f8df40b9a30 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8df40b95a0 0x7f8df40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f8df40b95a0 0x7f8df40b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54258/0 (socket says 172.21.15.64:54258) 2024-03-04T07:32:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 -- 172.21.15.64:0/587593754 learned_addr learned my addr 172.21.15.64:0/587593754 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df8bea640 1 --2- 172.21.15.64:0/587593754 >> v2:172.21.15.66:3300/0 conn(0x7f8df415df00 0x7f8df40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df93eb640 1 --2- 172.21.15.64:0/587593754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df40b9a30 0x7f8df40b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 -- 172.21.15.64:0/587593754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df40b9a30 msgr2=0x7f8df40b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 --2- 172.21.15.64:0/587593754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df40b9a30 0x7f8df40b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 -- 172.21.15.64:0/587593754 >> v2:172.21.15.66:3300/0 conn(0x7f8df415df00 msgr2=0x7f8df40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 --2- 172.21.15.64:0/587593754 >> v2:172.21.15.66:3300/0 conn(0x7f8df415df00 0x7f8df40d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.152+0000 7f8df3fff640 1 -- 172.21.15.64:0/587593754 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8df40cf0e0 con 0x7f8df40b95a0 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.153+0000 7f8df3fff640 1 --2- 172.21.15.64:0/587593754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df40b5700 secure :-1 s=READY pgs=1379 cs=0 l=1 rev1=1 crypto rx=0x7f8df40d11b0 tx=0x7f8de002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=260be0c893763dc8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.153+0000 7f8df37fe640 1 -- 172.21.15.64:0/587593754 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8de0035020 con 0x7f8df40b95a0 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.153+0000 7f8df37fe640 1 -- 172.21.15.64:0/587593754 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8de0031870 con 0x7f8df40b95a0 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.153+0000 7f8df37fe640 1 -- 172.21.15.64:0/587593754 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8de0031c40 con 0x7f8df40b95a0 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 -- 172.21.15.64:0/587593754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 msgr2=0x7f8df40b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:29.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/587593754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df40b5700 secure :-1 s=READY pgs=1379 cs=0 l=1 rev1=1 crypto rx=0x7f8df40d11b0 tx=0x7f8de002b910 comp rx=0 tx=0).stop 2024-03-04T07:32:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 -- 172.21.15.64:0/587593754 shutdown_connections 2024-03-04T07:32:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/587593754 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df40b9a30 0x7f8df40b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/587593754 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df40b5700 unknown :-1 s=CLOSED pgs=1379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/587593754 >> v2:172.21.15.66:3300/0 conn(0x7f8df415df00 0x7f8df40d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 -- 172.21.15.64:0/587593754 >> 172.21.15.64:0/587593754 conn(0x7f8df4071960 msgr2=0x7f8df4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 -- 172.21.15.64:0/587593754 shutdown_connections 2024-03-04T07:32:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.154+0000 7f8dfae4e640 1 -- 172.21.15.64:0/587593754 wait complete. 2024-03-04T07:32:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.155+0000 7f8dfae4e640 1 Processor -- start 2024-03-04T07:32:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.155+0000 7f8dfae4e640 1 -- start start 2024-03-04T07:32:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8dfae4e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df41551a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8dfae4e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8df415df00 0x7f8df41618b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8dfae4e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df415e2e0 0x7f8df4161df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8dfae4e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8df40cec70 con 0x7f8df40b95a0 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8dfae4e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8df40ce750 con 0x7f8df415df00 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8dfae4e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8df40cedf0 con 0x7f8df415e2e0 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df41551a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df41551a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:54260/0 (socket says 172.21.15.64:54260) 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 -- 172.21.15.64:0/118871333 learned_addr learned my addr 172.21.15.64:0/118871333 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df3fff640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8df415df00 0x7f8df41618b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df93eb640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df415e2e0 0x7f8df4161df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 -- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df415e2e0 msgr2=0x7f8df4161df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df415e2e0 0x7f8df4161df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 -- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8df415df00 msgr2=0x7f8df41618b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8df415df00 0x7f8df41618b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df8bea640 1 -- 172.21.15.64:0/118871333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8df41556e0 con 0x7f8df40b95a0 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.156+0000 7f8df3fff640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8df415df00 0x7f8df41618b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.157+0000 7f8df8bea640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df41551a0 secure :-1 s=READY pgs=1380 cs=0 l=1 rev1=1 crypto rx=0x7f8de800d620 tx=0x7f8de800dae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.157+0000 7f8df17fa640 1 -- 172.21.15.64:0/118871333 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8de800b620 con 0x7f8df40b95a0 2024-03-04T07:32:29.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.157+0000 7f8df17fa640 1 -- 172.21.15.64:0/118871333 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8de8019070 con 0x7f8df40b95a0 2024-03-04T07:32:29.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.157+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8df415e810 con 0x7f8df40b95a0 2024-03-04T07:32:29.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.157+0000 7f8df17fa640 1 -- 172.21.15.64:0/118871333 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8de800ba00 con 0x7f8df40b95a0 2024-03-04T07:32:29.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.157+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8df415edd0 con 0x7f8df40b95a0 2024-03-04T07:32:29.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.158+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8df4067ff0 con 0x7f8df40b95a0 2024-03-04T07:32:29.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.159+0000 7f8df17fa640 1 -- 172.21.15.64:0/118871333 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8de800bba0 con 0x7f8df40b95a0 2024-03-04T07:32:29.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.159+0000 7f8df17fa640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dd806b4d0 0x7f8dd806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:29.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.159+0000 7f8df17fa640 1 -- 172.21.15.64:0/118871333 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 5881+0+0 (secure 0 0 0) 0x7f8de808a5a0 con 0x7f8df40b95a0 2024-03-04T07:32:29.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.159+0000 7f8df3fff640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dd806b4d0 0x7f8dd806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:29.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.160+0000 7f8df3fff640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dd806b4d0 0x7f8dd806d990 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7f8de0036000 tx=0x7f8de003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:29.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.164+0000 7f8df17fa640 1 -- 172.21.15.64:0/118871333 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8df41556e0 con 0x7f8df40b95a0 2024-03-04T07:32:29.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.382+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8df40cfb40 con 0x7f8df40b95a0 2024-03-04T07:32:29.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.383+0000 7f8df17fa640 1 -- 172.21.15.64:0/118871333 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f8de8059400 con 0x7f8df40b95a0 2024-03-04T07:32:29.383 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:29.383 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:32:29.383 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dd806b4d0 msgr2=0x7f8dd806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dd806b4d0 0x7f8dd806d990 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7f8de0036000 tx=0x7f8de003c040 comp rx=0 tx=0).stop 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 msgr2=0x7f8df41551a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df41551a0 secure :-1 s=READY pgs=1380 cs=0 l=1 rev1=1 crypto rx=0x7f8de800d620 tx=0x7f8de800dae0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 shutdown_connections 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8dd806b4d0 0x7f8dd806d990 unknown :-1 s=CLOSED pgs=623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8df415e2e0 0x7f8df4161df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8df415df00 0x7f8df41618b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 --2- 172.21.15.64:0/118871333 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8df40b95a0 0x7f8df41551a0 unknown :-1 s=CLOSED pgs=1380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 >> 172.21.15.64:0/118871333 conn(0x7f8df4071960 msgr2=0x7f8df40b6f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 shutdown_connections 2024-03-04T07:32:29.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:29.385+0000 7f8dfae4e640 1 -- 172.21.15.64:0/118871333 wait complete. 2024-03-04T07:32:29.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_created 2024-03-04T07:32:29.513 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_created":1}} 2024-03-04T07:32:29.514 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 1 } } 2024-03-04T07:32:47.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:32:47.977 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:32:59.515 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data ls 2024-03-04T07:32:59.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.556+0000 7f00bb740f40 1 Processor -- start 2024-03-04T07:32:59.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.556+0000 7f00bb740f40 1 -- start start 2024-03-04T07:32:59.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.557+0000 7f00bb740f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f00a8003ab0 0x5556e45cf470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.557+0000 7f00bb740f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5556e45cfa00 0x5556e45fbe40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.557+0000 7f00bb740f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e46a29c0 0x5556e46a2da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.557+0000 7f00bb740f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5556e4574b70 con 0x5556e46a29c0 2024-03-04T07:32:59.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.557+0000 7f00bb740f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5556e4578ad0 con 0x5556e45cfa00 2024-03-04T07:32:59.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.557+0000 7f00bb740f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5556e45fd350 con 0x7f00a8003ab0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.557+0000 7f00b9a42640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e46a29c0 0x5556e46a2da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e46a29c0 0x5556e46a2da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41708/0 (socket says 172.21.15.64:41708) 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 -- 172.21.15.64:0/3350588807 learned_addr learned my addr 172.21.15.64:0/3350588807 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9241640 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f00a8003ab0 0x5556e45cf470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b3fff640 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5556e45cfa00 0x5556e45fbe40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 -- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f00a8003ab0 msgr2=0x5556e45cf470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f00a8003ab0 0x5556e45cf470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 -- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5556e45cfa00 msgr2=0x5556e45fbe40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5556e45cfa00 0x5556e45fbe40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 -- 172.21.15.64:0/3350588807 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5556e45ce900 con 0x5556e46a29c0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b9a42640 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e46a29c0 0x5556e46a2da0 secure :-1 s=READY pgs=1383 cs=0 l=1 rev1=1 crypto rx=0x7f00b4054040 tx=0x7f00b4003330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2b7ddf1dfb8f8895 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b37fe640 1 -- 172.21.15.64:0/3350588807 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00b4063070 con 0x5556e46a29c0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b37fe640 1 -- 172.21.15.64:0/3350588807 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f00b4052620 con 0x5556e46a29c0 2024-03-04T07:32:59.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00b37fe640 1 -- 172.21.15.64:0/3350588807 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00b40529f0 con 0x5556e46a29c0 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00bb740f40 1 -- 172.21.15.64:0/3350588807 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e46a29c0 msgr2=0x5556e46a2da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.558+0000 7f00bb740f40 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e46a29c0 0x5556e46a2da0 secure :-1 s=READY pgs=1383 cs=0 l=1 rev1=1 crypto rx=0x7f00b4054040 tx=0x7f00b4003330 comp rx=0 tx=0).stop 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 -- 172.21.15.64:0/3350588807 shutdown_connections 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e46a29c0 0x5556e46a2da0 unknown :-1 s=CLOSED pgs=1383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x5556e45cfa00 0x5556e45fbe40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 --2- 172.21.15.64:0/3350588807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f00a8003ab0 0x5556e45cf470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 -- 172.21.15.64:0/3350588807 >> 172.21.15.64:0/3350588807 conn(0x5556e45f7a50 msgr2=0x5556e45cdfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 -- 172.21.15.64:0/3350588807 shutdown_connections 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 -- 172.21.15.64:0/3350588807 wait complete. 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 Processor -- start 2024-03-04T07:32:59.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.559+0000 7f00bb740f40 1 -- start start 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00bb740f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e45cfa00 0x5556e47345d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00bb740f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5556e46a29c0 0x5556e46991f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00bb740f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f00a8003ab0 0x5556e4616a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00bb740f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x5556e46a66e0 con 0x5556e45cfa00 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00bb740f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x5556e467e140 con 0x7f00a8003ab0 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00bb740f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x5556e469d030 con 0x5556e46a29c0 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e45cfa00 0x5556e47345d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e45cfa00 0x5556e47345d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41712/0 (socket says 172.21.15.64:41712) 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 learned_addr learned my addr 172.21.15.64:0/1599120396 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:59.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5556e46a29c0 0x5556e46991f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f00a8003ab0 0x5556e4616a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5556e46a29c0 msgr2=0x5556e46991f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x5556e46a29c0 0x5556e46991f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f00a8003ab0 msgr2=0x5556e4616a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f00a8003ab0 0x5556e4616a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5556e4733e40 con 0x5556e45cfa00 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f00a8003ab0 0x5556e4616a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.560+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e45cfa00 0x5556e47345d0 secure :-1 s=READY pgs=1384 cs=0 l=1 rev1=1 crypto rx=0x7f00ac002e80 tx=0x7f00ac00f710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.561+0000 7f00b27fc640 1 -- 172.21.15.64:0/1599120396 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00ac012640 con 0x5556e45cfa00 2024-03-04T07:32:59.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.561+0000 7f00b27fc640 1 -- 172.21.15.64:0/1599120396 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f00ac016030 con 0x5556e45cfa00 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.561+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x5556e469d280 con 0x5556e45cfa00 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.561+0000 7f00b27fc640 1 -- 172.21.15.64:0/1599120396 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00ac0129e0 con 0x5556e45cfa00 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.561+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5556e4684420 con 0x5556e45cfa00 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.562+0000 7f00b27fc640 1 -- 172.21.15.64:0/1599120396 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f00ac00e050 con 0x5556e45cfa00 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.562+0000 7f00b27fc640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f009c069ee0 0x7f009c06c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.563+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f009c069ee0 0x7f009c06c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.563+0000 7f00b27fc640 1 -- 172.21.15.64:0/1599120396 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f00ac0892f0 con 0x5556e45cfa00 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.563+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x5556e46803f0 0x5556e4680830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.563+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:1 17.0 17:00000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.563+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f009c069ee0 0x7f009c06c3a0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x5556e473d140 tx=0x7f00a4001d90 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.563+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x5556e46803f0 0x5556e4680830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.564+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x5556e46803f0 0x5556e4680830 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.564+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 1 ==== osd_op_reply(1 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.564+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x5556e45f8e70 0x7f00b405c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.564+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:2 17.20 17:04000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e45f8e70 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.564+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x5556e45f8e70 0x7f00b405c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.564+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x5556e45f8e70 0x7f00b405c430 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.565+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.1 v2:172.21.15.64:6801/1190712954 1 ==== osd_op_reply(2 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac022070 con 0x5556e45f8e70 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.565+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f00ac057be0 0x7f00ac056840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.565+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:3 17.10 17:08000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.565+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f00ac057be0 0x7f00ac056840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.565+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f00ac057be0 0x7f00ac056840 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.566+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 1 ==== osd_op_reply(3 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a400f070 con 0x7f00ac057be0 2024-03-04T07:32:59.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.566+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f00a401b040 0x7f00a401d950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.566+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:4 17.30 17:0c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.566+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f00a401b040 0x7f00a401d950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.566+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f00a401b040 0x7f00a401d950 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.566+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== osd_op_reply(4 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.566+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:5 17.8 17:10000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.567+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 2 ==== osd_op_reply(5 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.567+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:6 17.28 17:14000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.567+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 2 ==== osd_op_reply(6 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a400f070 con 0x7f00ac057be0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.567+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:7 17.18 17:18000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.567+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 2 ==== osd_op_reply(7 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.567+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:8 17.38 17:1c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 3 ==== osd_op_reply(8 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:9 17.4 17:20000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 3 ==== osd_op_reply(9 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:10 17.24 17:24000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 3 ==== osd_op_reply(10 [pgnls start_epoch 111 out=49b] v105'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a400f070 con 0x7f00ac057be0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:11 17.14 17:28000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 4 ==== osd_op_reply(11 [pgnls start_epoch 111 out=49b] v105'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a400f070 con 0x7f00ac057be0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.568+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:12 17.34 17:2c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.569+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 4 ==== osd_op_reply(12 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.569+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:13 17.c 17:30000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.569+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 4 ==== osd_op_reply(13 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.569+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x5556e4685f90 0x7f00b406c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.569+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:14 17.2c 17:34000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.569+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x5556e4685f90 0x7f00b406c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.570+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x5556e4685f90 0x7f00b406c430 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.570+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 1 ==== osd_op_reply(14 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.570+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:15 17.1c 17:38000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e45f8e70 2024-03-04T07:32:59.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.1 v2:172.21.15.64:6801/1190712954 2 ==== osd_op_reply(15 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e45f8e70 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:16 17.3c 17:3c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 5 ==== osd_op_reply(16 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:17 17.2 17:40000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 2 ==== osd_op_reply(17 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f00ac05fc00 0x7f00ac05c9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:18 17.22 17:44000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac05fc00 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.571+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f00ac05fc00 0x7f00ac05c9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.572+0000 7f00b3fff640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f00ac05fc00 0x7f00ac05c9f0 crc :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.572+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.5 v2:172.21.15.66:6800/1583207443 1 ==== osd_op_reply(18 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac05fc00 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.572+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:19 17.12 17:48000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.572+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 5 ==== osd_op_reply(19 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac057be0 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.572+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:20 17.32 17:4c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.573+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 5 ==== osd_op_reply(20 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.573+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:21 17.a 17:50000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.573+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 6 ==== osd_op_reply(21 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.573+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:22 17.2a 17:54000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.573+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 6 ==== osd_op_reply(22 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac057be0 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.574+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:23 17.1a 17:58000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.574+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 6 ==== osd_op_reply(23 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.574+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:24 17.3a 17:5c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e45f8e70 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.574+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.1 v2:172.21.15.64:6801/1190712954 3 ==== osd_op_reply(24 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e45f8e70 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.574+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f00ac012b80 0x7f00ac06d950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.574+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:25 17.6 17:60000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac012b80 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.574+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f00ac012b80 0x7f00ac06d950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.575+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f00ac012b80 0x7f00ac06d950 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.575+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.6 v2:172.21.15.66:6816/177421408 1 ==== osd_op_reply(25 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00ac012b80 2024-03-04T07:32:59.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.575+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:26 17.26 17:64000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 7 ==== osd_op_reply(26 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:27 17.16 17:68000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac05fc00 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.5 v2:172.21.15.66:6800/1583207443 2 ==== osd_op_reply(27 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac05fc00 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:28 17.36 17:6c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 8 ==== osd_op_reply(28 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b9a42640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x5556e46875e0 0x7f00b4088970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- osd_op(unknown.0.0:29 17.e 17:70000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46875e0 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.576+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x5556e46875e0 0x7f00b4088970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.577+0000 7f00b9241640 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x5556e46875e0 0x7f00b4088970 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.577+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.2 v2:172.21.15.64:6824/2610379167 1 ==== osd_op_reply(29 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e46875e0 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.577+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:30 17.2e 17:74000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac012b80 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.577+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.6 v2:172.21.15.66:6816/177421408 2 ==== osd_op_reply(30 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00ac012b80 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.577+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:31 17.1e 17:78000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e45f8e70 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.578+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.1 v2:172.21.15.64:6801/1190712954 4 ==== osd_op_reply(31 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e45f8e70 2024-03-04T07:32:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.578+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:32 17.3e 17:7c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.578+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 3 ==== osd_op_reply(32 [pgnls start_epoch 111 out=49b] v105'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.578+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:33 17.1 17:80000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.578+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 9 ==== osd_op_reply(33 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.578+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:34 17.21 17:84000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.578+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 10 ==== osd_op_reply(34 [pgnls start_epoch 111 out=49b] v105'12 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.579+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:35 17.11 17:88000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.579+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 7 ==== osd_op_reply(35 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac057be0 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.579+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:36 17.31 17:8c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e45f8e70 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.579+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.1 v2:172.21.15.64:6801/1190712954 5 ==== osd_op_reply(36 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e45f8e70 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.579+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:37 17.9 17:90000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.579+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 8 ==== osd_op_reply(37 [pgnls start_epoch 111 out=49b] v105'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac057be0 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.579+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:38 17.29 17:94000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.580+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 4 ==== osd_op_reply(38 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.580+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:39 17.19 17:98000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac05fc00 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.580+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.5 v2:172.21.15.66:6800/1583207443 3 ==== osd_op_reply(39 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac05fc00 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.580+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:40 17.39 17:9c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac05fc00 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.580+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.5 v2:172.21.15.66:6800/1583207443 4 ==== osd_op_reply(40 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac05fc00 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.580+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:41 17.5 17:a0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac012b80 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.581+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.6 v2:172.21.15.66:6816/177421408 3 ==== osd_op_reply(41 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00ac012b80 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.581+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:42 17.25 17:a4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e45f8e70 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.581+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.1 v2:172.21.15.64:6801/1190712954 6 ==== osd_op_reply(42 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e45f8e70 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.581+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:43 17.15 17:a8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac05fc00 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.581+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.5 v2:172.21.15.66:6800/1583207443 5 ==== osd_op_reply(43 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac05fc00 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.581+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:44 17.35 17:ac000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.582+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 5 ==== osd_op_reply(44 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.582+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:45 17.d 17:b0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.582+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 7 ==== osd_op_reply(45 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.582+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:46 17.2d 17:b4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.582+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 8 ==== osd_op_reply(46 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.583+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:47 17.1d 17:b8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.583+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 9 ==== osd_op_reply(47 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.583+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:48 17.3d 17:bc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.583+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 10 ==== osd_op_reply(48 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.583+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:49 17.3 17:c0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.583+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 6 ==== osd_op_reply(49 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.583+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:50 17.23 17:c4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.584+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 7 ==== osd_op_reply(50 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.584+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:51 17.13 17:c8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac057be0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.584+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 <== osd.3 v2:172.21.15.64:6816/1402142346 9 ==== osd_op_reply(51 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00a402b040 con 0x7f00ac057be0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.584+0000 7f00b3fff640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:52 17.33 17:cc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.584+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.7 v2:172.21.15.66:6802/3006518207 11 ==== osd_op_reply(52 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x5556e46803f0 2024-03-04T07:32:59.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.584+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:53 17.b 17:d0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.585+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 11 ==== osd_op_reply(53 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.585+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:54 17.2b 17:d4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.585+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 8 ==== osd_op_reply(54 [pgnls start_epoch 111 out=49b] v105'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.585+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:55 17.1b 17:d8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.585+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 9 ==== osd_op_reply(55 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.585+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- osd_op(unknown.0.0:56 17.3b 17:dc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e46875e0 2024-03-04T07:32:59.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.586+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.2 v2:172.21.15.64:6824/2610379167 2 ==== osd_op_reply(56 [pgnls start_epoch 111 out=49b] v105'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e46875e0 2024-03-04T07:32:59.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.586+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:57 17.7 17:e0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.586+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 12 ==== osd_op_reply(57 [pgnls start_epoch 111 out=81b] v105'8 uv3 ondisk = 1) v8 ==== 144+0+81 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.586+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:58 17.27 17:e4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.586+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 10 ==== osd_op_reply(58 [pgnls start_epoch 111 out=49b] v105'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.586+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:59 17.17 17:e8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac012b80 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.587+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.6 v2:172.21.15.66:6816/177421408 4 ==== osd_op_reply(59 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00ac012b80 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.587+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:60 17.37 17:ec000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.587+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 13 ==== osd_op_reply(60 [pgnls start_epoch 111 out=49b] v105'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.587+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:61 17.f 17:f0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e45f8e70 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.587+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.1 v2:172.21.15.64:6801/1190712954 7 ==== osd_op_reply(61 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00ac087040 con 0x5556e45f8e70 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.587+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:62 17.2f 17:f4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x5556e4685f90 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 <== osd.4 v2:172.21.15.66:6824/1927515367 11 ==== osd_op_reply(62 [pgnls start_epoch 111 out=81b] v105'9 uv1 ondisk = 1) v8 ==== 144+0+81 (crc 0 0 0) 0x7f00ac087040 con 0x5556e4685f90 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00b9241640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:63 17.1f 17:f8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00ac012b80 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.6 v2:172.21.15.66:6816/177421408 5 ==== osd_op_reply(63 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00ac012b80 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:64 17.3f 17:fc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x5556e46819a0 con 0x7f00a401b040 2024-03-04T07:32:59.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 <== osd.0 v2:172.21.15.64:6800/2111658274 14 ==== osd_op_reply(64 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7f00b4067040 con 0x7f00a401b040 2024-03-04T07:32:59.593 INFO:teuthology.orchestra.run.smithi064.stdout:10000000001.00000000 2024-03-04T07:32:59.593 INFO:teuthology.orchestra.run.smithi064.stdout:10000000001.00000001 2024-03-04T07:32:59.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f00a401b040 msgr2=0x7f00a401d950 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f00a401b040 0x7f00a401d950 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x5556e45f8e70 msgr2=0x7f00b405c430 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.588+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x5556e45f8e70 0x7f00b405c430 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x5556e46875e0 msgr2=0x7f00b4088970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x5556e46875e0 0x7f00b4088970 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f00ac057be0 msgr2=0x7f00ac056840 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f00ac057be0 0x7f00ac056840 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 reap_dead start 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x5556e4685f90 msgr2=0x7f00b406c430 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x5556e4685f90 0x7f00b406c430 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 reap_dead start 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f00ac05fc00 msgr2=0x7f00ac05c9f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7f00ac05fc00 0x7f00ac05c9f0 crc :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f00ac012b80 msgr2=0x7f00ac06d950 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f00ac012b80 0x7f00ac06d950 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x5556e46803f0 msgr2=0x5556e4680830 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x5556e46803f0 0x5556e4680830 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f009c069ee0 msgr2=0x7f009c06c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f009c069ee0 0x7f009c06c3a0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x5556e473d140 tx=0x7f00a4001d90 comp rx=0 tx=0).stop 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e45cfa00 msgr2=0x5556e47345d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 --2- 172.21.15.64:0/1599120396 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x5556e45cfa00 0x5556e47345d0 secure :-1 s=READY pgs=1384 cs=0 l=1 rev1=1 crypto rx=0x7f00ac002e80 tx=0x7f00ac00f710 comp rx=0 tx=0).stop 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00b9a42640 1 -- 172.21.15.64:0/1599120396 reap_dead start 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 shutdown_connections 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 >> 172.21.15.64:0/1599120396 conn(0x5556e45f7a50 msgr2=0x5556e45cdfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 shutdown_connections 2024-03-04T07:32:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.589+0000 7f00bb740f40 1 -- 172.21.15.64:0/1599120396 wait complete. 2024-03-04T07:32:59.596 DEBUG:tasks.cephfs.filesystem:All objects for ino 1099511627777 size 8388608 found 2024-03-04T07:32:59.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:32:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.740+0000 7f816d1ac640 1 Processor -- start 2024-03-04T07:32:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.740+0000 7f816d1ac640 1 -- start start 2024-03-04T07:32:59.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816d1ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f816815e030 0x7f816806ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816d1ac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f816806ef70 0x7f81680b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816d1ac640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81680b2290 0x7f81680d0040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816d1ac640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8168162700 con 0x7f81680b2290 2024-03-04T07:32:59.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816d1ac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f816815e410 con 0x7f816806ef70 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816d1ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8168161af0 con 0x7f816815e030 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81680b2290 0x7f81680d0040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81680b2290 0x7f81680d0040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41714/0 (socket says 172.21.15.64:41714) 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 -- 172.21.15.64:0/1287493035 learned_addr learned my addr 172.21.15.64:0/1287493035 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f8166d9d640 1 --2- 172.21.15.64:0/1287493035 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f816815e030 0x7f816806ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f815ffff640 1 --2- 172.21.15.64:0/1287493035 >> v2:172.21.15.66:3300/0 conn(0x7f816806ef70 0x7f81680b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 -- 172.21.15.64:0/1287493035 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f816815e030 msgr2=0x7f816806ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 --2- 172.21.15.64:0/1287493035 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f816815e030 0x7f816806ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 -- 172.21.15.64:0/1287493035 >> v2:172.21.15.66:3300/0 conn(0x7f816806ef70 msgr2=0x7f81680b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 --2- 172.21.15.64:0/1287493035 >> v2:172.21.15.66:3300/0 conn(0x7f816806ef70 0x7f81680b1d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.741+0000 7f816759e640 1 -- 172.21.15.64:0/1287493035 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8168161dd0 con 0x7f81680b2290 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816759e640 1 --2- 172.21.15.64:0/1287493035 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680b2290 0x7f81680d0040 secure :-1 s=READY pgs=1385 cs=0 l=1 rev1=1 crypto rx=0x7f81680d1210 tx=0x7f816002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=38bdb705fb7fb73c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816659c640 1 -- 172.21.15.64:0/1287493035 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f816002ca70 con 0x7f81680b2290 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816659c640 1 -- 172.21.15.64:0/1287493035 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f816002cc10 con 0x7f81680b2290 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816659c640 1 -- 172.21.15.64:0/1287493035 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81600343f0 con 0x7f81680b2290 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816d1ac640 1 -- 172.21.15.64:0/1287493035 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680b2290 msgr2=0x7f81680d0040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816d1ac640 1 --2- 172.21.15.64:0/1287493035 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680b2290 0x7f81680d0040 secure :-1 s=READY pgs=1385 cs=0 l=1 rev1=1 crypto rx=0x7f81680d1210 tx=0x7f816002bed0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816d1ac640 1 -- 172.21.15.64:0/1287493035 shutdown_connections 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816d1ac640 1 --2- 172.21.15.64:0/1287493035 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81680b2290 0x7f81680d0040 unknown :-1 s=CLOSED pgs=1385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816d1ac640 1 --2- 172.21.15.64:0/1287493035 >> v2:172.21.15.66:3300/0 conn(0x7f816806ef70 0x7f81680b1d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816d1ac640 1 --2- 172.21.15.64:0/1287493035 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f816815e030 0x7f816806ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.742+0000 7f816d1ac640 1 -- 172.21.15.64:0/1287493035 >> 172.21.15.64:0/1287493035 conn(0x7f81680b55e0 msgr2=0x7f81680b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.743+0000 7f816d1ac640 1 -- 172.21.15.64:0/1287493035 shutdown_connections 2024-03-04T07:32:59.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.743+0000 7f816d1ac640 1 -- 172.21.15.64:0/1287493035 wait complete. 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.744+0000 7f816d1ac640 1 Processor -- start 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.744+0000 7f816d1ac640 1 -- start start 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.744+0000 7f816d1ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816806ef70 0x7f81680b13e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f816d1ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81680b2290 0x7f8168144cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f816d1ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f816815e030 0x7f8168146370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f816d1ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81680b31e0 con 0x7f816806ef70 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f816d1ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8168162390 con 0x7f816815e030 2024-03-04T07:32:59.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f816d1ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81680b3360 con 0x7f81680b2290 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816806ef70 0x7f81680b13e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816806ef70 0x7f81680b13e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41728/0 (socket says 172.21.15.64:41728) 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 -- 172.21.15.64:0/2198648631 learned_addr learned my addr 172.21.15.64:0/2198648631 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 -- 172.21.15.64:0/2198648631 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81680b2290 msgr2=0x7f8168144cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81680b2290 0x7f8168144cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 -- 172.21.15.64:0/2198648631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f816815e030 msgr2=0x7f8168146370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f816815e030 0x7f8168146370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 -- 172.21.15.64:0/2198648631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8168145620 con 0x7f816806ef70 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f8166d9d640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816806ef70 0x7f81680b13e0 secure :-1 s=READY pgs=1386 cs=0 l=1 rev1=1 crypto rx=0x7f815800d830 tx=0x7f815800dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f815f7fe640 1 -- 172.21.15.64:0/2198648631 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f815800b6c0 con 0x7f816806ef70 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f815f7fe640 1 -- 172.21.15.64:0/2198648631 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8158019070 con 0x7f816806ef70 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f815f7fe640 1 -- 172.21.15.64:0/2198648631 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f815800ba80 con 0x7f816806ef70 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8168145900 con 0x7f816806ef70 2024-03-04T07:32:59.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.745+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8168145ec0 con 0x7f816806ef70 2024-03-04T07:32:59.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.746+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81681460c0 con 0x7f816806ef70 2024-03-04T07:32:59.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.747+0000 7f815f7fe640 1 -- 172.21.15.64:0/2198648631 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f815801e460 con 0x7f816806ef70 2024-03-04T07:32:59.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.747+0000 7f815f7fe640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814806b500 0x7f814806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:32:59.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.747+0000 7f815ffff640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814806b500 0x7f814806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:32:59.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.747+0000 7f815f7fe640 1 -- 172.21.15.64:0/2198648631 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f815808a960 con 0x7f816806ef70 2024-03-04T07:32:59.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.748+0000 7f815ffff640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814806b500 0x7f814806d9c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f8150006fd0 tx=0x7f8150008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:32:59.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.752+0000 7f815f7fe640 1 -- 172.21.15.64:0/2198648631 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f81681460c0 con 0x7f816806ef70 2024-03-04T07:32:59.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.970+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8168067ff0 con 0x7f816806ef70 2024-03-04T07:32:59.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.971+0000 7f815f7fe640 1 -- 172.21.15.64:0/2198648631 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f8158059670 con 0x7f816806ef70 2024-03-04T07:32:59.971 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:32:59.971 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:32:59.971 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814806b500 msgr2=0x7f814806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814806b500 0x7f814806d9c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f8150006fd0 tx=0x7f8150008040 comp rx=0 tx=0).stop 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816806ef70 msgr2=0x7f81680b13e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816806ef70 0x7f81680b13e0 secure :-1 s=READY pgs=1386 cs=0 l=1 rev1=1 crypto rx=0x7f815800d830 tx=0x7f815800dcf0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 shutdown_connections 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f814806b500 0x7f814806d9c0 unknown :-1 s=CLOSED pgs=625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f816815e030 0x7f8168146370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81680b2290 0x7f8168144cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 --2- 172.21.15.64:0/2198648631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f816806ef70 0x7f81680b13e0 unknown :-1 s=CLOSED pgs=1386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:32:59.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 >> 172.21.15.64:0/2198648631 conn(0x7f81680b55e0 msgr2=0x7f816815ae20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:32:59.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 shutdown_connections 2024-03-04T07:32:59.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:32:59.973+0000 7f816d1ac640 1 -- 172.21.15.64:0/2198648631 wait complete. 2024-03-04T07:32:59.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:33:00.102 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-04T07:33:00.103 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-04T07:33:00.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:00.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.206+0000 7f4653839640 1 Processor -- start 2024-03-04T07:33:00.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.207+0000 7f4653839640 1 -- start start 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.207+0000 7f4653839640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c15df00 0x7f464c0d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.207+0000 7f4653839640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f464c0cea30 con 0x7f464c0b9a30 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.207+0000 7f4653839640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f464c0d2af0 con 0x7f464c0b95a0 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.207+0000 7f4653839640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f464c0cf3c0 con 0x7f464c15df00 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f4651dd6640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f464c0b9a30 0x7f464c0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42358/0 (socket says 172.21.15.64:42358) 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f4651dd6640 1 -- 172.21.15.64:0/4115647807 learned_addr learned my addr 172.21.15.64:0/4115647807 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f46515d5640 1 --2- 172.21.15.64:0/4115647807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c15df00 0x7f464c0d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f4643fff640 1 -- 172.21.15.64:0/4115647807 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2098723723 0 0) 0x7f464c0cea30 con 0x7f464c0b9a30 2024-03-04T07:33:00.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f4643fff640 1 -- 172.21.15.64:0/4115647807 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4638003520 con 0x7f464c0b9a30 2024-03-04T07:33:00.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f4643fff640 1 -- 172.21.15.64:0/4115647807 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3921057408 0 0) 0x7f464c0d2af0 con 0x7f464c0b95a0 2024-03-04T07:33:00.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f46515d5640 1 -- 172.21.15.64:0/4115647807 >> v1:172.21.15.66:6789/0 conn(0x7f464c0b95a0 legacy=0x7f464c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f46515d5640 1 -- 172.21.15.64:0/4115647807 >> v1:172.21.15.64:6789/0 conn(0x7f464c0b9a30 legacy=0x7f464c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.208+0000 7f46515d5640 1 -- 172.21.15.64:0/4115647807 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f464c0cf0e0 con 0x7f464c15df00 2024-03-04T07:33:00.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.209+0000 7f46515d5640 1 --2- 172.21.15.64:0/4115647807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c15df00 0x7f464c0d25b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f46480097d0 tx=0x7f464802b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f06044ffbf26bab4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4643fff640 1 -- 172.21.15.64:0/4115647807 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4648035020 con 0x7f464c15df00 2024-03-04T07:33:00.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4643fff640 1 -- 172.21.15.64:0/4115647807 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4648031890 con 0x7f464c15df00 2024-03-04T07:33:00.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4643fff640 1 -- 172.21.15.64:0/4115647807 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4648031c60 con 0x7f464c15df00 2024-03-04T07:33:00.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4653839640 1 -- 172.21.15.64:0/4115647807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c15df00 msgr2=0x7f464c0d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4653839640 1 --2- 172.21.15.64:0/4115647807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c15df00 0x7f464c0d25b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f46480097d0 tx=0x7f464802b930 comp rx=0 tx=0).stop 2024-03-04T07:33:00.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4653839640 1 -- 172.21.15.64:0/4115647807 shutdown_connections 2024-03-04T07:33:00.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4653839640 1 --2- 172.21.15.64:0/4115647807 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c15df00 0x7f464c0d25b0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4653839640 1 -- 172.21.15.64:0/4115647807 >> 172.21.15.64:0/4115647807 conn(0x7f464c071960 msgr2=0x7f464c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:00.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4653839640 1 -- 172.21.15.64:0/4115647807 shutdown_connections 2024-03-04T07:33:00.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.210+0000 7f4653839640 1 -- 172.21.15.64:0/4115647807 wait complete. 2024-03-04T07:33:00.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.211+0000 7f4653839640 1 Processor -- start 2024-03-04T07:33:00.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.211+0000 7f4653839640 1 -- start start 2024-03-04T07:33:00.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4653839640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c0b95a0 0x7f464c14e1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4653839640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f464c15df00 0x7f464c0b5310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4653839640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f464c0cd850 0x7f464c0b5850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4653839640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f464c0cec70 con 0x7f464c0cd850 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4653839640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f464c0ce750 con 0x7f464c15df00 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4653839640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f464c0cedf0 con 0x7f464c0b95a0 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f464c0cd850 0x7f464c0b5850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f464c0cd850 0x7f464c0b5850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41736/0 (socket says 172.21.15.64:41736) 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 -- 172.21.15.64:0/4113369854 learned_addr learned my addr 172.21.15.64:0/4113369854 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f46515d5640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c0b95a0 0x7f464c14e1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4650dd4640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f464c15df00 0x7f464c0b5310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 -- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c0b95a0 msgr2=0x7f464c14e1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c0b95a0 0x7f464c14e1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 -- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f464c15df00 msgr2=0x7f464c0b5310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f464c15df00 0x7f464c0b5310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.212+0000 7f4651dd6640 1 -- 172.21.15.64:0/4113369854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f464c0cc7b0 con 0x7f464c0cd850 2024-03-04T07:33:00.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.213+0000 7f4651dd6640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f464c0cd850 0x7f464c0b5850 secure :-1 s=READY pgs=1388 cs=0 l=1 rev1=1 crypto rx=0x7f464400cc10 tx=0x7f464400a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.213+0000 7f4641ffb640 1 -- 172.21.15.64:0/4113369854 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4644017070 con 0x7f464c0cd850 2024-03-04T07:33:00.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.213+0000 7f4641ffb640 1 -- 172.21.15.64:0/4113369854 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f464400ad40 con 0x7f464c0cd850 2024-03-04T07:33:00.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.213+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f464c0cca90 con 0x7f464c0cd850 2024-03-04T07:33:00.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.213+0000 7f4641ffb640 1 -- 172.21.15.64:0/4113369854 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4644012600 con 0x7f464c0cd850 2024-03-04T07:33:00.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.213+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f464c0cd050 con 0x7f464c0cd850 2024-03-04T07:33:00.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.214+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f464c067ff0 con 0x7f464c0cd850 2024-03-04T07:33:00.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.214+0000 7f4641ffb640 1 -- 172.21.15.64:0/4113369854 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f464401a070 con 0x7f464c0cd850 2024-03-04T07:33:00.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.215+0000 7f4641ffb640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f463006b500 0x7f463006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.215+0000 7f4641ffb640 1 -- 172.21.15.64:0/4113369854 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f4644089f90 con 0x7f464c0cd850 2024-03-04T07:33:00.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.215+0000 7f46515d5640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f463006b500 0x7f463006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.215+0000 7f46515d5640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f463006b500 0x7f463006d9c0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f4648036000 tx=0x7f4648008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.219+0000 7f4641ffb640 1 -- 172.21.15.64:0/4113369854 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f46440586d0 con 0x7f464c0cd850 2024-03-04T07:33:00.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.439+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f464c0b2ac0 con 0x7f464c0cd850 2024-03-04T07:33:00.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.440+0000 7f4641ffb640 1 -- 172.21.15.64:0/4113369854 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f464405c770 con 0x7f464c0cd850 2024-03-04T07:33:00.440 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:00.440 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:00.440 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f463006b500 msgr2=0x7f463006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f463006b500 0x7f463006d9c0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f4648036000 tx=0x7f4648008030 comp rx=0 tx=0).stop 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f464c0cd850 msgr2=0x7f464c0b5850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f464c0cd850 0x7f464c0b5850 secure :-1 s=READY pgs=1388 cs=0 l=1 rev1=1 crypto rx=0x7f464400cc10 tx=0x7f464400a430 comp rx=0 tx=0).stop 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 shutdown_connections 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f463006b500 0x7f463006d9c0 unknown :-1 s=CLOSED pgs=626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f464c0cd850 0x7f464c0b5850 unknown :-1 s=CLOSED pgs=1388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f464c15df00 0x7f464c0b5310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 --2- 172.21.15.64:0/4113369854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f464c0b95a0 0x7f464c14e1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.441+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 >> 172.21.15.64:0/4113369854 conn(0x7f464c071960 msgr2=0x7f464c0b8770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.442+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 shutdown_connections 2024-03-04T07:33:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.442+0000 7f4653839640 1 -- 172.21.15.64:0/4113369854 wait complete. 2024-03-04T07:33:00.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-04T07:33:00.582 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:33:00.583 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:33:00.583 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-04T07:33:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.634+0000 7f90919fef40 1 Processor -- start 2024-03-04T07:33:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.634+0000 7f90919fef40 1 -- start start 2024-03-04T07:33:00.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.634+0000 7f90919fef40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d5e5d50 0x557d2d687000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f90919fef40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d68b260 0x557d2d5e7c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f90919fef40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d689b70 0x557d2d68c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f90919fef40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2d5feba0 con 0x557d2d5e5d50 2024-03-04T07:33:00.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f90919fef40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2d5fe900 con 0x557d2d68b260 2024-03-04T07:33:00.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f90919fef40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x557d2d5fee40 con 0x557d2d689b70 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d5e5d50 0x557d2d687000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d5e5d50 0x557d2d687000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41752/0 (socket says 172.21.15.64:41752) 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 -- 172.21.15.64:0/1896555662 learned_addr learned my addr 172.21.15.64:0/1896555662 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908ecfe640 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d68b260 0x557d2d5e7c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908fd00640 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d689b70 0x557d2d68c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 -- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d689b70 msgr2=0x557d2d68c3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d689b70 0x557d2d68c3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 -- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d68b260 msgr2=0x557d2d5e7c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d68b260 0x557d2d5e7c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 -- 172.21.15.64:0/1896555662 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d2d4e4c50 con 0x557d2d5e5d50 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908f4ff640 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d5e5d50 0x557d2d687000 secure :-1 s=READY pgs=1390 cs=0 l=1 rev1=1 crypto rx=0x7f908001e400 tx=0x7f9080051000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=92d9ff4f00ab9c03 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908dcfc640 1 -- 172.21.15.64:0/1896555662 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9080068070 con 0x557d2d5e5d50 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908dcfc640 1 -- 172.21.15.64:0/1896555662 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f908004ec40 con 0x557d2d5e5d50 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.635+0000 7f908dcfc640 1 -- 172.21.15.64:0/1896555662 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9080060430 con 0x557d2d5e5d50 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 -- 172.21.15.64:0/1896555662 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d5e5d50 msgr2=0x557d2d687000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d5e5d50 0x557d2d687000 secure :-1 s=READY pgs=1390 cs=0 l=1 rev1=1 crypto rx=0x7f908001e400 tx=0x7f9080051000 comp rx=0 tx=0).stop 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 -- 172.21.15.64:0/1896555662 shutdown_connections 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d689b70 0x557d2d68c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d68b260 0x557d2d5e7c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 --2- 172.21.15.64:0/1896555662 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d5e5d50 0x557d2d687000 unknown :-1 s=CLOSED pgs=1390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 -- 172.21.15.64:0/1896555662 >> 172.21.15.64:0/1896555662 conn(0x557d2d5e1a60 msgr2=0x557d2d5b9e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 -- 172.21.15.64:0/1896555662 shutdown_connections 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 -- 172.21.15.64:0/1896555662 wait complete. 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 Processor -- start 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.636+0000 7f90919fef40 1 -- start start 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f90919fef40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d5e5d50 0x557d2d6799c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f90919fef40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d689b70 0x557d2d6847d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f90919fef40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d68b260 0x557d2d684d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f90919fef40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2d5b8800 con 0x557d2d689b70 2024-03-04T07:33:00.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f90919fef40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2d68c910 con 0x557d2d5e5d50 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f90919fef40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x557d2d679f00 con 0x557d2d68b260 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f908ecfe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d689b70 0x557d2d6847d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f908ecfe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d689b70 0x557d2d6847d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41764/0 (socket says 172.21.15.64:41764) 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f908ecfe640 1 -- 172.21.15.64:0/393862338 learned_addr learned my addr 172.21.15.64:0/393862338 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f908f4ff640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d5e5d50 0x557d2d6799c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.637+0000 7f908fd00640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d68b260 0x557d2d684d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ecfe640 1 -- 172.21.15.64:0/393862338 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d68b260 msgr2=0x557d2d684d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ecfe640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x557d2d68b260 0x557d2d684d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ecfe640 1 -- 172.21.15.64:0/393862338 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d5e5d50 msgr2=0x557d2d6799c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ecfe640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x557d2d5e5d50 0x557d2d6799c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ecfe640 1 -- 172.21.15.64:0/393862338 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d2d685340 con 0x557d2d689b70 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ecfe640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d689b70 0x557d2d6847d0 secure :-1 s=READY pgs=1391 cs=0 l=1 rev1=1 crypto rx=0x7f9084007f80 tx=0x7f9084006810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ccfa640 1 -- 172.21.15.64:0/393862338 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9084018070 con 0x557d2d689b70 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ccfa640 1 -- 172.21.15.64:0/393862338 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9084012770 con 0x557d2d689b70 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x557d2d685620 con 0x557d2d689b70 2024-03-04T07:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f908ccfa640 1 -- 172.21.15.64:0/393862338 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9084012b20 con 0x557d2d689b70 2024-03-04T07:33:00.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.638+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557d2d5e7f00 con 0x557d2d689b70 2024-03-04T07:33:00.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.639+0000 7f908ccfa640 1 -- 172.21.15.64:0/393862338 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f908401b070 con 0x557d2d689b70 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f908ccfa640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f907c069eb0 0x7f907c06c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f908ccfa640 1 -- 172.21.15.64:0/393862338 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f9084088570 con 0x557d2d689b70 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f908f4ff640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f907c069eb0 0x7f907c06c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f90919fef40 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x557d2d5f7420 0x557d2d5f7890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e111) v9 -- 0x557d2d5f7dd0 con 0x557d2d5f7420 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f908fd00640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x557d2d5f7420 0x557d2d5f7890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f908f4ff640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f907c069eb0 0x7f907c06c370 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f9080065000 tx=0x7f9080053000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.640+0000 7f908fd00640 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x557d2d5f7420 0x557d2d5f7890 crc :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f908fd00640 1 -- 172.21.15.64:0/393862338 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (99) out=99b] v0'0 uv9 ondisk = 0) v8 ==== 164+0+99 (crc 0 0 0) 0x7f908800f860 con 0x557d2d5f7420 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x557d2d5f7420 msgr2=0x557d2d5f7890 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x557d2d5f7420 0x557d2d5f7890 crc :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f907c069eb0 msgr2=0x7f907c06c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f907c069eb0 0x7f907c06c370 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f9080065000 tx=0x7f9080053000 comp rx=0 tx=0).stop 2024-03-04T07:33:00.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d689b70 msgr2=0x557d2d6847d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 --2- 172.21.15.64:0/393862338 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x557d2d689b70 0x557d2d6847d0 secure :-1 s=READY pgs=1391 cs=0 l=1 rev1=1 crypto rx=0x7f9084007f80 tx=0x7f9084006810 comp rx=0 tx=0).stop 2024-03-04T07:33:00.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f908fd00640 1 -- 172.21.15.64:0/393862338 reap_dead start 2024-03-04T07:33:00.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 shutdown_connections 2024-03-04T07:33:00.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 >> 172.21.15.64:0/393862338 conn(0x557d2d5e1a60 msgr2=0x557d2d689f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:00.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 shutdown_connections 2024-03-04T07:33:00.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.641+0000 7f90919fef40 1 -- 172.21.15.64:0/393862338 wait complete. 2024-03-04T07:33:00.646 INFO:teuthology.orchestra.run.smithi064.stdout:] 10000000001 stray0 2024-03-04T07:33:00.646 DEBUG:teuthology.orchestra.run.smithi064:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-04T07:33:00.704 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:33:00.704 INFO:teuthology.orchestra.run.smithi064.stdout: "ino": 1099511627777, 2024-03-04T07:33:00.704 INFO:teuthology.orchestra.run.smithi064.stdout: "ancestors": [ 2024-03-04T07:33:00.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 1536, 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "10000000001", 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 11 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 256, 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "stray0", 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 5 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "pool": 17, 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: "old_pools": [] 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:33:00.705 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:00.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.873+0000 7f647f4f4640 1 Processor -- start 2024-03-04T07:33:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.873+0000 7f647f4f4640 1 -- start start 2024-03-04T07:33:00.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.874+0000 7f647f4f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780cea70 0x7f64780d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.874+0000 7f647f4f4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f64780b0ed0 0x7f647806f290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.874+0000 7f647f4f4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f64780cd350 con 0x7f647815bda0 2024-03-04T07:33:00.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.874+0000 7f647f4f4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f64780b1360 con 0x7f64780b0ed0 2024-03-04T07:33:00.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.874+0000 7f647f4f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f647806eb20 con 0x7f64780cea70 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647d290640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f647815bda0 0x7f64780ce360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42362/0 (socket says 172.21.15.64:42362) 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647d290640 1 -- 172.21.15.64:0/3942498938 learned_addr learned my addr 172.21.15.64:0/3942498938 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647da91640 1 --2- 172.21.15.64:0/3942498938 >> v2:172.21.15.66:3300/0 conn(0x7f64780b0ed0 0x7f647806f290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647ca8f640 1 --2- 172.21.15.64:0/3942498938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780cea70 0x7f64780d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f646ffff640 1 -- 172.21.15.64:0/3942498938 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3929385517 0 0) 0x7f64780cd350 con 0x7f647815bda0 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f646ffff640 1 -- 172.21.15.64:0/3942498938 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6464003520 con 0x7f647815bda0 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647ca8f640 1 -- 172.21.15.64:0/3942498938 >> v2:172.21.15.66:3300/0 conn(0x7f64780b0ed0 msgr2=0x7f647806f290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647ca8f640 1 --2- 172.21.15.64:0/3942498938 >> v2:172.21.15.66:3300/0 conn(0x7f64780b0ed0 0x7f647806f290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647ca8f640 1 -- 172.21.15.64:0/3942498938 >> v1:172.21.15.64:6789/0 conn(0x7f647815bda0 legacy=0x7f64780ce360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f646ffff640 1 -- 172.21.15.64:0/3942498938 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 585761055 0 0) 0x7f6464003520 con 0x7f647815bda0 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647ca8f640 1 -- 172.21.15.64:0/3942498938 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64780cd630 con 0x7f64780cea70 2024-03-04T07:33:00.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.875+0000 7f647da91640 1 --2- 172.21.15.64:0/3942498938 >> v2:172.21.15.66:3300/0 conn(0x7f64780b0ed0 0x7f647806f290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.876+0000 7f647ca8f640 1 --2- 172.21.15.64:0/3942498938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780cea70 0x7f64780d25b0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f64680097a0 tx=0x7f646802b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4244f0bdef1fd9d9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.876+0000 7f646ffff640 1 -- 172.21.15.64:0/3942498938 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6468037070 con 0x7f64780cea70 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.876+0000 7f646ffff640 1 -- 172.21.15.64:0/3942498938 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6468030510 con 0x7f64780cea70 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.876+0000 7f646ffff640 1 -- 172.21.15.64:0/3942498938 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6474001670 con 0x7f64780cea70 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 -- 172.21.15.64:0/3942498938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780cea70 msgr2=0x7f64780d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 --2- 172.21.15.64:0/3942498938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780cea70 0x7f64780d25b0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f64680097a0 tx=0x7f646802b790 comp rx=0 tx=0).stop 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 -- 172.21.15.64:0/3942498938 shutdown_connections 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 --2- 172.21.15.64:0/3942498938 >> v2:172.21.15.66:3300/0 conn(0x7f64780b0ed0 0x7f647806f290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 --2- 172.21.15.64:0/3942498938 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780cea70 0x7f64780d25b0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 -- 172.21.15.64:0/3942498938 >> 172.21.15.64:0/3942498938 conn(0x7f64780d1a90 msgr2=0x7f64780d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 -- 172.21.15.64:0/3942498938 shutdown_connections 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.877+0000 7f647f4f4640 1 -- 172.21.15.64:0/3942498938 wait complete. 2024-03-04T07:33:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.878+0000 7f647f4f4640 1 Processor -- start 2024-03-04T07:33:00.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.878+0000 7f647f4f4640 1 -- start start 2024-03-04T07:33:00.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647f4f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780b0ed0 0x7f64780c6360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647f4f4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64780cea70 0x7f64781550f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647f4f4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f647815bda0 0x7f64780c42d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647f4f4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f647815c180 con 0x7f64780cea70 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647f4f4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f647806ed70 con 0x7f647815bda0 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647f4f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f647815c300 con 0x7f64780b0ed0 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64780cea70 0x7f64781550f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64780cea70 0x7f64781550f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41772/0 (socket says 172.21.15.64:41772) 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 -- 172.21.15.64:0/4272220696 learned_addr learned my addr 172.21.15.64:0/4272220696 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 -- 172.21.15.64:0/4272220696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780b0ed0 msgr2=0x7f64780c6360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780b0ed0 0x7f64780c6360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 -- 172.21.15.64:0/4272220696 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f647815bda0 msgr2=0x7f64780c42d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f647815bda0 0x7f64780c42d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 -- 172.21.15.64:0/4272220696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64780c60c0 con 0x7f64780cea70 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647ca8f640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64780cea70 0x7f64781550f0 secure :-1 s=READY pgs=1393 cs=0 l=1 rev1=1 crypto rx=0x7f64680098c0 tx=0x7f64680046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f646dffb640 1 -- 172.21.15.64:0/4272220696 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6468040020 con 0x7f64780cea70 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f646dffb640 1 -- 172.21.15.64:0/4272220696 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6468038040 con 0x7f64780cea70 2024-03-04T07:33:00.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f64780c69d0 con 0x7f64780cea70 2024-03-04T07:33:00.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.879+0000 7f646dffb640 1 -- 172.21.15.64:0/4272220696 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6468037070 con 0x7f64780cea70 2024-03-04T07:33:00.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.880+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64780b5a60 con 0x7f64780cea70 2024-03-04T07:33:00.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.881+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6478067ff0 con 0x7f64780cea70 2024-03-04T07:33:00.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.881+0000 7f646dffb640 1 -- 172.21.15.64:0/4272220696 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6468043020 con 0x7f64780cea70 2024-03-04T07:33:00.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.882+0000 7f646dffb640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f645c06b4d0 0x7f645c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:00.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.882+0000 7f646dffb640 1 -- 172.21.15.64:0/4272220696 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f64680abb40 con 0x7f64780cea70 2024-03-04T07:33:00.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.882+0000 7f647d290640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f645c06b4d0 0x7f645c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:00.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.882+0000 7f647d290640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f645c06b4d0 0x7f645c06d990 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f6474001eb0 tx=0x7f6474001a40 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:00.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:00.885+0000 7f646dffb640 1 -- 172.21.15.64:0/4272220696 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f646804a030 con 0x7f64780cea70 2024-03-04T07:33:01.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.104+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f647815f630 con 0x7f64780cea70 2024-03-04T07:33:01.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.104+0000 7f646dffb640 1 -- 172.21.15.64:0/4272220696 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f646807ab60 con 0x7f64780cea70 2024-03-04T07:33:01.104 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:01.105 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:01.105 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f645c06b4d0 msgr2=0x7f645c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f645c06b4d0 0x7f645c06d990 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f6474001eb0 tx=0x7f6474001a40 comp rx=0 tx=0).stop 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64780cea70 msgr2=0x7f64781550f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64780cea70 0x7f64781550f0 secure :-1 s=READY pgs=1393 cs=0 l=1 rev1=1 crypto rx=0x7f64680098c0 tx=0x7f64680046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 shutdown_connections 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f645c06b4d0 0x7f645c06d990 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f647815bda0 0x7f64780c42d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64780cea70 0x7f64781550f0 unknown :-1 s=CLOSED pgs=1393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 --2- 172.21.15.64:0/4272220696 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64780b0ed0 0x7f64780c6360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.106+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 >> 172.21.15.64:0/4272220696 conn(0x7f64780d1a90 msgr2=0x7f64780b3c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.107+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 shutdown_connections 2024-03-04T07:33:01.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.107+0000 7f647f4f4640 1 -- 172.21.15.64:0/4272220696 wait complete. 2024-03-04T07:33:01.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_reintegrated 2024-03-04T07:33:01.238 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_reintegrated":0}} 2024-03-04T07:33:01.238 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_reintegrated": 0 } } 2024-03-04T07:33:01.239 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:01.239 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec mv dir_2/file_b dir_2/file_c) 2024-03-04T07:33:01.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:01.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.406+0000 7f5460d1e640 1 Processor -- start 2024-03-04T07:33:01.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.406+0000 7f5460d1e640 1 -- start start 2024-03-04T07:33:01.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f5460d1e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f545c066d00 0x7f545c15be60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f5460d1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c15f740 0x7f545c0670e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f5460d1e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f545c06fdc0 0x7f545c0719b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f5460d1e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f545c067620 con 0x7f545c06fdc0 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f5460d1e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f545c15fcc0 con 0x7f545c066d00 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f5460d1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f545c15ffa0 con 0x7f545c15f740 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f545c06fdc0 0x7f545c0719b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f545c06fdc0 0x7f545c0719b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41778/0 (socket says 172.21.15.64:41778) 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 -- 172.21.15.64:0/3584939797 learned_addr learned my addr 172.21.15.64:0/3584939797 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f5453fff640 1 --2- 172.21.15.64:0/3584939797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c15f740 0x7f545c0670e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 -- 172.21.15.64:0/3584939797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c15f740 msgr2=0x7f545c0670e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 --2- 172.21.15.64:0/3584939797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c15f740 0x7f545c0670e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545a59c640 1 --2- 172.21.15.64:0/3584939797 >> v2:172.21.15.66:3300/0 conn(0x7f545c066d00 0x7f545c15be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 -- 172.21.15.64:0/3584939797 >> v2:172.21.15.66:3300/0 conn(0x7f545c066d00 msgr2=0x7f545c15be60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:01.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 --2- 172.21.15.64:0/3584939797 >> v2:172.21.15.66:3300/0 conn(0x7f545c066d00 0x7f545c15be60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.407+0000 7f545ad9d640 1 -- 172.21.15.64:0/3584939797 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f545c160280 con 0x7f545c06fdc0 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.408+0000 7f545ad9d640 1 --2- 172.21.15.64:0/3584939797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c0719b0 secure :-1 s=READY pgs=1394 cs=0 l=1 rev1=1 crypto rx=0x7f545c0b5e00 tx=0x7f545402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c2797fc7e3855e23 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.408+0000 7f5459d9b640 1 -- 172.21.15.64:0/3584939797 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f545402ca70 con 0x7f545c06fdc0 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.408+0000 7f5459d9b640 1 -- 172.21.15.64:0/3584939797 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f545402cc10 con 0x7f545c06fdc0 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.408+0000 7f5459d9b640 1 -- 172.21.15.64:0/3584939797 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54540343f0 con 0x7f545c06fdc0 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 -- 172.21.15.64:0/3584939797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 msgr2=0x7f545c0719b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 --2- 172.21.15.64:0/3584939797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c0719b0 secure :-1 s=READY pgs=1394 cs=0 l=1 rev1=1 crypto rx=0x7f545c0b5e00 tx=0x7f545402bed0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 -- 172.21.15.64:0/3584939797 shutdown_connections 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 --2- 172.21.15.64:0/3584939797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c0719b0 unknown :-1 s=CLOSED pgs=1394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 --2- 172.21.15.64:0/3584939797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c15f740 0x7f545c0670e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 --2- 172.21.15.64:0/3584939797 >> v2:172.21.15.66:3300/0 conn(0x7f545c066d00 0x7f545c15be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 -- 172.21.15.64:0/3584939797 >> 172.21.15.64:0/3584939797 conn(0x7f545c0d0990 msgr2=0x7f545c0d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:01.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 -- 172.21.15.64:0/3584939797 shutdown_connections 2024-03-04T07:33:01.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.409+0000 7f5460d1e640 1 -- 172.21.15.64:0/3584939797 wait complete. 2024-03-04T07:33:01.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.410+0000 7f5460d1e640 1 Processor -- start 2024-03-04T07:33:01.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.410+0000 7f5460d1e640 1 -- start start 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5460d1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c066d00 0x7f545c1f4b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5460d1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c1588e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5460d1e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f545c15f740 0x7f545c152bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5460d1e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f545c0703d0 con 0x7f545c06fdc0 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5460d1e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f545c070250 con 0x7f545c15f740 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5460d1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f545c071ef0 con 0x7f545c066d00 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c1588e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c1588e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41784/0 (socket says 172.21.15.64:41784) 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 -- 172.21.15.64:0/4177122354 learned_addr learned my addr 172.21.15.64:0/4177122354 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f545ad9d640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f545c15f740 0x7f545c152bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f545a59c640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c066d00 0x7f545c1f4b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 -- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c066d00 msgr2=0x7f545c1f4b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c066d00 0x7f545c1f4b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 -- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f545c15f740 msgr2=0x7f545c152bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f545c15f740 0x7f545c152bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f5453fff640 1 -- 172.21.15.64:0/4177122354 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f545c1f5050 con 0x7f545c06fdc0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.411+0000 7f545ad9d640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f545c15f740 0x7f545c152bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.412+0000 7f5453fff640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c1588e0 secure :-1 s=READY pgs=1395 cs=0 l=1 rev1=1 crypto rx=0x7f544400c7f0 tx=0x7f544400ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.412+0000 7f5452ffd640 1 -- 172.21.15.64:0/4177122354 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5444015070 con 0x7f545c06fdc0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.412+0000 7f5452ffd640 1 -- 172.21.15.64:0/4177122354 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5444004220 con 0x7f545c06fdc0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.412+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f545c159250 con 0x7f545c06fdc0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.412+0000 7f5452ffd640 1 -- 172.21.15.64:0/4177122354 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5444007620 con 0x7f545c06fdc0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.412+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f545c159810 con 0x7f545c06fdc0 2024-03-04T07:33:01.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.413+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f545c159fe0 con 0x7f545c06fdc0 2024-03-04T07:33:01.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.414+0000 7f5452ffd640 1 -- 172.21.15.64:0/4177122354 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f54440079d0 con 0x7f545c06fdc0 2024-03-04T07:33:01.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.414+0000 7f5452ffd640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f543c06b4d0 0x7f543c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.414+0000 7f5452ffd640 1 -- 172.21.15.64:0/4177122354 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f544408a580 con 0x7f545c06fdc0 2024-03-04T07:33:01.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.414+0000 7f545a59c640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f543c06b4d0 0x7f543c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.415+0000 7f545a59c640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f543c06b4d0 0x7f543c06d990 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f544c00a970 tx=0x7f544c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:01.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.418+0000 7f5452ffd640 1 -- 172.21.15.64:0/4177122354 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5444059310 con 0x7f545c06fdc0 2024-03-04T07:33:01.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.640+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f545c068700 con 0x7f545c06fdc0 2024-03-04T07:33:01.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.641+0000 7f5452ffd640 1 -- 172.21.15.64:0/4177122354 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f5444058c10 con 0x7f545c06fdc0 2024-03-04T07:33:01.641 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:01.641 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:01.641 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:01.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f543c06b4d0 msgr2=0x7f543c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f543c06b4d0 0x7f543c06d990 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f544c00a970 tx=0x7f544c008040 comp rx=0 tx=0).stop 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 msgr2=0x7f545c1588e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c1588e0 secure :-1 s=READY pgs=1395 cs=0 l=1 rev1=1 crypto rx=0x7f544400c7f0 tx=0x7f544400ccb0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 shutdown_connections 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f543c06b4d0 0x7f543c06d990 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f545c15f740 0x7f545c152bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f545c06fdc0 0x7f545c1588e0 unknown :-1 s=CLOSED pgs=1395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 --2- 172.21.15.64:0/4177122354 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f545c066d00 0x7f545c1f4b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.643+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 >> 172.21.15.64:0/4177122354 conn(0x7f545c0d0990 msgr2=0x7f545c0ceda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.644+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 shutdown_connections 2024-03-04T07:33:01.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.644+0000 7f5460d1e640 1 -- 172.21.15.64:0/4177122354 wait complete. 2024-03-04T07:33:01.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-04T07:33:01.772 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"num_strays":0}} 2024-03-04T07:33:01.772 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 0 } } 2024-03-04T07:33:01.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:01.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.874+0000 7f4246da0640 1 Processor -- start 2024-03-04T07:33:01.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.874+0000 7f4246da0640 1 -- start start 2024-03-04T07:33:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4246da0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f42401612b0 0x7f42400b2990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4246da0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f42400cf370 con 0x7f4240161690 2024-03-04T07:33:01.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4246da0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f42400d1040 con 0x7f42400b5c40 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4246da0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f42400d1320 con 0x7f42401612b0 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f424559d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4240161690 0x7f42400b5530 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42378/0 (socket says 172.21.15.64:42378) 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f424559d640 1 -- 172.21.15.64:0/1227323463 learned_addr learned my addr 172.21.15.64:0/1227323463 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 21015379 0 0) 0x7f42400cf370 con 0x7f4240161690 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f422c003520 con 0x7f4240161690 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4245d9e640 1 --2- 172.21.15.64:0/1227323463 >> v2:172.21.15.66:3300/0 conn(0x7f42401612b0 0x7f42400b2990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1546706398 0 0) 0x7f422c003520 con 0x7f4240161690 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 >> v2:172.21.15.66:3300/0 conn(0x7f42401612b0 msgr2=0x7f42400b2990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 --2- 172.21.15.64:0/1227323463 >> v2:172.21.15.66:3300/0 conn(0x7f42401612b0 0x7f42400b2990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 >> v1:172.21.15.66:6789/0 conn(0x7f42400b5c40 legacy=0x7f42400d19b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42400ced20 con 0x7f4240161690 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f423c001810 con 0x7f4240161690 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 >> v1:172.21.15.64:6789/0 conn(0x7f4240161690 legacy=0x7f42400b5530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f422c003a00 0x7f422c003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f422c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.875+0000 7f4244d9c640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f422c0052e0 0x7f422c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42400d1320 con 0x7f422c004800 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f422c00c200 con 0x7f422c0052e0 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f422c00c400 con 0x7f422c003a00 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424559d640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f422c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424659f640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f422c0052e0 0x7f422c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f4245d9e640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f422c003a00 0x7f422c003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424559d640 1 -- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f422c003a00 msgr2=0x7f422c003e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424559d640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f422c003a00 0x7f422c003e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424559d640 1 -- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f422c0052e0 msgr2=0x7f422c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424559d640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f422c0052e0 0x7f422c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424559d640 1 -- 172.21.15.64:0/1227323463 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f422c003520 con 0x7f422c004800 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424659f640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f422c0052e0 0x7f422c00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:01.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.876+0000 7f424659f640 1 -- 172.21.15.64:0/1227323463 reap_dead start 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f424559d640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f422c004c80 secure :-1 s=READY pgs=1397 cs=0 l=1 rev1=1 crypto rx=0x7f42400cf540 tx=0x7f422c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb1873359809c34f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f423c007c60 con 0x7f422c004800 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f423c030070 con 0x7f422c004800 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4244d9c640 1 -- 172.21.15.64:0/1227323463 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f423c007510 con 0x7f422c004800 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4246da0640 1 -- 172.21.15.64:0/1227323463 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 msgr2=0x7f422c004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4246da0640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f422c004c80 secure :-1 s=READY pgs=1397 cs=0 l=1 rev1=1 crypto rx=0x7f42400cf540 tx=0x7f422c005830 comp rx=0 tx=0).stop 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4246da0640 1 -- 172.21.15.64:0/1227323463 shutdown_connections 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4246da0640 1 --2- 172.21.15.64:0/1227323463 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f422c004c80 unknown :-1 s=CLOSED pgs=1397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4246da0640 1 -- 172.21.15.64:0/1227323463 >> 172.21.15.64:0/1227323463 conn(0x7f4240067120 msgr2=0x7f4240067530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4246da0640 1 -- 172.21.15.64:0/1227323463 shutdown_connections 2024-03-04T07:33:01.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.877+0000 7f4246da0640 1 -- 172.21.15.64:0/1227323463 wait complete. 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.878+0000 7f4246da0640 1 Processor -- start 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4246da0640 1 -- start start 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4246da0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f4240157610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4246da0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4240155230 0x7f42401f3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4246da0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42400b5010 0x7f42400b5490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4246da0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42400b3150 con 0x7f422c004800 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4246da0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42400b2fd0 con 0x7f4240155230 2024-03-04T07:33:01.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4246da0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f42400d3560 con 0x7f42400b5010 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4245d9e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f4240157610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4245d9e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f4240157610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41808/0 (socket says 172.21.15.64:41808) 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.879+0000 7f4245d9e640 1 -- 172.21.15.64:0/352202382 learned_addr learned my addr 172.21.15.64:0/352202382 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f424659f640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42400b5010 0x7f42400b5490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f424559d640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4240155230 0x7f42401f3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4245d9e640 1 -- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42400b5010 msgr2=0x7f42400b5490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4245d9e640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42400b5010 0x7f42400b5490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4245d9e640 1 -- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4240155230 msgr2=0x7f42401f3910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4245d9e640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4240155230 0x7f42401f3910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4245d9e640 1 -- 172.21.15.64:0/352202382 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4240156700 con 0x7f422c004800 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f424659f640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42400b5010 0x7f42400b5490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f424559d640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4240155230 0x7f42401f3910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4245d9e640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f4240157610 secure :-1 s=READY pgs=1398 cs=0 l=1 rev1=1 crypto rx=0x7f423000f660 tx=0x7f423000fb20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:01.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f42367fc640 1 -- 172.21.15.64:0/352202382 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4230019020 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f42367fc640 1 -- 172.21.15.64:0/352202382 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f423000d940 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4240155af0 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f42367fc640 1 -- 172.21.15.64:0/352202382 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f423000dcf0 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.880+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42401560b0 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.881+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f424004f330 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.882+0000 7f42367fc640 1 -- 172.21.15.64:0/352202382 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f423001c020 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.882+0000 7f42367fc640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f422406b500 0x7f422406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.882+0000 7f424559d640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f422406b500 0x7f422406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.882+0000 7f42367fc640 1 -- 172.21.15.64:0/352202382 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f423008aac0 con 0x7f422c004800 2024-03-04T07:33:01.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.883+0000 7f424559d640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f422406b500 0x7f422406d9c0 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7f423c034000 tx=0x7f423c039040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:01.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:01.887+0000 7f42367fc640 1 -- 172.21.15.64:0/352202382 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4230023020 con 0x7f422c004800 2024-03-04T07:33:02.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.105+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f42400b6990 con 0x7f422c004800 2024-03-04T07:33:02.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.106+0000 7f42367fc640 1 -- 172.21.15.64:0/352202382 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f4230059850 con 0x7f422c004800 2024-03-04T07:33:02.106 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:02.106 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:02.106 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:02.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f422406b500 msgr2=0x7f422406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f422406b500 0x7f422406d9c0 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7f423c034000 tx=0x7f423c039040 comp rx=0 tx=0).stop 2024-03-04T07:33:02.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 msgr2=0x7f4240157610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f4240157610 secure :-1 s=READY pgs=1398 cs=0 l=1 rev1=1 crypto rx=0x7f423000f660 tx=0x7f423000fb20 comp rx=0 tx=0).stop 2024-03-04T07:33:02.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 shutdown_connections 2024-03-04T07:33:02.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f422406b500 0x7f422406d9c0 unknown :-1 s=CLOSED pgs=630 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f42400b5010 0x7f42400b5490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4240155230 0x7f42401f3910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 --2- 172.21.15.64:0/352202382 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f422c004800 0x7f4240157610 unknown :-1 s=CLOSED pgs=1398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 >> 172.21.15.64:0/352202382 conn(0x7f4240067120 msgr2=0x7f42400b2790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:02.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 shutdown_connections 2024-03-04T07:33:02.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.108+0000 7f4246da0640 1 -- 172.21.15.64:0/352202382 wait complete. 2024-03-04T07:33:02.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_reintegrated 2024-03-04T07:33:02.238 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_reintegrated":1}} 2024-03-04T07:33:02.239 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_reintegrated": 1 } } 2024-03-04T07:33:02.239 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:02.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.342+0000 7fe3b05bc640 1 Processor -- start 2024-03-04T07:33:02.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.342+0000 7fe3b05bc640 1 -- start start 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3b05bc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3a815df00 0x7fe3a80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3b05bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a80b9a30 0x7fe3a80d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3b05bc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe3a80cea30 con 0x7fe3a80b95a0 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3b05bc640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe3a80d2af0 con 0x7fe3a815df00 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3b05bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a80cf3c0 con 0x7fe3a80b9a30 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3ae358640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3a815df00 0x7fe3a80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3ae358640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe3a815df00 0x7fe3a80d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41824/0 (socket says 172.21.15.64:41824) 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.343+0000 7fe3ae358640 1 -- 172.21.15.64:0/266087963 learned_addr learned my addr 172.21.15.64:0/266087963 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.344+0000 7fe3aeb59640 1 --2- 172.21.15.64:0/266087963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a80b9a30 0x7fe3a80d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.344+0000 7fe3ae358640 1 -- 172.21.15.64:0/266087963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a80b9a30 msgr2=0x7fe3a80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.344+0000 7fe3ae358640 1 --2- 172.21.15.64:0/266087963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a80b9a30 0x7fe3a80d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.344+0000 7fe3ae358640 1 -- 172.21.15.64:0/266087963 >> v1:172.21.15.66:6789/0 conn(0x7fe3a80b95a0 legacy=0x7fe3a80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.344+0000 7fe3ae358640 1 -- 172.21.15.64:0/266087963 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3a80cf0e0 con 0x7fe3a815df00 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.345+0000 7fe3ae358640 1 --2- 172.21.15.64:0/266087963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a815df00 0x7fe3a80d25b0 secure :-1 s=READY pgs=1399 cs=0 l=1 rev1=1 crypto rx=0x7fe3a0009870 tx=0x7fe3a002b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=68cfb11293a75a9a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.345+0000 7fe3ad356640 1 -- 172.21.15.64:0/266087963 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe3a0034070 con 0x7fe3a815df00 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.345+0000 7fe3ad356640 1 -- 172.21.15.64:0/266087963 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe3a0030840 con 0x7fe3a815df00 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.345+0000 7fe3ad356640 1 -- 172.21.15.64:0/266087963 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe3a0030c10 con 0x7fe3a815df00 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 -- 172.21.15.64:0/266087963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a815df00 msgr2=0x7fe3a80d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/266087963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a815df00 0x7fe3a80d25b0 secure :-1 s=READY pgs=1399 cs=0 l=1 rev1=1 crypto rx=0x7fe3a0009870 tx=0x7fe3a002b980 comp rx=0 tx=0).stop 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 -- 172.21.15.64:0/266087963 shutdown_connections 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/266087963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a80b9a30 0x7fe3a80d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/266087963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a815df00 0x7fe3a80d25b0 unknown :-1 s=CLOSED pgs=1399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 -- 172.21.15.64:0/266087963 >> 172.21.15.64:0/266087963 conn(0x7fe3a8071960 msgr2=0x7fe3a8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 -- 172.21.15.64:0/266087963 shutdown_connections 2024-03-04T07:33:02.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.346+0000 7fe3b05bc640 1 -- 172.21.15.64:0/266087963 wait complete. 2024-03-04T07:33:02.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.347+0000 7fe3b05bc640 1 Processor -- start 2024-03-04T07:33:02.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.347+0000 7fe3b05bc640 1 -- start start 2024-03-04T07:33:02.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3b05bc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3a80b95a0 0x7fe3a80c8870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3b05bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a815df00 0x7fe3a8150fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3b05bc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a80ca4a0 0x7fe3a815e9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3b05bc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a80cec70 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3b05bc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a80ce750 con 0x7fe3a80b95a0 2024-03-04T07:33:02.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3b05bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe3a80cedf0 con 0x7fe3a815df00 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a80ca4a0 0x7fe3a815e9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a80ca4a0 0x7fe3a815e9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41838/0 (socket says 172.21.15.64:41838) 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 -- 172.21.15.64:0/3229008894 learned_addr learned my addr 172.21.15.64:0/3229008894 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3adb57640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a815df00 0x7fe3a8150fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3ae358640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3a80b95a0 0x7fe3a80c8870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 -- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a815df00 msgr2=0x7fe3a8150fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a815df00 0x7fe3a8150fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 -- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3a80b95a0 msgr2=0x7fe3a80c8870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3a80b95a0 0x7fe3a80c8870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 -- 172.21.15.64:0/3229008894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3a80c9400 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3adb57640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a815df00 0x7fe3a8150fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3ae358640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3a80b95a0 0x7fe3a80c8870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:02.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3aeb59640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a80ca4a0 0x7fe3a815e9d0 secure :-1 s=READY pgs=1400 cs=0 l=1 rev1=1 crypto rx=0x7fe39800d830 tx=0x7fe39800dcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:02.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.348+0000 7fe3a6ffd640 1 -- 172.21.15.64:0/3229008894 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39800b6c0 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.349+0000 7fe3a6ffd640 1 -- 172.21.15.64:0/3229008894 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe398019070 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.349+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe3a80c96e0 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.349+0000 7fe3a6ffd640 1 -- 172.21.15.64:0/3229008894 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe39800bac0 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.349+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe3a80c9ca0 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.349+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe3a80d4680 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.350+0000 7fe3a6ffd640 1 -- 172.21.15.64:0/3229008894 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe39801e460 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.350+0000 7fe3a6ffd640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe39406b4d0 0x7fe39406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.351+0000 7fe3a6ffd640 1 -- 172.21.15.64:0/3229008894 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fe39808aa10 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.351+0000 7fe3ae358640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe39406b4d0 0x7fe39406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.351+0000 7fe3ae358640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe39406b4d0 0x7fe39406d990 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7fe3a0036000 tx=0x7fe3a003b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:02.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.355+0000 7fe3a6ffd640 1 -- 172.21.15.64:0/3229008894 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe398059700 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.575+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe3a8067ff0 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.575+0000 7fe3a6ffd640 1 -- 172.21.15.64:0/3229008894 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7fe398059000 con 0x7fe3a80ca4a0 2024-03-04T07:33:02.575 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:02.575 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:02.576 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:02.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe39406b4d0 msgr2=0x7fe39406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe39406b4d0 0x7fe39406d990 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7fe3a0036000 tx=0x7fe3a003b040 comp rx=0 tx=0).stop 2024-03-04T07:33:02.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a80ca4a0 msgr2=0x7fe3a815e9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a80ca4a0 0x7fe3a815e9d0 secure :-1 s=READY pgs=1400 cs=0 l=1 rev1=1 crypto rx=0x7fe39800d830 tx=0x7fe39800dcf0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 shutdown_connections 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe39406b4d0 0x7fe39406d990 unknown :-1 s=CLOSED pgs=631 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe3a80ca4a0 0x7fe3a815e9d0 unknown :-1 s=CLOSED pgs=1400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe3a815df00 0x7fe3a8150fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 --2- 172.21.15.64:0/3229008894 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe3a80b95a0 0x7fe3a80c8870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 >> 172.21.15.64:0/3229008894 conn(0x7fe3a8071960 msgr2=0x7fe3a80b7ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 shutdown_connections 2024-03-04T07:33:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.577+0000 7fe3b05bc640 1 -- 172.21.15.64:0/3229008894 wait complete. 2024-03-04T07:33:02.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-04T07:33:02.723 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:33:02.724 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:33:02.724 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-04T07:33:02.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.765+0000 7f3e36a38f40 1 Processor -- start 2024-03-04T07:33:02.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.765+0000 7f3e36a38f40 1 -- start start 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e36a38f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699d8ce80 0x560699e352d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e36a38f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699e30280 0x560699d91880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e36a38f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e39980 0x560699e1ab40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e36a38f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x560699d08b70 con 0x560699d8ce80 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e36a38f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x560699da8310 con 0x560699e39980 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e36a38f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x560699e1a6b0 con 0x560699e30280 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e33de1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699d8ce80 0x560699e352d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e33de1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699d8ce80 0x560699e352d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41860/0 (socket says 172.21.15.64:41860) 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e33de1640 1 -- 172.21.15.64:0/2506303465 learned_addr learned my addr 172.21.15.64:0/2506303465 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:02.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e33de1640 1 -- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699e30280 msgr2=0x560699d91880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e335e0640 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699e30280 0x560699d91880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e345e2640 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e39980 0x560699e1ab40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e33de1640 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699e30280 0x560699d91880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.766+0000 7f3e33de1640 1 -- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e39980 msgr2=0x560699e1ab40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e33de1640 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e39980 0x560699e1ab40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e33de1640 1 -- 172.21.15.64:0/2506303465 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560699e34e00 con 0x560699d8ce80 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e335e0640 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699e30280 0x560699d91880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e345e2640 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e39980 0x560699e1ab40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e33de1640 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699d8ce80 0x560699e352d0 secure :-1 s=READY pgs=1403 cs=0 l=1 rev1=1 crypto rx=0x7f3e2401ced0 tx=0x7f3e2404c000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a128220d1f093141 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e325de640 1 -- 172.21.15.64:0/2506303465 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e24057040 con 0x560699d8ce80 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e325de640 1 -- 172.21.15.64:0/2506303465 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e24062070 con 0x560699d8ce80 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e325de640 1 -- 172.21.15.64:0/2506303465 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e24052a40 con 0x560699d8ce80 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2506303465 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699d8ce80 msgr2=0x560699e352d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699d8ce80 0x560699e352d0 secure :-1 s=READY pgs=1403 cs=0 l=1 rev1=1 crypto rx=0x7f3e2401ced0 tx=0x7f3e2404c000 comp rx=0 tx=0).stop 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2506303465 shutdown_connections 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e39980 0x560699e1ab40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699e30280 0x560699d91880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2506303465 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699d8ce80 0x560699e352d0 unknown :-1 s=CLOSED pgs=1403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2506303465 >> 172.21.15.64:0/2506303465 conn(0x560699d8ba60 msgr2=0x560699e36600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.767+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2506303465 shutdown_connections 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.768+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2506303465 wait complete. 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.768+0000 7f3e36a38f40 1 Processor -- start 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.768+0000 7f3e36a38f40 1 -- start start 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e36a38f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699d8ce80 0x560699e2f2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e36a38f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e30280 0x560699da9e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e36a38f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699e39980 0x560699daa3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e36a38f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x560699d8b3d0 con 0x560699e39980 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e36a38f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x560699e350a0 con 0x560699e30280 2024-03-04T07:33:02.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e36a38f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x560699d8b690 con 0x560699d8ce80 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699e39980 0x560699daa3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699e39980 0x560699daa3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41870/0 (socket says 172.21.15.64:41870) 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 -- 172.21.15.64:0/2362592859 learned_addr learned my addr 172.21.15.64:0/2362592859 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e335e0640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e30280 0x560699da9e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e33de1640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699d8ce80 0x560699e2f2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 -- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699d8ce80 msgr2=0x560699e2f2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699d8ce80 0x560699e2f2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 -- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e30280 msgr2=0x560699da9e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e30280 0x560699da9e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 -- 172.21.15.64:0/2362592859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560699e19bd0 con 0x560699e39980 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e335e0640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560699e30280 0x560699da9e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e33de1640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560699d8ce80 0x560699e2f2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e345e2640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699e39980 0x560699daa3d0 secure :-1 s=READY pgs=1404 cs=0 l=1 rev1=1 crypto rx=0x7f3e2c00c430 tx=0x7f3e2c00c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:02.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e315dc640 1 -- 172.21.15.64:0/2362592859 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e2c017070 con 0x560699e39980 2024-03-04T07:33:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e315dc640 1 -- 172.21.15.64:0/2362592859 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3e2c0062d0 con 0x560699e39980 2024-03-04T07:33:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e315dc640 1 -- 172.21.15.64:0/2362592859 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e2c004760 con 0x560699e39980 2024-03-04T07:33:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.769+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x560699e19eb0 con 0x560699e39980 2024-03-04T07:33:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.770+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560699e2f800 con 0x560699e39980 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.771+0000 7f3e315dc640 1 -- 172.21.15.64:0/2362592859 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3e2c01a070 con 0x560699e39980 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.771+0000 7f3e315dc640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e20069ee0 0x7f3e2006c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.771+0000 7f3e33de1640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e20069ee0 0x7f3e2006c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.771+0000 7f3e315dc640 1 -- 172.21.15.64:0/2362592859 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f3e2c0887f0 con 0x560699e39980 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x560699e38bc0 0x560699ed2d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e111) v9 -- 0x560699e35220 con 0x560699e38bc0 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e335e0640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x560699e38bc0 0x560699ed2d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e33de1640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e20069ee0 0x7f3e2006c3a0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f3e24000c00 tx=0x7f3e2404f000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e335e0640 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x560699e38bc0 0x560699ed2d70 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e335e0640 1 -- 172.21.15.64:0/2362592859 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (93) out=93b] v0'0 uv10 ondisk = 0) v8 ==== 164+0+93 (crc 0 0 0) 0x7f3e2800b720 con 0x560699e38bc0 2024-03-04T07:33:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x560699e38bc0 msgr2=0x560699ed2d70 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.772+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x560699e38bc0 0x560699ed2d70 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e20069ee0 msgr2=0x7f3e2006c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3e20069ee0 0x7f3e2006c3a0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f3e24000c00 tx=0x7f3e2404f000 comp rx=0 tx=0).stop 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699e39980 msgr2=0x560699daa3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 --2- 172.21.15.64:0/2362592859 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560699e39980 0x560699daa3d0 secure :-1 s=READY pgs=1404 cs=0 l=1 rev1=1 crypto rx=0x7f3e2c00c430 tx=0x7f3e2c00c8f0 comp rx=0 tx=0).stop 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e345e2640 1 -- 172.21.15.64:0/2362592859 reap_dead start 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 shutdown_connections 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 >> 172.21.15.64:0/2362592859 conn(0x560699d8ba60 msgr2=0x560699e18380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 shutdown_connections 2024-03-04T07:33:02.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:02.773+0000 7f3e36a38f40 1 -- 172.21.15.64:0/2362592859 wait complete. 2024-03-04T07:33:02.778 INFO:teuthology.orchestra.run.smithi064.stdout:W�file_cdir_2 2024-03-04T07:33:02.778 DEBUG:teuthology.orchestra.run.smithi064:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "ino": 1099511627777, 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "ancestors": [ 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 1099511628278, 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "file_c", 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 15 2024-03-04T07:33:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 1, 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "dir_2", 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 31 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "pool": 17, 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "old_pools": [] 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:33:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:02.865 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:02.866 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec ln dir_2/file_c dir_2/file_d) 2024-03-04T07:33:02.926 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:02.926 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f dir_2/file_c) 2024-03-04T07:33:02.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:03.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.128+0000 7f2d539bf640 1 Processor -- start 2024-03-04T07:33:03.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.128+0000 7f2d539bf640 1 -- start start 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.129+0000 7f2d539bf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 0x7f2d4c0d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.129+0000 7f2d539bf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d4c0cea30 con 0x7f2d4c15df00 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.129+0000 7f2d539bf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d4c0cf3c0 con 0x7f2d4c0b95a0 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.129+0000 7f2d539bf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d4c0cf0e0 con 0x7f2d4c0b9a10 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.129+0000 7f2d5175b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2d4c15df00 0x7f2d4c0d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42392/0 (socket says 172.21.15.64:42392) 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.129+0000 7f2d5175b640 1 -- 172.21.15.64:0/3899503990 learned_addr learned my addr 172.21.15.64:0/3899503990 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.129+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3630998942 0 0) 0x7f2d4c0cea30 con 0x7f2d4c15df00 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d38003520 con 0x7f2d4c15df00 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d51f5c640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 0x7f2d4c0d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1973620599 0 0) 0x7f2d38003520 con 0x7f2d4c15df00 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 msgr2=0x7f2d4c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 0x7f2d4c0d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 >> v1:172.21.15.66:6789/0 conn(0x7f2d4c0b95a0 legacy=0x7f2d4c0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d4c0ce470 con 0x7f2d4c15df00 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2d48001670 con 0x7f2d4c15df00 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d51f5c640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 0x7f2d4c0d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 >> v1:172.21.15.64:6789/0 conn(0x7f2d4c15df00 legacy=0x7f2d4c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d380039e0 0x7f2d38003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d38004800 0x7f2d38004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 0x7f2d380051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d4c0cf0e0 con 0x7f2d380039e0 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d3800bac0 con 0x7f2d38004800 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d3800bcc0 con 0x7f2d4c0b9a10 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d5175b640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d380039e0 0x7f2d38003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d51f5c640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 0x7f2d380051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d50f5a640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d38004800 0x7f2d38004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d5175b640 1 -- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 msgr2=0x7f2d380051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d5175b640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d4c0b9a10 0x7f2d380051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d5175b640 1 -- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d38004800 msgr2=0x7f2d38004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d5175b640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d38004800 0x7f2d38004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.130+0000 7f2d5175b640 1 -- 172.21.15.64:0/3899503990 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d38003520 con 0x7f2d380039e0 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.131+0000 7f2d5175b640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d380039e0 0x7f2d38003e00 secure :-1 s=READY pgs=1406 cs=0 l=1 rev1=1 crypto rx=0x7f2d4800c210 tx=0x7f2d38005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1099e1ef0b18eec4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.131+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d48007d30 con 0x7f2d380039e0 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.131+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d48030070 con 0x7f2d380039e0 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.131+0000 7f2d3ffff640 1 -- 172.21.15.64:0/3899503990 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d4802cd60 con 0x7f2d380039e0 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.132+0000 7f2d539bf640 1 -- 172.21.15.64:0/3899503990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d380039e0 msgr2=0x7f2d38003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.132+0000 7f2d539bf640 1 --2- 172.21.15.64:0/3899503990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d380039e0 0x7f2d38003e00 secure :-1 s=READY pgs=1406 cs=0 l=1 rev1=1 crypto rx=0x7f2d4800c210 tx=0x7f2d38005830 comp rx=0 tx=0).stop 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.132+0000 7f2d51f5c640 1 -- 172.21.15.64:0/3899503990 reap_dead start 2024-03-04T07:33:03.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.132+0000 7f2d539bf640 1 -- 172.21.15.64:0/3899503990 shutdown_connections 2024-03-04T07:33:03.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.132+0000 7f2d539bf640 1 -- 172.21.15.64:0/3899503990 >> 172.21.15.64:0/3899503990 conn(0x7f2d4c071960 msgr2=0x7f2d4c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:03.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.132+0000 7f2d539bf640 1 -- 172.21.15.64:0/3899503990 shutdown_connections 2024-03-04T07:33:03.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.132+0000 7f2d539bf640 1 -- 172.21.15.64:0/3899503990 wait complete. 2024-03-04T07:33:03.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.133+0000 7f2d539bf640 1 Processor -- start 2024-03-04T07:33:03.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.133+0000 7f2d539bf640 1 -- start start 2024-03-04T07:33:03.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d539bf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d380039e0 0x7f2d4c160250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d539bf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d4c15ecf0 0x7f2d4c0cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d539bf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d4c160790 0x7f2d4c0cb290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d539bf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d4c0cedf0 con 0x7f2d4c160790 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d539bf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d4c0cec70 con 0x7f2d4c15ecf0 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d539bf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d4c0b5e10 con 0x7f2d380039e0 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d4c160790 0x7f2d4c0cb290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d4c160790 0x7f2d4c0cb290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41894/0 (socket says 172.21.15.64:41894) 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 -- 172.21.15.64:0/1449728573 learned_addr learned my addr 172.21.15.64:0/1449728573 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d5175b640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d380039e0 0x7f2d4c160250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d50f5a640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d4c15ecf0 0x7f2d4c0cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 -- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d380039e0 msgr2=0x7f2d4c160250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d380039e0 0x7f2d4c160250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 -- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d4c15ecf0 msgr2=0x7f2d4c0cad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d4c15ecf0 0x7f2d4c0cad50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 -- 172.21.15.64:0/1449728573 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d4c15f1b0 con 0x7f2d4c160790 2024-03-04T07:33:03.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d51f5c640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d4c160790 0x7f2d4c0cb290 secure :-1 s=READY pgs=1407 cs=0 l=1 rev1=1 crypto rx=0x7f2d4c0d0f40 tx=0x7f2d44004780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.134+0000 7f2d3dffb640 1 -- 172.21.15.64:0/1449728573 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d44007cf0 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.135+0000 7f2d3dffb640 1 -- 172.21.15.64:0/1449728573 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d44019070 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.135+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d4c15f490 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.135+0000 7f2d3dffb640 1 -- 172.21.15.64:0/1449728573 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d44014650 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.135+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d4c15fa50 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.136+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d4c067ff0 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.136+0000 7f2d3dffb640 1 -- 172.21.15.64:0/1449728573 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d440147f0 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.136+0000 7f2d3dffb640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3006b500 0x7f2d3006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.137+0000 7f2d3dffb640 1 -- 172.21.15.64:0/1449728573 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f2d4408a6a0 con 0x7f2d4c160790 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.137+0000 7f2d5175b640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3006b500 0x7f2d3006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.137+0000 7f2d5175b640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3006b500 0x7f2d3006d9c0 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f2d48034000 tx=0x7f2d48009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:03.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.141+0000 7f2d3dffb640 1 -- 172.21.15.64:0/1449728573 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d44058de0 con 0x7f2d4c160790 2024-03-04T07:33:03.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.358+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2d4c0b2bc0 con 0x7f2d4c160790 2024-03-04T07:33:03.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.359+0000 7f2d3dffb640 1 -- 172.21.15.64:0/1449728573 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f2d4405ce80 con 0x7f2d4c160790 2024-03-04T07:33:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:03.359 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:03.359 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:03.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.360+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3006b500 msgr2=0x7f2d3006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.360+0000 7f2d539bf640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3006b500 0x7f2d3006d9c0 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f2d48034000 tx=0x7f2d48009030 comp rx=0 tx=0).stop 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.360+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d4c160790 msgr2=0x7f2d4c0cb290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.360+0000 7f2d539bf640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d4c160790 0x7f2d4c0cb290 secure :-1 s=READY pgs=1407 cs=0 l=1 rev1=1 crypto rx=0x7f2d4c0d0f40 tx=0x7f2d44004780 comp rx=0 tx=0).stop 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.360+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 shutdown_connections 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.360+0000 7f2d539bf640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3006b500 0x7f2d3006d9c0 unknown :-1 s=CLOSED pgs=633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.361+0000 7f2d539bf640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d4c160790 0x7f2d4c0cb290 unknown :-1 s=CLOSED pgs=1407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.361+0000 7f2d539bf640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d4c15ecf0 0x7f2d4c0cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.361+0000 7f2d539bf640 1 --2- 172.21.15.64:0/1449728573 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d380039e0 0x7f2d4c160250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.361+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 >> 172.21.15.64:0/1449728573 conn(0x7f2d4c071960 msgr2=0x7f2d4c0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.361+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 shutdown_connections 2024-03-04T07:33:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:03.361+0000 7f2d539bf640 1 -- 172.21.15.64:0/1449728573 wait complete. 2024-03-04T07:33:03.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-04T07:33:03.489 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"num_strays":1}} 2024-03-04T07:33:03.604 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 1 } } 2024-03-04T07:33:03.604 DEBUG:tasks.ceph_test_case:wait_until_equal: 1 != 0, waiting (timeout=60)... 2024-03-04T07:33:08.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:08.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.706+0000 7f5bccd7a640 1 Processor -- start 2024-03-04T07:33:08.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.706+0000 7f5bccd7a640 1 -- start start 2024-03-04T07:33:08.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bccd7a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5bc815df00 0x7f5bc80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bccd7a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc80b95a0 0x7f5bc80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bccd7a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5bc80b9a30 0x7f5bc80b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bccd7a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5bc80cea30 con 0x7f5bc80b9a30 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bccd7a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5bc80d2af0 con 0x7f5bc815df00 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bccd7a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5bc80cf3c0 con 0x7f5bc80b95a0 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bc6d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5bc80b9a30 0x7f5bc80b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bc6d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5bc80b9a30 0x7f5bc80b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40882/0 (socket says 172.21.15.64:40882) 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.707+0000 7f5bc6d9d640 1 -- 172.21.15.64:0/3493447889 learned_addr learned my addr 172.21.15.64:0/3493447889 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bbdd9b640 1 --2- 172.21.15.64:0/3493447889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc80b95a0 0x7f5bc80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bc659c640 1 --2- 172.21.15.64:0/3493447889 >> v2:172.21.15.66:3300/0 conn(0x7f5bc815df00 0x7f5bc80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bc6d9d640 1 -- 172.21.15.64:0/3493447889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc80b95a0 msgr2=0x7f5bc80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bc6d9d640 1 --2- 172.21.15.64:0/3493447889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc80b95a0 0x7f5bc80b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bc6d9d640 1 -- 172.21.15.64:0/3493447889 >> v2:172.21.15.66:3300/0 conn(0x7f5bc815df00 msgr2=0x7f5bc80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:08.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bc6d9d640 1 --2- 172.21.15.64:0/3493447889 >> v2:172.21.15.66:3300/0 conn(0x7f5bc815df00 0x7f5bc80d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bc6d9d640 1 -- 172.21.15.64:0/3493447889 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5bc80cf0e0 con 0x7f5bc80b9a30 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.708+0000 7f5bc6d9d640 1 --2- 172.21.15.64:0/3493447889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc80b9a30 0x7f5bc80b5c40 secure :-1 s=READY pgs=1408 cs=0 l=1 rev1=1 crypto rx=0x7f5bc80d11b0 tx=0x7f5bc002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59756625ef365b58 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bc5d9b640 1 -- 172.21.15.64:0/3493447889 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bc002ca70 con 0x7f5bc80b9a30 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bc5d9b640 1 -- 172.21.15.64:0/3493447889 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5bc002cc10 con 0x7f5bc80b9a30 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bc5d9b640 1 -- 172.21.15.64:0/3493447889 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bc00343f0 con 0x7f5bc80b9a30 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 -- 172.21.15.64:0/3493447889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc80b9a30 msgr2=0x7f5bc80b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/3493447889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc80b9a30 0x7f5bc80b5c40 secure :-1 s=READY pgs=1408 cs=0 l=1 rev1=1 crypto rx=0x7f5bc80d11b0 tx=0x7f5bc002bed0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 -- 172.21.15.64:0/3493447889 shutdown_connections 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/3493447889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc80b9a30 0x7f5bc80b5c40 unknown :-1 s=CLOSED pgs=1408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/3493447889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc80b95a0 0x7f5bc80b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/3493447889 >> v2:172.21.15.66:3300/0 conn(0x7f5bc815df00 0x7f5bc80d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 -- 172.21.15.64:0/3493447889 >> 172.21.15.64:0/3493447889 conn(0x7f5bc8071960 msgr2=0x7f5bc8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 -- 172.21.15.64:0/3493447889 shutdown_connections 2024-03-04T07:33:08.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.709+0000 7f5bccd7a640 1 -- 172.21.15.64:0/3493447889 wait complete. 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 Processor -- start 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 -- start start 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bc80b95a0 0x7f5bc80b2df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc815df00 0x7f5bc81f7860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc8155230 0x7f5bc80b8740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5bc80cec70 con 0x7f5bc815df00 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5bc80ce750 con 0x7f5bc80b95a0 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bccd7a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5bc80cedf0 con 0x7f5bc8155230 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bbdd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc815df00 0x7f5bc81f7860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.711+0000 7f5bbdd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc815df00 0x7f5bc81f7860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40888/0 (socket says 172.21.15.64:40888) 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbdd9b640 1 -- 172.21.15.64:0/1491558598 learned_addr learned my addr 172.21.15.64:0/1491558598 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bc659c640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bc80b95a0 0x7f5bc80b2df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bc6d9d640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc8155230 0x7f5bc80b8740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbdd9b640 1 -- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc8155230 msgr2=0x7f5bc80b8740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbdd9b640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc8155230 0x7f5bc80b8740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbdd9b640 1 -- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bc80b95a0 msgr2=0x7f5bc80b2df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbdd9b640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bc80b95a0 0x7f5bc80b2df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbdd9b640 1 -- 172.21.15.64:0/1491558598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5bc80b3480 con 0x7f5bc815df00 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbdd9b640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc815df00 0x7f5bc81f7860 secure :-1 s=READY pgs=1409 cs=0 l=1 rev1=1 crypto rx=0x7f5bb000d5f0 tx=0x7f5bb000dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbf7fe640 1 -- 172.21.15.64:0/1491558598 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bb00125f0 con 0x7f5bc815df00 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbf7fe640 1 -- 172.21.15.64:0/1491558598 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5bb0019070 con 0x7f5bc815df00 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5bc80b3760 con 0x7f5bc815df00 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bbf7fe640 1 -- 172.21.15.64:0/1491558598 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bb00129d0 con 0x7f5bc815df00 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.712+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5bc80b3d20 con 0x7f5bc815df00 2024-03-04T07:33:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.713+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5bc8067ff0 con 0x7f5bc815df00 2024-03-04T07:33:08.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.714+0000 7f5bbf7fe640 1 -- 172.21.15.64:0/1491558598 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5bb0012b70 con 0x7f5bc815df00 2024-03-04T07:33:08.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.714+0000 7f5bbf7fe640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ba806b4d0 0x7f5ba806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:08.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.714+0000 7f5bbf7fe640 1 -- 172.21.15.64:0/1491558598 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f5bb008a560 con 0x7f5bc815df00 2024-03-04T07:33:08.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.714+0000 7f5bc659c640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ba806b4d0 0x7f5ba806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:08.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.715+0000 7f5bc659c640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ba806b4d0 0x7f5ba806d990 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f5bb8006fd0 tx=0x7f5bb8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:08.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.719+0000 7f5bbf7fe640 1 -- 172.21.15.64:0/1491558598 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5bb0058ca0 con 0x7f5bc815df00 2024-03-04T07:33:08.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.937+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5bc815d490 con 0x7f5bc815df00 2024-03-04T07:33:08.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.938+0000 7f5bbf7fe640 1 -- 172.21.15.64:0/1491558598 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f5bb005cd40 con 0x7f5bc815df00 2024-03-04T07:33:08.938 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:08.938 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:08.938 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:08.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ba806b4d0 msgr2=0x7f5ba806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:08.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ba806b4d0 0x7f5ba806d990 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f5bb8006fd0 tx=0x7f5bb8008040 comp rx=0 tx=0).stop 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc815df00 msgr2=0x7f5bc81f7860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc815df00 0x7f5bc81f7860 secure :-1 s=READY pgs=1409 cs=0 l=1 rev1=1 crypto rx=0x7f5bb000d5f0 tx=0x7f5bb000dab0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 shutdown_connections 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5ba806b4d0 0x7f5ba806d990 unknown :-1 s=CLOSED pgs=634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5bc8155230 0x7f5bc80b8740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5bc815df00 0x7f5bc81f7860 unknown :-1 s=CLOSED pgs=1409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 --2- 172.21.15.64:0/1491558598 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5bc80b95a0 0x7f5bc80b2df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 >> 172.21.15.64:0/1491558598 conn(0x7f5bc8071960 msgr2=0x7f5bc80d3ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 shutdown_connections 2024-03-04T07:33:08.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:08.940+0000 7f5bccd7a640 1 -- 172.21.15.64:0/1491558598 wait complete. 2024-03-04T07:33:08.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache num_strays 2024-03-04T07:33:09.069 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"num_strays":0}} 2024-03-04T07:33:09.070 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 0 } } 2024-03-04T07:33:09.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:09.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.171+0000 7f234078a640 1 Processor -- start 2024-03-04T07:33:09.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.171+0000 7f234078a640 1 -- start start 2024-03-04T07:33:09.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.172+0000 7f234078a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 0x7f23380b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.172+0000 7f234078a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f23380cea30 con 0x7f233815df00 2024-03-04T07:33:09.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.172+0000 7f234078a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f23380cf3c0 con 0x7f23380b9a10 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.172+0000 7f234078a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f23380cf0e0 con 0x7f23380b95a0 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233e526640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f233815df00 0x7f23380d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34712/0 (socket says 172.21.15.64:34712) 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233e526640 1 -- 172.21.15.64:0/649225025 learned_addr learned my addr 172.21.15.64:0/649225025 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233dd25640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 0x7f23380b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1904190523 0 0) 0x7f23380cea30 con 0x7f233815df00 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2324003520 con 0x7f233815df00 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3858219002 0 0) 0x7f2324003520 con 0x7f233815df00 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 msgr2=0x7f23380b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 0x7f23380b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 >> v1:172.21.15.66:6789/0 conn(0x7f23380b9a10 legacy=0x7f23380d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f23380ce470 con 0x7f233815df00 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f2334001670 con 0x7f233815df00 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 >> v1:172.21.15.64:6789/0 conn(0x7f233815df00 legacy=0x7f23380d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 0x7f23240039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2324003f60 0x7f2324004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2324004d40 0x7f23240051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f23380cf0e0 con 0x7f2324003f60 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f232400baa0 con 0x7f2324004d40 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f232400bca0 con 0x7f23380b95a0 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233dd25640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2324003f60 0x7f2324004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.173+0000 7f233e526640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 0x7f23240039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233ed27640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2324004d40 0x7f23240051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233dd25640 1 -- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 msgr2=0x7f23240039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233dd25640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 0x7f23240039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233dd25640 1 -- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2324004d40 msgr2=0x7f23240051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233dd25640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2324004d40 0x7f23240051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233dd25640 1 -- 172.21.15.64:0/649225025 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2324003520 con 0x7f2324003f60 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233e526640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f23380b95a0 0x7f23240039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233dd25640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2324003f60 0x7f2324004800 secure :-1 s=READY pgs=1411 cs=0 l=1 rev1=1 crypto rx=0x7f232c009ac0 tx=0x7f2324005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4db7bb9afc1de21f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.174+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f232c039070 con 0x7f2324003f60 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f232c02c6b0 con 0x7f2324003f60 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f233d524640 1 -- 172.21.15.64:0/649225025 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f232c009690 con 0x7f2324003f60 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f234078a640 1 -- 172.21.15.64:0/649225025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2324003f60 msgr2=0x7f2324004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f234078a640 1 --2- 172.21.15.64:0/649225025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2324003f60 0x7f2324004800 secure :-1 s=READY pgs=1411 cs=0 l=1 rev1=1 crypto rx=0x7f232c009ac0 tx=0x7f2324005830 comp rx=0 tx=0).stop 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f233ed27640 1 -- 172.21.15.64:0/649225025 reap_dead start 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f234078a640 1 -- 172.21.15.64:0/649225025 shutdown_connections 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f234078a640 1 -- 172.21.15.64:0/649225025 >> 172.21.15.64:0/649225025 conn(0x7f2338071960 msgr2=0x7f2338071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f234078a640 1 -- 172.21.15.64:0/649225025 shutdown_connections 2024-03-04T07:33:09.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.175+0000 7f234078a640 1 -- 172.21.15.64:0/649225025 wait complete. 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.176+0000 7f234078a640 1 Processor -- start 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.176+0000 7f234078a640 1 -- start start 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f234078a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2324003f60 0x7f233815be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f234078a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f233815f330 0x7f233815b5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f234078a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f23381e94a0 0x7f23381e9920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f234078a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f23380cedf0 con 0x7f23381e94a0 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f234078a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f23380cec70 con 0x7f233815f330 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f234078a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f23380d3930 con 0x7f2324003f60 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f23381e94a0 0x7f23381e9920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f23381e94a0 0x7f23381e9920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40906/0 (socket says 172.21.15.64:40906) 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 -- 172.21.15.64:0/4038439559 learned_addr learned my addr 172.21.15.64:0/4038439559 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233dd25640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f233815f330 0x7f233815b5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233e526640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2324003f60 0x7f233815be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 -- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2324003f60 msgr2=0x7f233815be30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2324003f60 0x7f233815be30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 -- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f233815f330 msgr2=0x7f233815b5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f233815f330 0x7f233815b5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.177+0000 7f233ed27640 1 -- 172.21.15.64:0/4038439559 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f233815fb60 con 0x7f23381e94a0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.178+0000 7f233ed27640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f23381e94a0 0x7f23381e9920 secure :-1 s=READY pgs=1412 cs=0 l=1 rev1=1 crypto rx=0x7f23380d0f40 tx=0x7f233000eb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.178+0000 7f232affd640 1 -- 172.21.15.64:0/4038439559 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2330003970 con 0x7f23381e94a0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.178+0000 7f232affd640 1 -- 172.21.15.64:0/4038439559 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2330018070 con 0x7f23381e94a0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.178+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f233815fe40 con 0x7f23381e94a0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.178+0000 7f232affd640 1 -- 172.21.15.64:0/4038439559 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2330003d50 con 0x7f23381e94a0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.178+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2338160400 con 0x7f23381e94a0 2024-03-04T07:33:09.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.179+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f233804f330 con 0x7f23381e94a0 2024-03-04T07:33:09.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.179+0000 7f232affd640 1 -- 172.21.15.64:0/4038439559 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f23300050d0 con 0x7f23381e94a0 2024-03-04T07:33:09.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.180+0000 7f232affd640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231c06b500 0x7f231c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.180+0000 7f232affd640 1 -- 172.21.15.64:0/4038439559 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f2330089fe0 con 0x7f23381e94a0 2024-03-04T07:33:09.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.180+0000 7f233e526640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231c06b500 0x7f231c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.180+0000 7f233e526640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231c06b500 0x7f231c06d9c0 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f2334000ff0 tx=0x7f2334009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:09.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.184+0000 7f232affd640 1 -- 172.21.15.64:0/4038439559 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f233804f330 con 0x7f23381e94a0 2024-03-04T07:33:09.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.402+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2338067ff0 con 0x7f23381e94a0 2024-03-04T07:33:09.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.403+0000 7f232affd640 1 -- 172.21.15.64:0/4038439559 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f2330058d70 con 0x7f23381e94a0 2024-03-04T07:33:09.403 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:09.403 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:09.403 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:09.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231c06b500 msgr2=0x7f231c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231c06b500 0x7f231c06d9c0 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f2334000ff0 tx=0x7f2334009040 comp rx=0 tx=0).stop 2024-03-04T07:33:09.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f23381e94a0 msgr2=0x7f23381e9920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f23381e94a0 0x7f23381e9920 secure :-1 s=READY pgs=1412 cs=0 l=1 rev1=1 crypto rx=0x7f23380d0f40 tx=0x7f233000eb90 comp rx=0 tx=0).stop 2024-03-04T07:33:09.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 shutdown_connections 2024-03-04T07:33:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f231c06b500 0x7f231c06d9c0 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f23381e94a0 0x7f23381e9920 unknown :-1 s=CLOSED pgs=1412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f233815f330 0x7f233815b5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 --2- 172.21.15.64:0/4038439559 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2324003f60 0x7f233815be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 >> 172.21.15.64:0/4038439559 conn(0x7f2338071960 msgr2=0x7f23380b7b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 shutdown_connections 2024-03-04T07:33:09.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.405+0000 7f234078a640 1 -- 172.21.15.64:0/4038439559 wait complete. 2024-03-04T07:33:09.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_reintegrated 2024-03-04T07:33:09.533 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_reintegrated":2}} 2024-03-04T07:33:09.533 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_reintegrated": 2 } } 2024-03-04T07:33:09.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:09.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.636+0000 7f3ab97bc640 1 Processor -- start 2024-03-04T07:33:09.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.636+0000 7f3ab97bc640 1 -- start start 2024-03-04T07:33:09.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab97bc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3ab415df70 0x7f3ab40d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab97bc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ab40d18c0 0x7f3ab4071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab97bc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3ab40b6f20 con 0x7f3ab4071a80 2024-03-04T07:33:09.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab97bc640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3ab4071eb0 con 0x7f3ab40d18c0 2024-03-04T07:33:09.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab97bc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3ab40d3030 con 0x7f3ab415df70 2024-03-04T07:33:09.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab27fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ab40d18c0 0x7f3ab4071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab27fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ab40d18c0 0x7f3ab4071540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40918/0 (socket says 172.21.15.64:40918) 2024-03-04T07:33:09.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab27fc640 1 -- 172.21.15.64:0/316636273 learned_addr learned my addr 172.21.15.64:0/316636273 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:09.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab27fc640 1 -- 172.21.15.64:0/316636273 >> v2:172.21.15.66:3300/0 conn(0x7f3ab415df70 msgr2=0x7f3ab40d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:09.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab27fc640 1 --2- 172.21.15.64:0/316636273 >> v2:172.21.15.66:3300/0 conn(0x7f3ab415df70 0x7f3ab40d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab27fc640 1 -- 172.21.15.64:0/316636273 >> v1:172.21.15.66:6789/0 conn(0x7f3ab4071a80 legacy=0x7f3ab40b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:09.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.637+0000 7f3ab27fc640 1 -- 172.21.15.64:0/316636273 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ab40d3310 con 0x7f3ab40d18c0 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.638+0000 7f3ab27fc640 1 --2- 172.21.15.64:0/316636273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab40d18c0 0x7f3ab4071540 secure :-1 s=READY pgs=1413 cs=0 l=1 rev1=1 crypto rx=0x7f3ab404f3a0 tx=0x7f3a9c02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=75ee0b19667f0315 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.638+0000 7f3ab1ffb640 1 -- 172.21.15.64:0/316636273 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a9c034070 con 0x7f3ab40d18c0 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.638+0000 7f3ab1ffb640 1 -- 172.21.15.64:0/316636273 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a9c030860 con 0x7f3ab40d18c0 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.638+0000 7f3ab1ffb640 1 -- 172.21.15.64:0/316636273 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a9c030c30 con 0x7f3ab40d18c0 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 -- 172.21.15.64:0/316636273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab40d18c0 msgr2=0x7f3ab4071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/316636273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab40d18c0 0x7f3ab4071540 secure :-1 s=READY pgs=1413 cs=0 l=1 rev1=1 crypto rx=0x7f3ab404f3a0 tx=0x7f3a9c02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 -- 172.21.15.64:0/316636273 shutdown_connections 2024-03-04T07:33:09.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/316636273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab40d18c0 0x7f3ab4071540 unknown :-1 s=CLOSED pgs=1413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/316636273 >> v2:172.21.15.66:3300/0 conn(0x7f3ab415df70 0x7f3ab40d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 -- 172.21.15.64:0/316636273 >> 172.21.15.64:0/316636273 conn(0x7f3ab40d3cf0 msgr2=0x7f3ab40d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:09.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 -- 172.21.15.64:0/316636273 shutdown_connections 2024-03-04T07:33:09.640 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.639+0000 7f3ab97bc640 1 -- 172.21.15.64:0/316636273 wait complete. 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.640+0000 7f3ab97bc640 1 Processor -- start 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.640+0000 7f3ab97bc640 1 -- start start 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab97bc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab4071a80 0x7f3ab4156c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab97bc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ab40d18c0 0x7f3ab415a4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab97bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ab415df70 0x7f3ab415a9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab97bc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ab40d2cf0 con 0x7f3ab4071a80 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab97bc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ab40d38d0 con 0x7f3ab40d18c0 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab97bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ab415e350 con 0x7f3ab415df70 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab4071a80 0x7f3ab4156c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab4071a80 0x7f3ab4156c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40928/0 (socket says 172.21.15.64:40928) 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 -- 172.21.15.64:0/2793388523 learned_addr learned my addr 172.21.15.64:0/2793388523 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 -- 172.21.15.64:0/2793388523 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ab415df70 msgr2=0x7f3ab415a9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ab415df70 0x7f3ab415a9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab27fc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ab40d18c0 0x7f3ab415a4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 -- 172.21.15.64:0/2793388523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ab40d18c0 msgr2=0x7f3ab415a4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ab40d18c0 0x7f3ab415a4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.641+0000 7f3ab2ffd640 1 -- 172.21.15.64:0/2793388523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ab40c9f50 con 0x7f3ab4071a80 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.642+0000 7f3ab2ffd640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab4071a80 0x7f3ab4156c80 secure :-1 s=READY pgs=1414 cs=0 l=1 rev1=1 crypto rx=0x7f3aac00ba50 tx=0x7f3aac00bf10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.642+0000 7f3aab7fe640 1 -- 172.21.15.64:0/2793388523 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3aac019070 con 0x7f3ab4071a80 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.642+0000 7f3aab7fe640 1 -- 172.21.15.64:0/2793388523 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3aac00cc00 con 0x7f3ab4071a80 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.642+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3ab4146100 con 0x7f3ab4071a80 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.642+0000 7f3aab7fe640 1 -- 172.21.15.64:0/2793388523 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3aac014440 con 0x7f3ab4071a80 2024-03-04T07:33:09.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.642+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3ab41571c0 con 0x7f3ab4071a80 2024-03-04T07:33:09.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.643+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3ab4067ff0 con 0x7f3ab4071a80 2024-03-04T07:33:09.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.644+0000 7f3aab7fe640 1 -- 172.21.15.64:0/2793388523 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3aac01c070 con 0x7f3ab4071a80 2024-03-04T07:33:09.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.644+0000 7f3aab7fe640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a9406b4d0 0x7f3a9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:09.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.644+0000 7f3aab7fe640 1 -- 172.21.15.64:0/2793388523 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f3aac08a040 con 0x7f3ab4071a80 2024-03-04T07:33:09.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.644+0000 7f3ab27fc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a9406b4d0 0x7f3a9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:09.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.644+0000 7f3ab27fc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a9406b4d0 0x7f3a9406d990 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f3a9c036000 tx=0x7f3a9c03b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:09.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.648+0000 7f3aab7fe640 1 -- 172.21.15.64:0/2793388523 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3aac058e60 con 0x7f3ab4071a80 2024-03-04T07:33:09.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.982+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3ab415d550 con 0x7f3ab4071a80 2024-03-04T07:33:09.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.982+0000 7f3aab7fe640 1 -- 172.21.15.64:0/2793388523 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f3aac058760 con 0x7f3ab4071a80 2024-03-04T07:33:09.982 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:09.982 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:09.982 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a9406b4d0 msgr2=0x7f3a9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a9406b4d0 0x7f3a9406d990 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f3a9c036000 tx=0x7f3a9c03b040 comp rx=0 tx=0).stop 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab4071a80 msgr2=0x7f3ab4156c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab4071a80 0x7f3ab4156c80 secure :-1 s=READY pgs=1414 cs=0 l=1 rev1=1 crypto rx=0x7f3aac00ba50 tx=0x7f3aac00bf10 comp rx=0 tx=0).stop 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 shutdown_connections 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a9406b4d0 0x7f3a9406d990 unknown :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ab415df70 0x7f3ab415a9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ab40d18c0 0x7f3ab415a4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 --2- 172.21.15.64:0/2793388523 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ab4071a80 0x7f3ab4156c80 unknown :-1 s=CLOSED pgs=1414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:09.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 >> 172.21.15.64:0/2793388523 conn(0x7f3ab40d3cf0 msgr2=0x7f3ab40b6020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:09.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 shutdown_connections 2024-03-04T07:33:09.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:09.984+0000 7f3ab97bc640 1 -- 172.21.15.64:0/2793388523 wait complete. 2024-03-04T07:33:09.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json flush journal 2024-03-04T07:33:10.123 INFO:teuthology.orchestra.run.smithi064.stdout:{"message":"","return_code":0} 2024-03-04T07:33:10.123 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:33:10.124 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data getxattr 10000000001.00000000 parent 2024-03-04T07:33:10.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.165+0000 7f725c663f40 1 Processor -- start 2024-03-04T07:33:10.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.165+0000 7f725c663f40 1 -- start start 2024-03-04T07:33:10.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725c663f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793cdb72f0 0x55793cddf630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725c663f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793ce8d020 0x55793cdfb850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725c663f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793cdf89f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725c663f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55793cdfdd30 con 0x55793cdfbd90 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725c663f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55793cd5bb70 con 0x55793ce8d020 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725c663f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55793cdfdfd0 con 0x55793cdb72f0 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725a965640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793cdf89f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725a965640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793cdf89f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40936/0 (socket says 172.21.15.64:40936) 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725a965640 1 -- 172.21.15.64:0/221666307 learned_addr learned my addr 172.21.15.64:0/221666307 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f7259963640 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793ce8d020 0x55793cdfb850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.166+0000 7f725a164640 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793cdb72f0 0x55793cddf630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725a965640 1 -- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793cdb72f0 msgr2=0x55793cddf630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725a965640 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793cdb72f0 0x55793cddf630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725a965640 1 -- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793ce8d020 msgr2=0x55793cdfb850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725a965640 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793ce8d020 0x55793cdfb850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725a965640 1 -- 172.21.15.64:0/221666307 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55793cde4340 con 0x55793cdfbd90 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f7259963640 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793ce8d020 0x55793cdfb850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725a164640 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793cdb72f0 0x55793cddf630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725a965640 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793cdf89f0 secure :-1 s=READY pgs=1415 cs=0 l=1 rev1=1 crypto rx=0x7f7254005b90 tx=0x7f7254051000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c771dc4ceaa87254 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f7258961640 1 -- 172.21.15.64:0/221666307 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7254066070 con 0x55793cdfbd90 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f7258961640 1 -- 172.21.15.64:0/221666307 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f725404e650 con 0x55793cdfbd90 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f7258961640 1 -- 172.21.15.64:0/221666307 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f725404ec20 con 0x55793cdfbd90 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 -- 172.21.15.64:0/221666307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 msgr2=0x55793cdf89f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793cdf89f0 secure :-1 s=READY pgs=1415 cs=0 l=1 rev1=1 crypto rx=0x7f7254005b90 tx=0x7f7254051000 comp rx=0 tx=0).stop 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 -- 172.21.15.64:0/221666307 shutdown_connections 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793cdf89f0 unknown :-1 s=CLOSED pgs=1415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793ce8d020 0x55793cdfb850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 --2- 172.21.15.64:0/221666307 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793cdb72f0 0x55793cddf630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 -- 172.21.15.64:0/221666307 >> 172.21.15.64:0/221666307 conn(0x55793cdb45e0 msgr2=0x55793ce8eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:10.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 -- 172.21.15.64:0/221666307 shutdown_connections 2024-03-04T07:33:10.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.167+0000 7f725c663f40 1 -- 172.21.15.64:0/221666307 wait complete. 2024-03-04T07:33:10.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.168+0000 7f725c663f40 1 Processor -- start 2024-03-04T07:33:10.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.168+0000 7f725c663f40 1 -- start start 2024-03-04T07:33:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725c663f40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793cdb72f0 0x55793ce8eea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725c663f40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793ce5a420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725c663f40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793ce8d020 0x55793ce5b560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725c663f40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x55793cde0740 con 0x55793cdfbd90 2024-03-04T07:33:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725c663f40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x55793cddff20 con 0x55793cdb72f0 2024-03-04T07:33:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725c663f40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x55793cde09e0 con 0x55793ce8d020 2024-03-04T07:33:10.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793ce5a420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793ce5a420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40950/0 (socket says 172.21.15.64:40950) 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 -- 172.21.15.64:0/3837308940 learned_addr learned my addr 172.21.15.64:0/3837308940 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725a965640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793ce8d020 0x55793ce5b560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725a164640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793cdb72f0 0x55793ce8eea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 -- 172.21.15.64:0/3837308940 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793ce8d020 msgr2=0x55793ce5b560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x55793ce8d020 0x55793ce5b560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 -- 172.21.15.64:0/3837308940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793cdb72f0 msgr2=0x55793ce8eea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793cdb72f0 0x55793ce8eea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 -- 172.21.15.64:0/3837308940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55793ce5c550 con 0x55793cdfbd90 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f725a164640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x55793cdb72f0 0x55793ce8eea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f7259963640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793ce5a420 secure :-1 s=READY pgs=1416 cs=0 l=1 rev1=1 crypto rx=0x7f7250001c80 tx=0x7f725000b6c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.169+0000 7f724b7fe640 1 -- 172.21.15.64:0/3837308940 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f725000f650 con 0x55793cdfbd90 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.170+0000 7f724b7fe640 1 -- 172.21.15.64:0/3837308940 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f725000f7f0 con 0x55793cdfbd90 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.170+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x55793cdfc3e0 con 0x55793cdfbd90 2024-03-04T07:33:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.170+0000 7f724b7fe640 1 -- 172.21.15.64:0/3837308940 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f725000fba0 con 0x55793cdfbd90 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.170+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55793ce5c810 con 0x55793cdfbd90 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.171+0000 7f724b7fe640 1 -- 172.21.15.64:0/3837308940 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f72500203f0 con 0x55793cdfbd90 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.171+0000 7f724b7fe640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f723c069ee0 0x7f723c06c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.171+0000 7f724b7fe640 1 -- 172.21.15.64:0/3837308940 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f7250089720 con 0x55793cdfbd90 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.171+0000 7f725a164640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f723c069ee0 0x7f723c06c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.172+0000 7f725c663f40 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x55793cf15c60 0x55793cf16070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.172+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:1 17.7 17:e2e07930:::10000000001.00000000:head [getxattr parent in=6b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e111) v9 -- 0x55793ce70390 con 0x55793cf15c60 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.172+0000 7f725a965640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x55793cf15c60 0x55793cf16070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.172+0000 7f725a164640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f723c069ee0 0x7f723c06c3a0 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7f724c005fd0 tx=0x7f724c00c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.172+0000 7f725a965640 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x55793cf15c60 0x55793cf16070 crc :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.172+0000 7f725a965640 1 -- 172.21.15.64:0/3837308940 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== osd_op_reply(1 10000000001.00000000 [getxattr (93) out=93b] v0'0 uv11 ondisk = 0) v8 ==== 164+0+93 (crc 0 0 0) 0x7f725404ea20 con 0x55793cf15c60 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x55793cf15c60 msgr2=0x55793cf16070 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x55793cf15c60 0x55793cf16070 crc :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f723c069ee0 msgr2=0x7f723c06c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f723c069ee0 0x7f723c06c3a0 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7f724c005fd0 tx=0x7f724c00c040 comp rx=0 tx=0).stop 2024-03-04T07:33:10.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 msgr2=0x55793ce5a420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 --2- 172.21.15.64:0/3837308940 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x55793cdfbd90 0x55793ce5a420 secure :-1 s=READY pgs=1416 cs=0 l=1 rev1=1 crypto rx=0x7f7250001c80 tx=0x7f725000b6c0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725a965640 1 -- 172.21.15.64:0/3837308940 reap_dead start 2024-03-04T07:33:10.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 shutdown_connections 2024-03-04T07:33:10.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 >> 172.21.15.64:0/3837308940 conn(0x55793cdb45e0 msgr2=0x55793cdfae10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:10.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 shutdown_connections 2024-03-04T07:33:10.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.173+0000 7f725c663f40 1 -- 172.21.15.64:0/3837308940 wait complete. 2024-03-04T07:33:10.177 INFO:teuthology.orchestra.run.smithi064.stdout:W�file_ddir_2* 2024-03-04T07:33:10.177 DEBUG:teuthology.orchestra.run.smithi064:> ceph-dencoder type inode_backtrace_t import - decode dump_json 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "ino": 1099511627777, 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "ancestors": [ 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 1099511628278, 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "file_d", 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 25 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "dirino": 1, 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "dname": "dir_2", 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: "version": 42 2024-03-04T07:33:10.262 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:10.263 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-03-04T07:33:10.263 INFO:teuthology.orchestra.run.smithi064.stdout: "pool": 17, 2024-03-04T07:33:10.263 INFO:teuthology.orchestra.run.smithi064.stdout: "old_pools": [] 2024-03-04T07:33:10.263 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:33:10.263 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:10.264 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:10.264 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec rm -f dir_2/file_d) 2024-03-04T07:33:10.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:10.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.464+0000 7fa2efa50640 1 Processor -- start 2024-03-04T07:33:10.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.464+0000 7fa2efa50640 1 -- start start 2024-03-04T07:33:10.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2efa50640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa2e80b95a0 0x7fa2e80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2efa50640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e80b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2efa50640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa2e80cea30 con 0x7fa2e815df00 2024-03-04T07:33:10.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2efa50640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa2e80cf3c0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2efa50640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa2e80cf0e0 con 0x7fa2e80b9a10 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2ecfeb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa2e80b95a0 0x7fa2e80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2ed7ec640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa2e815df00 0x7fa2e80d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34722/0 (socket says 172.21.15.64:34722) 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2ed7ec640 1 -- 172.21.15.64:0/3793134722 learned_addr learned my addr 172.21.15.64:0/3793134722 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2ecfeb640 1 -- 172.21.15.64:0/3793134722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 msgr2=0x7fa2e80b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2edfed640 1 --2- 172.21.15.64:0/3793134722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e80b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2dffff640 1 -- 172.21.15.64:0/3793134722 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1346678653 0 0) 0x7fa2e80cea30 con 0x7fa2e815df00 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2ecfeb640 1 --2- 172.21.15.64:0/3793134722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e80b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2ecfeb640 1 -- 172.21.15.64:0/3793134722 >> v1:172.21.15.64:6789/0 conn(0x7fa2e815df00 legacy=0x7fa2e80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2ecfeb640 1 -- 172.21.15.64:0/3793134722 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2e80ce470 con 0x7fa2e80b95a0 2024-03-04T07:33:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.465+0000 7fa2edfed640 1 --2- 172.21.15.64:0/3793134722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e80b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:10.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.466+0000 7fa2ecfeb640 1 --2- 172.21.15.64:0/3793134722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80b5700 secure :-1 s=READY pgs=1418 cs=0 l=1 rev1=1 crypto rx=0x7fa2d80097a0 tx=0x7fa2d802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7649c2af6ff3d6cf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:10.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.466+0000 7fa2dffff640 1 -- 172.21.15.64:0/3793134722 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2d8035020 con 0x7fa2e80b95a0 2024-03-04T07:33:10.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.466+0000 7fa2dffff640 1 -- 172.21.15.64:0/3793134722 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa2d8031830 con 0x7fa2e80b95a0 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.466+0000 7fa2dffff640 1 -- 172.21.15.64:0/3793134722 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2d8031c00 con 0x7fa2e80b95a0 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 -- 172.21.15.64:0/3793134722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 msgr2=0x7fa2e80b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 --2- 172.21.15.64:0/3793134722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80b5700 secure :-1 s=READY pgs=1418 cs=0 l=1 rev1=1 crypto rx=0x7fa2d80097a0 tx=0x7fa2d802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 -- 172.21.15.64:0/3793134722 shutdown_connections 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 --2- 172.21.15.64:0/3793134722 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e80b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 --2- 172.21.15.64:0/3793134722 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80b5700 unknown :-1 s=CLOSED pgs=1418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 -- 172.21.15.64:0/3793134722 >> 172.21.15.64:0/3793134722 conn(0x7fa2e8071960 msgr2=0x7fa2e8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 -- 172.21.15.64:0/3793134722 shutdown_connections 2024-03-04T07:33:10.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.467+0000 7fa2efa50640 1 -- 172.21.15.64:0/3793134722 wait complete. 2024-03-04T07:33:10.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.469+0000 7fa2efa50640 1 Processor -- start 2024-03-04T07:33:10.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.469+0000 7fa2efa50640 1 -- start start 2024-03-04T07:33:10.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2efa50640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80c94d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2efa50640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e81f6d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2efa50640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2e815df00 0x7fa2e81f7280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2efa50640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2e80cedf0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2efa50640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2e80cec70 con 0x7fa2e815df00 2024-03-04T07:33:10.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2efa50640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa2e80d3220 con 0x7fa2e80b9a10 2024-03-04T07:33:10.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2ed7ec640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80c94d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2ed7ec640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80c94d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40964/0 (socket says 172.21.15.64:40964) 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.470+0000 7fa2ed7ec640 1 -- 172.21.15.64:0/884544129 learned_addr learned my addr 172.21.15.64:0/884544129 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ed7ec640 1 -- 172.21.15.64:0/884544129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 msgr2=0x7fa2e81f6d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ecfeb640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e81f6d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ed7ec640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e81f6d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ed7ec640 1 -- 172.21.15.64:0/884544129 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2e815df00 msgr2=0x7fa2e81f7280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ed7ec640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2e815df00 0x7fa2e81f7280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ed7ec640 1 -- 172.21.15.64:0/884544129 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2e80c9f40 con 0x7fa2e80b95a0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ed7ec640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80c94d0 secure :-1 s=READY pgs=1419 cs=0 l=1 rev1=1 crypto rx=0x7fa2e400c430 tx=0x7fa2e400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ddffb640 1 -- 172.21.15.64:0/884544129 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2e40074e0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ddffb640 1 -- 172.21.15.64:0/884544129 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa2e40047e0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2ddffb640 1 -- 172.21.15.64:0/884544129 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2e4004bd0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa2e80ca220 con 0x7fa2e80b95a0 2024-03-04T07:33:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.471+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa2e8149ff0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.472+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa2e8067ff0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.472+0000 7fa2ddffb640 1 -- 172.21.15.64:0/884544129 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa2e4011070 con 0x7fa2e80b95a0 2024-03-04T07:33:10.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.473+0000 7fa2ddffb640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa2cc06b4d0 0x7fa2cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.473+0000 7fa2ddffb640 1 -- 172.21.15.64:0/884544129 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fa2e4089f20 con 0x7fa2e80b95a0 2024-03-04T07:33:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.473+0000 7fa2ecfeb640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa2cc06b4d0 0x7fa2cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:10.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.474+0000 7fa2ecfeb640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa2cc06b4d0 0x7fa2cc06d990 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7fa2d8036000 tx=0x7fa2d8008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:10.477 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.477+0000 7fa2ddffb640 1 -- 172.21.15.64:0/884544129 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa2e4058d40 con 0x7fa2e80b95a0 2024-03-04T07:33:10.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.696+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa2e80687a0 con 0x7fa2e80b95a0 2024-03-04T07:33:10.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.697+0000 7fa2ddffb640 1 -- 172.21.15.64:0/884544129 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7fa2e4058640 con 0x7fa2e80b95a0 2024-03-04T07:33:10.697 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:10.697 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:10.697 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:10.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa2cc06b4d0 msgr2=0x7fa2cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa2cc06b4d0 0x7fa2cc06d990 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7fa2d8036000 tx=0x7fa2d8008030 comp rx=0 tx=0).stop 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 msgr2=0x7fa2e80c94d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80c94d0 secure :-1 s=READY pgs=1419 cs=0 l=1 rev1=1 crypto rx=0x7fa2e400c430 tx=0x7fa2e400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 shutdown_connections 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa2cc06b4d0 0x7fa2cc06d990 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2e815df00 0x7fa2e81f7280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2e80b9a10 0x7fa2e81f6d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 --2- 172.21.15.64:0/884544129 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2e80b95a0 0x7fa2e80c94d0 unknown :-1 s=CLOSED pgs=1419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 >> 172.21.15.64:0/884544129 conn(0x7fa2e8071960 msgr2=0x7fa2e80b8530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 shutdown_connections 2024-03-04T07:33:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:10.699+0000 7fa2efa50640 1 -- 172.21.15.64:0/884544129 wait complete. 2024-03-04T07:33:10.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:33:10.831 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-04T07:33:10.831 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-04T07:33:10.832 DEBUG:tasks.ceph_test_case:wait_until_equal: 0 != 1, waiting (timeout=60)... 2024-03-04T07:33:15.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:15.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.935+0000 7f36f76a4640 1 Processor -- start 2024-03-04T07:33:15.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.936+0000 7f36f76a4640 1 -- start start 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.936+0000 7f36f76a4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f36f00aee10 0x7f36f0165a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.936+0000 7f36f76a4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36f0071240 con 0x7f36f0158520 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.936+0000 7f36f76a4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36f0071520 con 0x7f36f006df30 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.936+0000 7f36f76a4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f36f0071b70 con 0x7f36f00aee10 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.937+0000 7f36f5c41640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f36f00aee10 0x7f36f0165a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.937+0000 7f36f5440640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f36f0158520 0x7f36f006d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36324/0 (socket says 172.21.15.64:36324) 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.937+0000 7f36f5440640 1 -- 172.21.15.64:0/3370437293 learned_addr learned my addr 172.21.15.64:0/3370437293 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.937+0000 7f36f5c41640 1 -- 172.21.15.64:0/3370437293 >> v1:172.21.15.66:6789/0 conn(0x7f36f006df30 legacy=0x7f36f00ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.937+0000 7f36f5c41640 1 -- 172.21.15.64:0/3370437293 >> v1:172.21.15.64:6789/0 conn(0x7f36f0158520 legacy=0x7f36f006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:15.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.937+0000 7f36f5c41640 1 -- 172.21.15.64:0/3370437293 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36f0071800 con 0x7f36f00aee10 2024-03-04T07:33:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f5c41640 1 --2- 172.21.15.64:0/3370437293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f00aee10 0x7f36f0165a00 secure :-1 s=READY pgs=1421 cs=0 l=1 rev1=1 crypto rx=0x7f36e8009d90 tx=0x7f36e802be20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2ca7f80bdc7c8893 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:15.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36e7fff640 1 -- 172.21.15.64:0/3370437293 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36e8039070 con 0x7f36f00aee10 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36e7fff640 1 -- 172.21.15.64:0/3370437293 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36e802cbd0 con 0x7f36f00aee10 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36e7fff640 1 -- 172.21.15.64:0/3370437293 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36e8034440 con 0x7f36f00aee10 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f76a4640 1 -- 172.21.15.64:0/3370437293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f00aee10 msgr2=0x7f36f0165a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3370437293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f00aee10 0x7f36f0165a00 secure :-1 s=READY pgs=1421 cs=0 l=1 rev1=1 crypto rx=0x7f36e8009d90 tx=0x7f36e802be20 comp rx=0 tx=0).stop 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f76a4640 1 -- 172.21.15.64:0/3370437293 shutdown_connections 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3370437293 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f00aee10 0x7f36f0165a00 unknown :-1 s=CLOSED pgs=1421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f76a4640 1 -- 172.21.15.64:0/3370437293 >> 172.21.15.64:0/3370437293 conn(0x7f36f0066d00 msgr2=0x7f36f00670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f76a4640 1 -- 172.21.15.64:0/3370437293 shutdown_connections 2024-03-04T07:33:15.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.938+0000 7f36f76a4640 1 -- 172.21.15.64:0/3370437293 wait complete. 2024-03-04T07:33:15.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.939+0000 7f36f76a4640 1 Processor -- start 2024-03-04T07:33:15.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f76a4640 1 -- start start 2024-03-04T07:33:15.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f76a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36f006df30 0x7f36f0157780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f76a4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36f00aee10 0x7f36f01435a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f76a4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f0158520 0x7f36f015c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f76a4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36f0158900 con 0x7f36f0158520 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f76a4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36f0072000 con 0x7f36f00aee10 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f76a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36f0158a80 con 0x7f36f006df30 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f5c41640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f0158520 0x7f36f015c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f5c41640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f0158520 0x7f36f015c3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:36998/0 (socket says 172.21.15.64:36998) 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f5c41640 1 -- 172.21.15.64:0/3014556084 learned_addr learned my addr 172.21.15.64:0/3014556084 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f5440640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36f006df30 0x7f36f0157780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.940+0000 7f36f4c3f640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36f00aee10 0x7f36f01435a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f5c41640 1 -- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36f006df30 msgr2=0x7f36f0157780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f5c41640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36f006df30 0x7f36f0157780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f5c41640 1 -- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36f00aee10 msgr2=0x7f36f01435a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f5c41640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36f00aee10 0x7f36f01435a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f5c41640 1 -- 172.21.15.64:0/3014556084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36f01566e0 con 0x7f36f0158520 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f5440640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36f006df30 0x7f36f0157780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f5c41640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f0158520 0x7f36f015c3f0 secure :-1 s=READY pgs=1422 cs=0 l=1 rev1=1 crypto rx=0x7f36e8009d90 tx=0x7f36e8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36e5ffb640 1 -- 172.21.15.64:0/3014556084 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36e8044030 con 0x7f36f0158520 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36e5ffb640 1 -- 172.21.15.64:0/3014556084 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36e802c9d0 con 0x7f36f0158520 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36f01569c0 con 0x7f36f0158520 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36e5ffb640 1 -- 172.21.15.64:0/3014556084 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36e8039040 con 0x7f36f0158520 2024-03-04T07:33:15.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.941+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f36f0156f80 con 0x7f36f0158520 2024-03-04T07:33:15.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.942+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f36f0067ff0 con 0x7f36f0158520 2024-03-04T07:33:15.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.943+0000 7f36e5ffb640 1 -- 172.21.15.64:0/3014556084 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f36e8047070 con 0x7f36f0158520 2024-03-04T07:33:15.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.944+0000 7f36e5ffb640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36d406b4d0 0x7f36d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:15.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.944+0000 7f36e5ffb640 1 -- 172.21.15.64:0/3014556084 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f36e80acb60 con 0x7f36f0158520 2024-03-04T07:33:15.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.944+0000 7f36f5440640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36d406b4d0 0x7f36d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:15.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.944+0000 7f36f5440640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36d406b4d0 0x7f36d406d990 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f36ec000f60 tx=0x7f36ec009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:15.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:15.948+0000 7f36e5ffb640 1 -- 172.21.15.64:0/3014556084 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f36e804e070 con 0x7f36f0158520 2024-03-04T07:33:16.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.232+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f36f00b31f0 con 0x7f36f0158520 2024-03-04T07:33:16.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.232+0000 7f36e5ffb640 1 -- 172.21.15.64:0/3014556084 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f36e807ba40 con 0x7f36f0158520 2024-03-04T07:33:16.232 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:16.232 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:16.233 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:16.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36d406b4d0 msgr2=0x7f36d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36d406b4d0 0x7f36d406d990 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f36ec000f60 tx=0x7f36ec009040 comp rx=0 tx=0).stop 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f0158520 msgr2=0x7f36f015c3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f0158520 0x7f36f015c3f0 secure :-1 s=READY pgs=1422 cs=0 l=1 rev1=1 crypto rx=0x7f36e8009d90 tx=0x7f36e8004a40 comp rx=0 tx=0).stop 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 shutdown_connections 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36d406b4d0 0x7f36d406d990 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36f0158520 0x7f36f015c3f0 unknown :-1 s=CLOSED pgs=1422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36f00aee10 0x7f36f01435a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.234+0000 7f36f76a4640 1 --2- 172.21.15.64:0/3014556084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36f006df30 0x7f36f0157780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.235+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 >> 172.21.15.64:0/3014556084 conn(0x7f36f0066d00 msgr2=0x7f36f015b840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.235+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 shutdown_connections 2024-03-04T07:33:16.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.235+0000 7f36f76a4640 1 -- 172.21.15.64:0/3014556084 wait complete. 2024-03-04T07:33:16.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:33:16.363 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":1}} 2024-03-04T07:33:16.363 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 1 } } 2024-03-04T07:33:16.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:16.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.465+0000 7f324e7f4640 1 Processor -- start 2024-03-04T07:33:16.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.465+0000 7f324e7f4640 1 -- start start 2024-03-04T07:33:16.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f324e7f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 0x7f324806d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f324e7f4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f32480aee10 0x7f32480b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f324e7f4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3248071240 con 0x7f324806dd60 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f324e7f4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f324806e1c0 con 0x7f32480aee10 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f324e7f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3248071b70 con 0x7f3248158520 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f32477fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f324806dd60 0x7f32480ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36332/0 (socket says 172.21.15.64:36332) 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f32477fe640 1 -- 172.21.15.64:0/2706213284 learned_addr learned my addr 172.21.15.64:0/2706213284 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f324cd91640 1 --2- 172.21.15.64:0/2706213284 >> v2:172.21.15.66:3300/0 conn(0x7f32480aee10 0x7f32480b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f3247fff640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 0x7f324806d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3926643503 0 0) 0x7f3248071240 con 0x7f324806dd60 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.466+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3234003520 con 0x7f324806dd60 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2925757633 0 0) 0x7f3234003520 con 0x7f324806dd60 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 msgr2=0x7f324806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:16.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 0x7f324806d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 >> v2:172.21.15.66:3300/0 conn(0x7f32480aee10 msgr2=0x7f32480b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 --2- 172.21.15.64:0/2706213284 >> v2:172.21.15.66:3300/0 conn(0x7f32480aee10 0x7f32480b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3248071520 con 0x7f324806dd60 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f3230001670 con 0x7f324806dd60 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 >> v1:172.21.15.64:6789/0 conn(0x7f324806dd60 legacy=0x7f32480ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32340039b0 0x7f3234003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 0x7f3234004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3234004d40 0x7f32340051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3248071b70 con 0x7f3234004d40 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f323400b9c0 con 0x7f32340039b0 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f323400bb40 con 0x7f3248158520 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f324cd91640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3234004d40 0x7f32340051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3247fff640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32340039b0 0x7f3234003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f32477fe640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 0x7f3234004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f324cd91640 1 -- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 msgr2=0x7f3234004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f324cd91640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248158520 0x7f3234004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f324cd91640 1 -- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32340039b0 msgr2=0x7f3234003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f324cd91640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32340039b0 0x7f3234003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f324cd91640 1 -- 172.21.15.64:0/2706213284 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3234003520 con 0x7f3234004d40 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.467+0000 7f3247fff640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f32340039b0 0x7f3234003dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.468+0000 7f324cd91640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3234004d40 0x7f32340051c0 secure :-1 s=READY pgs=1424 cs=0 l=1 rev1=1 crypto rx=0x7f3240009fc0 tx=0x7f3234005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5f63ce80a5871cbc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.468+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3240039070 con 0x7f3234004d40 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.468+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f324002cc60 con 0x7f3234004d40 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.468+0000 7f3246ffd640 1 -- 172.21.15.64:0/2706213284 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3240009c10 con 0x7f3234004d40 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.469+0000 7f324e7f4640 1 -- 172.21.15.64:0/2706213284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3234004d40 msgr2=0x7f32340051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.469+0000 7f324e7f4640 1 --2- 172.21.15.64:0/2706213284 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3234004d40 0x7f32340051c0 secure :-1 s=READY pgs=1424 cs=0 l=1 rev1=1 crypto rx=0x7f3240009fc0 tx=0x7f3234005830 comp rx=0 tx=0).stop 2024-03-04T07:33:16.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.469+0000 7f324cd91640 1 -- 172.21.15.64:0/2706213284 reap_dead start 2024-03-04T07:33:16.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.469+0000 7f324e7f4640 1 -- 172.21.15.64:0/2706213284 shutdown_connections 2024-03-04T07:33:16.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.469+0000 7f324e7f4640 1 -- 172.21.15.64:0/2706213284 >> 172.21.15.64:0/2706213284 conn(0x7f3248066d00 msgr2=0x7f32480670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:16.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.469+0000 7f324e7f4640 1 -- 172.21.15.64:0/2706213284 shutdown_connections 2024-03-04T07:33:16.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.469+0000 7f324e7f4640 1 -- 172.21.15.64:0/2706213284 wait complete. 2024-03-04T07:33:16.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.470+0000 7f324e7f4640 1 Processor -- start 2024-03-04T07:33:16.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.470+0000 7f324e7f4640 1 -- start start 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324e7f4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3234004d40 0x7f32481f4080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324e7f4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32481e36a0 0x7f3248143580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324e7f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248155830 0x7f32480b3640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324e7f4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3248158900 con 0x7f32481e36a0 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324e7f4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3248072000 con 0x7f3234004d40 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324e7f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3248158a80 con 0x7f3248155830 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32481e36a0 0x7f3248143580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32481e36a0 0x7f3248143580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37004/0 (socket says 172.21.15.64:37004) 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 -- 172.21.15.64:0/2353774589 learned_addr learned my addr 172.21.15.64:0/2353774589 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f3247fff640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3234004d40 0x7f32481f4080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324cd91640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248155830 0x7f32480b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 -- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248155830 msgr2=0x7f32480b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248155830 0x7f32480b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 -- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3234004d40 msgr2=0x7f32481f4080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3234004d40 0x7f32481f4080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 -- 172.21.15.64:0/2353774589 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32481f48b0 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f324cd91640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248155830 0x7f32480b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.471+0000 7f32477fe640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32481e36a0 0x7f3248143580 secure :-1 s=READY pgs=1425 cs=0 l=1 rev1=1 crypto rx=0x7f3230004bd0 tx=0x7f3230007860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.472+0000 7f3244ff9640 1 -- 172.21.15.64:0/2353774589 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f323001c070 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.472+0000 7f3244ff9640 1 -- 172.21.15.64:0/2353774589 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3230011d30 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.472+0000 7f3244ff9640 1 -- 172.21.15.64:0/2353774589 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3230010040 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.472+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32481f4b90 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.472+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3248154380 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.473+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3248067ff0 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.473+0000 7f3244ff9640 1 -- 172.21.15.64:0/2353774589 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f323001f070 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.474+0000 7f3244ff9640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f322806b500 0x7f322806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.474+0000 7f3244ff9640 1 -- 172.21.15.64:0/2353774589 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f323008c2e0 con 0x7f32481e36a0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.474+0000 7f3247fff640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f322806b500 0x7f322806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.474+0000 7f3247fff640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f322806b500 0x7f322806d9c0 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f3238009ce0 tx=0x7f3238009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:16.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.482+0000 7f3244ff9640 1 -- 172.21.15.64:0/2353774589 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f323005b100 con 0x7f32481e36a0 2024-03-04T07:33:16.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.704+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f32480d07c0 con 0x7f32481e36a0 2024-03-04T07:33:16.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.705+0000 7f3244ff9640 1 -- 172.21.15.64:0/2353774589 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f323005aa00 con 0x7f32481e36a0 2024-03-04T07:33:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:16.705 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:16.705 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f322806b500 msgr2=0x7f322806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f322806b500 0x7f322806d9c0 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7f3238009ce0 tx=0x7f3238009190 comp rx=0 tx=0).stop 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32481e36a0 msgr2=0x7f3248143580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32481e36a0 0x7f3248143580 secure :-1 s=READY pgs=1425 cs=0 l=1 rev1=1 crypto rx=0x7f3230004bd0 tx=0x7f3230007860 comp rx=0 tx=0).stop 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 shutdown_connections 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f322806b500 0x7f322806d9c0 unknown :-1 s=CLOSED pgs=640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3248155830 0x7f32480b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f32481e36a0 0x7f3248143580 unknown :-1 s=CLOSED pgs=1425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 --2- 172.21.15.64:0/2353774589 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3234004d40 0x7f32481f4080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 >> 172.21.15.64:0/2353774589 conn(0x7f3248066d00 msgr2=0x7f32480b0390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 shutdown_connections 2024-03-04T07:33:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.707+0000 7f324e7f4640 1 -- 172.21.15.64:0/2353774589 wait complete. 2024-03-04T07:33:16.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump purge_queue pq_executed 2024-03-04T07:33:16.834 INFO:teuthology.orchestra.run.smithi064.stdout:{"purge_queue":{"pq_executed":1}} 2024-03-04T07:33:16.834 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 1 } } 2024-03-04T07:33:16.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:16.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.936+0000 7fa7ba5a0640 1 Processor -- start 2024-03-04T07:33:16.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.936+0000 7fa7ba5a0640 1 -- start start 2024-03-04T07:33:16.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.936+0000 7fa7ba5a0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa7b40d0ea0 0x7fa7b40b7350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.936+0000 7fa7ba5a0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa7b415e100 con 0x7fa7b415dd20 2024-03-04T07:33:16.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.936+0000 7fa7ba5a0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa7b40d22d0 con 0x7fa7b40d0a30 2024-03-04T07:33:16.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.936+0000 7fa7ba5a0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa7b40d3190 con 0x7fa7b40d0ea0 2024-03-04T07:33:16.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7b959e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa7b415dd20 0x7fa7b40d1bc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36334/0 (socket says 172.21.15.64:36334) 2024-03-04T07:33:16.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7b959e640 1 -- 172.21.15.64:0/4115689018 learned_addr learned my addr 172.21.15.64:0/4115689018 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7b9d9f640 1 --2- 172.21.15.64:0/4115689018 >> v2:172.21.15.66:3300/0 conn(0x7fa7b40d0ea0 0x7fa7b40b7350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4101028305 0 0) 0x7fa7b415e100 con 0x7fa7b415dd20 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa7a0003520 con 0x7fa7b415dd20 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2421253581 0 0) 0x7fa7a0003520 con 0x7fa7b415dd20 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 >> v2:172.21.15.66:3300/0 conn(0x7fa7b40d0ea0 msgr2=0x7fa7b40b7350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 --2- 172.21.15.64:0/4115689018 >> v2:172.21.15.66:3300/0 conn(0x7fa7b40d0ea0 0x7fa7b40b7350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 >> v1:172.21.15.66:6789/0 conn(0x7fa7b40d0a30 legacy=0x7fa7b40b6c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7b40d2580 con 0x7fa7b415dd20 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fa7b0001670 con 0x7fa7b415dd20 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 >> v1:172.21.15.64:6789/0 conn(0x7fa7b415dd20 legacy=0x7fa7b40d1bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7a0003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7a0004800 0x7fa7a0004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7a00051c0 0x7fa7a000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7b40d3190 con 0x7fa7a00039b0 2024-03-04T07:33:16.939 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a000c060 con 0x7fa7a00051c0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a000c260 con 0x7fa7a0004800 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.937+0000 7fa7b959e640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7a0003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b9d9f640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7a00051c0 0x7fa7a000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b8d9d640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7a0004800 0x7fa7a0004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b959e640 1 -- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7a0004800 msgr2=0x7fa7a0004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b959e640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7a0004800 0x7fa7a0004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b959e640 1 -- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7a00051c0 msgr2=0x7fa7a000b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b959e640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7a00051c0 0x7fa7a000b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b959e640 1 -- 172.21.15.64:0/4115689018 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7a0003520 con 0x7fa7a00039b0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b9d9f640 1 -- 172.21.15.64:0/4115689018 reap_dead start 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.938+0000 7fa7b959e640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7a0003dd0 secure :-1 s=READY pgs=1427 cs=0 l=1 rev1=1 crypto rx=0x7fa7b000c240 tx=0x7fa7a0005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=95d1c6522721aeec server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7b0007d10 con 0x7fa7a00039b0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa7b0030070 con 0x7fa7a00039b0 2024-03-04T07:33:16.940 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7abfff640 1 -- 172.21.15.64:0/4115689018 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7b0005d90 con 0x7fa7a00039b0 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/4115689018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 msgr2=0x7fa7a0003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7a0003dd0 secure :-1 s=READY pgs=1427 cs=0 l=1 rev1=1 crypto rx=0x7fa7b000c240 tx=0x7fa7a0005830 comp rx=0 tx=0).stop 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/4115689018 shutdown_connections 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/4115689018 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7a0003dd0 unknown :-1 s=CLOSED pgs=1427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/4115689018 >> 172.21.15.64:0/4115689018 conn(0x7fa7b4071960 msgr2=0x7fa7b4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/4115689018 shutdown_connections 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.939+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/4115689018 wait complete. 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.940+0000 7fa7ba5a0640 1 Processor -- start 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.940+0000 7fa7ba5a0640 1 -- start start 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7ba5a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7b4159aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7ba5a0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7b40c8d70 0x7fa7b40c91f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7ba5a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7b40cbad0 0x7fa7b40cbf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7ba5a0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7b40d2e20 con 0x7fa7a00039b0 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7ba5a0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7b40d2b40 con 0x7fa7b40c8d70 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7ba5a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa7b40d12a0 con 0x7fa7b40cbad0 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7b959e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7b4159aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7b959e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7b4159aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37012/0 (socket says 172.21.15.64:37012) 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7b959e640 1 -- 172.21.15.64:0/2601059378 learned_addr learned my addr 172.21.15.64:0/2601059378 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7b8d9d640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7b40c8d70 0x7fa7b40c91f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.941+0000 7fa7b9d9f640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7b40cbad0 0x7fa7b40cbf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7b959e640 1 -- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7b40cbad0 msgr2=0x7fa7b40cbf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7b959e640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7b40cbad0 0x7fa7b40cbf50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7b959e640 1 -- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7b40c8d70 msgr2=0x7fa7b40c91f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7b959e640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7b40c8d70 0x7fa7b40c91f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7b959e640 1 -- 172.21.15.64:0/2601059378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7b4160240 con 0x7fa7a00039b0 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7b9d9f640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7b40cbad0 0x7fa7b40cbf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7b959e640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7b4159aa0 secure :-1 s=READY pgs=1428 cs=0 l=1 rev1=1 crypto rx=0x7fa7b000c360 tx=0x7fa7b40d2750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:16.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7a9ffb640 1 -- 172.21.15.64:0/2601059378 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7b0045070 con 0x7fa7a00039b0 2024-03-04T07:33:16.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7a9ffb640 1 -- 172.21.15.64:0/2601059378 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa7b00049f0 con 0x7fa7a00039b0 2024-03-04T07:33:16.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7b4160520 con 0x7fa7a00039b0 2024-03-04T07:33:16.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7a9ffb640 1 -- 172.21.15.64:0/2601059378 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7b0007d10 con 0x7fa7a00039b0 2024-03-04T07:33:16.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.942+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7b4160ae0 con 0x7fa7a00039b0 2024-03-04T07:33:16.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.943+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa7b404f330 con 0x7fa7a00039b0 2024-03-04T07:33:16.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.944+0000 7fa7a9ffb640 1 -- 172.21.15.64:0/2601059378 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa7b0007070 con 0x7fa7a00039b0 2024-03-04T07:33:16.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.944+0000 7fa7a9ffb640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa79806b4d0 0x7fa79806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:16.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.945+0000 7fa7b8d9d640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa79806b4d0 0x7fa79806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:16.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.945+0000 7fa7a9ffb640 1 -- 172.21.15.64:0/2601059378 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fa7b00aae80 con 0x7fa7a00039b0 2024-03-04T07:33:16.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.945+0000 7fa7b8d9d640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa79806b4d0 0x7fa79806d990 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7fa7a4000c00 tx=0x7fa7a4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:16.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:16.949+0000 7fa7a9ffb640 1 -- 172.21.15.64:0/2601059378 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa7b0079650 con 0x7fa7a00039b0 2024-03-04T07:33:17.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.165+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa7b40b4650 con 0x7fa7a00039b0 2024-03-04T07:33:17.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.166+0000 7fa7a9ffb640 1 -- 172.21.15.64:0/2601059378 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7fa7b007d6f0 con 0x7fa7a00039b0 2024-03-04T07:33:17.166 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:17.166 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:17.166 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa79806b4d0 msgr2=0x7fa79806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa79806b4d0 0x7fa79806d990 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7fa7a4000c00 tx=0x7fa7a4009040 comp rx=0 tx=0).stop 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 msgr2=0x7fa7b4159aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7b4159aa0 secure :-1 s=READY pgs=1428 cs=0 l=1 rev1=1 crypto rx=0x7fa7b000c360 tx=0x7fa7b40d2750 comp rx=0 tx=0).stop 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 shutdown_connections 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa79806b4d0 0x7fa79806d990 unknown :-1 s=CLOSED pgs=641 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa7b40cbad0 0x7fa7b40cbf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa7b40c8d70 0x7fa7b40c91f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 --2- 172.21.15.64:0/2601059378 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa7a00039b0 0x7fa7b4159aa0 unknown :-1 s=CLOSED pgs=1428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 >> 172.21.15.64:0/2601059378 conn(0x7fa7b4071960 msgr2=0x7fa7b40b6c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:17.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 shutdown_connections 2024-03-04T07:33:17.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.168+0000 7fa7ba5a0640 1 -- 172.21.15.64:0/2601059378 wait complete. 2024-03-04T07:33:17.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache 2024-03-04T07:33:17.296 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"dir_update":0,"dir_update_receipt":0,"dir_try_discover":0,"dir_send_discover":0,"dir_handle_discover":0,"num_strays":0,"num_strays_delayed":0,"num_strays_enqueuing":0,"strays_created":3,"strays_enqueued":1,"strays_reintegrated":2,"strays_migrated":0,"num_recovering_processing":0,"num_recovering_enqueued":0,"num_recovering_prioritized":0,"recovery_started":0,"recovery_completed":0,"ireq_enqueue_scrub":0,"ireq_exportdir":0,"ireq_flush":0,"ireq_fragmentdir":0,"ireq_fragstats":0,"ireq_inodestats":0}} 2024-03-04T07:33:17.296 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "dir_handle_discover": 0, "dir_send_discover": 0, "dir_try_discover": 0, "dir_update": 0, "dir_update_receipt": 0, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 0, "ireq_fragstats": 0, "ireq_inodestats": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "num_recovering_processing": 0, "num_strays": 0, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "recovery_completed": 0, "recovery_started": 0, "strays_created": 3, "strays_enqueued": 1, "strays_migrated": 0, "strays_reintegrated": 2 } } 2024-03-04T07:33:17.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:17.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.398+0000 7eff96e66640 1 Processor -- start 2024-03-04T07:33:17.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.399+0000 7eff96e66640 1 -- start start 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.399+0000 7eff96e66640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff900b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.399+0000 7eff96e66640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7eff900b2b50 0x7eff900b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.399+0000 7eff96e66640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7eff900ce610 con 0x7eff900d3400 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.399+0000 7eff96e66640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7eff900d2f70 con 0x7eff900b2b50 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.399+0000 7eff96e66640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff900d4be0 con 0x7eff90071540 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff95403640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7eff900b2b50 0x7eff900b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff94c02640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff900b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff8ffff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7eff900d3400 0x7eff90071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:42188/0 (socket says 172.21.15.64:42188) 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff95403640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7eff900b2b50 0x7eff900b6e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:43400/0 (socket says 172.21.15.64:43400) 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff8ffff640 1 -- 172.21.15.64:0/3983223743 learned_addr learned my addr 172.21.15.64:0/3983223743 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff94c02640 1 -- 172.21.15.64:0/3983223743 >> v2:172.21.15.66:3300/0 conn(0x7eff900b2b50 msgr2=0x7eff900b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:17.400 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff94c02640 1 --2- 172.21.15.64:0/3983223743 >> v2:172.21.15.66:3300/0 conn(0x7eff900b2b50 0x7eff900b6e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff94c02640 1 -- 172.21.15.64:0/3983223743 >> v1:172.21.15.66:6789/0 conn(0x7eff900d3400 legacy=0x7eff90071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff94c02640 1 -- 172.21.15.64:0/3983223743 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff900d2920 con 0x7eff90071540 2024-03-04T07:33:17.401 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.400+0000 7eff95403640 1 --2- 172.21.15.64:0/3983223743 >> v2:172.21.15.66:3300/0 conn(0x7eff900b2b50 0x7eff900b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.401+0000 7eff94c02640 1 --2- 172.21.15.64:0/3983223743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff900b30a0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7eff840097a0 tx=0x7eff8402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b08ab83d84637c8e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.401+0000 7eff8f7fe640 1 -- 172.21.15.64:0/3983223743 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff84035020 con 0x7eff90071540 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.401+0000 7eff8f7fe640 1 -- 172.21.15.64:0/3983223743 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff84031830 con 0x7eff90071540 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.401+0000 7eff8f7fe640 1 -- 172.21.15.64:0/3983223743 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff84031c00 con 0x7eff90071540 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 -- 172.21.15.64:0/3983223743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 msgr2=0x7eff900b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 --2- 172.21.15.64:0/3983223743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff900b30a0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7eff840097a0 tx=0x7eff8402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 -- 172.21.15.64:0/3983223743 shutdown_connections 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 --2- 172.21.15.64:0/3983223743 >> v2:172.21.15.66:3300/0 conn(0x7eff900b2b50 0x7eff900b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 --2- 172.21.15.64:0/3983223743 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff900b30a0 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 -- 172.21.15.64:0/3983223743 >> 172.21.15.64:0/3983223743 conn(0x7eff900d3ca0 msgr2=0x7eff900d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 -- 172.21.15.64:0/3983223743 shutdown_connections 2024-03-04T07:33:17.402 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.402+0000 7eff96e66640 1 -- 172.21.15.64:0/3983223743 wait complete. 2024-03-04T07:33:17.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.403+0000 7eff96e66640 1 Processor -- start 2024-03-04T07:33:17.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.403+0000 7eff96e66640 1 -- start start 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff96e66640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff9015ccd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff96e66640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff900b2b50 0x7eff900ae380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff96e66640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff900d3400 0x7eff900ccf40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff96e66640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff9015e370 con 0x7eff900b2b50 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff96e66640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff900ce8f0 con 0x7eff900d3400 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff96e66640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff900aa840 con 0x7eff90071540 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff900b2b50 0x7eff900ae380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff900b2b50 0x7eff900ae380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37026/0 (socket says 172.21.15.64:37026) 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 -- 172.21.15.64:0/3612067365 learned_addr learned my addr 172.21.15.64:0/3612067365 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff94c02640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff9015ccd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff95403640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff900d3400 0x7eff900ccf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 -- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 msgr2=0x7eff9015ccd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff9015ccd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 -- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff900d3400 msgr2=0x7eff900ccf40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff900d3400 0x7eff900ccf40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.404+0000 7eff8ffff640 1 -- 172.21.15.64:0/3612067365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff900b27d0 con 0x7eff900b2b50 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.405+0000 7eff8ffff640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff900b2b50 0x7eff900ae380 secure :-1 s=READY pgs=1429 cs=0 l=1 rev1=1 crypto rx=0x7eff7c00c440 tx=0x7eff7c00c900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.405+0000 7eff8d7fa640 1 -- 172.21.15.64:0/3612067365 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff7c017070 con 0x7eff900b2b50 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.405+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff900b1f30 con 0x7eff900b2b50 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.405+0000 7eff8d7fa640 1 -- 172.21.15.64:0/3612067365 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff7c005ce0 con 0x7eff900b2b50 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.405+0000 7eff8d7fa640 1 -- 172.21.15.64:0/3612067365 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff7c0072f0 con 0x7eff900b2b50 2024-03-04T07:33:17.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.405+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff9015d210 con 0x7eff900b2b50 2024-03-04T07:33:17.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.406+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7eff90068700 con 0x7eff900b2b50 2024-03-04T07:33:17.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.406+0000 7eff8d7fa640 1 -- 172.21.15.64:0/3612067365 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7eff7c01a070 con 0x7eff900b2b50 2024-03-04T07:33:17.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.407+0000 7eff8d7fa640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7406b500 0x7eff7406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.407+0000 7eff8d7fa640 1 -- 172.21.15.64:0/3612067365 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7eff7c089630 con 0x7eff900b2b50 2024-03-04T07:33:17.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.407+0000 7eff94c02640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7406b500 0x7eff7406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.407+0000 7eff94c02640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7406b500 0x7eff7406d9c0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7eff84036000 tx=0x7eff84008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.411+0000 7eff8d7fa640 1 -- 172.21.15.64:0/3612067365 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7eff7c058340 con 0x7eff900b2b50 2024-03-04T07:33:17.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.630+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7eff900b22c0 con 0x7eff900b2b50 2024-03-04T07:33:17.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.631+0000 7eff8d7fa640 1 -- 172.21.15.64:0/3612067365 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7eff7c057c40 con 0x7eff900b2b50 2024-03-04T07:33:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:17.631 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:17.631 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7406b500 msgr2=0x7eff7406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7406b500 0x7eff7406d9c0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7eff84036000 tx=0x7eff84008030 comp rx=0 tx=0).stop 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff900b2b50 msgr2=0x7eff900ae380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff900b2b50 0x7eff900ae380 secure :-1 s=READY pgs=1429 cs=0 l=1 rev1=1 crypto rx=0x7eff7c00c440 tx=0x7eff7c00c900 comp rx=0 tx=0).stop 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 shutdown_connections 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff7406b500 0x7eff7406d9c0 unknown :-1 s=CLOSED pgs=642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff900d3400 0x7eff900ccf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff900b2b50 0x7eff900ae380 unknown :-1 s=CLOSED pgs=1429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 --2- 172.21.15.64:0/3612067365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff90071540 0x7eff9015ccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 >> 172.21.15.64:0/3612067365 conn(0x7eff900d3ca0 msgr2=0x7eff9015f4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 shutdown_connections 2024-03-04T07:33:17.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.633+0000 7eff96e66640 1 -- 172.21.15.64:0/3612067365 wait complete. 2024-03-04T07:33:17.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump purge_queue 2024-03-04T07:33:17.760 INFO:teuthology.orchestra.run.smithi064.stdout:{"purge_queue":{"pq_executing_ops":0,"pq_executing_ops_high_water":2,"pq_executing":0,"pq_executing_high_water":1,"pq_executed_ops":2,"pq_executed":1,"pq_item_in_journal":0}} 2024-03-04T07:33:17.761 DEBUG:tasks.cephfs.filesystem:_json_asok output { "purge_queue": { "pq_executed": 1, "pq_executed_ops": 2, "pq_executing": 0, "pq_executing_high_water": 1, "pq_executing_ops": 0, "pq_executing_ops_high_water": 2, "pq_item_in_journal": 0 } } 2024-03-04T07:33:17.761 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --cluster ceph --pool cephfs_data ls 2024-03-04T07:33:17.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.803+0000 7fb31363df40 1 Processor -- start 2024-03-04T07:33:17.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.803+0000 7fb31363df40 1 -- start start 2024-03-04T07:33:17.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb31363df40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d977a00 0x560d0da45a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb31363df40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0d9be640 0x560d0da47ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb31363df40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0da49d20 0x560d0da3dc10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb31363df40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d0d9bcc50 con 0x560d0d977a00 2024-03-04T07:33:17.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb31363df40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d0d91bb70 con 0x560d0d9be640 2024-03-04T07:33:17.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb31363df40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d0d9bcdd0 con 0x560d0da49d20 2024-03-04T07:33:17.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb310949640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d977a00 0x560d0da45a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb310949640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d977a00 0x560d0da45a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37030/0 (socket says 172.21.15.64:37030) 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.804+0000 7fb310949640 1 -- 172.21.15.64:0/1692710352 learned_addr learned my addr 172.21.15.64:0/1692710352 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb310949640 1 -- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0da49d20 msgr2=0x560d0da3dc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb310949640 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0da49d20 0x560d0da3dc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb310949640 1 -- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0d9be640 msgr2=0x560d0da47ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb30bfff640 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0d9be640 0x560d0da47ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb310949640 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0d9be640 0x560d0da47ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb310949640 1 -- 172.21.15.64:0/1692710352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d0da541b0 con 0x560d0d977a00 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb30bfff640 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0d9be640 0x560d0da47ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb310949640 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d977a00 0x560d0da45a60 secure :-1 s=READY pgs=1432 cs=0 l=1 rev1=1 crypto rx=0x7fb304006ec0 tx=0x7fb30400b6c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8f342148121bd53f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb30affd640 1 -- 172.21.15.64:0/1692710352 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb304012070 con 0x560d0d977a00 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb30affd640 1 -- 172.21.15.64:0/1692710352 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb30400e690 con 0x560d0d977a00 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.805+0000 7fb30affd640 1 -- 172.21.15.64:0/1692710352 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb30400eb00 con 0x560d0d977a00 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 -- 172.21.15.64:0/1692710352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d977a00 msgr2=0x560d0da45a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d977a00 0x560d0da45a60 secure :-1 s=READY pgs=1432 cs=0 l=1 rev1=1 crypto rx=0x7fb304006ec0 tx=0x7fb30400b6c0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 -- 172.21.15.64:0/1692710352 shutdown_connections 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0da49d20 0x560d0da3dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0d9be640 0x560d0da47ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 --2- 172.21.15.64:0/1692710352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d977a00 0x560d0da45a60 unknown :-1 s=CLOSED pgs=1432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 -- 172.21.15.64:0/1692710352 >> 172.21.15.64:0/1692710352 conn(0x560d0d9745d0 msgr2=0x560d0d9749b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 -- 172.21.15.64:0/1692710352 shutdown_connections 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.806+0000 7fb31363df40 1 -- 172.21.15.64:0/1692710352 wait complete. 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.807+0000 7fb31363df40 1 Processor -- start 2024-03-04T07:33:17.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.807+0000 7fb31363df40 1 -- start start 2024-03-04T07:33:17.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb31363df40 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0d977a00 0x560d0da2bff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb31363df40 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d9be640 0x560d0da3dc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb31363df40 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0da49d20 0x560d0da3e150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb31363df40 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d0da53f10 con 0x560d0d9be640 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb31363df40 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d0da45fa0 con 0x560d0da49d20 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb31363df40 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x560d0d99ec60 con 0x560d0d977a00 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d9be640 0x560d0da3dc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d9be640 0x560d0da3dc10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37032/0 (socket says 172.21.15.64:37032) 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 learned_addr learned my addr 172.21.15.64:0/751135854 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0d977a00 0x560d0da2bff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0d977a00 msgr2=0x560d0da2bff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0da49d20 0x560d0da3e150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x560d0d977a00 0x560d0da2bff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0da49d20 msgr2=0x560d0da3e150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x560d0da49d20 0x560d0da3e150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d0da4e940 con 0x560d0d9be640 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.808+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d9be640 0x560d0da3dc10 secure :-1 s=READY pgs=1433 cs=0 l=1 rev1=1 crypto rx=0x7fb2fc000c00 tx=0x7fb2fc002fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.809+0000 7fb309ffb640 1 -- 172.21.15.64:0/751135854 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2fc062020 con 0x560d0d9be640 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.809+0000 7fb309ffb640 1 -- 172.21.15.64:0/751135854 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb2fc052a40 con 0x560d0d9be640 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.809+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x560d0da4e0a0 con 0x560d0d9be640 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.809+0000 7fb309ffb640 1 -- 172.21.15.64:0/751135854 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2fc057040 con 0x560d0d9be640 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.809+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560d0da4e660 con 0x560d0d9be640 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.810+0000 7fb309ffb640 1 -- 172.21.15.64:0/751135854 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb2fc065020 con 0x560d0d9be640 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.810+0000 7fb309ffb640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb2f4069ee0 0x7fb2f406c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.810+0000 7fb309ffb640 1 -- 172.21.15.64:0/751135854 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fb2fc0cac30 con 0x560d0d9be640 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.810+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb2f4069ee0 0x7fb2f406c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.811+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x560d0da4d660 0x560d0daee7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.811+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:1 17.0 17:00000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x560d0da4d660 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.811+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb2f4069ee0 0x7fb2f406c3a0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fb30400b700 tx=0x7fb304019040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.811+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x560d0da4d660 0x560d0daee7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.811+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x560d0da4d660 0x560d0daee7e0 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.812+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 1 ==== osd_op_reply(1 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.812+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7fb30c0107d0 0x7fb30c00f250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.812+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:2 17.20 17:04000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb30c0107d0 2024-03-04T07:33:17.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.812+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7fb30c0107d0 0x7fb30c00f250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.812+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7fb30c0107d0 0x7fb30c00f250 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.1 v2:172.21.15.64:6801/1190712954 1 ==== osd_op_reply(2 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c0107d0 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7fb304017070 0x7fb304022410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:3 17.10 17:08000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb304017070 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7fb304017070 0x7fb304022410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7fb304017070 0x7fb304022410 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 1 ==== osd_op_reply(3 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7fb2fc067020 0x7fb2fc098210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.813+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:4 17.30 17:0c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb2fc067020 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.814+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7fb2fc067020 0x7fb2fc098210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.814+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7fb2fc067020 0x7fb2fc098210 crc :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.814+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== osd_op_reply(4 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.814+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:5 17.8 17:10000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x560d0da4d660 2024-03-04T07:33:17.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.815+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 2 ==== osd_op_reply(5 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.815+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:6 17.28 17:14000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb304017070 2024-03-04T07:33:17.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.815+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 2 ==== osd_op_reply(6 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.815+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:7 17.18 17:18000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb2fc067020 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 2 ==== osd_op_reply(7 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:8 17.38 17:1c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x560d0da4d660 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 3 ==== osd_op_reply(8 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:9 17.4 17:20000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb2fc067020 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 3 ==== osd_op_reply(9 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:10 17.24 17:24000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb304017070 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 3 ==== osd_op_reply(10 [pgnls start_epoch 111 out=49b] v105'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.816+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:11 17.14 17:28000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb304017070 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.817+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 4 ==== osd_op_reply(11 [pgnls start_epoch 111 out=49b] v105'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.817+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:12 17.34 17:2c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb2fc067020 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.817+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 4 ==== osd_op_reply(12 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.817+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:13 17.c 17:30000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x560d0da4d660 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.817+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 4 ==== osd_op_reply(13 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.817+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fb30c019970 0x7fb30c017440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.817+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:14 17.2c 17:34000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb30c019970 2024-03-04T07:33:17.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.818+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fb30c019970 0x7fb30c017440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.818+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fb30c019970 0x7fb30c017440 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.819+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 1 ==== osd_op_reply(14 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.819+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:15 17.1c 17:38000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb30c0107d0 2024-03-04T07:33:17.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.819+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.1 v2:172.21.15.64:6801/1190712954 2 ==== osd_op_reply(15 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c0107d0 2024-03-04T07:33:17.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.819+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:16 17.3c 17:3c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb2fc067020 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.819+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 5 ==== osd_op_reply(16 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.819+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:17 17.2 17:40000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb30c019970 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.820+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 2 ==== osd_op_reply(17 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.820+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7fb304037960 0x7fb3040363e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.820+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:18 17.22 17:44000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb304037960 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.820+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7fb304037960 0x7fb3040363e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.820+0000 7fb30bfff640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7fb304037960 0x7fb3040363e0 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.821+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.5 v2:172.21.15.66:6800/1583207443 1 ==== osd_op_reply(18 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304037960 2024-03-04T07:33:17.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.821+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:19 17.12 17:48000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x560d0da2c6a0 con 0x7fb304017070 2024-03-04T07:33:17.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.821+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 5 ==== osd_op_reply(19 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.821+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:20 17.32 17:4c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x560d0da4d660 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.821+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 5 ==== osd_op_reply(20 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.821+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:21 17.a 17:50000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x560d0da4d660 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 6 ==== osd_op_reply(21 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:22 17.2a 17:54000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304017070 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 6 ==== osd_op_reply(22 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:23 17.1a 17:58000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 6 ==== osd_op_reply(23 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:24 17.3a 17:5c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c0107d0 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.1 v2:172.21.15.64:6801/1190712954 3 ==== osd_op_reply(24 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c0107d0 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.822+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7fb30400d3d0 0x7fb30400d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.823+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:25 17.6 17:60000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30400d3d0 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.823+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7fb30400d3d0 0x7fb30400d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.823+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7fb30400d3d0 0x7fb30400d850 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.6 v2:172.21.15.66:6816/177421408 1 ==== osd_op_reply(25 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb30400d3d0 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:26 17.26 17:64000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 7 ==== osd_op_reply(26 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:27 17.16 17:68000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304037960 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.5 v2:172.21.15.66:6800/1583207443 2 ==== osd_op_reply(27 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304037960 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:28 17.36 17:6c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 8 ==== osd_op_reply(28 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb31114a640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7fb30c033910 0x7fb30c0253f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.824+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- osd_op(unknown.0.0:29 17.e 17:70000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c033910 2024-03-04T07:33:17.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.825+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7fb30c033910 0x7fb30c0253f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.825+0000 7fb310949640 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7fb30c033910 0x7fb30c0253f0 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.825+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.2 v2:172.21.15.64:6824/2610379167 1 ==== osd_op_reply(29 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c033910 2024-03-04T07:33:17.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.825+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:30 17.2e 17:74000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30400d3d0 2024-03-04T07:33:17.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.825+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.6 v2:172.21.15.66:6816/177421408 2 ==== osd_op_reply(30 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb30400d3d0 2024-03-04T07:33:17.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.825+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:31 17.1e 17:78000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c0107d0 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.826+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.1 v2:172.21.15.64:6801/1190712954 4 ==== osd_op_reply(31 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c0107d0 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.826+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:32 17.3e 17:7c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.826+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 3 ==== osd_op_reply(32 [pgnls start_epoch 111 out=49b] v105'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.826+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:33 17.1 17:80000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.826+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 9 ==== osd_op_reply(33 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.826+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:34 17.21 17:84000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 10 ==== osd_op_reply(34 [pgnls start_epoch 111 out=49b] v105'12 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:35 17.11 17:88000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304017070 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 7 ==== osd_op_reply(35 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:36 17.31 17:8c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c0107d0 2024-03-04T07:33:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.1 v2:172.21.15.64:6801/1190712954 5 ==== osd_op_reply(36 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c0107d0 2024-03-04T07:33:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:37 17.9 17:90000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304017070 2024-03-04T07:33:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 8 ==== osd_op_reply(37 [pgnls start_epoch 111 out=49b] v105'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.827+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:38 17.29 17:94000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.828+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 4 ==== osd_op_reply(38 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.828+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:39 17.19 17:98000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304037960 2024-03-04T07:33:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.828+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.5 v2:172.21.15.66:6800/1583207443 3 ==== osd_op_reply(39 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304037960 2024-03-04T07:33:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.828+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:40 17.39 17:9c000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304037960 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.829+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.5 v2:172.21.15.66:6800/1583207443 4 ==== osd_op_reply(40 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304037960 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.829+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:41 17.5 17:a0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30400d3d0 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.829+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.6 v2:172.21.15.66:6816/177421408 3 ==== osd_op_reply(41 [pgnls start_epoch 111 out=49b] v105'5 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb30400d3d0 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.829+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:42 17.25 17:a4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c0107d0 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.829+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.1 v2:172.21.15.64:6801/1190712954 6 ==== osd_op_reply(42 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c0107d0 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.829+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- osd_op(unknown.0.0:43 17.15 17:a8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304037960 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.830+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.5 v2:172.21.15.66:6800/1583207443 5 ==== osd_op_reply(43 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304037960 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.830+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:44 17.35 17:ac000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.830+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 5 ==== osd_op_reply(44 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.830+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:45 17.d 17:b0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.830+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 7 ==== osd_op_reply(45 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.830+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:46 17.2d 17:b4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.831+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 8 ==== osd_op_reply(46 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.831+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:47 17.1d 17:b8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.831+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 9 ==== osd_op_reply(47 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.831+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:48 17.3d 17:bc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.831+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 10 ==== osd_op_reply(48 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.831+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:49 17.3 17:c0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 6 ==== osd_op_reply(49 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:50 17.23 17:c4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 7 ==== osd_op_reply(50 [pgnls start_epoch 111 out=49b] v105'7 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- osd_op(unknown.0.0:51 17.13 17:c8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb304017070 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 <== osd.3 v2:172.21.15.64:6816/1402142346 9 ==== osd_op_reply(51 [pgnls start_epoch 111 out=49b] v105'15 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb2fc05a040 con 0x7fb304017070 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb30bfff640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- osd_op(unknown.0.0:52 17.33 17:cc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x560d0da4d660 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.7 v2:172.21.15.66:6802/3006518207 11 ==== osd_op_reply(52 [pgnls start_epoch 111 out=49b] v105'10 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x560d0da4d660 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.832+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:53 17.b 17:d0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.833+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 11 ==== osd_op_reply(53 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.833+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:54 17.2b 17:d4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.833+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 8 ==== osd_op_reply(54 [pgnls start_epoch 111 out=49b] v105'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.833+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:55 17.1b 17:d8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.833+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 9 ==== osd_op_reply(55 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.833+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- osd_op(unknown.0.0:56 17.3b 17:dc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c033910 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.834+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.2 v2:172.21.15.64:6824/2610379167 2 ==== osd_op_reply(56 [pgnls start_epoch 111 out=49b] v105'4 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c033910 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.834+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:57 17.7 17:e0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.834+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 12 ==== osd_op_reply(57 [pgnls start_epoch 111 out=49b] v111'12 uv12 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.835 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.834+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:58 17.27 17:e4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.835+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 10 ==== osd_op_reply(58 [pgnls start_epoch 111 out=49b] v105'2 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.835+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:59 17.17 17:e8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30400d3d0 2024-03-04T07:33:17.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.835+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.6 v2:172.21.15.66:6816/177421408 4 ==== osd_op_reply(59 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb30400d3d0 2024-03-04T07:33:17.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.835+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:60 17.37 17:ec000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.835+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 13 ==== osd_op_reply(60 [pgnls start_epoch 111 out=49b] v105'11 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.835+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- osd_op(unknown.0.0:61 17.f 17:f0000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c0107d0 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.836+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.1 v2:172.21.15.64:6801/1190712954 7 ==== osd_op_reply(61 [pgnls start_epoch 111 out=49b] v105'9 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c0107d0 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.836+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- osd_op(unknown.0.0:62 17.2f 17:f4000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30c019970 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.836+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 <== osd.4 v2:172.21.15.66:6824/1927515367 11 ==== osd_op_reply(62 [pgnls start_epoch 111 out=49b] v111'10 uv10 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb304014030 con 0x7fb30c019970 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.836+0000 7fb310949640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- osd_op(unknown.0.0:63 17.1f 17:f8000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb30400d3d0 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.836+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.6 v2:172.21.15.66:6816/177421408 5 ==== osd_op_reply(63 [pgnls start_epoch 111 out=49b] v105'6 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb30400d3d0 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.836+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- osd_op(unknown.0.0:64 17.3f 17:fc000000::::head [pgnls start_epoch 111 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected+supports_pool_eio e111) v9 -- 0x7fb2fc0bc040 con 0x7fb2fc067020 2024-03-04T07:33:17.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.836+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 <== osd.0 v2:172.21.15.64:6800/2111658274 14 ==== osd_op_reply(64 [pgnls start_epoch 111 out=49b] v105'8 uv0 ondisk = 1) v8 ==== 144+0+49 (crc 0 0 0) 0x7fb30c00d510 con 0x7fb2fc067020 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7fb2fc067020 msgr2=0x7fb2fc098210 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7fb2fc067020 0x7fb2fc098210 crc :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7fb30c0107d0 msgr2=0x7fb30c00f250 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7fb30c0107d0 0x7fb30c00f250 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7fb30c033910 msgr2=0x7fb30c0253f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7fb30c033910 0x7fb30c0253f0 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7fb304017070 msgr2=0x7fb304022410 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 reap_dead start 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7fb304017070 0x7fb304022410 crc :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fb30c019970 msgr2=0x7fb30c017440 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fb30c019970 0x7fb30c017440 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7fb304037960 msgr2=0x7fb3040363e0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7fb304037960 0x7fb3040363e0 crc :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7fb30400d3d0 msgr2=0x7fb30400d850 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7fb30400d3d0 0x7fb30400d850 crc :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x560d0da4d660 msgr2=0x560d0daee7e0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x560d0da4d660 0x560d0daee7e0 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31114a640 1 -- 172.21.15.64:0/751135854 reap_dead start 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb2f4069ee0 msgr2=0x7fb2f406c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb2f4069ee0 0x7fb2f406c3a0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fb30400b700 tx=0x7fb304019040 comp rx=0 tx=0).stop 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d9be640 msgr2=0x560d0da3dc10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d9be640 0x560d0da3dc10 secure :-1 s=READY pgs=1433 cs=0 l=1 rev1=1 crypto rx=0x7fb2fc000c00 tx=0x7fb2fc002fb0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 shutdown_connections 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb2f4069ee0 0x7fb2f406c3a0 unknown :-1 s=CLOSED pgs=643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 --2- 172.21.15.64:0/751135854 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x560d0d9be640 0x560d0da3dc10 unknown :-1 s=CLOSED pgs=1433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 >> 172.21.15.64:0/751135854 conn(0x560d0d9745d0 msgr2=0x560d0d9749b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 shutdown_connections 2024-03-04T07:33:17.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.837+0000 7fb31363df40 1 -- 172.21.15.64:0/751135854 wait complete. 2024-03-04T07:33:17.841 DEBUG:tasks.cephfs.filesystem:All objects for ino 1099511627777 size 8388608 are absent 2024-03-04T07:33:17.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:17.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.984+0000 7fab73ed4640 1 Processor -- start 2024-03-04T07:33:17.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.984+0000 7fab73ed4640 1 -- start start 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab73ed4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab73ed4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fab6c0b4260 0x7fab6c0b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab73ed4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fab6c0b0dd0 con 0x7fab6c0d06c0 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab73ed4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fab6c0b0b80 con 0x7fab6c0b4260 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab73ed4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab6c0aeec0 con 0x7fab6c0d02e0 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab7146f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fab6c0b4260 0x7fab6c0b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab7146f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fab6c0b4260 0x7fab6c0b1500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37046/0 (socket says 172.21.15.64:37046) 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.985+0000 7fab7146f640 1 -- 172.21.15.64:0/2074875390 learned_addr learned my addr 172.21.15.64:0/2074875390 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.986+0000 7fab71c70640 1 --2- 172.21.15.64:0/2074875390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.986+0000 7fab7146f640 1 -- 172.21.15.64:0/2074875390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 msgr2=0x7fab6c0cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.986+0000 7fab7146f640 1 --2- 172.21.15.64:0/2074875390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c0cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.986+0000 7fab7146f640 1 -- 172.21.15.64:0/2074875390 >> v1:172.21.15.66:6789/0 conn(0x7fab6c0d06c0 legacy=0x7fab6c0b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:17.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.986+0000 7fab7146f640 1 -- 172.21.15.64:0/2074875390 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab6c0b10b0 con 0x7fab6c0b4260 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab7146f640 1 --2- 172.21.15.64:0/2074875390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c0b1500 secure :-1 s=READY pgs=1434 cs=0 l=1 rev1=1 crypto rx=0x7fab5c009870 tx=0x7fab5c02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b676c85906de3d14 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab70c6e640 1 -- 172.21.15.64:0/2074875390 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab5c034070 con 0x7fab6c0b4260 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab70c6e640 1 -- 172.21.15.64:0/2074875390 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab5c030840 con 0x7fab6c0b4260 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab70c6e640 1 -- 172.21.15.64:0/2074875390 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab5c030c10 con 0x7fab6c0b4260 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 -- 172.21.15.64:0/2074875390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 msgr2=0x7fab6c0b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2074875390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c0b1500 secure :-1 s=READY pgs=1434 cs=0 l=1 rev1=1 crypto rx=0x7fab5c009870 tx=0x7fab5c02b980 comp rx=0 tx=0).stop 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 -- 172.21.15.64:0/2074875390 shutdown_connections 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2074875390 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c0b1500 unknown :-1 s=CLOSED pgs=1434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2074875390 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 -- 172.21.15.64:0/2074875390 >> 172.21.15.64:0/2074875390 conn(0x7fab6c0b36d0 msgr2=0x7fab6c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 -- 172.21.15.64:0/2074875390 shutdown_connections 2024-03-04T07:33:17.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.987+0000 7fab73ed4640 1 -- 172.21.15.64:0/2074875390 wait complete. 2024-03-04T07:33:17.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 Processor -- start 2024-03-04T07:33:17.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 -- start start 2024-03-04T07:33:17.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c1f36c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c13d940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab6c0d06c0 0x7fab6c140b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab6c0b22d0 con 0x7fab6c0b4260 2024-03-04T07:33:17.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab6c0b2150 con 0x7fab6c0d06c0 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab73ed4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fab6c0af350 con 0x7fab6c0d02e0 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab71c70640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c1f36c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab71c70640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c1f36c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37058/0 (socket says 172.21.15.64:37058) 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab71c70640 1 -- 172.21.15.64:0/2237384389 learned_addr learned my addr 172.21.15.64:0/2237384389 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab72471640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab6c0d06c0 0x7fab6c140b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.989+0000 7fab7146f640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c13d940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab71c70640 1 -- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 msgr2=0x7fab6c13d940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab71c70640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c13d940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab71c70640 1 -- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab6c0d06c0 msgr2=0x7fab6c140b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab71c70640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab6c0d06c0 0x7fab6c140b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab71c70640 1 -- 172.21.15.64:0/2237384389 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab6c13de80 con 0x7fab6c0b4260 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab71c70640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c1f36c0 secure :-1 s=READY pgs=1435 cs=0 l=1 rev1=1 crypto rx=0x7fab6800d800 tx=0x7fab6800dcc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab627fc640 1 -- 172.21.15.64:0/2237384389 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab6800b6c0 con 0x7fab6c0b4260 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab627fc640 1 -- 172.21.15.64:0/2237384389 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fab68019070 con 0x7fab6c0b4260 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab627fc640 1 -- 172.21.15.64:0/2237384389 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab6800ba80 con 0x7fab6c0b4260 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab6c06b580 con 0x7fab6c0b4260 2024-03-04T07:33:17.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.990+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab6c06bb40 con 0x7fab6c0b4260 2024-03-04T07:33:17.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.991+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab6c068700 con 0x7fab6c0b4260 2024-03-04T07:33:17.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.992+0000 7fab627fc640 1 -- 172.21.15.64:0/2237384389 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fab6800bc20 con 0x7fab6c0b4260 2024-03-04T07:33:17.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.992+0000 7fab627fc640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5006b500 0x7fab5006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:17.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.992+0000 7fab627fc640 1 -- 172.21.15.64:0/2237384389 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fab6808a5a0 con 0x7fab6c0b4260 2024-03-04T07:33:17.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.992+0000 7fab7146f640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5006b500 0x7fab5006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:17.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.993+0000 7fab7146f640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5006b500 0x7fab5006d9c0 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7fab5c036000 tx=0x7fab5c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:17.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:17.997+0000 7fab627fc640 1 -- 172.21.15.64:0/2237384389 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fab68058d70 con 0x7fab6c0b4260 2024-03-04T07:33:18.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:33:18.073 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:33:18.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.219+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fab6c15f410 con 0x7fab6c0b4260 2024-03-04T07:33:18.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.220+0000 7fab627fc640 1 -- 172.21.15.64:0/2237384389 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7fab6805f0c0 con 0x7fab6c0b4260 2024-03-04T07:33:18.220 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:18.220 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:18.220 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:18.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5006b500 msgr2=0x7fab5006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5006b500 0x7fab5006d9c0 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7fab5c036000 tx=0x7fab5c008030 comp rx=0 tx=0).stop 2024-03-04T07:33:18.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 msgr2=0x7fab6c1f36c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c1f36c0 secure :-1 s=READY pgs=1435 cs=0 l=1 rev1=1 crypto rx=0x7fab6800d800 tx=0x7fab6800dcc0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 shutdown_connections 2024-03-04T07:33:18.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fab5006b500 0x7fab5006d9c0 unknown :-1 s=CLOSED pgs=644 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fab6c0d06c0 0x7fab6c140b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fab6c0d02e0 0x7fab6c13d940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 --2- 172.21.15.64:0/2237384389 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fab6c0b4260 0x7fab6c1f36c0 unknown :-1 s=CLOSED pgs=1435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 >> 172.21.15.64:0/2237384389 conn(0x7fab6c0b36d0 msgr2=0x7fab6c0b2930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:18.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 shutdown_connections 2024-03-04T07:33:18.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.222+0000 7fab73ed4640 1 -- 172.21.15.64:0/2237384389 wait complete. 2024-03-04T07:33:18.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_created 2024-03-04T07:33:18.350 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_created":3}} 2024-03-04T07:33:18.351 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 3 } } 2024-03-04T07:33:18.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:18.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.452+0000 7f10dd684640 1 Processor -- start 2024-03-04T07:33:18.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.452+0000 7f10dd684640 1 -- start start 2024-03-04T07:33:18.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10dd684640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f10d80cebb0 0x7f10d80d0940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:18.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10dd684640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f10d8161c80 0x7f10d81620e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:18.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10dd684640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f10d8162ab0 con 0x7f10d8162620 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10dd684640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f10d80d1090 con 0x7f10d8161c80 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10dd684640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f10d806b0a0 con 0x7f10d80cebb0 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10d67fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f10d8161c80 0x7f10d81620e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10d77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f10d8162620 0x7f10d815d730 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36354/0 (socket says 172.21.15.64:36354) 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.453+0000 7f10d77fe640 1 -- 172.21.15.64:0/338894511 learned_addr learned my addr 172.21.15.64:0/338894511 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.454+0000 7f10d6ffd640 1 --2- 172.21.15.64:0/338894511 >> v2:172.21.15.66:3300/0 conn(0x7f10d80cebb0 0x7f10d80d0940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.454+0000 7f10d67fc640 1 -- 172.21.15.64:0/338894511 >> v2:172.21.15.66:3300/0 conn(0x7f10d80cebb0 msgr2=0x7f10d80d0940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.454+0000 7f10d67fc640 1 --2- 172.21.15.64:0/338894511 >> v2:172.21.15.66:3300/0 conn(0x7f10d80cebb0 0x7f10d80d0940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.454+0000 7f10d67fc640 1 -- 172.21.15.64:0/338894511 >> v1:172.21.15.64:6789/0 conn(0x7f10d8162620 legacy=0x7f10d815d730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.454+0000 7f10d67fc640 1 -- 172.21.15.64:0/338894511 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10d806b380 con 0x7f10d8161c80 2024-03-04T07:33:18.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.454+0000 7f10d67fc640 1 --2- 172.21.15.64:0/338894511 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8161c80 0x7f10d81620e0 secure :-1 s=READY pgs=1437 cs=0 l=1 rev1=1 crypto rx=0x7f10d8071b40 tx=0x7f10cc02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a3721d90241ff078 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:18.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10d5ffb640 1 -- 172.21.15.64:0/338894511 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10cc035020 con 0x7f10d8161c80 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10d5ffb640 1 -- 172.21.15.64:0/338894511 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f10cc031830 con 0x7f10d8161c80 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10d5ffb640 1 -- 172.21.15.64:0/338894511 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10cc031c00 con 0x7f10d8161c80 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 -- 172.21.15.64:0/338894511 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8161c80 msgr2=0x7f10d81620e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 --2- 172.21.15.64:0/338894511 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8161c80 0x7f10d81620e0 secure :-1 s=READY pgs=1437 cs=0 l=1 rev1=1 crypto rx=0x7f10d8071b40 tx=0x7f10cc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 -- 172.21.15.64:0/338894511 shutdown_connections 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 --2- 172.21.15.64:0/338894511 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8161c80 0x7f10d81620e0 unknown :-1 s=CLOSED pgs=1437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 --2- 172.21.15.64:0/338894511 >> v2:172.21.15.66:3300/0 conn(0x7f10d80cebb0 0x7f10d80d0940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 -- 172.21.15.64:0/338894511 >> 172.21.15.64:0/338894511 conn(0x7f10d80b35d0 msgr2=0x7f10d80b39e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 -- 172.21.15.64:0/338894511 shutdown_connections 2024-03-04T07:33:18.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.455+0000 7f10dd684640 1 -- 172.21.15.64:0/338894511 wait complete. 2024-03-04T07:33:18.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.457+0000 7f10dd684640 1 Processor -- start 2024-03-04T07:33:18.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.457+0000 7f10dd684640 1 -- start start 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10d80cebb0 0x7f10d815e570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10d8161c80 0x7f10d8157820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8162620 0x7f10d80cd120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10d815dfc0 con 0x7f10d8162620 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10d815de40 con 0x7f10d80cebb0 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10d815e140 con 0x7f10d8161c80 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8162620 0x7f10d80cd120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8162620 0x7f10d80cd120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37078/0 (socket says 172.21.15.64:37078) 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 -- 172.21.15.64:0/802805277 learned_addr learned my addr 172.21.15.64:0/802805277 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d6ffd640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10d80cebb0 0x7f10d815e570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d67fc640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10d8161c80 0x7f10d8157820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 -- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10d8161c80 msgr2=0x7f10d8157820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10d8161c80 0x7f10d8157820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 -- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10d80cebb0 msgr2=0x7f10d815e570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10d80cebb0 0x7f10d815e570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 -- 172.21.15.64:0/802805277 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10d80b5880 con 0x7f10d8162620 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10d77fe640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8162620 0x7f10d80cd120 secure :-1 s=READY pgs=1438 cs=0 l=1 rev1=1 crypto rx=0x7f10d000cab0 tx=0x7f10d000cf70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10bf7fe640 1 -- 172.21.15.64:0/802805277 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10d0017070 con 0x7f10d8162620 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10bf7fe640 1 -- 172.21.15.64:0/802805277 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f10d000ac30 con 0x7f10d8162620 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10d81f76f0 con 0x7f10d8162620 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10bf7fe640 1 -- 172.21.15.64:0/802805277 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10d0012400 con 0x7f10d8162620 2024-03-04T07:33:18.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.459+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10d81f7cb0 con 0x7f10d8162620 2024-03-04T07:33:18.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.460+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f10d80b5a50 con 0x7f10d8162620 2024-03-04T07:33:18.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.461+0000 7f10bf7fe640 1 -- 172.21.15.64:0/802805277 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f10d001a070 con 0x7f10d8162620 2024-03-04T07:33:18.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.461+0000 7f10bf7fe640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f10c406b500 0x7f10c406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:18.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.461+0000 7f10bf7fe640 1 -- 172.21.15.64:0/802805277 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f10d0089e80 con 0x7f10d8162620 2024-03-04T07:33:18.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.461+0000 7f10d6ffd640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f10c406b500 0x7f10c406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:18.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.462+0000 7f10d6ffd640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f10c406b500 0x7f10c406d9c0 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7f10c8006fd0 tx=0x7f10c8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:18.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.466+0000 7f10bf7fe640 1 -- 172.21.15.64:0/802805277 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f10d8162ab0 con 0x7f10d8162620 2024-03-04T07:33:18.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.687+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f10d80b7a40 con 0x7f10d8162620 2024-03-04T07:33:18.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.687+0000 7f10bf7fe640 1 -- 172.21.15.64:0/802805277 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f10d00585c0 con 0x7f10d8162620 2024-03-04T07:33:18.687 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:18.687 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:18.688 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:18.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f10c406b500 msgr2=0x7f10c406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f10c406b500 0x7f10c406d9c0 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7f10c8006fd0 tx=0x7f10c8008040 comp rx=0 tx=0).stop 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8162620 msgr2=0x7f10d80cd120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8162620 0x7f10d80cd120 secure :-1 s=READY pgs=1438 cs=0 l=1 rev1=1 crypto rx=0x7f10d000cab0 tx=0x7f10d000cf70 comp rx=0 tx=0).stop 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 shutdown_connections 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f10c406b500 0x7f10c406d9c0 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f10d8162620 0x7f10d80cd120 unknown :-1 s=CLOSED pgs=1438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10d8161c80 0x7f10d8157820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 --2- 172.21.15.64:0/802805277 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f10d80cebb0 0x7f10d815e570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 >> 172.21.15.64:0/802805277 conn(0x7f10d80b35d0 msgr2=0x7f10d806bb80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 shutdown_connections 2024-03-04T07:33:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:18.689+0000 7f10dd684640 1 -- 172.21.15.64:0/802805277 wait complete. 2024-03-04T07:33:18.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.c.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:33:18.817 INFO:teuthology.orchestra.run.smithi064.stdout:{"mds_cache":{"strays_enqueued":1}} 2024-03-04T07:33:18.817 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 1 } } 2024-03-04T07:33:18.817 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:33:18.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:33:18.858 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:33:18.912 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:33:18.912 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:33:18.912 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:18.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:19.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.068+0000 7f3c2d165640 1 Processor -- start 2024-03-04T07:33:19.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.068+0000 7f3c2d165640 1 -- start start 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.068+0000 7f3c2d165640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3c280d02e0 0x7f3c280cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.068+0000 7f3c2d165640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d06c0 0x7f3c280b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.068+0000 7f3c2d165640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c280b0dd0 con 0x7f3c280b4260 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.068+0000 7f3c2d165640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c280b0b80 con 0x7f3c280d02e0 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.068+0000 7f3c2d165640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c280aeec0 con 0x7f3c280d06c0 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.069+0000 7f3c26d9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3c280d02e0 0x7f3c280cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.069+0000 7f3c1ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3c280b4260 0x7f3c280b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36368/0 (socket says 172.21.15.64:36368) 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.069+0000 7f3c1ffff640 1 -- 172.21.15.64:0/2599719705 learned_addr learned my addr 172.21.15.64:0/2599719705 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.069+0000 7f3c26d9d640 1 -- 172.21.15.64:0/2599719705 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d06c0 msgr2=0x7f3c280b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.069+0000 7f3c26d9d640 1 --2- 172.21.15.64:0/2599719705 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d06c0 0x7f3c280b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.069+0000 7f3c26d9d640 1 -- 172.21.15.64:0/2599719705 >> v1:172.21.15.64:6789/0 conn(0x7f3c280b4260 legacy=0x7f3c280b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.069+0000 7f3c26d9d640 1 -- 172.21.15.64:0/2599719705 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c280b10b0 con 0x7f3c280d02e0 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c26d9d640 1 --2- 172.21.15.64:0/2599719705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280d02e0 0x7f3c280cfa00 secure :-1 s=READY pgs=1440 cs=0 l=1 rev1=1 crypto rx=0x7f3c180097a0 tx=0x7f3c1802b940 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7059db66f540dc43 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2659c640 1 -- 172.21.15.64:0/2599719705 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c18034070 con 0x7f3c280d02e0 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2659c640 1 -- 172.21.15.64:0/2599719705 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c18030800 con 0x7f3c280d02e0 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2659c640 1 -- 172.21.15.64:0/2599719705 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c18030bd0 con 0x7f3c280d02e0 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 -- 172.21.15.64:0/2599719705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280d02e0 msgr2=0x7f3c280cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 --2- 172.21.15.64:0/2599719705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280d02e0 0x7f3c280cfa00 secure :-1 s=READY pgs=1440 cs=0 l=1 rev1=1 crypto rx=0x7f3c180097a0 tx=0x7f3c1802b940 comp rx=0 tx=0).stop 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 -- 172.21.15.64:0/2599719705 shutdown_connections 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 --2- 172.21.15.64:0/2599719705 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d06c0 0x7f3c280b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 --2- 172.21.15.64:0/2599719705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280d02e0 0x7f3c280cfa00 unknown :-1 s=CLOSED pgs=1440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 -- 172.21.15.64:0/2599719705 >> 172.21.15.64:0/2599719705 conn(0x7f3c280b36d0 msgr2=0x7f3c280b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 -- 172.21.15.64:0/2599719705 shutdown_connections 2024-03-04T07:33:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.070+0000 7f3c2d165640 1 -- 172.21.15.64:0/2599719705 wait complete. 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 Processor -- start 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 -- start start 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280b4260 0x7f3c280c78f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d02e0 0x7f3c281f7b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c280d06c0 0x7f3c28159e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c280b22d0 con 0x7f3c280b4260 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c280b2150 con 0x7f3c280d06c0 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c2d165640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c280af350 con 0x7f3c280d02e0 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c26d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280b4260 0x7f3c280c78f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c26d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280b4260 0x7f3c280c78f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37108/0 (socket says 172.21.15.64:37108) 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c26d9d640 1 -- 172.21.15.64:0/3869961152 learned_addr learned my addr 172.21.15.64:0/3869961152 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.072+0000 7f3c1ffff640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d02e0 0x7f3c281f7b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c2759e640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c280d06c0 0x7f3c28159e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c26d9d640 1 -- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d02e0 msgr2=0x7f3c281f7b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c26d9d640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d02e0 0x7f3c281f7b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c26d9d640 1 -- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c280d06c0 msgr2=0x7f3c28159e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c26d9d640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c280d06c0 0x7f3c28159e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c26d9d640 1 -- 172.21.15.64:0/3869961152 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c281f8430 con 0x7f3c280b4260 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c26d9d640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280b4260 0x7f3c280c78f0 secure :-1 s=READY pgs=1441 cs=0 l=1 rev1=1 crypto rx=0x7f3c180098c0 tx=0x7f3c180046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c1f7fe640 1 -- 172.21.15.64:0/3869961152 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c18034040 con 0x7f3c280b4260 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c1f7fe640 1 -- 172.21.15.64:0/3869961152 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c18039d10 con 0x7f3c280b4260 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c281f8710 con 0x7f3c280b4260 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c1f7fe640 1 -- 172.21.15.64:0/3869961152 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c18042610 con 0x7f3c280b4260 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.073+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c281598f0 con 0x7f3c280b4260 2024-03-04T07:33:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.074+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c28068700 con 0x7f3c280b4260 2024-03-04T07:33:19.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.075+0000 7f3c1f7fe640 1 -- 172.21.15.64:0/3869961152 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3c18037070 con 0x7f3c280b4260 2024-03-04T07:33:19.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.075+0000 7f3c1f7fe640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c0806b4d0 0x7f3c0806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.075+0000 7f3c1f7fe640 1 -- 172.21.15.64:0/3869961152 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f3c180ac540 con 0x7f3c280b4260 2024-03-04T07:33:19.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.075+0000 7f3c1ffff640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c0806b4d0 0x7f3c0806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.076+0000 7f3c1ffff640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c0806b4d0 0x7f3c0806d990 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f3c2815b890 tx=0x7f3c10009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.080+0000 7f3c1f7fe640 1 -- 172.21.15.64:0/3869961152 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3c1807b360 con 0x7f3c280b4260 2024-03-04T07:33:19.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.263+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f3c2813e950 con 0x7f3c280b4260 2024-03-04T07:33:19.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.264+0000 7f3c1f7fe640 1 -- 172.21.15.64:0/3869961152 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v111) v1 ==== 74+0+16934 (secure 0 0 0) 0x7f3c1807ac60 con 0x7f3c280b4260 2024-03-04T07:33:19.264 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:19.264 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":111,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:35.558693+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"105","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.25","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.64:6835/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.66:6835/4084900379":"2024-03-05T07:32:21.508754+0000","172.21.15.66:6833/4084900379":"2024-03-05T07:32:21.508754+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c0806b4d0 msgr2=0x7f3c0806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c0806b4d0 0x7f3c0806d990 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f3c2815b890 tx=0x7f3c10009040 comp rx=0 tx=0).stop 2024-03-04T07:33:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280b4260 msgr2=0x7f3c280c78f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280b4260 0x7f3c280c78f0 secure :-1 s=READY pgs=1441 cs=0 l=1 rev1=1 crypto rx=0x7f3c180098c0 tx=0x7f3c180046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 shutdown_connections 2024-03-04T07:33:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c0806b4d0 0x7f3c0806d990 unknown :-1 s=CLOSED pgs=646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c280d06c0 0x7f3c28159e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c280d02e0 0x7f3c281f7b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 --2- 172.21.15.64:0/3869961152 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c280b4260 0x7f3c280c78f0 unknown :-1 s=CLOSED pgs=1441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 >> 172.21.15.64:0/3869961152 conn(0x7f3c280b36d0 msgr2=0x7f3c280cd680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 shutdown_connections 2024-03-04T07:33:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.266+0000 7f3c2d165640 1 -- 172.21.15.64:0/3869961152 wait complete. 2024-03-04T07:33:19.277 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 16, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:32:07.087121+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '104', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 17, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:32:07.785433+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '105', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:33:19.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:33:19.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.384+0000 7ff86d7a1640 1 Processor -- start 2024-03-04T07:33:19.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.385+0000 7ff86d7a1640 1 -- start start 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.385+0000 7ff86d7a1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff8680d37a0 0x7ff8680ceae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.385+0000 7ff86d7a1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680b9710 0x7ff8680cf020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.385+0000 7ff86d7a1640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8680d2700 con 0x7ff86815e000 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.385+0000 7ff86d7a1640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff8680b7ec0 con 0x7ff8680d37a0 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.385+0000 7ff86d7a1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8680d29e0 con 0x7ff8680b9710 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.385+0000 7ff8667fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff8680d37a0 0x7ff8680ceae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff8667fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff8680d37a0 0x7ff8680ceae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37120/0 (socket says 172.21.15.64:37120) 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff8667fc640 1 -- 172.21.15.64:0/3821374280 learned_addr learned my addr 172.21.15.64:0/3821374280 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3821374280 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680b9710 0x7ff8680cf020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff8667fc640 1 -- 172.21.15.64:0/3821374280 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680b9710 msgr2=0x7ff8680cf020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:19.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3821374280 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680b9710 0x7ff8680cf020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff8667fc640 1 -- 172.21.15.64:0/3821374280 >> v1:172.21.15.66:6789/0 conn(0x7ff86815e000 legacy=0x7ff8680d1e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff865ffb640 1 -- 172.21.15.64:0/3821374280 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1271018104 0 0) 0x7ff8680d2700 con 0x7ff86815e000 2024-03-04T07:33:19.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.386+0000 7ff8667fc640 1 -- 172.21.15.64:0/3821374280 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8680d2cc0 con 0x7ff8680d37a0 2024-03-04T07:33:19.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.387+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3821374280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff8680ceae0 secure :-1 s=READY pgs=1442 cs=0 l=1 rev1=1 crypto rx=0x7ff850009870 tx=0x7ff85002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=31e0d826f99b26bc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.387+0000 7ff865ffb640 1 -- 172.21.15.64:0/3821374280 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff85002f840 con 0x7ff8680d37a0 2024-03-04T07:33:19.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.387+0000 7ff865ffb640 1 -- 172.21.15.64:0/3821374280 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff85002f9e0 con 0x7ff8680d37a0 2024-03-04T07:33:19.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.387+0000 7ff865ffb640 1 -- 172.21.15.64:0/3821374280 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff85002fdb0 con 0x7ff8680d37a0 2024-03-04T07:33:19.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.387+0000 7ff86d7a1640 1 -- 172.21.15.64:0/3821374280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 msgr2=0x7ff8680ceae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.387+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/3821374280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff8680ceae0 secure :-1 s=READY pgs=1442 cs=0 l=1 rev1=1 crypto rx=0x7ff850009870 tx=0x7ff85002ba20 comp rx=0 tx=0).stop 2024-03-04T07:33:19.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.388+0000 7ff86d7a1640 1 -- 172.21.15.64:0/3821374280 shutdown_connections 2024-03-04T07:33:19.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.388+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/3821374280 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680b9710 0x7ff8680cf020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.388+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/3821374280 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff8680ceae0 unknown :-1 s=CLOSED pgs=1442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.388+0000 7ff86d7a1640 1 -- 172.21.15.64:0/3821374280 >> 172.21.15.64:0/3821374280 conn(0x7ff868066d00 msgr2=0x7ff868067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:19.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.388+0000 7ff86d7a1640 1 -- 172.21.15.64:0/3821374280 shutdown_connections 2024-03-04T07:33:19.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.388+0000 7ff86d7a1640 1 -- 172.21.15.64:0/3821374280 wait complete. 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.389+0000 7ff86d7a1640 1 Processor -- start 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.389+0000 7ff86d7a1640 1 -- start start 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8680b9710 0x7ff8680c9960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff868146440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff86815e000 0x7ff8680cd6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff86815e3e0 con 0x7ff8680d37a0 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8680d3280 con 0x7ff8680b9710 2024-03-04T07:33:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8680b6eb0 con 0x7ff86815e000 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff868146440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff868146440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37126/0 (socket says 172.21.15.64:37126) 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 -- 172.21.15.64:0/96189727 learned_addr learned my addr 172.21.15.64:0/96189727 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8677fe640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff86815e000 0x7ff8680cd6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff866ffd640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8680b9710 0x7ff8680c9960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 -- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff86815e000 msgr2=0x7ff8680cd6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff86815e000 0x7ff8680cd6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 -- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8680b9710 msgr2=0x7ff8680c9960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8680b9710 0x7ff8680c9960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 -- 172.21.15.64:0/96189727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff868146980 con 0x7ff8680d37a0 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff8667fc640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff868146440 secure :-1 s=READY pgs=1443 cs=0 l=1 rev1=1 crypto rx=0x7ff85002ba60 tx=0x7ff850004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff85f7fe640 1 -- 172.21.15.64:0/96189727 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff850047070 con 0x7ff8680d37a0 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff85f7fe640 1 -- 172.21.15.64:0/96189727 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff850037bc0 con 0x7ff8680d37a0 2024-03-04T07:33:19.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8680cb380 con 0x7ff8680d37a0 2024-03-04T07:33:19.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff85f7fe640 1 -- 172.21.15.64:0/96189727 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff850042480 con 0x7ff8680d37a0 2024-03-04T07:33:19.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.390+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8680cb940 con 0x7ff8680d37a0 2024-03-04T07:33:19.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.391+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff868072a30 con 0x7ff8680d37a0 2024-03-04T07:33:19.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.392+0000 7ff85f7fe640 1 -- 172.21.15.64:0/96189727 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff85004a070 con 0x7ff8680d37a0 2024-03-04T07:33:19.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.392+0000 7ff85f7fe640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff84806b4d0 0x7ff84806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.392+0000 7ff85f7fe640 1 -- 172.21.15.64:0/96189727 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7ff8500ac320 con 0x7ff8680d37a0 2024-03-04T07:33:19.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.392+0000 7ff866ffd640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff84806b4d0 0x7ff84806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.393+0000 7ff866ffd640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff84806b4d0 0x7ff84806d990 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7ff8580012e0 tx=0x7ff858009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.397+0000 7ff85f7fe640 1 -- 172.21.15.64:0/96189727 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff850033030 con 0x7ff8680d37a0 2024-03-04T07:33:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.576+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7ff8680b6a70 con 0x7ff8680d37a0 2024-03-04T07:33:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.576+0000 7ff85f7fe640 1 -- 172.21.15.64:0/96189727 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v111) v1 ==== 93+0+10 (secure 0 0 0) 0x7ff85007aaf0 con 0x7ff8680d37a0 2024-03-04T07:33:19.576 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:33:19.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff84806b4d0 msgr2=0x7ff84806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff84806b4d0 0x7ff84806d990 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7ff8580012e0 tx=0x7ff858009040 comp rx=0 tx=0).stop 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 msgr2=0x7ff868146440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff868146440 secure :-1 s=READY pgs=1443 cs=0 l=1 rev1=1 crypto rx=0x7ff85002ba60 tx=0x7ff850004730 comp rx=0 tx=0).stop 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 shutdown_connections 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff84806b4d0 0x7ff84806d990 unknown :-1 s=CLOSED pgs=647 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff86815e000 0x7ff8680cd6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8680d37a0 0x7ff868146440 unknown :-1 s=CLOSED pgs=1443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 --2- 172.21.15.64:0/96189727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8680b9710 0x7ff8680c9960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 >> 172.21.15.64:0/96189727 conn(0x7ff868066d00 msgr2=0x7ff86815e560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 shutdown_connections 2024-03-04T07:33:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.578+0000 7ff86d7a1640 1 -- 172.21.15.64:0/96189727 wait complete. 2024-03-04T07:33:19.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:33:19.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.694+0000 7f520ba81640 1 Processor -- start 2024-03-04T07:33:19.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.694+0000 7f520ba81640 1 -- start start 2024-03-04T07:33:19.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520ba81640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f520406ec40 0x7f5204071a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520ba81640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5204071f60 0x7f52040ce570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520ba81640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5204072370 con 0x7f52041445b0 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520ba81640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5204144990 con 0x7f5204071f60 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520ba81640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f520406f1e0 con 0x7f520406ec40 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520a01e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5204071f60 0x7f52040ce570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520a01e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5204071f60 0x7f52040ce570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37132/0 (socket says 172.21.15.64:37132) 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520a01e640 1 -- 172.21.15.64:0/315475814 learned_addr learned my addr 172.21.15.64:0/315475814 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.695+0000 7f520901c640 1 --2- 172.21.15.64:0/315475814 >> v2:172.21.15.66:3300/0 conn(0x7f520406ec40 0x7f5204071a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.696+0000 7f520a01e640 1 -- 172.21.15.64:0/315475814 >> v2:172.21.15.66:3300/0 conn(0x7f520406ec40 msgr2=0x7f5204071a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.696+0000 7f520a01e640 1 --2- 172.21.15.64:0/315475814 >> v2:172.21.15.66:3300/0 conn(0x7f520406ec40 0x7f5204071a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.696+0000 7f520881b640 1 -- 172.21.15.64:0/315475814 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2985879609 0 0) 0x7f5200001110 con 0x7f52041445b0 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.696+0000 7f520a01e640 1 -- 172.21.15.64:0/315475814 >> v1:172.21.15.66:6789/0 conn(0x7f52041445b0 legacy=0x7f52040b43c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.696+0000 7f520a01e640 1 -- 172.21.15.64:0/315475814 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f520406f4c0 con 0x7f5204071f60 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.696+0000 7f520a01e640 1 --2- 172.21.15.64:0/315475814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5204071f60 0x7f52040ce570 secure :-1 s=READY pgs=1444 cs=0 l=1 rev1=1 crypto rx=0x7f52040b56f0 tx=0x7f51fc02bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f11b9187cd7472c4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520881b640 1 -- 172.21.15.64:0/315475814 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51fc02cb20 con 0x7f5204071f60 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520881b640 1 -- 172.21.15.64:0/315475814 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f51fc02ccc0 con 0x7f5204071f60 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520881b640 1 -- 172.21.15.64:0/315475814 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51fc034610 con 0x7f5204071f60 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 -- 172.21.15.64:0/315475814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5204071f60 msgr2=0x7f52040ce570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 --2- 172.21.15.64:0/315475814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5204071f60 0x7f52040ce570 secure :-1 s=READY pgs=1444 cs=0 l=1 rev1=1 crypto rx=0x7f52040b56f0 tx=0x7f51fc02bf80 comp rx=0 tx=0).stop 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 -- 172.21.15.64:0/315475814 shutdown_connections 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 --2- 172.21.15.64:0/315475814 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5204071f60 0x7f52040ce570 unknown :-1 s=CLOSED pgs=1444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 --2- 172.21.15.64:0/315475814 >> v2:172.21.15.66:3300/0 conn(0x7f520406ec40 0x7f5204071a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 -- 172.21.15.64:0/315475814 >> 172.21.15.64:0/315475814 conn(0x7f52040d09c0 msgr2=0x7f52040d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:19.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 -- 172.21.15.64:0/315475814 shutdown_connections 2024-03-04T07:33:19.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.697+0000 7f520ba81640 1 -- 172.21.15.64:0/315475814 wait complete. 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.698+0000 7f520ba81640 1 Processor -- start 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.698+0000 7f520ba81640 1 -- start start 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520ba81640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f520406ec40 0x7f520415a5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520ba81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5204071f60 0x7f52040ab880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520ba81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52041445b0 0x7f52041f28e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520ba81640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f52040cdc70 con 0x7f52041445b0 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520ba81640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f52040ce050 con 0x7f520406ec40 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520ba81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f52040cddf0 con 0x7f5204071f60 2024-03-04T07:33:19.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520a01e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52041445b0 0x7f52041f28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520a01e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52041445b0 0x7f52041f28e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37142/0 (socket says 172.21.15.64:37142) 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520a01e640 1 -- 172.21.15.64:0/3325546890 learned_addr learned my addr 172.21.15.64:0/3325546890 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520981d640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f520406ec40 0x7f520415a5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.699+0000 7f520901c640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5204071f60 0x7f52040ab880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520a01e640 1 -- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5204071f60 msgr2=0x7f52040ab880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520a01e640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5204071f60 0x7f52040ab880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520a01e640 1 -- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f520406ec40 msgr2=0x7f520415a5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520a01e640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f520406ec40 0x7f520415a5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520a01e640 1 -- 172.21.15.64:0/3325546890 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f52041f3380 con 0x7f52041445b0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520a01e640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52041445b0 0x7f52041f28e0 secure :-1 s=READY pgs=1445 cs=0 l=1 rev1=1 crypto rx=0x7f51fc009f40 tx=0x7f51fc004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f51fa7fc640 1 -- 172.21.15.64:0/3325546890 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51fc043030 con 0x7f52041445b0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f51fa7fc640 1 -- 172.21.15.64:0/3325546890 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f51fc03e450 con 0x7f52041445b0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f52041f3660 con 0x7f52041445b0 2024-03-04T07:33:19.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f51fa7fc640 1 -- 172.21.15.64:0/3325546890 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51fc009510 con 0x7f52041445b0 2024-03-04T07:33:19.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.700+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f52041f3c20 con 0x7f52041445b0 2024-03-04T07:33:19.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.701+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f52040733d0 con 0x7f52041445b0 2024-03-04T07:33:19.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.701+0000 7f51fa7fc640 1 -- 172.21.15.64:0/3325546890 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f51fc046070 con 0x7f52041445b0 2024-03-04T07:33:19.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.702+0000 7f51fa7fc640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51e806b4d0 0x7f51e806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:19.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.702+0000 7f520981d640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51e806b4d0 0x7f51e806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:19.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.702+0000 7f51fa7fc640 1 -- 172.21.15.64:0/3325546890 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f51fc0ac510 con 0x7f52041445b0 2024-03-04T07:33:19.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.703+0000 7f520981d640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51e806b4d0 0x7f51e806d990 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f5200000f60 tx=0x7f5200009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:19.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.706+0000 7f51fa7fc640 1 -- 172.21.15.64:0/3325546890 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f51fc04d070 con 0x7f52041445b0 2024-03-04T07:33:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.886+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f520415e4c0 con 0x7f52041445b0 2024-03-04T07:33:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.886+0000 7f51fa7fc640 1 -- 172.21.15.64:0/3325546890 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v111) v1 ==== 104+0+11 (secure 0 0 0) 0x7f51fc07ad70 con 0x7f52041445b0 2024-03-04T07:33:19.886 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:33:19.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51e806b4d0 msgr2=0x7f51e806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51e806b4d0 0x7f51e806d990 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f5200000f60 tx=0x7f5200009040 comp rx=0 tx=0).stop 2024-03-04T07:33:19.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52041445b0 msgr2=0x7f52041f28e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52041445b0 0x7f52041f28e0 secure :-1 s=READY pgs=1445 cs=0 l=1 rev1=1 crypto rx=0x7f51fc009f40 tx=0x7f51fc004a40 comp rx=0 tx=0).stop 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 shutdown_connections 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f51e806b4d0 0x7f51e806d990 unknown :-1 s=CLOSED pgs=648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f52041445b0 0x7f52041f28e0 unknown :-1 s=CLOSED pgs=1445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5204071f60 0x7f52040ab880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 --2- 172.21.15.64:0/3325546890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f520406ec40 0x7f520415a5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 >> 172.21.15.64:0/3325546890 conn(0x7f52040d09c0 msgr2=0x7f5204142e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 shutdown_connections 2024-03-04T07:33:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:19.888+0000 7f520ba81640 1 -- 172.21.15.64:0/3325546890 wait complete. 2024-03-04T07:33:19.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:33:20.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.007+0000 7f4b97dac640 1 Processor -- start 2024-03-04T07:33:20.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.007+0000 7f4b97dac640 1 -- start start 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.008+0000 7f4b97dac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4b90143e60 0x7f4b900b3bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.008+0000 7f4b97dac640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4b900726e0 0x7f4b900719a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.008+0000 7f4b97dac640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b900722f0 con 0x7f4b90071ee0 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.008+0000 7f4b97dac640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b90072b50 con 0x7f4b900726e0 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.008+0000 7f4b97dac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b9006ecf0 con 0x7f4b90143e60 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95347640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4b900726e0 0x7f4b900719a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95347640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4b900726e0 0x7f4b900719a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37146/0 (socket says 172.21.15.64:37146) 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95347640 1 -- 172.21.15.64:0/2652595009 learned_addr learned my addr 172.21.15.64:0/2652595009 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95b48640 1 --2- 172.21.15.64:0/2652595009 >> v2:172.21.15.66:3300/0 conn(0x7f4b90143e60 0x7f4b900b3bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b96349640 1 --1- 172.21.15.64:0/2652595009 >> v1:172.21.15.66:6789/0 conn(0x7f4b90071ee0 0x7f4b9006da00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:42218/0 (socket says 172.21.15.64:42218) 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95347640 1 -- 172.21.15.64:0/2652595009 >> v2:172.21.15.66:3300/0 conn(0x7f4b90143e60 msgr2=0x7f4b900b3bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95347640 1 --2- 172.21.15.64:0/2652595009 >> v2:172.21.15.66:3300/0 conn(0x7f4b90143e60 0x7f4b900b3bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95347640 1 -- 172.21.15.64:0/2652595009 >> v1:172.21.15.66:6789/0 conn(0x7f4b90071ee0 legacy=0x7f4b9006da00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.009+0000 7f4b95347640 1 -- 172.21.15.64:0/2652595009 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b9006efd0 con 0x7f4b900726e0 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b95347640 1 --2- 172.21.15.64:0/2652595009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b900726e0 0x7f4b900719a0 secure :-1 s=READY pgs=1446 cs=0 l=1 rev1=1 crypto rx=0x7f4b900715c0 tx=0x7f4b8002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d2ace5f5dfaac3b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b94b46640 1 -- 172.21.15.64:0/2652595009 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b80034070 con 0x7f4b900726e0 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b94b46640 1 -- 172.21.15.64:0/2652595009 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b80030860 con 0x7f4b900726e0 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b94b46640 1 -- 172.21.15.64:0/2652595009 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b80030c30 con 0x7f4b900726e0 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 -- 172.21.15.64:0/2652595009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b900726e0 msgr2=0x7f4b900719a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 --2- 172.21.15.64:0/2652595009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b900726e0 0x7f4b900719a0 secure :-1 s=READY pgs=1446 cs=0 l=1 rev1=1 crypto rx=0x7f4b900715c0 tx=0x7f4b8002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 -- 172.21.15.64:0/2652595009 shutdown_connections 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 --2- 172.21.15.64:0/2652595009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b900726e0 0x7f4b900719a0 unknown :-1 s=CLOSED pgs=1446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 --2- 172.21.15.64:0/2652595009 >> v2:172.21.15.66:3300/0 conn(0x7f4b90143e60 0x7f4b900b3bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 -- 172.21.15.64:0/2652595009 >> 172.21.15.64:0/2652595009 conn(0x7f4b900b2790 msgr2=0x7f4b900b2ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 -- 172.21.15.64:0/2652595009 shutdown_connections 2024-03-04T07:33:20.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.010+0000 7f4b97dac640 1 -- 172.21.15.64:0/2652595009 wait complete. 2024-03-04T07:33:20.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.012+0000 7f4b97dac640 1 Processor -- start 2024-03-04T07:33:20.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.012+0000 7f4b97dac640 1 -- start start 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.013+0000 7f4b97dac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b90071ee0 0x7f4b90143010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.013+0000 7f4b97dac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b900726e0 0x7f4b900afc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.013+0000 7f4b97dac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b90143e60 0x7f4b900a9fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.013+0000 7f4b97dac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b90144240 con 0x7f4b90143e60 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.013+0000 7f4b97dac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b9006f590 con 0x7f4b900726e0 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.013+0000 7f4b97dac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b901443c0 con 0x7f4b90071ee0 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b90143e60 0x7f4b900a9fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b90143e60 0x7f4b900a9fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37152/0 (socket says 172.21.15.64:37152) 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 -- 172.21.15.64:0/687058276 learned_addr learned my addr 172.21.15.64:0/687058276 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b95b48640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b90071ee0 0x7f4b90143010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b95347640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b900726e0 0x7f4b900afc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 -- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b90071ee0 msgr2=0x7f4b90143010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b90071ee0 0x7f4b90143010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 -- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b900726e0 msgr2=0x7f4b900afc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b900726e0 0x7f4b900afc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 -- 172.21.15.64:0/687058276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b900ae940 con 0x7f4b90143e60 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b96349640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b90143e60 0x7f4b900a9fa0 secure :-1 s=READY pgs=1447 cs=0 l=1 rev1=1 crypto rx=0x7f4b8800c390 tx=0x7f4b8800c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b867fc640 1 -- 172.21.15.64:0/687058276 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b880074e0 con 0x7f4b90143e60 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b867fc640 1 -- 172.21.15.64:0/687058276 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b88007680 con 0x7f4b90143e60 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b900aebc0 con 0x7f4b90143e60 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b867fc640 1 -- 172.21.15.64:0/687058276 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b880049e0 con 0x7f4b90143e60 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.014+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b900af460 con 0x7f4b90143e60 2024-03-04T07:33:20.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.015+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b900aefb0 con 0x7f4b90143e60 2024-03-04T07:33:20.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.016+0000 7f4b867fc640 1 -- 172.21.15.64:0/687058276 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b8801a020 con 0x7f4b90143e60 2024-03-04T07:33:20.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.016+0000 7f4b867fc640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7406b4d0 0x7f4b7406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.016+0000 7f4b867fc640 1 -- 172.21.15.64:0/687058276 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f4b88089910 con 0x7f4b90143e60 2024-03-04T07:33:20.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.016+0000 7f4b95b48640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7406b4d0 0x7f4b7406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.017+0000 7f4b95b48640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7406b4d0 0x7f4b7406d990 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7f4b901e80a0 tx=0x7f4b8c00b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:20.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.020+0000 7f4b867fc640 1 -- 172.21.15.64:0/687058276 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b900ae940 con 0x7f4b90143e60 2024-03-04T07:33:20.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.203+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f4b9004f410 con 0x7f4b90143e60 2024-03-04T07:33:20.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.203+0000 7f4b867fc640 1 -- 172.21.15.64:0/687058276 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v111) v1 ==== 100+0+11 (secure 0 0 0) 0x7f4b900722f0 con 0x7f4b90143e60 2024-03-04T07:33:20.204 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:33:20.205 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7406b4d0 msgr2=0x7f4b7406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7406b4d0 0x7f4b7406d990 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7f4b901e80a0 tx=0x7f4b8c00b040 comp rx=0 tx=0).stop 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b90143e60 msgr2=0x7f4b900a9fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b90143e60 0x7f4b900a9fa0 secure :-1 s=READY pgs=1447 cs=0 l=1 rev1=1 crypto rx=0x7f4b8800c390 tx=0x7f4b8800c850 comp rx=0 tx=0).stop 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 shutdown_connections 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7406b4d0 0x7f4b7406d990 unknown :-1 s=CLOSED pgs=649 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b90143e60 0x7f4b900a9fa0 unknown :-1 s=CLOSED pgs=1447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b900726e0 0x7f4b900afc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.205+0000 7f4b97dac640 1 --2- 172.21.15.64:0/687058276 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b90071ee0 0x7f4b90143010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.206+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 >> 172.21.15.64:0/687058276 conn(0x7f4b900b2790 msgr2=0x7f4b900b4970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.206+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 shutdown_connections 2024-03-04T07:33:20.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.206+0000 7f4b97dac640 1 -- 172.21.15.64:0/687058276 wait complete. 2024-03-04T07:33:20.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:33:20.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.319+0000 7ffac269a640 1 Processor -- start 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.319+0000 7ffac269a640 1 -- start start 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffac269a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 0x7ffabc0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffac269a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffabc0cea30 con 0x7ffabc0b9a10 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffac269a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffabc0cf3c0 con 0x7ffabc15df00 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffac269a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffabc0cf0e0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffac0c37640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ffabc0b9a10 0x7ffabc0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36384/0 (socket says 172.21.15.64:36384) 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffac0c37640 1 -- 172.21.15.64:0/326866908 learned_addr learned my addr 172.21.15.64:0/326866908 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 588924580 0 0) 0x7ffabc0cea30 con 0x7ffabc0b9a10 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 0x7ffabc0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ffaa8003520 con 0x7ffabc0b9a10 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1502692129 0 0) 0x7ffaa8003520 con 0x7ffabc0b9a10 2024-03-04T07:33:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 msgr2=0x7ffabc0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 0x7ffabc0b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 >> v1:172.21.15.66:6789/0 conn(0x7ffabc15df00 legacy=0x7ffabc0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffabc0ce470 con 0x7ffabc0b9a10 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7ffab40019f0 con 0x7ffabc0b9a10 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.320+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 >> v1:172.21.15.64:6789/0 conn(0x7ffabc0b9a10 legacy=0x7ffabc0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabaffd640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffaa80039e0 0x7ffaa8003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabaffd640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 0x7ffaa8004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabaffd640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffaa8004d40 0x7ffaa80051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffabc0cf0e0 con 0x7ffaa8004d40 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffaa800b9f0 con 0x7ffaa80039e0 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffaa800bbf0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabbfff640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffaa80039e0 0x7ffaa8003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 0x7ffaa8004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffac0c37640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffaa8004d40 0x7ffaa80051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabb7fe640 1 -- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffaa80039e0 msgr2=0x7ffaa8003dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:20.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffaa80039e0 0x7ffaa8003dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabb7fe640 1 -- 172.21.15.64:0/326866908 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffaa8004d40 msgr2=0x7ffaa80051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:20.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffaa8004d40 0x7ffaa80051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffabb7fe640 1 -- 172.21.15.64:0/326866908 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffaa8003520 con 0x7ffabc0b95a0 2024-03-04T07:33:20.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.321+0000 7ffac0c37640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffaa8004d40 0x7ffaa80051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.322+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 0x7ffaa8004800 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7ffaa4009ac0 tx=0x7ffaa8005830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=70306095af6a2709 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.322+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaa4039070 con 0x7ffabc0b95a0 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.322+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffaa402c630 con 0x7ffabc0b95a0 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.322+0000 7ffabaffd640 1 -- 172.21.15.64:0/326866908 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaa4009690 con 0x7ffabc0b95a0 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.323+0000 7ffac269a640 1 -- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 msgr2=0x7ffaa8004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.323+0000 7ffac269a640 1 --2- 172.21.15.64:0/326866908 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0b95a0 0x7ffaa8004800 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7ffaa4009ac0 tx=0x7ffaa8005830 comp rx=0 tx=0).stop 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.323+0000 7ffac0c37640 1 -- 172.21.15.64:0/326866908 reap_dead start 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.323+0000 7ffac269a640 1 -- 172.21.15.64:0/326866908 shutdown_connections 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.323+0000 7ffac269a640 1 -- 172.21.15.64:0/326866908 >> 172.21.15.64:0/326866908 conn(0x7ffabc071960 msgr2=0x7ffabc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:20.328 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.323+0000 7ffac269a640 1 -- 172.21.15.64:0/326866908 shutdown_connections 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.323+0000 7ffac269a640 1 -- 172.21.15.64:0/326866908 wait complete. 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.324+0000 7ffac269a640 1 Processor -- start 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.324+0000 7ffac269a640 1 -- start start 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac269a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffabc0b95a0 0x7ffabc0cc8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac269a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffabc15a3b0 0x7ffabc160b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac269a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0c77b0 0x7ffabc0c7c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac269a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffabc0cedf0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac269a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffabc0cec70 con 0x7ffabc15a3b0 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac269a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffabc0d3930 con 0x7ffabc0c77b0 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffabc0b95a0 0x7ffabc0cc8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffabc0b95a0 0x7ffabc0cc8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37178/0 (socket says 172.21.15.64:37178) 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 -- 172.21.15.64:0/1242596244 learned_addr learned my addr 172.21.15.64:0/1242596244 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac0c37640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0c77b0 0x7ffabc0c7c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffabc15a3b0 0x7ffabc160b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 -- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0c77b0 msgr2=0x7ffabc0c7c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0c77b0 0x7ffabc0c7c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 -- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffabc15a3b0 msgr2=0x7ffabc160b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffabc15a3b0 0x7ffabc160b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabbfff640 1 -- 172.21.15.64:0/1242596244 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffabc0cce20 con 0x7ffabc0b95a0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffac0c37640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0c77b0 0x7ffabc0c7c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.325+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffabc15a3b0 0x7ffabc160b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.326+0000 7ffabbfff640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffabc0b95a0 0x7ffabc0cc8e0 secure :-1 s=READY pgs=1449 cs=0 l=1 rev1=1 crypto rx=0x7ffaac00f7c0 tx=0x7ffaac00fc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.326+0000 7ffab8ff9640 1 -- 172.21.15.64:0/1242596244 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaac00fe70 con 0x7ffabc0b95a0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.326+0000 7ffab8ff9640 1 -- 172.21.15.64:0/1242596244 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffaac018070 con 0x7ffabc0b95a0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.326+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffabc0cd0f0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.326+0000 7ffab8ff9640 1 -- 172.21.15.64:0/1242596244 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffaac00d6d0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.326+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffabc0cd6b0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.327+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffabc067ff0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.327+0000 7ffab8ff9640 1 -- 172.21.15.64:0/1242596244 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffaac00d870 con 0x7ffabc0b95a0 2024-03-04T07:33:20.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.328+0000 7ffab8ff9640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa9c06b500 0x7ffa9c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:20.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.328+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa9c06b500 0x7ffa9c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:20.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.328+0000 7ffab8ff9640 1 -- 172.21.15.64:0/1242596244 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7ffaac08a250 con 0x7ffabc0b95a0 2024-03-04T07:33:20.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.328+0000 7ffabb7fe640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa9c06b500 0x7ffa9c06d9c0 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7ffaa4036000 tx=0x7ffaa4030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:20.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.332+0000 7ffab8ff9640 1 -- 172.21.15.64:0/1242596244 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffaac058fe0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.531+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7ffabc0b6940 con 0x7ffabc0b95a0 2024-03-04T07:33:20.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.531+0000 7ffab8ff9640 1 -- 172.21.15.64:0/1242596244 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 4 entries v111) v1 ==== 80+0+242 (secure 0 0 0) 0x7ffaac0588e0 con 0x7ffabc0b95a0 2024-03-04T07:33:20.531 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.64:6837/209754994 2024-03-05T07:32:22.510444+0000 2024-03-04T07:33:20.531 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.64:6835/209754994 2024-03-05T07:32:22.510444+0000 2024-03-04T07:33:20.531 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.66:6835/4084900379 2024-03-05T07:32:21.508754+0000 2024-03-04T07:33:20.531 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.15.66:6833/4084900379 2024-03-05T07:32:21.508754+0000 2024-03-04T07:33:20.531 INFO:teuthology.orchestra.run.smithi064.stderr:listed 4 entries 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa9c06b500 msgr2=0x7ffa9c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa9c06b500 0x7ffa9c06d9c0 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7ffaa4036000 tx=0x7ffaa4030000 comp rx=0 tx=0).stop 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffabc0b95a0 msgr2=0x7ffabc0cc8e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffabc0b95a0 0x7ffabc0cc8e0 secure :-1 s=READY pgs=1449 cs=0 l=1 rev1=1 crypto rx=0x7ffaac00f7c0 tx=0x7ffaac00fc80 comp rx=0 tx=0).stop 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 shutdown_connections 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa9c06b500 0x7ffa9c06d9c0 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffabc0c77b0 0x7ffabc0c7c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffabc15a3b0 0x7ffabc160b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 --2- 172.21.15.64:0/1242596244 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffabc0b95a0 0x7ffabc0cc8e0 unknown :-1 s=CLOSED pgs=1449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:20.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 >> 172.21.15.64:0/1242596244 conn(0x7ffabc071960 msgr2=0x7ffabc0b5810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:20.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 shutdown_connections 2024-03-04T07:33:20.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:20.533+0000 7ffac269a640 1 -- 172.21.15.64:0/1242596244 wait complete. 2024-03-04T07:33:20.544 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:33:20.544 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:20.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:33:20.569 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[74376]: fuse finished with error 0 and tester_r 0 2024-03-04T07:33:20.571 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:33:20.571 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:20.571 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:33:20.738 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:33:20.738 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:20.757 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:33:20.757 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:33:20.757 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:20.758 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:33:20.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:33:20.816 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:33:20.816 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:33:20.816 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:20.831 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:33:20.831 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:33:20.831 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:20.831 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:33:20.885 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:33:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:33:20.887 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:33:20.887 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:20.905 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:33:20.905 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:33:20.905 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:20.906 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:33:20.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:33:20.963 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:33:20.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:21.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.066+0000 7f5c30239640 1 Processor -- start 2024-03-04T07:33:21.066 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.066+0000 7f5c30239640 1 -- start start 2024-03-04T07:33:21.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c30239640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5c280b95a0 0x7f5c280b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c30239640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c280b9a10 0x7f5c280b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c30239640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5c280cea30 con 0x7f5c2815df00 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c30239640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5c280cf3c0 con 0x7f5c280b95a0 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c30239640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c280cf0e0 con 0x7f5c280b9a10 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2d7d4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5c280b95a0 0x7f5c280b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2d7d4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5c280b95a0 0x7f5c280b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37186/0 (socket says 172.21.15.64:37186) 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1592168071 learned_addr learned my addr 172.21.15.64:0/1592168071 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2e7d6640 1 --2- 172.21.15.64:0/1592168071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c280b9a10 0x7f5c280b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1592168071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c280b9a10 msgr2=0x7f5c280b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2d7d4640 1 --2- 172.21.15.64:0/1592168071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c280b9a10 0x7f5c280b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1592168071 >> v1:172.21.15.66:6789/0 conn(0x7f5c2815df00 legacy=0x7f5c280d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1592168071 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c280ce470 con 0x7f5c280b95a0 2024-03-04T07:33:21.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.067+0000 7f5c2e7d6640 1 --2- 172.21.15.64:0/1592168071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c280b9a10 0x7f5c280b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.068+0000 7f5c2d7d4640 1 --2- 172.21.15.64:0/1592168071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b95a0 0x7f5c280b5700 secure :-1 s=READY pgs=1450 cs=0 l=1 rev1=1 crypto rx=0x7f5c18009870 tx=0x7f5c1802b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d8e5395b7e9fd5eb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.068+0000 7f5c2cfd3640 1 -- 172.21.15.64:0/1592168071 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c18035020 con 0x7f5c280b95a0 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.068+0000 7f5c2cfd3640 1 -- 172.21.15.64:0/1592168071 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c18031640 con 0x7f5c280b95a0 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.068+0000 7f5c2cfd3640 1 -- 172.21.15.64:0/1592168071 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c1802d040 con 0x7f5c280b95a0 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 -- 172.21.15.64:0/1592168071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b95a0 msgr2=0x7f5c280b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 --2- 172.21.15.64:0/1592168071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b95a0 0x7f5c280b5700 secure :-1 s=READY pgs=1450 cs=0 l=1 rev1=1 crypto rx=0x7f5c18009870 tx=0x7f5c1802b8f0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 -- 172.21.15.64:0/1592168071 shutdown_connections 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 --2- 172.21.15.64:0/1592168071 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c280b9a10 0x7f5c280b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 --2- 172.21.15.64:0/1592168071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b95a0 0x7f5c280b5700 unknown :-1 s=CLOSED pgs=1450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 -- 172.21.15.64:0/1592168071 >> 172.21.15.64:0/1592168071 conn(0x7f5c28071960 msgr2=0x7f5c28071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:21.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 -- 172.21.15.64:0/1592168071 shutdown_connections 2024-03-04T07:33:21.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.069+0000 7f5c30239640 1 -- 172.21.15.64:0/1592168071 wait complete. 2024-03-04T07:33:21.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.070+0000 7f5c30239640 1 Processor -- start 2024-03-04T07:33:21.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.070+0000 7f5c30239640 1 -- start start 2024-03-04T07:33:21.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c280b95a0 0x7f5c28158720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b9a10 0x7f5c281e8110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c2815df00 0x7f5c280ccd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c280cedf0 con 0x7f5c280b9a10 2024-03-04T07:33:21.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c280cec70 con 0x7f5c280b95a0 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5c280d3220 con 0x7f5c2815df00 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b9a10 0x7f5c281e8110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b9a10 0x7f5c281e8110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37188/0 (socket says 172.21.15.64:37188) 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1547361442 learned_addr learned my addr 172.21.15.64:0/1547361442 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2e7d6640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c2815df00 0x7f5c280ccd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2dfd5640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c280b95a0 0x7f5c28158720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c2815df00 msgr2=0x7f5c280ccd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c2815df00 0x7f5c280ccd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c280b95a0 msgr2=0x7f5c28158720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c280b95a0 0x7f5c28158720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 -- 172.21.15.64:0/1547361442 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c281e8650 con 0x7f5c280b9a10 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2e7d6640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c2815df00 0x7f5c280ccd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2dfd5640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c280b95a0 0x7f5c28158720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:21.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c2d7d4640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b9a10 0x7f5c281e8110 secure :-1 s=READY pgs=1451 cs=0 l=1 rev1=1 crypto rx=0x7f5c1802b930 tx=0x7f5c180046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c1e7fc640 1 -- 172.21.15.64:0/1547361442 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c18040070 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c1e7fc640 1 -- 172.21.15.64:0/1547361442 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5c18039c10 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c281e88d0 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c1e7fc640 1 -- 172.21.15.64:0/1547361442 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c18035040 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.071+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c281e8e90 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.072+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c28067ff0 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.073+0000 7f5c1e7fc640 1 -- 172.21.15.64:0/1547361442 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5c1804b060 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.073+0000 7f5c1e7fc640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c0c06b500 0x7f5c0c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.073+0000 7f5c1e7fc640 1 -- 172.21.15.64:0/1547361442 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f5c180abd50 con 0x7f5c280b9a10 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.073+0000 7f5c2dfd5640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c0c06b500 0x7f5c0c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.074+0000 7f5c2dfd5640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c0c06b500 0x7f5c0c06d9c0 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7f5c280b5750 tx=0x7f5c24008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.081 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.082+0000 7f5c1e7fc640 1 -- 172.21.15.64:0/1547361442 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5c180af030 con 0x7f5c280b9a10 2024-03-04T07:33:21.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.300+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5c280d3000 con 0x7f5c280b9a10 2024-03-04T07:33:21.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.301+0000 7f5c1e7fc640 1 -- 172.21.15.64:0/1547361442 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f5c281e8650 con 0x7f5c280b9a10 2024-03-04T07:33:21.301 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:21.301 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:21.301 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c0c06b500 msgr2=0x7f5c0c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c0c06b500 0x7f5c0c06d9c0 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7f5c280b5750 tx=0x7f5c24008040 comp rx=0 tx=0).stop 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b9a10 msgr2=0x7f5c281e8110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b9a10 0x7f5c281e8110 secure :-1 s=READY pgs=1451 cs=0 l=1 rev1=1 crypto rx=0x7f5c1802b930 tx=0x7f5c180046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 shutdown_connections 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5c0c06b500 0x7f5c0c06d9c0 unknown :-1 s=CLOSED pgs=651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5c2815df00 0x7f5c280ccd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5c280b9a10 0x7f5c281e8110 unknown :-1 s=CLOSED pgs=1451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 --2- 172.21.15.64:0/1547361442 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5c280b95a0 0x7f5c28158720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.302+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 >> 172.21.15.64:0/1547361442 conn(0x7f5c28071960 msgr2=0x7f5c280b8a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.303+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 shutdown_connections 2024-03-04T07:33:21.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.303+0000 7f5c30239640 1 -- 172.21.15.64:0/1547361442 wait complete. 2024-03-04T07:33:21.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:21.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.415+0000 7f96b0615640 1 Processor -- start 2024-03-04T07:33:21.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.415+0000 7f96b0615640 1 -- start start 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96b0615640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 0x7f96a80cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96b0615640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f96a80b2320 con 0x7f96a80d06c0 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96b0615640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f96a80b0dd0 con 0x7f96a80b4260 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96b0615640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96a80b0b80 con 0x7f96a80d02e0 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96aebb2640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f96a80d06c0 0x7f96a80b1c10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36394/0 (socket says 172.21.15.64:36394) 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96aebb2640 1 -- 172.21.15.64:0/844307897 learned_addr learned my addr 172.21.15.64:0/844307897 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2382534736 0 0) 0x7f96a80b2320 con 0x7f96a80d06c0 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ae3b1640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 0x7f96a80cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9694003520 con 0x7f96a80d06c0 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4025806391 0 0) 0x7f9694003520 con 0x7f96a80d06c0 2024-03-04T07:33:21.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 msgr2=0x7f96a80cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 0x7f96a80cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 >> v1:172.21.15.66:6789/0 conn(0x7f96a80b4260 legacy=0x7f96a80b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96a80b10b0 con 0x7f96a80d06c0 2024-03-04T07:33:21.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f96a00019f0 con 0x7f96a80d06c0 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 >> v1:172.21.15.64:6789/0 conn(0x7f96a80d06c0 legacy=0x7f96a80b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96940039e0 0x7f9694003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9694004800 0x7f9694004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 0x7f96940051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.417+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96a80b0b80 con 0x7f9694004800 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f969400bb30 con 0x7f96940039e0 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f969400bd30 con 0x7f96a80d02e0 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96adbb0640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9694004800 0x7f9694004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96ae3b1640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96940039e0 0x7f9694003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96aebb2640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 0x7f96940051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96adbb0640 1 -- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 msgr2=0x7f96940051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96adbb0640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 0x7f96940051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96adbb0640 1 -- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96940039e0 msgr2=0x7f9694003e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96adbb0640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96940039e0 0x7f9694003e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96adbb0640 1 -- 172.21.15.64:0/844307897 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9694003520 con 0x7f9694004800 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.418+0000 7f96aebb2640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a80d02e0 0x7f96940051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96adbb0640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9694004800 0x7f9694004c80 secure :-1 s=READY pgs=1453 cs=0 l=1 rev1=1 crypto rx=0x7f96a80b46a0 tx=0x7f9694005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cfcf59ed2cf7ab28 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969c001370 con 0x7f9694004800 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f969c0052d0 con 0x7f9694004800 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96ad3af640 1 -- 172.21.15.64:0/844307897 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969c003830 con 0x7f9694004800 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96b0615640 1 -- 172.21.15.64:0/844307897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9694004800 msgr2=0x7f9694004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96b0615640 1 --2- 172.21.15.64:0/844307897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9694004800 0x7f9694004c80 secure :-1 s=READY pgs=1453 cs=0 l=1 rev1=1 crypto rx=0x7f96a80b46a0 tx=0x7f9694005830 comp rx=0 tx=0).stop 2024-03-04T07:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96aebb2640 1 -- 172.21.15.64:0/844307897 reap_dead start 2024-03-04T07:33:21.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96b0615640 1 -- 172.21.15.64:0/844307897 shutdown_connections 2024-03-04T07:33:21.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96b0615640 1 -- 172.21.15.64:0/844307897 >> 172.21.15.64:0/844307897 conn(0x7f96a80b36d0 msgr2=0x7f96a80b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:21.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96b0615640 1 -- 172.21.15.64:0/844307897 shutdown_connections 2024-03-04T07:33:21.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.419+0000 7f96b0615640 1 -- 172.21.15.64:0/844307897 wait complete. 2024-03-04T07:33:21.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 Processor -- start 2024-03-04T07:33:21.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 -- start start 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9694004800 0x7f96a81f1710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a8147c90 0x7f96a8147350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a81f5300 0x7f96a81f5780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96a80b0360 con 0x7f96a8147c90 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96a80aeec0 con 0x7f9694004800 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96b0615640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96a80b04e0 con 0x7f96a81f5300 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96adbb0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a8147c90 0x7f96a8147350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96adbb0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a8147c90 0x7f96a8147350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37212/0 (socket says 172.21.15.64:37212) 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96adbb0640 1 -- 172.21.15.64:0/23330265 learned_addr learned my addr 172.21.15.64:0/23330265 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96ae3b1640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9694004800 0x7f96a81f1710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.421+0000 7f96aebb2640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a81f5300 0x7f96a81f5780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96adbb0640 1 -- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a81f5300 msgr2=0x7f96a81f5780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96adbb0640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a81f5300 0x7f96a81f5780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96adbb0640 1 -- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9694004800 msgr2=0x7f96a81f1710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96adbb0640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9694004800 0x7f96a81f1710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96adbb0640 1 -- 172.21.15.64:0/23330265 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96a8147890 con 0x7f96a8147c90 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96ae3b1640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9694004800 0x7f96a81f1710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96aebb2640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a81f5300 0x7f96a81f5780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96adbb0640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a8147c90 0x7f96a8147350 secure :-1 s=READY pgs=1454 cs=0 l=1 rev1=1 crypto rx=0x7f969c000c00 tx=0x7f969c001120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f969affd640 1 -- 172.21.15.64:0/23330265 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969c041070 con 0x7f96a8147c90 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f969affd640 1 -- 172.21.15.64:0/23330265 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f969c005480 con 0x7f96a8147c90 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f969affd640 1 -- 172.21.15.64:0/23330265 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969c007a30 con 0x7f96a8147c90 2024-03-04T07:33:21.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96a8158370 con 0x7f96a8147c90 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.422+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96a8158930 con 0x7f96a8147c90 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.423+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f96a8068700 con 0x7f96a8147c90 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.423+0000 7f969affd640 1 -- 172.21.15.64:0/23330265 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f969c044070 con 0x7f96a8147c90 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.424+0000 7f969affd640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f968c06b500 0x7f968c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.424+0000 7f969affd640 1 -- 172.21.15.64:0/23330265 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f969c0abd40 con 0x7f96a8147c90 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.424+0000 7f96ae3b1640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f968c06b500 0x7f968c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.425+0000 7f96ae3b1640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f968c06b500 0x7f968c06d9c0 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f96a4009bc0 tx=0x7f96a4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.428+0000 7f969affd640 1 -- 172.21.15.64:0/23330265 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f96a8147890 con 0x7f96a8147c90 2024-03-04T07:33:21.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.616+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f96a80b1b00 con 0x7f96a8147c90 2024-03-04T07:33:21.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.617+0000 7f969affd640 1 -- 172.21.15.64:0/23330265 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v111) v1 ==== 74+0+16934 (secure 0 0 0) 0x7f969c006e20 con 0x7f96a8147c90 2024-03-04T07:33:21.617 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:21.617 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":111,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:35.558693+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"105","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.25","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.64:6835/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.66:6835/4084900379":"2024-03-05T07:32:21.508754+0000","172.21.15.66:6833/4084900379":"2024-03-05T07:32:21.508754+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:21.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f968c06b500 msgr2=0x7f968c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f968c06b500 0x7f968c06d9c0 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f96a4009bc0 tx=0x7f96a4008040 comp rx=0 tx=0).stop 2024-03-04T07:33:21.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a8147c90 msgr2=0x7f96a8147350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a8147c90 0x7f96a8147350 secure :-1 s=READY pgs=1454 cs=0 l=1 rev1=1 crypto rx=0x7f969c000c00 tx=0x7f969c001120 comp rx=0 tx=0).stop 2024-03-04T07:33:21.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 shutdown_connections 2024-03-04T07:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f968c06b500 0x7f968c06d9c0 unknown :-1 s=CLOSED pgs=652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96a81f5300 0x7f96a81f5780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96a8147c90 0x7f96a8147350 unknown :-1 s=CLOSED pgs=1454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 --2- 172.21.15.64:0/23330265 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9694004800 0x7f96a81f1710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 >> 172.21.15.64:0/23330265 conn(0x7f96a80b36d0 msgr2=0x7f96a80d2480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 shutdown_connections 2024-03-04T07:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.619+0000 7f96b0615640 1 -- 172.21.15.64:0/23330265 wait complete. 2024-03-04T07:33:21.630 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 16, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:32:07.087121+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '104', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 17, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:32:07.785433+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '105', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.75, 'score_stable': 1.75, 'optimal_score': 1, 'raw_score_acting': 1.75, 'raw_score_stable': 1.75, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:33:21.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:33:21.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.735+0000 7feae7a39640 1 Processor -- start 2024-03-04T07:33:21.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.736+0000 7feae7a39640 1 -- start start 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.736+0000 7feae7a39640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feae00b2cd0 0x7feae00b1c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7feae7a39640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feae0161a40 con 0x7feae00cdcd0 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7feae7a39640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feae00b2a20 con 0x7feae006b2c0 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7feae7a39640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feae00d11e0 con 0x7feae00b2cd0 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7feae57d5640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7feae00cdcd0 0x7feae00b1550 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36400/0 (socket says 172.21.15.64:36400) 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7feae57d5640 1 -- 172.21.15.64:0/2680280304 learned_addr learned my addr 172.21.15.64:0/2680280304 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7feae4fd4640 1 --2- 172.21.15.64:0/2680280304 >> v2:172.21.15.66:3300/0 conn(0x7feae00b2cd0 0x7feae00b1c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 603051149 0 0) 0x7feae0161a40 con 0x7feae00cdcd0 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7feacc003520 con 0x7feae00cdcd0 2024-03-04T07:33:21.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 42905328 0 0) 0x7feacc003520 con 0x7feae00cdcd0 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 >> v2:172.21.15.66:3300/0 conn(0x7feae00b2cd0 msgr2=0x7feae00b1c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 --2- 172.21.15.64:0/2680280304 >> v2:172.21.15.66:3300/0 conn(0x7feae00b2cd0 0x7feae00b1c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 >> v1:172.21.15.66:6789/0 conn(0x7feae006b2c0 legacy=0x7feae006b710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feae00d0940 con 0x7feae00cdcd0 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7feadc001670 con 0x7feae00cdcd0 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 >> v1:172.21.15.64:6789/0 conn(0x7feae00cdcd0 legacy=0x7feae00b1550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feacc003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feacc0043b0 0x7feacc004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feacc005120 0x7feacc00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feae00d11e0 con 0x7feacc0039b0 2024-03-04T07:33:21.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feacc00c060 con 0x7feacc005120 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feacc00c260 con 0x7feacc0043b0 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.737+0000 7feae57d5640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feacc003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae5fd6640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feacc005120 0x7feacc00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae4fd4640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feacc0043b0 0x7feacc004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae57d5640 1 -- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feacc0043b0 msgr2=0x7feacc004880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae57d5640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feacc0043b0 0x7feacc004880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae57d5640 1 -- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feacc005120 msgr2=0x7feacc00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae57d5640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feacc005120 0x7feacc00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae57d5640 1 -- 172.21.15.64:0/2680280304 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feacc003520 con 0x7feacc0039b0 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.738+0000 7feae5fd6640 1 -- 172.21.15.64:0/2680280304 reap_dead start 2024-03-04T07:33:21.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.739+0000 7feae57d5640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feacc003d90 secure :-1 s=READY pgs=1456 cs=0 l=1 rev1=1 crypto rx=0x7feadc00c210 tx=0x7feacc005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d9236bdde487380 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.739+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feadc007d30 con 0x7feacc0039b0 2024-03-04T07:33:21.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.739+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feadc005af0 con 0x7feacc0039b0 2024-03-04T07:33:21.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.739+0000 7fead7fff640 1 -- 172.21.15.64:0/2680280304 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feadc007070 con 0x7feacc0039b0 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.739+0000 7feae7a39640 1 -- 172.21.15.64:0/2680280304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 msgr2=0x7feacc003d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.739+0000 7feae7a39640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feacc003d90 secure :-1 s=READY pgs=1456 cs=0 l=1 rev1=1 crypto rx=0x7feadc00c210 tx=0x7feacc005830 comp rx=0 tx=0).stop 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.740+0000 7feae7a39640 1 -- 172.21.15.64:0/2680280304 shutdown_connections 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.740+0000 7feae7a39640 1 --2- 172.21.15.64:0/2680280304 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feacc003d90 unknown :-1 s=CLOSED pgs=1456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.740+0000 7feae7a39640 1 -- 172.21.15.64:0/2680280304 >> 172.21.15.64:0/2680280304 conn(0x7feae00b3700 msgr2=0x7feae00b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.740+0000 7feae7a39640 1 -- 172.21.15.64:0/2680280304 shutdown_connections 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.740+0000 7feae7a39640 1 -- 172.21.15.64:0/2680280304 wait complete. 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.741+0000 7feae7a39640 1 Processor -- start 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.741+0000 7feae7a39640 1 -- start start 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae7a39640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feae00af380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae7a39640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae00c9580 0x7feae0152b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae7a39640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae0153080 0x7feae00caf10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae7a39640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feae00cc1c0 con 0x7feacc0039b0 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae7a39640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feae00d0f00 con 0x7feae0153080 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae7a39640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feae00cc340 con 0x7feae00c9580 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feae00af380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feae00af380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37232/0 (socket says 172.21.15.64:37232) 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 -- 172.21.15.64:0/154936486 learned_addr learned my addr 172.21.15.64:0/154936486 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 -- 172.21.15.64:0/154936486 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae00c9580 msgr2=0x7feae0152b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae00c9580 0x7feae0152b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 -- 172.21.15.64:0/154936486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae0153080 msgr2=0x7feae00caf10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae5fd6640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae0153080 0x7feae00caf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae0153080 0x7feae00caf10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 -- 172.21.15.64:0/154936486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feae00afa60 con 0x7feacc0039b0 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae57d5640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feae00af380 secure :-1 s=READY pgs=1457 cs=0 l=1 rev1=1 crypto rx=0x7feadc00c330 tx=0x7feae00d0b10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7fead5ffb640 1 -- 172.21.15.64:0/154936486 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feadc042040 con 0x7feacc0039b0 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7fead5ffb640 1 -- 172.21.15.64:0/154936486 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feadc04c070 con 0x7feacc0039b0 2024-03-04T07:33:21.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7fead5ffb640 1 -- 172.21.15.64:0/154936486 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feadc007d30 con 0x7feacc0039b0 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.742+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feae00afd40 con 0x7feacc0039b0 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.743+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feae0153500 con 0x7feacc0039b0 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.743+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feae00b33c0 con 0x7feacc0039b0 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.744+0000 7fead5ffb640 1 -- 172.21.15.64:0/154936486 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feadc007070 con 0x7feacc0039b0 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.744+0000 7fead5ffb640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feac406b4d0 0x7feac406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.745+0000 7feae4fd4640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feac406b4d0 0x7feac406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.745+0000 7fead5ffb640 1 -- 172.21.15.64:0/154936486 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7feadc0b4490 con 0x7feacc0039b0 2024-03-04T07:33:21.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.745+0000 7feae4fd4640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feac406b4d0 0x7feac406d990 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7feae0070a90 tx=0x7fead0008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:21.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.750+0000 7fead5ffb640 1 -- 172.21.15.64:0/154936486 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feae00b33c0 con 0x7feacc0039b0 2024-03-04T07:33:21.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.929+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7feae00cbfa0 con 0x7feacc0039b0 2024-03-04T07:33:21.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.929+0000 7fead5ffb640 1 -- 172.21.15.64:0/154936486 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v111) v1 ==== 93+0+10 (secure 0 0 0) 0x7feadc0833d0 con 0x7feacc0039b0 2024-03-04T07:33:21.929 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:33:21.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feac406b4d0 msgr2=0x7feac406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feac406b4d0 0x7feac406d990 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7feae0070a90 tx=0x7fead0008040 comp rx=0 tx=0).stop 2024-03-04T07:33:21.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 msgr2=0x7feae00af380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:21.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feae00af380 secure :-1 s=READY pgs=1457 cs=0 l=1 rev1=1 crypto rx=0x7feadc00c330 tx=0x7feae00d0b10 comp rx=0 tx=0).stop 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 shutdown_connections 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feac406b4d0 0x7feac406d990 unknown :-1 s=CLOSED pgs=653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feae0153080 0x7feae00caf10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feae00c9580 0x7feae0152b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 --2- 172.21.15.64:0/154936486 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feacc0039b0 0x7feae00af380 unknown :-1 s=CLOSED pgs=1457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 >> 172.21.15.64:0/154936486 conn(0x7feae00b3700 msgr2=0x7feae015d310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 shutdown_connections 2024-03-04T07:33:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:21.931+0000 7feae7a39640 1 -- 172.21.15.64:0/154936486 wait complete. 2024-03-04T07:33:21.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:33:22.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.047+0000 7fbf43981640 1 Processor -- start 2024-03-04T07:33:22.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.047+0000 7fbf43981640 1 -- start start 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf43981640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 0x7fbf3c0b1c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf43981640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbf3c161a40 con 0x7fbf3c06b2c0 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf43981640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbf3c0b2a20 con 0x7fbf3c0cdcd0 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf43981640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbf3c0d11e0 con 0x7fbf3c0b2cd0 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf41f1e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbf3c06b2c0 0x7fbf3c06b750 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36410/0 (socket says 172.21.15.64:36410) 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf41f1e640 1 -- 172.21.15.64:0/1411485450 learned_addr learned my addr 172.21.15.64:0/1411485450 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf40f1c640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 0x7fbf3c0b1c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3461910060 0 0) 0x7fbf3c161a40 con 0x7fbf3c06b2c0 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.048+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fbf28003520 con 0x7fbf3c06b2c0 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 688263191 0 0) 0x7fbf28003520 con 0x7fbf3c06b2c0 2024-03-04T07:33:22.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 msgr2=0x7fbf3c0b1c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 0x7fbf3c0b1c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 >> v1:172.21.15.66:6789/0 conn(0x7fbf3c0cdcd0 legacy=0x7fbf3c0b1550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf3c0d0940 con 0x7fbf3c06b2c0 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fbf340019f0 con 0x7fbf3c06b2c0 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 >> v1:172.21.15.64:6789/0 conn(0x7fbf3c06b2c0 legacy=0x7fbf3c06b750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf28003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf28004390 0x7fbf28004860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 0x7fbf28005160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf3c0d11e0 con 0x7fbf280039e0 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf2800ba70 con 0x7fbf28004390 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbf2800bc70 con 0x7fbf3c0b2cd0 2024-03-04T07:33:22.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf4171d640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf28003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf40f1c640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf28004390 0x7fbf28004860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf41f1e640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 0x7fbf28005160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf4171d640 1 -- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 msgr2=0x7fbf28005160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf4171d640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c0b2cd0 0x7fbf28005160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf4171d640 1 -- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf28004390 msgr2=0x7fbf28004860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf4171d640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf28004390 0x7fbf28004860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf4171d640 1 -- 172.21.15.64:0/1411485450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf28003520 con 0x7fbf280039e0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.049+0000 7fbf40f1c640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf28004390 0x7fbf28004860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.050+0000 7fbf4171d640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf28003dc0 secure :-1 s=READY pgs=1459 cs=0 l=1 rev1=1 crypto rx=0x7fbf38001860 tx=0x7fbf28005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c1f10354e4628b59 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.050+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf3802f050 con 0x7fbf280039e0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.050+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf38033650 con 0x7fbf280039e0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.050+0000 7fbf2ffff640 1 -- 172.21.15.64:0/1411485450 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf38001430 con 0x7fbf280039e0 2024-03-04T07:33:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.051+0000 7fbf43981640 1 -- 172.21.15.64:0/1411485450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 msgr2=0x7fbf28003dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.051+0000 7fbf43981640 1 --2- 172.21.15.64:0/1411485450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf28003dc0 secure :-1 s=READY pgs=1459 cs=0 l=1 rev1=1 crypto rx=0x7fbf38001860 tx=0x7fbf28005830 comp rx=0 tx=0).stop 2024-03-04T07:33:22.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.051+0000 7fbf41f1e640 1 -- 172.21.15.64:0/1411485450 reap_dead start 2024-03-04T07:33:22.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.051+0000 7fbf43981640 1 -- 172.21.15.64:0/1411485450 shutdown_connections 2024-03-04T07:33:22.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.051+0000 7fbf43981640 1 -- 172.21.15.64:0/1411485450 >> 172.21.15.64:0/1411485450 conn(0x7fbf3c0b3700 msgr2=0x7fbf3c0b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:22.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.051+0000 7fbf43981640 1 -- 172.21.15.64:0/1411485450 shutdown_connections 2024-03-04T07:33:22.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.051+0000 7fbf43981640 1 -- 172.21.15.64:0/1411485450 wait complete. 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.052+0000 7fbf43981640 1 Processor -- start 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.052+0000 7fbf43981640 1 -- start start 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.053+0000 7fbf43981640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf3c0b47f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.053+0000 7fbf43981640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c1f7720 0x7fbf3c1567c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.053+0000 7fbf43981640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf3c0cec20 0x7fbf3c14fbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.053+0000 7fbf43981640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf3c0cc1c0 con 0x7fbf280039e0 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.053+0000 7fbf43981640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf3c0d0f00 con 0x7fbf3c0cec20 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.053+0000 7fbf43981640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbf3c0cc340 con 0x7fbf3c1f7720 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf3c0b47f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf3c0b47f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37256/0 (socket says 172.21.15.64:37256) 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 -- 172.21.15.64:0/827277682 learned_addr learned my addr 172.21.15.64:0/827277682 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf40f1c640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c1f7720 0x7fbf3c1567c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf41f1e640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf3c0cec20 0x7fbf3c14fbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 -- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c1f7720 msgr2=0x7fbf3c1567c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c1f7720 0x7fbf3c1567c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 -- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf3c0cec20 msgr2=0x7fbf3c14fbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf3c0cec20 0x7fbf3c14fbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 -- 172.21.15.64:0/827277682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf3c156d00 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf4171d640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf3c0b47f0 secure :-1 s=READY pgs=1460 cs=0 l=1 rev1=1 crypto rx=0x7fbf3802bdf0 tx=0x7fbf38001230 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf2dffb640 1 -- 172.21.15.64:0/827277682 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf38042070 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf2dffb640 1 -- 172.21.15.64:0/827277682 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf380037f0 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbf3c1f7ce0 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.054+0000 7fbf2dffb640 1 -- 172.21.15.64:0/827277682 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf38001430 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.055+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbf3c1f82a0 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.056+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbf3c09fcf0 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.056+0000 7fbf2dffb640 1 -- 172.21.15.64:0/827277682 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbf3803b040 con 0x7fbf280039e0 2024-03-04T07:33:22.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.056+0000 7fbf2dffb640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf2006b4d0 0x7fbf2006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.056+0000 7fbf2dffb640 1 -- 172.21.15.64:0/827277682 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fbf380ac6d0 con 0x7fbf280039e0 2024-03-04T07:33:22.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.056+0000 7fbf40f1c640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf2006b4d0 0x7fbf2006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.057+0000 7fbf40f1c640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf2006b4d0 0x7fbf2006d990 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7fbf3c1609e0 tx=0x7fbf30008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.061+0000 7fbf2dffb640 1 -- 172.21.15.64:0/827277682 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbf3804b060 con 0x7fbf280039e0 2024-03-04T07:33:22.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.395+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fbf3c04f410 con 0x7fbf280039e0 2024-03-04T07:33:22.395 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.395+0000 7fbf2dffb640 1 -- 172.21.15.64:0/827277682 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v111) v1 ==== 104+0+11 (secure 0 0 0) 0x7fbf3c156d00 con 0x7fbf280039e0 2024-03-04T07:33:22.395 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:33:22.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf2006b4d0 msgr2=0x7fbf2006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf2006b4d0 0x7fbf2006d990 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7fbf3c1609e0 tx=0x7fbf30008040 comp rx=0 tx=0).stop 2024-03-04T07:33:22.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 msgr2=0x7fbf3c0b47f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf3c0b47f0 secure :-1 s=READY pgs=1460 cs=0 l=1 rev1=1 crypto rx=0x7fbf3802bdf0 tx=0x7fbf38001230 comp rx=0 tx=0).stop 2024-03-04T07:33:22.398 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 shutdown_connections 2024-03-04T07:33:22.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf2006b4d0 0x7fbf2006d990 unknown :-1 s=CLOSED pgs=654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf3c0cec20 0x7fbf3c14fbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf3c1f7720 0x7fbf3c1567c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 --2- 172.21.15.64:0/827277682 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf280039e0 0x7fbf3c0b47f0 unknown :-1 s=CLOSED pgs=1460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 >> 172.21.15.64:0/827277682 conn(0x7fbf3c0b3700 msgr2=0x7fbf3c0cd350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:22.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 shutdown_connections 2024-03-04T07:33:22.399 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.398+0000 7fbf43981640 1 -- 172.21.15.64:0/827277682 wait complete. 2024-03-04T07:33:22.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:33:22.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.545+0000 7fc5b6efb640 1 Processor -- start 2024-03-04T07:33:22.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.545+0000 7fc5b6efb640 1 -- start start 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b6efb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc5b0071540 0x7fc5b0071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b6efb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5b0071e60 con 0x7fc5b00cee80 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b6efb640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5b00d4960 con 0x7fc5b00b2d30 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b6efb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5b00d3230 con 0x7fc5b0071540 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b4c97640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc5b0071540 0x7fc5b0071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b5498640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc5b00cee80 0x7fc5b0168900 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36424/0 (socket says 172.21.15.64:36424) 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b4c97640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc5b0071540 0x7fc5b0071920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37266/0 (socket says 172.21.15.64:37266) 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.546+0000 7fc5b5498640 1 -- 172.21.15.64:0/3786375843 learned_addr learned my addr 172.21.15.64:0/3786375843 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.547+0000 7fc5b4c97640 1 -- 172.21.15.64:0/3786375843 >> v1:172.21.15.66:6789/0 conn(0x7fc5b00b2d30 legacy=0x7fc5b00ce770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.547+0000 7fc5b4c97640 1 -- 172.21.15.64:0/3786375843 >> v1:172.21.15.64:6789/0 conn(0x7fc5b00cee80 legacy=0x7fc5b0168900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.547+0000 7fc5af7fe640 1 -- 172.21.15.64:0/3786375843 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2976771123 0 0) 0x7fc5b0071e60 con 0x7fc5b00cee80 2024-03-04T07:33:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.547+0000 7fc5b4c97640 1 -- 172.21.15.64:0/3786375843 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b00d36c0 con 0x7fc5b0071540 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.548+0000 7fc5b4c97640 1 --2- 172.21.15.64:0/3786375843 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b0071540 0x7fc5b0071920 secure :-1 s=READY pgs=1462 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8009d90 tx=0x7fc5a802beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9224175eb6908ca server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.548+0000 7fc5af7fe640 1 -- 172.21.15.64:0/3786375843 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a8039070 con 0x7fc5b0071540 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.548+0000 7fc5af7fe640 1 -- 172.21.15.64:0/3786375843 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5a802cc60 con 0x7fc5b0071540 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.548+0000 7fc5af7fe640 1 -- 172.21.15.64:0/3786375843 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a8034660 con 0x7fc5b0071540 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.548+0000 7fc5b6efb640 1 -- 172.21.15.64:0/3786375843 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b0071540 msgr2=0x7fc5b0071920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.548+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/3786375843 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b0071540 0x7fc5b0071920 secure :-1 s=READY pgs=1462 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8009d90 tx=0x7fc5a802beb0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.549+0000 7fc5b6efb640 1 -- 172.21.15.64:0/3786375843 shutdown_connections 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.549+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/3786375843 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b0071540 0x7fc5b0071920 unknown :-1 s=CLOSED pgs=1462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.549+0000 7fc5b6efb640 1 -- 172.21.15.64:0/3786375843 >> 172.21.15.64:0/3786375843 conn(0x7fc5b00d3d60 msgr2=0x7fc5b00d4170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.549+0000 7fc5b6efb640 1 -- 172.21.15.64:0/3786375843 shutdown_connections 2024-03-04T07:33:22.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.549+0000 7fc5b6efb640 1 -- 172.21.15.64:0/3786375843 wait complete. 2024-03-04T07:33:22.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.550+0000 7fc5b6efb640 1 Processor -- start 2024-03-04T07:33:22.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.550+0000 7fc5b6efb640 1 -- start start 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0071540 0x7fc5b00cd3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b2d30 0x7fc5b0156d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00cee80 0x7fc5b01572d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00d2be0 con 0x7fc5b00cee80 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00d2900 con 0x7fc5b00b2d30 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00d2ec0 con 0x7fc5b0071540 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00cee80 0x7fc5b01572d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00cee80 0x7fc5b01572d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37270/0 (socket says 172.21.15.64:37270) 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 -- 172.21.15.64:0/1476985168 learned_addr learned my addr 172.21.15.64:0/1476985168 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5affff640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b2d30 0x7fc5b0156d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b4c97640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0071540 0x7fc5b00cd3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 -- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0071540 msgr2=0x7fc5b00cd3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0071540 0x7fc5b00cd3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 -- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b2d30 msgr2=0x7fc5b0156d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b2d30 0x7fc5b0156d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 -- 172.21.15.64:0/1476985168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b00cc310 con 0x7fc5b00cee80 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b5498640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00cee80 0x7fc5b01572d0 secure :-1 s=READY pgs=1463 cs=0 l=1 rev1=1 crypto rx=0x7fc59800c450 tx=0x7fc59800c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5ad7fa640 1 -- 172.21.15.64:0/1476985168 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc598018070 con 0x7fc5b00cee80 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5ad7fa640 1 -- 172.21.15.64:0/1476985168 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc598007500 con 0x7fc5b00cee80 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5b00cc5f0 con 0x7fc5b00cee80 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5ad7fa640 1 -- 172.21.15.64:0/1476985168 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc598004a20 con 0x7fc5b00cee80 2024-03-04T07:33:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.551+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5b00ccbb0 con 0x7fc5b00cee80 2024-03-04T07:33:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.552+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5b0072070 con 0x7fc5b00cee80 2024-03-04T07:33:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.553+0000 7fc5ad7fa640 1 -- 172.21.15.64:0/1476985168 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc59801b070 con 0x7fc5b00cee80 2024-03-04T07:33:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.553+0000 7fc5ad7fa640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59006b4d0 0x7fc59006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.553+0000 7fc5ad7fa640 1 -- 172.21.15.64:0/1476985168 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fc598089430 con 0x7fc5b00cee80 2024-03-04T07:33:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.553+0000 7fc5b4c97640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59006b4d0 0x7fc59006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.554+0000 7fc5b4c97640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59006b4d0 0x7fc59006d990 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8036000 tx=0x7fc5a8030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.558+0000 7fc5ad7fa640 1 -- 172.21.15.64:0/1476985168 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc598057c00 con 0x7fc5b00cee80 2024-03-04T07:33:22.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.739+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fc5b00d1c90 con 0x7fc5b00cee80 2024-03-04T07:33:22.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.740+0000 7fc5ad7fa640 1 -- 172.21.15.64:0/1476985168 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v111) v1 ==== 100+0+11 (secure 0 0 0) 0x7fc59805bca0 con 0x7fc5b00cee80 2024-03-04T07:33:22.740 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59006b4d0 msgr2=0x7fc59006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59006b4d0 0x7fc59006d990 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8036000 tx=0x7fc5a8030000 comp rx=0 tx=0).stop 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00cee80 msgr2=0x7fc5b01572d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00cee80 0x7fc5b01572d0 secure :-1 s=READY pgs=1463 cs=0 l=1 rev1=1 crypto rx=0x7fc59800c450 tx=0x7fc59800c910 comp rx=0 tx=0).stop 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 shutdown_connections 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59006b4d0 0x7fc59006d990 unknown :-1 s=CLOSED pgs=655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00cee80 0x7fc5b01572d0 unknown :-1 s=CLOSED pgs=1463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b2d30 0x7fc5b0156d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 --2- 172.21.15.64:0/1476985168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0071540 0x7fc5b00cd3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 >> 172.21.15.64:0/1476985168 conn(0x7fc5b00d3d60 msgr2=0x7fc5b00b9380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:22.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 shutdown_connections 2024-03-04T07:33:22.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.742+0000 7fc5b6efb640 1 -- 172.21.15.64:0/1476985168 wait complete. 2024-03-04T07:33:22.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.854+0000 7f644b2c3640 1 Processor -- start 2024-03-04T07:33:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.855+0000 7f644b2c3640 1 -- start start 2024-03-04T07:33:22.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644b2c3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f64440b0e90 0x7f644406b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644b2c3640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f64440cee90 con 0x7f644406b5c0 2024-03-04T07:33:22.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644b2c3640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f64440cfc50 con 0x7f64440ae9b0 2024-03-04T07:33:22.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644b2c3640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f64440cfa00 con 0x7f64440b0e90 2024-03-04T07:33:22.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644885e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f64440b0e90 0x7f644406b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644885e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f64440b0e90 0x7f644406b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37278/0 (socket says 172.21.15.64:37278) 2024-03-04T07:33:22.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644885e640 1 -- 172.21.15.64:0/3023442875 learned_addr learned my addr 172.21.15.64:0/3023442875 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:22.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644885e640 1 -- 172.21.15.64:0/3023442875 >> v1:172.21.15.66:6789/0 conn(0x7f64440ae9b0 legacy=0x7f64440d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:22.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644885e640 1 -- 172.21.15.64:0/3023442875 >> v1:172.21.15.64:6789/0 conn(0x7f644406b5c0 legacy=0x7f64440b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.856+0000 7f644885e640 1 -- 172.21.15.64:0/3023442875 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64440cff30 con 0x7f64440b0e90 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.857+0000 7f644885e640 1 --2- 172.21.15.64:0/3023442875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64440b0e90 0x7f644406b9a0 secure :-1 s=READY pgs=1465 cs=0 l=1 rev1=1 crypto rx=0x7f6434009870 tx=0x7f643402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=983380c1d9c8e5be server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.857+0000 7f643bfff640 1 -- 172.21.15.64:0/3023442875 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6434035020 con 0x7f64440b0e90 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.857+0000 7f643bfff640 1 -- 172.21.15.64:0/3023442875 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6434031870 con 0x7f64440b0e90 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.857+0000 7f643bfff640 1 -- 172.21.15.64:0/3023442875 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6434031c40 con 0x7f64440b0e90 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.858+0000 7f644b2c3640 1 -- 172.21.15.64:0/3023442875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64440b0e90 msgr2=0x7f644406b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.858+0000 7f644b2c3640 1 --2- 172.21.15.64:0/3023442875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64440b0e90 0x7f644406b9a0 secure :-1 s=READY pgs=1465 cs=0 l=1 rev1=1 crypto rx=0x7f6434009870 tx=0x7f643402b910 comp rx=0 tx=0).stop 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.858+0000 7f644b2c3640 1 -- 172.21.15.64:0/3023442875 shutdown_connections 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.858+0000 7f644b2c3640 1 --2- 172.21.15.64:0/3023442875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f64440b0e90 0x7f644406b9a0 unknown :-1 s=CLOSED pgs=1465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.858+0000 7f644b2c3640 1 -- 172.21.15.64:0/3023442875 >> 172.21.15.64:0/3023442875 conn(0x7f64440b36d0 msgr2=0x7f64440b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:22.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.858+0000 7f644b2c3640 1 -- 172.21.15.64:0/3023442875 shutdown_connections 2024-03-04T07:33:22.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.858+0000 7f644b2c3640 1 -- 172.21.15.64:0/3023442875 wait complete. 2024-03-04T07:33:22.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.859+0000 7f644b2c3640 1 Processor -- start 2024-03-04T07:33:22.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.859+0000 7f644b2c3640 1 -- start start 2024-03-04T07:33:22.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644b2c3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f644406b5c0 0x7f64441f4690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644b2c3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64440ae9b0 0x7f644415ae00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644b2c3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64440b0e90 0x7f644406fdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644b2c3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f644406c910 con 0x7f644406b5c0 2024-03-04T07:33:22.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644b2c3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f644406c790 con 0x7f64440b0e90 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644b2c3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f64440d0ab0 con 0x7f64440ae9b0 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f644406b5c0 0x7f64441f4690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f644406b5c0 0x7f64441f4690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37290/0 (socket says 172.21.15.64:37290) 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 -- 172.21.15.64:0/2558602061 learned_addr learned my addr 172.21.15.64:0/2558602061 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644885e640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64440ae9b0 0x7f644415ae00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f6449860640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64440b0e90 0x7f644406fdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 -- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64440ae9b0 msgr2=0x7f644415ae00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64440ae9b0 0x7f644415ae00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 -- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64440b0e90 msgr2=0x7f644406fdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64440b0e90 0x7f644406fdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.860+0000 7f644905f640 1 -- 172.21.15.64:0/2558602061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64440c71b0 con 0x7f644406b5c0 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.861+0000 7f644905f640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f644406b5c0 0x7f64441f4690 secure :-1 s=READY pgs=1466 cs=0 l=1 rev1=1 crypto rx=0x7f644000c3f0 tx=0x7f644000c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.861+0000 7f6439ffb640 1 -- 172.21.15.64:0/2558602061 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64400052d0 con 0x7f644406b5c0 2024-03-04T07:33:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.861+0000 7f6439ffb640 1 -- 172.21.15.64:0/2558602061 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6440005490 con 0x7f644406b5c0 2024-03-04T07:33:22.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.861+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f64441f4bd0 con 0x7f644406b5c0 2024-03-04T07:33:22.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.861+0000 7f6439ffb640 1 -- 172.21.15.64:0/2558602061 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6440007260 con 0x7f644406b5c0 2024-03-04T07:33:22.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.861+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64440c5dd0 con 0x7f644406b5c0 2024-03-04T07:33:22.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.862+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6444068700 con 0x7f644406b5c0 2024-03-04T07:33:22.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.863+0000 7f6439ffb640 1 -- 172.21.15.64:0/2558602061 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6440007400 con 0x7f644406b5c0 2024-03-04T07:33:22.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.863+0000 7f6439ffb640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f642806b4d0 0x7f642806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:22.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.863+0000 7f6439ffb640 1 -- 172.21.15.64:0/2558602061 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f6440089720 con 0x7f644406b5c0 2024-03-04T07:33:22.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.863+0000 7f644885e640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f642806b4d0 0x7f642806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:22.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.864+0000 7f644885e640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f642806b4d0 0x7f642806d990 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x7f6434036000 tx=0x7f643403c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:22.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:22.868+0000 7f6439ffb640 1 -- 172.21.15.64:0/2558602061 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f64440c71b0 con 0x7f644406b5c0 2024-03-04T07:33:23.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.090+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f644406daf0 con 0x7f644406b5c0 2024-03-04T07:33:23.090 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.091+0000 7f6439ffb640 1 -- 172.21.15.64:0/2558602061 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f644005ce20 con 0x7f644406b5c0 2024-03-04T07:33:23.091 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:23.091 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:23.091 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:23.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f642806b4d0 msgr2=0x7f642806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f642806b4d0 0x7f642806d990 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x7f6434036000 tx=0x7f643403c040 comp rx=0 tx=0).stop 2024-03-04T07:33:23.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f644406b5c0 msgr2=0x7f64441f4690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f644406b5c0 0x7f64441f4690 secure :-1 s=READY pgs=1466 cs=0 l=1 rev1=1 crypto rx=0x7f644000c3f0 tx=0x7f644000c8b0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 shutdown_connections 2024-03-04T07:33:23.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f642806b4d0 0x7f642806d990 unknown :-1 s=CLOSED pgs=656 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f64440b0e90 0x7f644406fdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f64440ae9b0 0x7f644415ae00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 --2- 172.21.15.64:0/2558602061 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f644406b5c0 0x7f64441f4690 unknown :-1 s=CLOSED pgs=1466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 >> 172.21.15.64:0/2558602061 conn(0x7f64440b36d0 msgr2=0x7f6444068e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:23.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 shutdown_connections 2024-03-04T07:33:23.094 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.093+0000 7f644b2c3640 1 -- 172.21.15.64:0/2558602061 wait complete. 2024-03-04T07:33:23.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:23.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.207+0000 7fda0de64640 1 Processor -- start 2024-03-04T07:33:23.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.207+0000 7fda0de64640 1 -- start start 2024-03-04T07:33:23.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.207+0000 7fda0de64640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d02e0 0x7fda080cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.207+0000 7fda0de64640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fda080d06c0 0x7fda080b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.207+0000 7fda0de64640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fda080b0dd0 con 0x7fda080b4260 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.207+0000 7fda0de64640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fda080b0b80 con 0x7fda080d06c0 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.207+0000 7fda0de64640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fda080aeec0 con 0x7fda080d02e0 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda07fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fda080d06c0 0x7fda080b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda06ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fda080b4260 0x7fda080b1500 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:36436/0 (socket says 172.21.15.64:36436) 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda06ffd640 1 -- 172.21.15.64:0/2451849596 learned_addr learned my addr 172.21.15.64:0/2451849596 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda077fe640 1 --2- 172.21.15.64:0/2451849596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d02e0 0x7fda080cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda07fff640 1 -- 172.21.15.64:0/2451849596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d02e0 msgr2=0x7fda080cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda07fff640 1 --2- 172.21.15.64:0/2451849596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d02e0 0x7fda080cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda07fff640 1 -- 172.21.15.64:0/2451849596 >> v1:172.21.15.64:6789/0 conn(0x7fda080b4260 legacy=0x7fda080b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda07fff640 1 -- 172.21.15.64:0/2451849596 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fda080b10b0 con 0x7fda080d06c0 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda067fc640 1 -- 172.21.15.64:0/2451849596 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2420286295 0 0) 0x7fda080b0dd0 con 0x7fda080b4260 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.208+0000 7fda077fe640 1 --2- 172.21.15.64:0/2451849596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d02e0 0x7fda080cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda07fff640 1 --2- 172.21.15.64:0/2451849596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d06c0 0x7fda080b1c10 secure :-1 s=READY pgs=1468 cs=0 l=1 rev1=1 crypto rx=0x7fda00009d90 tx=0x7fda0002bf30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4dba2e44bd7fc3a0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda067fc640 1 -- 172.21.15.64:0/2451849596 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda00039070 con 0x7fda080d06c0 2024-03-04T07:33:23.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda067fc640 1 -- 172.21.15.64:0/2451849596 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fda0002cce0 con 0x7fda080d06c0 2024-03-04T07:33:23.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda067fc640 1 -- 172.21.15.64:0/2451849596 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda00034660 con 0x7fda080d06c0 2024-03-04T07:33:23.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 -- 172.21.15.64:0/2451849596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d06c0 msgr2=0x7fda080b1c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.212 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 --2- 172.21.15.64:0/2451849596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d06c0 0x7fda080b1c10 secure :-1 s=READY pgs=1468 cs=0 l=1 rev1=1 crypto rx=0x7fda00009d90 tx=0x7fda0002bf30 comp rx=0 tx=0).stop 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 -- 172.21.15.64:0/2451849596 shutdown_connections 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 --2- 172.21.15.64:0/2451849596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d06c0 0x7fda080b1c10 unknown :-1 s=CLOSED pgs=1468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 --2- 172.21.15.64:0/2451849596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d02e0 0x7fda080cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 -- 172.21.15.64:0/2451849596 >> 172.21.15.64:0/2451849596 conn(0x7fda080b36d0 msgr2=0x7fda080b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 -- 172.21.15.64:0/2451849596 shutdown_connections 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.209+0000 7fda0de64640 1 -- 172.21.15.64:0/2451849596 wait complete. 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.210+0000 7fda0de64640 1 Processor -- start 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda0de64640 1 -- start start 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda0de64640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fda080b4260 0x7fda08157df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda0de64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d02e0 0x7fda08152c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda0de64640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d06c0 0x7fda081f47a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda0de64640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda080b22d0 con 0x7fda080d02e0 2024-03-04T07:33:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda0de64640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda080b2150 con 0x7fda080b4260 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda0de64640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fda080af350 con 0x7fda080d06c0 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.211+0000 7fda06ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d02e0 0x7fda08152c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d02e0 0x7fda08152c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37302/0 (socket says 172.21.15.64:37302) 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 -- 172.21.15.64:0/1342740392 learned_addr learned my addr 172.21.15.64:0/1342740392 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda077fe640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fda080b4260 0x7fda08157df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda07fff640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d06c0 0x7fda081f47a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 -- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d06c0 msgr2=0x7fda081f47a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d06c0 0x7fda081f47a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 -- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fda080b4260 msgr2=0x7fda08157df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fda080b4260 0x7fda08157df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 -- 172.21.15.64:0/1342740392 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fda08157660 con 0x7fda080d02e0 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda07fff640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d06c0 0x7fda081f47a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda077fe640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fda080b4260 0x7fda08157df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda06ffd640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d02e0 0x7fda08152c80 secure :-1 s=READY pgs=1469 cs=0 l=1 rev1=1 crypto rx=0x7fd9fc00c4e0 tx=0x7fd9fc00c9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fd9effff640 1 -- 172.21.15.64:0/1342740392 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9fc018070 con 0x7fda080d02e0 2024-03-04T07:33:23.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fd9effff640 1 -- 172.21.15.64:0/1342740392 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd9fc007500 con 0x7fda080d02e0 2024-03-04T07:33:23.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fd9effff640 1 -- 172.21.15.64:0/1342740392 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9fc004a20 con 0x7fda080d02e0 2024-03-04T07:33:23.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fda081f4ce0 con 0x7fda080d02e0 2024-03-04T07:33:23.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.212+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fda08157380 con 0x7fda080d02e0 2024-03-04T07:33:23.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.213+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fda08068700 con 0x7fda080d02e0 2024-03-04T07:33:23.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.214+0000 7fd9effff640 1 -- 172.21.15.64:0/1342740392 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd9fc004bc0 con 0x7fda080d02e0 2024-03-04T07:33:23.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.214+0000 7fd9effff640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9f406b4d0 0x7fd9f406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.214+0000 7fd9effff640 1 -- 172.21.15.64:0/1342740392 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fd9fc0896a0 con 0x7fda080d02e0 2024-03-04T07:33:23.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.214+0000 7fda077fe640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9f406b4d0 0x7fd9f406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.215+0000 7fda077fe640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9f406b4d0 0x7fd9f406d990 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fd9f800aa30 tx=0x7fd9f8008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.222 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.223+0000 7fd9effff640 1 -- 172.21.15.64:0/1342740392 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd9fc0584f0 con 0x7fda080d02e0 2024-03-04T07:33:23.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.403+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fda080d12e0 con 0x7fda080d02e0 2024-03-04T07:33:23.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.405+0000 7fd9effff640 1 -- 172.21.15.64:0/1342740392 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v111) v1 ==== 74+0+16934 (secure 0 0 0) 0x7fd9fc057df0 con 0x7fda080d02e0 2024-03-04T07:33:23.405 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:23.405 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":111,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:35.558693+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"105","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.25","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.64:6835/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.66:6835/4084900379":"2024-03-05T07:32:21.508754+0000","172.21.15.66:6833/4084900379":"2024-03-05T07:32:21.508754+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:23.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9f406b4d0 msgr2=0x7fd9f406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9f406b4d0 0x7fd9f406d990 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fd9f800aa30 tx=0x7fd9f8008040 comp rx=0 tx=0).stop 2024-03-04T07:33:23.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d02e0 msgr2=0x7fda08152c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d02e0 0x7fda08152c80 secure :-1 s=READY pgs=1469 cs=0 l=1 rev1=1 crypto rx=0x7fd9fc00c4e0 tx=0x7fd9fc00c9a0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 shutdown_connections 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd9f406b4d0 0x7fd9f406d990 unknown :-1 s=CLOSED pgs=657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fda080d06c0 0x7fda081f47a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fda080d02e0 0x7fda08152c80 unknown :-1 s=CLOSED pgs=1469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 --2- 172.21.15.64:0/1342740392 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fda080b4260 0x7fda08157df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 >> 172.21.15.64:0/1342740392 conn(0x7fda080b36d0 msgr2=0x7fda0806bcb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 shutdown_connections 2024-03-04T07:33:23.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.407+0000 7fda0de64640 1 -- 172.21.15.64:0/1342740392 wait complete. 2024-03-04T07:33:23.419 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:33:23.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:23.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.520+0000 7fa0e5b13640 1 Processor -- start 2024-03-04T07:33:23.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.520+0000 7fa0e5b13640 1 -- start start 2024-03-04T07:33:23.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0e5b13640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa0e015df00 0x7fa0e00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0e5b13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b9a30 0x7fa0e00d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0e5b13640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa0e00cea30 con 0x7fa0e00b95a0 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0e5b13640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa0e00d2af0 con 0x7fa0e015df00 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0e5b13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e00cf3c0 con 0x7fa0e00b9a30 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0df7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa0e015df00 0x7fa0e00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0df7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa0e015df00 0x7fa0e00d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51326/0 (socket says 172.21.15.64:51326) 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0df7fe640 1 -- 172.21.15.64:0/2211536941 learned_addr learned my addr 172.21.15.64:0/2211536941 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0dffff640 1 --2- 172.21.15.64:0/2211536941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b9a30 0x7fa0e00d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0df7fe640 1 -- 172.21.15.64:0/2211536941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b9a30 msgr2=0x7fa0e00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0df7fe640 1 --2- 172.21.15.64:0/2211536941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b9a30 0x7fa0e00d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0df7fe640 1 -- 172.21.15.64:0/2211536941 >> v1:172.21.15.66:6789/0 conn(0x7fa0e00b95a0 legacy=0x7fa0e00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.521+0000 7fa0df7fe640 1 -- 172.21.15.64:0/2211536941 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa0e00cf0e0 con 0x7fa0e015df00 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.522+0000 7fa0df7fe640 1 --2- 172.21.15.64:0/2211536941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e00d25b0 secure :-1 s=READY pgs=1470 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0009870 tx=0x7fa0d002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=428e56729362df77 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.522+0000 7fa0de7fc640 1 -- 172.21.15.64:0/2211536941 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d0035020 con 0x7fa0e015df00 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.522+0000 7fa0de7fc640 1 -- 172.21.15.64:0/2211536941 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa0d0031640 con 0x7fa0e015df00 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.522+0000 7fa0de7fc640 1 -- 172.21.15.64:0/2211536941 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d002d040 con 0x7fa0e015df00 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 -- 172.21.15.64:0/2211536941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 msgr2=0x7fa0e00d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/2211536941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e00d25b0 secure :-1 s=READY pgs=1470 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0009870 tx=0x7fa0d002b8f0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 -- 172.21.15.64:0/2211536941 shutdown_connections 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/2211536941 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b9a30 0x7fa0e00d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/2211536941 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e00d25b0 unknown :-1 s=CLOSED pgs=1470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 -- 172.21.15.64:0/2211536941 >> 172.21.15.64:0/2211536941 conn(0x7fa0e0071960 msgr2=0x7fa0e0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 -- 172.21.15.64:0/2211536941 shutdown_connections 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.523+0000 7fa0e5b13640 1 -- 172.21.15.64:0/2211536941 wait complete. 2024-03-04T07:33:23.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.524+0000 7fa0e5b13640 1 Processor -- start 2024-03-04T07:33:23.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.524+0000 7fa0e5b13640 1 -- start start 2024-03-04T07:33:23.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0e5b13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b95a0 0x7fa0e0160cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0e5b13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e01e7ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0e5b13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e015ab40 0x7fa0e015afb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0e5b13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e00cec70 con 0x7fa0e015df00 2024-03-04T07:33:23.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0e5b13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e00ce750 con 0x7fa0e015ab40 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0e5b13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e00cedf0 con 0x7fa0e00b95a0 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e01e7ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e01e7ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51338/0 (socket says 172.21.15.64:51338) 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 -- 172.21.15.64:0/1109053671 learned_addr learned my addr 172.21.15.64:0/1109053671 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0dffff640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e015ab40 0x7fa0e015afb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0df7fe640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b95a0 0x7fa0e0160cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 -- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b95a0 msgr2=0x7fa0e0160cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b95a0 0x7fa0e0160cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 -- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e015ab40 msgr2=0x7fa0e015afb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e015ab40 0x7fa0e015afb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 -- 172.21.15.64:0/1109053671 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa0e015b4f0 con 0x7fa0e015df00 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.525+0000 7fa0deffd640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e01e7ef0 secure :-1 s=READY pgs=1471 cs=0 l=1 rev1=1 crypto rx=0x7fa0d400c430 tx=0x7fa0d400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.526+0000 7fa0c7fff640 1 -- 172.21.15.64:0/1109053671 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d4011790 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.526+0000 7fa0c7fff640 1 -- 172.21.15.64:0/1109053671 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa0d4017070 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.526+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa0e015fc10 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.526+0000 7fa0c7fff640 1 -- 172.21.15.64:0/1109053671 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d4011bb0 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.526+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa0e01601d0 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.527+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa0e0067ff0 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.527+0000 7fa0c7fff640 1 -- 172.21.15.64:0/1109053671 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa0d401c460 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.527+0000 7fa0c7fff640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0cc06b500 0x7fa0cc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.528+0000 7fa0c7fff640 1 -- 172.21.15.64:0/1109053671 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fa0d408a0e0 con 0x7fa0e015df00 2024-03-04T07:33:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.528+0000 7fa0df7fe640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0cc06b500 0x7fa0cc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.528+0000 7fa0df7fe640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0cc06b500 0x7fa0cc06d9c0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0036000 tx=0x7fa0d003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.536+0000 7fa0c7fff640 1 -- 172.21.15.64:0/1109053671 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa0d4058f00 con 0x7fa0e015df00 2024-03-04T07:33:23.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.809+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa0e0162670 con 0x7fa0e015df00 2024-03-04T07:33:23.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.809+0000 7fa0c7fff640 1 -- 172.21.15.64:0/1109053671 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7fa0d4058800 con 0x7fa0e015df00 2024-03-04T07:33:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:23.810 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:23.810 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:23.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0cc06b500 msgr2=0x7fa0cc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0cc06b500 0x7fa0cc06d9c0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0036000 tx=0x7fa0d003c040 comp rx=0 tx=0).stop 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 msgr2=0x7fa0e01e7ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e01e7ef0 secure :-1 s=READY pgs=1471 cs=0 l=1 rev1=1 crypto rx=0x7fa0d400c430 tx=0x7fa0d400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 shutdown_connections 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa0cc06b500 0x7fa0cc06d9c0 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa0e015ab40 0x7fa0e015afb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa0e015df00 0x7fa0e01e7ef0 unknown :-1 s=CLOSED pgs=1471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 --2- 172.21.15.64:0/1109053671 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa0e00b95a0 0x7fa0e0160cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.811+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 >> 172.21.15.64:0/1109053671 conn(0x7fa0e0071960 msgr2=0x7fa0e00b8b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.812+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 shutdown_connections 2024-03-04T07:33:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.812+0000 7fa0e5b13640 1 -- 172.21.15.64:0/1109053671 wait complete. 2024-03-04T07:33:23.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:23.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.924+0000 7f4d4e688640 1 Processor -- start 2024-03-04T07:33:23.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d4e688640 1 -- start start 2024-03-04T07:33:23.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d4e688640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4d480b1020 0x7f4d4806af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d4e688640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4d4806ed10 0x7f4d480cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d4e688640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d480cd6c0 0x7f4d480cdad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d4e688640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4d480d28b0 con 0x7f4d4806ed10 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d4e688640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4d480cc0e0 con 0x7f4d480b1020 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d4e688640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d480cc3c0 con 0x7f4d480cd6c0 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d477fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4d4806ed10 0x7f4d480cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d477fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4d4806ed10 0x7f4d480cd180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51342/0 (socket says 172.21.15.64:51342) 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.925+0000 7f4d477fe640 1 -- 172.21.15.64:0/4130800804 learned_addr learned my addr 172.21.15.64:0/4130800804 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d4cc25640 1 --2- 172.21.15.64:0/4130800804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d480cd6c0 0x7f4d480cdad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d477fe640 1 -- 172.21.15.64:0/4130800804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d480cd6c0 msgr2=0x7f4d480cdad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d47fff640 1 --2- 172.21.15.64:0/4130800804 >> v2:172.21.15.66:3300/0 conn(0x7f4d480b1020 0x7f4d4806af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d477fe640 1 --2- 172.21.15.64:0/4130800804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d480cd6c0 0x7f4d480cdad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d477fe640 1 -- 172.21.15.64:0/4130800804 >> v2:172.21.15.66:3300/0 conn(0x7f4d480b1020 msgr2=0x7f4d4806af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d477fe640 1 --2- 172.21.15.64:0/4130800804 >> v2:172.21.15.66:3300/0 conn(0x7f4d480b1020 0x7f4d4806af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d477fe640 1 -- 172.21.15.64:0/4130800804 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d480cc6a0 con 0x7f4d4806ed10 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d4cc25640 1 --2- 172.21.15.64:0/4130800804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d480cd6c0 0x7f4d480cdad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d47fff640 1 --2- 172.21.15.64:0/4130800804 >> v2:172.21.15.66:3300/0 conn(0x7f4d480b1020 0x7f4d4806af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:23.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.926+0000 7f4d477fe640 1 --2- 172.21.15.64:0/4130800804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d4806ed10 0x7f4d480cd180 secure :-1 s=READY pgs=1472 cs=0 l=1 rev1=1 crypto rx=0x7f4d48070d50 tx=0x7f4d3c02b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c06bedded9f1429 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d46ffd640 1 -- 172.21.15.64:0/4130800804 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d3c035020 con 0x7f4d4806ed10 2024-03-04T07:33:23.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d46ffd640 1 -- 172.21.15.64:0/4130800804 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d3c02d040 con 0x7f4d4806ed10 2024-03-04T07:33:23.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d46ffd640 1 -- 172.21.15.64:0/4130800804 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d3c031a80 con 0x7f4d4806ed10 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 -- 172.21.15.64:0/4130800804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d4806ed10 msgr2=0x7f4d480cd180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 --2- 172.21.15.64:0/4130800804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d4806ed10 0x7f4d480cd180 secure :-1 s=READY pgs=1472 cs=0 l=1 rev1=1 crypto rx=0x7f4d48070d50 tx=0x7f4d3c02b900 comp rx=0 tx=0).stop 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 -- 172.21.15.64:0/4130800804 shutdown_connections 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 --2- 172.21.15.64:0/4130800804 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d480cd6c0 0x7f4d480cdad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 --2- 172.21.15.64:0/4130800804 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d4806ed10 0x7f4d480cd180 unknown :-1 s=CLOSED pgs=1472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 --2- 172.21.15.64:0/4130800804 >> v2:172.21.15.66:3300/0 conn(0x7f4d480b1020 0x7f4d4806af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 -- 172.21.15.64:0/4130800804 >> 172.21.15.64:0/4130800804 conn(0x7f4d480d1a90 msgr2=0x7f4d480d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 -- 172.21.15.64:0/4130800804 shutdown_connections 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.927+0000 7f4d4e688640 1 -- 172.21.15.64:0/4130800804 wait complete. 2024-03-04T07:33:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.929+0000 7f4d4e688640 1 Processor -- start 2024-03-04T07:33:23.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.929+0000 7f4d4e688640 1 -- start start 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.930+0000 7f4d4e688640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d4806ed10 0x7f4d4815dc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.930+0000 7f4d4e688640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d480b1020 0x7f4d48158220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.930+0000 7f4d4e688640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d480cd6c0 0x7f4d48160700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.930+0000 7f4d4e688640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d4815d700 con 0x7f4d480cd6c0 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.930+0000 7f4d4e688640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d480ccc60 con 0x7f4d480b1020 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.930+0000 7f4d4e688640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d4815d880 con 0x7f4d4806ed10 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d480cd6c0 0x7f4d48160700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d480cd6c0 0x7f4d48160700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51348/0 (socket says 172.21.15.64:51348) 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 -- 172.21.15.64:0/2926286708 learned_addr learned my addr 172.21.15.64:0/2926286708 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d47fff640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d4806ed10 0x7f4d4815dc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d477fe640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d480b1020 0x7f4d48158220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 -- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d4806ed10 msgr2=0x7f4d4815dc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d4806ed10 0x7f4d4815dc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 -- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d480b1020 msgr2=0x7f4d48158220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d480b1020 0x7f4d48158220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 -- 172.21.15.64:0/2926286708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d4815e360 con 0x7f4d480cd6c0 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d47fff640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d4806ed10 0x7f4d4815dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d477fe640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d480b1020 0x7f4d48158220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4cc25640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d480cd6c0 0x7f4d48160700 secure :-1 s=READY pgs=1473 cs=0 l=1 rev1=1 crypto rx=0x7f4d4000cd80 tx=0x7f4d4000a650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d44ff9640 1 -- 172.21.15.64:0/2926286708 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d4000f680 con 0x7f4d480cd6c0 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d44ff9640 1 -- 172.21.15.64:0/2926286708 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d4000f840 con 0x7f4d480cd6c0 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d44ff9640 1 -- 172.21.15.64:0/2926286708 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d4000fc30 con 0x7f4d480cd6c0 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.931+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d4815e640 con 0x7f4d480cd6c0 2024-03-04T07:33:23.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.932+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4d4815ec00 con 0x7f4d480cd6c0 2024-03-04T07:33:23.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.932+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4d48067ff0 con 0x7f4d480cd6c0 2024-03-04T07:33:23.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.933+0000 7f4d44ff9640 1 -- 172.21.15.64:0/2926286708 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4d4000e3e0 con 0x7f4d480cd6c0 2024-03-04T07:33:23.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.933+0000 7f4d44ff9640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d3406b4d0 0x7f4d3406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:23.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.933+0000 7f4d44ff9640 1 -- 172.21.15.64:0/2926286708 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f4d4008afe0 con 0x7f4d480cd6c0 2024-03-04T07:33:23.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.933+0000 7f4d47fff640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d3406b4d0 0x7f4d3406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:23.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.934+0000 7f4d47fff640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d3406b4d0 0x7f4d3406d990 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7f4d4815d300 tx=0x7f4d38008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:23.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:23.938+0000 7f4d44ff9640 1 -- 172.21.15.64:0/2926286708 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4d40014030 con 0x7f4d480cd6c0 2024-03-04T07:33:24.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.163+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4d4806f2f0 con 0x7f4d480cd6c0 2024-03-04T07:33:24.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.164+0000 7f4d44ff9640 1 -- 172.21.15.64:0/2926286708 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 74 v74) v1 ==== 94+0+4978 (secure 0 0 0) 0x7f4d40059e00 con 0x7f4d480cd6c0 2024-03-04T07:33:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:24.164 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":74,"default_fscid":8,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72}],"filesystems":[{"mdsmap":{"epoch":74,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:32:08.826165+0000","modified":"2024-03-04T07:32:25.534407+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":109,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8139},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8139":{"gid":8139,"name":"c","rank":0,"incarnation":71,"state":"up:active","state_seq":4,"addr":"172.21.15.64:6835/1389682232","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":1389682232},{"type":"v1","addr":"172.21.15.64:6835","nonce":1389682232}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[17],"metadata_pool":16,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8139,"qdb_cluster":[8139]},"id":8}]} 2024-03-04T07:33:24.164 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 74 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d3406b4d0 msgr2=0x7f4d3406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d3406b4d0 0x7f4d3406d990 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7f4d4815d300 tx=0x7f4d38008040 comp rx=0 tx=0).stop 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d480cd6c0 msgr2=0x7f4d48160700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d480cd6c0 0x7f4d48160700 secure :-1 s=READY pgs=1473 cs=0 l=1 rev1=1 crypto rx=0x7f4d4000cd80 tx=0x7f4d4000a650 comp rx=0 tx=0).stop 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 shutdown_connections 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d3406b4d0 0x7f4d3406d990 unknown :-1 s=CLOSED pgs=659 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d480cd6c0 0x7f4d48160700 unknown :-1 s=CLOSED pgs=1473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d480b1020 0x7f4d48158220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 --2- 172.21.15.64:0/2926286708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d4806ed10 0x7f4d4815dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 >> 172.21.15.64:0/2926286708 conn(0x7f4d480d1a90 msgr2=0x7f4d48070120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:24.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 shutdown_connections 2024-03-04T07:33:24.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.166+0000 7f4d4e688640 1 -- 172.21.15.64:0/2926286708 wait complete. 2024-03-04T07:33:24.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:24.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.281+0000 7fdb0e7f7640 1 Processor -- start 2024-03-04T07:33:24.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.281+0000 7fdb0e7f7640 1 -- start start 2024-03-04T07:33:24.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb0e7f7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fdb080d02e0 0x7fdb080cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb0e7f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fdb080d06c0 0x7fdb080b1c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb0e7f7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdb080b0dd0 con 0x7fdb080b4260 2024-03-04T07:33:24.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb0e7f7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdb080b0b80 con 0x7fdb080d02e0 2024-03-04T07:33:24.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb0e7f7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fdb080aeec0 con 0x7fdb080d06c0 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb07fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fdb080d02e0 0x7fdb080cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb07fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fdb080d02e0 0x7fdb080cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51356/0 (socket says 172.21.15.64:51356) 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb07fff640 1 -- 172.21.15.64:0/2997347357 learned_addr learned my addr 172.21.15.64:0/2997347357 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.282+0000 7fdb0cd94640 1 --2- 172.21.15.64:0/2997347357 >> v2:172.21.15.66:3300/0 conn(0x7fdb080d06c0 0x7fdb080b1c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.283+0000 7fdb07fff640 1 -- 172.21.15.64:0/2997347357 >> v2:172.21.15.66:3300/0 conn(0x7fdb080d06c0 msgr2=0x7fdb080b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.283+0000 7fdb07fff640 1 --2- 172.21.15.64:0/2997347357 >> v2:172.21.15.66:3300/0 conn(0x7fdb080d06c0 0x7fdb080b1c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.283+0000 7fdb07fff640 1 -- 172.21.15.64:0/2997347357 >> v1:172.21.15.66:6789/0 conn(0x7fdb080b4260 legacy=0x7fdb080b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.283+0000 7fdb07fff640 1 -- 172.21.15.64:0/2997347357 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb080b10b0 con 0x7fdb080d02e0 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.283+0000 7fdb0cd94640 1 --2- 172.21.15.64:0/2997347357 >> v2:172.21.15.66:3300/0 conn(0x7fdb080d06c0 0x7fdb080b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.283+0000 7fdb07fff640 1 --2- 172.21.15.64:0/2997347357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080d02e0 0x7fdb080cfa00 secure :-1 s=READY pgs=1474 cs=0 l=1 rev1=1 crypto rx=0x7fdb080cc0e0 tx=0x7fdaf802ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=30f7ee4d869d406e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb06ffd640 1 -- 172.21.15.64:0/2997347357 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdaf802f850 con 0x7fdb080d02e0 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb06ffd640 1 -- 172.21.15.64:0/2997347357 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdaf802f9f0 con 0x7fdb080d02e0 2024-03-04T07:33:24.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb06ffd640 1 -- 172.21.15.64:0/2997347357 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdaf802fdc0 con 0x7fdb080d02e0 2024-03-04T07:33:24.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2997347357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080d02e0 msgr2=0x7fdb080cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2997347357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080d02e0 0x7fdb080cfa00 secure :-1 s=READY pgs=1474 cs=0 l=1 rev1=1 crypto rx=0x7fdb080cc0e0 tx=0x7fdaf802ba30 comp rx=0 tx=0).stop 2024-03-04T07:33:24.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2997347357 shutdown_connections 2024-03-04T07:33:24.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2997347357 >> v2:172.21.15.66:3300/0 conn(0x7fdb080d06c0 0x7fdb080b1c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2997347357 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080d02e0 0x7fdb080cfa00 unknown :-1 s=CLOSED pgs=1474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2997347357 >> 172.21.15.64:0/2997347357 conn(0x7fdb080b36d0 msgr2=0x7fdb080b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2997347357 shutdown_connections 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.284+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2997347357 wait complete. 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.285+0000 7fdb0e7f7640 1 Processor -- start 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb0e7f7640 1 -- start start 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb0e7f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080b4260 0x7fdb080c82e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb0e7f7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb080d02e0 0x7fdb08143c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb0e7f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb080d06c0 0x7fdb0815a9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb0e7f7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb080b22d0 con 0x7fdb080b4260 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb0e7f7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb080b2150 con 0x7fdb080d02e0 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb0e7f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdb080af350 con 0x7fdb080d06c0 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb07fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080b4260 0x7fdb080c82e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb07fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080b4260 0x7fdb080c82e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51360/0 (socket says 172.21.15.64:51360) 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb07fff640 1 -- 172.21.15.64:0/2468747885 learned_addr learned my addr 172.21.15.64:0/2468747885 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:24.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.286+0000 7fdb077fe640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb080d02e0 0x7fdb08143c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb0cd94640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb080d06c0 0x7fdb0815a9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb07fff640 1 -- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb080d06c0 msgr2=0x7fdb0815a9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb07fff640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb080d06c0 0x7fdb0815a9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb07fff640 1 -- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb080d02e0 msgr2=0x7fdb08143c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb07fff640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb080d02e0 0x7fdb08143c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb07fff640 1 -- 172.21.15.64:0/2468747885 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb08152f50 con 0x7fdb080b4260 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb07fff640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080b4260 0x7fdb080c82e0 secure :-1 s=READY pgs=1475 cs=0 l=1 rev1=1 crypto rx=0x7fdaf802ba70 tx=0x7fdaf8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb04ff9640 1 -- 172.21.15.64:0/2468747885 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdaf8037c90 con 0x7fdb080b4260 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb04ff9640 1 -- 172.21.15.64:0/2468747885 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdaf8046070 con 0x7fdb080b4260 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdb08153230 con 0x7fdb080b4260 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb04ff9640 1 -- 172.21.15.64:0/2468747885 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdaf8041640 con 0x7fdb080b4260 2024-03-04T07:33:24.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.287+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdb081537f0 con 0x7fdb080b4260 2024-03-04T07:33:24.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.288+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdb08072d90 con 0x7fdb080b4260 2024-03-04T07:33:24.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.289+0000 7fdb04ff9640 1 -- 172.21.15.64:0/2468747885 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdaf8040070 con 0x7fdb080b4260 2024-03-04T07:33:24.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.289+0000 7fdb04ff9640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaf406b4d0 0x7fdaf406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.289+0000 7fdb04ff9640 1 -- 172.21.15.64:0/2468747885 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7fdaf80ac9a0 con 0x7fdb080b4260 2024-03-04T07:33:24.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.289+0000 7fdb077fe640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaf406b4d0 0x7fdaf406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.289+0000 7fdb077fe640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaf406b4d0 0x7fdaf406d990 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7fdb0815b6c0 tx=0x7fdafc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.293+0000 7fdb04ff9640 1 -- 172.21.15.64:0/2468747885 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdb08072d90 con 0x7fdb080b4260 2024-03-04T07:33:24.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.475+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fdb08068700 con 0x7fdb080b4260 2024-03-04T07:33:24.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.476+0000 7fdb04ff9640 1 -- 172.21.15.64:0/2468747885 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v111) v1 ==== 74+0+16934 (secure 0 0 0) 0x7fdaf8033030 con 0x7fdb080b4260 2024-03-04T07:33:24.476 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:24.476 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":111,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:32:35.558693+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":16,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:32:07.087121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"104","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":17,"pool_name":"cephfs_data","create_time":"2024-03-04T07:32:07.785433+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"105","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"17.24","mappings":[{"from":0,"to":5}]},{"pgid":"17.25","mappings":[{"from":0,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6837/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.64:6835/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.66:6835/4084900379":"2024-03-05T07:32:21.508754+0000","172.21.15.66:6833/4084900379":"2024-03-05T07:32:21.508754+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaf406b4d0 msgr2=0x7fdaf406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaf406b4d0 0x7fdaf406d990 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7fdb0815b6c0 tx=0x7fdafc009040 comp rx=0 tx=0).stop 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080b4260 msgr2=0x7fdb080c82e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080b4260 0x7fdb080c82e0 secure :-1 s=READY pgs=1475 cs=0 l=1 rev1=1 crypto rx=0x7fdaf802ba70 tx=0x7fdaf8004730 comp rx=0 tx=0).stop 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 shutdown_connections 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaf406b4d0 0x7fdaf406d990 unknown :-1 s=CLOSED pgs=660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdb080d06c0 0x7fdb0815a9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdb080d02e0 0x7fdb08143c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 --2- 172.21.15.64:0/2468747885 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdb080b4260 0x7fdb080c82e0 unknown :-1 s=CLOSED pgs=1475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 >> 172.21.15.64:0/2468747885 conn(0x7fdb080b36d0 msgr2=0x7fdb080b1920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:24.478 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 shutdown_connections 2024-03-04T07:33:24.479 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.478+0000 7fdb0e7f7640 1 -- 172.21.15.64:0/2468747885 wait complete. 2024-03-04T07:33:24.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:33:24.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.595+0000 7f802ea0e640 1 Processor -- start 2024-03-04T07:33:24.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.595+0000 7f802ea0e640 1 -- start start 2024-03-04T07:33:24.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f802ea0e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f802815e000 0x7f8028071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f802ea0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280d35e0 0x7f8028071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f802ea0e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f80280ce970 con 0x7f80280b2c50 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f802ea0e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f80280cece0 con 0x7f802815e000 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f802ea0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f80280cef30 con 0x7f80280d35e0 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f8027fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f802815e000 0x7f8028071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f802cfab640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280d35e0 0x7f8028071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f80277fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f80280b2c50 0x7f8028161cb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38850/0 (socket says 172.21.15.64:38850) 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f80277fe640 1 -- 172.21.15.64:0/3919985878 learned_addr learned my addr 172.21.15.64:0/3919985878 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f8027fff640 1 -- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280d35e0 msgr2=0x7f8028071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f8027fff640 1 --2- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280d35e0 0x7f8028071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f8027fff640 1 -- 172.21.15.64:0/3919985878 >> v1:172.21.15.66:6789/0 conn(0x7f80280b2c50 legacy=0x7f8028161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f8027fff640 1 -- 172.21.15.64:0/3919985878 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80280cf210 con 0x7f802815e000 2024-03-04T07:33:24.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.596+0000 7f802cfab640 1 --2- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280d35e0 0x7f8028071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.598+0000 7f8027fff640 1 --2- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f802815e000 0x7f8028071540 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f80100097a0 tx=0x7f801002b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c89945896ae6d480 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.598+0000 7f8026ffd640 1 -- 172.21.15.64:0/3919985878 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80100315a0 con 0x7f802815e000 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.598+0000 7f8026ffd640 1 -- 172.21.15.64:0/3919985878 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8010037070 con 0x7f802815e000 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.598+0000 7f8026ffd640 1 -- 172.21.15.64:0/3919985878 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8010031960 con 0x7f802815e000 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 -- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f802815e000 msgr2=0x7f8028071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f802815e000 0x7f8028071540 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f80100097a0 tx=0x7f801002b840 comp rx=0 tx=0).stop 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 -- 172.21.15.64:0/3919985878 shutdown_connections 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f80280d35e0 0x7f8028071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3919985878 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f802815e000 0x7f8028071540 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 -- 172.21.15.64:0/3919985878 >> 172.21.15.64:0/3919985878 conn(0x7f80280d3c80 msgr2=0x7f80280d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 -- 172.21.15.64:0/3919985878 shutdown_connections 2024-03-04T07:33:24.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.599+0000 7f802ea0e640 1 -- 172.21.15.64:0/3919985878 wait complete. 2024-03-04T07:33:24.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.600+0000 7f802ea0e640 1 Processor -- start 2024-03-04T07:33:24.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.601+0000 7f802ea0e640 1 -- start start 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f802ea0e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80280b2c50 0x7f80280cc100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f802ea0e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280d35e0 0x7f8028154cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f802ea0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802815e000 0x7f80281556b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f802ea0e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80280b3320 con 0x7f80280d35e0 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f802ea0e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80280b31a0 con 0x7f80280b2c50 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f802ea0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f802815e3e0 con 0x7f802815e000 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280d35e0 0x7f8028154cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280d35e0 0x7f8028154cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51368/0 (socket says 172.21.15.64:51368) 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 -- 172.21.15.64:0/3892396317 learned_addr learned my addr 172.21.15.64:0/3892396317 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f802cfab640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802815e000 0x7f80281556b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f8027fff640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80280b2c50 0x7f80280cc100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 -- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802815e000 msgr2=0x7f80281556b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802815e000 0x7f80281556b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 -- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80280b2c50 msgr2=0x7f80280cc100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80280b2c50 0x7f80280cc100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.602+0000 7f80277fe640 1 -- 172.21.15.64:0/3892396317 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80280b63f0 con 0x7f80280d35e0 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.603+0000 7f80277fe640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280d35e0 0x7f8028154cc0 secure :-1 s=READY pgs=1476 cs=0 l=1 rev1=1 crypto rx=0x7f801800c3c0 tx=0x7f801800c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.603+0000 7f8024ff9640 1 -- 172.21.15.64:0/3892396317 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8018017070 con 0x7f80280d35e0 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.603+0000 7f8024ff9640 1 -- 172.21.15.64:0/3892396317 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8018005ce0 con 0x7f80280d35e0 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.603+0000 7f8024ff9640 1 -- 172.21.15.64:0/3892396317 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80180072f0 con 0x7f80280d35e0 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.603+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80281541e0 con 0x7f80280d35e0 2024-03-04T07:33:24.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.603+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80281547a0 con 0x7f80280d35e0 2024-03-04T07:33:24.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.604+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f80281549a0 con 0x7f80280d35e0 2024-03-04T07:33:24.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.604+0000 7f8024ff9640 1 -- 172.21.15.64:0/3892396317 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f801801a070 con 0x7f80280d35e0 2024-03-04T07:33:24.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.605+0000 7f8024ff9640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.605+0000 7f8024ff9640 1 -- 172.21.15.64:0/3892396317 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f80180896d0 con 0x7f80280d35e0 2024-03-04T07:33:24.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.605+0000 7f8027fff640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.605+0000 7f8027fff640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f8010036000 tx=0x7f801003d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.609 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.609+0000 7f8024ff9640 1 -- 172.21.15.64:0/3892396317 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f80281549a0 con 0x7f80280d35e0 2024-03-04T07:33:24.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.827+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f8028067ff0 con 0x7f80280d35e0 2024-03-04T07:33:24.827 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:33:24.827+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:33:24.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.830+0000 7f8024ff9640 1 -- 172.21.15.64:0/3892396317 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v75) v1 ==== 161+0+0 (secure 0 0 0) 0x7f8018058460 con 0x7f80280d35e0 2024-03-04T07:33:24.830 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 msgr2=0x7f800806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f8010036000 tx=0x7f801003d040 comp rx=0 tx=0).stop 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280d35e0 msgr2=0x7f8028154cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280d35e0 0x7f8028154cc0 secure :-1 s=READY pgs=1476 cs=0 l=1 rev1=1 crypto rx=0x7f801800c3c0 tx=0x7f801800c880 comp rx=0 tx=0).stop 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 shutdown_connections 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f800806b4d0 0x7f800806d990 unknown :-1 s=CLOSED pgs=661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f802815e000 0x7f80281556b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f80280d35e0 0x7f8028154cc0 unknown :-1 s=CLOSED pgs=1476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 --2- 172.21.15.64:0/3892396317 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f80280b2c50 0x7f80280cc100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.832+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 >> 172.21.15.64:0/3892396317 conn(0x7f80280d3c80 msgr2=0x7f802815e950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:24.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.833+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 shutdown_connections 2024-03-04T07:33:24.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.833+0000 7f802ea0e640 1 -- 172.21.15.64:0/3892396317 wait complete. 2024-03-04T07:33:24.836 INFO:tasks.ceph.mds.c.smithi064.stderr: -8794> 2024-03-04T07:32:27.520+0000 7fedfc44e640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:33:24.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:33:24.868 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:33:24.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f3955ba0640 1 Processor -- start 2024-03-04T07:33:24.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f3955ba0640 1 -- start start 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f3955ba0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f39500cef10 0x7f39500cd4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f3955ba0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39500cc6f0 con 0x7f39500cff40 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f3955ba0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39500cc9d0 con 0x7f395015f1f0 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f3955ba0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f395015f5d0 con 0x7f39500cef10 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f395539f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f39500cff40 0x7f39500d03f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51406/0 (socket says 172.21.15.64:51406) 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.950+0000 7f395539f640 1 -- 172.21.15.64:0/3087124588 learned_addr learned my addr 172.21.15.64:0/3087124588 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394ffff640 1 --2- 172.21.15.64:0/3087124588 >> v2:172.21.15.66:3300/0 conn(0x7f39500cef10 0x7f39500cd4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1743767582 0 0) 0x7f39500cc6f0 con 0x7f39500cff40 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f393c003520 con 0x7f39500cff40 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4233822893 0 0) 0x7f393c003520 con 0x7f39500cff40 2024-03-04T07:33:24.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 >> v2:172.21.15.66:3300/0 conn(0x7f39500cef10 msgr2=0x7f39500cd4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 --2- 172.21.15.64:0/3087124588 >> v2:172.21.15.66:3300/0 conn(0x7f39500cef10 0x7f39500cd4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 >> v1:172.21.15.66:6789/0 conn(0x7f395015f1f0 legacy=0x7f39500cf530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39500cccb0 con 0x7f39500cff40 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f3948001670 con 0x7f39500cff40 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 >> v1:172.21.15.64:6789/0 conn(0x7f39500cff40 legacy=0x7f39500d03f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f393c0039b0 0x7f393c003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0043b0 0x7f393c004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f393c005080 0x7f393c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f393c00bf00 con 0x7f393c0039b0 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f393c00c0b0 con 0x7f393c0043b0 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f393c00c2b0 con 0x7f393c005080 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f3954b9e640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f393c0039b0 0x7f393c003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f394ffff640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0043b0 0x7f393c004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.951+0000 7f395539f640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f393c005080 0x7f393c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f3954b9e640 1 -- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f393c005080 msgr2=0x7f393c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f3954b9e640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f393c005080 0x7f393c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f3954b9e640 1 -- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0043b0 msgr2=0x7f393c004880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f3954b9e640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0043b0 0x7f393c004880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f3954b9e640 1 -- 172.21.15.64:0/3087124588 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f393c003520 con 0x7f393c0039b0 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f395539f640 1 -- 172.21.15.64:0/3087124588 reap_dead start 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f3954b9e640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f393c0039b0 0x7f393c003d90 secure :-1 s=READY pgs=1481 cs=0 l=1 rev1=1 crypto rx=0x7f3938001860 tx=0x7f393c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f8c9c291a8e6c589 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f393802f050 con 0x7f393c0039b0 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3938033650 con 0x7f393c0039b0 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.952+0000 7f394f7fe640 1 -- 172.21.15.64:0/3087124588 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3938001430 con 0x7f393c0039b0 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.953+0000 7f3955ba0640 1 -- 172.21.15.64:0/3087124588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f393c0039b0 msgr2=0x7f393c003d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.953+0000 7f3955ba0640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f393c0039b0 0x7f393c003d90 secure :-1 s=READY pgs=1481 cs=0 l=1 rev1=1 crypto rx=0x7f3938001860 tx=0x7f393c005830 comp rx=0 tx=0).stop 2024-03-04T07:33:24.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.953+0000 7f3955ba0640 1 -- 172.21.15.64:0/3087124588 shutdown_connections 2024-03-04T07:33:24.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.953+0000 7f3955ba0640 1 --2- 172.21.15.64:0/3087124588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f393c0039b0 0x7f393c003d90 unknown :-1 s=CLOSED pgs=1481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.953+0000 7f3955ba0640 1 -- 172.21.15.64:0/3087124588 >> 172.21.15.64:0/3087124588 conn(0x7f39500d1960 msgr2=0x7f39500d1d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:24.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.953+0000 7f3955ba0640 1 -- 172.21.15.64:0/3087124588 shutdown_connections 2024-03-04T07:33:24.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.953+0000 7f3955ba0640 1 -- 172.21.15.64:0/3087124588 wait complete. 2024-03-04T07:33:24.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.954+0000 7f3955ba0640 1 Processor -- start 2024-03-04T07:33:24.957 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.954+0000 7f3955ba0640 1 -- start start 2024-03-04T07:33:24.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f3955ba0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0039b0 0x7f3950152f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f3955ba0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39500b6f10 0x7f39500b2340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f3955ba0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39500b04b0 0x7f39500b0940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f3955ba0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39500cd840 con 0x7f39500b04b0 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f3955ba0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39500cd6c0 con 0x7f393c0039b0 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f3955ba0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39500b2cd0 con 0x7f39500b6f10 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39500b04b0 0x7f39500b0940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39500b04b0 0x7f39500b0940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51400/0 (socket says 172.21.15.64:51400) 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 -- 172.21.15.64:0/1942796990 learned_addr learned my addr 172.21.15.64:0/1942796990 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f394ffff640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39500b6f10 0x7f39500b2340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f3954b9e640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0039b0 0x7f3950152f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 -- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39500b6f10 msgr2=0x7f39500b2340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39500b6f10 0x7f39500b2340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 -- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0039b0 msgr2=0x7f3950152f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0039b0 0x7f3950152f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.955+0000 7f395539f640 1 -- 172.21.15.64:0/1942796990 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39500b2fb0 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.956+0000 7f395539f640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39500b04b0 0x7f39500b0940 secure :-1 s=READY pgs=1482 cs=0 l=1 rev1=1 crypto rx=0x7f3948000c00 tx=0x7f3948007430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.956+0000 7f394d7fa640 1 -- 172.21.15.64:0/1942796990 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3948007740 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.956+0000 7f394d7fa640 1 -- 172.21.15.64:0/1942796990 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3948018070 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.956+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39500b3290 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.956+0000 7f394d7fa640 1 -- 172.21.15.64:0/1942796990 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f394800fad0 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.956+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39500b3850 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.957+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f395004f330 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.957+0000 7f394d7fa640 1 -- 172.21.15.64:0/1942796990 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f394800fc70 con 0x7f39500b04b0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.958+0000 7f394d7fa640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393006b670 0x7f393006db30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.958+0000 7f3954b9e640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393006b670 0x7f393006db30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:24.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.958+0000 7f394d7fa640 1 -- 172.21.15.64:0/1942796990 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(112..112 src has 1..112) v4 ==== 6025+0+0 (secure 0 0 0) 0x7f394808a0f0 con 0x7f39500b04b0 2024-03-04T07:33:24.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.958+0000 7f3954b9e640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393006b670 0x7f393006db30 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f3938036000 tx=0x7f3938035040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:24.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:24.962+0000 7f394d7fa640 1 -- 172.21.15.64:0/1942796990 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f39480587e0 con 0x7f39500b04b0 2024-03-04T07:33:25.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.182+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f39500b3a20 con 0x7f39500b04b0 2024-03-04T07:33:25.836 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.836+0000 7f394d7fa640 1 -- 172.21.15.64:0/1942796990 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v76) v1 ==== 104+0+0 (secure 0 0 0) 0x7f394805c880 con 0x7f39500b04b0 2024-03-04T07:33:25.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393006b670 msgr2=0x7f393006db30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:25.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393006b670 0x7f393006db30 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f3938036000 tx=0x7f3938035040 comp rx=0 tx=0).stop 2024-03-04T07:33:25.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39500b04b0 msgr2=0x7f39500b0940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:25.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39500b04b0 0x7f39500b0940 secure :-1 s=READY pgs=1482 cs=0 l=1 rev1=1 crypto rx=0x7f3948000c00 tx=0x7f3948007430 comp rx=0 tx=0).stop 2024-03-04T07:33:25.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 shutdown_connections 2024-03-04T07:33:25.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393006b670 0x7f393006db30 unknown :-1 s=CLOSED pgs=662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:25.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39500b04b0 0x7f39500b0940 unknown :-1 s=CLOSED pgs=1482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:25.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39500b6f10 0x7f39500b2340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:25.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 --2- 172.21.15.64:0/1942796990 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f393c0039b0 0x7f3950152f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:25.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 >> 172.21.15.64:0/1942796990 conn(0x7f39500d1960 msgr2=0x7f39501537e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:25.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 shutdown_connections 2024-03-04T07:33:25.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:25.839+0000 7f3955ba0640 1 -- 172.21.15.64:0/1942796990 wait complete. 2024-03-04T07:33:25.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:33:26.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.005+0000 7f8e382a4640 1 Processor -- start 2024-03-04T07:33:26.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.005+0000 7f8e382a4640 1 -- start start 2024-03-04T07:33:26.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e382a4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8e300b4340 0x7f8e3015d9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e382a4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e300d0a30 con 0x7f8e3015fb60 2024-03-04T07:33:26.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e382a4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e300d01d0 con 0x7f8e3015d2c0 2024-03-04T07:33:26.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e382a4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e300cc5c0 con 0x7f8e300b4340 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e36841640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8e3015fb60 0x7f8e3015ff70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51414/0 (socket says 172.21.15.64:51414) 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e36841640 1 -- 172.21.15.64:0/2372587137 learned_addr learned my addr 172.21.15.64:0/2372587137 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3583f640 1 --2- 172.21.15.64:0/2372587137 >> v2:172.21.15.66:3300/0 conn(0x7f8e300b4340 0x7f8e3015d9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1369239410 0 0) 0x7f8e300d0a30 con 0x7f8e3015fb60 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8e18003520 con 0x7f8e3015fb60 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 236818893 0 0) 0x7f8e18003520 con 0x7f8e3015fb60 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 >> v2:172.21.15.66:3300/0 conn(0x7f8e300b4340 msgr2=0x7f8e3015d9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 --2- 172.21.15.64:0/2372587137 >> v2:172.21.15.66:3300/0 conn(0x7f8e300b4340 0x7f8e3015d9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 >> v1:172.21.15.66:6789/0 conn(0x7f8e3015d2c0 legacy=0x7f8e300b2ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e300d0470 con 0x7f8e3015fb60 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f8e28001670 con 0x7f8e3015fb60 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 >> v1:172.21.15.64:6789/0 conn(0x7f8e3015fb60 legacy=0x7f8e3015ff70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e180039b0 0x7f8e18003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e180043b0 0x7f8e18004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.006+0000 7f8e3503e640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e18005080 0x7f8e1800b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e300cc5c0 con 0x7f8e18005080 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e1800bfe0 con 0x7f8e180039b0 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e1800c1e0 con 0x7f8e180043b0 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e36841640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e18005080 0x7f8e1800b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e36040640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e180039b0 0x7f8e18003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e3583f640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e180043b0 0x7f8e18004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e36841640 1 -- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e180043b0 msgr2=0x7f8e18004880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e36841640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e180043b0 0x7f8e18004880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e36841640 1 -- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e180039b0 msgr2=0x7f8e18003d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e36841640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e180039b0 0x7f8e18003d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.007+0000 7f8e36841640 1 -- 172.21.15.64:0/2372587137 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e18003520 con 0x7f8e18005080 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e36841640 1 -- 172.21.15.64:0/2372587137 reap_dead start 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e36841640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e18005080 0x7f8e1800b9c0 secure :-1 s=READY pgs=1484 cs=0 l=1 rev1=1 crypto rx=0x7f8e2800c190 tx=0x7f8e18005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c61b7b48540ad682 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e28007cc0 con 0x7f8e18005080 2024-03-04T07:33:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e28030070 con 0x7f8e18005080 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e3503e640 1 -- 172.21.15.64:0/2372587137 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e28005d50 con 0x7f8e18005080 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e382a4640 1 -- 172.21.15.64:0/2372587137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e18005080 msgr2=0x7f8e1800b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e382a4640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e18005080 0x7f8e1800b9c0 secure :-1 s=READY pgs=1484 cs=0 l=1 rev1=1 crypto rx=0x7f8e2800c190 tx=0x7f8e18005830 comp rx=0 tx=0).stop 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.008+0000 7f8e382a4640 1 -- 172.21.15.64:0/2372587137 shutdown_connections 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.009+0000 7f8e382a4640 1 --2- 172.21.15.64:0/2372587137 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e18005080 0x7f8e1800b9c0 unknown :-1 s=CLOSED pgs=1484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.009+0000 7f8e382a4640 1 -- 172.21.15.64:0/2372587137 >> 172.21.15.64:0/2372587137 conn(0x7f8e300b3660 msgr2=0x7f8e300b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.009+0000 7f8e382a4640 1 -- 172.21.15.64:0/2372587137 shutdown_connections 2024-03-04T07:33:26.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.009+0000 7f8e382a4640 1 -- 172.21.15.64:0/2372587137 wait complete. 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 Processor -- start 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 -- start start 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e18005080 0x7f8e3014cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e30158360 0x7f8e300b4720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b64c0 0x7f8e300b6940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e3015d9d0 con 0x7f8e30158360 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e300cc860 con 0x7f8e300b64c0 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.010+0000 7f8e382a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e3015db50 con 0x7f8e18005080 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e30158360 0x7f8e300b4720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e30158360 0x7f8e300b4720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51418/0 (socket says 172.21.15.64:51418) 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 -- 172.21.15.64:0/3243133950 learned_addr learned my addr 172.21.15.64:0/3243133950 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e36040640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e18005080 0x7f8e3014cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e36841640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b64c0 0x7f8e300b6940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 -- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e18005080 msgr2=0x7f8e3014cbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e18005080 0x7f8e3014cbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 -- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b64c0 msgr2=0x7f8e300b6940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b64c0 0x7f8e300b6940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 -- 172.21.15.64:0/3243133950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e300b5000 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e3583f640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e30158360 0x7f8e300b4720 secure :-1 s=READY pgs=1485 cs=0 l=1 rev1=1 crypto rx=0x7f8e240118e0 tx=0x7f8e24011da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e2effd640 1 -- 172.21.15.64:0/3243133950 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e2400faf0 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e2effd640 1 -- 172.21.15.64:0/3243133950 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e2400fc90 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e300b52e0 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e2effd640 1 -- 172.21.15.64:0/3243133950 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e24014440 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.011+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e300b58a0 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.012+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e3009fc50 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.013+0000 7f8e2effd640 1 -- 172.21.15.64:0/3243133950 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8e24012040 con 0x7f8e30158360 2024-03-04T07:33:26.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.013+0000 7f8e2effd640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b500 0x7f8e1c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.013+0000 7f8e2effd640 1 -- 172.21.15.64:0/3243133950 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(112..112 src has 1..112) v4 ==== 6025+0+0 (secure 0 0 0) 0x7f8e2408a980 con 0x7f8e30158360 2024-03-04T07:33:26.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.013+0000 7f8e36040640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b500 0x7f8e1c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.014+0000 7f8e36040640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b500 0x7f8e1c06d9c0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7f8e20003ea0 tx=0x7f8e20009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:26.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.017+0000 7f8e2effd640 1 -- 172.21.15.64:0/3243133950 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8e24019030 con 0x7f8e30158360 2024-03-04T07:33:26.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.200+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f8e300b3440 con 0x7f8e30158360 2024-03-04T07:33:26.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.838+0000 7f8e2effd640 1 -- 172.21.15.64:0/3243133950 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v113) v1 ==== 181+0+0 (secure 0 0 0) 0x7f8e24059100 con 0x7f8e30158360 2024-03-04T07:33:26.838 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:33:26.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.840+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b500 msgr2=0x7f8e1c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.840+0000 7f8e382a4640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b500 0x7f8e1c06d9c0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7f8e20003ea0 tx=0x7f8e20009040 comp rx=0 tx=0).stop 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e30158360 msgr2=0x7f8e300b4720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e30158360 0x7f8e300b4720 secure :-1 s=READY pgs=1485 cs=0 l=1 rev1=1 crypto rx=0x7f8e240118e0 tx=0x7f8e24011da0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 shutdown_connections 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e1c06b500 0x7f8e1c06d9c0 unknown :-1 s=CLOSED pgs=664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e300b64c0 0x7f8e300b6940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e30158360 0x7f8e300b4720 unknown :-1 s=CLOSED pgs=1485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 --2- 172.21.15.64:0/3243133950 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e18005080 0x7f8e3014cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 >> 172.21.15.64:0/3243133950 conn(0x7f8e300b3660 msgr2=0x7f8e300ce130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 shutdown_connections 2024-03-04T07:33:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.841+0000 7f8e382a4640 1 -- 172.21.15.64:0/3243133950 wait complete. 2024-03-04T07:33:26.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:33:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.966+0000 7f8c87f93640 1 Processor -- start 2024-03-04T07:33:26.966 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.967+0000 7f8c87f93640 1 -- start start 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.967+0000 7f8c87f93640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8c800ceea0 0x7f8c800d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.967+0000 7f8c87f93640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8c800d3740 con 0x7f8c8015dd90 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.967+0000 7f8c87f93640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8c800d3120 con 0x7f8c800b6d10 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.967+0000 7f8c87f93640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8c800d3400 con 0x7f8c800ceea0 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.967+0000 7f8c85d2f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8c8015dd90 0x7f8c800d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51430/0 (socket says 172.21.15.64:51430) 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.967+0000 7f8c85d2f640 1 -- 172.21.15.64:0/693989656 learned_addr learned my addr 172.21.15.64:0/693989656 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3971868368 0 0) 0x7f8c800d3740 con 0x7f8c8015dd90 2024-03-04T07:33:26.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8c6c003520 con 0x7f8c8015dd90 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c8552e640 1 --2- 172.21.15.64:0/693989656 >> v2:172.21.15.66:3300/0 conn(0x7f8c800ceea0 0x7f8c800d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3223310853 0 0) 0x7f8c6c003520 con 0x7f8c8015dd90 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 >> v2:172.21.15.66:3300/0 conn(0x7f8c800ceea0 msgr2=0x7f8c800d29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 --2- 172.21.15.64:0/693989656 >> v2:172.21.15.66:3300/0 conn(0x7f8c800ceea0 0x7f8c800d29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 >> v1:172.21.15.66:6789/0 conn(0x7f8c800b6d10 legacy=0x7f8c800d0c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c800d39e0 con 0x7f8c8015dd90 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f8c7c001670 con 0x7f8c8015dd90 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 >> v1:172.21.15.64:6789/0 conn(0x7f8c8015dd90 legacy=0x7f8c800d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c6c003a00 0x7f8c6c003e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6c004800 0x7f8c6c004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c0051c0 0x7f8c6c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c800d3400 con 0x7f8c6c003a00 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c6c00c0a0 con 0x7f8c6c004800 2024-03-04T07:33:26.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c6c00c2a0 con 0x7f8c6c0051c0 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c85d2f640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c6c003a00 0x7f8c6c003e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c8552e640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6c004800 0x7f8c6c004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c86530640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c0051c0 0x7f8c6c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c85d2f640 1 -- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c0051c0 msgr2=0x7f8c6c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c85d2f640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c0051c0 0x7f8c6c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c85d2f640 1 -- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6c004800 msgr2=0x7f8c6c004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c85d2f640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c6c004800 0x7f8c6c004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c85d2f640 1 -- 172.21.15.64:0/693989656 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c6c003520 con 0x7f8c6c003a00 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.968+0000 7f8c86530640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c0051c0 0x7f8c6c00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.969+0000 7f8c86530640 1 -- 172.21.15.64:0/693989656 reap_dead start 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.969+0000 7f8c85d2f640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c6c003a00 0x7f8c6c003e20 secure :-1 s=READY pgs=1487 cs=0 l=1 rev1=1 crypto rx=0x7f8c7c00c0e0 tx=0x7f8c6c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f32121f1bbf6717b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.969+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c7c007c20 con 0x7f8c6c003a00 2024-03-04T07:33:26.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.969+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c7c030070 con 0x7f8c6c003a00 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.969+0000 7f8c84d2d640 1 -- 172.21.15.64:0/693989656 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c7c005d50 con 0x7f8c6c003a00 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.970+0000 7f8c87f93640 1 -- 172.21.15.64:0/693989656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c6c003a00 msgr2=0x7f8c6c003e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.970+0000 7f8c87f93640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c6c003a00 0x7f8c6c003e20 secure :-1 s=READY pgs=1487 cs=0 l=1 rev1=1 crypto rx=0x7f8c7c00c0e0 tx=0x7f8c6c005830 comp rx=0 tx=0).stop 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.970+0000 7f8c87f93640 1 -- 172.21.15.64:0/693989656 shutdown_connections 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.970+0000 7f8c87f93640 1 --2- 172.21.15.64:0/693989656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c6c003a00 0x7f8c6c003e20 unknown :-1 s=CLOSED pgs=1487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.970+0000 7f8c87f93640 1 -- 172.21.15.64:0/693989656 >> 172.21.15.64:0/693989656 conn(0x7f8c80071960 msgr2=0x7f8c80071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.970+0000 7f8c87f93640 1 -- 172.21.15.64:0/693989656 shutdown_connections 2024-03-04T07:33:26.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.970+0000 7f8c87f93640 1 -- 172.21.15.64:0/693989656 wait complete. 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.971+0000 7f8c87f93640 1 Processor -- start 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.971+0000 7f8c87f93640 1 -- start start 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c87f93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c003a00 0x7f8c800cd0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c87f93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c8015e2f0 0x7f8c8015e750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c87f93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c8015ec90 0x7f8c8015f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c87f93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c800b71a0 con 0x7f8c8015ec90 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c87f93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c8015e170 con 0x7f8c8015e2f0 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c87f93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8c800b7320 con 0x7f8c6c003a00 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c8015ec90 0x7f8c8015f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c8015ec90 0x7f8c8015f110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51436/0 (socket says 172.21.15.64:51436) 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 -- 172.21.15.64:0/1715906193 learned_addr learned my addr 172.21.15.64:0/1715906193 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c85d2f640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c003a00 0x7f8c800cd0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c8552e640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c8015e2f0 0x7f8c8015e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 -- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c003a00 msgr2=0x7f8c800cd0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c003a00 0x7f8c800cd0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 -- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c8015e2f0 msgr2=0x7f8c8015e750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c8015e2f0 0x7f8c8015e750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 -- 172.21.15.64:0/1715906193 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c800cad70 con 0x7f8c8015ec90 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c8552e640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c8015e2f0 0x7f8c8015e750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.972+0000 7f8c86530640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c8015ec90 0x7f8c8015f110 secure :-1 s=READY pgs=1488 cs=0 l=1 rev1=1 crypto rx=0x7f8c800b9df0 tx=0x7f8c7800dc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.973+0000 7f8c767fc640 1 -- 172.21.15.64:0/1715906193 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c78018070 con 0x7f8c8015ec90 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.973+0000 7f8c767fc640 1 -- 172.21.15.64:0/1715906193 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8c78003b70 con 0x7f8c8015ec90 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.973+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c8015f850 con 0x7f8c8015ec90 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.973+0000 7f8c767fc640 1 -- 172.21.15.64:0/1715906193 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c78005070 con 0x7f8c8015ec90 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.973+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c800cb030 con 0x7f8c8015ec90 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.974+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c80072a30 con 0x7f8c8015ec90 2024-03-04T07:33:26.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.974+0000 7f8c767fc640 1 -- 172.21.15.64:0/1715906193 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8c7801b070 con 0x7f8c8015ec90 2024-03-04T07:33:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.974+0000 7f8c767fc640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c6406b4d0 0x7f8c6406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.975+0000 7f8c85d2f640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c6406b4d0 0x7f8c6406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.975+0000 7f8c767fc640 1 -- 172.21.15.64:0/1715906193 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(113..113 src has 1..113) v4 ==== 5587+0+0 (secure 0 0 0) 0x7f8c78089880 con 0x7f8c8015ec90 2024-03-04T07:33:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.975+0000 7f8c85d2f640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c6406b4d0 0x7f8c6406d990 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f8c7c034000 tx=0x7f8c7c039040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:26.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:26.979+0000 7f8c767fc640 1 -- 172.21.15.64:0/1715906193 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8c78058770 con 0x7f8c8015ec90 2024-03-04T07:33:27.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.160+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f8c800c7570 con 0x7f8c8015ec90 2024-03-04T07:33:27.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.846+0000 7f8c767fc640 1 -- 172.21.15.64:0/1715906193 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v114) v1 ==== 169+0+0 (secure 0 0 0) 0x7f8c78058070 con 0x7f8c8015ec90 2024-03-04T07:33:27.846 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c6406b4d0 msgr2=0x7f8c6406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c6406b4d0 0x7f8c6406d990 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f8c7c034000 tx=0x7f8c7c039040 comp rx=0 tx=0).stop 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c8015ec90 msgr2=0x7f8c8015f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c8015ec90 0x7f8c8015f110 secure :-1 s=READY pgs=1488 cs=0 l=1 rev1=1 crypto rx=0x7f8c800b9df0 tx=0x7f8c7800dc00 comp rx=0 tx=0).stop 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 shutdown_connections 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8c6406b4d0 0x7f8c6406d990 unknown :-1 s=CLOSED pgs=665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8c8015ec90 0x7f8c8015f110 unknown :-1 s=CLOSED pgs=1488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.848+0000 7f8c87f93640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8c8015e2f0 0x7f8c8015e750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.849+0000 7f8c87f93640 1 --2- 172.21.15.64:0/1715906193 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8c6c003a00 0x7f8c800cd0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.849+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 >> 172.21.15.64:0/1715906193 conn(0x7f8c80071960 msgr2=0x7f8c800b4b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.849+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 shutdown_connections 2024-03-04T07:33:27.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.849+0000 7f8c87f93640 1 -- 172.21.15.64:0/1715906193 wait complete. 2024-03-04T07:33:27.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:33:27.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.974+0000 7f7d3cc0e640 1 Processor -- start 2024-03-04T07:33:27.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.974+0000 7f7d3cc0e640 1 -- start start 2024-03-04T07:33:27.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d3cc0e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7d3815dbd0 0x7f7d3815ee70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d3cc0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 0x7f7d38066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d3cc0e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7d3815f3b0 con 0x7f7d380b5840 2024-03-04T07:33:27.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d3cc0e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7d3806f070 con 0x7f7d3815dbd0 2024-03-04T07:33:27.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d3cc0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d38071940 con 0x7f7d380b40c0 2024-03-04T07:33:27.981 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d2ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7d380b5840 0x7f7d380b39b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51436/0 (socket says 172.21.15.64:51436) 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3834772253 learned_addr learned my addr 172.21.15.64:0/3834772253 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d36d9d640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 0x7f7d38066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d3659c640 1 --2- 172.21.15.64:0/3834772253 >> v2:172.21.15.66:3300/0 conn(0x7f7d3815dbd0 0x7f7d3815ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 936474200 0 0) 0x7f7d3815f3b0 con 0x7f7d380b5840 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7d24003520 con 0x7f7d380b5840 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.975+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1000007921 0 0) 0x7f7d24003520 con 0x7f7d380b5840 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 msgr2=0x7f7d38066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 0x7f7d38066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 >> v2:172.21.15.66:3300/0 conn(0x7f7d3815dbd0 msgr2=0x7f7d3815ee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 --2- 172.21.15.64:0/3834772253 >> v2:172.21.15.66:3300/0 conn(0x7f7d3815dbd0 0x7f7d3815ee70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d380b5db0 con 0x7f7d380b5840 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f7d20001670 con 0x7f7d380b5840 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 >> v1:172.21.15.64:6789/0 conn(0x7f7d380b5840 legacy=0x7f7d380b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:27.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d240039b0 0x7f7d24003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d24004800 0x7f7d24004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 0x7f7d240051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d3806f070 con 0x7f7d24004800 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d38071940 con 0x7f7d240039b0 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d2400b9c0 con 0x7f7d380b40c0 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d2ffff640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d24004800 0x7f7d24004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d3659c640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d240039b0 0x7f7d24003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d36d9d640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 0x7f7d240051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 msgr2=0x7f7d240051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d2ffff640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 0x7f7d240051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d240039b0 msgr2=0x7f7d24003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d2ffff640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d240039b0 0x7f7d24003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3834772253 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d24003520 con 0x7f7d24004800 2024-03-04T07:33:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.976+0000 7f7d36d9d640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380b40c0 0x7f7d240051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d2ffff640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d24004800 0x7f7d24004c80 secure :-1 s=READY pgs=1490 cs=0 l=1 rev1=1 crypto rx=0x7f7d20007fd0 tx=0x7f7d24005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c8bb1221c05e859 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d20007d50 con 0x7f7d24004800 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d20030070 con 0x7f7d24004800 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d35d9b640 1 -- 172.21.15.64:0/3834772253 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d2002cd40 con 0x7f7d24004800 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3834772253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d24004800 msgr2=0x7f7d24004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d3cc0e640 1 --2- 172.21.15.64:0/3834772253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d24004800 0x7f7d24004c80 secure :-1 s=READY pgs=1490 cs=0 l=1 rev1=1 crypto rx=0x7f7d20007fd0 tx=0x7f7d24005830 comp rx=0 tx=0).stop 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d36d9d640 1 -- 172.21.15.64:0/3834772253 reap_dead start 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3834772253 shutdown_connections 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.977+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3834772253 >> 172.21.15.64:0/3834772253 conn(0x7f7d380b26b0 msgr2=0x7f7d380b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.978+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3834772253 shutdown_connections 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.978+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3834772253 wait complete. 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.979+0000 7f7d3cc0e640 1 Processor -- start 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.979+0000 7f7d3cc0e640 1 -- start start 2024-03-04T07:33:27.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d24004800 0x7f7d38157770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d38067690 0x7f7d3814ce90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380c5920 0x7f7d380c5da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d380721c0 con 0x7f7d38067690 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d38071ee0 con 0x7f7d24004800 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7d380726e0 con 0x7f7d380c5920 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d38067690 0x7f7d3814ce90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d38067690 0x7f7d3814ce90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51454/0 (socket says 172.21.15.64:51454) 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3662264088 learned_addr learned my addr 172.21.15.64:0/3662264088 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3659c640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d24004800 0x7f7d38157770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d36d9d640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380c5920 0x7f7d380c5da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380c5920 msgr2=0x7f7d380c5da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380c5920 0x7f7d380c5da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d24004800 msgr2=0x7f7d38157770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:27.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d24004800 0x7f7d38157770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 -- 172.21.15.64:0/3662264088 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d380c6810 con 0x7f7d38067690 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3659c640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d24004800 0x7f7d38157770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d36d9d640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380c5920 0x7f7d380c5da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2ffff640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d38067690 0x7f7d3814ce90 secure :-1 s=READY pgs=1491 cs=0 l=1 rev1=1 crypto rx=0x7f7d2002c570 tx=0x7f7d20000fe0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2effd640 1 -- 172.21.15.64:0/3662264088 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d20001cb0 con 0x7f7d38067690 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2effd640 1 -- 172.21.15.64:0/3662264088 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7d20004bf0 con 0x7f7d38067690 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d381f4c90 con 0x7f7d38067690 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d2effd640 1 -- 172.21.15.64:0/3662264088 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d20007070 con 0x7f7d38067690 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.980+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d381f5250 con 0x7f7d38067690 2024-03-04T07:33:27.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.981+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d38073380 con 0x7f7d38067690 2024-03-04T07:33:27.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.982+0000 7f7d2effd640 1 -- 172.21.15.64:0/3662264088 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7d20007210 con 0x7f7d38067690 2024-03-04T07:33:27.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.982+0000 7f7d2effd640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d1806b4d0 0x7f7d1806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:27.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.982+0000 7f7d2effd640 1 -- 172.21.15.64:0/3662264088 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f7d200ab490 con 0x7f7d38067690 2024-03-04T07:33:27.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.982+0000 7f7d3659c640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d1806b4d0 0x7f7d1806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:27.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.983+0000 7f7d3659c640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d1806b4d0 0x7f7d1806d990 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f7d28009bc0 tx=0x7f7d28008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:27.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:27.987+0000 7f7d2effd640 1 -- 172.21.15.64:0/3662264088 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7d2007a6d0 con 0x7f7d38067690 2024-03-04T07:33:28.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.169+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7f7d3804f410 con 0x7f7d38067690 2024-03-04T07:33:28.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.175+0000 7f7d2effd640 1 -- 172.21.15.64:0/3662264088 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v31) v1 ==== 104+0+0 (secure 0 0 0) 0x7f7d20079fd0 con 0x7f7d38067690 2024-03-04T07:33:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d1806b4d0 msgr2=0x7f7d1806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d1806b4d0 0x7f7d1806d990 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f7d28009bc0 tx=0x7f7d28008040 comp rx=0 tx=0).stop 2024-03-04T07:33:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d38067690 msgr2=0x7f7d3814ce90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d38067690 0x7f7d3814ce90 secure :-1 s=READY pgs=1491 cs=0 l=1 rev1=1 crypto rx=0x7f7d2002c570 tx=0x7f7d20000fe0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 shutdown_connections 2024-03-04T07:33:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7d1806b4d0 0x7f7d1806d990 unknown :-1 s=CLOSED pgs=666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7d380c5920 0x7f7d380c5da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7d38067690 0x7f7d3814ce90 unknown :-1 s=CLOSED pgs=1491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 --2- 172.21.15.64:0/3662264088 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7d24004800 0x7f7d38157770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 >> 172.21.15.64:0/3662264088 conn(0x7f7d380b26b0 msgr2=0x7f7d3815cb80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:28.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 shutdown_connections 2024-03-04T07:33:28.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.177+0000 7f7d3cc0e640 1 -- 172.21.15.64:0/3662264088 wait complete. 2024-03-04T07:33:28.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration' 2024-03-04T07:33:28.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.290+0000 7f9555fbf640 1 Processor -- start 2024-03-04T07:33:28.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.290+0000 7f9555fbf640 1 -- start start 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.290+0000 7f9555fbf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9550157b80 0x7f955015b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.290+0000 7f9555fbf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f955006c7b0 con 0x7f95500cbab0 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.290+0000 7f9555fbf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f955006ca90 con 0x7f9550157f60 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.290+0000 7f9555fbf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f955006cd70 con 0x7f9550157b80 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f95500cbab0 0x7f9550070b40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51438/0 (socket says 172.21.15.64:51438) 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954ffff640 1 -- 172.21.15.64:0/229399328 learned_addr learned my addr 172.21.15.64:0/229399328 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954f7fe640 1 --2- 172.21.15.64:0/229399328 >> v2:172.21.15.66:3300/0 conn(0x7f9550157b80 0x7f955015b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954e7fc640 1 -- 172.21.15.64:0/229399328 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4217346440 0 0) 0x7f955006c7b0 con 0x7f95500cbab0 2024-03-04T07:33:28.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954e7fc640 1 -- 172.21.15.64:0/229399328 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9538003520 con 0x7f95500cbab0 2024-03-04T07:33:28.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954e7fc640 1 -- 172.21.15.64:0/229399328 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1976396053 0 0) 0x7f9538003520 con 0x7f95500cbab0 2024-03-04T07:33:28.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954f7fe640 1 -- 172.21.15.64:0/229399328 >> v1:172.21.15.66:6789/0 conn(0x7f9550157f60 legacy=0x7f955015b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954f7fe640 1 -- 172.21.15.64:0/229399328 >> v1:172.21.15.64:6789/0 conn(0x7f95500cbab0 legacy=0x7f9550070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.291+0000 7f954f7fe640 1 -- 172.21.15.64:0/229399328 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f955006d050 con 0x7f9550157b80 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.292+0000 7f954f7fe640 1 --2- 172.21.15.64:0/229399328 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f955015b260 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7f954000a940 tx=0x7f954002ccf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fb08064d2b8f4e89 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.292+0000 7f954e7fc640 1 -- 172.21.15.64:0/229399328 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f954003a070 con 0x7f9550157b80 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.292+0000 7f954e7fc640 1 -- 172.21.15.64:0/229399328 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f954002d820 con 0x7f9550157b80 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.292+0000 7f954e7fc640 1 -- 172.21.15.64:0/229399328 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95480019f0 con 0x7f9550157b80 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.293+0000 7f9555fbf640 1 -- 172.21.15.64:0/229399328 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 msgr2=0x7f955015b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.293+0000 7f9555fbf640 1 --2- 172.21.15.64:0/229399328 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f955015b260 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7f954000a940 tx=0x7f954002ccf0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.293+0000 7f9555fbf640 1 -- 172.21.15.64:0/229399328 shutdown_connections 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.293+0000 7f9555fbf640 1 --2- 172.21.15.64:0/229399328 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f955015b260 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.293+0000 7f9555fbf640 1 -- 172.21.15.64:0/229399328 >> 172.21.15.64:0/229399328 conn(0x7f9550066d00 msgr2=0x7f95500670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.293+0000 7f9555fbf640 1 -- 172.21.15.64:0/229399328 shutdown_connections 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.293+0000 7f9555fbf640 1 -- 172.21.15.64:0/229399328 wait complete. 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.294+0000 7f9555fbf640 1 Processor -- start 2024-03-04T07:33:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.294+0000 7f9555fbf640 1 -- start start 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9555fbf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500cbab0 0x7f9550145310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9555fbf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f9550155270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9555fbf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9550157f60 0x7f9550153980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9555fbf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9550071250 con 0x7f9550157f60 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9555fbf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95500cc260 con 0x7f9550157b80 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9555fbf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95500713d0 con 0x7f95500cbab0 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9550157f60 0x7f9550153980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9550157f60 0x7f9550153980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51466/0 (socket says 172.21.15.64:51466) 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 -- 172.21.15.64:0/3411730656 learned_addr learned my addr 172.21.15.64:0/3411730656 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954f7fe640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500cbab0 0x7f9550145310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954effd640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f9550155270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 -- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500cbab0 msgr2=0x7f9550145310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500cbab0 0x7f9550145310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 -- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 msgr2=0x7f9550155270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f9550155270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 -- 172.21.15.64:0/3411730656 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9550145850 con 0x7f9550157f60 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954effd640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f9550155270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f954ffff640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9550157f60 0x7f9550153980 secure :-1 s=READY pgs=1493 cs=0 l=1 rev1=1 crypto rx=0x7f95480018c0 tx=0x7f954800a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9537fff640 1 -- 172.21.15.64:0/3411730656 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9548012400 con 0x7f9550157f60 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9537fff640 1 -- 172.21.15.64:0/3411730656 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95480125c0 con 0x7f9550157f60 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9537fff640 1 -- 172.21.15.64:0/3411730656 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95480129b0 con 0x7f9550157f60 2024-03-04T07:33:28.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.295+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95501541d0 con 0x7f9550157f60 2024-03-04T07:33:28.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.296+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9550154790 con 0x7f9550157f60 2024-03-04T07:33:28.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.296+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9550074980 con 0x7f9550157f60 2024-03-04T07:33:28.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.297+0000 7f9537fff640 1 -- 172.21.15.64:0/3411730656 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9548012b50 con 0x7f9550157f60 2024-03-04T07:33:28.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.297+0000 7f9537fff640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953c06b4d0 0x7f953c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.297+0000 7f9537fff640 1 -- 172.21.15.64:0/3411730656 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f954808a6e0 con 0x7f9550157f60 2024-03-04T07:33:28.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.297+0000 7f954f7fe640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953c06b4d0 0x7f953c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.298+0000 7f954f7fe640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953c06b4d0 0x7f953c06d990 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f9540037000 tx=0x7f9540008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:28.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.302+0000 7f9537fff640 1 -- 172.21.15.64:0/3411730656 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9550074980 con 0x7f9550157f60 2024-03-04T07:33:28.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.567+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]} v 0) v1 -- 0x7f95500725f0 con 0x7f9550157f60 2024-03-04T07:33:28.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.850+0000 7f9537fff640 1 -- 172.21.15.64:0/3411730656 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_hardlink_reintegration"]}]=0 v355) v1 ==== 142+0+0 (secure 0 0 0) 0x7f95500725f0 con 0x7f9550157f60 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.852+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953c06b4d0 msgr2=0x7f953c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.852+0000 7f9555fbf640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953c06b4d0 0x7f953c06d990 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f9540037000 tx=0x7f9540008040 comp rx=0 tx=0).stop 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.852+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9550157f60 msgr2=0x7f9550153980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.852+0000 7f9555fbf640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9550157f60 0x7f9550153980 secure :-1 s=READY pgs=1493 cs=0 l=1 rev1=1 crypto rx=0x7f95480018c0 tx=0x7f954800a430 comp rx=0 tx=0).stop 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 shutdown_connections 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f953c06b4d0 0x7f953c06d990 unknown :-1 s=CLOSED pgs=667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9550157f60 0x7f9550153980 unknown :-1 s=CLOSED pgs=1493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9550157b80 0x7f9550155270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 --2- 172.21.15.64:0/3411730656 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95500cbab0 0x7f9550145310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 >> 172.21.15.64:0/3411730656 conn(0x7f9550066d00 msgr2=0x7f955006d570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 shutdown_connections 2024-03-04T07:33:28.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.853+0000 7f9555fbf640 1 -- 172.21.15.64:0/3411730656 wait complete. 2024-03-04T07:33:28.864 INFO:tasks.cephfs_test_runner:That removal of primary dentry of hardlinked inode results ... ok 2024-03-04T07:33:28.865 INFO:tasks.cephfs_test_runner:Starting test: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) Reproduce https://tracker.ceph.com/issues/53597 2024-03-04T07:33:28.865 INFO:tasks.cephfs_test_runner:test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:33:28.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:28.967 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.967+0000 7fcbb8ced640 1 Processor -- start 2024-03-04T07:33:28.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.967+0000 7fcbb8ced640 1 -- start start 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.968+0000 7fcbb8ced640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb415df00 0x7fcbb40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.968+0000 7fcbb8ced640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcbb40b9a30 0x7fcbb40d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.968+0000 7fcbb8ced640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcbb40cea30 con 0x7fcbb40b95a0 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.968+0000 7fcbb8ced640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcbb40d2af0 con 0x7fcbb40b9a30 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.968+0000 7fcbb8ced640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbb40cf3c0 con 0x7fcbb415df00 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.968+0000 7fcbb1d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcbb40b95a0 0x7fcbb40b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51448/0 (socket says 172.21.15.64:51448) 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb1d9b640 1 -- 172.21.15.64:0/1927319023 learned_addr learned my addr 172.21.15.64:0/1927319023 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb2d9d640 1 --2- 172.21.15.64:0/1927319023 >> v2:172.21.15.66:3300/0 conn(0x7fcbb40b9a30 0x7fcbb40d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb259c640 1 --2- 172.21.15.64:0/1927319023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb415df00 0x7fcbb40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb159a640 1 -- 172.21.15.64:0/1927319023 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3468347903 0 0) 0x7fcbb40cea30 con 0x7fcbb40b95a0 2024-03-04T07:33:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb159a640 1 -- 172.21.15.64:0/1927319023 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcba0003520 con 0x7fcbb40b95a0 2024-03-04T07:33:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb259c640 1 -- 172.21.15.64:0/1927319023 >> v2:172.21.15.66:3300/0 conn(0x7fcbb40b9a30 msgr2=0x7fcbb40d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb259c640 1 --2- 172.21.15.64:0/1927319023 >> v2:172.21.15.66:3300/0 conn(0x7fcbb40b9a30 0x7fcbb40d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb259c640 1 -- 172.21.15.64:0/1927319023 >> v1:172.21.15.64:6789/0 conn(0x7fcbb40b95a0 legacy=0x7fcbb40b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.969+0000 7fcbb259c640 1 -- 172.21.15.64:0/1927319023 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbb40cf0e0 con 0x7fcbb415df00 2024-03-04T07:33:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.970+0000 7fcbb259c640 1 --2- 172.21.15.64:0/1927319023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb415df00 0x7fcbb40d25b0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fcbac0097a0 tx=0x7fcbac02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7224ed6c642ce431 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.970+0000 7fcbb159a640 1 -- 172.21.15.64:0/1927319023 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbac035020 con 0x7fcbb415df00 2024-03-04T07:33:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.970+0000 7fcbb159a640 1 -- 172.21.15.64:0/1927319023 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcbac031830 con 0x7fcbb415df00 2024-03-04T07:33:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.970+0000 7fcbb159a640 1 -- 172.21.15.64:0/1927319023 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbac031be0 con 0x7fcbb415df00 2024-03-04T07:33:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.970+0000 7fcbb8ced640 1 -- 172.21.15.64:0/1927319023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb415df00 msgr2=0x7fcbb40d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.970+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/1927319023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb415df00 0x7fcbb40d25b0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fcbac0097a0 tx=0x7fcbac02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.971+0000 7fcbb8ced640 1 -- 172.21.15.64:0/1927319023 shutdown_connections 2024-03-04T07:33:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.971+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/1927319023 >> v2:172.21.15.66:3300/0 conn(0x7fcbb40b9a30 0x7fcbb40d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.971+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/1927319023 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb415df00 0x7fcbb40d25b0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.971+0000 7fcbb8ced640 1 -- 172.21.15.64:0/1927319023 >> 172.21.15.64:0/1927319023 conn(0x7fcbb4071960 msgr2=0x7fcbb4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.971+0000 7fcbb8ced640 1 -- 172.21.15.64:0/1927319023 shutdown_connections 2024-03-04T07:33:28.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.971+0000 7fcbb8ced640 1 -- 172.21.15.64:0/1927319023 wait complete. 2024-03-04T07:33:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.972+0000 7fcbb8ced640 1 Processor -- start 2024-03-04T07:33:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.972+0000 7fcbb8ced640 1 -- start start 2024-03-04T07:33:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb40b95a0 0x7fcbb4151280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbb415df00 0x7fcbb4158720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbb4158c60 0x7fcbb40b6e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbb40cec70 con 0x7fcbb415df00 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbb40ce750 con 0x7fcbb4158c60 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcbb40cedf0 con 0x7fcbb40b95a0 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbb415df00 0x7fcbb4158720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbb415df00 0x7fcbb4158720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51470/0 (socket says 172.21.15.64:51470) 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 -- 172.21.15.64:0/74889445 learned_addr learned my addr 172.21.15.64:0/74889445 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:28.975 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb2d9d640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbb4158c60 0x7fcbb40b6e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb259c640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb40b95a0 0x7fcbb4151280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 -- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb40b95a0 msgr2=0x7fcbb4151280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb40b95a0 0x7fcbb4151280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 -- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbb4158c60 msgr2=0x7fcbb40b6e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbb4158c60 0x7fcbb40b6e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 -- 172.21.15.64:0/74889445 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbb41573a0 con 0x7fcbb415df00 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb259c640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb40b95a0 0x7fcbb4151280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb1d9b640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbb415df00 0x7fcbb4158720 secure :-1 s=READY pgs=1495 cs=0 l=1 rev1=1 crypto rx=0x7fcb9c00c460 tx=0x7fcb9c00c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbaaffd640 1 -- 172.21.15.64:0/74889445 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcb9c017070 con 0x7fcbb415df00 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbaaffd640 1 -- 172.21.15.64:0/74889445 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcb9c005ce0 con 0x7fcbb415df00 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbb4157680 con 0x7fcbb415df00 2024-03-04T07:33:28.976 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbaaffd640 1 -- 172.21.15.64:0/74889445 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcb9c007290 con 0x7fcbb415df00 2024-03-04T07:33:28.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.973+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbb4157c40 con 0x7fcbb415df00 2024-03-04T07:33:28.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.974+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbb4067ff0 con 0x7fcbb415df00 2024-03-04T07:33:28.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.975+0000 7fcbaaffd640 1 -- 172.21.15.64:0/74889445 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcb9c01a070 con 0x7fcbb415df00 2024-03-04T07:33:28.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.975+0000 7fcbaaffd640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcb9406b500 0x7fcb9406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:28.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.975+0000 7fcbaaffd640 1 -- 172.21.15.64:0/74889445 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fcb9c089240 con 0x7fcbb415df00 2024-03-04T07:33:28.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.975+0000 7fcbb259c640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcb9406b500 0x7fcb9406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:28.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.976+0000 7fcbb259c640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcb9406b500 0x7fcb9406d9c0 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7fcbac036000 tx=0x7fcbac03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:28.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:28.980+0000 7fcbaaffd640 1 -- 172.21.15.64:0/74889445 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcb9c057cc0 con 0x7fcbb415df00 2024-03-04T07:33:29.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.163+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fcbb40d3290 con 0x7fcbb415df00 2024-03-04T07:33:29.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.164+0000 7fcbaaffd640 1 -- 172.21.15.64:0/74889445 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v114) v1 ==== 74+0+13240 (secure 0 0 0) 0x7fcb9c05bd60 con 0x7fcbb415df00 2024-03-04T07:33:29.164 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:29.164 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":114,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:33:27.839562+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6835/1389682232":"2024-03-05T07:33:24.828216+0000","172.21.15.64:6834/1389682232":"2024-03-05T07:33:24.828216+0000","172.21.15.64:6837/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.64:6835/209754994":"2024-03-05T07:32:22.510444+0000","172.21.15.66:6835/4084900379":"2024-03-05T07:32:21.508754+0000","172.21.15.66:6833/4084900379":"2024-03-05T07:32:21.508754+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:29.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcb9406b500 msgr2=0x7fcb9406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcb9406b500 0x7fcb9406d9c0 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7fcbac036000 tx=0x7fcbac03c040 comp rx=0 tx=0).stop 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbb415df00 msgr2=0x7fcbb4158720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbb415df00 0x7fcbb4158720 secure :-1 s=READY pgs=1495 cs=0 l=1 rev1=1 crypto rx=0x7fcb9c00c460 tx=0x7fcb9c00c920 comp rx=0 tx=0).stop 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 shutdown_connections 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcb9406b500 0x7fcb9406d9c0 unknown :-1 s=CLOSED pgs=668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcbb4158c60 0x7fcbb40b6e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcbb415df00 0x7fcbb4158720 unknown :-1 s=CLOSED pgs=1495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 --2- 172.21.15.64:0/74889445 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcbb40b95a0 0x7fcbb4151280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 >> 172.21.15.64:0/74889445 conn(0x7fcbb4071960 msgr2=0x7fcbb415ba00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 shutdown_connections 2024-03-04T07:33:29.167 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.166+0000 7fcbb8ced640 1 -- 172.21.15.64:0/74889445 wait complete. 2024-03-04T07:33:29.177 INFO:tasks.ceph_test_case.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:33:29.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:33:29.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.282+0000 7f1459437640 1 Processor -- start 2024-03-04T07:33:29.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.282+0000 7f1459437640 1 -- start start 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1459437640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f14540b3060 0x7f1454071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1459437640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14540b2b10 0x7f14540d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1459437640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f145415e4b0 con 0x7f14540717b0 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1459437640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14540d2e10 con 0x7f14540b2b10 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1459437640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f14540b97e0 con 0x7f14540b3060 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f14537fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f14540b2b10 0x7f14540d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1452ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f14540717b0 0x7f14540b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51460/0 (socket says 172.21.15.64:51460) 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1452ffd640 1 -- 172.21.15.64:0/3350002071 learned_addr learned my addr 172.21.15.64:0/3350002071 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f14537fe640 1 -- 172.21.15.64:0/3350002071 >> v2:172.21.15.66:3300/0 conn(0x7f14540b3060 msgr2=0x7f1454071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f14527fc640 1 --2- 172.21.15.64:0/3350002071 >> v2:172.21.15.66:3300/0 conn(0x7f14540b3060 0x7f1454071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f1451ffb640 1 -- 172.21.15.64:0/3350002071 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1815016290 0 0) 0x7f144c001110 con 0x7f14540717b0 2024-03-04T07:33:29.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f14537fe640 1 --2- 172.21.15.64:0/3350002071 >> v2:172.21.15.66:3300/0 conn(0x7f14540b3060 0x7f1454071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f14537fe640 1 -- 172.21.15.64:0/3350002071 >> v1:172.21.15.64:6789/0 conn(0x7f14540717b0 legacy=0x7f14540b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f14537fe640 1 -- 172.21.15.64:0/3350002071 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14540b9a80 con 0x7f14540b2b10 2024-03-04T07:33:29.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.283+0000 7f14527fc640 1 --2- 172.21.15.64:0/3350002071 >> v2:172.21.15.66:3300/0 conn(0x7f14540b3060 0x7f1454071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:29.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.284+0000 7f14537fe640 1 --2- 172.21.15.64:0/3350002071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540b2b10 0x7f14540d32a0 secure :-1 s=READY pgs=1497 cs=0 l=1 rev1=1 crypto rx=0x7f145406ce10 tx=0x7f144402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8b1ba8fa5d796225 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:29.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.284+0000 7f1451ffb640 1 -- 172.21.15.64:0/3350002071 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1444039070 con 0x7f14540b2b10 2024-03-04T07:33:29.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.284+0000 7f1451ffb640 1 -- 172.21.15.64:0/3350002071 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f144402cc80 con 0x7f14540b2b10 2024-03-04T07:33:29.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.284+0000 7f1451ffb640 1 -- 172.21.15.64:0/3350002071 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1444034660 con 0x7f14540b2b10 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 -- 172.21.15.64:0/3350002071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540b2b10 msgr2=0x7f14540d32a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 --2- 172.21.15.64:0/3350002071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540b2b10 0x7f14540d32a0 secure :-1 s=READY pgs=1497 cs=0 l=1 rev1=1 crypto rx=0x7f145406ce10 tx=0x7f144402bed0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 -- 172.21.15.64:0/3350002071 shutdown_connections 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 --2- 172.21.15.64:0/3350002071 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540b2b10 0x7f14540d32a0 unknown :-1 s=CLOSED pgs=1497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 --2- 172.21.15.64:0/3350002071 >> v2:172.21.15.66:3300/0 conn(0x7f14540b3060 0x7f1454071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 -- 172.21.15.64:0/3350002071 >> 172.21.15.64:0/3350002071 conn(0x7f14540d3d20 msgr2=0x7f14540d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 -- 172.21.15.64:0/3350002071 shutdown_connections 2024-03-04T07:33:29.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.285+0000 7f1459437640 1 -- 172.21.15.64:0/3350002071 wait complete. 2024-03-04T07:33:29.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.286+0000 7f1459437640 1 Processor -- start 2024-03-04T07:33:29.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.286+0000 7f1459437640 1 -- start start 2024-03-04T07:33:29.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1459437640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540717b0 0x7f14540c9810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1459437640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14540b2b10 0x7f145414ae00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1459437640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14540b3060 0x7f1454148e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1459437640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14540b7560 con 0x7f14540717b0 2024-03-04T07:33:29.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1459437640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14540d4bf0 con 0x7f14540b2b10 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1459437640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14540b76e0 con 0x7f14540b3060 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540717b0 0x7f14540c9810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540717b0 0x7f14540c9810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51488/0 (socket says 172.21.15.64:51488) 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 -- 172.21.15.64:0/1408599713 learned_addr learned my addr 172.21.15.64:0/1408599713 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 -- 172.21.15.64:0/1408599713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14540b3060 msgr2=0x7f1454148e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f14527fc640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14540b2b10 0x7f145414ae00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14540b3060 0x7f1454148e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 -- 172.21.15.64:0/1408599713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14540b2b10 msgr2=0x7f145414ae00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14540b2b10 0x7f145414ae00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 -- 172.21.15.64:0/1408599713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14540c9d50 con 0x7f14540717b0 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f1452ffd640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540717b0 0x7f14540c9810 secure :-1 s=READY pgs=1498 cs=0 l=1 rev1=1 crypto rx=0x7f144c00c7b0 tx=0x7f144c00cc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:29.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.287+0000 7f144b7fe640 1 -- 172.21.15.64:0/1408599713 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f144c018070 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.288+0000 7f144b7fe640 1 -- 172.21.15.64:0/1408599713 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f144c0072d0 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.288+0000 7f144b7fe640 1 -- 172.21.15.64:0/1408599713 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f144c0047e0 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.288+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1454147d60 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.288+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1454148320 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.288+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f145406ce40 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.289+0000 7f144b7fe640 1 -- 172.21.15.64:0/1408599713 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f144c01b070 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.289+0000 7f144b7fe640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f143406b4d0 0x7f143406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.290+0000 7f144b7fe640 1 -- 172.21.15.64:0/1408599713 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f144c089240 con 0x7f14540717b0 2024-03-04T07:33:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.290+0000 7f14527fc640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f143406b4d0 0x7f143406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.292 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.290+0000 7f14527fc640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f143406b4d0 0x7f143406d990 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f14541f6e20 tx=0x7f143c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:29.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.294+0000 7f144b7fe640 1 -- 172.21.15.64:0/1408599713 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f145415e4b0 con 0x7f14540717b0 2024-03-04T07:33:29.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.472+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f14540d33d0 con 0x7f14540717b0 2024-03-04T07:33:29.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.472+0000 7f144b7fe640 1 -- 172.21.15.64:0/1408599713 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v114) v1 ==== 93+0+10 (secure 0 0 0) 0x7f144c0583a0 con 0x7f14540717b0 2024-03-04T07:33:29.472 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:33:29.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f143406b4d0 msgr2=0x7f143406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f143406b4d0 0x7f143406d990 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f14541f6e20 tx=0x7f143c008040 comp rx=0 tx=0).stop 2024-03-04T07:33:29.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540717b0 msgr2=0x7f14540c9810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540717b0 0x7f14540c9810 secure :-1 s=READY pgs=1498 cs=0 l=1 rev1=1 crypto rx=0x7f144c00c7b0 tx=0x7f144c00cc70 comp rx=0 tx=0).stop 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 shutdown_connections 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f143406b4d0 0x7f143406d990 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f14540b3060 0x7f1454148e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f14540b2b10 0x7f145414ae00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 --2- 172.21.15.64:0/1408599713 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f14540717b0 0x7f14540c9810 unknown :-1 s=CLOSED pgs=1498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 >> 172.21.15.64:0/1408599713 conn(0x7f14540d3d20 msgr2=0x7f145415a4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 shutdown_connections 2024-03-04T07:33:29.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.474+0000 7f1459437640 1 -- 172.21.15.64:0/1408599713 wait complete. 2024-03-04T07:33:29.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Starting test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir' 2024-03-04T07:33:29.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.587+0000 7fcf00b13640 1 Processor -- start 2024-03-04T07:33:29.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.587+0000 7fcf00b13640 1 -- start start 2024-03-04T07:33:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcf00b13640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcefc0d02e0 0x7fcefc0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcf00b13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0b4260 0x7fcefc0b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcf00b13640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcefc0b0dd0 con 0x7fcefc0d06c0 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcf00b13640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcefc0b0b80 con 0x7fcefc0d02e0 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcf00b13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcefc0aeec0 con 0x7fcefc0b4260 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcefa59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcefc0d02e0 0x7fcefc0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcefad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcefc0d06c0 0x7fcefc0b1a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51470/0 (socket says 172.21.15.64:51470) 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcefa59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcefc0d02e0 0x7fcefc0cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51494/0 (socket says 172.21.15.64:51494) 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.588+0000 7fcefad9d640 1 -- 172.21.15.64:0/181968829 learned_addr learned my addr 172.21.15.64:0/181968829 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.589+0000 7fcef9d9b640 1 --2- 172.21.15.64:0/181968829 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0b4260 0x7fcefc0b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.589+0000 7fcefa59c640 1 -- 172.21.15.64:0/181968829 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0b4260 msgr2=0x7fcefc0b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.589+0000 7fcefa59c640 1 --2- 172.21.15.64:0/181968829 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0b4260 0x7fcefc0b1500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.589+0000 7fcefa59c640 1 -- 172.21.15.64:0/181968829 >> v1:172.21.15.64:6789/0 conn(0x7fcefc0d06c0 legacy=0x7fcefc0b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.589+0000 7fcefa59c640 1 -- 172.21.15.64:0/181968829 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcefc0b10b0 con 0x7fcefc0d02e0 2024-03-04T07:33:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.589+0000 7fcef959a640 1 -- 172.21.15.64:0/181968829 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1780829251 0 0) 0x7fcefc0b0dd0 con 0x7fcefc0d06c0 2024-03-04T07:33:29.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.589+0000 7fcefa59c640 1 --2- 172.21.15.64:0/181968829 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc0cfa00 secure :-1 s=READY pgs=1500 cs=0 l=1 rev1=1 crypto rx=0x7fceec009d90 tx=0x7fceec02bf30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=226026a6a3930630 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:29.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcef959a640 1 -- 172.21.15.64:0/181968829 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fceec039070 con 0x7fcefc0d02e0 2024-03-04T07:33:29.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcef959a640 1 -- 172.21.15.64:0/181968829 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fceec02cce0 con 0x7fcefc0d02e0 2024-03-04T07:33:29.591 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcef959a640 1 -- 172.21.15.64:0/181968829 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fceec034660 con 0x7fcefc0d02e0 2024-03-04T07:33:29.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 -- 172.21.15.64:0/181968829 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 msgr2=0x7fcefc0cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 --2- 172.21.15.64:0/181968829 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc0cfa00 secure :-1 s=READY pgs=1500 cs=0 l=1 rev1=1 crypto rx=0x7fceec009d90 tx=0x7fceec02bf30 comp rx=0 tx=0).stop 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 -- 172.21.15.64:0/181968829 shutdown_connections 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 --2- 172.21.15.64:0/181968829 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0b4260 0x7fcefc0b1500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 --2- 172.21.15.64:0/181968829 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc0cfa00 unknown :-1 s=CLOSED pgs=1500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 -- 172.21.15.64:0/181968829 >> 172.21.15.64:0/181968829 conn(0x7fcefc0b36d0 msgr2=0x7fcefc0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 -- 172.21.15.64:0/181968829 shutdown_connections 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.590+0000 7fcf00b13640 1 -- 172.21.15.64:0/181968829 wait complete. 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.591+0000 7fcf00b13640 1 Processor -- start 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcf00b13640 1 -- start start 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcf00b13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcefc0b4260 0x7fcefc06b0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcf00b13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc06b620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcf00b13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0d06c0 0x7fcefc154510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcf00b13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcefc0b22d0 con 0x7fcefc0d02e0 2024-03-04T07:33:29.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcf00b13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcefc0b2150 con 0x7fcefc0b4260 2024-03-04T07:33:29.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcf00b13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcefc0af350 con 0x7fcefc0d06c0 2024-03-04T07:33:29.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcef9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc06b620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcef9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc06b620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51502/0 (socket says 172.21.15.64:51502) 2024-03-04T07:33:29.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcef9d9b640 1 -- 172.21.15.64:0/2479886775 learned_addr learned my addr 172.21.15.64:0/2479886775 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:29.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcefad9d640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0d06c0 0x7fcefc154510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.592+0000 7fcefa59c640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcefc0b4260 0x7fcefc06b0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef9d9b640 1 -- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0d06c0 msgr2=0x7fcefc154510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef9d9b640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0d06c0 0x7fcefc154510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef9d9b640 1 -- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcefc0b4260 msgr2=0x7fcefc06b0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:29.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef9d9b640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcefc0b4260 0x7fcefc06b0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:29.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef9d9b640 1 -- 172.21.15.64:0/2479886775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcefc06bb60 con 0x7fcefc0d02e0 2024-03-04T07:33:29.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef9d9b640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc06b620 secure :-1 s=READY pgs=1501 cs=0 l=1 rev1=1 crypto rx=0x7fcee400d7d0 tx=0x7fcee400dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef2ffd640 1 -- 172.21.15.64:0/2479886775 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcee4004020 con 0x7fcefc0d02e0 2024-03-04T07:33:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef2ffd640 1 -- 172.21.15.64:0/2479886775 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcee40041e0 con 0x7fcefc0d02e0 2024-03-04T07:33:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcefc06be30 con 0x7fcefc0d02e0 2024-03-04T07:33:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcef2ffd640 1 -- 172.21.15.64:0/2479886775 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcee40043d0 con 0x7fcefc0d02e0 2024-03-04T07:33:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.593+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcefc06c3f0 con 0x7fcefc0d02e0 2024-03-04T07:33:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.594+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcefc068700 con 0x7fcefc0d02e0 2024-03-04T07:33:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.594+0000 7fcef2ffd640 1 -- 172.21.15.64:0/2479886775 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcee401b020 con 0x7fcefc0d02e0 2024-03-04T07:33:29.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.595+0000 7fcef2ffd640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcedc06b4d0 0x7fcedc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:29.601 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.595+0000 7fcef2ffd640 1 -- 172.21.15.64:0/2479886775 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fcee4089c10 con 0x7fcefc0d02e0 2024-03-04T07:33:29.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.595+0000 7fcefa59c640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcedc06b4d0 0x7fcedc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:29.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.595+0000 7fcefa59c640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcedc06b4d0 0x7fcedc06d990 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7fceec036000 tx=0x7fceec030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:29.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.599+0000 7fcef2ffd640 1 -- 172.21.15.64:0/2479886775 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcee4058720 con 0x7fcefc0d02e0 2024-03-04T07:33:29.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:29.853+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir"]} v 0) v1 -- 0x7fcefc15c0e0 con 0x7fcefc0d02e0 2024-03-04T07:33:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.856+0000 7fcef2ffd640 1 -- 172.21.15.64:0/2479886775 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Starting test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir"]}]=0 v357) v1 ==== 143+0+0 (secure 0 0 0) 0x7fcee405c7c0 con 0x7fcefc0d02e0 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcedc06b4d0 msgr2=0x7fcedc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcedc06b4d0 0x7fcedc06d990 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7fceec036000 tx=0x7fceec030000 comp rx=0 tx=0).stop 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 msgr2=0x7fcefc06b620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc06b620 secure :-1 s=READY pgs=1501 cs=0 l=1 rev1=1 crypto rx=0x7fcee400d7d0 tx=0x7fcee400dc90 comp rx=0 tx=0).stop 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 shutdown_connections 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcedc06b4d0 0x7fcedc06d990 unknown :-1 s=CLOSED pgs=670 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcefc0d06c0 0x7fcefc154510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcefc0d02e0 0x7fcefc06b620 unknown :-1 s=CLOSED pgs=1501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 --2- 172.21.15.64:0/2479886775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcefc0b4260 0x7fcefc06b0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.858+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 >> 172.21.15.64:0/2479886775 conn(0x7fcefc0b36d0 msgr2=0x7fcefc151f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:30.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.859+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 shutdown_connections 2024-03-04T07:33:30.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.859+0000 7fcf00b13640 1 -- 172.21.15.64:0/2479886775 wait complete. 2024-03-04T07:33:30.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config set mon mon_allow_pool_delete True 2024-03-04T07:33:30.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 Processor -- start 2024-03-04T07:33:30.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 -- start start 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb4b015f5c0 0x7fb4b00cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b00cff40 0x7fb4b00d03d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4b00b1470 0x7fb4b00b18a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4b00b0af0 con 0x7fb4b00b1470 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4b00b33b0 con 0x7fb4b015f5c0 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.984+0000 7fb4b48fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4b00b0d40 con 0x7fb4b00cff40 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4b00b1470 0x7fb4b00b18a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb4b00b1470 0x7fb4b00b18a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51518/0 (socket says 172.21.15.64:51518) 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 -- 172.21.15.64:0/458208025 learned_addr learned my addr 172.21.15.64:0/458208025 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 -- 172.21.15.64:0/458208025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b00cff40 msgr2=0x7fb4b00d03d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:30.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 --2- 172.21.15.64:0/458208025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b00cff40 0x7fb4b00d03d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 -- 172.21.15.64:0/458208025 >> v2:172.21.15.66:3300/0 conn(0x7fb4b015f5c0 msgr2=0x7fb4b00cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:30.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 --2- 172.21.15.64:0/458208025 >> v2:172.21.15.66:3300/0 conn(0x7fb4b015f5c0 0x7fb4b00cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.985+0000 7fb4aed9d640 1 -- 172.21.15.64:0/458208025 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4b00b1020 con 0x7fb4b00b1470 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4aed9d640 1 --2- 172.21.15.64:0/458208025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b00b18a0 secure :-1 s=READY pgs=1502 cs=0 l=1 rev1=1 crypto rx=0x7fb4b0071e70 tx=0x7fb4a802be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fdc008461dc24e4d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4add9b640 1 -- 172.21.15.64:0/458208025 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4a802c9e0 con 0x7fb4b00b1470 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4add9b640 1 -- 172.21.15.64:0/458208025 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb4a802cb80 con 0x7fb4b00b1470 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4add9b640 1 -- 172.21.15.64:0/458208025 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4a80343f0 con 0x7fb4b00b1470 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 -- 172.21.15.64:0/458208025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 msgr2=0x7fb4b00b18a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/458208025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b00b18a0 secure :-1 s=READY pgs=1502 cs=0 l=1 rev1=1 crypto rx=0x7fb4b0071e70 tx=0x7fb4a802be40 comp rx=0 tx=0).stop 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 -- 172.21.15.64:0/458208025 shutdown_connections 2024-03-04T07:33:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/458208025 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b00b18a0 unknown :-1 s=CLOSED pgs=1502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/458208025 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b00cff40 0x7fb4b00d03d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/458208025 >> v2:172.21.15.66:3300/0 conn(0x7fb4b015f5c0 0x7fb4b00cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 -- 172.21.15.64:0/458208025 >> 172.21.15.64:0/458208025 conn(0x7fb4b00b36d0 msgr2=0x7fb4b00b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:30.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 -- 172.21.15.64:0/458208025 shutdown_connections 2024-03-04T07:33:30.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.986+0000 7fb4b48fb640 1 -- 172.21.15.64:0/458208025 wait complete. 2024-03-04T07:33:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.988+0000 7fb4b48fb640 1 Processor -- start 2024-03-04T07:33:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.988+0000 7fb4b48fb640 1 -- start start 2024-03-04T07:33:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.988+0000 7fb4b48fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b015e5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4b48fb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4b00cff40 0x7fb4b00c43f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4b48fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b015f5c0 0x7fb4b00c4930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4b48fb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4b015f9a0 con 0x7fb4b00b1470 2024-03-04T07:33:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4b48fb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4b00ced80 con 0x7fb4b00cff40 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4b48fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4b00cf2a0 con 0x7fb4b015f5c0 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b015e5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b015e5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51534/0 (socket says 172.21.15.64:51534) 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 -- 172.21.15.64:0/1779926590 learned_addr learned my addr 172.21.15.64:0/1779926590 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4aed9d640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b015f5c0 0x7fb4b00c4930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4a7fff640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4b00cff40 0x7fb4b00c43f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 -- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b015f5c0 msgr2=0x7fb4b00c4930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b015f5c0 0x7fb4b00c4930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 -- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4b00cff40 msgr2=0x7fb4b00c43f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4b00cff40 0x7fb4b00c43f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 -- 172.21.15.64:0/1779926590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4b01f48f0 con 0x7fb4b00b1470 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4ae59c640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b015e5f0 secure :-1 s=READY pgs=1503 cs=0 l=1 rev1=1 crypto rx=0x7fb4a000b4a0 tx=0x7fb4a000b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4a6ffd640 1 -- 172.21.15.64:0/1779926590 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4a0014070 con 0x7fb4b00b1470 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4a6ffd640 1 -- 172.21.15.64:0/1779926590 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb4a0010610 con 0x7fb4b00b1470 2024-03-04T07:33:30.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb4b00c9dc0 con 0x7fb4b00b1470 2024-03-04T07:33:30.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4a6ffd640 1 -- 172.21.15.64:0/1779926590 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4a0010a00 con 0x7fb4b00b1470 2024-03-04T07:33:30.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.989+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb4b00ca380 con 0x7fb4b00b1470 2024-03-04T07:33:30.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.990+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb4b009fd30 con 0x7fb4b00b1470 2024-03-04T07:33:30.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.991+0000 7fb4a6ffd640 1 -- 172.21.15.64:0/1779926590 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb4a0017070 con 0x7fb4b00b1470 2024-03-04T07:33:30.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.991+0000 7fb4a6ffd640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb49006b4d0 0x7fb49006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:30.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.991+0000 7fb4a7fff640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb49006b4d0 0x7fb49006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:30.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.991+0000 7fb4a6ffd640 1 -- 172.21.15.64:0/1779926590 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fb4a0089a90 con 0x7fb4b00b1470 2024-03-04T07:33:30.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.992+0000 7fb4a7fff640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb49006b4d0 0x7fb49006d990 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7fb4b0070310 tx=0x7fb498008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:30.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:30.996+0000 7fb4a6ffd640 1 -- 172.21.15.64:0/1779926590 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb4a0058ae0 con 0x7fb4b00b1470 2024-03-04T07:33:31.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.177+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_allow_pool_delete}] v 0) v1 -- 0x7fb4b004f410 con 0x7fb4b00b1470 2024-03-04T07:33:31.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.182+0000 7fb4a6ffd640 1 -- 172.21.15.64:0/1779926590 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_allow_pool_delete}]=0 v32)=0 v32) v1 ==== 122+0+0 (secure 0 0 0) 0x7fb4a00583e0 con 0x7fb4b00b1470 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb49006b4d0 msgr2=0x7fb49006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb49006b4d0 0x7fb49006d990 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7fb4b0070310 tx=0x7fb498008040 comp rx=0 tx=0).stop 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 msgr2=0x7fb4b015e5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b015e5f0 secure :-1 s=READY pgs=1503 cs=0 l=1 rev1=1 crypto rx=0x7fb4a000b4a0 tx=0x7fb4a000b960 comp rx=0 tx=0).stop 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 shutdown_connections 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb49006b4d0 0x7fb49006d990 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4b015f5c0 0x7fb4b00c4930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb4b00cff40 0x7fb4b00c43f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 --2- 172.21.15.64:0/1779926590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb4b00b1470 0x7fb4b015e5f0 unknown :-1 s=CLOSED pgs=1503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 >> 172.21.15.64:0/1779926590 conn(0x7fb4b00b36d0 msgr2=0x7fb4b00cd1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 shutdown_connections 2024-03-04T07:33:31.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.185+0000 7fb4b48fb640 1 -- 172.21.15.64:0/1779926590 wait complete. 2024-03-04T07:33:31.196 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:33:31.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:33:31.234 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:33:31.293 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:33:31.293 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:33:31.293 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:31.308 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:33:31.308 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:33:31.308 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:31.326 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:33:31.326 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:33:31.326 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:31.363 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:33:31.363 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:33:31.363 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:33:31.383 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:33:31.384 INFO:tasks.cephfs.cephfs_test_case:[, , , ] 2024-03-04T07:33:31.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:31.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.504+0000 7fc5b5dbc640 1 Processor -- start 2024-03-04T07:33:31.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.504+0000 7fc5b5dbc640 1 -- start start 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5b5dbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 0x7fc5b015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5b5dbc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5b00b4160 con 0x7fc5b006e700 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5b5dbc640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5b00b5b40 con 0x7fc5b0072660 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5b5dbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00b3ba0 con 0x7fc5b0072090 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5af7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc5b006e700 0x7fc5b006d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51474/0 (socket says 172.21.15.64:51474) 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5af7fe640 1 -- 172.21.15.64:0/1549018429 learned_addr learned my addr 172.21.15.64:0/1549018429 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5affff640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 0x7fc5b015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1158559984 0 0) 0x7fc5b00b4160 con 0x7fc5b006e700 2024-03-04T07:33:31.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc598003520 con 0x7fc5b006e700 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 4043092738 0 0) 0x7fc598003520 con 0x7fc5b006e700 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 msgr2=0x7fc5b015ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 0x7fc5b015ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 >> v1:172.21.15.66:6789/0 conn(0x7fc5b0072660 legacy=0x7fc5b0071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b00b3e80 con 0x7fc5b006e700 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fc5a4001670 con 0x7fc5b006e700 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 >> v1:172.21.15.64:6789/0 conn(0x7fc5b006e700 legacy=0x7fc5b006d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5980039e0 0x7fc598003e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc598004800 0x7fc598004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 0x7fc5980051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00b3ba0 con 0x7fc5980039e0 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc59800baa0 con 0x7fc598004800 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc59800bca0 con 0x7fc5b0072090 2024-03-04T07:33:31.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5af7fe640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5980039e0 0x7fc598003e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.505+0000 7fc5aeffd640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc598004800 0x7fc598004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5affff640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 0x7fc5980051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5af7fe640 1 -- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 msgr2=0x7fc5980051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5af7fe640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 0x7fc5980051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5af7fe640 1 -- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc598004800 msgr2=0x7fc598004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5af7fe640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc598004800 0x7fc598004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5af7fe640 1 -- 172.21.15.64:0/1549018429 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc598003520 con 0x7fc5980039e0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5affff640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5b0072090 0x7fc5980051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5af7fe640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5980039e0 0x7fc598003e00 secure :-1 s=READY pgs=1505 cs=0 l=1 rev1=1 crypto rx=0x7fc5a4001eb0 tx=0x7fc598005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4eab7b687db97d00 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.506+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a4007d30 con 0x7fc5980039e0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5a4030070 con 0x7fc5980039e0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5ae7fc640 1 -- 172.21.15.64:0/1549018429 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a4005d30 con 0x7fc5980039e0 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/1549018429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5980039e0 msgr2=0x7fc598003e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5b5dbc640 1 --2- 172.21.15.64:0/1549018429 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5980039e0 0x7fc598003e00 secure :-1 s=READY pgs=1505 cs=0 l=1 rev1=1 crypto rx=0x7fc5a4001eb0 tx=0x7fc598005830 comp rx=0 tx=0).stop 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5affff640 1 -- 172.21.15.64:0/1549018429 reap_dead start 2024-03-04T07:33:31.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/1549018429 shutdown_connections 2024-03-04T07:33:31.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/1549018429 >> 172.21.15.64:0/1549018429 conn(0x7fc5b00b2720 msgr2=0x7fc5b00b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:31.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/1549018429 shutdown_connections 2024-03-04T07:33:31.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.507+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/1549018429 wait complete. 2024-03-04T07:33:31.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.508+0000 7fc5b5dbc640 1 Processor -- start 2024-03-04T07:33:31.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.508+0000 7fc5b5dbc640 1 -- start start 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5b5dbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5980039e0 0x7fc5b00cf860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5b5dbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b0b50 0x7fc5b00b0fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5b5dbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00ad8b0 0x7fc5b00add60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5b5dbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00b5750 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5b5dbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00b55d0 con 0x7fc5b00b0b50 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5b5dbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5b00b58d0 con 0x7fc5980039e0 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5affff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00ad8b0 0x7fc5b00add60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5affff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00ad8b0 0x7fc5b00add60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51544/0 (socket says 172.21.15.64:51544) 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5affff640 1 -- 172.21.15.64:0/3846564164 learned_addr learned my addr 172.21.15.64:0/3846564164 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5af7fe640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5980039e0 0x7fc5b00cf860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.509+0000 7fc5aeffd640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b0b50 0x7fc5b00b0fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5affff640 1 -- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5980039e0 msgr2=0x7fc5b00cf860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5affff640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5980039e0 0x7fc5b00cf860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5affff640 1 -- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b0b50 msgr2=0x7fc5b00b0fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5affff640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b0b50 0x7fc5b00b0fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5affff640 1 -- 172.21.15.64:0/3846564164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5b0158cf0 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5affff640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00ad8b0 0x7fc5b00add60 secure :-1 s=READY pgs=1506 cs=0 l=1 rev1=1 crypto rx=0x7fc5b006d380 tx=0x7fc5a80115a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc597fff640 1 -- 172.21.15.64:0/3846564164 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a801a020 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc597fff640 1 -- 172.21.15.64:0/3846564164 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc5a8016470 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5b0158fd0 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc597fff640 1 -- 172.21.15.64:0/3846564164 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5a8016860 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.510+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5b0159590 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.511+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5b0159790 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.511+0000 7fc597fff640 1 -- 172.21.15.64:0/3846564164 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc5a8005d10 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.512+0000 7fc597fff640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59c06b500 0x7fc59c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.512+0000 7fc597fff640 1 -- 172.21.15.64:0/3846564164 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7fc5a808a9c0 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.512+0000 7fc5af7fe640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59c06b500 0x7fc59c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.512+0000 7fc5af7fe640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59c06b500 0x7fc59c06d9c0 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7fc5a4034000 tx=0x7fc5a4009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:31.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.516+0000 7fc597fff640 1 -- 172.21.15.64:0/3846564164 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc5b0159790 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.734+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc5b0067ff0 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.735+0000 7fc597fff640 1 -- 172.21.15.64:0/3846564164 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 76 v76) v1 ==== 94+0+3696 (secure 0 0 0) 0x7fc5a808ce20 con 0x7fc5b00ad8b0 2024-03-04T07:33:31.735 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:31.735 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":76,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7315,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6833/2204670061","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":2204670061},{"type":"v1","addr":"172.21.15.66:6833","nonce":2204670061}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":7318,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/3746326588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6834","nonce":3746326588},{"type":"v1","addr":"172.21.15.66:6835","nonce":3746326588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":70},{"gid":8157,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/463950643","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6836","nonce":463950643},{"type":"v1","addr":"172.21.15.64:6837","nonce":463950643}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":72},{"gid":8490,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6835/2069549223","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":2069549223},{"type":"v1","addr":"172.21.15.64:6835","nonce":2069549223}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":76}],"filesystems":[]} 2024-03-04T07:33:31.735 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 76 2024-03-04T07:33:31.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59c06b500 msgr2=0x7fc59c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59c06b500 0x7fc59c06d9c0 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7fc5a4034000 tx=0x7fc5a4009030 comp rx=0 tx=0).stop 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00ad8b0 msgr2=0x7fc5b00add60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00ad8b0 0x7fc5b00add60 secure :-1 s=READY pgs=1506 cs=0 l=1 rev1=1 crypto rx=0x7fc5b006d380 tx=0x7fc5a80115a0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 shutdown_connections 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc59c06b500 0x7fc59c06d9c0 unknown :-1 s=CLOSED pgs=672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5b00ad8b0 0x7fc5b00add60 unknown :-1 s=CLOSED pgs=1506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5b00b0b50 0x7fc5b00b0fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 --2- 172.21.15.64:0/3846564164 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5980039e0 0x7fc5b00cf860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.736+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 >> 172.21.15.64:0/3846564164 conn(0x7fc5b00b2720 msgr2=0x7fc5b0071980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.737+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 shutdown_connections 2024-03-04T07:33:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.737+0000 7fc5b5dbc640 1 -- 172.21.15.64:0/3846564164 wait complete. 2024-03-04T07:33:31.748 INFO:tasks.ceph.mds.a:Restarting daemon 2024-03-04T07:33:31.748 INFO:tasks.ceph.mds.a:Stopping old one... 2024-03-04T07:33:31.748 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:33:31.748 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:33:31.748 INFO:tasks.ceph.mds.c:Restarting daemon 2024-03-04T07:33:31.749 INFO:tasks.ceph.mds.c:Stopping old one... 2024-03-04T07:33:31.749 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:33:31.749 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:33:31.749 INFO:tasks.ceph.mds.b:Restarting daemon 2024-03-04T07:33:31.749 INFO:tasks.ceph.mds.b:Stopping old one... 2024-03-04T07:33:31.749 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:33:31.749 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:33:31.749 INFO:tasks.ceph.mds.d:Restarting daemon 2024-03-04T07:33:31.750 INFO:tasks.ceph.mds.d:Stopping old one... 2024-03-04T07:33:31.750 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:33:31.750 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:33:31.758 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:33:31.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i c 2024-03-04T07:33:31.758 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:33:31.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i a 2024-03-04T07:33:31.761 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:33:31.762 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i d 2024-03-04T07:33:31.762 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:33:31.762 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mds -f --cluster ceph -i b 2024-03-04T07:33:31.800 INFO:tasks.ceph.mds.c:Started 2024-03-04T07:33:31.800 DEBUG:teuthology.parallel:result is None 2024-03-04T07:33:31.801 INFO:tasks.ceph.mds.a:Started 2024-03-04T07:33:31.801 DEBUG:teuthology.parallel:result is None 2024-03-04T07:33:31.805 INFO:tasks.ceph.mds.d:Started 2024-03-04T07:33:31.805 DEBUG:teuthology.parallel:result is None 2024-03-04T07:33:31.806 INFO:tasks.ceph.mds.b:Started 2024-03-04T07:33:31.806 DEBUG:teuthology.parallel:result is None 2024-03-04T07:33:31.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist clear 2024-03-04T07:33:31.876 INFO:tasks.ceph.mds.c.smithi064.stdout:starting mds.c at 2024-03-04T07:33:31.876 INFO:tasks.ceph.mds.a.smithi064.stdout:starting mds.a at 2024-03-04T07:33:31.897 INFO:tasks.ceph.mds.b.smithi066.stdout:starting mds.b at 2024-03-04T07:33:31.897 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:33:31.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.913+0000 7f090190c640 1 Processor -- start 2024-03-04T07:33:31.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.913+0000 7f090190c640 1 -- start start 2024-03-04T07:33:31.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f090190c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f08fc0b95a0 0x7f08fc0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f090190c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f08fc0b9a10 0x7f08fc0b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f090190c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f08fc0cea30 con 0x7f08fc15df00 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f090190c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f08fc0cf3c0 con 0x7f08fc0b9a10 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f090190c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f08fc0cf0e0 con 0x7f08fc0b95a0 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f08fb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f08fc0b9a10 0x7f08fc0b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f08fb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f08fc0b9a10 0x7f08fc0b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51570/0 (socket says 172.21.15.64:51570) 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f08faffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f08fc15df00 0x7f08fc0d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51494/0 (socket says 172.21.15.64:51494) 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f08fb7fe640 1 -- 172.21.15.64:0/1238374006 learned_addr learned my addr 172.21.15.64:0/1238374006 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.914+0000 7f08f27fc640 1 --2- 172.21.15.64:0/1238374006 >> v2:172.21.15.66:3300/0 conn(0x7f08fc0b95a0 0x7f08fc0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.915+0000 7f08fb7fe640 1 -- 172.21.15.64:0/1238374006 >> v2:172.21.15.66:3300/0 conn(0x7f08fc0b95a0 msgr2=0x7f08fc0b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.915+0000 7f08fb7fe640 1 --2- 172.21.15.64:0/1238374006 >> v2:172.21.15.66:3300/0 conn(0x7f08fc0b95a0 0x7f08fc0b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.915+0000 7f08fb7fe640 1 -- 172.21.15.64:0/1238374006 >> v1:172.21.15.64:6789/0 conn(0x7f08fc15df00 legacy=0x7f08fc0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.915+0000 7f08fb7fe640 1 -- 172.21.15.64:0/1238374006 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08fc0ce470 con 0x7f08fc0b9a10 2024-03-04T07:33:31.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.915+0000 7f08fb7fe640 1 --2- 172.21.15.64:0/1238374006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b9a10 0x7f08fc0b5c40 secure :-1 s=READY pgs=1512 cs=0 l=1 rev1=1 crypto rx=0x7f08fc0d1760 tx=0x7f08f402bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=93a91b1d37c45209 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f08fa7fc640 1 -- 172.21.15.64:0/1238374006 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08f402cb20 con 0x7f08fc0b9a10 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f08fa7fc640 1 -- 172.21.15.64:0/1238374006 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08f402ccc0 con 0x7f08fc0b9a10 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f08fa7fc640 1 -- 172.21.15.64:0/1238374006 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08f4034610 con 0x7f08fc0b9a10 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 -- 172.21.15.64:0/1238374006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b9a10 msgr2=0x7f08fc0b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 --2- 172.21.15.64:0/1238374006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b9a10 0x7f08fc0b5c40 secure :-1 s=READY pgs=1512 cs=0 l=1 rev1=1 crypto rx=0x7f08fc0d1760 tx=0x7f08f402bf80 comp rx=0 tx=0).stop 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 -- 172.21.15.64:0/1238374006 shutdown_connections 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 --2- 172.21.15.64:0/1238374006 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b9a10 0x7f08fc0b5c40 unknown :-1 s=CLOSED pgs=1512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 --2- 172.21.15.64:0/1238374006 >> v2:172.21.15.66:3300/0 conn(0x7f08fc0b95a0 0x7f08fc0b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 -- 172.21.15.64:0/1238374006 >> 172.21.15.64:0/1238374006 conn(0x7f08fc071960 msgr2=0x7f08fc071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 -- 172.21.15.64:0/1238374006 shutdown_connections 2024-03-04T07:33:31.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.916+0000 7f090190c640 1 -- 172.21.15.64:0/1238374006 wait complete. 2024-03-04T07:33:31.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.917+0000 7f090190c640 1 Processor -- start 2024-03-04T07:33:31.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f090190c640 1 -- start start 2024-03-04T07:33:31.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f090190c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b95a0 0x7f08fc0cd5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f090190c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08fc0b9a10 0x7f08fc15f180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f090190c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08fc15df00 0x7f08fc1e8aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f090190c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08fc0cedf0 con 0x7f08fc0b95a0 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f090190c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08fc0cec70 con 0x7f08fc15df00 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f090190c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f08fc0d3220 con 0x7f08fc0b9a10 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f08faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b95a0 0x7f08fc0cd5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f08faffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b95a0 0x7f08fc0cd5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51582/0 (socket says 172.21.15.64:51582) 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f08faffd640 1 -- 172.21.15.64:0/479124013 learned_addr learned my addr 172.21.15.64:0/479124013 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f08fb7fe640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08fc15df00 0x7f08fc1e8aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.918+0000 7f08f27fc640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08fc0b9a10 0x7f08fc15f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08faffd640 1 -- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08fc0b9a10 msgr2=0x7f08fc15f180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08faffd640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08fc0b9a10 0x7f08fc15f180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08faffd640 1 -- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08fc15df00 msgr2=0x7f08fc1e8aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08faffd640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08fc15df00 0x7f08fc1e8aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:31.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08faffd640 1 -- 172.21.15.64:0/479124013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08fc160c70 con 0x7f08fc0b95a0 2024-03-04T07:33:31.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08faffd640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b95a0 0x7f08fc0cd5a0 secure :-1 s=READY pgs=1513 cs=0 l=1 rev1=1 crypto rx=0x7f08ec00c3c0 tx=0x7f08ec00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:31.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08f8ff9640 1 -- 172.21.15.64:0/479124013 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08ec0074e0 con 0x7f08fc0b95a0 2024-03-04T07:33:31.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08f8ff9640 1 -- 172.21.15.64:0/479124013 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08ec007680 con 0x7f08fc0b95a0 2024-03-04T07:33:31.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08fc15fbd0 con 0x7f08fc0b95a0 2024-03-04T07:33:31.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f08f8ff9640 1 -- 172.21.15.64:0/479124013 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08ec004a00 con 0x7f08fc0b95a0 2024-03-04T07:33:31.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.919+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08fc160190 con 0x7f08fc0b95a0 2024-03-04T07:33:31.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.920+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f08fc0b6d70 con 0x7f08fc0b95a0 2024-03-04T07:33:31.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.920+0000 7f08f8ff9640 1 -- 172.21.15.64:0/479124013 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f08ec01a020 con 0x7f08fc0b95a0 2024-03-04T07:33:31.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.921+0000 7f08f8ff9640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08dc06b500 0x7f08dc06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:31.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.921+0000 7f08f8ff9640 1 -- 172.21.15.64:0/479124013 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 5115+0+0 (secure 0 0 0) 0x7f08ec089580 con 0x7f08fc0b95a0 2024-03-04T07:33:31.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.921+0000 7f08f27fc640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08dc06b500 0x7f08dc06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:31.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.921+0000 7f08f27fc640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08dc06b500 0x7f08dc06d9c0 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f08fc15b6d0 tx=0x7f08e4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:31.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:31.925+0000 7f08f8ff9640 1 -- 172.21.15.64:0/479124013 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f08fc0b6d70 con 0x7f08fc0b95a0 2024-03-04T07:33:32.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:32.120+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist clear"} v 0) v1 -- 0x7f08fc162810 con 0x7f08fc0b95a0 2024-03-04T07:33:33.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.210+0000 7f08f8ff9640 1 -- 172.21.15.64:0/479124013 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist clear"}]=0 removed all blocklist entries v119) v1 ==== 97+0+0 (secure 0 0 0) 0x7f08fc162810 con 0x7f08fc0b95a0 2024-03-04T07:33:33.211 INFO:teuthology.orchestra.run.smithi064.stderr: removed all blocklist entries 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08dc06b500 msgr2=0x7f08dc06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08dc06b500 0x7f08dc06d9c0 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f08fc15b6d0 tx=0x7f08e4009190 comp rx=0 tx=0).stop 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b95a0 msgr2=0x7f08fc0cd5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b95a0 0x7f08fc0cd5a0 secure :-1 s=READY pgs=1513 cs=0 l=1 rev1=1 crypto rx=0x7f08ec00c3c0 tx=0x7f08ec00c880 comp rx=0 tx=0).stop 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 shutdown_connections 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08dc06b500 0x7f08dc06d9c0 unknown :-1 s=CLOSED pgs=673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08fc15df00 0x7f08fc1e8aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08fc0b9a10 0x7f08fc15f180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 --2- 172.21.15.64:0/479124013 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08fc0b95a0 0x7f08fc0cd5a0 unknown :-1 s=CLOSED pgs=1513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 >> 172.21.15.64:0/479124013 conn(0x7f08fc071960 msgr2=0x7f08fc15d1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 shutdown_connections 2024-03-04T07:33:33.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.214+0000 7f090190c640 1 -- 172.21.15.64:0/479124013 wait complete. 2024-03-04T07:33:33.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth ls --format=json-pretty 2024-03-04T07:33:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.334+0000 7f712050c640 1 Processor -- start 2024-03-04T07:33:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.334+0000 7f712050c640 1 -- start start 2024-03-04T07:33:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f712050c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f71180b2c50 0x7f71180d2ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f712050c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71180ce970 con 0x7f711815e000 2024-03-04T07:33:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f712050c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f71180cef30 con 0x7f7118071c50 2024-03-04T07:33:33.335 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f712050c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f71180cece0 con 0x7f71180b2c50 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f711eaa9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f71180b2c50 0x7f71180d2ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f711e2a8640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f711815e000 0x7f7118071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51500/0 (socket says 172.21.15.64:51500) 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f711eaa9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f71180b2c50 0x7f71180d2ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51590/0 (socket says 172.21.15.64:51590) 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.335+0000 7f711e2a8640 1 -- 172.21.15.64:0/1524126400 learned_addr learned my addr 172.21.15.64:0/1524126400 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.336+0000 7f711eaa9640 1 -- 172.21.15.64:0/1524126400 >> v1:172.21.15.66:6789/0 conn(0x7f7118071c50 legacy=0x7f71180d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.336+0000 7f711eaa9640 1 -- 172.21.15.64:0/1524126400 >> v1:172.21.15.64:6789/0 conn(0x7f711815e000 legacy=0x7f7118071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.336+0000 7f711d2a6640 1 -- 172.21.15.64:0/1524126400 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1644910797 0 0) 0x7f71180ce970 con 0x7f711815e000 2024-03-04T07:33:33.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.336+0000 7f711eaa9640 1 -- 172.21.15.64:0/1524126400 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71180cf210 con 0x7f71180b2c50 2024-03-04T07:33:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.336+0000 7f711eaa9640 1 --2- 172.21.15.64:0/1524126400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71180b2c50 0x7f71180d2ce0 secure :-1 s=READY pgs=1515 cs=0 l=1 rev1=1 crypto rx=0x7f71100097a0 tx=0x7f711002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be85f88da0ed53ab server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f711d2a6640 1 -- 172.21.15.64:0/1524126400 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7110035020 con 0x7f71180b2c50 2024-03-04T07:33:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f711d2a6640 1 -- 172.21.15.64:0/1524126400 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7110031830 con 0x7f71180b2c50 2024-03-04T07:33:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f711d2a6640 1 -- 172.21.15.64:0/1524126400 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7110031c00 con 0x7f71180b2c50 2024-03-04T07:33:33.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f712050c640 1 -- 172.21.15.64:0/1524126400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71180b2c50 msgr2=0x7f71180d2ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f712050c640 1 --2- 172.21.15.64:0/1524126400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71180b2c50 0x7f71180d2ce0 secure :-1 s=READY pgs=1515 cs=0 l=1 rev1=1 crypto rx=0x7f71100097a0 tx=0x7f711002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f712050c640 1 -- 172.21.15.64:0/1524126400 shutdown_connections 2024-03-04T07:33:33.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f712050c640 1 --2- 172.21.15.64:0/1524126400 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f71180b2c50 0x7f71180d2ce0 unknown :-1 s=CLOSED pgs=1515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f712050c640 1 -- 172.21.15.64:0/1524126400 >> 172.21.15.64:0/1524126400 conn(0x7f71180d3c80 msgr2=0x7f71180d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:33.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f712050c640 1 -- 172.21.15.64:0/1524126400 shutdown_connections 2024-03-04T07:33:33.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.337+0000 7f712050c640 1 -- 172.21.15.64:0/1524126400 wait complete. 2024-03-04T07:33:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.338+0000 7f712050c640 1 Processor -- start 2024-03-04T07:33:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.338+0000 7f712050c640 1 -- start start 2024-03-04T07:33:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f712050c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7118071c50 0x7f71180aa680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f712050c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71180b2c50 0x7f7118150170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f712050c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f711815e000 0x7f71180ccc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f712050c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71180b3320 con 0x7f7118071c50 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f712050c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71180b31a0 con 0x7f71180b2c50 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f712050c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f711815e3e0 con 0x7f711815e000 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f711e2a8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7118071c50 0x7f71180aa680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f711e2a8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7118071c50 0x7f71180aa680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51600/0 (socket says 172.21.15.64:51600) 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f711e2a8640 1 -- 172.21.15.64:0/4256104011 learned_addr learned my addr 172.21.15.64:0/4256104011 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f711daa7640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71180b2c50 0x7f7118150170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.339+0000 7f711eaa9640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f711815e000 0x7f71180ccc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f711e2a8640 1 -- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f711815e000 msgr2=0x7f71180ccc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f711e2a8640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f711815e000 0x7f71180ccc10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f711e2a8640 1 -- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71180b2c50 msgr2=0x7f7118150170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f711e2a8640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71180b2c50 0x7f7118150170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f711e2a8640 1 -- 172.21.15.64:0/4256104011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71180cbe90 con 0x7f7118071c50 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f711daa7640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71180b2c50 0x7f7118150170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:33.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f711e2a8640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7118071c50 0x7f71180aa680 secure :-1 s=READY pgs=1516 cs=0 l=1 rev1=1 crypto rx=0x7f710800c8f0 tx=0x7f710800cdb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:33.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f7116ffd640 1 -- 172.21.15.64:0/4256104011 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7108018070 con 0x7f7118071c50 2024-03-04T07:33:33.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f7116ffd640 1 -- 172.21.15.64:0/4256104011 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f710800a640 con 0x7f7118071c50 2024-03-04T07:33:33.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f71180cc130 con 0x7f7118071c50 2024-03-04T07:33:33.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f7116ffd640 1 -- 172.21.15.64:0/4256104011 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f710800aa30 con 0x7f7118071c50 2024-03-04T07:33:33.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.340+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f71180aac60 con 0x7f7118071c50 2024-03-04T07:33:33.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.341+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7118067ff0 con 0x7f7118071c50 2024-03-04T07:33:33.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.341+0000 7f7116ffd640 1 -- 172.21.15.64:0/4256104011 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f710801b070 con 0x7f7118071c50 2024-03-04T07:33:33.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.342+0000 7f7116ffd640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f710406b500 0x7f710406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.342+0000 7f7116ffd640 1 -- 172.21.15.64:0/4256104011 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f7108089410 con 0x7f7118071c50 2024-03-04T07:33:33.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.342+0000 7f711daa7640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f710406b500 0x7f710406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.342+0000 7f711daa7640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f710406b500 0x7f710406d9c0 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7f710c007fd0 tx=0x7f710c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:33.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.346+0000 7f7116ffd640 1 -- 172.21.15.64:0/4256104011 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7108058670 con 0x7f7118071c50 2024-03-04T07:33:33.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.599+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth ls", "format": "json-pretty"} v 0) v1 -- 0x7f7118161400 con 0x7f7118071c50 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.599+0000 7f7116ffd640 1 -- 172.21.15.64:0/4256104011 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth ls", "format": "json-pretty"}]=0 v21) v1 ==== 80+0+5304 (secure 0 0 0) 0x7f7108057f70 con 0x7f7118071c50 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_dump": [ 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.a", 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vl7cUBKhAA9PgOhE13erbyX+ySTVcWPQ==", 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.599 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.b", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlYPnXNhAADbIO8yKOj0FDDi4B6J11dA==", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.c", 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlm9BaMRAA6C5Dm303Q/QbBcv8GVt4lQ==", 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mds.d", 2024-03-04T07:33:33.601 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlVzlkAxAA81N6gg+8jkDnbIEXdh9FoQ==", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.0", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCBd+VlbG9WIxAAKkl5KR3jnIbEO/Sz67NE1A==", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.602 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.1", 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCEd+VloYR8IRAALF9ddA84oLU0IstqEsP1IA==", 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.2", 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCHd+VlV77mHxAA7AJwVv1U0vzceAr4QmM0Dw==", 2024-03-04T07:33:33.603 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.3", 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCKd+VlmlIiHhAAckskCgQ6GPPHtVoKCreg0A==", 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.604 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.4", 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCQd+Vl20N0KhAAbOXlvIx8spsimrmd/ytsmg==", 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.5", 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCTd+Vl2F+0KRAAVrtAGjxRr9UkHdR2chgeQg==", 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.605 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.6", 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCWd+Vl/x+oKBAAflXbQdUuNZec7uj9gmiNPA==", 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.606 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "osd.7", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQCZd+VlGeWMKBAAKpfslP6/RXZFiKd0vdjEWg==", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow profile osd", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile osd", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.0", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA==", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow r", 2024-03-04T07:33:33.607 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rw tag cephfs data=cephfs" 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.1", 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+VlI9FfERAAfmGQ5rWQ3IyoHpsQRjSS+Q==", 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.2", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vlt0ivDRAAZitiAHjj0C4N7RqeNi1B7Q==", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.3", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB+d+Vl3ihEFRAAxs1aAshf0ottCOrXAQeD3A==", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow", 2024-03-04T07:33:33.609 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow r", 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow rw", 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow rwx" 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "client.admin", 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlLh/iChAA26wa+BrqBIFu6zUVXX7n+A==", 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": "allow *", 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow *", 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.x", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+VlK+uKJhAAo6F84F3bycxbHl1gpSWI2Q==", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "entity": "mgr.y", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "AQB9d+Vlhf52IBAA+8R/gU18DLVHovuGTnLing==", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "caps": { 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": "allow *", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": "allow profile mgr", 2024-03-04T07:33:33.611 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": "allow *" 2024-03-04T07:33:33.612 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.612 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:33.612 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-03-04T07:33:33.612 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:33:33.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f710406b500 msgr2=0x7f710406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f710406b500 0x7f710406d9c0 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7f710c007fd0 tx=0x7f710c009040 comp rx=0 tx=0).stop 2024-03-04T07:33:33.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7118071c50 msgr2=0x7f71180aa680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7118071c50 0x7f71180aa680 secure :-1 s=READY pgs=1516 cs=0 l=1 rev1=1 crypto rx=0x7f710800c8f0 tx=0x7f710800cdb0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 shutdown_connections 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f710406b500 0x7f710406d9c0 unknown :-1 s=CLOSED pgs=678 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f711815e000 0x7f71180ccc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f71180b2c50 0x7f7118150170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 --2- 172.21.15.64:0/4256104011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7118071c50 0x7f71180aa680 unknown :-1 s=CLOSED pgs=1516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 >> 172.21.15.64:0/4256104011 conn(0x7f71180d3c80 msgr2=0x7f71180cf750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 shutdown_connections 2024-03-04T07:33:33.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.601+0000 7f712050c640 1 -- 172.21.15.64:0/4256104011 wait complete. 2024-03-04T07:33:33.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:33.715 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.716+0000 7f7ec07e2640 1 Processor -- start 2024-03-04T07:33:33.716 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.716+0000 7f7ec07e2640 1 -- start start 2024-03-04T07:33:33.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ec07e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 0x7f7eb80cea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ec07e2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7eb80d0880 0x7f7eb80ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ec07e2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7eb80d0dd0 con 0x7f7eb815e060 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ec07e2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7eb80d10b0 con 0x7f7eb80d0880 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ec07e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb80cc0e0 con 0x7f7eb806fe20 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebe57e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7eb815e060 0x7f7eb80ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54676/0 (socket says 172.21.15.64:54676) 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebe57e640 1 -- 172.21.15.64:0/4036275762 learned_addr learned my addr 172.21.15.64:0/4036275762 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebed7f640 1 --2- 172.21.15.64:0/4036275762 >> v2:172.21.15.66:3300/0 conn(0x7f7eb80d0880 0x7f7eb80ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2924229798 0 0) 0x7f7eb80d0dd0 con 0x7f7eb815e060 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7ea4003520 con 0x7f7eb815e060 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 0x7f7eb80cea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2690772301 0 0) 0x7f7ea4003520 con 0x7f7eb815e060 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 msgr2=0x7f7eb80cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 0x7f7eb80cea20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 >> v2:172.21.15.66:3300/0 conn(0x7f7eb80d0880 msgr2=0x7f7eb80ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 --2- 172.21.15.64:0/4036275762 >> v2:172.21.15.66:3300/0 conn(0x7f7eb80d0880 0x7f7eb80ccbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7eb80cc3c0 con 0x7f7eb815e060 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f7eb4001670 con 0x7f7eb815e060 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.717+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 >> v1:172.21.15.64:6789/0 conn(0x7f7eb815e060 legacy=0x7f7eb80ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebd57c640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 0x7f7ea40039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebd57c640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea4003f30 0x7f7ea4004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebd57c640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea4004d40 0x7f7ea40051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb80d10b0 con 0x7f7ea4003f30 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb80cc0e0 con 0x7f7ea4004d40 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ea400ba10 con 0x7f7eb806fe20 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea4003f30 0x7f7ea4004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebe57e640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 0x7f7ea40039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebed7f640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea4004d40 0x7f7ea40051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebdd7d640 1 -- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 msgr2=0x7f7ea40039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7eb806fe20 0x7f7ea40039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebdd7d640 1 -- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea4004d40 msgr2=0x7f7ea40051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ea4004d40 0x7f7ea40051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.718+0000 7f7ebdd7d640 1 -- 172.21.15.64:0/4036275762 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ea4003520 con 0x7f7ea4003f30 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea4003f30 0x7f7ea4004800 secure :-1 s=READY pgs=1518 cs=0 l=1 rev1=1 crypto rx=0x7f7ea800ba00 tx=0x7f7ea4005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a0b93a7dbe7bd2d4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ea8036020 con 0x7f7ea4003f30 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ea8032620 con 0x7f7ea4003f30 2024-03-04T07:33:33.721 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ebd57c640 1 -- 172.21.15.64:0/4036275762 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ea800b600 con 0x7f7ea4003f30 2024-03-04T07:33:33.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ec07e2640 1 -- 172.21.15.64:0/4036275762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea4003f30 msgr2=0x7f7ea4004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ec07e2640 1 --2- 172.21.15.64:0/4036275762 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ea4003f30 0x7f7ea4004800 secure :-1 s=READY pgs=1518 cs=0 l=1 rev1=1 crypto rx=0x7f7ea800ba00 tx=0x7f7ea4005830 comp rx=0 tx=0).stop 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ebed7f640 1 -- 172.21.15.64:0/4036275762 reap_dead start 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ec07e2640 1 -- 172.21.15.64:0/4036275762 shutdown_connections 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ec07e2640 1 -- 172.21.15.64:0/4036275762 >> 172.21.15.64:0/4036275762 conn(0x7f7eb80d1a90 msgr2=0x7f7eb80d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.719+0000 7f7ec07e2640 1 -- 172.21.15.64:0/4036275762 shutdown_connections 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.720+0000 7f7ec07e2640 1 -- 172.21.15.64:0/4036275762 wait complete. 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 Processor -- start 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 -- start start 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea4003f30 0x7f7eb81f3110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7eb815f020 0x7f7eb806c030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7eb806c570 0x7f7eb80af890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb80cea20 con 0x7f7eb815f020 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb815e440 con 0x7f7eb806c570 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ec07e2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb80ceba0 con 0x7f7ea4003f30 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ebdd7d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7eb815f020 0x7f7eb806c030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.721+0000 7f7ebdd7d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7eb815f020 0x7f7eb806c030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60504/0 (socket says 172.21.15.64:60504) 2024-03-04T07:33:33.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebdd7d640 1 -- 172.21.15.64:0/1125986019 learned_addr learned my addr 172.21.15.64:0/1125986019 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebed7f640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7eb806c570 0x7f7eb80af890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebe57e640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea4003f30 0x7f7eb81f3110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebdd7d640 1 -- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea4003f30 msgr2=0x7f7eb81f3110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea4003f30 0x7f7eb81f3110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebdd7d640 1 -- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7eb806c570 msgr2=0x7f7eb80af890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7eb806c570 0x7f7eb80af890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebdd7d640 1 -- 172.21.15.64:0/1125986019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7eb81f3650 con 0x7f7eb815f020 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebed7f640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7eb806c570 0x7f7eb80af890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ebdd7d640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7eb815f020 0x7f7eb806c030 secure :-1 s=READY pgs=1519 cs=0 l=1 rev1=1 crypto rx=0x7f7ea802bf70 tx=0x7f7ea8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7eaeffd640 1 -- 172.21.15.64:0/1125986019 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ea8041070 con 0x7f7eb815f020 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7eaeffd640 1 -- 172.21.15.64:0/1125986019 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ea8032e10 con 0x7f7eb815f020 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7eaeffd640 1 -- 172.21.15.64:0/1125986019 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ea8036040 con 0x7f7eb815f020 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7eb81f7010 con 0x7f7eb815f020 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.722+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7eb81f75d0 con 0x7f7eb815f020 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.723+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7eb8067ff0 con 0x7f7eb815f020 2024-03-04T07:33:33.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.724+0000 7f7eaeffd640 1 -- 172.21.15.64:0/1125986019 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7ea8039020 con 0x7f7eb815f020 2024-03-04T07:33:33.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.724+0000 7f7eaeffd640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e9c06b500 0x7f7e9c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:33.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.724+0000 7f7eaeffd640 1 -- 172.21.15.64:0/1125986019 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f7ea80ab960 con 0x7f7eb815f020 2024-03-04T07:33:33.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.724+0000 7f7ebe57e640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e9c06b500 0x7f7e9c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:33.727 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.724+0000 7f7ebe57e640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e9c06b500 0x7f7e9c06d9c0 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f7eb40014f0 tx=0x7f7eb4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:33.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.728+0000 7f7eaeffd640 1 -- 172.21.15.64:0/1125986019 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7ea807aaa0 con 0x7f7eb815f020 2024-03-04T07:33:33.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.909+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7eb80cf060 con 0x7f7eb815f020 2024-03-04T07:33:33.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.910+0000 7f7eaeffd640 1 -- 172.21.15.64:0/1125986019 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v119) v1 ==== 74+0+12853 (secure 0 0 0) 0x7f7ea807a3a0 con 0x7f7eb815f020 2024-03-04T07:33:33.910 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:33.910 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":119,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:33:33.209750+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":17,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.912+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e9c06b500 msgr2=0x7f7e9c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.912+0000 7f7ec07e2640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e9c06b500 0x7f7e9c06d9c0 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f7eb40014f0 tx=0x7f7eb4009040 comp rx=0 tx=0).stop 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.912+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7eb815f020 msgr2=0x7f7eb806c030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.912+0000 7f7ec07e2640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7eb815f020 0x7f7eb806c030 secure :-1 s=READY pgs=1519 cs=0 l=1 rev1=1 crypto rx=0x7f7ea802bf70 tx=0x7f7ea8004730 comp rx=0 tx=0).stop 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 shutdown_connections 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e9c06b500 0x7f7e9c06d9c0 unknown :-1 s=CLOSED pgs=679 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7eb806c570 0x7f7eb80af890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7eb815f020 0x7f7eb806c030 unknown :-1 s=CLOSED pgs=1519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 --2- 172.21.15.64:0/1125986019 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ea4003f30 0x7f7eb81f3110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 >> 172.21.15.64:0/1125986019 conn(0x7f7eb80d1a90 msgr2=0x7f7eb815d100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 shutdown_connections 2024-03-04T07:33:33.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:33.913+0000 7f7ec07e2640 1 -- 172.21.15.64:0/1125986019 wait complete. 2024-03-04T07:33:33.923 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:33:33.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:33:34.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.028+0000 7f36a693a640 1 Processor -- start 2024-03-04T07:33:34.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.028+0000 7f36a693a640 1 -- start start 2024-03-04T07:33:34.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f36a693a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 0x7f36a00cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f36a693a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f36a00cd720 0x7f36a00d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f36a693a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36a00b33b0 con 0x7f36a00cff40 2024-03-04T07:33:34.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f36a693a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f36a00b0af0 con 0x7f36a00cd720 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f36a693a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36a00cdb30 con 0x7f36a015f5c0 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369f7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f36a00cff40 0x7f36a00b1470 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54690/0 (socket says 172.21.15.64:54690) 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369f7fe640 1 -- 172.21.15.64:0/1482208134 learned_addr learned my addr 172.21.15.64:0/1482208134 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369ffff640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 0x7f36a00cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f36a4ed7640 1 --2- 172.21.15.64:0/1482208134 >> v2:172.21.15.66:3300/0 conn(0x7f36a00cd720 0x7f36a00d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1898593288 0 0) 0x7f36a00b33b0 con 0x7f36a00cff40 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3688003520 con 0x7f36a00cff40 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2597525858 0 0) 0x7f3688003520 con 0x7f36a00cff40 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 msgr2=0x7f36a00cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 0x7f36a00cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 >> v2:172.21.15.66:3300/0 conn(0x7f36a00cd720 msgr2=0x7f36a00d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:34.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 --2- 172.21.15.64:0/1482208134 >> v2:172.21.15.66:3300/0 conn(0x7f36a00cd720 0x7f36a00d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36a00b0d40 con 0x7f36a00cff40 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f3690001690 con 0x7f36a00cff40 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 >> v1:172.21.15.64:6789/0 conn(0x7f36a00cff40 legacy=0x7f36a00b1470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.029+0000 7f369effd640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 0x7f36880039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369effd640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3688003f30 0x7f3688004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369effd640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688004d40 0x7f36880051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36a00b0af0 con 0x7f3688003f30 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36a00cdb30 con 0x7f3688004d40 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f368800ba10 con 0x7f36a015f5c0 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369f7fe640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3688003f30 0x7f3688004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f36a4ed7640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688004d40 0x7f36880051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369ffff640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 0x7f36880039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369f7fe640 1 -- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 msgr2=0x7f36880039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369f7fe640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a015f5c0 0x7f36880039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369f7fe640 1 -- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688004d40 msgr2=0x7f36880051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369f7fe640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688004d40 0x7f36880051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.030+0000 7f369f7fe640 1 -- 172.21.15.64:0/1482208134 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3688003520 con 0x7f3688003f30 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f369f7fe640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3688003f30 0x7f3688004800 secure :-1 s=READY pgs=1521 cs=0 l=1 rev1=1 crypto rx=0x7f369000c2f0 tx=0x7f3688005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59849516b89f4012 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3690007d90 con 0x7f3688003f30 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3690030070 con 0x7f3688003f30 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f369effd640 1 -- 172.21.15.64:0/1482208134 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f369002cd40 con 0x7f3688003f30 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f36a693a640 1 -- 172.21.15.64:0/1482208134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3688003f30 msgr2=0x7f3688004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f36a693a640 1 --2- 172.21.15.64:0/1482208134 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3688003f30 0x7f3688004800 secure :-1 s=READY pgs=1521 cs=0 l=1 rev1=1 crypto rx=0x7f369000c2f0 tx=0x7f3688005830 comp rx=0 tx=0).stop 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f36a4ed7640 1 -- 172.21.15.64:0/1482208134 reap_dead start 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f36a693a640 1 -- 172.21.15.64:0/1482208134 shutdown_connections 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f36a693a640 1 -- 172.21.15.64:0/1482208134 >> 172.21.15.64:0/1482208134 conn(0x7f36a00b36d0 msgr2=0x7f36a00b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f36a693a640 1 -- 172.21.15.64:0/1482208134 shutdown_connections 2024-03-04T07:33:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.031+0000 7f36a693a640 1 -- 172.21.15.64:0/1482208134 wait complete. 2024-03-04T07:33:34.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.032+0000 7f36a693a640 1 Processor -- start 2024-03-04T07:33:34.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f36a693a640 1 -- start start 2024-03-04T07:33:34.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f36a693a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688003f30 0x7f36a014fd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f36a693a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36a009f660 0x7f36a009dc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f36a693a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a0158df0 0x7f36a009bac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f36a693a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36a00ced80 con 0x7f36a009f660 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f36a693a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36a00ceaa0 con 0x7f3688003f30 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f36a693a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36a015f9a0 con 0x7f36a0158df0 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.033+0000 7f369f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36a009f660 0x7f36a009dc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36a009f660 0x7f36a009dc70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60518/0 (socket says 172.21.15.64:60518) 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 -- 172.21.15.64:0/2924726131 learned_addr learned my addr 172.21.15.64:0/2924726131 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369ffff640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688003f30 0x7f36a014fd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f36a4ed7640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a0158df0 0x7f36a009bac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 -- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a0158df0 msgr2=0x7f36a009bac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a0158df0 0x7f36a009bac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 -- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688003f30 msgr2=0x7f36a014fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688003f30 0x7f36a014fd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 -- 172.21.15.64:0/2924726131 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36a009e5c0 con 0x7f36a009f660 2024-03-04T07:33:34.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f36a4ed7640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a0158df0 0x7f36a009bac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369f7fe640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36a009f660 0x7f36a009dc70 secure :-1 s=READY pgs=1522 cs=0 l=1 rev1=1 crypto rx=0x7f369002c570 tx=0x7f3690001a60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369cff9640 1 -- 172.21.15.64:0/2924726131 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3690045070 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369cff9640 1 -- 172.21.15.64:0/2924726131 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3690004bf0 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36a009e8a0 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f369cff9640 1 -- 172.21.15.64:0/2924726131 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3690007d70 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.034+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f36a009ee60 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.035+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f36a009fd30 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.036+0000 7f369cff9640 1 -- 172.21.15.64:0/2924726131 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3690007070 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.036+0000 7f369cff9640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f368c06b500 0x7f368c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.036+0000 7f369cff9640 1 -- 172.21.15.64:0/2924726131 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 4857+0+0 (secure 0 0 0) 0x7f36900aae30 con 0x7f36a009f660 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.036+0000 7f369ffff640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f368c06b500 0x7f368c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.036+0000 7f369ffff640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f368c06b500 0x7f368c06d9c0 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f3694009ae0 tx=0x7f3694009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.040+0000 7f369cff9640 1 -- 172.21.15.64:0/2924726131 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f369007a050 con 0x7f36a009f660 2024-03-04T07:33:34.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.221+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f36a004f410 con 0x7f36a009f660 2024-03-04T07:33:34.279 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.221+0000 7f369cff9640 1 -- 172.21.15.64:0/2924726131 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v119) v1 ==== 93+0+10 (secure 0 0 0) 0x7f3690079950 con 0x7f36a009f660 2024-03-04T07:33:34.280 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:33:34.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f368c06b500 msgr2=0x7f368c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f368c06b500 0x7f368c06d9c0 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f3694009ae0 tx=0x7f3694009190 comp rx=0 tx=0).stop 2024-03-04T07:33:34.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36a009f660 msgr2=0x7f36a009dc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36a009f660 0x7f36a009dc70 secure :-1 s=READY pgs=1522 cs=0 l=1 rev1=1 crypto rx=0x7f369002c570 tx=0x7f3690001a60 comp rx=0 tx=0).stop 2024-03-04T07:33:34.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 shutdown_connections 2024-03-04T07:33:34.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f368c06b500 0x7f368c06d9c0 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36a0158df0 0x7f36a009bac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36a009f660 0x7f36a009dc70 unknown :-1 s=CLOSED pgs=1522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.223+0000 7f36a693a640 1 --2- 172.21.15.64:0/2924726131 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3688003f30 0x7f36a014fd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.224+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 >> 172.21.15.64:0/2924726131 conn(0x7f36a00b36d0 msgr2=0x7f36a00b1470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:34.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.224+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 shutdown_connections 2024-03-04T07:33:34.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.224+0000 7f36a693a640 1 -- 172.21.15.64:0/2924726131 wait complete. 2024-03-04T07:33:34.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-04T07:33:34.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5ca5ae640 1 Processor -- start 2024-03-04T07:33:34.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5ca5ae640 1 -- start start 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5ca5ae640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb5c40d02e0 0x7fb5c40cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5ca5ae640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5c40b2320 con 0x7fb5c40d06c0 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5ca5ae640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb5c40b0dd0 con 0x7fb5c40b4260 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5ca5ae640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb5c40b0b80 con 0x7fb5c40d02e0 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5c8b4b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb5c40d06c0 0x7fb5c40b1c10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54706/0 (socket says 172.21.15.64:54706) 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.385+0000 7fb5c8b4b640 1 -- 172.21.15.64:0/993863890 learned_addr learned my addr 172.21.15.64:0/993863890 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.386+0000 7fb5c3fff640 1 --2- 172.21.15.64:0/993863890 >> v2:172.21.15.66:3300/0 conn(0x7fb5c40d02e0 0x7fb5c40cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.386+0000 7fb5c2ffd640 1 -- 172.21.15.64:0/993863890 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3513621775 0 0) 0x7fb5c40b2320 con 0x7fb5c40d06c0 2024-03-04T07:33:34.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.386+0000 7fb5c2ffd640 1 -- 172.21.15.64:0/993863890 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb5b0003520 con 0x7fb5c40d06c0 2024-03-04T07:33:34.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.386+0000 7fb5c3fff640 1 -- 172.21.15.64:0/993863890 >> v1:172.21.15.66:6789/0 conn(0x7fb5c40b4260 legacy=0x7fb5c40b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.386+0000 7fb5c2ffd640 1 -- 172.21.15.64:0/993863890 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2380128458 0 0) 0x7fb5c40b0dd0 con 0x7fb5c40b4260 2024-03-04T07:33:34.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.386+0000 7fb5c3fff640 1 -- 172.21.15.64:0/993863890 >> v1:172.21.15.64:6789/0 conn(0x7fb5c40d06c0 legacy=0x7fb5c40b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.386+0000 7fb5c3fff640 1 -- 172.21.15.64:0/993863890 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5c40b10b0 con 0x7fb5c40d02e0 2024-03-04T07:33:34.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.387+0000 7fb5c3fff640 1 --2- 172.21.15.64:0/993863890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40d02e0 0x7fb5c40cfa00 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7fb5b40097a0 tx=0x7fb5b402b9f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f8e0826e53a810dd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.387+0000 7fb5c2ffd640 1 -- 172.21.15.64:0/993863890 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5b402f810 con 0x7fb5c40d02e0 2024-03-04T07:33:34.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.387+0000 7fb5c2ffd640 1 -- 172.21.15.64:0/993863890 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5b402f9b0 con 0x7fb5c40d02e0 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.387+0000 7fb5c2ffd640 1 -- 172.21.15.64:0/993863890 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5b402fd80 con 0x7fb5c40d02e0 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.387+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/993863890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40d02e0 msgr2=0x7fb5c40cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.387+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/993863890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40d02e0 0x7fb5c40cfa00 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7fb5b40097a0 tx=0x7fb5b402b9f0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.388+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/993863890 shutdown_connections 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.388+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/993863890 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40d02e0 0x7fb5c40cfa00 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.388+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/993863890 >> 172.21.15.64:0/993863890 conn(0x7fb5c40b36d0 msgr2=0x7fb5c40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.388+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/993863890 shutdown_connections 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.388+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/993863890 wait complete. 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.389+0000 7fb5ca5ae640 1 Processor -- start 2024-03-04T07:33:34.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.389+0000 7fb5ca5ae640 1 -- start start 2024-03-04T07:33:34.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40b4260 0x7fb5c40cb680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5c40d02e0 0x7fb5c415ce40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5c40d06c0 0x7fb5c40ca0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5c40b0360 con 0x7fb5c40d06c0 2024-03-04T07:33:34.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5c40aeec0 con 0x7fb5c40b4260 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb5c40b04e0 con 0x7fb5c40d02e0 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5c40d06c0 0x7fb5c40ca0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5c40d06c0 0x7fb5c40ca0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60530/0 (socket says 172.21.15.64:60530) 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 -- 172.21.15.64:0/2240754127 learned_addr learned my addr 172.21.15.64:0/2240754127 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c3fff640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40b4260 0x7fb5c40cb680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c37fe640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5c40d02e0 0x7fb5c415ce40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 -- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5c40d02e0 msgr2=0x7fb5c415ce40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5c40d02e0 0x7fb5c415ce40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 -- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40b4260 msgr2=0x7fb5c40cb680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40b4260 0x7fb5c40cb680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 -- 172.21.15.64:0/2240754127 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5c40c9020 con 0x7fb5c40d06c0 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c37fe640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5c40d02e0 0x7fb5c415ce40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:34.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c8b4b640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5c40d06c0 0x7fb5c40ca0c0 secure :-1 s=READY pgs=1524 cs=0 l=1 rev1=1 crypto rx=0x7fb5bc00cb40 tx=0x7fb5bc00a060 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c0ff9640 1 -- 172.21.15.64:0/2240754127 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5bc017070 con 0x7fb5c40d06c0 2024-03-04T07:33:34.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c0ff9640 1 -- 172.21.15.64:0/2240754127 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb5bc00aca0 con 0x7fb5c40d06c0 2024-03-04T07:33:34.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5c40c9300 con 0x7fb5c40d06c0 2024-03-04T07:33:34.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5c0ff9640 1 -- 172.21.15.64:0/2240754127 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5bc012620 con 0x7fb5c40d06c0 2024-03-04T07:33:34.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.390+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5c40c98c0 con 0x7fb5c40d06c0 2024-03-04T07:33:34.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.391+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5c40d10e0 con 0x7fb5c40d06c0 2024-03-04T07:33:34.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.392+0000 7fb5c0ff9640 1 -- 172.21.15.64:0/2240754127 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb5bc01a070 con 0x7fb5c40d06c0 2024-03-04T07:33:34.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.392+0000 7fb5c0ff9640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5a406b500 0x7fb5a406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.392+0000 7fb5c0ff9640 1 -- 172.21.15.64:0/2240754127 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 4857+0+0 (secure 0 0 0) 0x7fb5bc089ce0 con 0x7fb5c40d06c0 2024-03-04T07:33:34.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.392+0000 7fb5c3fff640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5a406b500 0x7fb5a406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.394 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.393+0000 7fb5c3fff640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5a406b500 0x7fb5a406d9c0 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7fb5b4036000 tx=0x7fb5b4008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.397+0000 7fb5c0ff9640 1 -- 172.21.15.64:0/2240754127 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb5c40d10e0 con 0x7fb5c40d06c0 2024-03-04T07:33:34.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.580+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7fb5c40b1c60 con 0x7fb5c40d06c0 2024-03-04T07:33:34.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.580+0000 7fb5c0ff9640 1 -- 172.21.15.64:0/2240754127 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v119) v1 ==== 84+0+65 (secure 0 0 0) 0x7fb5bc0587e0 con 0x7fb5c40d06c0 2024-03-04T07:33:34.580 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:34.580 INFO:teuthology.orchestra.run.smithi064.stdout:[ 2024-03-04T07:33:34.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-03-04T07:33:34.581 INFO:teuthology.orchestra.run.smithi064.stdout: "poolnum": 1, 2024-03-04T07:33:34.581 INFO:teuthology.orchestra.run.smithi064.stdout: "poolname": ".mgr" 2024-03-04T07:33:34.581 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:34.581 INFO:teuthology.orchestra.run.smithi064.stdout:] 2024-03-04T07:33:34.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5a406b500 msgr2=0x7fb5a406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5a406b500 0x7fb5a406d9c0 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7fb5b4036000 tx=0x7fb5b4008030 comp rx=0 tx=0).stop 2024-03-04T07:33:34.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5c40d06c0 msgr2=0x7fb5c40ca0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5c40d06c0 0x7fb5c40ca0c0 secure :-1 s=READY pgs=1524 cs=0 l=1 rev1=1 crypto rx=0x7fb5bc00cb40 tx=0x7fb5bc00a060 comp rx=0 tx=0).stop 2024-03-04T07:33:34.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 shutdown_connections 2024-03-04T07:33:34.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb5a406b500 0x7fb5a406d9c0 unknown :-1 s=CLOSED pgs=681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb5c40d06c0 0x7fb5c40ca0c0 unknown :-1 s=CLOSED pgs=1524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb5c40d02e0 0x7fb5c415ce40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 --2- 172.21.15.64:0/2240754127 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb5c40b4260 0x7fb5c40cb680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 >> 172.21.15.64:0/2240754127 conn(0x7fb5c40b36d0 msgr2=0x7fb5c415c000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:34.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 shutdown_connections 2024-03-04T07:33:34.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.582+0000 7fb5ca5ae640 1 -- 172.21.15.64:0/2240754127 wait complete. 2024-03-04T07:33:34.593 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-04T07:33:34.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-04T07:33:34.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.699+0000 7feec6d81640 1 Processor -- start 2024-03-04T07:33:34.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.699+0000 7feec6d81640 1 -- start start 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec6d81640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feec01618e0 0x7feec006b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec6d81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec6d81640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7feec0161d40 0x7feec01621d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec6d81640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feec00cfce0 con 0x7feec0161d40 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec6d81640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feec00cee70 con 0x7feec01618e0 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec6d81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feec015f730 con 0x7feec00ce4f0 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7feec0161d40 0x7feec01621d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7feec0161d40 0x7feec01621d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60536/0 (socket says 172.21.15.64:60536) 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 -- 172.21.15.64:0/1416075818 learned_addr learned my addr 172.21.15.64:0/1416075818 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec4b1d640 1 --2- 172.21.15.64:0/1416075818 >> v2:172.21.15.66:3300/0 conn(0x7feec01618e0 0x7feec006b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feeb7fff640 1 --2- 172.21.15.64:0/1416075818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 -- 172.21.15.64:0/1416075818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 msgr2=0x7feec00ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 --2- 172.21.15.64:0/1416075818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00ce930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 -- 172.21.15.64:0/1416075818 >> v2:172.21.15.66:3300/0 conn(0x7feec01618e0 msgr2=0x7feec006b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 --2- 172.21.15.64:0/1416075818 >> v2:172.21.15.66:3300/0 conn(0x7feec01618e0 0x7feec006b130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feec531e640 1 -- 172.21.15.64:0/1416075818 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feec015ebb0 con 0x7feec0161d40 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.700+0000 7feeb7fff640 1 --2- 172.21.15.64:0/1416075818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.701+0000 7feec531e640 1 --2- 172.21.15.64:0/1416075818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec01621d0 secure :-1 s=READY pgs=1525 cs=0 l=1 rev1=1 crypto rx=0x7feec0071250 tx=0x7feeb802bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad41f30b5471e2bc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.701+0000 7feeb77fe640 1 -- 172.21.15.64:0/1416075818 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feeb802cb00 con 0x7feec0161d40 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.701+0000 7feeb77fe640 1 -- 172.21.15.64:0/1416075818 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feeb802cca0 con 0x7feec0161d40 2024-03-04T07:33:34.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.701+0000 7feeb77fe640 1 -- 172.21.15.64:0/1416075818 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feeb8034610 con 0x7feec0161d40 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.701+0000 7feec6d81640 1 -- 172.21.15.64:0/1416075818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 msgr2=0x7feec01621d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.701+0000 7feec6d81640 1 --2- 172.21.15.64:0/1416075818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec01621d0 secure :-1 s=READY pgs=1525 cs=0 l=1 rev1=1 crypto rx=0x7feec0071250 tx=0x7feeb802bf60 comp rx=0 tx=0).stop 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.702+0000 7feec6d81640 1 -- 172.21.15.64:0/1416075818 shutdown_connections 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.702+0000 7feec6d81640 1 --2- 172.21.15.64:0/1416075818 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec01621d0 unknown :-1 s=CLOSED pgs=1525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.702+0000 7feec6d81640 1 --2- 172.21.15.64:0/1416075818 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.702+0000 7feec6d81640 1 --2- 172.21.15.64:0/1416075818 >> v2:172.21.15.66:3300/0 conn(0x7feec01618e0 0x7feec006b130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.702+0000 7feec6d81640 1 -- 172.21.15.64:0/1416075818 >> 172.21.15.64:0/1416075818 conn(0x7feec00b3660 msgr2=0x7feec00b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.702+0000 7feec6d81640 1 -- 172.21.15.64:0/1416075818 shutdown_connections 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.702+0000 7feec6d81640 1 -- 172.21.15.64:0/1416075818 wait complete. 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.703+0000 7feec6d81640 1 Processor -- start 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.703+0000 7feec6d81640 1 -- start start 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00cb5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec00c8c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec0154270 0x7feec0153390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feec015f450 con 0x7feec0161d40 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feec015f170 con 0x7feec0154270 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feec006b670 con 0x7feec00ce4f0 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec00c8c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec00c8c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60548/0 (socket says 172.21.15.64:60548) 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 -- 172.21.15.64:0/2579717545 learned_addr learned my addr 172.21.15.64:0/2579717545 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec531e640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec0154270 0x7feec0153390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec4b1d640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00cb5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 -- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 msgr2=0x7feec00cb5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00cb5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 -- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec0154270 msgr2=0x7feec0153390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec0154270 0x7feec0153390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 -- 172.21.15.64:0/2579717545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feec00b7250 con 0x7feec0161d40 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb7fff640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec00c8c20 secure :-1 s=READY pgs=1526 cs=0 l=1 rev1=1 crypto rx=0x7feeb000c7f0 tx=0x7feeb000ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb57fa640 1 -- 172.21.15.64:0/2579717545 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feeb0015070 con 0x7feec0161d40 2024-03-04T07:33:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb57fa640 1 -- 172.21.15.64:0/2579717545 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feeb0007820 con 0x7feec0161d40 2024-03-04T07:33:34.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feec00b7530 con 0x7feec0161d40 2024-03-04T07:33:34.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feeb57fa640 1 -- 172.21.15.64:0/2579717545 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feeb0007c10 con 0x7feec0161d40 2024-03-04T07:33:34.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.704+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feec00b5620 con 0x7feec0161d40 2024-03-04T07:33:34.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.705+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feec009fc50 con 0x7feec0161d40 2024-03-04T07:33:34.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.706+0000 7feeb57fa640 1 -- 172.21.15.64:0/2579717545 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feeb0018070 con 0x7feec0161d40 2024-03-04T07:33:34.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.706+0000 7feeb57fa640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeac06b500 0x7feeac06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:34.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.706+0000 7feeb57fa640 1 -- 172.21.15.64:0/2579717545 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(119..119 src has 1..119) v4 ==== 4857+0+0 (secure 0 0 0) 0x7feeb0089d30 con 0x7feec0161d40 2024-03-04T07:33:34.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.706+0000 7feec4b1d640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeac06b500 0x7feeac06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:34.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.707+0000 7feec4b1d640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeac06b500 0x7feeac06d9c0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7feebc006fd0 tx=0x7feebc008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:34.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.711+0000 7feeb57fa640 1 -- 172.21.15.64:0/2579717545 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feeb0058f10 con 0x7feec0161d40 2024-03-04T07:33:34.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:34.895+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7feec004f410 con 0x7feec0161d40 2024-03-04T07:33:35.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.870+0000 7feeb57fa640 1 -- 172.21.15.64:0/2579717545 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v120) v1 ==== 138+0+0 (secure 0 0 0) 0x7feeb0058810 con 0x7feec0161d40 2024-03-04T07:33:35.870 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' created 2024-03-04T07:33:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeac06b500 msgr2=0x7feeac06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeac06b500 0x7feeac06d9c0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7feebc006fd0 tx=0x7feebc008040 comp rx=0 tx=0).stop 2024-03-04T07:33:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 msgr2=0x7feec00c8c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec00c8c20 secure :-1 s=READY pgs=1526 cs=0 l=1 rev1=1 crypto rx=0x7feeb000c7f0 tx=0x7feeb000ccb0 comp rx=0 tx=0).stop 2024-03-04T07:33:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 shutdown_connections 2024-03-04T07:33:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feeac06b500 0x7feeac06d9c0 unknown :-1 s=CLOSED pgs=682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feec0154270 0x7feec0153390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:35.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feec0161d40 0x7feec00c8c20 unknown :-1 s=CLOSED pgs=1526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:35.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 --2- 172.21.15.64:0/2579717545 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feec00ce4f0 0x7feec00cb5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:35.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 >> 172.21.15.64:0/2579717545 conn(0x7feec00b3660 msgr2=0x7feec00b3a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:35.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 shutdown_connections 2024-03-04T07:33:35.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:35.873+0000 7feec6d81640 1 -- 172.21.15.64:0/2579717545 wait complete. 2024-03-04T07:33:35.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-04T07:33:36.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.005+0000 7fa606d63640 1 Processor -- start 2024-03-04T07:33:36.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.005+0000 7fa606d63640 1 -- start start 2024-03-04T07:33:36.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa606d63640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6000717b0 0x7fa600071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:36.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa606d63640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6000d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:36.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa606d63640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa60015e4b0 con 0x7fa6000b3060 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa606d63640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa6000d2e10 con 0x7fa6000717b0 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa606d63640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6000b97e0 con 0x7fa6000b2b10 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa604aff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa6000717b0 0x7fa600071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa5fffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa6000b3060 0x7fa6000b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54714/0 (socket says 172.21.15.64:54714) 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa5fffff640 1 -- 172.21.15.64:0/1161752999 learned_addr learned my addr 172.21.15.64:0/1161752999 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa605300640 1 --2- 172.21.15.64:0/1161752999 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6000d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.006+0000 7fa604aff640 1 -- 172.21.15.64:0/1161752999 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 msgr2=0x7fa6000d32a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa604aff640 1 --2- 172.21.15.64:0/1161752999 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6000d32a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa604aff640 1 -- 172.21.15.64:0/1161752999 >> v1:172.21.15.64:6789/0 conn(0x7fa6000b3060 legacy=0x7fa6000b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa604aff640 1 -- 172.21.15.64:0/1161752999 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6000b9a80 con 0x7fa6000717b0 2024-03-04T07:33:36.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa5ff7fe640 1 -- 172.21.15.64:0/1161752999 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4180734051 0 0) 0x7fa5e8001110 con 0x7fa6000b3060 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa604aff640 1 --2- 172.21.15.64:0/1161752999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa600071b90 secure :-1 s=READY pgs=1528 cs=0 l=1 rev1=1 crypto rx=0x7fa5f8009d90 tx=0x7fa5f802bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=acf36cb28f0046b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa5ff7fe640 1 -- 172.21.15.64:0/1161752999 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f8039070 con 0x7fa6000717b0 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa5ff7fe640 1 -- 172.21.15.64:0/1161752999 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa5f802cc50 con 0x7fa6000717b0 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.007+0000 7fa5ff7fe640 1 -- 172.21.15.64:0/1161752999 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f8034440 con 0x7fa6000717b0 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 -- 172.21.15.64:0/1161752999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 msgr2=0x7fa600071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 --2- 172.21.15.64:0/1161752999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa600071b90 secure :-1 s=READY pgs=1528 cs=0 l=1 rev1=1 crypto rx=0x7fa5f8009d90 tx=0x7fa5f802bea0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 -- 172.21.15.64:0/1161752999 shutdown_connections 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 --2- 172.21.15.64:0/1161752999 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6000d32a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 --2- 172.21.15.64:0/1161752999 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa600071b90 unknown :-1 s=CLOSED pgs=1528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 -- 172.21.15.64:0/1161752999 >> 172.21.15.64:0/1161752999 conn(0x7fa6000d3d20 msgr2=0x7fa6000d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 -- 172.21.15.64:0/1161752999 shutdown_connections 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.008+0000 7fa606d63640 1 -- 172.21.15.64:0/1161752999 wait complete. 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.009+0000 7fa606d63640 1 Processor -- start 2024-03-04T07:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.009+0000 7fa606d63640 1 -- start start 2024-03-04T07:33:36.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa606d63640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa6000ce090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:36.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa606d63640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6001e3db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa606d63640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6000b3060 0x7fa600149fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa606d63640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6000b7560 con 0x7fa6000717b0 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa606d63640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6000d4bf0 con 0x7fa6000b3060 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa606d63640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa6000b76e0 con 0x7fa6000b2b10 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa6000ce090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa6000ce090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60566/0 (socket says 172.21.15.64:60566) 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 -- 172.21.15.64:0/1212013783 learned_addr learned my addr 172.21.15.64:0/1212013783 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa5fffff640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6001e3db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa605300640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6000b3060 0x7fa600149fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 -- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 msgr2=0x7fa6001e3db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6001e3db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 -- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6000b3060 msgr2=0x7fa600149fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6000b3060 0x7fa600149fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.010+0000 7fa604aff640 1 -- 172.21.15.64:0/1212013783 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6000cd2d0 con 0x7fa6000717b0 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.011+0000 7fa604aff640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa6000ce090 secure :-1 s=READY pgs=1529 cs=0 l=1 rev1=1 crypto rx=0x7fa5f8009d90 tx=0x7fa5f8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:36.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.011+0000 7fa5fd7fa640 1 -- 172.21.15.64:0/1212013783 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f8039050 con 0x7fa6000717b0 2024-03-04T07:33:36.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.011+0000 7fa5fd7fa640 1 -- 172.21.15.64:0/1212013783 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa5f803a040 con 0x7fa6000717b0 2024-03-04T07:33:36.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.011+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa6000cd5b0 con 0x7fa6000717b0 2024-03-04T07:33:36.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.011+0000 7fa5fd7fa640 1 -- 172.21.15.64:0/1212013783 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5f802ca50 con 0x7fa6000717b0 2024-03-04T07:33:36.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.011+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6001e42f0 con 0x7fa6000717b0 2024-03-04T07:33:36.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.012+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa60006ce40 con 0x7fa6000717b0 2024-03-04T07:33:36.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.012+0000 7fa5fd7fa640 1 -- 172.21.15.64:0/1212013783 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa5f8047020 con 0x7fa6000717b0 2024-03-04T07:33:36.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.013+0000 7fa5fd7fa640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa5e006b4d0 0x7fa5e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:36.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.013+0000 7fa5fd7fa640 1 -- 172.21.15.64:0/1212013783 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(120..120 src has 1..120) v4 ==== 5227+0+0 (secure 0 0 0) 0x7fa5f80ac1a0 con 0x7fa6000717b0 2024-03-04T07:33:36.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.013+0000 7fa5fffff640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa5e006b4d0 0x7fa5e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:36.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.013+0000 7fa5fffff640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa5e006b4d0 0x7fa5e006d990 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7fa6000b9660 tx=0x7fa5e8007c00 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:36.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.017+0000 7fa5fd7fa640 1 -- 172.21.15.64:0/1212013783 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa5f804e020 con 0x7fa6000717b0 2024-03-04T07:33:36.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.199+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fa60004f410 con 0x7fa6000717b0 2024-03-04T07:33:36.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.871+0000 7fa5fd7fa640 1 -- 172.21.15.64:0/1212013783 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v121) v1 ==== 170+0+0 (secure 0 0 0) 0x7fa5f807b200 con 0x7fa6000717b0 2024-03-04T07:33:36.872 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' created 2024-03-04T07:33:36.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa5e006b4d0 msgr2=0x7fa5e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:36.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa5e006b4d0 0x7fa5e006d990 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7fa6000b9660 tx=0x7fa5e8007c00 comp rx=0 tx=0).stop 2024-03-04T07:33:36.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 msgr2=0x7fa6000ce090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:36.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa6000ce090 secure :-1 s=READY pgs=1529 cs=0 l=1 rev1=1 crypto rx=0x7fa5f8009d90 tx=0x7fa5f8004a40 comp rx=0 tx=0).stop 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 shutdown_connections 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa5e006b4d0 0x7fa5e006d990 unknown :-1 s=CLOSED pgs=683 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa6000b3060 0x7fa600149fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa6000b2b10 0x7fa6001e3db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 --2- 172.21.15.64:0/1212013783 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa6000717b0 0x7fa6000ce090 unknown :-1 s=CLOSED pgs=1529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.874+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 >> 172.21.15.64:0/1212013783 conn(0x7fa6000d3d20 msgr2=0x7fa6000c9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.875+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 shutdown_connections 2024-03-04T07:33:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:36.875+0000 7fa606d63640 1 -- 172.21.15.64:0/1212013783 wait complete. 2024-03-04T07:33:36.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-04T07:33:37.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.014+0000 7f8e61e95640 1 Processor -- start 2024-03-04T07:33:37.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.014+0000 7f8e61e95640 1 -- start start 2024-03-04T07:33:37.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e61e95640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8e5c15df70 0x7f8e5c0d3210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e61e95640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0ce530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e61e95640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8e5c0d3b10 con 0x7f8e5c0cea70 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e61e95640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8e5c0b6d60 con 0x7f8e5c15df70 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e61e95640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e5c0d3df0 con 0x7f8e5c0d1dd0 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e5bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8e5c0cea70 0x7f8e5c0d06b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54728/0 (socket says 172.21.15.64:54728) 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e5bfff640 1 -- 172.21.15.64:0/2001970380 learned_addr learned my addr 172.21.15.64:0/2001970380 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e5b7fe640 1 --2- 172.21.15.64:0/2001970380 >> v2:172.21.15.66:3300/0 conn(0x7f8e5c15df70 0x7f8e5c0d3210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.015+0000 7f8e5affd640 1 --2- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0ce530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.016+0000 7f8e5a7fc640 1 -- 172.21.15.64:0/2001970380 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4167796362 0 0) 0x7f8e5c0d3b10 con 0x7f8e5c0cea70 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.016+0000 7f8e5a7fc640 1 -- 172.21.15.64:0/2001970380 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8e44003520 con 0x7f8e5c0cea70 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.016+0000 7f8e5b7fe640 1 -- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c0d1dd0 msgr2=0x7f8e5c0ce530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.016+0000 7f8e5b7fe640 1 --2- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0ce530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.016+0000 7f8e5b7fe640 1 -- 172.21.15.64:0/2001970380 >> v1:172.21.15.64:6789/0 conn(0x7f8e5c0cea70 legacy=0x7f8e5c0d06b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:37.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.016+0000 7f8e5b7fe640 1 -- 172.21.15.64:0/2001970380 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e5c0d40d0 con 0x7f8e5c15df70 2024-03-04T07:33:37.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.016+0000 7f8e5affd640 1 --2- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0ce530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e5b7fe640 1 --2- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c15df70 0x7f8e5c0d3210 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f8e4c0097a0 tx=0x7f8e4c02b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=341a271c71939c7b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e5a7fc640 1 -- 172.21.15.64:0/2001970380 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e4c035020 con 0x7f8e5c15df70 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e5a7fc640 1 -- 172.21.15.64:0/2001970380 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e4c031830 con 0x7f8e5c15df70 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e5a7fc640 1 -- 172.21.15.64:0/2001970380 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e4c031c00 con 0x7f8e5c15df70 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 -- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c15df70 msgr2=0x7f8e5c0d3210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 --2- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c15df70 0x7f8e5c0d3210 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f8e4c0097a0 tx=0x7f8e4c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 -- 172.21.15.64:0/2001970380 shutdown_connections 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 --2- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0ce530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 --2- 172.21.15.64:0/2001970380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c15df70 0x7f8e5c0d3210 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 -- 172.21.15.64:0/2001970380 >> 172.21.15.64:0/2001970380 conn(0x7f8e5c071960 msgr2=0x7f8e5c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 -- 172.21.15.64:0/2001970380 shutdown_connections 2024-03-04T07:33:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.017+0000 7f8e61e95640 1 -- 172.21.15.64:0/2001970380 wait complete. 2024-03-04T07:33:37.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 Processor -- start 2024-03-04T07:33:37.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 -- start start 2024-03-04T07:33:37.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e5c0cea70 0x7f8e5c1f3630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:37.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0cc780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:37.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c15df70 0x7f8e5c158af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e5c0d38d0 con 0x7f8e5c0cea70 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e5c0d3750 con 0x7f8e5c0d1dd0 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.019+0000 7f8e61e95640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e5c0b81e0 con 0x7f8e5c15df70 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e5c0cea70 0x7f8e5c1f3630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e5c0cea70 0x7f8e5c1f3630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60582/0 (socket says 172.21.15.64:60582) 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 -- 172.21.15.64:0/3159337510 learned_addr learned my addr 172.21.15.64:0/3159337510 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5affd640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0cc780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 -- 172.21.15.64:0/3159337510 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c15df70 msgr2=0x7f8e5c158af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c15df70 0x7f8e5c158af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 -- 172.21.15.64:0/3159337510 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c0d1dd0 msgr2=0x7f8e5c0cc780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0cc780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 -- 172.21.15.64:0/3159337510 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e5c0cd5a0 con 0x7f8e5c0cea70 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e5b7fe640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e5c0cea70 0x7f8e5c1f3630 secure :-1 s=READY pgs=1531 cs=0 l=1 rev1=1 crypto rx=0x7f8e4c034040 tx=0x7f8e4c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e43fff640 1 -- 172.21.15.64:0/3159337510 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e4c040070 con 0x7f8e5c0cea70 2024-03-04T07:33:37.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e43fff640 1 -- 172.21.15.64:0/3159337510 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8e4c03a910 con 0x7f8e5c0cea70 2024-03-04T07:33:37.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e5c0cd880 con 0x7f8e5c0cea70 2024-03-04T07:33:37.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e43fff640 1 -- 172.21.15.64:0/3159337510 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e4c035040 con 0x7f8e5c0cea70 2024-03-04T07:33:37.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.020+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e5c1582f0 con 0x7f8e5c0cea70 2024-03-04T07:33:37.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.021+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e5c072ac0 con 0x7f8e5c0cea70 2024-03-04T07:33:37.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.022+0000 7f8e43fff640 1 -- 172.21.15.64:0/3159337510 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8e4c038020 con 0x7f8e5c0cea70 2024-03-04T07:33:37.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.022+0000 7f8e43fff640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e4806b4d0 0x7f8e4806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:37.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.022+0000 7f8e5affd640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e4806b4d0 0x7f8e4806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:37.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.022+0000 7f8e43fff640 1 -- 172.21.15.64:0/3159337510 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(121..121 src has 1..121) v4 ==== 5609+0+0 (secure 0 0 0) 0x7f8e4c0ab9b0 con 0x7f8e5c0cea70 2024-03-04T07:33:37.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.023+0000 7f8e5affd640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e4806b4d0 0x7f8e4806d990 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f8e50009560 tx=0x7f8e50008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:37.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.027+0000 7f8e43fff640 1 -- 172.21.15.64:0/3159337510 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8e4c07a890 con 0x7f8e5c0cea70 2024-03-04T07:33:37.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.245+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7f8e5c15cdd0 con 0x7f8e5c0cea70 2024-03-04T07:33:37.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.899+0000 7f8e43fff640 1 -- 172.21.15.64:0/3159337510 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 18 and data pool 19 v78) v1 ==== 174+0+0 (secure 0 0 0) 0x7f8e4c07a190 con 0x7f8e5c0cea70 2024-03-04T07:33:37.900 INFO:teuthology.orchestra.run.smithi064.stderr:new fs with metadata pool 18 and data pool 19 2024-03-04T07:33:37.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.906+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e4806b4d0 msgr2=0x7f8e4806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:37.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.906+0000 7f8e61e95640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e4806b4d0 0x7f8e4806d990 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f8e50009560 tx=0x7f8e50008040 comp rx=0 tx=0).stop 2024-03-04T07:33:37.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.906+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e5c0cea70 msgr2=0x7f8e5c1f3630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:37.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.906+0000 7f8e61e95640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e5c0cea70 0x7f8e5c1f3630 secure :-1 s=READY pgs=1531 cs=0 l=1 rev1=1 crypto rx=0x7f8e4c034040 tx=0x7f8e4c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.907+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 shutdown_connections 2024-03-04T07:33:37.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.907+0000 7f8e61e95640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8e4806b4d0 0x7f8e4806d990 unknown :-1 s=CLOSED pgs=684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.907+0000 7f8e61e95640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8e5c15df70 0x7f8e5c158af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.907+0000 7f8e61e95640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8e5c0d1dd0 0x7f8e5c0cc780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.907+0000 7f8e61e95640 1 --2- 172.21.15.64:0/3159337510 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8e5c0cea70 0x7f8e5c1f3630 unknown :-1 s=CLOSED pgs=1531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:37.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.907+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 >> 172.21.15.64:0/3159337510 conn(0x7f8e5c071960 msgr2=0x7f8e5c0d0690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:37.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.909+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 shutdown_connections 2024-03-04T07:33:37.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:37.909+0000 7f8e61e95640 1 -- 172.21.15.64:0/3159337510 wait complete. 2024-03-04T07:33:37.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:38.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.032+0000 7f6a35876640 1 Processor -- start 2024-03-04T07:33:38.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.032+0000 7f6a35876640 1 -- start start 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.032+0000 7f6a35876640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a3015b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.032+0000 7f6a35876640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6a300cbab0 0x7f6a30070b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.032+0000 7f6a35876640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6a3006c7b0 con 0x7f6a30157b80 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.032+0000 7f6a35876640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6a3006ca90 con 0x7f6a300cbab0 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.032+0000 7f6a35876640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a3006cd70 con 0x7f6a30157f60 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6a30157b80 0x7f6a3015b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54738/0 (socket says 172.21.15.64:54738) 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2effd640 1 -- 172.21.15.64:0/1985996749 learned_addr learned my addr 172.21.15.64:0/1985996749 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:38.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a27fff640 1 --2- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a3015b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2f7fe640 1 --2- 172.21.15.64:0/1985996749 >> v2:172.21.15.66:3300/0 conn(0x7f6a300cbab0 0x7f6a30070b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2e7fc640 1 -- 172.21.15.64:0/1985996749 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3614766566 0 0) 0x7f6a3006c7b0 con 0x7f6a30157b80 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2e7fc640 1 -- 172.21.15.64:0/1985996749 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6a1c003520 con 0x7f6a30157b80 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2e7fc640 1 -- 172.21.15.64:0/1985996749 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1311287858 0 0) 0x7f6a1c003520 con 0x7f6a30157b80 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2f7fe640 1 -- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 msgr2=0x7f6a3015b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2f7fe640 1 --2- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a3015b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2f7fe640 1 -- 172.21.15.64:0/1985996749 >> v1:172.21.15.64:6789/0 conn(0x7f6a30157b80 legacy=0x7f6a3015b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.033+0000 7f6a2f7fe640 1 -- 172.21.15.64:0/1985996749 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a3006d050 con 0x7f6a300cbab0 2024-03-04T07:33:38.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.034+0000 7f6a2f7fe640 1 --2- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 0x7f6a30070b40 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f6a2800aea0 tx=0x7f6a2802ce80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=eeb1f1860e0b29b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.034+0000 7f6a2e7fc640 1 -- 172.21.15.64:0/1985996749 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a20001670 con 0x7f6a300cbab0 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.034+0000 7f6a2e7fc640 1 -- 172.21.15.64:0/1985996749 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a2802d7a0 con 0x7f6a300cbab0 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.034+0000 7f6a2e7fc640 1 -- 172.21.15.64:0/1985996749 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a2802db70 con 0x7f6a300cbab0 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 -- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 msgr2=0x7f6a30070b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 --2- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 0x7f6a30070b40 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f6a2800aea0 tx=0x7f6a2802ce80 comp rx=0 tx=0).stop 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 -- 172.21.15.64:0/1985996749 shutdown_connections 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 --2- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 0x7f6a30070b40 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 --2- 172.21.15.64:0/1985996749 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a3015b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 -- 172.21.15.64:0/1985996749 >> 172.21.15.64:0/1985996749 conn(0x7f6a30066d00 msgr2=0x7f6a300670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 -- 172.21.15.64:0/1985996749 shutdown_connections 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.035+0000 7f6a35876640 1 -- 172.21.15.64:0/1985996749 wait complete. 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.036+0000 7f6a35876640 1 Processor -- start 2024-03-04T07:33:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.036+0000 7f6a35876640 1 -- start start 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 0x7f6a300adb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a30157b80 0x7f6a301f39e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a300b0140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a30071080 con 0x7f6a30157b80 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a300cc260 con 0x7f6a300cbab0 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6a30071200 con 0x7f6a30157f60 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a30157b80 0x7f6a301f39e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a30157b80 0x7f6a301f39e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60590/0 (socket says 172.21.15.64:60590) 2024-03-04T07:33:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 -- 172.21.15.64:0/1826177819 learned_addr learned my addr 172.21.15.64:0/1826177819 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a2effd640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 0x7f6a300adb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a2f7fe640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a300b0140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 -- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 msgr2=0x7f6a300b0140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a300b0140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 -- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 msgr2=0x7f6a300adb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 0x7f6a300adb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 -- 172.21.15.64:0/1826177819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a3015c6d0 con 0x7f6a30157b80 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a2f7fe640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a300b0140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a27fff640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a30157b80 0x7f6a301f39e0 secure :-1 s=READY pgs=1533 cs=0 l=1 rev1=1 crypto rx=0x7f6a1800e870 tx=0x7f6a1800ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a2cff9640 1 -- 172.21.15.64:0/1826177819 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a18017070 con 0x7f6a30157b80 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a2cff9640 1 -- 172.21.15.64:0/1826177819 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6a1800c940 con 0x7f6a30157b80 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a3015c9b0 con 0x7f6a30157b80 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a2cff9640 1 -- 172.21.15.64:0/1826177819 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a1800cd30 con 0x7f6a30157b80 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.037+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a301f6fd0 con 0x7f6a30157b80 2024-03-04T07:33:38.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.038+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6a30067ff0 con 0x7f6a30157b80 2024-03-04T07:33:38.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.039+0000 7f6a2cff9640 1 -- 172.21.15.64:0/1826177819 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6a1801a070 con 0x7f6a30157b80 2024-03-04T07:33:38.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.039+0000 7f6a2cff9640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a1006b4d0 0x7f6a1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.039+0000 7f6a2cff9640 1 -- 172.21.15.64:0/1826177819 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(122..122 src has 1..122) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6a18089fe0 con 0x7f6a30157b80 2024-03-04T07:33:38.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.039+0000 7f6a2effd640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a1006b4d0 0x7f6a1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.040+0000 7f6a2effd640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a1006b4d0 0x7f6a1006d990 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f6a200013c0 tx=0x7f6a20001350 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.044+0000 7f6a2cff9640 1 -- 172.21.15.64:0/1826177819 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6a18058840 con 0x7f6a30157b80 2024-03-04T07:33:38.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.224+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6a30153c10 con 0x7f6a30157b80 2024-03-04T07:33:38.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.226+0000 7f6a2cff9640 1 -- 172.21.15.64:0/1826177819 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v122) v1 ==== 74+0+16644 (secure 0 0 0) 0x7f6a1805c8e0 con 0x7f6a30157b80 2024-03-04T07:33:38.227 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:38.227 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":122,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:33:37.247377+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:33:34.914683+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-04T07:33:36.218855+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:38.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.227+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a1006b4d0 msgr2=0x7f6a1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.227+0000 7f6a35876640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a1006b4d0 0x7f6a1006d990 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f6a200013c0 tx=0x7f6a20001350 comp rx=0 tx=0).stop 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.227+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a30157b80 msgr2=0x7f6a301f39e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.227+0000 7f6a35876640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a30157b80 0x7f6a301f39e0 secure :-1 s=READY pgs=1533 cs=0 l=1 rev1=1 crypto rx=0x7f6a1800e870 tx=0x7f6a1800ed30 comp rx=0 tx=0).stop 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 shutdown_connections 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6a1006b4d0 0x7f6a1006d990 unknown :-1 s=CLOSED pgs=685 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6a30157f60 0x7f6a300b0140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6a30157b80 0x7f6a301f39e0 unknown :-1 s=CLOSED pgs=1533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 --2- 172.21.15.64:0/1826177819 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6a300cbab0 0x7f6a300adb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 >> 172.21.15.64:0/1826177819 conn(0x7f6a30066d00 msgr2=0x7f6a301548b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 shutdown_connections 2024-03-04T07:33:38.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.228+0000 7f6a35876640 1 -- 172.21.15.64:0/1826177819 wait complete. 2024-03-04T07:33:38.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:38.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.340+0000 7f3424681640 1 Processor -- start 2024-03-04T07:33:38.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.340+0000 7f3424681640 1 -- start start 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.341+0000 7f3424681640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f341c071540 0x7f341c0b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.341+0000 7f3424681640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 0x7f341c071920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.341+0000 7f3424681640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f341c071e60 con 0x7f341c0b2b50 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.341+0000 7f3424681640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f341c0d2f70 con 0x7f341c071540 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.341+0000 7f3424681640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f341c0d4be0 con 0x7f341c0d3400 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.341+0000 7f3422c1e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f341c0b2b50 0x7f341c0d1870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54752/0 (socket says 172.21.15.64:54752) 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.341+0000 7f3422c1e640 1 -- 172.21.15.64:0/2910465782 learned_addr learned my addr 172.21.15.64:0/2910465782 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342241d640 1 --2- 172.21.15.64:0/2910465782 >> v2:172.21.15.66:3300/0 conn(0x7f341c071540 0x7f341c0b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f3421c1c640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 0x7f341c071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 824159724 0 0) 0x7f341c071e60 con 0x7f341c0b2b50 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3404003520 con 0x7f341c0b2b50 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2421767637 0 0) 0x7f3404003520 con 0x7f341c0b2b50 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 msgr2=0x7f341c071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:38.345 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 0x7f341c071920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 >> v2:172.21.15.66:3300/0 conn(0x7f341c071540 msgr2=0x7f341c0b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 --2- 172.21.15.64:0/2910465782 >> v2:172.21.15.66:3300/0 conn(0x7f341c071540 0x7f341c0b30a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f341c0ce610 con 0x7f341c0b2b50 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f340c0019f0 con 0x7f341c0b2b50 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 >> v1:172.21.15.64:6789/0 conn(0x7f341c0b2b50 legacy=0x7f341c0d1870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f34040039b0 0x7f3404003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3404004800 0x7f3404004c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 0x7f34040051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f341c0d2f70 con 0x7f34040039b0 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f341c0d4be0 con 0x7f3404004800 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f340400b9c0 con 0x7f341c0d3400 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342241d640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f34040039b0 0x7f3404003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f3421c1c640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3404004800 0x7f3404004c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f3422c1e640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 0x7f34040051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342241d640 1 -- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 msgr2=0x7f34040051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:38.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342241d640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f341c0d3400 0x7f34040051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342241d640 1 -- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3404004800 msgr2=0x7f3404004c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342241d640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3404004800 0x7f3404004c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.342+0000 7f342241d640 1 -- 172.21.15.64:0/2910465782 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3404003520 con 0x7f34040039b0 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.343+0000 7f3421c1c640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3404004800 0x7f3404004c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.343+0000 7f342241d640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f34040039b0 0x7f3404003dd0 secure :-1 s=READY pgs=1535 cs=0 l=1 rev1=1 crypto rx=0x7f3414009ac0 tx=0x7f3404005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec2a726afcd59850 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.343+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3414039070 con 0x7f34040039b0 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.343+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f341402c710 con 0x7f34040039b0 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.343+0000 7f342141b640 1 -- 172.21.15.64:0/2910465782 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3414009690 con 0x7f34040039b0 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.344+0000 7f3424681640 1 -- 172.21.15.64:0/2910465782 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f34040039b0 msgr2=0x7f3404003dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.344+0000 7f3424681640 1 --2- 172.21.15.64:0/2910465782 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f34040039b0 0x7f3404003dd0 secure :-1 s=READY pgs=1535 cs=0 l=1 rev1=1 crypto rx=0x7f3414009ac0 tx=0x7f3404005830 comp rx=0 tx=0).stop 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.344+0000 7f3422c1e640 1 -- 172.21.15.64:0/2910465782 reap_dead start 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.344+0000 7f3424681640 1 -- 172.21.15.64:0/2910465782 shutdown_connections 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.344+0000 7f3424681640 1 -- 172.21.15.64:0/2910465782 >> 172.21.15.64:0/2910465782 conn(0x7f341c0d3ca0 msgr2=0x7f341c0d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.344+0000 7f3424681640 1 -- 172.21.15.64:0/2910465782 shutdown_connections 2024-03-04T07:33:38.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.344+0000 7f3424681640 1 -- 172.21.15.64:0/2910465782 wait complete. 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.345+0000 7f3424681640 1 Processor -- start 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.345+0000 7f3424681640 1 -- start start 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3424681640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34040039b0 0x7f341c158180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3424681640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f341c0cda70 0x7f341c0cb780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3424681640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f341c15a230 0x7f341c15a6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3424681640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f341c0ce8f0 con 0x7f341c15a230 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3424681640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f341c0d2c00 con 0x7f341c0cda70 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3424681640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f341c15e370 con 0x7f34040039b0 2024-03-04T07:33:38.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f341c15a230 0x7f341c15a6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f341c15a230 0x7f341c15a6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60620/0 (socket says 172.21.15.64:60620) 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 -- 172.21.15.64:0/1934639588 learned_addr learned my addr 172.21.15.64:0/1934639588 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f342241d640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34040039b0 0x7f341c158180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3421c1c640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f341c0cda70 0x7f341c0cb780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 -- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34040039b0 msgr2=0x7f341c158180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34040039b0 0x7f341c158180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 -- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f341c0cda70 msgr2=0x7f341c0cb780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f341c0cda70 0x7f341c0cb780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 -- 172.21.15.64:0/1934639588 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f341c0cc080 con 0x7f341c15a230 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.346+0000 7f3422c1e640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f341c15a230 0x7f341c15a6b0 secure :-1 s=READY pgs=1536 cs=0 l=1 rev1=1 crypto rx=0x7f341c0b6860 tx=0x7f340c00e530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.347+0000 7f341affd640 1 -- 172.21.15.64:0/1934639588 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f340c00eb30 con 0x7f341c15a230 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.347+0000 7f341affd640 1 -- 172.21.15.64:0/1934639588 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f340c00ecd0 con 0x7f341c15a230 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.347+0000 7f341affd640 1 -- 172.21.15.64:0/1934639588 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f340c014670 con 0x7f341c15a230 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.347+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f341c0cc360 con 0x7f341c15a230 2024-03-04T07:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.347+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f341c1f3c30 con 0x7f341c15a230 2024-03-04T07:33:38.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.348+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f341c1f3e30 con 0x7f341c15a230 2024-03-04T07:33:38.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.348+0000 7f341affd640 1 -- 172.21.15.64:0/1934639588 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f340c014810 con 0x7f341c15a230 2024-03-04T07:33:38.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.348+0000 7f341affd640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f340806b4d0 0x7f340806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.349+0000 7f341affd640 1 -- 172.21.15.64:0/1934639588 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(122..122 src has 1..122) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f340c089e70 con 0x7f341c15a230 2024-03-04T07:33:38.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.349+0000 7f342241d640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f340806b4d0 0x7f340806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.349+0000 7f342241d640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f340806b4d0 0x7f340806d990 secure :-1 s=READY pgs=686 cs=0 l=1 rev1=1 crypto rx=0x7f3414036000 tx=0x7f3414030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.353+0000 7f341affd640 1 -- 172.21.15.64:0/1934639588 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f341c1f3e30 con 0x7f341c15a230 2024-03-04T07:33:38.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.533+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f341c068700 con 0x7f341c15a230 2024-03-04T07:33:38.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.534+0000 7f341affd640 1 -- 172.21.15.64:0/1934639588 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v122) v1 ==== 74+0+16644 (secure 0 0 0) 0x7f340c0586a0 con 0x7f341c15a230 2024-03-04T07:33:38.535 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:38.535 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":122,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:33:37.247377+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:33:34.914683+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-04T07:33:36.218855+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f340806b4d0 msgr2=0x7f340806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f340806b4d0 0x7f340806d990 secure :-1 s=READY pgs=686 cs=0 l=1 rev1=1 crypto rx=0x7f3414036000 tx=0x7f3414030000 comp rx=0 tx=0).stop 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f341c15a230 msgr2=0x7f341c15a6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f341c15a230 0x7f341c15a6b0 secure :-1 s=READY pgs=1536 cs=0 l=1 rev1=1 crypto rx=0x7f341c0b6860 tx=0x7f340c00e530 comp rx=0 tx=0).stop 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 shutdown_connections 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f340806b4d0 0x7f340806d990 unknown :-1 s=CLOSED pgs=686 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f341c15a230 0x7f341c15a6b0 unknown :-1 s=CLOSED pgs=1536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f341c0cda70 0x7f341c0cb780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 --2- 172.21.15.64:0/1934639588 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34040039b0 0x7f341c158180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 >> 172.21.15.64:0/1934639588 conn(0x7f341c0d3ca0 msgr2=0x7f341c15f140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 shutdown_connections 2024-03-04T07:33:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.536+0000 7f3424681640 1 -- 172.21.15.64:0/1934639588 wait complete. 2024-03-04T07:33:38.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-04T07:33:38.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.651+0000 7fb63e078640 1 Processor -- start 2024-03-04T07:33:38.652 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.651+0000 7fb63e078640 1 -- start start 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.652+0000 7fb63e078640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb63815f500 0x7fb6380ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.652+0000 7fb63e078640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6380cd310 con 0x7fb6380cf740 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.652+0000 7fb63e078640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb6380cd5b0 con 0x7fb6380c5e30 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.652+0000 7fb63e078640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb63815f8e0 con 0x7fb63815f500 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.652+0000 7fb636ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb6380cf740 0x7fb6380ce840 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54754/0 (socket says 172.21.15.64:54754) 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.652+0000 7fb636ffd640 1 -- 172.21.15.64:0/483955236 learned_addr learned my addr 172.21.15.64:0/483955236 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.652+0000 7fb6377fe640 1 --2- 172.21.15.64:0/483955236 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb63815f500 0x7fb6380ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.653+0000 7fb6367fc640 1 -- 172.21.15.64:0/483955236 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1286764838 0 0) 0x7fb6380cd310 con 0x7fb6380cf740 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.653+0000 7fb6367fc640 1 -- 172.21.15.64:0/483955236 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb624003520 con 0x7fb6380cf740 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.653+0000 7fb6367fc640 1 -- 172.21.15.64:0/483955236 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 680582840 0 0) 0x7fb6380cd5b0 con 0x7fb6380c5e30 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.653+0000 7fb6367fc640 1 -- 172.21.15.64:0/483955236 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb6380cd310 con 0x7fb6380c5e30 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.653+0000 7fb6377fe640 1 -- 172.21.15.64:0/483955236 >> v1:172.21.15.66:6789/0 conn(0x7fb6380c5e30 legacy=0x7fb6380c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.653+0000 7fb6377fe640 1 -- 172.21.15.64:0/483955236 >> v1:172.21.15.64:6789/0 conn(0x7fb6380cf740 legacy=0x7fb6380ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.654 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.653+0000 7fb6377fe640 1 -- 172.21.15.64:0/483955236 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6380cd890 con 0x7fb63815f500 2024-03-04T07:33:38.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb6377fe640 1 --2- 172.21.15.64:0/483955236 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb63815f500 0x7fb6380ce300 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fb628009dc0 tx=0x7fb62802be90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=be8b038a3701b62d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb6367fc640 1 -- 172.21.15.64:0/483955236 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb628039070 con 0x7fb63815f500 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb6367fc640 1 -- 172.21.15.64:0/483955236 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb62802cc40 con 0x7fb63815f500 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb6367fc640 1 -- 172.21.15.64:0/483955236 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb628034440 con 0x7fb63815f500 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb63e078640 1 -- 172.21.15.64:0/483955236 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb63815f500 msgr2=0x7fb6380ce300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb63e078640 1 --2- 172.21.15.64:0/483955236 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb63815f500 0x7fb6380ce300 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fb628009dc0 tx=0x7fb62802be90 comp rx=0 tx=0).stop 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb63e078640 1 -- 172.21.15.64:0/483955236 shutdown_connections 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb63e078640 1 --2- 172.21.15.64:0/483955236 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb63815f500 0x7fb6380ce300 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb63e078640 1 -- 172.21.15.64:0/483955236 >> 172.21.15.64:0/483955236 conn(0x7fb6380b3700 msgr2=0x7fb6380b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb63e078640 1 -- 172.21.15.64:0/483955236 shutdown_connections 2024-03-04T07:33:38.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.654+0000 7fb63e078640 1 -- 172.21.15.64:0/483955236 wait complete. 2024-03-04T07:33:38.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.655+0000 7fb63e078640 1 Processor -- start 2024-03-04T07:33:38.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb63e078640 1 -- start start 2024-03-04T07:33:38.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb63e078640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6380c5e30 0x7fb638151ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb63e078640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6380cf740 0x7fb6380b66d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb63e078640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb63815f500 0x7fb63815e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb63e078640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6380c59a0 con 0x7fb6380c5e30 2024-03-04T07:33:38.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb63e078640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6380c56c0 con 0x7fb63815f500 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb63e078640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6380cef50 con 0x7fb6380cf740 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb6377fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6380c5e30 0x7fb638151ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb6377fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6380c5e30 0x7fb638151ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60632/0 (socket says 172.21.15.64:60632) 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb6377fe640 1 -- 172.21.15.64:0/2583174067 learned_addr learned my addr 172.21.15.64:0/2583174067 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb636ffd640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6380cf740 0x7fb6380b66d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.656+0000 7fb637fff640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb63815f500 0x7fb63815e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb6377fe640 1 -- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6380cf740 msgr2=0x7fb6380b66d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb6377fe640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6380cf740 0x7fb6380b66d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb6377fe640 1 -- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb63815f500 msgr2=0x7fb63815e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb6377fe640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb63815f500 0x7fb63815e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb6377fe640 1 -- 172.21.15.64:0/2583174067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6380c8d70 con 0x7fb6380c5e30 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb637fff640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb63815f500 0x7fb63815e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb6377fe640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6380c5e30 0x7fb638151ff0 secure :-1 s=READY pgs=1538 cs=0 l=1 rev1=1 crypto rx=0x7fb628037040 tx=0x7fb628004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb62ffff640 1 -- 172.21.15.64:0/2583174067 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb628039050 con 0x7fb6380c5e30 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb62ffff640 1 -- 172.21.15.64:0/2583174067 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb628045070 con 0x7fb6380c5e30 2024-03-04T07:33:38.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6380c9050 con 0x7fb6380c5e30 2024-03-04T07:33:38.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb62ffff640 1 -- 172.21.15.64:0/2583174067 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb628009450 con 0x7fb6380c5e30 2024-03-04T07:33:38.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.657+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6380c98f0 con 0x7fb6380c5e30 2024-03-04T07:33:38.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.658+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb6380c9440 con 0x7fb6380c5e30 2024-03-04T07:33:38.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.658+0000 7fb62ffff640 1 -- 172.21.15.64:0/2583174067 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb628040670 con 0x7fb6380c5e30 2024-03-04T07:33:38.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.659+0000 7fb62ffff640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb61806b4d0 0x7fb61806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:38.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.659+0000 7fb62ffff640 1 -- 172.21.15.64:0/2583174067 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(122..122 src has 1..122) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb6280acb20 con 0x7fb6380c5e30 2024-03-04T07:33:38.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.659+0000 7fb636ffd640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb61806b4d0 0x7fb61806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:38.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.659+0000 7fb636ffd640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb61806b4d0 0x7fb61806d990 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7fb6380d0b10 tx=0x7fb620009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:38.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.663+0000 7fb62ffff640 1 -- 172.21.15.64:0/2583174067 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb62807b350 con 0x7fb6380c5e30 2024-03-04T07:33:38.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:38.885+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7fb63809fcf0 con 0x7fb6380c5e30 2024-03-04T07:33:39.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.549+0000 7fb62ffff640 1 -- 172.21.15.64:0/2583174067 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v80) v1 ==== 118+0+0 (secure 0 0 0) 0x7fb62807f3f0 con 0x7fb6380c5e30 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb61806b4d0 msgr2=0x7fb61806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb61806b4d0 0x7fb61806d990 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7fb6380d0b10 tx=0x7fb620009040 comp rx=0 tx=0).stop 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6380c5e30 msgr2=0x7fb638151ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6380c5e30 0x7fb638151ff0 secure :-1 s=READY pgs=1538 cs=0 l=1 rev1=1 crypto rx=0x7fb628037040 tx=0x7fb628004a40 comp rx=0 tx=0).stop 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 shutdown_connections 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb61806b4d0 0x7fb61806d990 unknown :-1 s=CLOSED pgs=687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb63815f500 0x7fb63815e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6380cf740 0x7fb6380b66d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 --2- 172.21.15.64:0/2583174067 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6380c5e30 0x7fb638151ff0 unknown :-1 s=CLOSED pgs=1538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 >> 172.21.15.64:0/2583174067 conn(0x7fb6380b3700 msgr2=0x7fb63806f340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 shutdown_connections 2024-03-04T07:33:39.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.553+0000 7fb63e078640 1 -- 172.21.15.64:0/2583174067 wait complete. 2024-03-04T07:33:39.567 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-03-04T07:33:39.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay false 2024-03-04T07:33:39.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.678+0000 7f29e202a640 1 Processor -- start 2024-03-04T07:33:39.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.678+0000 7f29e202a640 1 -- start start 2024-03-04T07:33:39.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29e202a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0717b0 0x7f29dc071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:39.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29e202a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f29dc0b3060 0x7f29dc0d32a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29e202a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29dc15e4b0 con 0x7f29dc0b2b10 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29e202a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f29dc0d2e10 con 0x7f29dc0b3060 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29e202a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc0b97e0 con 0x7f29dc0717b0 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29dbfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f29dc0b2b10 0x7f29dc0b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54770/0 (socket says 172.21.15.64:54770) 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29dbfff640 1 -- 172.21.15.64:0/2246684691 learned_addr learned my addr 172.21.15.64:0/2246684691 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29daffd640 1 --2- 172.21.15.64:0/2246684691 >> v2:172.21.15.66:3300/0 conn(0x7f29dc0b3060 0x7f29dc0d32a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29db7fe640 1 --2- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0717b0 0x7f29dc071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29da7fc640 1 -- 172.21.15.64:0/2246684691 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3019512232 0 0) 0x7f29d4001490 con 0x7f29dc0b2b10 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29da7fc640 1 -- 172.21.15.64:0/2246684691 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f29c4003520 con 0x7f29dc0b2b10 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29daffd640 1 -- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0717b0 msgr2=0x7f29dc071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:39.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29daffd640 1 --2- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0717b0 0x7f29dc071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29daffd640 1 -- 172.21.15.64:0/2246684691 >> v1:172.21.15.64:6789/0 conn(0x7f29dc0b2b10 legacy=0x7f29dc0b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29daffd640 1 -- 172.21.15.64:0/2246684691 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29dc0b9a80 con 0x7f29dc0b3060 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.679+0000 7f29db7fe640 1 --2- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0717b0 0x7f29dc071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29daffd640 1 --2- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0b3060 0x7f29dc0d32a0 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f29cc0097a0 tx=0x7f29cc02b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7ee523d5ef94a844 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29da7fc640 1 -- 172.21.15.64:0/2246684691 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29cc035020 con 0x7f29dc0b3060 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29da7fc640 1 -- 172.21.15.64:0/2246684691 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29cc031830 con 0x7f29dc0b3060 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29da7fc640 1 -- 172.21.15.64:0/2246684691 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29cc031be0 con 0x7f29dc0b3060 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29e202a640 1 -- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0b3060 msgr2=0x7f29dc0d32a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29e202a640 1 --2- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0b3060 0x7f29dc0d32a0 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f29cc0097a0 tx=0x7f29cc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29e202a640 1 -- 172.21.15.64:0/2246684691 shutdown_connections 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.680+0000 7f29e202a640 1 --2- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0b3060 0x7f29dc0d32a0 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.681+0000 7f29e202a640 1 --2- 172.21.15.64:0/2246684691 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0717b0 0x7f29dc071b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.681+0000 7f29e202a640 1 -- 172.21.15.64:0/2246684691 >> 172.21.15.64:0/2246684691 conn(0x7f29dc0d3d20 msgr2=0x7f29dc0d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:39.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.681+0000 7f29e202a640 1 -- 172.21.15.64:0/2246684691 shutdown_connections 2024-03-04T07:33:39.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.681+0000 7f29e202a640 1 -- 172.21.15.64:0/2246684691 wait complete. 2024-03-04T07:33:39.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.682+0000 7f29e202a640 1 Processor -- start 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.682+0000 7f29e202a640 1 -- start start 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0717b0 0x7f29dc0c96d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0b2b10 0x7f29dc1e33a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29dc0b3060 0x7f29dc14afc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc0b7560 con 0x7f29dc0b3060 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc0d4bf0 con 0x7f29dc0717b0 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29dc0b76e0 con 0x7f29dc0b2b10 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29dc0b3060 0x7f29dc14afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29dc0b3060 0x7f29dc14afc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60642/0 (socket says 172.21.15.64:60642) 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 -- 172.21.15.64:0/505150774 learned_addr learned my addr 172.21.15.64:0/505150774 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29daffd640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0b2b10 0x7f29dc1e33a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:39.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29db7fe640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0717b0 0x7f29dc0c96d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 -- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0b2b10 msgr2=0x7f29dc1e33a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0b2b10 0x7f29dc1e33a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 -- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0717b0 msgr2=0x7f29dc0c96d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0717b0 0x7f29dc0c96d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 -- 172.21.15.64:0/505150774 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29dc158210 con 0x7f29dc0b3060 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29dbfff640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29dc0b3060 0x7f29dc14afc0 secure :-1 s=READY pgs=1540 cs=0 l=1 rev1=1 crypto rx=0x7f29d400cb50 tx=0x7f29d400a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29c3fff640 1 -- 172.21.15.64:0/505150774 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29d4017070 con 0x7f29dc0b3060 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29c3fff640 1 -- 172.21.15.64:0/505150774 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29d400ad40 con 0x7f29dc0b3060 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29dc1584f0 con 0x7f29dc0b3060 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29c3fff640 1 -- 172.21.15.64:0/505150774 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29d4012620 con 0x7f29dc0b3060 2024-03-04T07:33:39.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.683+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f29dc158ab0 con 0x7f29dc0b3060 2024-03-04T07:33:39.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.684+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f29dc06ce40 con 0x7f29dc0b3060 2024-03-04T07:33:39.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.685+0000 7f29c3fff640 1 -- 172.21.15.64:0/505150774 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f29d401a070 con 0x7f29dc0b3060 2024-03-04T07:33:39.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.685+0000 7f29c3fff640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29c806b4d0 0x7f29c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:39.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.685+0000 7f29c3fff640 1 -- 172.21.15.64:0/505150774 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f29d4089fd0 con 0x7f29dc0b3060 2024-03-04T07:33:39.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.685+0000 7f29db7fe640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29c806b4d0 0x7f29c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:39.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.686+0000 7f29db7fe640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29c806b4d0 0x7f29c806d990 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f29d0009560 tx=0x7f29d0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:39.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.690+0000 7f29c3fff640 1 -- 172.21.15.64:0/505150774 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f29dc15e4b0 con 0x7f29dc0b3060 2024-03-04T07:33:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:39.911+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"} v 0) v1 -- 0x7f29dc0b6030 con 0x7f29dc0b3060 2024-03-04T07:33:40.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.910+0000 7f29c3fff640 1 -- 172.21.15.64:0/505150774 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]=0 v81) v1 ==== 122+0+0 (secure 0 0 0) 0x7f29d4058800 con 0x7f29dc0b3060 2024-03-04T07:33:40.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29c806b4d0 msgr2=0x7f29c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:40.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29c806b4d0 0x7f29c806d990 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f29d0009560 tx=0x7f29d0009190 comp rx=0 tx=0).stop 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29dc0b3060 msgr2=0x7f29dc14afc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29dc0b3060 0x7f29dc14afc0 secure :-1 s=READY pgs=1540 cs=0 l=1 rev1=1 crypto rx=0x7f29d400cb50 tx=0x7f29d400a430 comp rx=0 tx=0).stop 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 shutdown_connections 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29c806b4d0 0x7f29c806d990 unknown :-1 s=CLOSED pgs=688 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29dc0b3060 0x7f29dc14afc0 unknown :-1 s=CLOSED pgs=1540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29dc0b2b10 0x7f29dc1e33a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 --2- 172.21.15.64:0/505150774 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29dc0717b0 0x7f29dc0c96d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.912+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 >> 172.21.15.64:0/505150774 conn(0x7f29dc0d3d20 msgr2=0x7f29dc15c140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.913+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 shutdown_connections 2024-03-04T07:33:40.914 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:40.913+0000 7f29e202a640 1 -- 172.21.15.64:0/505150774 wait complete. 2024-03-04T07:33:40.927 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-03-04T07:33:40.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:41.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.033+0000 7fb479ceb640 1 Processor -- start 2024-03-04T07:33:41.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.033+0000 7fb479ceb640 1 -- start start 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb479ceb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb47415df00 0x7fb4740d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb479ceb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb4740b95a0 0x7fb4740b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb479ceb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb4740cea30 con 0x7fb4740b9a30 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb479ceb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4740d2af0 con 0x7fb47415df00 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb479ceb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb4740cf3c0 con 0x7fb4740b95a0 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb4737fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb47415df00 0x7fb4740d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb473fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb4740b9a30 0x7fb4740d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54784/0 (socket says 172.21.15.64:54784) 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb4737fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb47415df00 0x7fb4740d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60654/0 (socket says 172.21.15.64:60654) 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb473fff640 1 -- 172.21.15.64:0/1773026683 learned_addr learned my addr 172.21.15.64:0/1773026683 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:41.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.034+0000 7fb472ffd640 1 --2- 172.21.15.64:0/1773026683 >> v2:172.21.15.66:3300/0 conn(0x7fb4740b95a0 0x7fb4740b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.035+0000 7fb4737fe640 1 -- 172.21.15.64:0/1773026683 >> v2:172.21.15.66:3300/0 conn(0x7fb4740b95a0 msgr2=0x7fb4740b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:41.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.035+0000 7fb4737fe640 1 --2- 172.21.15.64:0/1773026683 >> v2:172.21.15.66:3300/0 conn(0x7fb4740b95a0 0x7fb4740b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.035+0000 7fb4737fe640 1 -- 172.21.15.64:0/1773026683 >> v1:172.21.15.64:6789/0 conn(0x7fb4740b9a30 legacy=0x7fb4740d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.035+0000 7fb4737fe640 1 -- 172.21.15.64:0/1773026683 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4740cf0e0 con 0x7fb47415df00 2024-03-04T07:33:41.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.035+0000 7fb4727fc640 1 -- 172.21.15.64:0/1773026683 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1665599139 0 0) 0x7fb4740cea30 con 0x7fb4740b9a30 2024-03-04T07:33:41.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.035+0000 7fb4737fe640 1 --2- 172.21.15.64:0/1773026683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415df00 0x7fb4740d25b0 secure :-1 s=READY pgs=1542 cs=0 l=1 rev1=1 crypto rx=0x7fb4740d1760 tx=0x7fb46402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=676016c10d6d0912 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb4727fc640 1 -- 172.21.15.64:0/1773026683 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb464035020 con 0x7fb47415df00 2024-03-04T07:33:41.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb4727fc640 1 -- 172.21.15.64:0/1773026683 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb464031830 con 0x7fb47415df00 2024-03-04T07:33:41.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb4727fc640 1 -- 172.21.15.64:0/1773026683 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb464031c00 con 0x7fb47415df00 2024-03-04T07:33:41.038 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 -- 172.21.15.64:0/1773026683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415df00 msgr2=0x7fb4740d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 --2- 172.21.15.64:0/1773026683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415df00 0x7fb4740d25b0 secure :-1 s=READY pgs=1542 cs=0 l=1 rev1=1 crypto rx=0x7fb4740d1760 tx=0x7fb46402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 -- 172.21.15.64:0/1773026683 shutdown_connections 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 --2- 172.21.15.64:0/1773026683 >> v2:172.21.15.66:3300/0 conn(0x7fb4740b95a0 0x7fb4740b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 --2- 172.21.15.64:0/1773026683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415df00 0x7fb4740d25b0 unknown :-1 s=CLOSED pgs=1542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 -- 172.21.15.64:0/1773026683 >> 172.21.15.64:0/1773026683 conn(0x7fb474071960 msgr2=0x7fb474071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 -- 172.21.15.64:0/1773026683 shutdown_connections 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.036+0000 7fb479ceb640 1 -- 172.21.15.64:0/1773026683 wait complete. 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.037+0000 7fb479ceb640 1 Processor -- start 2024-03-04T07:33:41.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb479ceb640 1 -- start start 2024-03-04T07:33:41.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb479ceb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4740b95a0 0x7fb4741f4290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb479ceb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb47415df00 0x7fb4741f0c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb479ceb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415e2e0 0x7fb47415a190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb479ceb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4740cec70 con 0x7fb47415e2e0 2024-03-04T07:33:41.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb479ceb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4740ce750 con 0x7fb47415df00 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb479ceb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb4740cedf0 con 0x7fb4740b95a0 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb473fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415e2e0 0x7fb47415a190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb4737fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4740b95a0 0x7fb4741f4290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.038+0000 7fb473fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415e2e0 0x7fb47415a190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60666/0 (socket says 172.21.15.64:60666) 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb473fff640 1 -- 172.21.15.64:0/2794371380 learned_addr learned my addr 172.21.15.64:0/2794371380 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb472ffd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb47415df00 0x7fb4741f0c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb4737fe640 1 -- 172.21.15.64:0/2794371380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb47415df00 msgr2=0x7fb4741f0c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb4737fe640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb47415df00 0x7fb4741f0c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb4737fe640 1 -- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415e2e0 msgr2=0x7fb47415a190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb4737fe640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415e2e0 0x7fb47415a190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb4737fe640 1 -- 172.21.15.64:0/2794371380 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4741f47d0 con 0x7fb4740b95a0 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb473fff640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415e2e0 0x7fb47415a190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb4737fe640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4740b95a0 0x7fb4741f4290 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fb464033000 tx=0x7fb4640046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb45bfff640 1 -- 172.21.15.64:0/2794371380 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb464040070 con 0x7fb4740b95a0 2024-03-04T07:33:41.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb45bfff640 1 -- 172.21.15.64:0/2794371380 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb46403a830 con 0x7fb4740b95a0 2024-03-04T07:33:41.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb47415e7e0 con 0x7fb4740b95a0 2024-03-04T07:33:41.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb45bfff640 1 -- 172.21.15.64:0/2794371380 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb464035040 con 0x7fb4740b95a0 2024-03-04T07:33:41.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.039+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb47415eda0 con 0x7fb4740b95a0 2024-03-04T07:33:41.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.040+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb474067ff0 con 0x7fb4740b95a0 2024-03-04T07:33:41.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.041+0000 7fb45bfff640 1 -- 172.21.15.64:0/2794371380 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb464038020 con 0x7fb4740b95a0 2024-03-04T07:33:41.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.041+0000 7fb45bfff640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb46006b4d0 0x7fb46006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.041+0000 7fb45bfff640 1 -- 172.21.15.64:0/2794371380 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb4640aba60 con 0x7fb4740b95a0 2024-03-04T07:33:41.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.041+0000 7fb472ffd640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb46006b4d0 0x7fb46006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.042+0000 7fb472ffd640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb46006b4d0 0x7fb46006d990 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7fb4741f6d60 tx=0x7fb468009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.046+0000 7fb45bfff640 1 -- 172.21.15.64:0/2794371380 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb4741f47d0 con 0x7fb4740b95a0 2024-03-04T07:33:41.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.266+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb4740689e0 con 0x7fb4740b95a0 2024-03-04T07:33:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.266+0000 7fb45bfff640 1 -- 172.21.15.64:0/2794371380 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 81 v81) v1 ==== 94+0+4976 (secure 0 0 0) 0x7fb464031800 con 0x7fb4740b95a0 2024-03-04T07:33:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:41.267 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":81,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6713,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3217233400","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3217233400},{"type":"v1","addr":"172.21.15.64:6836","nonce":3217233400}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77},{"gid":6716,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2523962873","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2523962873},{"type":"v1","addr":"172.21.15.64:6837","nonce":2523962873}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77},{"gid":6728,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77}],"filesystems":[{"mdsmap":{"epoch":81,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:39.913726+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8577},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:33:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 81 2024-03-04T07:33:41.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb46006b4d0 msgr2=0x7fb46006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb46006b4d0 0x7fb46006d990 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7fb4741f6d60 tx=0x7fb468009190 comp rx=0 tx=0).stop 2024-03-04T07:33:41.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4740b95a0 msgr2=0x7fb4741f4290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4740b95a0 0x7fb4741f4290 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fb464033000 tx=0x7fb4640046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 shutdown_connections 2024-03-04T07:33:41.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb46006b4d0 0x7fb46006d990 unknown :-1 s=CLOSED pgs=689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb47415e2e0 0x7fb47415a190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb47415df00 0x7fb4741f0c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 --2- 172.21.15.64:0/2794371380 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb4740b95a0 0x7fb4741f4290 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 >> 172.21.15.64:0/2794371380 conn(0x7fb474071960 msgr2=0x7fb4740b8ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:41.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 shutdown_connections 2024-03-04T07:33:41.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.268+0000 7fb479ceb640 1 -- 172.21.15.64:0/2794371380 wait complete. 2024-03-04T07:33:41.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:33:41.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.380+0000 7fa486e0e640 1 Processor -- start 2024-03-04T07:33:41.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.381+0000 7fa486e0e640 1 -- start start 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.381+0000 7fa486e0e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa4800b1020 0x7fa48006af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.381+0000 7fa486e0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4800cd890 0x7fa48015d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.381+0000 7fa486e0e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4800cc0e0 con 0x7fa48006ed10 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.381+0000 7fa486e0e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4800d28b0 con 0x7fa4800b1020 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.381+0000 7fa486e0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4800cf320 con 0x7fa4800cd890 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.381+0000 7fa484baa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa4800b1020 0x7fa48006af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa4853ab640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4800cd890 0x7fa48015d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa484baa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa4800b1020 0x7fa48006af20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60680/0 (socket says 172.21.15.64:60680) 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa484baa640 1 -- 172.21.15.64:0/1130809922 learned_addr learned my addr 172.21.15.64:0/1130809922 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa47ffff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fa48006ed10 0x7fa4800cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49696/0 (socket says 172.21.15.64:49696) 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa484baa640 1 -- 172.21.15.64:0/1130809922 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4800cd890 msgr2=0x7fa48015d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:41.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa484baa640 1 --2- 172.21.15.64:0/1130809922 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4800cd890 0x7fa48015d700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa484baa640 1 -- 172.21.15.64:0/1130809922 >> v1:172.21.15.66:6789/0 conn(0x7fa48006ed10 legacy=0x7fa4800cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa47f7fe640 1 -- 172.21.15.64:0/1130809922 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3087516663 0 0) 0x7fa4800cc0e0 con 0x7fa48006ed10 2024-03-04T07:33:41.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.382+0000 7fa484baa640 1 -- 172.21.15.64:0/1130809922 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4800cc3c0 con 0x7fa4800b1020 2024-03-04T07:33:41.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa484baa640 1 --2- 172.21.15.64:0/1130809922 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa48006af20 secure :-1 s=READY pgs=1543 cs=0 l=1 rev1=1 crypto rx=0x7fa474009870 tx=0x7fa47402b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3ffb522819b493a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa47f7fe640 1 -- 172.21.15.64:0/1130809922 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa474034070 con 0x7fa4800b1020 2024-03-04T07:33:41.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa47f7fe640 1 -- 172.21.15.64:0/1130809922 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa474030840 con 0x7fa4800b1020 2024-03-04T07:33:41.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa47f7fe640 1 -- 172.21.15.64:0/1130809922 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa474030c10 con 0x7fa4800b1020 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 -- 172.21.15.64:0/1130809922 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 msgr2=0x7fa48006af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1130809922 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa48006af20 secure :-1 s=READY pgs=1543 cs=0 l=1 rev1=1 crypto rx=0x7fa474009870 tx=0x7fa47402b980 comp rx=0 tx=0).stop 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 -- 172.21.15.64:0/1130809922 shutdown_connections 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1130809922 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4800cd890 0x7fa48015d700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1130809922 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa48006af20 unknown :-1 s=CLOSED pgs=1543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 -- 172.21.15.64:0/1130809922 >> 172.21.15.64:0/1130809922 conn(0x7fa4800d1a90 msgr2=0x7fa4800d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 -- 172.21.15.64:0/1130809922 shutdown_connections 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.383+0000 7fa486e0e640 1 -- 172.21.15.64:0/1130809922 wait complete. 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.384+0000 7fa486e0e640 1 Processor -- start 2024-03-04T07:33:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa486e0e640 1 -- start start 2024-03-04T07:33:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa486e0e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa48006ed10 0x7fa48015b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa486e0e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa4800cb440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa486e0e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4800cd890 0x7fa4800b18a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa486e0e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4800ccc60 con 0x7fa4800b1020 2024-03-04T07:33:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa486e0e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4800cc980 con 0x7fa4800cd890 2024-03-04T07:33:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa486e0e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4800cdca0 con 0x7fa48006ed10 2024-03-04T07:33:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.385+0000 7fa47ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa4800cb440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa4800cb440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60694/0 (socket says 172.21.15.64:60694) 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa484baa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa48006ed10 0x7fa48015b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 -- 172.21.15.64:0/1976798352 learned_addr learned my addr 172.21.15.64:0/1976798352 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa4853ab640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4800cd890 0x7fa4800b18a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 -- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa48006ed10 msgr2=0x7fa48015b510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa48006ed10 0x7fa48015b510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 -- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4800cd890 msgr2=0x7fa4800b18a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4800cd890 0x7fa4800b18a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 -- 172.21.15.64:0/1976798352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4800cb980 con 0x7fa4800b1020 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa484baa640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa48006ed10 0x7fa48015b510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47ffff640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa4800cb440 secure :-1 s=READY pgs=1544 cs=0 l=1 rev1=1 crypto rx=0x7fa46c00c4a0 tx=0x7fa46c00c960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47d7fa640 1 -- 172.21.15.64:0/1976798352 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa46c018070 con 0x7fa4800b1020 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47d7fa640 1 -- 172.21.15.64:0/1976798352 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa46c0074e0 con 0x7fa4800b1020 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4800c98a0 con 0x7fa4800b1020 2024-03-04T07:33:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa47d7fa640 1 -- 172.21.15.64:0/1976798352 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa46c004a20 con 0x7fa4800b1020 2024-03-04T07:33:41.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.386+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4800c9e60 con 0x7fa4800b1020 2024-03-04T07:33:41.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.387+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa48006fd20 con 0x7fa4800b1020 2024-03-04T07:33:41.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.388+0000 7fa47d7fa640 1 -- 172.21.15.64:0/1976798352 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa46c004bc0 con 0x7fa4800b1020 2024-03-04T07:33:41.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.388+0000 7fa47d7fa640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa46406b4d0 0x7fa46406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.388+0000 7fa47d7fa640 1 -- 172.21.15.64:0/1976798352 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa46c089560 con 0x7fa4800b1020 2024-03-04T07:33:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.388+0000 7fa484baa640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa46406b4d0 0x7fa46406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.389+0000 7fa484baa640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa46406b4d0 0x7fa46406d990 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7fa474036000 tx=0x7fa474008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.392+0000 7fa47d7fa640 1 -- 172.21.15.64:0/1976798352 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa46c057d10 con 0x7fa4800b1020 2024-03-04T07:33:41.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.575+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa48006f2f0 con 0x7fa4800b1020 2024-03-04T07:33:41.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.576+0000 7fa47d7fa640 1 -- 172.21.15.64:0/1976798352 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v123) v1 ==== 74+0+16631 (secure 0 0 0) 0x7fa46c05bdb0 con 0x7fa4800b1020 2024-03-04T07:33:41.577 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:41.577 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":123,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:33:38.899330+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:33:34.914683+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-04T07:33:36.218855+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"123","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:33:41.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa46406b4d0 msgr2=0x7fa46406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa46406b4d0 0x7fa46406d990 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7fa474036000 tx=0x7fa474008030 comp rx=0 tx=0).stop 2024-03-04T07:33:41.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 msgr2=0x7fa4800cb440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa4800cb440 secure :-1 s=READY pgs=1544 cs=0 l=1 rev1=1 crypto rx=0x7fa46c00c4a0 tx=0x7fa46c00c960 comp rx=0 tx=0).stop 2024-03-04T07:33:41.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 shutdown_connections 2024-03-04T07:33:41.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa46406b4d0 0x7fa46406d990 unknown :-1 s=CLOSED pgs=690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4800cd890 0x7fa4800b18a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4800b1020 0x7fa4800cb440 unknown :-1 s=CLOSED pgs=1544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 --2- 172.21.15.64:0/1976798352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa48006ed10 0x7fa48015b510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 >> 172.21.15.64:0/1976798352 conn(0x7fa4800d1a90 msgr2=0x7fa4800af090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:41.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 shutdown_connections 2024-03-04T07:33:41.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.578+0000 7fa486e0e640 1 -- 172.21.15.64:0/1976798352 wait complete. 2024-03-04T07:33:41.590 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-04T07:33:41.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:33:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.691+0000 7f70a7833640 1 Processor -- start 2024-03-04T07:33:41.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.691+0000 7f70a7833640 1 -- start start 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.692+0000 7f70a7833640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a015df00 0x7f70a00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.692+0000 7f70a7833640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f70a00cea30 con 0x7f70a00b9a30 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.692+0000 7f70a7833640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f70a00d2af0 con 0x7f70a00b95a0 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.692+0000 7f70a7833640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70a00cf3c0 con 0x7f70a015df00 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.692+0000 7f70a5dd0640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f70a00b9a30 0x7f70a00d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54800/0 (socket says 172.21.15.64:54800) 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.692+0000 7f70a5dd0640 1 -- 172.21.15.64:0/1205687943 learned_addr learned my addr 172.21.15.64:0/1205687943 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.692+0000 7f70a55cf640 1 --2- 172.21.15.64:0/1205687943 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a015df00 0x7f70a00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.693+0000 7f709ffff640 1 -- 172.21.15.64:0/1205687943 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3306323535 0 0) 0x7f70a00cea30 con 0x7f70a00b9a30 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.693+0000 7f709ffff640 1 -- 172.21.15.64:0/1205687943 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7088003520 con 0x7f70a00b9a30 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.693+0000 7f709ffff640 1 -- 172.21.15.64:0/1205687943 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1989355919 0 0) 0x7f70a00d2af0 con 0x7f70a00b95a0 2024-03-04T07:33:41.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.693+0000 7f709ffff640 1 -- 172.21.15.64:0/1205687943 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f70a00cea30 con 0x7f70a00b95a0 2024-03-04T07:33:41.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.693+0000 7f70a55cf640 1 -- 172.21.15.64:0/1205687943 >> v1:172.21.15.66:6789/0 conn(0x7f70a00b95a0 legacy=0x7f70a00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.693+0000 7f70a55cf640 1 -- 172.21.15.64:0/1205687943 >> v1:172.21.15.64:6789/0 conn(0x7f70a00b9a30 legacy=0x7f70a00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.693+0000 7f70a55cf640 1 -- 172.21.15.64:0/1205687943 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70a00cf0e0 con 0x7f70a015df00 2024-03-04T07:33:41.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.694+0000 7f70a55cf640 1 --2- 172.21.15.64:0/1205687943 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a015df00 0x7f70a00d25b0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f70980097d0 tx=0x7f709802b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=13c87c30cf814f1c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.694+0000 7f709ffff640 1 -- 172.21.15.64:0/1205687943 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7098035020 con 0x7f70a015df00 2024-03-04T07:33:41.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.694+0000 7f709ffff640 1 -- 172.21.15.64:0/1205687943 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7098031890 con 0x7f70a015df00 2024-03-04T07:33:41.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.694+0000 7f709ffff640 1 -- 172.21.15.64:0/1205687943 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7098031c60 con 0x7f70a015df00 2024-03-04T07:33:41.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.695+0000 7f70a7833640 1 -- 172.21.15.64:0/1205687943 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a015df00 msgr2=0x7f70a00d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.695+0000 7f70a7833640 1 --2- 172.21.15.64:0/1205687943 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a015df00 0x7f70a00d25b0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f70980097d0 tx=0x7f709802b930 comp rx=0 tx=0).stop 2024-03-04T07:33:41.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.695+0000 7f70a7833640 1 -- 172.21.15.64:0/1205687943 shutdown_connections 2024-03-04T07:33:41.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.695+0000 7f70a7833640 1 --2- 172.21.15.64:0/1205687943 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a015df00 0x7f70a00d25b0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.695+0000 7f70a7833640 1 -- 172.21.15.64:0/1205687943 >> 172.21.15.64:0/1205687943 conn(0x7f70a0071960 msgr2=0x7f70a0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:41.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.695+0000 7f70a7833640 1 -- 172.21.15.64:0/1205687943 shutdown_connections 2024-03-04T07:33:41.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.695+0000 7f70a7833640 1 -- 172.21.15.64:0/1205687943 wait complete. 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.696+0000 7f70a7833640 1 Processor -- start 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.696+0000 7f70a7833640 1 -- start start 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.696+0000 7f70a7833640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a00b95a0 0x7f70a015e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.696+0000 7f70a7833640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70a015df00 0x7f70a0160b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a7833640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70a015a6f0 0x7f70a0159b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a7833640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70a00cec70 con 0x7f70a015a6f0 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a7833640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70a00ce750 con 0x7f70a015df00 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a7833640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f70a00cedf0 con 0x7f70a00b95a0 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70a015a6f0 0x7f70a0159b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70a015a6f0 0x7f70a0159b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60710/0 (socket says 172.21.15.64:60710) 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 -- 172.21.15.64:0/2418004767 learned_addr learned my addr 172.21.15.64:0/2418004767 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:41.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a55cf640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a00b95a0 0x7f70a015e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a4dce640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70a015df00 0x7f70a0160b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 -- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a00b95a0 msgr2=0x7f70a015e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a00b95a0 0x7f70a015e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 -- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70a015df00 msgr2=0x7f70a0160b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70a015df00 0x7f70a0160b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 -- 172.21.15.64:0/2418004767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70a015f330 con 0x7f70a015a6f0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a5dd0640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70a015a6f0 0x7f70a0159b30 secure :-1 s=READY pgs=1546 cs=0 l=1 rev1=1 crypto rx=0x7f709000cb70 tx=0x7f709000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f709dffb640 1 -- 172.21.15.64:0/2418004767 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7090017070 con 0x7f70a015a6f0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f709dffb640 1 -- 172.21.15.64:0/2418004767 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f709000ace0 con 0x7f70a015a6f0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70a015ed70 con 0x7f70a015a6f0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f709dffb640 1 -- 172.21.15.64:0/2418004767 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7090012620 con 0x7f70a015a6f0 2024-03-04T07:33:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.697+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70a00c7950 con 0x7f70a015a6f0 2024-03-04T07:33:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.699+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f70a0067ff0 con 0x7f70a015a6f0 2024-03-04T07:33:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.699+0000 7f709dffb640 1 -- 172.21.15.64:0/2418004767 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f709001a070 con 0x7f70a015a6f0 2024-03-04T07:33:41.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.699+0000 7f709dffb640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f708c06b500 0x7f708c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:41.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.699+0000 7f709dffb640 1 -- 172.21.15.64:0/2418004767 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7090089e90 con 0x7f70a015a6f0 2024-03-04T07:33:41.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.699+0000 7f70a55cf640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f708c06b500 0x7f708c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:41.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.700+0000 7f70a55cf640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f708c06b500 0x7f708c06d9c0 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7f7098036000 tx=0x7f7098008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:41.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.704+0000 7f709dffb640 1 -- 172.21.15.64:0/2418004767 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7090058750 con 0x7f70a015a6f0 2024-03-04T07:33:41.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.880+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f70a00b7090 con 0x7f708c06b500 2024-03-04T07:33:41.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.886+0000 7f709dffb640 1 -- 172.21.15.64:0/2418004767 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252482 (secure 0 0 0) 0x7f7098087a60 con 0x7f708c06b500 2024-03-04T07:33:41.890 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:41.891 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:33:41.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f708c06b500 msgr2=0x7f708c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f708c06b500 0x7f708c06d9c0 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7f7098036000 tx=0x7f7098008030 comp rx=0 tx=0).stop 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70a015a6f0 msgr2=0x7f70a0159b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70a015a6f0 0x7f70a0159b30 secure :-1 s=READY pgs=1546 cs=0 l=1 rev1=1 crypto rx=0x7f709000cb70 tx=0x7f709000a430 comp rx=0 tx=0).stop 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 shutdown_connections 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f708c06b500 0x7f708c06d9c0 unknown :-1 s=CLOSED pgs=691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f70a015a6f0 0x7f70a0159b30 unknown :-1 s=CLOSED pgs=1546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f70a015df00 0x7f70a0160b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 --2- 172.21.15.64:0/2418004767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f70a00b95a0 0x7f70a015e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 >> 172.21.15.64:0/2418004767 conn(0x7f70a0071960 msgr2=0x7f70a00b8760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 shutdown_connections 2024-03-04T07:33:41.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:41.889+0000 7f70a7833640 1 -- 172.21.15.64:0/2418004767 wait complete. 2024-03-04T07:33:41.901 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":326,"stamp":"2024-03-04T07:33:41.216027+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":165,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":325100,"kb_used_data":13224,"kb_used_omap":12,"kb_used_meta":311731,"kb_avail":754649620,"statfs":{"total":773094113280,"available":772761210880,"internally_reserved":0,"allocated":13541376,"data_stored":6541100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":319213104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":26,"apply_latency_ms":26,"commit_latency_ns":26000000,"apply_latency_ns":26000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.343918"},"pg_stats":[{"pgid":"19.3f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984742+0000","last_change":"2024-03-04T07:33:37.914110+0000","last_active":"2024-03-04T07:33:38.984742+0000","last_peered":"2024-03-04T07:33:38.984742+0000","last_clean":"2024-03-04T07:33:38.984742+0000","last_became_active":"2024-03-04T07:33:37.913537+0000","last_became_peered":"2024-03-04T07:33:37.913537+0000","last_unstale":"2024-03-04T07:33:38.984742+0000","last_undegraded":"2024-03-04T07:33:38.984742+0000","last_fullsized":"2024-03-04T07:33:38.984742+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:32:32.966372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104175+0000","last_change":"2024-03-04T07:33:37.912529+0000","last_active":"2024-03-04T07:33:39.104175+0000","last_peered":"2024-03-04T07:33:39.104175+0000","last_clean":"2024-03-04T07:33:39.104175+0000","last_became_active":"2024-03-04T07:33:37.912195+0000","last_became_peered":"2024-03-04T07:33:37.912195+0000","last_unstale":"2024-03-04T07:33:39.104175+0000","last_undegraded":"2024-03-04T07:33:39.104175+0000","last_fullsized":"2024-03-04T07:33:39.104175+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:18:25.553843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046187+0000","last_change":"2024-03-04T07:33:37.912728+0000","last_active":"2024-03-04T07:33:39.046187+0000","last_peered":"2024-03-04T07:33:39.046187+0000","last_clean":"2024-03-04T07:33:39.046187+0000","last_became_active":"2024-03-04T07:33:37.912440+0000","last_became_peered":"2024-03-04T07:33:37.912440+0000","last_unstale":"2024-03-04T07:33:39.046187+0000","last_undegraded":"2024-03-04T07:33:39.046187+0000","last_fullsized":"2024-03-04T07:33:39.046187+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:18:23.211938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.3c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984544+0000","last_change":"2024-03-04T07:33:37.910256+0000","last_active":"2024-03-04T07:33:38.984544+0000","last_peered":"2024-03-04T07:33:38.984544+0000","last_clean":"2024-03-04T07:33:38.984544+0000","last_became_active":"2024-03-04T07:33:37.909991+0000","last_became_peered":"2024-03-04T07:33:37.909991+0000","last_unstale":"2024-03-04T07:33:38.984544+0000","last_undegraded":"2024-03-04T07:33:38.984544+0000","last_fullsized":"2024-03-04T07:33:38.984544+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:08:01.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913412+0000","last_change":"2024-03-04T07:33:37.913405+0000","last_active":"2024-03-04T07:33:38.913412+0000","last_peered":"2024-03-04T07:33:38.913412+0000","last_clean":"2024-03-04T07:33:38.913412+0000","last_became_active":"2024-03-04T07:33:37.913102+0000","last_became_peered":"2024-03-04T07:33:37.913102+0000","last_unstale":"2024-03-04T07:33:38.913412+0000","last_undegraded":"2024-03-04T07:33:38.913412+0000","last_fullsized":"2024-03-04T07:33:38.913412+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:11:58.665258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.3a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905449+0000","last_change":"2024-03-04T07:33:37.908785+0000","last_active":"2024-03-04T07:33:38.905449+0000","last_peered":"2024-03-04T07:33:38.905449+0000","last_clean":"2024-03-04T07:33:38.905449+0000","last_became_active":"2024-03-04T07:33:37.908481+0000","last_became_peered":"2024-03-04T07:33:37.908481+0000","last_unstale":"2024-03-04T07:33:38.905449+0000","last_undegraded":"2024-03-04T07:33:38.905449+0000","last_fullsized":"2024-03-04T07:33:38.905449+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:18:23.211938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.39","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904939+0000","last_change":"2024-03-04T07:33:37.917400+0000","last_active":"2024-03-04T07:33:38.904939+0000","last_peered":"2024-03-04T07:33:38.904939+0000","last_clean":"2024-03-04T07:33:38.904939+0000","last_became_active":"2024-03-04T07:33:37.917122+0000","last_became_peered":"2024-03-04T07:33:37.917122+0000","last_unstale":"2024-03-04T07:33:38.904939+0000","last_undegraded":"2024-03-04T07:33:38.904939+0000","last_fullsized":"2024-03-04T07:33:38.904939+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:05:45.003617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.38","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045926+0000","last_change":"2024-03-04T07:33:37.909512+0000","last_active":"2024-03-04T07:33:39.045926+0000","last_peered":"2024-03-04T07:33:39.045926+0000","last_clean":"2024-03-04T07:33:39.045926+0000","last_became_active":"2024-03-04T07:33:37.908921+0000","last_became_peered":"2024-03-04T07:33:37.908921+0000","last_unstale":"2024-03-04T07:33:39.045926+0000","last_undegraded":"2024-03-04T07:33:39.045926+0000","last_fullsized":"2024-03-04T07:33:39.045926+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:18:25.553843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.37","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046221+0000","last_change":"2024-03-04T07:33:37.908464+0000","last_active":"2024-03-04T07:33:39.046221+0000","last_peered":"2024-03-04T07:33:39.046221+0000","last_clean":"2024-03-04T07:33:39.046221+0000","last_became_active":"2024-03-04T07:33:37.908121+0000","last_became_peered":"2024-03-04T07:33:37.908121+0000","last_unstale":"2024-03-04T07:33:39.046221+0000","last_undegraded":"2024-03-04T07:33:39.046221+0000","last_fullsized":"2024-03-04T07:33:39.046221+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:00.401735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.36","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104089+0000","last_change":"2024-03-04T07:33:37.912520+0000","last_active":"2024-03-04T07:33:39.104089+0000","last_peered":"2024-03-04T07:33:39.104089+0000","last_clean":"2024-03-04T07:33:39.104089+0000","last_became_active":"2024-03-04T07:33:37.912217+0000","last_became_peered":"2024-03-04T07:33:37.912217+0000","last_unstale":"2024-03-04T07:33:39.104089+0000","last_undegraded":"2024-03-04T07:33:39.104089+0000","last_fullsized":"2024-03-04T07:33:39.104089+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:16.285680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.35","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905137+0000","last_change":"2024-03-04T07:33:37.902831+0000","last_active":"2024-03-04T07:33:38.905137+0000","last_peered":"2024-03-04T07:33:38.905137+0000","last_clean":"2024-03-04T07:33:38.905137+0000","last_became_active":"2024-03-04T07:33:37.902552+0000","last_became_peered":"2024-03-04T07:33:37.902552+0000","last_unstale":"2024-03-04T07:33:38.905137+0000","last_undegraded":"2024-03-04T07:33:38.905137+0000","last_fullsized":"2024-03-04T07:33:38.905137+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:14:43.052402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.34","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045919+0000","last_change":"2024-03-04T07:33:37.909272+0000","last_active":"2024-03-04T07:33:39.045919+0000","last_peered":"2024-03-04T07:33:39.045919+0000","last_clean":"2024-03-04T07:33:39.045919+0000","last_became_active":"2024-03-04T07:33:37.907842+0000","last_became_peered":"2024-03-04T07:33:37.907842+0000","last_unstale":"2024-03-04T07:33:39.045919+0000","last_undegraded":"2024-03-04T07:33:39.045919+0000","last_fullsized":"2024-03-04T07:33:39.045919+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:02.194334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.33","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984521+0000","last_change":"2024-03-04T07:33:37.912671+0000","last_active":"2024-03-04T07:33:38.984521+0000","last_peered":"2024-03-04T07:33:38.984521+0000","last_clean":"2024-03-04T07:33:38.984521+0000","last_became_active":"2024-03-04T07:33:37.912411+0000","last_became_peered":"2024-03-04T07:33:37.912411+0000","last_unstale":"2024-03-04T07:33:38.984521+0000","last_undegraded":"2024-03-04T07:33:38.984521+0000","last_fullsized":"2024-03-04T07:33:38.984521+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:01:40.656467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.32","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103916+0000","last_change":"2024-03-04T07:33:37.903644+0000","last_active":"2024-03-04T07:33:39.103916+0000","last_peered":"2024-03-04T07:33:39.103916+0000","last_clean":"2024-03-04T07:33:39.103916+0000","last_became_active":"2024-03-04T07:33:37.902950+0000","last_became_peered":"2024-03-04T07:33:37.902950+0000","last_unstale":"2024-03-04T07:33:39.103916+0000","last_undegraded":"2024-03-04T07:33:39.103916+0000","last_fullsized":"2024-03-04T07:33:39.103916+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:17:04.630164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.31","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984316+0000","last_change":"2024-03-04T07:33:37.917184+0000","last_active":"2024-03-04T07:33:38.984316+0000","last_peered":"2024-03-04T07:33:38.984316+0000","last_clean":"2024-03-04T07:33:38.984316+0000","last_became_active":"2024-03-04T07:33:37.916900+0000","last_became_peered":"2024-03-04T07:33:37.916900+0000","last_unstale":"2024-03-04T07:33:38.984316+0000","last_undegraded":"2024-03-04T07:33:38.984316+0000","last_fullsized":"2024-03-04T07:33:38.984316+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:15:54.012933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.30","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538385+0000","last_change":"2024-03-04T07:33:37.903213+0000","last_active":"2024-03-04T07:33:39.538385+0000","last_peered":"2024-03-04T07:33:39.538385+0000","last_clean":"2024-03-04T07:33:39.538385+0000","last_became_active":"2024-03-04T07:33:37.902923+0000","last_became_peered":"2024-03-04T07:33:37.902923+0000","last_unstale":"2024-03-04T07:33:39.538385+0000","last_undegraded":"2024-03-04T07:33:39.538385+0000","last_fullsized":"2024-03-04T07:33:39.538385+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:24:32.795500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905521+0000","last_change":"2024-03-04T07:33:37.912006+0000","last_active":"2024-03-04T07:33:38.905521+0000","last_peered":"2024-03-04T07:33:38.905521+0000","last_clean":"2024-03-04T07:33:38.905521+0000","last_became_active":"2024-03-04T07:33:37.911693+0000","last_became_peered":"2024-03-04T07:33:37.911693+0000","last_unstale":"2024-03-04T07:33:38.905521+0000","last_undegraded":"2024-03-04T07:33:38.905521+0000","last_fullsized":"2024-03-04T07:33:38.905521+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:00.401735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.2e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538311+0000","last_change":"2024-03-04T07:33:37.915702+0000","last_active":"2024-03-04T07:33:39.538311+0000","last_peered":"2024-03-04T07:33:39.538311+0000","last_clean":"2024-03-04T07:33:39.538311+0000","last_became_active":"2024-03-04T07:33:37.915436+0000","last_became_peered":"2024-03-04T07:33:37.915436+0000","last_unstale":"2024-03-04T07:33:39.538311+0000","last_undegraded":"2024-03-04T07:33:39.538311+0000","last_fullsized":"2024-03-04T07:33:39.538311+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:11:58.665258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904977+0000","last_change":"2024-03-04T07:33:37.913337+0000","last_active":"2024-03-04T07:33:38.904977+0000","last_peered":"2024-03-04T07:33:38.904977+0000","last_clean":"2024-03-04T07:33:38.904977+0000","last_became_active":"2024-03-04T07:33:37.913066+0000","last_became_peered":"2024-03-04T07:33:37.913066+0000","last_unstale":"2024-03-04T07:33:38.904977+0000","last_undegraded":"2024-03-04T07:33:38.904977+0000","last_fullsized":"2024-03-04T07:33:38.904977+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:15:17.532604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538146+0000","last_change":"2024-03-04T07:33:37.907244+0000","last_active":"2024-03-04T07:33:39.538146+0000","last_peered":"2024-03-04T07:33:39.538146+0000","last_clean":"2024-03-04T07:33:39.538146+0000","last_became_active":"2024-03-04T07:33:37.906932+0000","last_became_peered":"2024-03-04T07:33:37.906932+0000","last_unstale":"2024-03-04T07:33:39.538146+0000","last_undegraded":"2024-03-04T07:33:39.538146+0000","last_fullsized":"2024-03-04T07:33:39.538146+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:40.210998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913375+0000","last_change":"2024-03-04T07:33:37.912993+0000","last_active":"2024-03-04T07:33:38.913375+0000","last_peered":"2024-03-04T07:33:38.913375+0000","last_clean":"2024-03-04T07:33:38.913375+0000","last_became_active":"2024-03-04T07:33:37.912664+0000","last_became_peered":"2024-03-04T07:33:37.912664+0000","last_unstale":"2024-03-04T07:33:38.913375+0000","last_undegraded":"2024-03-04T07:33:38.913375+0000","last_fullsized":"2024-03-04T07:33:38.913375+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:53:17.975211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.2a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904875+0000","last_change":"2024-03-04T07:33:37.903158+0000","last_active":"2024-03-04T07:33:38.904875+0000","last_peered":"2024-03-04T07:33:38.904875+0000","last_clean":"2024-03-04T07:33:38.904875+0000","last_became_active":"2024-03-04T07:33:37.902890+0000","last_became_peered":"2024-03-04T07:33:37.902890+0000","last_unstale":"2024-03-04T07:33:38.904875+0000","last_undegraded":"2024-03-04T07:33:38.904875+0000","last_fullsized":"2024-03-04T07:33:38.904875+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:51:40.231265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.29","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045971+0000","last_change":"2024-03-04T07:33:37.912088+0000","last_active":"2024-03-04T07:33:39.045971+0000","last_peered":"2024-03-04T07:33:39.045971+0000","last_clean":"2024-03-04T07:33:39.045971+0000","last_became_active":"2024-03-04T07:33:37.911354+0000","last_became_peered":"2024-03-04T07:33:37.911354+0000","last_unstale":"2024-03-04T07:33:39.045971+0000","last_undegraded":"2024-03-04T07:33:39.045971+0000","last_fullsized":"2024-03-04T07:33:39.045971+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:00.753618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.28","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904789+0000","last_change":"2024-03-04T07:33:37.900514+0000","last_active":"2024-03-04T07:33:38.904789+0000","last_peered":"2024-03-04T07:33:38.904789+0000","last_clean":"2024-03-04T07:33:38.904789+0000","last_became_active":"2024-03-04T07:33:37.900237+0000","last_became_peered":"2024-03-04T07:33:37.900237+0000","last_unstale":"2024-03-04T07:33:38.904789+0000","last_undegraded":"2024-03-04T07:33:38.904789+0000","last_fullsized":"2024-03-04T07:33:38.904789+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:58.274479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.27","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104058+0000","last_change":"2024-03-04T07:33:37.911065+0000","last_active":"2024-03-04T07:33:39.104058+0000","last_peered":"2024-03-04T07:33:39.104058+0000","last_clean":"2024-03-04T07:33:39.104058+0000","last_became_active":"2024-03-04T07:33:37.910704+0000","last_became_peered":"2024-03-04T07:33:37.910704+0000","last_unstale":"2024-03-04T07:33:39.104058+0000","last_undegraded":"2024-03-04T07:33:39.104058+0000","last_fullsized":"2024-03-04T07:33:39.104058+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:27:37.193234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.26","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538384+0000","last_change":"2024-03-04T07:33:37.905780+0000","last_active":"2024-03-04T07:33:39.538384+0000","last_peered":"2024-03-04T07:33:39.538384+0000","last_clean":"2024-03-04T07:33:39.538384+0000","last_became_active":"2024-03-04T07:33:37.905504+0000","last_became_peered":"2024-03-04T07:33:37.905504+0000","last_unstale":"2024-03-04T07:33:39.538384+0000","last_undegraded":"2024-03-04T07:33:39.538384+0000","last_fullsized":"2024-03-04T07:33:39.538384+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:56.623265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.25","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984590+0000","last_change":"2024-03-04T07:33:37.913487+0000","last_active":"2024-03-04T07:33:38.984590+0000","last_peered":"2024-03-04T07:33:38.984590+0000","last_clean":"2024-03-04T07:33:38.984590+0000","last_became_active":"2024-03-04T07:33:37.913075+0000","last_became_peered":"2024-03-04T07:33:37.913075+0000","last_unstale":"2024-03-04T07:33:38.984590+0000","last_undegraded":"2024-03-04T07:33:38.984590+0000","last_fullsized":"2024-03-04T07:33:38.984590+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:46.139300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.24","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905215+0000","last_change":"2024-03-04T07:33:37.905387+0000","last_active":"2024-03-04T07:33:38.905215+0000","last_peered":"2024-03-04T07:33:38.905215+0000","last_clean":"2024-03-04T07:33:38.905215+0000","last_became_active":"2024-03-04T07:33:37.905113+0000","last_became_peered":"2024-03-04T07:33:37.905113+0000","last_unstale":"2024-03-04T07:33:38.905215+0000","last_undegraded":"2024-03-04T07:33:38.905215+0000","last_fullsized":"2024-03-04T07:33:38.905215+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:18:02.209734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.23","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984463+0000","last_change":"2024-03-04T07:33:37.909222+0000","last_active":"2024-03-04T07:33:38.984463+0000","last_peered":"2024-03-04T07:33:38.984463+0000","last_clean":"2024-03-04T07:33:38.984463+0000","last_became_active":"2024-03-04T07:33:37.908938+0000","last_became_peered":"2024-03-04T07:33:37.908938+0000","last_unstale":"2024-03-04T07:33:38.984463+0000","last_undegraded":"2024-03-04T07:33:38.984463+0000","last_fullsized":"2024-03-04T07:33:38.984463+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:11:07.853604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.22","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984423+0000","last_change":"2024-03-04T07:33:37.909264+0000","last_active":"2024-03-04T07:33:38.984423+0000","last_peered":"2024-03-04T07:33:38.984423+0000","last_clean":"2024-03-04T07:33:38.984423+0000","last_became_active":"2024-03-04T07:33:37.908926+0000","last_became_peered":"2024-03-04T07:33:37.908926+0000","last_unstale":"2024-03-04T07:33:38.984423+0000","last_undegraded":"2024-03-04T07:33:38.984423+0000","last_fullsized":"2024-03-04T07:33:38.984423+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:45:49.809896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.21","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913264+0000","last_change":"2024-03-04T07:33:37.918002+0000","last_active":"2024-03-04T07:33:38.913264+0000","last_peered":"2024-03-04T07:33:38.913264+0000","last_clean":"2024-03-04T07:33:38.913264+0000","last_became_active":"2024-03-04T07:33:37.917713+0000","last_became_peered":"2024-03-04T07:33:37.917713+0000","last_unstale":"2024-03-04T07:33:38.913264+0000","last_undegraded":"2024-03-04T07:33:38.913264+0000","last_fullsized":"2024-03-04T07:33:38.913264+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:20:10.870050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.20","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904957+0000","last_change":"2024-03-04T07:33:37.907998+0000","last_active":"2024-03-04T07:33:38.904957+0000","last_peered":"2024-03-04T07:33:38.904957+0000","last_clean":"2024-03-04T07:33:38.904957+0000","last_became_active":"2024-03-04T07:33:37.907696+0000","last_became_peered":"2024-03-04T07:33:37.907696+0000","last_unstale":"2024-03-04T07:33:38.904957+0000","last_undegraded":"2024-03-04T07:33:38.904957+0000","last_fullsized":"2024-03-04T07:33:38.904957+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:00.753618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.1e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538475+0000","last_change":"2024-03-04T07:33:37.903250+0000","last_active":"2024-03-04T07:33:39.538475+0000","last_peered":"2024-03-04T07:33:39.538475+0000","last_clean":"2024-03-04T07:33:39.538475+0000","last_became_active":"2024-03-04T07:33:37.902982+0000","last_became_peered":"2024-03-04T07:33:37.902982+0000","last_unstale":"2024-03-04T07:33:39.538475+0000","last_undegraded":"2024-03-04T07:33:39.538475+0000","last_fullsized":"2024-03-04T07:33:39.538475+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:20:30.177289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1f","version":"122'4","reported_seq":18,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913555+0000","last_change":"2024-03-04T07:33:36.896087+0000","last_active":"2024-03-04T07:33:38.913555+0000","last_peered":"2024-03-04T07:33:38.913555+0000","last_clean":"2024-03-04T07:33:38.913555+0000","last_became_active":"2024-03-04T07:33:36.895794+0000","last_became_peered":"2024-03-04T07:33:36.895794+0000","last_unstale":"2024-03-04T07:33:38.913555+0000","last_undegraded":"2024-03-04T07:33:38.913555+0000","last_fullsized":"2024-03-04T07:33:38.913555+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:20.643636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.1f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046257+0000","last_change":"2024-03-04T07:33:37.909561+0000","last_active":"2024-03-04T07:33:39.046257+0000","last_peered":"2024-03-04T07:33:39.046257+0000","last_clean":"2024-03-04T07:33:39.046257+0000","last_became_active":"2024-03-04T07:33:37.908713+0000","last_became_peered":"2024-03-04T07:33:37.908713+0000","last_unstale":"2024-03-04T07:33:39.046257+0000","last_undegraded":"2024-03-04T07:33:39.046257+0000","last_fullsized":"2024-03-04T07:33:39.046257+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:09.891054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.1e","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905136+0000","last_change":"2024-03-04T07:33:36.889533+0000","last_active":"2024-03-04T07:33:38.905136+0000","last_peered":"2024-03-04T07:33:38.905136+0000","last_clean":"2024-03-04T07:33:38.905136+0000","last_became_active":"2024-03-04T07:33:36.889212+0000","last_became_peered":"2024-03-04T07:33:36.889212+0000","last_unstale":"2024-03-04T07:33:38.905136+0000","last_undegraded":"2024-03-04T07:33:38.905136+0000","last_fullsized":"2024-03-04T07:33:38.905136+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:32:31.960949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.1c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913462+0000","last_change":"2024-03-04T07:33:37.915174+0000","last_active":"2024-03-04T07:33:38.913462+0000","last_peered":"2024-03-04T07:33:38.913462+0000","last_clean":"2024-03-04T07:33:38.913462+0000","last_became_active":"2024-03-04T07:33:37.914876+0000","last_became_peered":"2024-03-04T07:33:37.914876+0000","last_unstale":"2024-03-04T07:33:38.913462+0000","last_undegraded":"2024-03-04T07:33:38.913462+0000","last_fullsized":"2024-03-04T07:33:38.913462+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:24:32.795500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1d","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905033+0000","last_change":"2024-03-04T07:33:36.887177+0000","last_active":"2024-03-04T07:33:38.905033+0000","last_peered":"2024-03-04T07:33:38.905033+0000","last_clean":"2024-03-04T07:33:38.905033+0000","last_became_active":"2024-03-04T07:33:36.886913+0000","last_became_peered":"2024-03-04T07:33:36.886913+0000","last_unstale":"2024-03-04T07:33:38.905033+0000","last_undegraded":"2024-03-04T07:33:38.905033+0000","last_fullsized":"2024-03-04T07:33:38.905033+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:20:29.171867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.1d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913373+0000","last_change":"2024-03-04T07:33:37.913498+0000","last_active":"2024-03-04T07:33:38.913373+0000","last_peered":"2024-03-04T07:33:38.913373+0000","last_clean":"2024-03-04T07:33:38.913373+0000","last_became_active":"2024-03-04T07:33:37.913176+0000","last_became_peered":"2024-03-04T07:33:37.913176+0000","last_unstale":"2024-03-04T07:33:38.913373+0000","last_undegraded":"2024-03-04T07:33:38.913373+0000","last_fullsized":"2024-03-04T07:33:38.913373+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:40.210998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1c","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104041+0000","last_change":"2024-03-04T07:33:36.892967+0000","last_active":"2024-03-04T07:33:39.104041+0000","last_peered":"2024-03-04T07:33:39.104041+0000","last_clean":"2024-03-04T07:33:39.104041+0000","last_became_active":"2024-03-04T07:33:36.892674+0000","last_became_peered":"2024-03-04T07:33:36.892674+0000","last_unstale":"2024-03-04T07:33:39.104041+0000","last_undegraded":"2024-03-04T07:33:39.104041+0000","last_fullsized":"2024-03-04T07:33:39.104041+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:40.669156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538022+0000","last_change":"2024-03-04T07:33:37.916198+0000","last_active":"2024-03-04T07:33:39.538022+0000","last_peered":"2024-03-04T07:33:39.538022+0000","last_clean":"2024-03-04T07:33:39.538022+0000","last_became_active":"2024-03-04T07:33:37.915917+0000","last_became_peered":"2024-03-04T07:33:37.915917+0000","last_unstale":"2024-03-04T07:33:39.538022+0000","last_undegraded":"2024-03-04T07:33:39.538022+0000","last_fullsized":"2024-03-04T07:33:39.538022+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:44:54.577910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1b","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103794+0000","last_change":"2024-03-04T07:33:36.896139+0000","last_active":"2024-03-04T07:33:39.103794+0000","last_peered":"2024-03-04T07:33:39.103794+0000","last_clean":"2024-03-04T07:33:39.103794+0000","last_became_active":"2024-03-04T07:33:36.895861+0000","last_became_peered":"2024-03-04T07:33:36.895861+0000","last_unstale":"2024-03-04T07:33:39.103794+0000","last_undegraded":"2024-03-04T07:33:39.103794+0000","last_fullsized":"2024-03-04T07:33:39.103794+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:24:47.194656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905446+0000","last_change":"2024-03-04T07:33:37.911148+0000","last_active":"2024-03-04T07:33:38.905446+0000","last_peered":"2024-03-04T07:33:38.905446+0000","last_clean":"2024-03-04T07:33:38.905446+0000","last_became_active":"2024-03-04T07:33:37.910748+0000","last_became_peered":"2024-03-04T07:33:37.910748+0000","last_unstale":"2024-03-04T07:33:38.905446+0000","last_undegraded":"2024-03-04T07:33:38.905446+0000","last_fullsized":"2024-03-04T07:33:38.905446+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:21.649059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.1a","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.537870+0000","last_change":"2024-03-04T07:33:36.896984+0000","last_active":"2024-03-04T07:33:39.537870+0000","last_peered":"2024-03-04T07:33:39.537870+0000","last_clean":"2024-03-04T07:33:39.537870+0000","last_became_active":"2024-03-04T07:33:36.896671+0000","last_became_peered":"2024-03-04T07:33:36.896671+0000","last_unstale":"2024-03-04T07:33:39.537870+0000","last_undegraded":"2024-03-04T07:33:39.537870+0000","last_fullsized":"2024-03-04T07:33:39.537870+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:08.885632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.18","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045982+0000","last_change":"2024-03-04T07:33:37.909215+0000","last_active":"2024-03-04T07:33:39.045982+0000","last_peered":"2024-03-04T07:33:39.045982+0000","last_clean":"2024-03-04T07:33:39.045982+0000","last_became_active":"2024-03-04T07:33:37.908575+0000","last_became_peered":"2024-03-04T07:33:37.908575+0000","last_unstale":"2024-03-04T07:33:39.045982+0000","last_undegraded":"2024-03-04T07:33:39.045982+0000","last_fullsized":"2024-03-04T07:33:39.045982+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:21.649059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.19","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904989+0000","last_change":"2024-03-04T07:33:36.890198+0000","last_active":"2024-03-04T07:33:38.904989+0000","last_peered":"2024-03-04T07:33:38.904989+0000","last_clean":"2024-03-04T07:33:38.904989+0000","last_became_active":"2024-03-04T07:33:36.889931+0000","last_became_peered":"2024-03-04T07:33:36.889931+0000","last_unstale":"2024-03-04T07:33:38.904989+0000","last_undegraded":"2024-03-04T07:33:38.904989+0000","last_fullsized":"2024-03-04T07:33:38.904989+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:04:50.686002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.19","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103845+0000","last_change":"2024-03-04T07:33:37.906198+0000","last_active":"2024-03-04T07:33:39.103845+0000","last_peered":"2024-03-04T07:33:39.103845+0000","last_clean":"2024-03-04T07:33:39.103845+0000","last_became_active":"2024-03-04T07:33:37.905903+0000","last_became_peered":"2024-03-04T07:33:37.905903+0000","last_unstale":"2024-03-04T07:33:39.103845+0000","last_undegraded":"2024-03-04T07:33:39.103845+0000","last_fullsized":"2024-03-04T07:33:39.103845+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:50:06.210807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.18","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103857+0000","last_change":"2024-03-04T07:33:36.899080+0000","last_active":"2024-03-04T07:33:39.103857+0000","last_peered":"2024-03-04T07:33:39.103857+0000","last_clean":"2024-03-04T07:33:39.103857+0000","last_became_active":"2024-03-04T07:33:36.898726+0000","last_became_peered":"2024-03-04T07:33:36.898726+0000","last_unstale":"2024-03-04T07:33:39.103857+0000","last_undegraded":"2024-03-04T07:33:39.103857+0000","last_fullsized":"2024-03-04T07:33:39.103857+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:35.513831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.16","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984679+0000","last_change":"2024-03-04T07:33:37.899684+0000","last_active":"2024-03-04T07:33:38.984679+0000","last_peered":"2024-03-04T07:33:38.984679+0000","last_clean":"2024-03-04T07:33:38.984679+0000","last_became_active":"2024-03-04T07:33:37.899410+0000","last_became_peered":"2024-03-04T07:33:37.899410+0000","last_unstale":"2024-03-04T07:33:38.984679+0000","last_undegraded":"2024-03-04T07:33:38.984679+0000","last_fullsized":"2024-03-04T07:33:38.984679+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:29:35.430461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.17","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905705+0000","last_change":"2024-03-04T07:33:36.892246+0000","last_active":"2024-03-04T07:33:38.905705+0000","last_peered":"2024-03-04T07:33:38.905705+0000","last_clean":"2024-03-04T07:33:38.905705+0000","last_became_active":"2024-03-04T07:33:36.891730+0000","last_became_peered":"2024-03-04T07:33:36.891730+0000","last_unstale":"2024-03-04T07:33:38.905705+0000","last_undegraded":"2024-03-04T07:33:38.905705+0000","last_fullsized":"2024-03-04T07:33:38.905705+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:48:22.544870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.17","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905508+0000","last_change":"2024-03-04T07:33:37.905329+0000","last_active":"2024-03-04T07:33:38.905508+0000","last_peered":"2024-03-04T07:33:38.905508+0000","last_clean":"2024-03-04T07:33:38.905508+0000","last_became_active":"2024-03-04T07:33:37.904515+0000","last_became_peered":"2024-03-04T07:33:37.904515+0000","last_unstale":"2024-03-04T07:33:38.905508+0000","last_undegraded":"2024-03-04T07:33:38.905508+0000","last_fullsized":"2024-03-04T07:33:38.905508+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:23.279200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.16","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538444+0000","last_change":"2024-03-04T07:33:36.895331+0000","last_active":"2024-03-04T07:33:39.538444+0000","last_peered":"2024-03-04T07:33:39.538444+0000","last_clean":"2024-03-04T07:33:39.538444+0000","last_became_active":"2024-03-04T07:33:36.895043+0000","last_became_peered":"2024-03-04T07:33:36.895043+0000","last_unstale":"2024-03-04T07:33:39.538444+0000","last_undegraded":"2024-03-04T07:33:39.538444+0000","last_fullsized":"2024-03-04T07:33:39.538444+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:21:04.079411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.14","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913578+0000","last_change":"2024-03-04T07:33:37.914473+0000","last_active":"2024-03-04T07:33:38.913578+0000","last_peered":"2024-03-04T07:33:38.913578+0000","last_clean":"2024-03-04T07:33:38.913578+0000","last_became_active":"2024-03-04T07:33:37.914137+0000","last_became_peered":"2024-03-04T07:33:37.914137+0000","last_unstale":"2024-03-04T07:33:38.913578+0000","last_undegraded":"2024-03-04T07:33:38.913578+0000","last_fullsized":"2024-03-04T07:33:38.913578+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:28.004888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.15","version":"122'3","reported_seq":17,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905399+0000","last_change":"2024-03-04T07:33:36.886916+0000","last_active":"2024-03-04T07:33:38.905399+0000","last_peered":"2024-03-04T07:33:38.905399+0000","last_clean":"2024-03-04T07:33:38.905399+0000","last_became_active":"2024-03-04T07:33:36.886570+0000","last_became_peered":"2024-03-04T07:33:36.886570+0000","last_unstale":"2024-03-04T07:33:38.905399+0000","last_undegraded":"2024-03-04T07:33:38.905399+0000","last_fullsized":"2024-03-04T07:33:38.905399+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:07.972587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.15","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913413+0000","last_change":"2024-03-04T07:33:37.916073+0000","last_active":"2024-03-04T07:33:38.913413+0000","last_peered":"2024-03-04T07:33:38.913413+0000","last_clean":"2024-03-04T07:33:38.913413+0000","last_became_active":"2024-03-04T07:33:37.915795+0000","last_became_peered":"2024-03-04T07:33:37.915795+0000","last_unstale":"2024-03-04T07:33:38.913413+0000","last_undegraded":"2024-03-04T07:33:38.913413+0000","last_fullsized":"2024-03-04T07:33:38.913413+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:56.623265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.14","version":"122'6","reported_seq":20,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045910+0000","last_change":"2024-03-04T07:33:36.890663+0000","last_active":"2024-03-04T07:33:39.045910+0000","last_peered":"2024-03-04T07:33:39.045910+0000","last_clean":"2024-03-04T07:33:39.045910+0000","last_became_active":"2024-03-04T07:33:36.890191+0000","last_became_peered":"2024-03-04T07:33:36.890191+0000","last_unstale":"2024-03-04T07:33:39.045910+0000","last_undegraded":"2024-03-04T07:33:39.045910+0000","last_fullsized":"2024-03-04T07:33:39.045910+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:04:20.034596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.13","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984648+0000","last_change":"2024-03-04T07:33:37.906790+0000","last_active":"2024-03-04T07:33:38.984648+0000","last_peered":"2024-03-04T07:33:38.984648+0000","last_clean":"2024-03-04T07:33:38.984648+0000","last_became_active":"2024-03-04T07:33:37.906529+0000","last_became_peered":"2024-03-04T07:33:37.906529+0000","last_unstale":"2024-03-04T07:33:38.984648+0000","last_undegraded":"2024-03-04T07:33:38.984648+0000","last_fullsized":"2024-03-04T07:33:38.984648+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:54:13.459318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.12","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984474+0000","last_change":"2024-03-04T07:33:36.890915+0000","last_active":"2024-03-04T07:33:38.984474+0000","last_peered":"2024-03-04T07:33:38.984474+0000","last_clean":"2024-03-04T07:33:38.984474+0000","last_became_active":"2024-03-04T07:33:36.890147+0000","last_became_peered":"2024-03-04T07:33:36.890147+0000","last_unstale":"2024-03-04T07:33:38.984474+0000","last_undegraded":"2024-03-04T07:33:38.984474+0000","last_fullsized":"2024-03-04T07:33:38.984474+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:19:13.138364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.10","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103801+0000","last_change":"2024-03-04T07:33:37.910399+0000","last_active":"2024-03-04T07:33:39.103801+0000","last_peered":"2024-03-04T07:33:39.103801+0000","last_clean":"2024-03-04T07:33:39.103801+0000","last_became_active":"2024-03-04T07:33:37.910019+0000","last_became_peered":"2024-03-04T07:33:37.910019+0000","last_unstale":"2024-03-04T07:33:39.103801+0000","last_undegraded":"2024-03-04T07:33:39.103801+0000","last_fullsized":"2024-03-04T07:33:39.103801+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:34:53.283404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.11","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103958+0000","last_change":"2024-03-04T07:33:36.898941+0000","last_active":"2024-03-04T07:33:39.103958+0000","last_peered":"2024-03-04T07:33:39.103958+0000","last_clean":"2024-03-04T07:33:39.103958+0000","last_became_active":"2024-03-04T07:33:36.898634+0000","last_became_peered":"2024-03-04T07:33:36.898634+0000","last_unstale":"2024-03-04T07:33:39.103958+0000","last_undegraded":"2024-03-04T07:33:39.103958+0000","last_fullsized":"2024-03-04T07:33:39.103958+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:07.972587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.11","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045933+0000","last_change":"2024-03-04T07:33:37.911825+0000","last_active":"2024-03-04T07:33:39.045933+0000","last_peered":"2024-03-04T07:33:39.045933+0000","last_clean":"2024-03-04T07:33:39.045933+0000","last_became_active":"2024-03-04T07:33:37.911010+0000","last_became_peered":"2024-03-04T07:33:37.911010+0000","last_unstale":"2024-03-04T07:33:39.045933+0000","last_undegraded":"2024-03-04T07:33:39.045933+0000","last_fullsized":"2024-03-04T07:33:39.045933+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:05:45.003617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.10","version":"122'8","reported_seq":22,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046025+0000","last_change":"2024-03-04T07:33:36.887508+0000","last_active":"2024-03-04T07:33:39.046025+0000","last_peered":"2024-03-04T07:33:39.046025+0000","last_clean":"2024-03-04T07:33:39.046025+0000","last_became_active":"2024-03-04T07:33:36.887175+0000","last_became_peered":"2024-03-04T07:33:36.887175+0000","last_unstale":"2024-03-04T07:33:39.046025+0000","last_undegraded":"2024-03-04T07:33:39.046025+0000","last_fullsized":"2024-03-04T07:33:39.046025+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:33:16.648451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046237+0000","last_change":"2024-03-04T07:33:37.908194+0000","last_active":"2024-03-04T07:33:39.046237+0000","last_peered":"2024-03-04T07:33:39.046237+0000","last_clean":"2024-03-04T07:33:39.046237+0000","last_became_active":"2024-03-04T07:33:37.907827+0000","last_became_peered":"2024-03-04T07:33:37.907827+0000","last_unstale":"2024-03-04T07:33:39.046237+0000","last_undegraded":"2024-03-04T07:33:39.046237+0000","last_fullsized":"2024-03-04T07:33:39.046237+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:29:53.755911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.f","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905751+0000","last_change":"2024-03-04T07:33:36.891109+0000","last_active":"2024-03-04T07:33:38.905751+0000","last_peered":"2024-03-04T07:33:38.905751+0000","last_clean":"2024-03-04T07:33:38.905751+0000","last_became_active":"2024-03-04T07:33:36.890835+0000","last_became_peered":"2024-03-04T07:33:36.890835+0000","last_unstale":"2024-03-04T07:33:38.905751+0000","last_undegraded":"2024-03-04T07:33:38.905751+0000","last_fullsized":"2024-03-04T07:33:38.905751+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:40.669156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905657+0000","last_change":"2024-03-04T07:33:37.908074+0000","last_active":"2024-03-04T07:33:38.905657+0000","last_peered":"2024-03-04T07:33:38.905657+0000","last_clean":"2024-03-04T07:33:38.905657+0000","last_became_active":"2024-03-04T07:33:37.907792+0000","last_became_peered":"2024-03-04T07:33:37.907792+0000","last_unstale":"2024-03-04T07:33:38.905657+0000","last_undegraded":"2024-03-04T07:33:38.905657+0000","last_fullsized":"2024-03-04T07:33:38.905657+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:49:53.593860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.e","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904981+0000","last_change":"2024-03-04T07:33:36.885659+0000","last_active":"2024-03-04T07:33:38.904981+0000","last_peered":"2024-03-04T07:33:38.904981+0000","last_clean":"2024-03-04T07:33:38.904981+0000","last_became_active":"2024-03-04T07:33:36.885397+0000","last_became_peered":"2024-03-04T07:33:36.885397+0000","last_unstale":"2024-03-04T07:33:38.904981+0000","last_undegraded":"2024-03-04T07:33:38.904981+0000","last_fullsized":"2024-03-04T07:33:38.904981+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:07:59.549085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538314+0000","last_change":"2024-03-04T07:33:37.904029+0000","last_active":"2024-03-04T07:33:39.538314+0000","last_peered":"2024-03-04T07:33:39.538314+0000","last_clean":"2024-03-04T07:33:39.538314+0000","last_became_active":"2024-03-04T07:33:37.903779+0000","last_became_peered":"2024-03-04T07:33:37.903779+0000","last_unstale":"2024-03-04T07:33:39.538314+0000","last_undegraded":"2024-03-04T07:33:39.538314+0000","last_fullsized":"2024-03-04T07:33:39.538314+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:53:17.975211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.d","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905802+0000","last_change":"2024-03-04T07:33:36.886413+0000","last_active":"2024-03-04T07:33:38.905802+0000","last_peered":"2024-03-04T07:33:38.905802+0000","last_clean":"2024-03-04T07:33:38.905802+0000","last_became_active":"2024-03-04T07:33:36.886072+0000","last_became_peered":"2024-03-04T07:33:36.886072+0000","last_unstale":"2024-03-04T07:33:38.905802+0000","last_undegraded":"2024-03-04T07:33:38.905802+0000","last_fullsized":"2024-03-04T07:33:38.905802+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:34:52.277982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104151+0000","last_change":"2024-03-04T07:33:37.905074+0000","last_active":"2024-03-04T07:33:39.104151+0000","last_peered":"2024-03-04T07:33:39.104151+0000","last_clean":"2024-03-04T07:33:39.104151+0000","last_became_active":"2024-03-04T07:33:37.904818+0000","last_became_peered":"2024-03-04T07:33:37.904818+0000","last_unstale":"2024-03-04T07:33:39.104151+0000","last_undegraded":"2024-03-04T07:33:39.104151+0000","last_fullsized":"2024-03-04T07:33:39.104151+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:02.194334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.c","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905135+0000","last_change":"2024-03-04T07:33:36.884363+0000","last_active":"2024-03-04T07:33:38.905135+0000","last_peered":"2024-03-04T07:33:38.905135+0000","last_clean":"2024-03-04T07:33:38.905135+0000","last_became_active":"2024-03-04T07:33:36.884033+0000","last_became_peered":"2024-03-04T07:33:36.884033+0000","last_unstale":"2024-03-04T07:33:38.905135+0000","last_undegraded":"2024-03-04T07:33:38.905135+0000","last_fullsized":"2024-03-04T07:33:38.905135+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:35.513831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984509+0000","last_change":"2024-03-04T07:33:37.914335+0000","last_active":"2024-03-04T07:33:38.984509+0000","last_peered":"2024-03-04T07:33:38.984509+0000","last_clean":"2024-03-04T07:33:38.984509+0000","last_became_active":"2024-03-04T07:33:37.913846+0000","last_became_peered":"2024-03-04T07:33:37.913846+0000","last_unstale":"2024-03-04T07:33:38.984509+0000","last_undegraded":"2024-03-04T07:33:38.984509+0000","last_fullsized":"2024-03-04T07:33:38.984509+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:42.731656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.b","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905883+0000","last_change":"2024-03-04T07:33:36.888826+0000","last_active":"2024-03-04T07:33:38.905883+0000","last_peered":"2024-03-04T07:33:38.905883+0000","last_clean":"2024-03-04T07:33:38.905883+0000","last_became_active":"2024-03-04T07:33:36.888511+0000","last_became_peered":"2024-03-04T07:33:36.888511+0000","last_unstale":"2024-03-04T07:33:38.905883+0000","last_undegraded":"2024-03-04T07:33:38.905883+0000","last_fullsized":"2024-03-04T07:33:38.905883+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:14.074056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905609+0000","last_change":"2024-03-04T07:33:37.911388+0000","last_active":"2024-03-04T07:33:38.905609+0000","last_peered":"2024-03-04T07:33:38.905609+0000","last_clean":"2024-03-04T07:33:38.905609+0000","last_became_active":"2024-03-04T07:33:37.910390+0000","last_became_peered":"2024-03-04T07:33:37.910390+0000","last_unstale":"2024-03-04T07:33:38.905609+0000","last_undegraded":"2024-03-04T07:33:38.905609+0000","last_fullsized":"2024-03-04T07:33:38.905609+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:29:53.755911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.a","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538090+0000","last_change":"2024-03-04T07:33:36.893389+0000","last_active":"2024-03-04T07:33:39.538090+0000","last_peered":"2024-03-04T07:33:39.538090+0000","last_clean":"2024-03-04T07:33:39.538090+0000","last_became_active":"2024-03-04T07:33:36.892895+0000","last_became_peered":"2024-03-04T07:33:36.892895+0000","last_unstale":"2024-03-04T07:33:39.538090+0000","last_undegraded":"2024-03-04T07:33:39.538090+0000","last_fullsized":"2024-03-04T07:33:39.538090+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:55:04.199714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.8","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103814+0000","last_change":"2024-03-04T07:33:37.906257+0000","last_active":"2024-03-04T07:33:39.103814+0000","last_peered":"2024-03-04T07:33:39.103814+0000","last_clean":"2024-03-04T07:33:39.103814+0000","last_became_active":"2024-03-04T07:33:37.905950+0000","last_became_peered":"2024-03-04T07:33:37.905950+0000","last_unstale":"2024-03-04T07:33:39.103814+0000","last_undegraded":"2024-03-04T07:33:39.103814+0000","last_fullsized":"2024-03-04T07:33:39.103814+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:04:51.691425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.9","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.537846+0000","last_change":"2024-03-04T07:33:36.889358+0000","last_active":"2024-03-04T07:33:39.537846+0000","last_peered":"2024-03-04T07:33:39.537846+0000","last_clean":"2024-03-04T07:33:39.537846+0000","last_became_active":"2024-03-04T07:33:36.889089+0000","last_became_peered":"2024-03-04T07:33:36.889089+0000","last_unstale":"2024-03-04T07:33:39.537846+0000","last_undegraded":"2024-03-04T07:33:39.537846+0000","last_fullsized":"2024-03-04T07:33:39.537846+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:10:59.396312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.9","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913266+0000","last_change":"2024-03-04T07:33:37.917234+0000","last_active":"2024-03-04T07:33:38.913266+0000","last_peered":"2024-03-04T07:33:38.913266+0000","last_clean":"2024-03-04T07:33:38.913266+0000","last_became_active":"2024-03-04T07:33:37.916937+0000","last_became_peered":"2024-03-04T07:33:37.916937+0000","last_unstale":"2024-03-04T07:33:38.913266+0000","last_undegraded":"2024-03-04T07:33:38.913266+0000","last_fullsized":"2024-03-04T07:33:38.913266+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:26:55.634674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.8","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904645+0000","last_change":"2024-03-04T07:33:36.887238+0000","last_active":"2024-03-04T07:33:38.904645+0000","last_peered":"2024-03-04T07:33:38.904645+0000","last_clean":"2024-03-04T07:33:38.904645+0000","last_became_active":"2024-03-04T07:33:36.886700+0000","last_became_peered":"2024-03-04T07:33:36.886700+0000","last_unstale":"2024-03-04T07:33:38.904645+0000","last_undegraded":"2024-03-04T07:33:38.904645+0000","last_fullsized":"2024-03-04T07:33:38.904645+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:55.617843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.6","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905976+0000","last_change":"2024-03-04T07:33:37.911299+0000","last_active":"2024-03-04T07:33:38.905976+0000","last_peered":"2024-03-04T07:33:38.905976+0000","last_clean":"2024-03-04T07:33:38.905976+0000","last_became_active":"2024-03-04T07:33:37.910974+0000","last_became_peered":"2024-03-04T07:33:37.910974+0000","last_unstale":"2024-03-04T07:33:38.905976+0000","last_undegraded":"2024-03-04T07:33:38.905976+0000","last_fullsized":"2024-03-04T07:33:38.905976+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:55:05.205137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.7","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984774+0000","last_change":"2024-03-04T07:33:36.891449+0000","last_active":"2024-03-04T07:33:38.984774+0000","last_peered":"2024-03-04T07:33:38.984774+0000","last_clean":"2024-03-04T07:33:38.984774+0000","last_became_active":"2024-03-04T07:33:36.891146+0000","last_became_peered":"2024-03-04T07:33:36.891146+0000","last_unstale":"2024-03-04T07:33:38.984774+0000","last_undegraded":"2024-03-04T07:33:38.984774+0000","last_fullsized":"2024-03-04T07:33:38.984774+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:35:09.536484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.7","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984761+0000","last_change":"2024-03-04T07:33:37.917184+0000","last_active":"2024-03-04T07:33:38.984761+0000","last_peered":"2024-03-04T07:33:38.984761+0000","last_clean":"2024-03-04T07:33:38.984761+0000","last_became_active":"2024-03-04T07:33:37.916913+0000","last_became_peered":"2024-03-04T07:33:37.916913+0000","last_unstale":"2024-03-04T07:33:38.984761+0000","last_undegraded":"2024-03-04T07:33:38.984761+0000","last_fullsized":"2024-03-04T07:33:38.984761+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:15.079478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.6","version":"123'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913515+0000","last_change":"2024-03-04T07:33:36.892044+0000","last_active":"2024-03-04T07:33:38.913515+0000","last_peered":"2024-03-04T07:33:38.913515+0000","last_clean":"2024-03-04T07:33:38.913515+0000","last_became_active":"2024-03-04T07:33:36.891642+0000","last_became_peered":"2024-03-04T07:33:36.891642+0000","last_unstale":"2024-03-04T07:33:38.913515+0000","last_undegraded":"2024-03-04T07:33:38.913515+0000","last_fullsized":"2024-03-04T07:33:38.913515+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:05:43.998194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.4","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104117+0000","last_change":"2024-03-04T07:33:37.915697+0000","last_active":"2024-03-04T07:33:39.104117+0000","last_peered":"2024-03-04T07:33:39.104117+0000","last_clean":"2024-03-04T07:33:39.104117+0000","last_became_active":"2024-03-04T07:33:37.915239+0000","last_became_peered":"2024-03-04T07:33:37.915239+0000","last_unstale":"2024-03-04T07:33:39.104117+0000","last_undegraded":"2024-03-04T07:33:39.104117+0000","last_fullsized":"2024-03-04T07:33:39.104117+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:08.415455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.5","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104119+0000","last_change":"2024-03-04T07:33:36.899212+0000","last_active":"2024-03-04T07:33:39.104119+0000","last_peered":"2024-03-04T07:33:39.104119+0000","last_clean":"2024-03-04T07:33:39.104119+0000","last_became_active":"2024-03-04T07:33:36.898920+0000","last_became_peered":"2024-03-04T07:33:36.898920+0000","last_unstale":"2024-03-04T07:33:39.104119+0000","last_undegraded":"2024-03-04T07:33:39.104119+0000","last_fullsized":"2024-03-04T07:33:39.104119+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:48:22.544870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.5","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905870+0000","last_change":"2024-03-04T07:33:37.911195+0000","last_active":"2024-03-04T07:33:38.905870+0000","last_peered":"2024-03-04T07:33:38.905870+0000","last_clean":"2024-03-04T07:33:38.905870+0000","last_became_active":"2024-03-04T07:33:37.910737+0000","last_became_peered":"2024-03-04T07:33:37.910737+0000","last_unstale":"2024-03-04T07:33:38.905870+0000","last_undegraded":"2024-03-04T07:33:38.905870+0000","last_fullsized":"2024-03-04T07:33:38.905870+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:09.891054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.4","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904946+0000","last_change":"2024-03-04T07:33:36.896850+0000","last_active":"2024-03-04T07:33:38.904946+0000","last_peered":"2024-03-04T07:33:38.904946+0000","last_clean":"2024-03-04T07:33:38.904946+0000","last_became_active":"2024-03-04T07:33:36.896256+0000","last_became_peered":"2024-03-04T07:33:36.896256+0000","last_unstale":"2024-03-04T07:33:38.904946+0000","last_undegraded":"2024-03-04T07:33:38.904946+0000","last_fullsized":"2024-03-04T07:33:38.904946+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:24:31.790077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913270+0000","last_change":"2024-03-04T07:33:37.918848+0000","last_active":"2024-03-04T07:33:38.913270+0000","last_peered":"2024-03-04T07:33:38.913270+0000","last_clean":"2024-03-04T07:33:38.913270+0000","last_became_active":"2024-03-04T07:33:37.918205+0000","last_became_peered":"2024-03-04T07:33:37.918205+0000","last_unstale":"2024-03-04T07:33:38.913270+0000","last_undegraded":"2024-03-04T07:33:38.913270+0000","last_fullsized":"2024-03-04T07:33:38.913270+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:44:54.577910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.3","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103913+0000","last_change":"2024-03-04T07:33:36.899140+0000","last_active":"2024-03-04T07:33:39.103913+0000","last_peered":"2024-03-04T07:33:39.103913+0000","last_clean":"2024-03-04T07:33:39.103913+0000","last_became_active":"2024-03-04T07:33:36.898503+0000","last_became_peered":"2024-03-04T07:33:36.898503+0000","last_unstale":"2024-03-04T07:33:39.103913+0000","last_undegraded":"2024-03-04T07:33:39.103913+0000","last_fullsized":"2024-03-04T07:33:39.103913+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:03:55.760196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905832+0000","last_change":"2024-03-04T07:33:37.903555+0000","last_active":"2024-03-04T07:33:38.905832+0000","last_peered":"2024-03-04T07:33:38.905832+0000","last_clean":"2024-03-04T07:33:38.905832+0000","last_became_active":"2024-03-04T07:33:37.903114+0000","last_became_peered":"2024-03-04T07:33:37.903114+0000","last_unstale":"2024-03-04T07:33:38.905832+0000","last_undegraded":"2024-03-04T07:33:38.905832+0000","last_fullsized":"2024-03-04T07:33:38.905832+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:12:37.381738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.2","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913256+0000","last_change":"2024-03-04T07:33:36.891284+0000","last_active":"2024-03-04T07:33:38.913256+0000","last_peered":"2024-03-04T07:33:38.913256+0000","last_clean":"2024-03-04T07:33:38.913256+0000","last_became_active":"2024-03-04T07:33:36.890998+0000","last_became_peered":"2024-03-04T07:33:36.890998+0000","last_unstale":"2024-03-04T07:33:38.913256+0000","last_undegraded":"2024-03-04T07:33:38.913256+0000","last_fullsized":"2024-03-04T07:33:38.913256+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:18:22.206516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.0","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538393+0000","last_change":"2024-03-04T07:33:37.910265+0000","last_active":"2024-03-04T07:33:39.538393+0000","last_peered":"2024-03-04T07:33:39.538393+0000","last_clean":"2024-03-04T07:33:39.538393+0000","last_became_active":"2024-03-04T07:33:37.909790+0000","last_became_peered":"2024-03-04T07:33:37.909790+0000","last_unstale":"2024-03-04T07:33:39.538393+0000","last_undegraded":"2024-03-04T07:33:39.538393+0000","last_fullsized":"2024-03-04T07:33:39.538393+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:28.004888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.537999+0000","last_change":"2024-03-04T07:33:36.894317+0000","last_active":"2024-03-04T07:33:39.537999+0000","last_peered":"2024-03-04T07:33:39.537999+0000","last_clean":"2024-03-04T07:33:39.537999+0000","last_became_active":"2024-03-04T07:33:36.894013+0000","last_became_peered":"2024-03-04T07:33:36.894013+0000","last_unstale":"2024-03-04T07:33:39.537999+0000","last_undegraded":"2024-03-04T07:33:39.537999+0000","last_fullsized":"2024-03-04T07:33:39.537999+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:29:52.750489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.1","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103904+0000","last_change":"2024-03-04T07:33:37.911071+0000","last_active":"2024-03-04T07:33:39.103904+0000","last_peered":"2024-03-04T07:33:39.103904+0000","last_clean":"2024-03-04T07:33:39.103904+0000","last_became_active":"2024-03-04T07:33:37.910740+0000","last_became_peered":"2024-03-04T07:33:37.910740+0000","last_unstale":"2024-03-04T07:33:39.103904+0000","last_undegraded":"2024-03-04T07:33:39.103904+0000","last_fullsized":"2024-03-04T07:33:39.103904+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:10.425201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.0","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904847+0000","last_change":"2024-03-04T07:33:36.887020+0000","last_active":"2024-03-04T07:33:38.904847+0000","last_peered":"2024-03-04T07:33:38.904847+0000","last_clean":"2024-03-04T07:33:38.904847+0000","last_became_active":"2024-03-04T07:33:36.886326+0000","last_became_peered":"2024-03-04T07:33:36.886326+0000","last_unstale":"2024-03-04T07:33:38.904847+0000","last_undegraded":"2024-03-04T07:33:38.904847+0000","last_fullsized":"2024-03-04T07:33:38.904847+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:26.999465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.12","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913195+0000","last_change":"2024-03-04T07:33:37.919080+0000","last_active":"2024-03-04T07:33:38.913195+0000","last_peered":"2024-03-04T07:33:38.913195+0000","last_clean":"2024-03-04T07:33:38.913195+0000","last_became_active":"2024-03-04T07:33:37.918578+0000","last_became_peered":"2024-03-04T07:33:37.918578+0000","last_unstale":"2024-03-04T07:33:38.913195+0000","last_undegraded":"2024-03-04T07:33:38.913195+0000","last_fullsized":"2024-03-04T07:33:38.913195+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:14:10.409068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.13","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913476+0000","last_change":"2024-03-04T07:33:36.895678+0000","last_active":"2024-03-04T07:33:38.913476+0000","last_peered":"2024-03-04T07:33:38.913476+0000","last_clean":"2024-03-04T07:33:38.913476+0000","last_became_active":"2024-03-04T07:33:36.895368+0000","last_became_peered":"2024-03-04T07:33:36.895368+0000","last_unstale":"2024-03-04T07:33:38.913476+0000","last_undegraded":"2024-03-04T07:33:38.913476+0000","last_fullsized":"2024-03-04T07:33:38.913476+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:57:59.748196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":356,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913012+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:33:38.913012+0000","last_peered":"2024-03-04T07:33:38.913012+0000","last_clean":"2024-03-04T07:33:38.913012+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:33:38.913012+0000","last_undegraded":"2024-03-04T07:33:38.913012+0000","last_fullsized":"2024-03-04T07:33:38.913012+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":19,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":18,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574937,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37452,"kb_used_data":2104,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334388,"statfs":{"total":96636764160,"available":96598413312,"internally_reserved":0,"allocated":2154496,"data_stored":1262061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574936,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41020,"kb_used_data":1512,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330820,"statfs":{"total":96636764160,"available":96594759680,"internally_reserved":0,"allocated":1548288,"data_stored":671055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574936,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41016,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330824,"statfs":{"total":96636764160,"available":96594763776,"internally_reserved":0,"allocated":1544192,"data_stored":669555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574937,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41008,"kb_used_data":1504,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330832,"statfs":{"total":96636764160,"available":96594771968,"internally_reserved":0,"allocated":1540096,"data_stored":669513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574936,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41008,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330832,"statfs":{"total":96636764160,"available":96594771968,"internally_reserved":0,"allocated":1544192,"data_stored":669603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574936,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41016,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330824,"statfs":{"total":96636764160,"available":96594763776,"internally_reserved":0,"allocated":1544192,"data_stored":670059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574936,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41000,"kb_used_data":1500,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330840,"statfs":{"total":96636764160,"available":96594780160,"internally_reserved":0,"allocated":1536000,"data_stored":669443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574936,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41580,"kb_used_data":2080,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330260,"statfs":{"total":96636764160,"available":96594186240,"internally_reserved":0,"allocated":2129920,"data_stored":1259811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":2250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:33:41.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:33:42.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.042+0000 7f1cb92b5640 1 Processor -- start 2024-03-04T07:33:42.046 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.042+0000 7f1cb92b5640 1 -- start start 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb92b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d02e0 0x7f1cb40cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb92b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1cb40b4260 0x7f1cb40b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb92b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1cb40b0dd0 con 0x7f1cb40d06c0 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb92b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1cb40b0b80 con 0x7f1cb40b4260 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb92b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1cb40aeec0 con 0x7f1cb40d02e0 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb2ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d02e0 0x7f1cb40cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb27fc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1cb40b4260 0x7f1cb40b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb2ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d02e0 0x7f1cb40cfa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:39632/0 (socket says 172.21.15.64:39632) 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb2ffd640 1 -- 172.21.15.64:0/2513936607 learned_addr learned my addr 172.21.15.64:0/2513936607 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb27fc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1cb40b4260 0x7f1cb40b1500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:44934/0 (socket says 172.21.15.64:44934) 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.043+0000 7f1cb37fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f1cb40d06c0 0x7f1cb40b1a40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49708/0 (socket says 172.21.15.64:49708) 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.044+0000 7f1cb2ffd640 1 -- 172.21.15.64:0/2513936607 >> v2:172.21.15.66:3300/0 conn(0x7f1cb40b4260 msgr2=0x7f1cb40b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:42.047 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.044+0000 7f1cb2ffd640 1 --2- 172.21.15.64:0/2513936607 >> v2:172.21.15.66:3300/0 conn(0x7f1cb40b4260 0x7f1cb40b1500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.044+0000 7f1cb2ffd640 1 -- 172.21.15.64:0/2513936607 >> v1:172.21.15.66:6789/0 conn(0x7f1cb40d06c0 legacy=0x7f1cb40b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.044+0000 7f1cb2ffd640 1 -- 172.21.15.64:0/2513936607 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cb40b10b0 con 0x7f1cb40d02e0 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.044+0000 7f1cb2ffd640 1 --2- 172.21.15.64:0/2513936607 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d02e0 0x7f1cb40cfa00 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f1cac009870 tx=0x7f1cac02ba20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bb70d9f66e2af236 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb1ffb640 1 -- 172.21.15.64:0/2513936607 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cac02f840 con 0x7f1cb40d02e0 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb1ffb640 1 -- 172.21.15.64:0/2513936607 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cac02f9e0 con 0x7f1cb40d02e0 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb1ffb640 1 -- 172.21.15.64:0/2513936607 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cac02fdb0 con 0x7f1cb40d02e0 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 -- 172.21.15.64:0/2513936607 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d02e0 msgr2=0x7f1cb40cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/2513936607 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d02e0 0x7f1cb40cfa00 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f1cac009870 tx=0x7f1cac02ba20 comp rx=0 tx=0).stop 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 -- 172.21.15.64:0/2513936607 shutdown_connections 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/2513936607 >> v2:172.21.15.66:3300/0 conn(0x7f1cb40b4260 0x7f1cb40b1500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/2513936607 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d02e0 0x7f1cb40cfa00 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 -- 172.21.15.64:0/2513936607 >> 172.21.15.64:0/2513936607 conn(0x7f1cb40b36d0 msgr2=0x7f1cb40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 -- 172.21.15.64:0/2513936607 shutdown_connections 2024-03-04T07:33:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.045+0000 7f1cb92b5640 1 -- 172.21.15.64:0/2513936607 wait complete. 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.046+0000 7f1cb92b5640 1 Processor -- start 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.046+0000 7f1cb92b5640 1 -- start start 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb92b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1cb40b4260 0x7f1cb4154d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb92b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1cb40d02e0 0x7f1cb413fd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb92b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d06c0 0x7f1cb415a410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb92b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cb40b22d0 con 0x7f1cb40d02e0 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb92b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cb40b2150 con 0x7f1cb40b4260 2024-03-04T07:33:42.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb92b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1cb40af350 con 0x7f1cb40d06c0 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb27fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1cb40d02e0 0x7f1cb413fd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb2ffd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1cb40b4260 0x7f1cb4154d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb27fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1cb40d02e0 0x7f1cb413fd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60720/0 (socket says 172.21.15.64:60720) 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb27fc640 1 -- 172.21.15.64:0/883349122 learned_addr learned my addr 172.21.15.64:0/883349122 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.047+0000 7f1cb37fe640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d06c0 0x7f1cb415a410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb2ffd640 1 -- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d06c0 msgr2=0x7f1cb415a410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb2ffd640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d06c0 0x7f1cb415a410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb2ffd640 1 -- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1cb40d02e0 msgr2=0x7f1cb413fd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb2ffd640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1cb40d02e0 0x7f1cb413fd60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb2ffd640 1 -- 172.21.15.64:0/883349122 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cb4153530 con 0x7f1cb40b4260 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb2ffd640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1cb40b4260 0x7f1cb4154d20 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f1cac033000 tx=0x7f1cac004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cab7fe640 1 -- 172.21.15.64:0/883349122 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cac037bc0 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cab7fe640 1 -- 172.21.15.64:0/883349122 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1cac046070 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1cb4154240 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cab7fe640 1 -- 172.21.15.64:0/883349122 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cac041400 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.048+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1cb4154800 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.049+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1cb4068700 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.049+0000 7f1cab7fe640 1 -- 172.21.15.64:0/883349122 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1cac040070 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.050+0000 7f1cab7fe640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c9406b4d0 0x7f1c9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.050+0000 7f1cab7fe640 1 -- 172.21.15.64:0/883349122 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1cac0ac6c0 con 0x7f1cb40b4260 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.050+0000 7f1cb27fc640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c9406b4d0 0x7f1c9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.050+0000 7f1cb27fc640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c9406b4d0 0x7f1c9406d990 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f1c9c006a90 tx=0x7f1c9c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.059+0000 7f1cab7fe640 1 -- 172.21.15.64:0/883349122 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1cac033030 con 0x7f1cb40b4260 2024-03-04T07:33:42.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.236+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f1cb41402a0 con 0x7f1c9406b4d0 2024-03-04T07:33:42.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.242+0000 7f1cab7fe640 1 -- 172.21.15.64:0/883349122 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+252482 (secure 0 0 0) 0x7f1cb41402a0 con 0x7f1c9406b4d0 2024-03-04T07:33:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:42.247 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:33:42.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c9406b4d0 msgr2=0x7f1c9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c9406b4d0 0x7f1c9406d990 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f1c9c006a90 tx=0x7f1c9c009190 comp rx=0 tx=0).stop 2024-03-04T07:33:42.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1cb40b4260 msgr2=0x7f1cb4154d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1cb40b4260 0x7f1cb4154d20 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f1cac033000 tx=0x7f1cac004730 comp rx=0 tx=0).stop 2024-03-04T07:33:42.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 shutdown_connections 2024-03-04T07:33:42.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c9406b4d0 0x7f1c9406d990 unknown :-1 s=CLOSED pgs=692 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1cb40d06c0 0x7f1cb415a410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1cb40d02e0 0x7f1cb413fd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 --2- 172.21.15.64:0/883349122 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1cb40b4260 0x7f1cb4154d20 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 >> 172.21.15.64:0/883349122 conn(0x7f1cb40b36d0 msgr2=0x7f1cb4140710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:42.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 shutdown_connections 2024-03-04T07:33:42.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.245+0000 7f1cb92b5640 1 -- 172.21.15.64:0/883349122 wait complete. 2024-03-04T07:33:42.257 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":326,"stamp":"2024-03-04T07:33:41.216027+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592778,"num_objects":24,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":204,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156,"ondisk_log_size":156,"up":194,"acting":194,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":165,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":325100,"kb_used_data":13224,"kb_used_omap":12,"kb_used_meta":311731,"kb_avail":754649620,"statfs":{"total":773094113280,"available":772761210880,"internally_reserved":0,"allocated":13541376,"data_stored":6541100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":319213104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":26,"apply_latency_ms":26,"commit_latency_ns":26000000,"apply_latency_ns":26000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.343918"},"pg_stats":[{"pgid":"19.3f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984742+0000","last_change":"2024-03-04T07:33:37.914110+0000","last_active":"2024-03-04T07:33:38.984742+0000","last_peered":"2024-03-04T07:33:38.984742+0000","last_clean":"2024-03-04T07:33:38.984742+0000","last_became_active":"2024-03-04T07:33:37.913537+0000","last_became_peered":"2024-03-04T07:33:37.913537+0000","last_unstale":"2024-03-04T07:33:38.984742+0000","last_undegraded":"2024-03-04T07:33:38.984742+0000","last_fullsized":"2024-03-04T07:33:38.984742+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:32:32.966372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104175+0000","last_change":"2024-03-04T07:33:37.912529+0000","last_active":"2024-03-04T07:33:39.104175+0000","last_peered":"2024-03-04T07:33:39.104175+0000","last_clean":"2024-03-04T07:33:39.104175+0000","last_became_active":"2024-03-04T07:33:37.912195+0000","last_became_peered":"2024-03-04T07:33:37.912195+0000","last_unstale":"2024-03-04T07:33:39.104175+0000","last_undegraded":"2024-03-04T07:33:39.104175+0000","last_fullsized":"2024-03-04T07:33:39.104175+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:18:25.553843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046187+0000","last_change":"2024-03-04T07:33:37.912728+0000","last_active":"2024-03-04T07:33:39.046187+0000","last_peered":"2024-03-04T07:33:39.046187+0000","last_clean":"2024-03-04T07:33:39.046187+0000","last_became_active":"2024-03-04T07:33:37.912440+0000","last_became_peered":"2024-03-04T07:33:37.912440+0000","last_unstale":"2024-03-04T07:33:39.046187+0000","last_undegraded":"2024-03-04T07:33:39.046187+0000","last_fullsized":"2024-03-04T07:33:39.046187+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:18:23.211938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.3c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984544+0000","last_change":"2024-03-04T07:33:37.910256+0000","last_active":"2024-03-04T07:33:38.984544+0000","last_peered":"2024-03-04T07:33:38.984544+0000","last_clean":"2024-03-04T07:33:38.984544+0000","last_became_active":"2024-03-04T07:33:37.909991+0000","last_became_peered":"2024-03-04T07:33:37.909991+0000","last_unstale":"2024-03-04T07:33:38.984544+0000","last_undegraded":"2024-03-04T07:33:38.984544+0000","last_fullsized":"2024-03-04T07:33:38.984544+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:08:01.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.3b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913412+0000","last_change":"2024-03-04T07:33:37.913405+0000","last_active":"2024-03-04T07:33:38.913412+0000","last_peered":"2024-03-04T07:33:38.913412+0000","last_clean":"2024-03-04T07:33:38.913412+0000","last_became_active":"2024-03-04T07:33:37.913102+0000","last_became_peered":"2024-03-04T07:33:37.913102+0000","last_unstale":"2024-03-04T07:33:38.913412+0000","last_undegraded":"2024-03-04T07:33:38.913412+0000","last_fullsized":"2024-03-04T07:33:38.913412+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:11:58.665258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.3a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905449+0000","last_change":"2024-03-04T07:33:37.908785+0000","last_active":"2024-03-04T07:33:38.905449+0000","last_peered":"2024-03-04T07:33:38.905449+0000","last_clean":"2024-03-04T07:33:38.905449+0000","last_became_active":"2024-03-04T07:33:37.908481+0000","last_became_peered":"2024-03-04T07:33:37.908481+0000","last_unstale":"2024-03-04T07:33:38.905449+0000","last_undegraded":"2024-03-04T07:33:38.905449+0000","last_fullsized":"2024-03-04T07:33:38.905449+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:18:23.211938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.39","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904939+0000","last_change":"2024-03-04T07:33:37.917400+0000","last_active":"2024-03-04T07:33:38.904939+0000","last_peered":"2024-03-04T07:33:38.904939+0000","last_clean":"2024-03-04T07:33:38.904939+0000","last_became_active":"2024-03-04T07:33:37.917122+0000","last_became_peered":"2024-03-04T07:33:37.917122+0000","last_unstale":"2024-03-04T07:33:38.904939+0000","last_undegraded":"2024-03-04T07:33:38.904939+0000","last_fullsized":"2024-03-04T07:33:38.904939+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:05:45.003617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.38","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045926+0000","last_change":"2024-03-04T07:33:37.909512+0000","last_active":"2024-03-04T07:33:39.045926+0000","last_peered":"2024-03-04T07:33:39.045926+0000","last_clean":"2024-03-04T07:33:39.045926+0000","last_became_active":"2024-03-04T07:33:37.908921+0000","last_became_peered":"2024-03-04T07:33:37.908921+0000","last_unstale":"2024-03-04T07:33:39.045926+0000","last_undegraded":"2024-03-04T07:33:39.045926+0000","last_fullsized":"2024-03-04T07:33:39.045926+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:18:25.553843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.37","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046221+0000","last_change":"2024-03-04T07:33:37.908464+0000","last_active":"2024-03-04T07:33:39.046221+0000","last_peered":"2024-03-04T07:33:39.046221+0000","last_clean":"2024-03-04T07:33:39.046221+0000","last_became_active":"2024-03-04T07:33:37.908121+0000","last_became_peered":"2024-03-04T07:33:37.908121+0000","last_unstale":"2024-03-04T07:33:39.046221+0000","last_undegraded":"2024-03-04T07:33:39.046221+0000","last_fullsized":"2024-03-04T07:33:39.046221+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:00.401735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.36","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104089+0000","last_change":"2024-03-04T07:33:37.912520+0000","last_active":"2024-03-04T07:33:39.104089+0000","last_peered":"2024-03-04T07:33:39.104089+0000","last_clean":"2024-03-04T07:33:39.104089+0000","last_became_active":"2024-03-04T07:33:37.912217+0000","last_became_peered":"2024-03-04T07:33:37.912217+0000","last_unstale":"2024-03-04T07:33:39.104089+0000","last_undegraded":"2024-03-04T07:33:39.104089+0000","last_fullsized":"2024-03-04T07:33:39.104089+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:59:16.285680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.35","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905137+0000","last_change":"2024-03-04T07:33:37.902831+0000","last_active":"2024-03-04T07:33:38.905137+0000","last_peered":"2024-03-04T07:33:38.905137+0000","last_clean":"2024-03-04T07:33:38.905137+0000","last_became_active":"2024-03-04T07:33:37.902552+0000","last_became_peered":"2024-03-04T07:33:37.902552+0000","last_unstale":"2024-03-04T07:33:38.905137+0000","last_undegraded":"2024-03-04T07:33:38.905137+0000","last_fullsized":"2024-03-04T07:33:38.905137+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:14:43.052402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.34","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045919+0000","last_change":"2024-03-04T07:33:37.909272+0000","last_active":"2024-03-04T07:33:39.045919+0000","last_peered":"2024-03-04T07:33:39.045919+0000","last_clean":"2024-03-04T07:33:39.045919+0000","last_became_active":"2024-03-04T07:33:37.907842+0000","last_became_peered":"2024-03-04T07:33:37.907842+0000","last_unstale":"2024-03-04T07:33:39.045919+0000","last_undegraded":"2024-03-04T07:33:39.045919+0000","last_fullsized":"2024-03-04T07:33:39.045919+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:02.194334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.33","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984521+0000","last_change":"2024-03-04T07:33:37.912671+0000","last_active":"2024-03-04T07:33:38.984521+0000","last_peered":"2024-03-04T07:33:38.984521+0000","last_clean":"2024-03-04T07:33:38.984521+0000","last_became_active":"2024-03-04T07:33:37.912411+0000","last_became_peered":"2024-03-04T07:33:37.912411+0000","last_unstale":"2024-03-04T07:33:38.984521+0000","last_undegraded":"2024-03-04T07:33:38.984521+0000","last_fullsized":"2024-03-04T07:33:38.984521+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:01:40.656467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.32","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103916+0000","last_change":"2024-03-04T07:33:37.903644+0000","last_active":"2024-03-04T07:33:39.103916+0000","last_peered":"2024-03-04T07:33:39.103916+0000","last_clean":"2024-03-04T07:33:39.103916+0000","last_became_active":"2024-03-04T07:33:37.902950+0000","last_became_peered":"2024-03-04T07:33:37.902950+0000","last_unstale":"2024-03-04T07:33:39.103916+0000","last_undegraded":"2024-03-04T07:33:39.103916+0000","last_fullsized":"2024-03-04T07:33:39.103916+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:17:04.630164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.31","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984316+0000","last_change":"2024-03-04T07:33:37.917184+0000","last_active":"2024-03-04T07:33:38.984316+0000","last_peered":"2024-03-04T07:33:38.984316+0000","last_clean":"2024-03-04T07:33:38.984316+0000","last_became_active":"2024-03-04T07:33:37.916900+0000","last_became_peered":"2024-03-04T07:33:37.916900+0000","last_unstale":"2024-03-04T07:33:38.984316+0000","last_undegraded":"2024-03-04T07:33:38.984316+0000","last_fullsized":"2024-03-04T07:33:38.984316+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:15:54.012933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.30","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538385+0000","last_change":"2024-03-04T07:33:37.903213+0000","last_active":"2024-03-04T07:33:39.538385+0000","last_peered":"2024-03-04T07:33:39.538385+0000","last_clean":"2024-03-04T07:33:39.538385+0000","last_became_active":"2024-03-04T07:33:37.902923+0000","last_became_peered":"2024-03-04T07:33:37.902923+0000","last_unstale":"2024-03-04T07:33:39.538385+0000","last_undegraded":"2024-03-04T07:33:39.538385+0000","last_fullsized":"2024-03-04T07:33:39.538385+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:24:32.795500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905521+0000","last_change":"2024-03-04T07:33:37.912006+0000","last_active":"2024-03-04T07:33:38.905521+0000","last_peered":"2024-03-04T07:33:38.905521+0000","last_clean":"2024-03-04T07:33:38.905521+0000","last_became_active":"2024-03-04T07:33:37.911693+0000","last_became_peered":"2024-03-04T07:33:37.911693+0000","last_unstale":"2024-03-04T07:33:38.905521+0000","last_undegraded":"2024-03-04T07:33:38.905521+0000","last_fullsized":"2024-03-04T07:33:38.905521+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:11:00.401735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.2e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538311+0000","last_change":"2024-03-04T07:33:37.915702+0000","last_active":"2024-03-04T07:33:39.538311+0000","last_peered":"2024-03-04T07:33:39.538311+0000","last_clean":"2024-03-04T07:33:39.538311+0000","last_became_active":"2024-03-04T07:33:37.915436+0000","last_became_peered":"2024-03-04T07:33:37.915436+0000","last_unstale":"2024-03-04T07:33:39.538311+0000","last_undegraded":"2024-03-04T07:33:39.538311+0000","last_fullsized":"2024-03-04T07:33:39.538311+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:11:58.665258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904977+0000","last_change":"2024-03-04T07:33:37.913337+0000","last_active":"2024-03-04T07:33:38.904977+0000","last_peered":"2024-03-04T07:33:38.904977+0000","last_clean":"2024-03-04T07:33:38.904977+0000","last_became_active":"2024-03-04T07:33:37.913066+0000","last_became_peered":"2024-03-04T07:33:37.913066+0000","last_unstale":"2024-03-04T07:33:38.904977+0000","last_undegraded":"2024-03-04T07:33:38.904977+0000","last_fullsized":"2024-03-04T07:33:38.904977+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:15:17.532604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538146+0000","last_change":"2024-03-04T07:33:37.907244+0000","last_active":"2024-03-04T07:33:39.538146+0000","last_peered":"2024-03-04T07:33:39.538146+0000","last_clean":"2024-03-04T07:33:39.538146+0000","last_became_active":"2024-03-04T07:33:37.906932+0000","last_became_peered":"2024-03-04T07:33:37.906932+0000","last_unstale":"2024-03-04T07:33:39.538146+0000","last_undegraded":"2024-03-04T07:33:39.538146+0000","last_fullsized":"2024-03-04T07:33:39.538146+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:40.210998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.2b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913375+0000","last_change":"2024-03-04T07:33:37.912993+0000","last_active":"2024-03-04T07:33:38.913375+0000","last_peered":"2024-03-04T07:33:38.913375+0000","last_clean":"2024-03-04T07:33:38.913375+0000","last_became_active":"2024-03-04T07:33:37.912664+0000","last_became_peered":"2024-03-04T07:33:37.912664+0000","last_unstale":"2024-03-04T07:33:38.913375+0000","last_undegraded":"2024-03-04T07:33:38.913375+0000","last_fullsized":"2024-03-04T07:33:38.913375+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:53:17.975211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.2a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904875+0000","last_change":"2024-03-04T07:33:37.903158+0000","last_active":"2024-03-04T07:33:38.904875+0000","last_peered":"2024-03-04T07:33:38.904875+0000","last_clean":"2024-03-04T07:33:38.904875+0000","last_became_active":"2024-03-04T07:33:37.902890+0000","last_became_peered":"2024-03-04T07:33:37.902890+0000","last_unstale":"2024-03-04T07:33:38.904875+0000","last_undegraded":"2024-03-04T07:33:38.904875+0000","last_fullsized":"2024-03-04T07:33:38.904875+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:51:40.231265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.29","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045971+0000","last_change":"2024-03-04T07:33:37.912088+0000","last_active":"2024-03-04T07:33:39.045971+0000","last_peered":"2024-03-04T07:33:39.045971+0000","last_clean":"2024-03-04T07:33:39.045971+0000","last_became_active":"2024-03-04T07:33:37.911354+0000","last_became_peered":"2024-03-04T07:33:37.911354+0000","last_unstale":"2024-03-04T07:33:39.045971+0000","last_undegraded":"2024-03-04T07:33:39.045971+0000","last_fullsized":"2024-03-04T07:33:39.045971+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:00.753618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.28","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904789+0000","last_change":"2024-03-04T07:33:37.900514+0000","last_active":"2024-03-04T07:33:38.904789+0000","last_peered":"2024-03-04T07:33:38.904789+0000","last_clean":"2024-03-04T07:33:38.904789+0000","last_became_active":"2024-03-04T07:33:37.900237+0000","last_became_peered":"2024-03-04T07:33:37.900237+0000","last_unstale":"2024-03-04T07:33:38.904789+0000","last_undegraded":"2024-03-04T07:33:38.904789+0000","last_fullsized":"2024-03-04T07:33:38.904789+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:30:58.274479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.27","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104058+0000","last_change":"2024-03-04T07:33:37.911065+0000","last_active":"2024-03-04T07:33:39.104058+0000","last_peered":"2024-03-04T07:33:39.104058+0000","last_clean":"2024-03-04T07:33:39.104058+0000","last_became_active":"2024-03-04T07:33:37.910704+0000","last_became_peered":"2024-03-04T07:33:37.910704+0000","last_unstale":"2024-03-04T07:33:39.104058+0000","last_undegraded":"2024-03-04T07:33:39.104058+0000","last_fullsized":"2024-03-04T07:33:39.104058+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:27:37.193234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.26","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538384+0000","last_change":"2024-03-04T07:33:37.905780+0000","last_active":"2024-03-04T07:33:39.538384+0000","last_peered":"2024-03-04T07:33:39.538384+0000","last_clean":"2024-03-04T07:33:39.538384+0000","last_became_active":"2024-03-04T07:33:37.905504+0000","last_became_peered":"2024-03-04T07:33:37.905504+0000","last_unstale":"2024-03-04T07:33:39.538384+0000","last_undegraded":"2024-03-04T07:33:39.538384+0000","last_fullsized":"2024-03-04T07:33:39.538384+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:56.623265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.25","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984590+0000","last_change":"2024-03-04T07:33:37.913487+0000","last_active":"2024-03-04T07:33:38.984590+0000","last_peered":"2024-03-04T07:33:38.984590+0000","last_clean":"2024-03-04T07:33:38.984590+0000","last_became_active":"2024-03-04T07:33:37.913075+0000","last_became_peered":"2024-03-04T07:33:37.913075+0000","last_unstale":"2024-03-04T07:33:38.984590+0000","last_undegraded":"2024-03-04T07:33:38.984590+0000","last_fullsized":"2024-03-04T07:33:38.984590+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:52:46.139300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.24","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905215+0000","last_change":"2024-03-04T07:33:37.905387+0000","last_active":"2024-03-04T07:33:38.905215+0000","last_peered":"2024-03-04T07:33:38.905215+0000","last_clean":"2024-03-04T07:33:38.905215+0000","last_became_active":"2024-03-04T07:33:37.905113+0000","last_became_peered":"2024-03-04T07:33:37.905113+0000","last_unstale":"2024-03-04T07:33:38.905215+0000","last_undegraded":"2024-03-04T07:33:38.905215+0000","last_fullsized":"2024-03-04T07:33:38.905215+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:18:02.209734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.23","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984463+0000","last_change":"2024-03-04T07:33:37.909222+0000","last_active":"2024-03-04T07:33:38.984463+0000","last_peered":"2024-03-04T07:33:38.984463+0000","last_clean":"2024-03-04T07:33:38.984463+0000","last_became_active":"2024-03-04T07:33:37.908938+0000","last_became_peered":"2024-03-04T07:33:37.908938+0000","last_unstale":"2024-03-04T07:33:38.984463+0000","last_undegraded":"2024-03-04T07:33:38.984463+0000","last_fullsized":"2024-03-04T07:33:38.984463+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:11:07.853604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.22","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984423+0000","last_change":"2024-03-04T07:33:37.909264+0000","last_active":"2024-03-04T07:33:38.984423+0000","last_peered":"2024-03-04T07:33:38.984423+0000","last_clean":"2024-03-04T07:33:38.984423+0000","last_became_active":"2024-03-04T07:33:37.908926+0000","last_became_peered":"2024-03-04T07:33:37.908926+0000","last_unstale":"2024-03-04T07:33:38.984423+0000","last_undegraded":"2024-03-04T07:33:38.984423+0000","last_fullsized":"2024-03-04T07:33:38.984423+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:45:49.809896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.21","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913264+0000","last_change":"2024-03-04T07:33:37.918002+0000","last_active":"2024-03-04T07:33:38.913264+0000","last_peered":"2024-03-04T07:33:38.913264+0000","last_clean":"2024-03-04T07:33:38.913264+0000","last_became_active":"2024-03-04T07:33:37.917713+0000","last_became_peered":"2024-03-04T07:33:37.917713+0000","last_unstale":"2024-03-04T07:33:38.913264+0000","last_undegraded":"2024-03-04T07:33:38.913264+0000","last_fullsized":"2024-03-04T07:33:38.913264+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:20:10.870050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.20","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904957+0000","last_change":"2024-03-04T07:33:37.907998+0000","last_active":"2024-03-04T07:33:38.904957+0000","last_peered":"2024-03-04T07:33:38.904957+0000","last_clean":"2024-03-04T07:33:38.904957+0000","last_became_active":"2024-03-04T07:33:37.907696+0000","last_became_peered":"2024-03-04T07:33:37.907696+0000","last_unstale":"2024-03-04T07:33:38.904957+0000","last_undegraded":"2024-03-04T07:33:38.904957+0000","last_fullsized":"2024-03-04T07:33:38.904957+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:58:00.753618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.1e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538475+0000","last_change":"2024-03-04T07:33:37.903250+0000","last_active":"2024-03-04T07:33:39.538475+0000","last_peered":"2024-03-04T07:33:39.538475+0000","last_clean":"2024-03-04T07:33:39.538475+0000","last_became_active":"2024-03-04T07:33:37.902982+0000","last_became_peered":"2024-03-04T07:33:37.902982+0000","last_unstale":"2024-03-04T07:33:39.538475+0000","last_undegraded":"2024-03-04T07:33:39.538475+0000","last_fullsized":"2024-03-04T07:33:39.538475+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:20:30.177289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1f","version":"122'4","reported_seq":18,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913555+0000","last_change":"2024-03-04T07:33:36.896087+0000","last_active":"2024-03-04T07:33:38.913555+0000","last_peered":"2024-03-04T07:33:38.913555+0000","last_clean":"2024-03-04T07:33:38.913555+0000","last_became_active":"2024-03-04T07:33:36.895794+0000","last_became_peered":"2024-03-04T07:33:36.895794+0000","last_unstale":"2024-03-04T07:33:38.913555+0000","last_undegraded":"2024-03-04T07:33:38.913555+0000","last_fullsized":"2024-03-04T07:33:38.913555+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:20.643636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.1f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046257+0000","last_change":"2024-03-04T07:33:37.909561+0000","last_active":"2024-03-04T07:33:39.046257+0000","last_peered":"2024-03-04T07:33:39.046257+0000","last_clean":"2024-03-04T07:33:39.046257+0000","last_became_active":"2024-03-04T07:33:37.908713+0000","last_became_peered":"2024-03-04T07:33:37.908713+0000","last_unstale":"2024-03-04T07:33:39.046257+0000","last_undegraded":"2024-03-04T07:33:39.046257+0000","last_fullsized":"2024-03-04T07:33:39.046257+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:09.891054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.1e","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905136+0000","last_change":"2024-03-04T07:33:36.889533+0000","last_active":"2024-03-04T07:33:38.905136+0000","last_peered":"2024-03-04T07:33:38.905136+0000","last_clean":"2024-03-04T07:33:38.905136+0000","last_became_active":"2024-03-04T07:33:36.889212+0000","last_became_peered":"2024-03-04T07:33:36.889212+0000","last_unstale":"2024-03-04T07:33:38.905136+0000","last_undegraded":"2024-03-04T07:33:38.905136+0000","last_fullsized":"2024-03-04T07:33:38.905136+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:32:31.960949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.1c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913462+0000","last_change":"2024-03-04T07:33:37.915174+0000","last_active":"2024-03-04T07:33:38.913462+0000","last_peered":"2024-03-04T07:33:38.913462+0000","last_clean":"2024-03-04T07:33:38.913462+0000","last_became_active":"2024-03-04T07:33:37.914876+0000","last_became_peered":"2024-03-04T07:33:37.914876+0000","last_unstale":"2024-03-04T07:33:38.913462+0000","last_undegraded":"2024-03-04T07:33:38.913462+0000","last_fullsized":"2024-03-04T07:33:38.913462+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:24:32.795500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1d","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905033+0000","last_change":"2024-03-04T07:33:36.887177+0000","last_active":"2024-03-04T07:33:38.905033+0000","last_peered":"2024-03-04T07:33:38.905033+0000","last_clean":"2024-03-04T07:33:38.905033+0000","last_became_active":"2024-03-04T07:33:36.886913+0000","last_became_peered":"2024-03-04T07:33:36.886913+0000","last_unstale":"2024-03-04T07:33:38.905033+0000","last_undegraded":"2024-03-04T07:33:38.905033+0000","last_fullsized":"2024-03-04T07:33:38.905033+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:20:29.171867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.1d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913373+0000","last_change":"2024-03-04T07:33:37.913498+0000","last_active":"2024-03-04T07:33:38.913373+0000","last_peered":"2024-03-04T07:33:38.913373+0000","last_clean":"2024-03-04T07:33:38.913373+0000","last_became_active":"2024-03-04T07:33:37.913176+0000","last_became_peered":"2024-03-04T07:33:37.913176+0000","last_unstale":"2024-03-04T07:33:38.913373+0000","last_undegraded":"2024-03-04T07:33:38.913373+0000","last_fullsized":"2024-03-04T07:33:38.913373+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:35:40.210998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.1c","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104041+0000","last_change":"2024-03-04T07:33:36.892967+0000","last_active":"2024-03-04T07:33:39.104041+0000","last_peered":"2024-03-04T07:33:39.104041+0000","last_clean":"2024-03-04T07:33:39.104041+0000","last_became_active":"2024-03-04T07:33:36.892674+0000","last_became_peered":"2024-03-04T07:33:36.892674+0000","last_unstale":"2024-03-04T07:33:39.104041+0000","last_undegraded":"2024-03-04T07:33:39.104041+0000","last_fullsized":"2024-03-04T07:33:39.104041+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:40.669156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538022+0000","last_change":"2024-03-04T07:33:37.916198+0000","last_active":"2024-03-04T07:33:39.538022+0000","last_peered":"2024-03-04T07:33:39.538022+0000","last_clean":"2024-03-04T07:33:39.538022+0000","last_became_active":"2024-03-04T07:33:37.915917+0000","last_became_peered":"2024-03-04T07:33:37.915917+0000","last_unstale":"2024-03-04T07:33:39.538022+0000","last_undegraded":"2024-03-04T07:33:39.538022+0000","last_fullsized":"2024-03-04T07:33:39.538022+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:44:54.577910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1b","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103794+0000","last_change":"2024-03-04T07:33:36.896139+0000","last_active":"2024-03-04T07:33:39.103794+0000","last_peered":"2024-03-04T07:33:39.103794+0000","last_clean":"2024-03-04T07:33:39.103794+0000","last_became_active":"2024-03-04T07:33:36.895861+0000","last_became_peered":"2024-03-04T07:33:36.895861+0000","last_unstale":"2024-03-04T07:33:39.103794+0000","last_undegraded":"2024-03-04T07:33:39.103794+0000","last_fullsized":"2024-03-04T07:33:39.103794+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:24:47.194656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.1b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905446+0000","last_change":"2024-03-04T07:33:37.911148+0000","last_active":"2024-03-04T07:33:38.905446+0000","last_peered":"2024-03-04T07:33:38.905446+0000","last_clean":"2024-03-04T07:33:38.905446+0000","last_became_active":"2024-03-04T07:33:37.910748+0000","last_became_peered":"2024-03-04T07:33:37.910748+0000","last_unstale":"2024-03-04T07:33:38.905446+0000","last_undegraded":"2024-03-04T07:33:38.905446+0000","last_fullsized":"2024-03-04T07:33:38.905446+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:21.649059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.1a","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.537870+0000","last_change":"2024-03-04T07:33:36.896984+0000","last_active":"2024-03-04T07:33:39.537870+0000","last_peered":"2024-03-04T07:33:39.537870+0000","last_clean":"2024-03-04T07:33:39.537870+0000","last_became_active":"2024-03-04T07:33:36.896671+0000","last_became_peered":"2024-03-04T07:33:36.896671+0000","last_unstale":"2024-03-04T07:33:39.537870+0000","last_undegraded":"2024-03-04T07:33:39.537870+0000","last_fullsized":"2024-03-04T07:33:39.537870+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:08.885632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.18","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045982+0000","last_change":"2024-03-04T07:33:37.909215+0000","last_active":"2024-03-04T07:33:39.045982+0000","last_peered":"2024-03-04T07:33:39.045982+0000","last_clean":"2024-03-04T07:33:39.045982+0000","last_became_active":"2024-03-04T07:33:37.908575+0000","last_became_peered":"2024-03-04T07:33:37.908575+0000","last_unstale":"2024-03-04T07:33:39.045982+0000","last_undegraded":"2024-03-04T07:33:39.045982+0000","last_fullsized":"2024-03-04T07:33:39.045982+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:21.649059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.19","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904989+0000","last_change":"2024-03-04T07:33:36.890198+0000","last_active":"2024-03-04T07:33:38.904989+0000","last_peered":"2024-03-04T07:33:38.904989+0000","last_clean":"2024-03-04T07:33:38.904989+0000","last_became_active":"2024-03-04T07:33:36.889931+0000","last_became_peered":"2024-03-04T07:33:36.889931+0000","last_unstale":"2024-03-04T07:33:38.904989+0000","last_undegraded":"2024-03-04T07:33:38.904989+0000","last_fullsized":"2024-03-04T07:33:38.904989+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:04:50.686002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.19","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103845+0000","last_change":"2024-03-04T07:33:37.906198+0000","last_active":"2024-03-04T07:33:39.103845+0000","last_peered":"2024-03-04T07:33:39.103845+0000","last_clean":"2024-03-04T07:33:39.103845+0000","last_became_active":"2024-03-04T07:33:37.905903+0000","last_became_peered":"2024-03-04T07:33:37.905903+0000","last_unstale":"2024-03-04T07:33:39.103845+0000","last_undegraded":"2024-03-04T07:33:39.103845+0000","last_fullsized":"2024-03-04T07:33:39.103845+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:50:06.210807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.18","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103857+0000","last_change":"2024-03-04T07:33:36.899080+0000","last_active":"2024-03-04T07:33:39.103857+0000","last_peered":"2024-03-04T07:33:39.103857+0000","last_clean":"2024-03-04T07:33:39.103857+0000","last_became_active":"2024-03-04T07:33:36.898726+0000","last_became_peered":"2024-03-04T07:33:36.898726+0000","last_unstale":"2024-03-04T07:33:39.103857+0000","last_undegraded":"2024-03-04T07:33:39.103857+0000","last_fullsized":"2024-03-04T07:33:39.103857+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:35.513831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.16","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984679+0000","last_change":"2024-03-04T07:33:37.899684+0000","last_active":"2024-03-04T07:33:38.984679+0000","last_peered":"2024-03-04T07:33:38.984679+0000","last_clean":"2024-03-04T07:33:38.984679+0000","last_became_active":"2024-03-04T07:33:37.899410+0000","last_became_peered":"2024-03-04T07:33:37.899410+0000","last_unstale":"2024-03-04T07:33:38.984679+0000","last_undegraded":"2024-03-04T07:33:38.984679+0000","last_fullsized":"2024-03-04T07:33:38.984679+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:29:35.430461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.17","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905705+0000","last_change":"2024-03-04T07:33:36.892246+0000","last_active":"2024-03-04T07:33:38.905705+0000","last_peered":"2024-03-04T07:33:38.905705+0000","last_clean":"2024-03-04T07:33:38.905705+0000","last_became_active":"2024-03-04T07:33:36.891730+0000","last_became_peered":"2024-03-04T07:33:36.891730+0000","last_unstale":"2024-03-04T07:33:38.905705+0000","last_undegraded":"2024-03-04T07:33:38.905705+0000","last_fullsized":"2024-03-04T07:33:38.905705+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:48:22.544870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.17","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905508+0000","last_change":"2024-03-04T07:33:37.905329+0000","last_active":"2024-03-04T07:33:38.905508+0000","last_peered":"2024-03-04T07:33:38.905508+0000","last_clean":"2024-03-04T07:33:38.905508+0000","last_became_active":"2024-03-04T07:33:37.904515+0000","last_became_peered":"2024-03-04T07:33:37.904515+0000","last_unstale":"2024-03-04T07:33:38.905508+0000","last_undegraded":"2024-03-04T07:33:38.905508+0000","last_fullsized":"2024-03-04T07:33:38.905508+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:13:23.279200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.16","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538444+0000","last_change":"2024-03-04T07:33:36.895331+0000","last_active":"2024-03-04T07:33:39.538444+0000","last_peered":"2024-03-04T07:33:39.538444+0000","last_clean":"2024-03-04T07:33:39.538444+0000","last_became_active":"2024-03-04T07:33:36.895043+0000","last_became_peered":"2024-03-04T07:33:36.895043+0000","last_unstale":"2024-03-04T07:33:39.538444+0000","last_undegraded":"2024-03-04T07:33:39.538444+0000","last_fullsized":"2024-03-04T07:33:39.538444+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:21:04.079411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.14","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913578+0000","last_change":"2024-03-04T07:33:37.914473+0000","last_active":"2024-03-04T07:33:38.913578+0000","last_peered":"2024-03-04T07:33:38.913578+0000","last_clean":"2024-03-04T07:33:38.913578+0000","last_became_active":"2024-03-04T07:33:37.914137+0000","last_became_peered":"2024-03-04T07:33:37.914137+0000","last_unstale":"2024-03-04T07:33:38.913578+0000","last_undegraded":"2024-03-04T07:33:38.913578+0000","last_fullsized":"2024-03-04T07:33:38.913578+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:28.004888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.15","version":"122'3","reported_seq":17,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905399+0000","last_change":"2024-03-04T07:33:36.886916+0000","last_active":"2024-03-04T07:33:38.905399+0000","last_peered":"2024-03-04T07:33:38.905399+0000","last_clean":"2024-03-04T07:33:38.905399+0000","last_became_active":"2024-03-04T07:33:36.886570+0000","last_became_peered":"2024-03-04T07:33:36.886570+0000","last_unstale":"2024-03-04T07:33:38.905399+0000","last_undegraded":"2024-03-04T07:33:38.905399+0000","last_fullsized":"2024-03-04T07:33:38.905399+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:07.972587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.15","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913413+0000","last_change":"2024-03-04T07:33:37.916073+0000","last_active":"2024-03-04T07:33:38.913413+0000","last_peered":"2024-03-04T07:33:38.913413+0000","last_clean":"2024-03-04T07:33:38.913413+0000","last_became_active":"2024-03-04T07:33:37.915795+0000","last_became_peered":"2024-03-04T07:33:37.915795+0000","last_unstale":"2024-03-04T07:33:38.913413+0000","last_undegraded":"2024-03-04T07:33:38.913413+0000","last_fullsized":"2024-03-04T07:33:38.913413+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:56.623265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.14","version":"122'6","reported_seq":20,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045910+0000","last_change":"2024-03-04T07:33:36.890663+0000","last_active":"2024-03-04T07:33:39.045910+0000","last_peered":"2024-03-04T07:33:39.045910+0000","last_clean":"2024-03-04T07:33:39.045910+0000","last_became_active":"2024-03-04T07:33:36.890191+0000","last_became_peered":"2024-03-04T07:33:36.890191+0000","last_unstale":"2024-03-04T07:33:39.045910+0000","last_undegraded":"2024-03-04T07:33:39.045910+0000","last_fullsized":"2024-03-04T07:33:39.045910+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:04:20.034596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.13","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984648+0000","last_change":"2024-03-04T07:33:37.906790+0000","last_active":"2024-03-04T07:33:38.984648+0000","last_peered":"2024-03-04T07:33:38.984648+0000","last_clean":"2024-03-04T07:33:38.984648+0000","last_became_active":"2024-03-04T07:33:37.906529+0000","last_became_peered":"2024-03-04T07:33:37.906529+0000","last_unstale":"2024-03-04T07:33:38.984648+0000","last_undegraded":"2024-03-04T07:33:38.984648+0000","last_fullsized":"2024-03-04T07:33:38.984648+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:54:13.459318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.12","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984474+0000","last_change":"2024-03-04T07:33:36.890915+0000","last_active":"2024-03-04T07:33:38.984474+0000","last_peered":"2024-03-04T07:33:38.984474+0000","last_clean":"2024-03-04T07:33:38.984474+0000","last_became_active":"2024-03-04T07:33:36.890147+0000","last_became_peered":"2024-03-04T07:33:36.890147+0000","last_unstale":"2024-03-04T07:33:38.984474+0000","last_undegraded":"2024-03-04T07:33:38.984474+0000","last_fullsized":"2024-03-04T07:33:38.984474+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:19:13.138364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.10","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103801+0000","last_change":"2024-03-04T07:33:37.910399+0000","last_active":"2024-03-04T07:33:39.103801+0000","last_peered":"2024-03-04T07:33:39.103801+0000","last_clean":"2024-03-04T07:33:39.103801+0000","last_became_active":"2024-03-04T07:33:37.910019+0000","last_became_peered":"2024-03-04T07:33:37.910019+0000","last_unstale":"2024-03-04T07:33:39.103801+0000","last_undegraded":"2024-03-04T07:33:39.103801+0000","last_fullsized":"2024-03-04T07:33:39.103801+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:34:53.283404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.11","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103958+0000","last_change":"2024-03-04T07:33:36.898941+0000","last_active":"2024-03-04T07:33:39.103958+0000","last_peered":"2024-03-04T07:33:39.103958+0000","last_clean":"2024-03-04T07:33:39.103958+0000","last_became_active":"2024-03-04T07:33:36.898634+0000","last_became_peered":"2024-03-04T07:33:36.898634+0000","last_unstale":"2024-03-04T07:33:39.103958+0000","last_undegraded":"2024-03-04T07:33:39.103958+0000","last_fullsized":"2024-03-04T07:33:39.103958+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:46:07.972587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.11","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.045933+0000","last_change":"2024-03-04T07:33:37.911825+0000","last_active":"2024-03-04T07:33:39.045933+0000","last_peered":"2024-03-04T07:33:39.045933+0000","last_clean":"2024-03-04T07:33:39.045933+0000","last_became_active":"2024-03-04T07:33:37.911010+0000","last_became_peered":"2024-03-04T07:33:37.911010+0000","last_unstale":"2024-03-04T07:33:39.045933+0000","last_undegraded":"2024-03-04T07:33:39.045933+0000","last_fullsized":"2024-03-04T07:33:39.045933+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:05:45.003617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.10","version":"122'8","reported_seq":22,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046025+0000","last_change":"2024-03-04T07:33:36.887508+0000","last_active":"2024-03-04T07:33:39.046025+0000","last_peered":"2024-03-04T07:33:39.046025+0000","last_clean":"2024-03-04T07:33:39.046025+0000","last_became_active":"2024-03-04T07:33:36.887175+0000","last_became_peered":"2024-03-04T07:33:36.887175+0000","last_unstale":"2024-03-04T07:33:39.046025+0000","last_undegraded":"2024-03-04T07:33:39.046025+0000","last_fullsized":"2024-03-04T07:33:39.046025+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T11:33:16.648451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"19.e","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.046237+0000","last_change":"2024-03-04T07:33:37.908194+0000","last_active":"2024-03-04T07:33:39.046237+0000","last_peered":"2024-03-04T07:33:39.046237+0000","last_clean":"2024-03-04T07:33:39.046237+0000","last_became_active":"2024-03-04T07:33:37.907827+0000","last_became_peered":"2024-03-04T07:33:37.907827+0000","last_unstale":"2024-03-04T07:33:39.046237+0000","last_undegraded":"2024-03-04T07:33:39.046237+0000","last_fullsized":"2024-03-04T07:33:39.046237+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:29:53.755911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"18.f","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905751+0000","last_change":"2024-03-04T07:33:36.891109+0000","last_active":"2024-03-04T07:33:38.905751+0000","last_peered":"2024-03-04T07:33:38.905751+0000","last_clean":"2024-03-04T07:33:38.905751+0000","last_became_active":"2024-03-04T07:33:36.890835+0000","last_became_peered":"2024-03-04T07:33:36.890835+0000","last_unstale":"2024-03-04T07:33:38.905751+0000","last_undegraded":"2024-03-04T07:33:38.905751+0000","last_fullsized":"2024-03-04T07:33:38.905751+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:40.669156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.f","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905657+0000","last_change":"2024-03-04T07:33:37.908074+0000","last_active":"2024-03-04T07:33:38.905657+0000","last_peered":"2024-03-04T07:33:38.905657+0000","last_clean":"2024-03-04T07:33:38.905657+0000","last_became_active":"2024-03-04T07:33:37.907792+0000","last_became_peered":"2024-03-04T07:33:37.907792+0000","last_unstale":"2024-03-04T07:33:38.905657+0000","last_undegraded":"2024-03-04T07:33:38.905657+0000","last_fullsized":"2024-03-04T07:33:38.905657+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:49:53.593860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.e","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904981+0000","last_change":"2024-03-04T07:33:36.885659+0000","last_active":"2024-03-04T07:33:38.904981+0000","last_peered":"2024-03-04T07:33:38.904981+0000","last_clean":"2024-03-04T07:33:38.904981+0000","last_became_active":"2024-03-04T07:33:36.885397+0000","last_became_peered":"2024-03-04T07:33:36.885397+0000","last_unstale":"2024-03-04T07:33:38.904981+0000","last_undegraded":"2024-03-04T07:33:38.904981+0000","last_fullsized":"2024-03-04T07:33:38.904981+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:07:59.549085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.c","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538314+0000","last_change":"2024-03-04T07:33:37.904029+0000","last_active":"2024-03-04T07:33:39.538314+0000","last_peered":"2024-03-04T07:33:39.538314+0000","last_clean":"2024-03-04T07:33:39.538314+0000","last_became_active":"2024-03-04T07:33:37.903779+0000","last_became_peered":"2024-03-04T07:33:37.903779+0000","last_unstale":"2024-03-04T07:33:39.538314+0000","last_undegraded":"2024-03-04T07:33:39.538314+0000","last_fullsized":"2024-03-04T07:33:39.538314+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T07:53:17.975211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.d","version":"122'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905802+0000","last_change":"2024-03-04T07:33:36.886413+0000","last_active":"2024-03-04T07:33:38.905802+0000","last_peered":"2024-03-04T07:33:38.905802+0000","last_clean":"2024-03-04T07:33:38.905802+0000","last_became_active":"2024-03-04T07:33:36.886072+0000","last_became_peered":"2024-03-04T07:33:36.886072+0000","last_unstale":"2024-03-04T07:33:38.905802+0000","last_undegraded":"2024-03-04T07:33:38.905802+0000","last_fullsized":"2024-03-04T07:33:38.905802+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:34:52.277982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"19.d","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104151+0000","last_change":"2024-03-04T07:33:37.905074+0000","last_active":"2024-03-04T07:33:39.104151+0000","last_peered":"2024-03-04T07:33:39.104151+0000","last_clean":"2024-03-04T07:33:39.104151+0000","last_became_active":"2024-03-04T07:33:37.904818+0000","last_became_peered":"2024-03-04T07:33:37.904818+0000","last_unstale":"2024-03-04T07:33:39.104151+0000","last_undegraded":"2024-03-04T07:33:39.104151+0000","last_fullsized":"2024-03-04T07:33:39.104151+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:02:02.194334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.c","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905135+0000","last_change":"2024-03-04T07:33:36.884363+0000","last_active":"2024-03-04T07:33:38.905135+0000","last_peered":"2024-03-04T07:33:38.905135+0000","last_clean":"2024-03-04T07:33:38.905135+0000","last_became_active":"2024-03-04T07:33:36.884033+0000","last_became_peered":"2024-03-04T07:33:36.884033+0000","last_unstale":"2024-03-04T07:33:38.905135+0000","last_undegraded":"2024-03-04T07:33:38.905135+0000","last_fullsized":"2024-03-04T07:33:38.905135+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:35.513831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.a","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984509+0000","last_change":"2024-03-04T07:33:37.914335+0000","last_active":"2024-03-04T07:33:38.984509+0000","last_peered":"2024-03-04T07:33:38.984509+0000","last_clean":"2024-03-04T07:33:38.984509+0000","last_became_active":"2024-03-04T07:33:37.913846+0000","last_became_peered":"2024-03-04T07:33:37.913846+0000","last_unstale":"2024-03-04T07:33:38.984509+0000","last_undegraded":"2024-03-04T07:33:38.984509+0000","last_fullsized":"2024-03-04T07:33:38.984509+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:15:42.731656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.b","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905883+0000","last_change":"2024-03-04T07:33:36.888826+0000","last_active":"2024-03-04T07:33:38.905883+0000","last_peered":"2024-03-04T07:33:38.905883+0000","last_clean":"2024-03-04T07:33:38.905883+0000","last_became_active":"2024-03-04T07:33:36.888511+0000","last_became_peered":"2024-03-04T07:33:36.888511+0000","last_unstale":"2024-03-04T07:33:38.905883+0000","last_undegraded":"2024-03-04T07:33:38.905883+0000","last_fullsized":"2024-03-04T07:33:38.905883+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:14.074056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"19.b","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905609+0000","last_change":"2024-03-04T07:33:37.911388+0000","last_active":"2024-03-04T07:33:38.905609+0000","last_peered":"2024-03-04T07:33:38.905609+0000","last_clean":"2024-03-04T07:33:38.905609+0000","last_became_active":"2024-03-04T07:33:37.910390+0000","last_became_peered":"2024-03-04T07:33:37.910390+0000","last_unstale":"2024-03-04T07:33:38.905609+0000","last_undegraded":"2024-03-04T07:33:38.905609+0000","last_fullsized":"2024-03-04T07:33:38.905609+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:29:53.755911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.a","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538090+0000","last_change":"2024-03-04T07:33:36.893389+0000","last_active":"2024-03-04T07:33:39.538090+0000","last_peered":"2024-03-04T07:33:39.538090+0000","last_clean":"2024-03-04T07:33:39.538090+0000","last_became_active":"2024-03-04T07:33:36.892895+0000","last_became_peered":"2024-03-04T07:33:36.892895+0000","last_unstale":"2024-03-04T07:33:39.538090+0000","last_undegraded":"2024-03-04T07:33:39.538090+0000","last_fullsized":"2024-03-04T07:33:39.538090+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:55:04.199714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.8","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103814+0000","last_change":"2024-03-04T07:33:37.906257+0000","last_active":"2024-03-04T07:33:39.103814+0000","last_peered":"2024-03-04T07:33:39.103814+0000","last_clean":"2024-03-04T07:33:39.103814+0000","last_became_active":"2024-03-04T07:33:37.905950+0000","last_became_peered":"2024-03-04T07:33:37.905950+0000","last_unstale":"2024-03-04T07:33:39.103814+0000","last_undegraded":"2024-03-04T07:33:39.103814+0000","last_fullsized":"2024-03-04T07:33:39.103814+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:04:51.691425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.9","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.537846+0000","last_change":"2024-03-04T07:33:36.889358+0000","last_active":"2024-03-04T07:33:39.537846+0000","last_peered":"2024-03-04T07:33:39.537846+0000","last_clean":"2024-03-04T07:33:39.537846+0000","last_became_active":"2024-03-04T07:33:36.889089+0000","last_became_peered":"2024-03-04T07:33:36.889089+0000","last_unstale":"2024-03-04T07:33:39.537846+0000","last_undegraded":"2024-03-04T07:33:39.537846+0000","last_fullsized":"2024-03-04T07:33:39.537846+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T15:10:59.396312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.9","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913266+0000","last_change":"2024-03-04T07:33:37.917234+0000","last_active":"2024-03-04T07:33:38.913266+0000","last_peered":"2024-03-04T07:33:38.913266+0000","last_clean":"2024-03-04T07:33:38.913266+0000","last_became_active":"2024-03-04T07:33:37.916937+0000","last_became_peered":"2024-03-04T07:33:37.916937+0000","last_unstale":"2024-03-04T07:33:38.913266+0000","last_undegraded":"2024-03-04T07:33:38.913266+0000","last_fullsized":"2024-03-04T07:33:38.913266+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:26:55.634674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.8","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904645+0000","last_change":"2024-03-04T07:33:36.887238+0000","last_active":"2024-03-04T07:33:38.904645+0000","last_peered":"2024-03-04T07:33:38.904645+0000","last_clean":"2024-03-04T07:33:38.904645+0000","last_became_active":"2024-03-04T07:33:36.886700+0000","last_became_peered":"2024-03-04T07:33:36.886700+0000","last_unstale":"2024-03-04T07:33:38.904645+0000","last_undegraded":"2024-03-04T07:33:38.904645+0000","last_fullsized":"2024-03-04T07:33:38.904645+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T17:36:55.617843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.6","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905976+0000","last_change":"2024-03-04T07:33:37.911299+0000","last_active":"2024-03-04T07:33:38.905976+0000","last_peered":"2024-03-04T07:33:38.905976+0000","last_clean":"2024-03-04T07:33:38.905976+0000","last_became_active":"2024-03-04T07:33:37.910974+0000","last_became_peered":"2024-03-04T07:33:37.910974+0000","last_unstale":"2024-03-04T07:33:38.905976+0000","last_undegraded":"2024-03-04T07:33:38.905976+0000","last_fullsized":"2024-03-04T07:33:38.905976+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:55:05.205137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.7","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984774+0000","last_change":"2024-03-04T07:33:36.891449+0000","last_active":"2024-03-04T07:33:38.984774+0000","last_peered":"2024-03-04T07:33:38.984774+0000","last_clean":"2024-03-04T07:33:38.984774+0000","last_became_active":"2024-03-04T07:33:36.891146+0000","last_became_peered":"2024-03-04T07:33:36.891146+0000","last_unstale":"2024-03-04T07:33:38.984774+0000","last_undegraded":"2024-03-04T07:33:38.984774+0000","last_fullsized":"2024-03-04T07:33:38.984774+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T10:35:09.536484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"19.7","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.984761+0000","last_change":"2024-03-04T07:33:37.917184+0000","last_active":"2024-03-04T07:33:38.984761+0000","last_peered":"2024-03-04T07:33:38.984761+0000","last_clean":"2024-03-04T07:33:38.984761+0000","last_became_active":"2024-03-04T07:33:37.916913+0000","last_became_peered":"2024-03-04T07:33:37.916913+0000","last_unstale":"2024-03-04T07:33:38.984761+0000","last_undegraded":"2024-03-04T07:33:38.984761+0000","last_fullsized":"2024-03-04T07:33:38.984761+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:46:15.079478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"18.6","version":"123'1","reported_seq":15,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913515+0000","last_change":"2024-03-04T07:33:36.892044+0000","last_active":"2024-03-04T07:33:38.913515+0000","last_peered":"2024-03-04T07:33:38.913515+0000","last_clean":"2024-03-04T07:33:38.913515+0000","last_became_active":"2024-03-04T07:33:36.891642+0000","last_became_peered":"2024-03-04T07:33:36.891642+0000","last_unstale":"2024-03-04T07:33:38.913515+0000","last_undegraded":"2024-03-04T07:33:38.913515+0000","last_fullsized":"2024-03-04T07:33:38.913515+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:05:43.998194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.4","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104117+0000","last_change":"2024-03-04T07:33:37.915697+0000","last_active":"2024-03-04T07:33:39.104117+0000","last_peered":"2024-03-04T07:33:39.104117+0000","last_clean":"2024-03-04T07:33:39.104117+0000","last_became_active":"2024-03-04T07:33:37.915239+0000","last_became_peered":"2024-03-04T07:33:37.915239+0000","last_unstale":"2024-03-04T07:33:39.104117+0000","last_undegraded":"2024-03-04T07:33:39.104117+0000","last_fullsized":"2024-03-04T07:33:39.104117+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:45:08.415455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.5","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.104119+0000","last_change":"2024-03-04T07:33:36.899212+0000","last_active":"2024-03-04T07:33:39.104119+0000","last_peered":"2024-03-04T07:33:39.104119+0000","last_clean":"2024-03-04T07:33:39.104119+0000","last_became_active":"2024-03-04T07:33:36.898920+0000","last_became_peered":"2024-03-04T07:33:36.898920+0000","last_unstale":"2024-03-04T07:33:39.104119+0000","last_undegraded":"2024-03-04T07:33:39.104119+0000","last_fullsized":"2024-03-04T07:33:39.104119+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:48:22.544870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.5","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905870+0000","last_change":"2024-03-04T07:33:37.911195+0000","last_active":"2024-03-04T07:33:38.905870+0000","last_peered":"2024-03-04T07:33:38.905870+0000","last_clean":"2024-03-04T07:33:38.905870+0000","last_became_active":"2024-03-04T07:33:37.910737+0000","last_became_peered":"2024-03-04T07:33:37.910737+0000","last_unstale":"2024-03-04T07:33:38.905870+0000","last_undegraded":"2024-03-04T07:33:38.905870+0000","last_fullsized":"2024-03-04T07:33:38.905870+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:33:09.891054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"18.4","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904946+0000","last_change":"2024-03-04T07:33:36.896850+0000","last_active":"2024-03-04T07:33:38.904946+0000","last_peered":"2024-03-04T07:33:38.904946+0000","last_clean":"2024-03-04T07:33:38.904946+0000","last_became_active":"2024-03-04T07:33:36.896256+0000","last_became_peered":"2024-03-04T07:33:36.896256+0000","last_unstale":"2024-03-04T07:33:38.904946+0000","last_undegraded":"2024-03-04T07:33:38.904946+0000","last_fullsized":"2024-03-04T07:33:38.904946+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:24:31.790077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.2","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913270+0000","last_change":"2024-03-04T07:33:37.918848+0000","last_active":"2024-03-04T07:33:38.913270+0000","last_peered":"2024-03-04T07:33:38.913270+0000","last_clean":"2024-03-04T07:33:38.913270+0000","last_became_active":"2024-03-04T07:33:37.918205+0000","last_became_peered":"2024-03-04T07:33:37.918205+0000","last_unstale":"2024-03-04T07:33:38.913270+0000","last_undegraded":"2024-03-04T07:33:38.913270+0000","last_fullsized":"2024-03-04T07:33:38.913270+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:44:54.577910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.3","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103913+0000","last_change":"2024-03-04T07:33:36.899140+0000","last_active":"2024-03-04T07:33:39.103913+0000","last_peered":"2024-03-04T07:33:39.103913+0000","last_clean":"2024-03-04T07:33:39.103913+0000","last_became_active":"2024-03-04T07:33:36.898503+0000","last_became_peered":"2024-03-04T07:33:36.898503+0000","last_unstale":"2024-03-04T07:33:39.103913+0000","last_undegraded":"2024-03-04T07:33:39.103913+0000","last_fullsized":"2024-03-04T07:33:39.103913+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T16:03:55.760196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"19.3","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.905832+0000","last_change":"2024-03-04T07:33:37.903555+0000","last_active":"2024-03-04T07:33:38.905832+0000","last_peered":"2024-03-04T07:33:38.905832+0000","last_clean":"2024-03-04T07:33:38.905832+0000","last_became_active":"2024-03-04T07:33:37.903114+0000","last_became_peered":"2024-03-04T07:33:37.903114+0000","last_unstale":"2024-03-04T07:33:38.905832+0000","last_undegraded":"2024-03-04T07:33:38.905832+0000","last_fullsized":"2024-03-04T07:33:38.905832+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T08:12:37.381738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"18.2","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913256+0000","last_change":"2024-03-04T07:33:36.891284+0000","last_active":"2024-03-04T07:33:38.913256+0000","last_peered":"2024-03-04T07:33:38.913256+0000","last_clean":"2024-03-04T07:33:38.913256+0000","last_became_active":"2024-03-04T07:33:36.890998+0000","last_became_peered":"2024-03-04T07:33:36.890998+0000","last_unstale":"2024-03-04T07:33:38.913256+0000","last_undegraded":"2024-03-04T07:33:38.913256+0000","last_fullsized":"2024-03-04T07:33:38.913256+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T14:18:22.206516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"19.0","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.538393+0000","last_change":"2024-03-04T07:33:37.910265+0000","last_active":"2024-03-04T07:33:39.538393+0000","last_peered":"2024-03-04T07:33:39.538393+0000","last_clean":"2024-03-04T07:33:39.538393+0000","last_became_active":"2024-03-04T07:33:37.909790+0000","last_became_peered":"2024-03-04T07:33:37.909790+0000","last_unstale":"2024-03-04T07:33:39.538393+0000","last_undegraded":"2024-03-04T07:33:39.538393+0000","last_fullsized":"2024-03-04T07:33:39.538393+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:28.004888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"18.1","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.537999+0000","last_change":"2024-03-04T07:33:36.894317+0000","last_active":"2024-03-04T07:33:39.537999+0000","last_peered":"2024-03-04T07:33:39.537999+0000","last_clean":"2024-03-04T07:33:39.537999+0000","last_became_active":"2024-03-04T07:33:36.894013+0000","last_became_peered":"2024-03-04T07:33:36.894013+0000","last_unstale":"2024-03-04T07:33:39.537999+0000","last_undegraded":"2024-03-04T07:33:39.537999+0000","last_fullsized":"2024-03-04T07:33:39.537999+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T19:29:52.750489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"19.1","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:39.103904+0000","last_change":"2024-03-04T07:33:37.911071+0000","last_active":"2024-03-04T07:33:39.103904+0000","last_peered":"2024-03-04T07:33:39.103904+0000","last_clean":"2024-03-04T07:33:39.103904+0000","last_became_active":"2024-03-04T07:33:37.910740+0000","last_became_peered":"2024-03-04T07:33:37.910740+0000","last_unstale":"2024-03-04T07:33:39.103904+0000","last_undegraded":"2024-03-04T07:33:39.103904+0000","last_fullsized":"2024-03-04T07:33:39.103904+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:17:10.425201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"18.0","version":"0'0","reported_seq":14,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.904847+0000","last_change":"2024-03-04T07:33:36.887020+0000","last_active":"2024-03-04T07:33:38.904847+0000","last_peered":"2024-03-04T07:33:38.904847+0000","last_clean":"2024-03-04T07:33:38.904847+0000","last_became_active":"2024-03-04T07:33:36.886326+0000","last_became_peered":"2024-03-04T07:33:36.886326+0000","last_unstale":"2024-03-04T07:33:38.904847+0000","last_undegraded":"2024-03-04T07:33:38.904847+0000","last_fullsized":"2024-03-04T07:33:38.904847+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T13:28:26.999465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"19.12","version":"0'0","reported_seq":12,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913195+0000","last_change":"2024-03-04T07:33:37.919080+0000","last_active":"2024-03-04T07:33:38.913195+0000","last_peered":"2024-03-04T07:33:38.913195+0000","last_clean":"2024-03-04T07:33:38.913195+0000","last_became_active":"2024-03-04T07:33:37.918578+0000","last_became_peered":"2024-03-04T07:33:37.918578+0000","last_unstale":"2024-03-04T07:33:38.913195+0000","last_undegraded":"2024-03-04T07:33:38.913195+0000","last_fullsized":"2024-03-04T07:33:38.913195+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":121,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:36.871157+0000","last_clean_scrub_stamp":"2024-03-04T07:33:36.871157+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:14:10.409068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"18.13","version":"122'2","reported_seq":16,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913476+0000","last_change":"2024-03-04T07:33:36.895678+0000","last_active":"2024-03-04T07:33:38.913476+0000","last_peered":"2024-03-04T07:33:38.913476+0000","last_clean":"2024-03-04T07:33:38.913476+0000","last_became_active":"2024-03-04T07:33:36.895368+0000","last_became_peered":"2024-03-04T07:33:36.895368+0000","last_unstale":"2024-03-04T07:33:38.913476+0000","last_undegraded":"2024-03-04T07:33:38.913476+0000","last_fullsized":"2024-03-04T07:33:38.913476+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":120,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:33:35.865734+0000","last_clean_scrub_stamp":"2024-03-04T07:33:35.865734+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T09:57:59.748196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"15'117","reported_seq":356,"reported_epoch":123,"state":"active+clean","last_fresh":"2024-03-04T07:33:38.913012+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:33:38.913012+0000","last_peered":"2024-03-04T07:33:38.913012+0000","last_clean":"2024-03-04T07:33:38.913012+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:33:38.913012+0000","last_undegraded":"2024-03-04T07:33:38.913012+0000","last_fullsized":"2024-03-04T07:33:38.913012+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":19,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":8},{"poolid":18,"num_pg":32,"stat_sum":{"num_bytes":2410,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":64,"acting":64,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574937,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":37452,"kb_used_data":2104,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94334388,"statfs":{"total":96636764160,"available":96598413312,"internally_reserved":0,"allocated":2154496,"data_stored":1262061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":36174282},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574936,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41020,"kb_used_data":1512,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330820,"statfs":{"total":96636764160,"available":96594759680,"internally_reserved":0,"allocated":1548288,"data_stored":671055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574936,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41016,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330824,"statfs":{"total":96636764160,"available":96594763776,"internally_reserved":0,"allocated":1544192,"data_stored":669555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574937,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41008,"kb_used_data":1504,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330832,"statfs":{"total":96636764160,"available":96594771968,"internally_reserved":0,"allocated":1540096,"data_stored":669513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574936,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41008,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330832,"statfs":{"total":96636764160,"available":96594771968,"internally_reserved":0,"allocated":1544192,"data_stored":669603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574936,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41016,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330824,"statfs":{"total":96636764160,"available":96594763776,"internally_reserved":0,"allocated":1544192,"data_stored":670059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574936,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41000,"kb_used_data":1500,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330840,"statfs":{"total":96636764160,"available":96594780160,"internally_reserved":0,"allocated":1536000,"data_stored":669443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574936,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41580,"kb_used_data":2080,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330260,"statfs":{"total":96636764160,"available":96594186240,"internally_reserved":0,"allocated":2129920,"data_stored":1259811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":18,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":2250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":19,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:33:42.291 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-04T07:33:42.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph auth caps client.0 mon 'allow r' osd 'allow rw tag cephfs data=cephfs' mds allow 2024-03-04T07:33:42.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.406+0000 7f99fb0af640 1 Processor -- start 2024-03-04T07:33:42.407 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.406+0000 7f99fb0af640 1 -- start start 2024-03-04T07:33:42.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.406+0000 7f99fb0af640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41614d0 0x7f99f40cec80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.406+0000 7f99fb0af640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99f40d1260 0x7f99f40d3180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.406+0000 7f99fb0af640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f99f40cab70 con 0x7f99f41618b0 2024-03-04T07:33:42.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.406+0000 7f99fb0af640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f99f40d23c0 con 0x7f99f40d1260 2024-03-04T07:33:42.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.406+0000 7f99fb0af640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99f40d26a0 con 0x7f99f41614d0 2024-03-04T07:33:42.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f964c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99f40d1260 0x7f99f40d3180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f8e4b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41614d0 0x7f99f40cec80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f3fff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f99f41618b0 0x7f99f40b6de0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49722/0 (socket says 172.21.15.64:49722) 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f964c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f99f40d1260 0x7f99f40d3180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60728/0 (socket says 172.21.15.64:60728) 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f3fff640 1 -- 172.21.15.64:0/4290268738 learned_addr learned my addr 172.21.15.64:0/4290268738 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f964c640 1 -- 172.21.15.64:0/4290268738 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41614d0 msgr2=0x7f99f40cec80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f964c640 1 --2- 172.21.15.64:0/4290268738 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41614d0 0x7f99f40cec80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f964c640 1 -- 172.21.15.64:0/4290268738 >> v1:172.21.15.66:6789/0 conn(0x7f99f41618b0 legacy=0x7f99f40b6de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f964c640 1 -- 172.21.15.64:0/4290268738 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99f40d2980 con 0x7f99f40d1260 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f37fe640 1 -- 172.21.15.64:0/4290268738 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 759212170 0 0) 0x7f99f40cab70 con 0x7f99f41618b0 2024-03-04T07:33:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.407+0000 7f99f8e4b640 1 --2- 172.21.15.64:0/4290268738 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41614d0 0x7f99f40cec80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99f964c640 1 --2- 172.21.15.64:0/4290268738 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f40d1260 0x7f99f40d3180 secure :-1 s=READY pgs=1547 cs=0 l=1 rev1=1 crypto rx=0x7f99ec009d90 tx=0x7f99ec02bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e1711360879db4c7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99f37fe640 1 -- 172.21.15.64:0/4290268738 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99ec039070 con 0x7f99f40d1260 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99f37fe640 1 -- 172.21.15.64:0/4290268738 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99ec02cc50 con 0x7f99f40d1260 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99f37fe640 1 -- 172.21.15.64:0/4290268738 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99ec034440 con 0x7f99f40d1260 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 -- 172.21.15.64:0/4290268738 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f40d1260 msgr2=0x7f99f40d3180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 --2- 172.21.15.64:0/4290268738 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f40d1260 0x7f99f40d3180 secure :-1 s=READY pgs=1547 cs=0 l=1 rev1=1 crypto rx=0x7f99ec009d90 tx=0x7f99ec02bea0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 -- 172.21.15.64:0/4290268738 shutdown_connections 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 --2- 172.21.15.64:0/4290268738 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f40d1260 0x7f99f40d3180 unknown :-1 s=CLOSED pgs=1547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 --2- 172.21.15.64:0/4290268738 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41614d0 0x7f99f40cec80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 -- 172.21.15.64:0/4290268738 >> 172.21.15.64:0/4290268738 conn(0x7f99f4067120 msgr2=0x7f99f4067530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 -- 172.21.15.64:0/4290268738 shutdown_connections 2024-03-04T07:33:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.408+0000 7f99fb0af640 1 -- 172.21.15.64:0/4290268738 wait complete. 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.409+0000 7f99fb0af640 1 Processor -- start 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99fb0af640 1 -- start start 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99fb0af640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99f40d1260 0x7f99f4154000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99fb0af640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f41614d0 0x7f99f40d3080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99fb0af640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41618b0 0x7f99f40d35c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99fb0af640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99f40b74f0 con 0x7f99f41614d0 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99fb0af640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99f40cf380 con 0x7f99f40d1260 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99fb0af640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f99f40b7670 con 0x7f99f41618b0 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99f3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f41614d0 0x7f99f40d3080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99f3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f41614d0 0x7f99f40d3080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60738/0 (socket says 172.21.15.64:60738) 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99f3fff640 1 -- 172.21.15.64:0/3845136779 learned_addr learned my addr 172.21.15.64:0/3845136779 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.410+0000 7f99f8e4b640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99f40d1260 0x7f99f4154000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.414 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f964c640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41618b0 0x7f99f40d35c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f3fff640 1 -- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41618b0 msgr2=0x7f99f40d35c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f3fff640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41618b0 0x7f99f40d35c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f3fff640 1 -- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99f40d1260 msgr2=0x7f99f4154000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f3fff640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99f40d1260 0x7f99f4154000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f3fff640 1 -- 172.21.15.64:0/3845136779 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99f4158e00 con 0x7f99f41614d0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f3fff640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f41614d0 0x7f99f40d3080 secure :-1 s=READY pgs=1548 cs=0 l=1 rev1=1 crypto rx=0x7f99e000c410 tx=0x7f99e000c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f17fa640 1 -- 172.21.15.64:0/3845136779 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99e00074e0 con 0x7f99f41614d0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f17fa640 1 -- 172.21.15.64:0/3845136779 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f99e00047e0 con 0x7f99f41614d0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99f41590e0 con 0x7f99f41614d0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99f17fa640 1 -- 172.21.15.64:0/3845136779 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99e0004bd0 con 0x7f99f41614d0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.411+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99f41596a0 con 0x7f99f41614d0 2024-03-04T07:33:42.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.412+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f99f404f410 con 0x7f99f41614d0 2024-03-04T07:33:42.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.413+0000 7f99f17fa640 1 -- 172.21.15.64:0/3845136779 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f99e0011070 con 0x7f99f41614d0 2024-03-04T07:33:42.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.413+0000 7f99f17fa640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99d806b4d0 0x7f99d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.413+0000 7f99f17fa640 1 -- 172.21.15.64:0/3845136779 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f99e0089d70 con 0x7f99f41614d0 2024-03-04T07:33:42.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.413+0000 7f99f8e4b640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99d806b4d0 0x7f99d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.413+0000 7f99f8e4b640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99d806b4d0 0x7f99d806d990 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f99e8009560 tx=0x7f99e8009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.417+0000 7f99f17fa640 1 -- 172.21.15.64:0/3845136779 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f99e0058520 con 0x7f99f41614d0 2024-03-04T07:33:42.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.664+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]} v 0) v1 -- 0x7f99f40d1d10 con 0x7f99f41614d0 2024-03-04T07:33:42.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.667+0000 7f99f17fa640 1 -- 172.21.15.64:0/3845136779 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "auth caps", "entity": "client.0", "caps": ["mon", "allow r", "osd", "allow rw tag cephfs data=cephfs", "mds", "allow"]}]=0 updated caps for client.0 v22) v1 ==== 190+0+147 (secure 0 0 0) 0x7f99e005c5c0 con 0x7f99f41614d0 2024-03-04T07:33:42.668 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-03-04T07:33:42.668 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQB+d+VlLdzhBxAA4hhHh3ps8ERTrOPa4kmWLA== 2024-03-04T07:33:42.668 INFO:teuthology.orchestra.run.smithi064.stdout: caps mds = "allow" 2024-03-04T07:33:42.668 INFO:teuthology.orchestra.run.smithi064.stdout: caps mon = "allow r" 2024-03-04T07:33:42.668 INFO:teuthology.orchestra.run.smithi064.stdout: caps osd = "allow rw tag cephfs data=cephfs" 2024-03-04T07:33:42.668 INFO:teuthology.orchestra.run.smithi064.stderr:updated caps for client.0 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99d806b4d0 msgr2=0x7f99d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99d806b4d0 0x7f99d806d990 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f99e8009560 tx=0x7f99e8009190 comp rx=0 tx=0).stop 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f41614d0 msgr2=0x7f99f40d3080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f41614d0 0x7f99f40d3080 secure :-1 s=READY pgs=1548 cs=0 l=1 rev1=1 crypto rx=0x7f99e000c410 tx=0x7f99e000c8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 shutdown_connections 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f99d806b4d0 0x7f99d806d990 unknown :-1 s=CLOSED pgs=693 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f99f41618b0 0x7f99f40d35c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f99f41614d0 0x7f99f40d3080 unknown :-1 s=CLOSED pgs=1548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 --2- 172.21.15.64:0/3845136779 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f99f40d1260 0x7f99f4154000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 >> 172.21.15.64:0/3845136779 conn(0x7f99f4067120 msgr2=0x7f99f4160750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:42.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 shutdown_connections 2024-03-04T07:33:42.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.669+0000 7f99fb0af640 1 -- 172.21.15.64:0/3845136779 wait complete. 2024-03-04T07:33:42.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:42.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.784+0000 7f2d1e665640 1 Processor -- start 2024-03-04T07:33:42.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.784+0000 7f2d1e665640 1 -- start start 2024-03-04T07:33:42.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d1e665640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d1815e000 0x7f2d18071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d1e665640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2d180d35e0 0x7f2d18071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d1e665640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d180ce970 con 0x7f2d180b2c50 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d1e665640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d180cece0 con 0x7f2d180d35e0 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d1e665640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d180cef30 con 0x7f2d1815e000 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d17fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d1815e000 0x7f2d18071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d177fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f2d180b2c50 0x7f2d18161cb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49738/0 (socket says 172.21.15.64:49738) 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d177fe640 1 -- 172.21.15.64:0/4015338221 learned_addr learned my addr 172.21.15.64:0/4015338221 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d1cc02640 1 --2- 172.21.15.64:0/4015338221 >> v2:172.21.15.64:3300/0 conn(0x7f2d180d35e0 0x7f2d18071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d16ffd640 1 -- 172.21.15.64:0/4015338221 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4014905278 0 0) 0x7f2d180ce970 con 0x7f2d180b2c50 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d16ffd640 1 -- 172.21.15.64:0/4015338221 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d00003520 con 0x7f2d180b2c50 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d17fff640 1 -- 172.21.15.64:0/4015338221 >> v2:172.21.15.64:3300/0 conn(0x7f2d180d35e0 msgr2=0x7f2d18071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d17fff640 1 --2- 172.21.15.64:0/4015338221 >> v2:172.21.15.64:3300/0 conn(0x7f2d180d35e0 0x7f2d18071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d17fff640 1 -- 172.21.15.64:0/4015338221 >> v1:172.21.15.66:6789/0 conn(0x7f2d180b2c50 legacy=0x7f2d18161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.785+0000 7f2d17fff640 1 -- 172.21.15.64:0/4015338221 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d180cf210 con 0x7f2d1815e000 2024-03-04T07:33:42.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.786+0000 7f2d17fff640 1 --2- 172.21.15.64:0/4015338221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d1815e000 0x7f2d18071540 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f2d0c0097a0 tx=0x7f2d0c02b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8d150c4a0d61c7a6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.786+0000 7f2d16ffd640 1 -- 172.21.15.64:0/4015338221 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d0c0315a0 con 0x7f2d1815e000 2024-03-04T07:33:42.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.786+0000 7f2d16ffd640 1 -- 172.21.15.64:0/4015338221 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d0c037070 con 0x7f2d1815e000 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.786+0000 7f2d16ffd640 1 -- 172.21.15.64:0/4015338221 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d0c031940 con 0x7f2d1815e000 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 -- 172.21.15.64:0/4015338221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d1815e000 msgr2=0x7f2d18071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 --2- 172.21.15.64:0/4015338221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d1815e000 0x7f2d18071540 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f2d0c0097a0 tx=0x7f2d0c02b840 comp rx=0 tx=0).stop 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 -- 172.21.15.64:0/4015338221 shutdown_connections 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 --2- 172.21.15.64:0/4015338221 >> v2:172.21.15.64:3300/0 conn(0x7f2d180d35e0 0x7f2d18071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 --2- 172.21.15.64:0/4015338221 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d1815e000 0x7f2d18071540 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 -- 172.21.15.64:0/4015338221 >> 172.21.15.64:0/4015338221 conn(0x7f2d180d3c80 msgr2=0x7f2d180d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 -- 172.21.15.64:0/4015338221 shutdown_connections 2024-03-04T07:33:42.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.787+0000 7f2d1e665640 1 -- 172.21.15.64:0/4015338221 wait complete. 2024-03-04T07:33:42.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.788+0000 7f2d1e665640 1 Processor -- start 2024-03-04T07:33:42.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.788+0000 7f2d1e665640 1 -- start start 2024-03-04T07:33:42.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1e665640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d180b2c50 0x7f2d18152320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1e665640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d180d35e0 0x7f2d18071540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1e665640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d1815e000 0x7f2d18071a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1e665640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d180b3320 con 0x7f2d1815e000 2024-03-04T07:33:42.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1e665640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d180b31a0 con 0x7f2d180b2c50 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1e665640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d1815e3e0 con 0x7f2d180d35e0 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1cc02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d1815e000 0x7f2d18071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1cc02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d1815e000 0x7f2d18071a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60762/0 (socket says 172.21.15.64:60762) 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1cc02640 1 -- 172.21.15.64:0/2530412726 learned_addr learned my addr 172.21.15.64:0/2530412726 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d17fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d180b2c50 0x7f2d18152320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d177fe640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d180d35e0 0x7f2d18071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1cc02640 1 -- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d180d35e0 msgr2=0x7f2d18071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.789+0000 7f2d1cc02640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d180d35e0 0x7f2d18071540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d1cc02640 1 -- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d180b2c50 msgr2=0x7f2d18152320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d1cc02640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d180b2c50 0x7f2d18152320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d1cc02640 1 -- 172.21.15.64:0/2530412726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d18156300 con 0x7f2d1815e000 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d17fff640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d180b2c50 0x7f2d18152320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d1cc02640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d1815e000 0x7f2d18071a80 secure :-1 s=READY pgs=1549 cs=0 l=1 rev1=1 crypto rx=0x7f2d10009cb0 tx=0x7f2d1000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d14ff9640 1 -- 172.21.15.64:0/2530412726 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d10018020 con 0x7f2d1815e000 2024-03-04T07:33:42.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d14ff9640 1 -- 172.21.15.64:0/2530412726 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d10014480 con 0x7f2d1815e000 2024-03-04T07:33:42.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d181565e0 con 0x7f2d1815e000 2024-03-04T07:33:42.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d14ff9640 1 -- 172.21.15.64:0/2530412726 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d10014870 con 0x7f2d1815e000 2024-03-04T07:33:42.793 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.790+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d180b4e50 con 0x7f2d1815e000 2024-03-04T07:33:42.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.791+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d18067ff0 con 0x7f2d1815e000 2024-03-04T07:33:42.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.791+0000 7f2d14ff9640 1 -- 172.21.15.64:0/2530412726 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d1001b020 con 0x7f2d1815e000 2024-03-04T07:33:42.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.792+0000 7f2d14ff9640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b4d0 0x7f2d0406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:42.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.792+0000 7f2d14ff9640 1 -- 172.21.15.64:0/2530412726 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2d1008adf0 con 0x7f2d1815e000 2024-03-04T07:33:42.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.792+0000 7f2d17fff640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b4d0 0x7f2d0406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:42.794 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.792+0000 7f2d17fff640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b4d0 0x7f2d0406d990 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f2d0c036000 tx=0x7f2d0c03d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:42.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:42.796+0000 7f2d14ff9640 1 -- 172.21.15.64:0/2530412726 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d10022020 con 0x7f2d1815e000 2024-03-04T07:33:42.868 INFO:tasks.ceph.mds.d.smithi066.stderr:2024-03-04T07:33:42.868+0000 7f246d66b640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-04T07:33:43.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.024+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2d180cfb70 con 0x7f2d1815e000 2024-03-04T07:33:43.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.024+0000 7f2d14ff9640 1 -- 172.21.15.64:0/2530412726 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 81 v81) v1 ==== 94+0+4976 (secure 0 0 0) 0x7f2d10010040 con 0x7f2d1815e000 2024-03-04T07:33:43.025 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:43.025 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":81,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6713,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3217233400","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3217233400},{"type":"v1","addr":"172.21.15.64:6836","nonce":3217233400}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77},{"gid":6716,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2523962873","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2523962873},{"type":"v1","addr":"172.21.15.64:6837","nonce":2523962873}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77},{"gid":6728,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77}],"filesystems":[{"mdsmap":{"epoch":81,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:39.913726+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0],"up":{"mds_0":8577},"failed":[],"damaged":[],"stopped":[],"info":{"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:33:43.026 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 81 2024-03-04T07:33:43.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b4d0 msgr2=0x7f2d0406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b4d0 0x7f2d0406d990 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f2d0c036000 tx=0x7f2d0c03d040 comp rx=0 tx=0).stop 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d1815e000 msgr2=0x7f2d18071a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d1815e000 0x7f2d18071a80 secure :-1 s=READY pgs=1549 cs=0 l=1 rev1=1 crypto rx=0x7f2d10009cb0 tx=0x7f2d1000c490 comp rx=0 tx=0).stop 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 shutdown_connections 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d0406b4d0 0x7f2d0406d990 unknown :-1 s=CLOSED pgs=694 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d1815e000 0x7f2d18071a80 unknown :-1 s=CLOSED pgs=1549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d180d35e0 0x7f2d18071540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 --2- 172.21.15.64:0/2530412726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d180b2c50 0x7f2d18152320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 >> 172.21.15.64:0/2530412726 conn(0x7f2d180d3c80 msgr2=0x7f2d18160b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 shutdown_connections 2024-03-04T07:33:43.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.026+0000 7f2d1e665640 1 -- 172.21.15.64:0/2530412726 wait complete. 2024-03-04T07:33:43.441 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 81, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:33:39.913726+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 8577}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:33:43.441 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-03-04T07:33:43.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.d status 2024-03-04T07:33:43.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.545+0000 7f4b3f37a640 1 Processor -- start 2024-03-04T07:33:43.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.545+0000 7f4b3f37a640 1 -- start start 2024-03-04T07:33:43.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3f37a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b3806af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3f37a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4b3806ed10 0x7f4b380cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3f37a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b380cc0e0 con 0x7f4b380cd6c0 2024-03-04T07:33:43.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3f37a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b380d28b0 con 0x7f4b3806ed10 2024-03-04T07:33:43.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3f37a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b380cf320 con 0x7f4b380b1020 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3d917640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4b380cd6c0 0x7f4b3815d700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60594/0 (socket says 172.21.15.64:60594) 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3d917640 1 -- 172.21.15.64:0/504683293 learned_addr learned my addr 172.21.15.64:0/504683293 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3c915640 1 --2- 172.21.15.64:0/504683293 >> v2:172.21.15.66:3300/0 conn(0x7f4b3806ed10 0x7f4b380cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3d116640 1 --2- 172.21.15.64:0/504683293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b3806af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b37fff640 1 -- 172.21.15.64:0/504683293 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1042105349 0 0) 0x7f4b380cc0e0 con 0x7f4b380cd6c0 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b37fff640 1 -- 172.21.15.64:0/504683293 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4b20003520 con 0x7f4b380cd6c0 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3d116640 1 -- 172.21.15.64:0/504683293 >> v2:172.21.15.66:3300/0 conn(0x7f4b3806ed10 msgr2=0x7f4b380cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3d116640 1 --2- 172.21.15.64:0/504683293 >> v2:172.21.15.66:3300/0 conn(0x7f4b3806ed10 0x7f4b380cd180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3d116640 1 -- 172.21.15.64:0/504683293 >> v1:172.21.15.64:6789/0 conn(0x7f4b380cd6c0 legacy=0x7f4b3815d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3d116640 1 -- 172.21.15.64:0/504683293 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b380cc3c0 con 0x7f4b380b1020 2024-03-04T07:33:43.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.546+0000 7f4b3c915640 1 --2- 172.21.15.64:0/504683293 >> v2:172.21.15.66:3300/0 conn(0x7f4b3806ed10 0x7f4b380cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.547+0000 7f4b3d116640 1 --2- 172.21.15.64:0/504683293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b3806af20 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f4b300097a0 tx=0x7f4b3002b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=156e13e9060253fc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.547+0000 7f4b37fff640 1 -- 172.21.15.64:0/504683293 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b30035020 con 0x7f4b380b1020 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.547+0000 7f4b37fff640 1 -- 172.21.15.64:0/504683293 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b30031830 con 0x7f4b380b1020 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.547+0000 7f4b37fff640 1 -- 172.21.15.64:0/504683293 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b30031c00 con 0x7f4b380b1020 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 -- 172.21.15.64:0/504683293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 msgr2=0x7f4b3806af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/504683293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b3806af20 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f4b300097a0 tx=0x7f4b3002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 -- 172.21.15.64:0/504683293 shutdown_connections 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/504683293 >> v2:172.21.15.66:3300/0 conn(0x7f4b3806ed10 0x7f4b380cd180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/504683293 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b3806af20 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 -- 172.21.15.64:0/504683293 >> 172.21.15.64:0/504683293 conn(0x7f4b380d1a90 msgr2=0x7f4b380d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 -- 172.21.15.64:0/504683293 shutdown_connections 2024-03-04T07:33:43.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.548+0000 7f4b3f37a640 1 -- 172.21.15.64:0/504683293 wait complete. 2024-03-04T07:33:43.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.549+0000 7f4b3f37a640 1 Processor -- start 2024-03-04T07:33:43.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.549+0000 7f4b3f37a640 1 -- start start 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3f37a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b3806ed10 0x7f4b380cb690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3f37a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b380b6b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3f37a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b380cd6c0 0x7f4b380b7060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3f37a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b380ccc60 con 0x7f4b3806ed10 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3f37a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b380cc980 con 0x7f4b380cd6c0 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3f37a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b380cdaf0 con 0x7f4b380b1020 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b3806ed10 0x7f4b380cb690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b3806ed10 0x7f4b380cb690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47686/0 (socket says 172.21.15.64:47686) 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 -- 172.21.15.64:0/3083118121 learned_addr learned my addr 172.21.15.64:0/3083118121 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d917640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b380cd6c0 0x7f4b380b7060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3c915640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b380b6b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 -- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 msgr2=0x7f4b380b6b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b380b6b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 -- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b380cd6c0 msgr2=0x7f4b380b7060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b380cd6c0 0x7f4b380b7060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 -- 172.21.15.64:0/3083118121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b380cbbd0 con 0x7f4b3806ed10 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.550+0000 7f4b3d116640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b3806ed10 0x7f4b380cb690 secure :-1 s=READY pgs=1551 cs=0 l=1 rev1=1 crypto rx=0x7f4b30009890 tx=0x7f4b300046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.551+0000 7f4b35ffb640 1 -- 172.21.15.64:0/3083118121 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b30035040 con 0x7f4b3806ed10 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.551+0000 7f4b35ffb640 1 -- 172.21.15.64:0/3083118121 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b3003b920 con 0x7f4b3806ed10 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.551+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b380b5590 con 0x7f4b3806ed10 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.551+0000 7f4b35ffb640 1 -- 172.21.15.64:0/3083118121 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b3003bd10 con 0x7f4b3806ed10 2024-03-04T07:33:43.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.551+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b380b5b50 con 0x7f4b3806ed10 2024-03-04T07:33:43.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.553+0000 7f4b35ffb640 1 -- 172.21.15.64:0/3083118121 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b30038020 con 0x7f4b3806ed10 2024-03-04T07:33:43.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.553+0000 7f4b35ffb640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b2406b4d0 0x7f4b2406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.553+0000 7f4b35ffb640 1 -- 172.21.15.64:0/3083118121 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4b300ac0d0 con 0x7f4b3806ed10 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.553+0000 7f4b3c915640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b2406b4d0 0x7f4b2406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.554+0000 7f4b3c915640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b2406b4d0 0x7f4b2406d990 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7f4b2c009560 tx=0x7f4b2c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.554+0000 7f4b41bfd740 1 Processor -- start 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.554+0000 7f4b41bfd740 1 -- start start 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.554+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d1a870 0x559d17d1ac50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d1e8c0 0x559d17d20360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d4aa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x559d17c846b0 con 0x559d17d4a600 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x559d17c2da60 con 0x559d17d1a870 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x559d17c80750 con 0x559d17d1e8c0 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d917640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d4aa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d1a870 0x559d17d1ac50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d1a870 0x559d17d1ac50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:54982/0 (socket says 172.21.15.64:54982) 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d917640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d4aa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47688/0 (socket says 172.21.15.64:47688) 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3c915640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d1e8c0 0x559d17d20360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 -- 172.21.15.64:0/3844713408 learned_addr learned my addr 172.21.15.64:0/3844713408 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 -- 172.21.15.64:0/3844713408 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d1e8c0 msgr2=0x559d17d20360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d1e8c0 0x559d17d20360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 -- 172.21.15.64:0/3844713408 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 msgr2=0x559d17d4aa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d4aa80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 -- 172.21.15.64:0/3844713408 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559d17c4bf30 con 0x559d17d1a870 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d917640 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d4aa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b3d116640 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d1a870 0x559d17d1ac50 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f4b30082ac0 tx=0x7f4b3007a3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4f762ba87084028 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b1effd640 1 -- 172.21.15.64:0/3844713408 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b30082ff0 con 0x559d17d1a870 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b1effd640 1 -- 172.21.15.64:0/3844713408 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b30078a30 con 0x559d17d1a870 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.555+0000 7f4b1effd640 1 -- 172.21.15.64:0/3844713408 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b30077000 con 0x559d17d1a870 2024-03-04T07:33:43.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 -- 172.21.15.64:0/3844713408 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d1a870 msgr2=0x559d17d1ac50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d1a870 0x559d17d1ac50 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f4b30082ac0 tx=0x7f4b3007a3b0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 -- 172.21.15.64:0/3844713408 shutdown_connections 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d4aa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d1e8c0 0x559d17d20360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/3844713408 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d1a870 0x559d17d1ac50 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 -- 172.21.15.64:0/3844713408 >> 172.21.15.64:0/3844713408 conn(0x559d17d1c8f0 msgr2=0x559d17d1e1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 -- 172.21.15.64:0/3844713408 shutdown_connections 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 -- 172.21.15.64:0/3844713408 wait complete. 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.556+0000 7f4b41bfd740 1 -- start start 2024-03-04T07:33:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.557+0000 7f4b41bfd740 1 Processor -- start 2024-03-04T07:33:43.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.557+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17d49790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.557+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d3ea70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.557+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d3efb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.557+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x559d17c846b0 con 0x559d17d4e270 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.557+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x559d17c40ea0 con 0x559d17d3fb80 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.557+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x559d17c2da60 con 0x559d17d4a600 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b41bfd740 10 client.0 _ll_register_callbacks cb 0x559d17c40870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d3ea70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d3ea70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47704/0 (socket says 172.21.15.64:47704) 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 -- 172.21.15.64:0/2335916103 learned_addr learned my addr 172.21.15.64:0/2335916103 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3d116640 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17d49790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3d917640 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d3efb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 -- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 msgr2=0x559d17d49790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17d49790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 -- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 msgr2=0x559d17d3efb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d3efb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 -- 172.21.15.64:0/2335916103 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x559d17c41270 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b3c915640 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d3ea70 secure :-1 s=READY pgs=1552 cs=0 l=1 rev1=1 crypto rx=0x7f4b380d0f10 tx=0x7f4b2c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2f4077f426053c4c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b17fff640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b17fff640 1 -- 172.21.15.64:0/2335916103 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b2c027070 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b17fff640 1 -- 172.21.15.64:0/2335916103 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4b2c014440 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f4b30078a30 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b17fff640 1 -- 172.21.15.64:0/2335916103 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4b2c032050 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b41bfd740 10 client.8709 fetch_fsmap learned FSMap version 81 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=81,monmap=2+}) v3 -- 0x559d179992d0 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.558+0000 7f4b17fff640 1 -- 172.21.15.64:0/2335916103 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b30078a30 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b17fff640 1 -- 172.21.15.64:0/2335916103 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 81) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f4b2c035020 con 0x559d17d4e270 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b41bfd740 10 client.8709 fetch_fsmap finished waiting for FSMap version 81 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b41bfd740 10 client.8709 resolve_mds: resolved name 'd' to daemon mds.d 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b41bfd740 20 client.8709 populate_metadata read hostname 'smithi064' 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d43010 0x559d17d84b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b41bfd740 4 client.8709 mds_command: new command op to 8577 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-04T07:33:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 --> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559d17c91bf0 con 0x559d17d43010 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b3d116640 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d43010 0x559d17d84b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b17fff640 10 client.8709 ms_handle_connect on v2:172.21.15.66:6832/546842609 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.559+0000 7f4b3d116640 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d43010 0x559d17d84b00 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d0d0c1c02f80a82e server_cookie=6b2493f1b0490e74 in_seq=0 out_seq=0 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b17fff640 1 -- 172.21.15.64:0/2335916103 <== mds.0 v2:172.21.15.66:6832/546842609 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+21660 (crc 0 0 0) 0x559d17c91bf0 con 0x559d17d43010 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b17fff640 10 client.8709 handle_command_reply: tid=0 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 client.8709 shutdown 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 msgr2=0x559d17d3ea70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d3ea70 secure :-1 s=READY pgs=1552 cs=0 l=1 rev1=1 crypto rx=0x7f4b380d0f10 tx=0x7f4b2c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 shutdown_connections 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d43010 0x559d17d84b00 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d3efb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b17fff640 0 client.8709 ms_handle_reset on v2:172.21.15.66:6832/546842609 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d3ea70 unknown :-1 s=CLOSED pgs=1552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2335916103 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17d49790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.560+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 >> 172.21.15.64:0/2335916103 conn(0x559d17d1a870 msgr2=0x559d17d485c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.561+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 shutdown_connections 2024-03-04T07:33:43.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.561+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2335916103 wait complete. 2024-03-04T07:33:43.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.561+0000 7f4b41bfd740 20 client.8709 trim_cache size 0 max 16384 2024-03-04T07:33:43.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.577+0000 7f4b41bfd740 1 Processor -- start 2024-03-04T07:33:43.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.577+0000 7f4b41bfd740 1 -- start start 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d3fb80 0x559d17d387f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d1ae40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d4e270 0x559d17d47580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b381ffff0 con 0x559d17d4a600 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b380c7df0 con 0x559d17d4e270 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b381f7d00 con 0x559d17d3fb80 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b3c915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d1ae40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b3c915640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d1ae40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47714/0 (socket says 172.21.15.64:47714) 2024-03-04T07:33:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b3c915640 1 -- 172.21.15.64:0/2301733542 learned_addr learned my addr 172.21.15.64:0/2301733542 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b3d116640 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d3fb80 0x559d17d387f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.578+0000 7f4b3d917640 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d4e270 0x559d17d47580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b3c915640 1 -- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d3fb80 msgr2=0x559d17d387f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b3c915640 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d3fb80 0x559d17d387f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b3c915640 1 -- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d4e270 msgr2=0x559d17d47580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b3c915640 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d4e270 0x559d17d47580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b3c915640 1 -- 172.21.15.64:0/2301733542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559d17d450f0 con 0x559d17d4a600 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b3c915640 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d1ae40 secure :-1 s=READY pgs=1553 cs=0 l=1 rev1=1 crypto rx=0x7f4b2c0117e0 tx=0x7f4b2c024000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9bc5ae7f2f9ba5c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b1e7fc640 1 -- 172.21.15.64:0/2301733542 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b2c027070 con 0x559d17d4a600 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b1e7fc640 1 -- 172.21.15.64:0/2301733542 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b2c03b070 con 0x559d17d4a600 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b1e7fc640 1 -- 172.21.15.64:0/2301733542 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b2c014bc0 con 0x559d17d4a600 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2301733542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 msgr2=0x559d17d1ae40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d1ae40 secure :-1 s=READY pgs=1553 cs=0 l=1 rev1=1 crypto rx=0x7f4b2c0117e0 tx=0x7f4b2c024000 comp rx=0 tx=0).stop 2024-03-04T07:33:43.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2301733542 shutdown_connections 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d4e270 0x559d17d47580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4a600 0x559d17d1ae40 unknown :-1 s=CLOSED pgs=1553 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/2301733542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d3fb80 0x559d17d387f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2301733542 >> 172.21.15.64:0/2301733542 conn(0x559d17d1a870 msgr2=0x559d17d4d1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2301733542 shutdown_connections 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.579+0000 7f4b41bfd740 1 -- 172.21.15.64:0/2301733542 wait complete. 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.580+0000 7f4b41bfd740 1 -- start start 2024-03-04T07:33:43.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.580+0000 7f4b41bfd740 1 Processor -- start 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d433f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17dd5920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b41bfd740 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d1f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x559d17d42010 con 0x559d17d4e270 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b380c77f0 con 0x559d17d3fb80 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b41bfd740 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x559d17d63200 con 0x559d17d4a600 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b41bfd740 10 client.0 _ll_register_callbacks cb 0x7f4b2c004320 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d1f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d1f520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47724/0 (socket says 172.21.15.64:47724) 2024-03-04T07:33:43.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 -- 172.21.15.64:0/115903617 learned_addr learned my addr 172.21.15.64:0/115903617 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d116640 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d433f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3c915640 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17dd5920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 -- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 msgr2=0x559d17dd5920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17dd5920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 -- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 msgr2=0x559d17d433f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d433f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 -- 172.21.15.64:0/115903617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x559d17d212a0 con 0x559d17d4e270 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b3d917640 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d1f520 secure :-1 s=READY pgs=1554 cs=0 l=1 rev1=1 crypto rx=0x7f4b28016b70 tx=0x7f4b28007410 comp rx=0 tx=0).ready entity=mon.0 client_cookie=92c72046d3160ba0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.581+0000 7f4b17fff640 10 client.0 ms_handle_connect on v2:172.21.15.64:3300/0 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b17fff640 1 -- 172.21.15.64:0/115903617 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b28015630 con 0x559d17d4e270 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b17fff640 1 -- 172.21.15.64:0/115903617 <== mon.0 v2:172.21.15.64:3300/0 2 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4b28010040 con 0x559d17d4e270 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f4b2c03b070 con 0x559d17d4e270 2024-03-04T07:33:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b17fff640 1 -- 172.21.15.64:0/115903617 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4b28028050 con 0x559d17d4e270 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 10 client.8715 fetch_fsmap learned FSMap version 81 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,fsmap=81,monmap=2+}) v3 -- 0x559d17a636b0 con 0x559d17d4e270 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b17fff640 1 -- 172.21.15.64:0/115903617 <== mon.0 v2:172.21.15.64:3300/0 4 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b2c03b070 con 0x559d17d4e270 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b17fff640 1 -- 172.21.15.64:0/115903617 <== mon.0 v2:172.21.15.64:3300/0 5 ==== fsmap(e 81) v1 ==== 3314+0+0 (secure 0 0 0) 0x7f4b2802b020 con 0x559d17d4e270 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 10 client.8715 fetch_fsmap finished waiting for FSMap version 81 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 10 client.8715 resolve_mds: resolved name 'd' to daemon mds.d 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 20 client.8715 populate_metadata read hostname 'smithi064' 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d800d0 0x559d17d35270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 4 client.8715 mds_command: new command op to 8577 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 --> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] -- command(tid 0: {"prefix": "status"}) v1 -- 0x559d17d1d050 con 0x559d17d800d0 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.582+0000 7f4b3d116640 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d800d0 0x559d17d35270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b3d116640 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d800d0 0x559d17d35270 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=640ddd27042e9126 server_cookie=b9ca87c058058ae9 in_seq=0 out_seq=0 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b17fff640 10 client.8715 ms_handle_connect on v2:172.21.15.66:6832/546842609 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b17fff640 1 -- 172.21.15.64:0/115903617 <== mds.0 v2:172.21.15.66:6832/546842609 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+333 (crc 0 0 0) 0x559d17d1d050 con 0x559d17d800d0 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b17fff640 10 client.8715 handle_command_reply: tid=0 2024-03-04T07:33:43.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 client.8715 shutdown 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 msgr2=0x559d17d1f520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d1f520 secure :-1 s=READY pgs=1554 cs=0 l=1 rev1=1 crypto rx=0x7f4b28016b70 tx=0x7f4b28007410 comp rx=0 tx=0).stop 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 shutdown_connections 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:6832/546842609,v1:172.21.15.66:6834/546842609] conn(0x559d17d800d0 0x559d17d35270 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x559d17d4e270 0x559d17d1f520 unknown :-1 s=CLOSED pgs=1554 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x559d17d4a600 0x559d17dd5920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b17fff640 0 client.8715 ms_handle_reset on v2:172.21.15.66:6832/546842609 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 --2- 172.21.15.64:0/115903617 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x559d17d3fb80 0x559d17d433f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 >> 172.21.15.64:0/115903617 conn(0x559d17d1a870 msgr2=0x559d17d1ae40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 shutdown_connections 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 1 -- 172.21.15.64:0/115903617 wait complete. 2024-03-04T07:33:43.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.583+0000 7f4b41bfd740 20 client.8715 trim_cache size 0 max 16384 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_fsid": "c3ef085c-fd40-4f87-8c92-fdb42a102a8c", 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "whoami": 0, 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 8577, 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "want_state": "up:active", 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "state": "up:active", 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs", 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "rank_uptime": 5.6834284149999998, 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "mdsmap_epoch": 81, 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch": 123, 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "osdmap_epoch_barrier": 123, 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout: "uptime": 11.684573179999999 2024-03-04T07:33:43.591 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b2406b4d0 msgr2=0x7f4b2406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b2406b4d0 0x7f4b2406d990 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7f4b2c009560 tx=0x7f4b2c009190 comp rx=0 tx=0).stop 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b3806ed10 msgr2=0x7f4b380cb690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b3806ed10 0x7f4b380cb690 secure :-1 s=READY pgs=1551 cs=0 l=1 rev1=1 crypto rx=0x7f4b30009890 tx=0x7f4b300046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 shutdown_connections 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b2406b4d0 0x7f4b2406d990 unknown :-1 s=CLOSED pgs=695 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b380cd6c0 0x7f4b380b7060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b380b1020 0x7f4b380b6b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 --2- 172.21.15.64:0/3083118121 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b3806ed10 0x7f4b380cb690 unknown :-1 s=CLOSED pgs=1551 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 >> 172.21.15.64:0/3083118121 conn(0x7f4b380d1a90 msgr2=0x7f4b380b01d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 shutdown_connections 2024-03-04T07:33:43.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.584+0000 7f4b3f37a640 1 -- 172.21.15.64:0/3083118121 wait complete. 2024-03-04T07:33:43.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs ls 2024-03-04T07:33:43.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.696+0000 7f9235e2a640 1 Processor -- start 2024-03-04T07:33:43.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.696+0000 7f9235e2a640 1 -- start start 2024-03-04T07:33:43.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f9235e2a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f923015e060 0x7f92300ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f9235e2a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f92300d0dd0 con 0x7f923006fe20 2024-03-04T07:33:43.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f9235e2a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f92300d10b0 con 0x7f92300d0880 2024-03-04T07:33:43.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f9235e2a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f92300cc0e0 con 0x7f923015e060 2024-03-04T07:33:43.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f923006fe20 0x7f92300ce850 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60606/0 (socket says 172.21.15.64:60606) 2024-03-04T07:33:43.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922effd640 1 -- 172.21.15.64:0/4007254541 learned_addr learned my addr 172.21.15.64:0/4007254541 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922f7fe640 1 --2- 172.21.15.64:0/4007254541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f923015e060 0x7f92300ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922e7fc640 1 -- 172.21.15.64:0/4007254541 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3493449518 0 0) 0x7f92300d0dd0 con 0x7f923006fe20 2024-03-04T07:33:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922e7fc640 1 -- 172.21.15.64:0/4007254541 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9218003520 con 0x7f923006fe20 2024-03-04T07:33:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922e7fc640 1 -- 172.21.15.64:0/4007254541 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 250716344 0 0) 0x7f92300d10b0 con 0x7f92300d0880 2024-03-04T07:33:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922f7fe640 1 -- 172.21.15.64:0/4007254541 >> v1:172.21.15.66:6789/0 conn(0x7f92300d0880 legacy=0x7f92300ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922f7fe640 1 -- 172.21.15.64:0/4007254541 >> v1:172.21.15.64:6789/0 conn(0x7f923006fe20 legacy=0x7f92300ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.697+0000 7f922f7fe640 1 -- 172.21.15.64:0/4007254541 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f92300cc3c0 con 0x7f923015e060 2024-03-04T07:33:43.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.698+0000 7f922f7fe640 1 --2- 172.21.15.64:0/4007254541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f923015e060 0x7f92300ce310 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f92200097d0 tx=0x7f922002b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c6a901a3e8b4c1ac server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.698+0000 7f922e7fc640 1 -- 172.21.15.64:0/4007254541 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9220035020 con 0x7f923015e060 2024-03-04T07:33:43.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.698+0000 7f922e7fc640 1 -- 172.21.15.64:0/4007254541 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9220031890 con 0x7f923015e060 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.698+0000 7f922e7fc640 1 -- 172.21.15.64:0/4007254541 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9220031c40 con 0x7f923015e060 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.699+0000 7f9235e2a640 1 -- 172.21.15.64:0/4007254541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f923015e060 msgr2=0x7f92300ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.699+0000 7f9235e2a640 1 --2- 172.21.15.64:0/4007254541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f923015e060 0x7f92300ce310 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f92200097d0 tx=0x7f922002b930 comp rx=0 tx=0).stop 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.699+0000 7f9235e2a640 1 -- 172.21.15.64:0/4007254541 shutdown_connections 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.699+0000 7f9235e2a640 1 --2- 172.21.15.64:0/4007254541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f923015e060 0x7f92300ce310 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.699+0000 7f9235e2a640 1 -- 172.21.15.64:0/4007254541 >> 172.21.15.64:0/4007254541 conn(0x7f92300d1a90 msgr2=0x7f92300d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.699+0000 7f9235e2a640 1 -- 172.21.15.64:0/4007254541 shutdown_connections 2024-03-04T07:33:43.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.699+0000 7f9235e2a640 1 -- 172.21.15.64:0/4007254541 wait complete. 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.700+0000 7f9235e2a640 1 Processor -- start 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.700+0000 7f9235e2a640 1 -- start start 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f9235e2a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f923006fe20 0x7f923015eeb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f9235e2a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f92300d0880 0x7f923006bef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f9235e2a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f923015e060 0x7f923006c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f9235e2a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f92300cef60 con 0x7f923015e060 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f9235e2a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f923015e440 con 0x7f923006fe20 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f9235e2a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f92300cf0e0 con 0x7f92300d0880 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f923006fe20 0x7f923015eeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f923006fe20 0x7f923015eeb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:55010/0 (socket says 172.21.15.64:55010) 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 -- 172.21.15.64:0/2653721050 learned_addr learned my addr 172.21.15.64:0/2653721050 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f923015e060 0x7f923006c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922effd640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f92300d0880 0x7f923006bef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 -- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f92300d0880 msgr2=0x7f923006bef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f92300d0880 0x7f923006bef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 -- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f923015e060 msgr2=0x7f923006c430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f923015e060 0x7f923006c430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 -- 172.21.15.64:0/2653721050 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f92300b4a80 con 0x7f923006fe20 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922ffff640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f923015e060 0x7f923006c430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.701+0000 7f922f7fe640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f923006fe20 0x7f923015eeb0 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f922002b970 tx=0x7f92200046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.702+0000 7f9217fff640 1 -- 172.21.15.64:0/2653721050 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9220035050 con 0x7f923006fe20 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.702+0000 7f9217fff640 1 -- 172.21.15.64:0/2653721050 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9220003820 con 0x7f923006fe20 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.702+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f92300b4d60 con 0x7f923006fe20 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.702+0000 7f9217fff640 1 -- 172.21.15.64:0/2653721050 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9220003bf0 con 0x7f923006fe20 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.702+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f92300b5320 con 0x7f923006fe20 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.703+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9230067ff0 con 0x7f923006fe20 2024-03-04T07:33:43.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.703+0000 7f9217fff640 1 -- 172.21.15.64:0/2653721050 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9220038020 con 0x7f923006fe20 2024-03-04T07:33:43.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.703+0000 7f9217fff640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f921c06b4d0 0x7f921c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:43.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.704+0000 7f9217fff640 1 -- 172.21.15.64:0/2653721050 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f92200ac450 con 0x7f923006fe20 2024-03-04T07:33:43.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.704+0000 7f922effd640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f921c06b4d0 0x7f921c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:43.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.704+0000 7f922effd640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f921c06b4d0 0x7f921c06d990 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f92300b4560 tx=0x7f9224009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:43.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.708+0000 7f9217fff640 1 -- 172.21.15.64:0/2653721050 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f922002d040 con 0x7f923006fe20 2024-03-04T07:33:43.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.928+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f923006b9d0 con 0x7f923006fe20 2024-03-04T07:33:43.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.928+0000 7f9217fff640 1 -- 172.21.15.64:0/2653721050 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v81) v1 ==== 53+0+73 (secure 0 0 0) 0x7f922007ac90 con 0x7f923006fe20 2024-03-04T07:33:43.929 INFO:teuthology.orchestra.run.smithi064.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-04T07:33:43.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f921c06b4d0 msgr2=0x7f921c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f921c06b4d0 0x7f921c06d990 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f92300b4560 tx=0x7f9224009040 comp rx=0 tx=0).stop 2024-03-04T07:33:43.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f923006fe20 msgr2=0x7f923015eeb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f923006fe20 0x7f923015eeb0 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f922002b970 tx=0x7f92200046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 shutdown_connections 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f921c06b4d0 0x7f921c06d990 unknown :-1 s=CLOSED pgs=696 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f923015e060 0x7f923006c430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f92300d0880 0x7f923006bef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 --2- 172.21.15.64:0/2653721050 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f923006fe20 0x7f923015eeb0 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 >> 172.21.15.64:0/2653721050 conn(0x7f92300d1a90 msgr2=0x7f923006af20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 shutdown_connections 2024-03-04T07:33:43.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:43.930+0000 7f9235e2a640 1 -- 172.21.15.64:0/2653721050 wait complete. 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi064.front.sepia.ceph.com 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-04T07:33:43.942 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-04T07:33:43.943 INFO:tasks.cephfs.mount:Setting the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:33:43.943 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:43.943 DEBUG:teuthology.orchestra.run.smithi064:> ip addr 2024-03-04T07:33:43.958 INFO:teuthology.orchestra.run.smithi064.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-04T07:33:43.958 INFO:teuthology.orchestra.run.smithi064.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-04T07:33:43.958 INFO:teuthology.orchestra.run.smithi064.stdout: inet 127.0.0.1/8 scope host lo 2024-03-04T07:33:43.958 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:33:43.958 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 ::1/128 scope host 2024-03-04T07:33:43.958 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:33:43.958 INFO:teuthology.orchestra.run.smithi064.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:78 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f0 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:a9:a3:79 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: altname enp7s0f1 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:de brd ff:ff:ff:ff:ff:ff 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 0c:c4:7a:b6:f8:df brd ff:ff:ff:ff:ff:ff 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: inet 172.21.15.64/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft 42288sec preferred_lft 42288sec 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:33:43.959 INFO:teuthology.orchestra.run.smithi064.stdout:6: ceph-brx: mtu 1500 qdisc noqueue state UP group default qlen 1000 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: inet 192.168.255.254/16 brd 192.168.255.255 scope global ceph-brx 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::709f:2aff:fe4a:d7c/64 scope link 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout:7: brx.0@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether 56:fb:b6:60:a0:c5 brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::54fb:b6ff:fe60:a0c5/64 scope link 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout:8: brx.1@if2: mtu 1500 qdisc noqueue master ceph-brx state UP group default qlen 1000 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: link/ether b6:f3:ac:37:22:ea brd ff:ff:ff:ff:ff:ff link-netns ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: inet6 fe80::b4f3:acff:fe37:22ea/64 scope link 2024-03-04T07:33:43.960 INFO:teuthology.orchestra.run.smithi064.stdout: valid_lft forever preferred_lft forever 2024-03-04T07:33:43.961 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:43.961 DEBUG:teuthology.orchestra.run.smithi064:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-04T07:33:44.023 INFO:teuthology.orchestra.run.smithi064.stdout:1 2024-03-04T07:33:44.025 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.025 DEBUG:teuthology.orchestra.run.smithi064:> route 2024-03-04T07:33:44.081 INFO:teuthology.orchestra.run.smithi064.stdout:Kernel IP routing table 2024-03-04T07:33:44.081 INFO:teuthology.orchestra.run.smithi064.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-04T07:33:44.081 INFO:teuthology.orchestra.run.smithi064.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-04T07:33:44.082 INFO:teuthology.orchestra.run.smithi064.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-04T07:33:44.082 INFO:teuthology.orchestra.run.smithi064.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-04T07:33:44.082 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.082 DEBUG:teuthology.orchestra.run.smithi064:> (cd / && exec bash -c ' 2024-03-04T07:33:44.082 DEBUG:teuthology.orchestra.run.smithi064:> set -e 2024-03-04T07:33:44.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-04T07:33:44.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-04T07:33:44.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-04T07:33:44.082 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:33:44.176 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.176 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:33:44.190 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:33:44.190 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:33:44.190 INFO:tasks.cephfs.mount:Resuming the 'ceph-ns--home-ubuntu-cephtest-mnt.0' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:33:44.190 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link set brx.0 up 2024-03-04T07:33:44.257 INFO:tasks.cephfs.fuse_mount:Client client.0 config is {} 2024-03-04T07:33:44.257 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:33:44.257 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-04T07:33:44.312 INFO:teuthology.orchestra.run.smithi064.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-04T07:33:44.312 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-04T07:33:44.312 DEBUG:teuthology.orchestra.run.smithi064:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:33:44.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:33:44.433 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:33:44.489 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:33:44.490 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:33:44.491 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.491 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:33:44.543 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2024-03-04T07:33:44.543 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.0 --id 0) 2024-03-04T07:33:44.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo modprobe fuse 2024-03-04T07:33:44.612 DEBUG:teuthology.orchestra.run.smithi064:> cat /proc/self/mounts | awk '{print $2}' 2024-03-04T07:33:44.667 INFO:teuthology.orchestra.run.smithi064.stdout:/proc 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/sys 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/dev 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/security 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/shm 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/pts 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/run 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/cgroup 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/pstore 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/bpf 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/ 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/selinux 2024-03-04T07:33:44.668 INFO:teuthology.orchestra.run.smithi064.stdout:/proc/sys/fs/binfmt_misc 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/mqueue 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/debug 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/dev/hugepages 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/tracing 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/fs/fuse/connections 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/sys/kernel/config 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-sysctl.service 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup-dev.service 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/run/credentials/systemd-tmpfiles-setup.service 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/nfs/rpc_pipefs 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/run/user/1000 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-0 2024-03-04T07:33:44.669 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-1 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-2 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:/var/lib/ceph/osd/ceph-3 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:33:44.670 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.670 DEBUG:teuthology.orchestra.run.smithi064:> ls /sys/fs/fuse/connections 2024-03-04T07:33:44.671 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:2024-03-04T07:33:44.671+0000 7f4d2ae47480 -1 init, newargv = 0x55dc49d6d2c0 newargc=15 2024-03-04T07:33:44.671 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[78502]: starting ceph client 2024-03-04T07:33:44.676 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[78502]: starting fuse 2024-03-04T07:33:44.686 INFO:teuthology.orchestra.run.smithi064.stdout:41 2024-03-04T07:33:44.686 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [41] 2024-03-04T07:33:44.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo stdin-killer -- python3 -c ' 2024-03-04T07:33:44.686 DEBUG:teuthology.orchestra.run.smithi064:> import glob 2024-03-04T07:33:44.686 DEBUG:teuthology.orchestra.run.smithi064:> import re 2024-03-04T07:33:44.686 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> import subprocess 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> def _find_admin_socket(client_name): 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> asok_path = "/var/run/ceph/ceph-client.0.*.asok" 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> files = glob.glob(asok_path) 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> mountpoint = "/home/ubuntu/cephtest/mnt.0" 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> # Given a non-glob path, it better be there 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> if "*" not in asok_path: 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> assert(len(files) == 1) 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> return files[0] 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> for f in files: 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2024-03-04T07:33:44.687 DEBUG:teuthology.orchestra.run.smithi064:> if os.path.exists("/proc/{0}".format(pid)): 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> contents = proc_f.read() 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> if mountpoint in contents: 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> return f 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> print(_find_admin_socket("client.0")) 2024-03-04T07:33:44.688 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:33:44.806 INFO:teuthology.orchestra.run.smithi064.stdout:/var/run/ceph/ceph-client.0.78502.asok 2024-03-04T07:33:44.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:44 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:33:44.816 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.0.78502.asok 2024-03-04T07:33:44.816 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.0.78502.asok status 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "metadata": { 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_sha1": "1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd", 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 19.0.0-1739-g1bc250c7 (1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd) squid (dev)", 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "entity_id": "0", 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "hostname": "smithi064", 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.0", 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "pid": "78502", 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "root": "/" 2024-03-04T07:33:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_count": 0, 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "dentry_pinned_count": 0, 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "id": 8727, 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "inst": { 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "name": { 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "client", 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "num": 8727 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 933806186 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": { 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "type": "v1", 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "addr": "192.168.0.1:0", 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "nonce": 933806186 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "inst_str": "client.8727 192.168.0.1:0/933806186", 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "addr_str": "192.168.0.1:0/933806186", 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "inode_count": 1, 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_epoch": 81, 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch": 123, 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_epoch_barrier": 0, 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "blocklisted": false, 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: "fs_name": "cephfs" 2024-03-04T07:33:44.958 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:33:44.964 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.964 DEBUG:teuthology.orchestra.run.smithi064:> stat --file-system '--printf=%T 2024-03-04T07:33:44.964 DEBUG:teuthology.orchestra.run.smithi064:> ' -- /home/ubuntu/cephtest/mnt.0 2024-03-04T07:33:44.979 INFO:teuthology.orchestra.run.smithi064.stdout:fuseblk 2024-03-04T07:33:44.979 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.0 2024-03-04T07:33:44.979 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:44.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-04T07:33:45.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 2 2024-03-04T07:33:45.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.190+0000 7f5018a88640 1 Processor -- start 2024-03-04T07:33:45.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.190+0000 7f5018a88640 1 -- start start 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f5018a88640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f501415dd90 0x7f50140ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f5018a88640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f50140d29d0 0x7f50140d41e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f5018a88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d4720 0x7f50140b6d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f5018a88640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f50140d3400 con 0x7f50140d29d0 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f5018a88640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f50140d3740 con 0x7f501415dd90 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f5018a88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f50140d3120 con 0x7f50140d4720 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f50140d29d0 0x7f50140d41e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:45.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f501259c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f501415dd90 0x7f50140ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f5012d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d4720 0x7f50140b6d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f50140d29d0 0x7f50140d41e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47746/0 (socket says 172.21.15.64:47746) 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 -- 172.21.15.64:0/2155464148 learned_addr learned my addr 172.21.15.64:0/2155464148 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 -- 172.21.15.64:0/2155464148 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d4720 msgr2=0x7f50140b6d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 --2- 172.21.15.64:0/2155464148 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d4720 0x7f50140b6d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 -- 172.21.15.64:0/2155464148 >> v2:172.21.15.66:3300/0 conn(0x7f501415dd90 msgr2=0x7f50140ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 --2- 172.21.15.64:0/2155464148 >> v2:172.21.15.66:3300/0 conn(0x7f501415dd90 0x7f50140ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.191+0000 7f500bfff640 1 -- 172.21.15.64:0/2155464148 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50140d39e0 con 0x7f50140d29d0 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.192+0000 7f500bfff640 1 --2- 172.21.15.64:0/2155464148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d29d0 0x7f50140d41e0 secure :-1 s=READY pgs=1558 cs=0 l=1 rev1=1 crypto rx=0x7f50140d1f90 tx=0x7f4ffc02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c044cc1048528255 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.192+0000 7f5011d9b640 1 -- 172.21.15.64:0/2155464148 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ffc034070 con 0x7f50140d29d0 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.192+0000 7f5011d9b640 1 -- 172.21.15.64:0/2155464148 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ffc030860 con 0x7f50140d29d0 2024-03-04T07:33:45.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.192+0000 7f5011d9b640 1 -- 172.21.15.64:0/2155464148 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ffc030c30 con 0x7f50140d29d0 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 -- 172.21.15.64:0/2155464148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d29d0 msgr2=0x7f50140d41e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 --2- 172.21.15.64:0/2155464148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d29d0 0x7f50140d41e0 secure :-1 s=READY pgs=1558 cs=0 l=1 rev1=1 crypto rx=0x7f50140d1f90 tx=0x7f4ffc02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 -- 172.21.15.64:0/2155464148 shutdown_connections 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 --2- 172.21.15.64:0/2155464148 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d4720 0x7f50140b6d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 --2- 172.21.15.64:0/2155464148 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d29d0 0x7f50140d41e0 unknown :-1 s=CLOSED pgs=1558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 --2- 172.21.15.64:0/2155464148 >> v2:172.21.15.66:3300/0 conn(0x7f501415dd90 0x7f50140ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 -- 172.21.15.64:0/2155464148 >> 172.21.15.64:0/2155464148 conn(0x7f5014071960 msgr2=0x7f5014071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 -- 172.21.15.64:0/2155464148 shutdown_connections 2024-03-04T07:33:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.193+0000 7f5018a88640 1 -- 172.21.15.64:0/2155464148 wait complete. 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.194+0000 7f5018a88640 1 Processor -- start 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.194+0000 7f5018a88640 1 -- start start 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f5018a88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d29d0 0x7f50141e7760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f5018a88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d4720 0x7f50141603d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f5018a88640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f501415dd90 0x7f5014160910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f5018a88640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50140d2df0 con 0x7f50140d4720 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f5018a88640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f501415e170 con 0x7f501415dd90 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f5018a88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f50140b7250 con 0x7f50140d29d0 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d4720 0x7f50141603d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d4720 0x7f50141603d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47748/0 (socket says 172.21.15.64:47748) 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 -- 172.21.15.64:0/3212100497 learned_addr learned my addr 172.21.15.64:0/3212100497 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f5012d9d640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f501415dd90 0x7f5014160910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:45.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f501259c640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d29d0 0x7f50141e7760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 -- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d29d0 msgr2=0x7f50141e7760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d29d0 0x7f50141e7760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 -- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f501415dd90 msgr2=0x7f5014160910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f501415dd90 0x7f5014160910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 -- 172.21.15.64:0/3212100497 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50140ccf80 con 0x7f50140d4720 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.195+0000 7f500bfff640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d4720 0x7f50141603d0 secure :-1 s=READY pgs=1559 cs=0 l=1 rev1=1 crypto rx=0x7f4ffc02b9e0 tx=0x7f4ffc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.196+0000 7f500affd640 1 -- 172.21.15.64:0/3212100497 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ffc048070 con 0x7f50140d4720 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.196+0000 7f500affd640 1 -- 172.21.15.64:0/3212100497 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ffc030e20 con 0x7f50140d4720 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.196+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f501415e840 con 0x7f50140d4720 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.196+0000 7f500affd640 1 -- 172.21.15.64:0/3212100497 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ffc034040 con 0x7f50140d4720 2024-03-04T07:33:45.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.196+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f501415ee00 con 0x7f50140d4720 2024-03-04T07:33:45.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.197+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5014072a30 con 0x7f50140d4720 2024-03-04T07:33:45.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.197+0000 7f500affd640 1 -- 172.21.15.64:0/3212100497 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4ffc04b070 con 0x7f50140d4720 2024-03-04T07:33:45.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.198+0000 7f500affd640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ff406b4d0 0x7f4ff406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:45.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.198+0000 7f501259c640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ff406b4d0 0x7f4ff406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:45.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.198+0000 7f500affd640 1 -- 172.21.15.64:0/3212100497 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4ffc0ac090 con 0x7f50140d4720 2024-03-04T07:33:45.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.198+0000 7f501259c640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ff406b4d0 0x7f4ff406d990 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f50140b47d0 tx=0x7f5004009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:45.203 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.202+0000 7f500affd640 1 -- 172.21.15.64:0/3212100497 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4ffc07a8c0 con 0x7f50140d4720 2024-03-04T07:33:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.425+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"} v 0) v1 -- 0x7f501404f410 con 0x7f50140d4720 2024-03-04T07:33:45.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.928+0000 7f500affd640 1 -- 172.21.15.64:0/3212100497 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"}]=0 v82) v1 ==== 105+0+0 (secure 0 0 0) 0x7f4ffc07e960 con 0x7f50140d4720 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ff406b4d0 msgr2=0x7f4ff406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ff406b4d0 0x7f4ff406d990 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f50140b47d0 tx=0x7f5004009190 comp rx=0 tx=0).stop 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d4720 msgr2=0x7f50141603d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d4720 0x7f50141603d0 secure :-1 s=READY pgs=1559 cs=0 l=1 rev1=1 crypto rx=0x7f4ffc02b9e0 tx=0x7f4ffc004730 comp rx=0 tx=0).stop 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 shutdown_connections 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ff406b4d0 0x7f4ff406d990 unknown :-1 s=CLOSED pgs=697 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f501415dd90 0x7f5014160910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f50140d4720 0x7f50141603d0 unknown :-1 s=CLOSED pgs=1559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 --2- 172.21.15.64:0/3212100497 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f50140d29d0 0x7f50141e7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.930+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 >> 172.21.15.64:0/3212100497 conn(0x7f5014071960 msgr2=0x7f50140b3640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:45.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.931+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 shutdown_connections 2024-03-04T07:33:45.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:45.931+0000 7f5018a88640 1 -- 172.21.15.64:0/3212100497 wait complete. 2024-03-04T07:33:45.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:46.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.048+0000 7f6cd0fe9640 1 Processor -- start 2024-03-04T07:33:46.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.048+0000 7f6cd0fe9640 1 -- start start 2024-03-04T07:33:46.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cd0fe9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc0b1020 0x7f6ccc06af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:46.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cd0fe9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ccc06ed10 0x7f6ccc0cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cd0fe9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6ccc0cc0e0 con 0x7f6ccc0cd6c0 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cd0fe9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ccc0d28b0 con 0x7f6ccc06ed10 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cd0fe9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ccc0cf320 con 0x7f6ccc0b1020 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cc9d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ccc06ed10 0x7f6ccc0cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6ccad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6ccc0cd6c0 0x7f6ccc15d700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60618/0 (socket says 172.21.15.64:60618) 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6ccad9d640 1 -- 172.21.15.64:0/2887932158 learned_addr learned my addr 172.21.15.64:0/2887932158 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cca59c640 1 --2- 172.21.15.64:0/2887932158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc0b1020 0x7f6ccc06af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cc9d9b640 1 -- 172.21.15.64:0/2887932158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc0b1020 msgr2=0x7f6ccc06af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cc9d9b640 1 --2- 172.21.15.64:0/2887932158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc0b1020 0x7f6ccc06af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cc9d9b640 1 -- 172.21.15.64:0/2887932158 >> v1:172.21.15.64:6789/0 conn(0x7f6ccc0cd6c0 legacy=0x7f6ccc15d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cc9d9b640 1 -- 172.21.15.64:0/2887932158 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ccc0cc3c0 con 0x7f6ccc06ed10 2024-03-04T07:33:46.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.049+0000 7f6cc959a640 1 -- 172.21.15.64:0/2887932158 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 643140999 0 0) 0x7f6ccc0cc0e0 con 0x7f6ccc0cd6c0 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.050+0000 7f6cc9d9b640 1 --2- 172.21.15.64:0/2887932158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc06ed10 0x7f6ccc0cd180 secure :-1 s=READY pgs=1561 cs=0 l=1 rev1=1 crypto rx=0x7f6cb40097a0 tx=0x7f6cb402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d4d51b099273180c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.050+0000 7f6cc959a640 1 -- 172.21.15.64:0/2887932158 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cb4035020 con 0x7f6ccc06ed10 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.050+0000 7f6cc959a640 1 -- 172.21.15.64:0/2887932158 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cb4031830 con 0x7f6ccc06ed10 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.050+0000 7f6cc959a640 1 -- 172.21.15.64:0/2887932158 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cb4031c00 con 0x7f6ccc06ed10 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2887932158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc06ed10 msgr2=0x7f6ccc0cd180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2887932158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc06ed10 0x7f6ccc0cd180 secure :-1 s=READY pgs=1561 cs=0 l=1 rev1=1 crypto rx=0x7f6cb40097a0 tx=0x7f6cb402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2887932158 shutdown_connections 2024-03-04T07:33:46.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2887932158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc06ed10 0x7f6ccc0cd180 unknown :-1 s=CLOSED pgs=1561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2887932158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc0b1020 0x7f6ccc06af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2887932158 >> 172.21.15.64:0/2887932158 conn(0x7f6ccc0d1a90 msgr2=0x7f6ccc0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:46.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2887932158 shutdown_connections 2024-03-04T07:33:46.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.051+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2887932158 wait complete. 2024-03-04T07:33:46.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.052+0000 7f6cd0fe9640 1 Processor -- start 2024-03-04T07:33:46.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.052+0000 7f6cd0fe9640 1 -- start start 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cd0fe9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc06ed10 0x7f6ccc150010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cd0fe9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc0b1020 0x7f6ccc0c58b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cd0fe9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ccc0cd6c0 0x7f6ccc0c5df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cd0fe9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ccc0ccc60 con 0x7f6ccc0b1020 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cd0fe9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ccc0cc980 con 0x7f6ccc0cd6c0 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cd0fe9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ccc0cdaf0 con 0x7f6ccc06ed10 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc0b1020 0x7f6ccc0c58b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc0b1020 0x7f6ccc0c58b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47772/0 (socket says 172.21.15.64:47772) 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 -- 172.21.15.64:0/2842735397 learned_addr learned my addr 172.21.15.64:0/2842735397 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cca59c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc06ed10 0x7f6ccc150010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6ccad9d640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ccc0cd6c0 0x7f6ccc0c5df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 -- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc06ed10 msgr2=0x7f6ccc150010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc06ed10 0x7f6ccc150010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 -- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ccc0cd6c0 msgr2=0x7f6ccc0c5df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ccc0cd6c0 0x7f6ccc0c5df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cc9d9b640 1 -- 172.21.15.64:0/2842735397 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ccc0b3240 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6ccad9d640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ccc0cd6c0 0x7f6ccc0c5df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.053+0000 7f6cca59c640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc06ed10 0x7f6ccc150010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.054+0000 7f6cc9d9b640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc0b1020 0x7f6ccc0c58b0 secure :-1 s=READY pgs=1562 cs=0 l=1 rev1=1 crypto rx=0x7f6cb4009890 tx=0x7f6cb40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.054+0000 7f6cc2ffd640 1 -- 172.21.15.64:0/2842735397 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cb4035040 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.054+0000 7f6cc2ffd640 1 -- 172.21.15.64:0/2842735397 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6cb403b920 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.054+0000 7f6cc2ffd640 1 -- 172.21.15.64:0/2842735397 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cb403bd10 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.054+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ccc0b2f60 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.054+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ccc0b5610 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.055+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ccc067ff0 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.055+0000 7f6cc2ffd640 1 -- 172.21.15.64:0/2842735397 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6cb4038020 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.056+0000 7f6cc2ffd640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cac06b500 0x7f6cac06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.056+0000 7f6cc2ffd640 1 -- 172.21.15.64:0/2842735397 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f6cb40ac380 con 0x7f6ccc0b1020 2024-03-04T07:33:46.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.056+0000 7f6cca59c640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cac06b500 0x7f6cac06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:46.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.056+0000 7f6cca59c640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cac06b500 0x7f6cac06d9c0 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f6cc4009f30 tx=0x7f6cc4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:46.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.060+0000 7f6cc2ffd640 1 -- 172.21.15.64:0/2842735397 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6cb407b170 con 0x7f6ccc0b1020 2024-03-04T07:33:46.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.280+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6ccc0d1e70 con 0x7f6ccc0b1020 2024-03-04T07:33:46.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.280+0000 7f6cc2ffd640 1 -- 172.21.15.64:0/2842735397 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 83 v83) v1 ==== 94+0+4992 (secure 0 0 0) 0x7f6cb4031800 con 0x7f6ccc0b1020 2024-03-04T07:33:46.281 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:46.282 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":83,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6713,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3217233400","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3217233400},{"type":"v1","addr":"172.21.15.64:6836","nonce":3217233400}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77},{"gid":6716,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2523962873","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2523962873},{"type":"v1","addr":"172.21.15.64:6837","nonce":2523962873}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77}],"filesystems":[{"mdsmap":{"epoch":83,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:45.931887+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:creating","state_seq":1,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:33:46.282 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 83 2024-03-04T07:33:46.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cac06b500 msgr2=0x7f6cac06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:46.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cac06b500 0x7f6cac06d9c0 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f6cc4009f30 tx=0x7f6cc4009190 comp rx=0 tx=0).stop 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc0b1020 msgr2=0x7f6ccc0c58b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc0b1020 0x7f6ccc0c58b0 secure :-1 s=READY pgs=1562 cs=0 l=1 rev1=1 crypto rx=0x7f6cb4009890 tx=0x7f6cb40046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 shutdown_connections 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6cac06b500 0x7f6cac06d9c0 unknown :-1 s=CLOSED pgs=698 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ccc0cd6c0 0x7f6ccc0c5df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ccc0b1020 0x7f6ccc0c58b0 unknown :-1 s=CLOSED pgs=1562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.282+0000 7f6cd0fe9640 1 --2- 172.21.15.64:0/2842735397 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ccc06ed10 0x7f6ccc150010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.283+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 >> 172.21.15.64:0/2842735397 conn(0x7f6ccc0d1a90 msgr2=0x7f6ccc0b0310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.283+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 shutdown_connections 2024-03-04T07:33:46.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:46.283+0000 7f6cd0fe9640 1 -- 172.21.15.64:0/2842735397 wait complete. 2024-03-04T07:33:46.294 DEBUG:tasks.ceph_test_case:wait_until_equal: 1 != 2, waiting (timeout=30)... 2024-03-04T07:33:47.869 INFO:tasks.ceph.mds.d.smithi066.stderr:2024-03-04T07:33:47.868+0000 7f246d66b640 -1 mds.pinger is_rank_lagging: rank=1 was never sent ping request. 2024-03-04T07:33:48.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:33:48.139 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:33:51.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:51.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.408+0000 7fcd32a47640 1 Processor -- start 2024-03-04T07:33:51.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.408+0000 7fcd32a47640 1 -- start start 2024-03-04T07:33:51.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd32a47640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcd2c06df30 0x7fcd2c0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.410 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd32a47640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcd2c071240 con 0x7fcd2c158520 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd32a47640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcd2c071520 con 0x7fcd2c0aec40 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd32a47640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcd2c0af0f0 con 0x7fcd2c06df30 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcd2c158520 0x7fcd2c06d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60626/0 (socket says 172.21.15.64:60626) 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2bfff640 1 -- 172.21.15.64:0/3375612045 learned_addr learned my addr 172.21.15.64:0/3375612045 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2b7fe640 1 --2- 172.21.15.64:0/3375612045 >> v2:172.21.15.66:3300/0 conn(0x7fcd2c06df30 0x7fcd2c0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2affd640 1 -- 172.21.15.64:0/3375612045 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3508423623 0 0) 0x7fcd2c071240 con 0x7fcd2c158520 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2affd640 1 -- 172.21.15.64:0/3375612045 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcd14003520 con 0x7fcd2c158520 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2affd640 1 -- 172.21.15.64:0/3375612045 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4226056536 0 0) 0x7fcd2c071520 con 0x7fcd2c0aec40 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2affd640 1 -- 172.21.15.64:0/3375612045 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fcd2c071240 con 0x7fcd2c0aec40 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2b7fe640 1 -- 172.21.15.64:0/3375612045 >> v1:172.21.15.66:6789/0 conn(0x7fcd2c0aec40 legacy=0x7fcd2c0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2b7fe640 1 -- 172.21.15.64:0/3375612045 >> v1:172.21.15.64:6789/0 conn(0x7fcd2c158520 legacy=0x7fcd2c06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.411 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.409+0000 7fcd2b7fe640 1 -- 172.21.15.64:0/3375612045 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcd2c071800 con 0x7fcd2c06df30 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.410+0000 7fcd2b7fe640 1 --2- 172.21.15.64:0/3375612045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c06df30 0x7fcd2c0ae700 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fcd1c0097d0 tx=0x7fcd1c02b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3fc4450d5e12a31c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.410+0000 7fcd2affd640 1 -- 172.21.15.64:0/3375612045 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd1c034070 con 0x7fcd2c06df30 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd2affd640 1 -- 172.21.15.64:0/3375612045 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcd1c030610 con 0x7fcd2c06df30 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd2affd640 1 -- 172.21.15.64:0/3375612045 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd1c0309c0 con 0x7fcd2c06df30 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd32a47640 1 -- 172.21.15.64:0/3375612045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c06df30 msgr2=0x7fcd2c0ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd32a47640 1 --2- 172.21.15.64:0/3375612045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c06df30 0x7fcd2c0ae700 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fcd1c0097d0 tx=0x7fcd1c02b960 comp rx=0 tx=0).stop 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd32a47640 1 -- 172.21.15.64:0/3375612045 shutdown_connections 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd32a47640 1 --2- 172.21.15.64:0/3375612045 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c06df30 0x7fcd2c0ae700 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd32a47640 1 -- 172.21.15.64:0/3375612045 >> 172.21.15.64:0/3375612045 conn(0x7fcd2c066d00 msgr2=0x7fcd2c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd32a47640 1 -- 172.21.15.64:0/3375612045 shutdown_connections 2024-03-04T07:33:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.411+0000 7fcd32a47640 1 -- 172.21.15.64:0/3375612045 wait complete. 2024-03-04T07:33:51.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.412+0000 7fcd32a47640 1 Processor -- start 2024-03-04T07:33:51.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd32a47640 1 -- start start 2024-03-04T07:33:51.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd32a47640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcd2c06df30 0x7fcd2c14feb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.415 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd32a47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcd2c0aec40 0x7fcd2c141010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd32a47640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c158520 0x7fcd2c1442c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd32a47640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd2c158900 con 0x7fcd2c06df30 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd32a47640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd2c071b70 con 0x7fcd2c158520 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd32a47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcd2c158a80 con 0x7fcd2c0aec40 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcd2c06df30 0x7fcd2c14feb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd2bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcd2c06df30 0x7fcd2c14feb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47778/0 (socket says 172.21.15.64:47778) 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd2bfff640 1 -- 172.21.15.64:0/411439281 learned_addr learned my addr 172.21.15.64:0/411439281 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd2b7fe640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcd2c0aec40 0x7fcd2c141010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.413+0000 7fcd30fe4640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c158520 0x7fcd2c1442c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd2bfff640 1 -- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcd2c0aec40 msgr2=0x7fcd2c141010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd2bfff640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcd2c0aec40 0x7fcd2c141010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd2bfff640 1 -- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c158520 msgr2=0x7fcd2c1442c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd2bfff640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c158520 0x7fcd2c1442c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd2bfff640 1 -- 172.21.15.64:0/411439281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcd2c150e50 con 0x7fcd2c06df30 2024-03-04T07:33:51.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd2bfff640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcd2c06df30 0x7fcd2c14feb0 secure :-1 s=READY pgs=1564 cs=0 l=1 rev1=1 crypto rx=0x7fcd2000c500 tx=0x7fcd2000c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:51.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd28ff9640 1 -- 172.21.15.64:0/411439281 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd20017070 con 0x7fcd2c06df30 2024-03-04T07:33:51.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcd2c1505b0 con 0x7fcd2c06df30 2024-03-04T07:33:51.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd28ff9640 1 -- 172.21.15.64:0/411439281 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcd20005ce0 con 0x7fcd2c06df30 2024-03-04T07:33:51.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd28ff9640 1 -- 172.21.15.64:0/411439281 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcd200072f0 con 0x7fcd2c06df30 2024-03-04T07:33:51.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.414+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcd2c150b70 con 0x7fcd2c06df30 2024-03-04T07:33:51.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.415+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcd2c067ff0 con 0x7fcd2c06df30 2024-03-04T07:33:51.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.416+0000 7fcd28ff9640 1 -- 172.21.15.64:0/411439281 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcd2001a070 con 0x7fcd2c06df30 2024-03-04T07:33:51.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.416+0000 7fcd28ff9640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcd1806b500 0x7fcd1806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.416+0000 7fcd2b7fe640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcd1806b500 0x7fcd1806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.416+0000 7fcd28ff9640 1 -- 172.21.15.64:0/411439281 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fcd200894e0 con 0x7fcd2c06df30 2024-03-04T07:33:51.418 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.417+0000 7fcd2b7fe640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcd1806b500 0x7fcd1806d9c0 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7fcd1c036000 tx=0x7fcd1c03c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:51.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.421+0000 7fcd28ff9640 1 -- 172.21.15.64:0/411439281 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcd20057d20 con 0x7fcd2c06df30 2024-03-04T07:33:51.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.640+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fcd2c0b31f0 con 0x7fcd2c06df30 2024-03-04T07:33:51.641 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.640+0000 7fcd28ff9640 1 -- 172.21.15.64:0/411439281 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4995 (secure 0 0 0) 0x7fcd2005bdc0 con 0x7fcd2c06df30 2024-03-04T07:33:51.641 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:51.641 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":84,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6713,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3217233400","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3217233400},{"type":"v1","addr":"172.21.15.64:6836","nonce":3217233400}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77},{"gid":6716,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2523962873","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2523962873},{"type":"v1","addr":"172.21.15.64:6837","nonce":2523962873}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:46.932881+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:active","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[6728,8577]},"id":9}]} 2024-03-04T07:33:51.642 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 84 2024-03-04T07:33:51.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcd1806b500 msgr2=0x7fcd1806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcd1806b500 0x7fcd1806d9c0 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7fcd1c036000 tx=0x7fcd1c03c040 comp rx=0 tx=0).stop 2024-03-04T07:33:51.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcd2c06df30 msgr2=0x7fcd2c14feb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcd2c06df30 0x7fcd2c14feb0 secure :-1 s=READY pgs=1564 cs=0 l=1 rev1=1 crypto rx=0x7fcd2000c500 tx=0x7fcd2000c9c0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 shutdown_connections 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcd1806b500 0x7fcd1806d9c0 unknown :-1 s=CLOSED pgs=699 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcd2c158520 0x7fcd2c1442c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcd2c0aec40 0x7fcd2c141010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 --2- 172.21.15.64:0/411439281 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcd2c06df30 0x7fcd2c14feb0 unknown :-1 s=CLOSED pgs=1564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 >> 172.21.15.64:0/411439281 conn(0x7fcd2c066d00 msgr2=0x7fcd2c158ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 shutdown_connections 2024-03-04T07:33:51.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.642+0000 7fcd32a47640 1 -- 172.21.15.64:0/411439281 wait complete. 2024-03-04T07:33:51.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:51.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.756+0000 7f72d81c9640 1 Processor -- start 2024-03-04T07:33:51.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.756+0000 7f72d81c9640 1 -- start start 2024-03-04T07:33:51.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d81c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d81c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f72d00b55d0 0x7f72d0071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d81c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72d00b4160 con 0x7f72d006e700 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d81c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f72d00b5b40 con 0x7f72d00b55d0 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d81c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72d00b3ba0 con 0x7f72d0072660 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d5f65640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f72d006e700 0x7f72d006d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60628/0 (socket says 172.21.15.64:60628) 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d5f65640 1 -- 172.21.15.64:0/3832199346 learned_addr learned my addr 172.21.15.64:0/3832199346 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d6766640 1 --2- 172.21.15.64:0/3832199346 >> v2:172.21.15.66:3300/0 conn(0x7f72d00b55d0 0x7f72d0071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d5764640 1 --2- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d4f63640 1 -- 172.21.15.64:0/3832199346 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1033583238 0 0) 0x7f72d00b4160 con 0x7f72d006e700 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d4f63640 1 -- 172.21.15.64:0/3832199346 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f72b8003520 con 0x7f72d006e700 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d6766640 1 -- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 msgr2=0x7f72d015ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d6766640 1 --2- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d015ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d6766640 1 -- 172.21.15.64:0/3832199346 >> v1:172.21.15.64:6789/0 conn(0x7f72d006e700 legacy=0x7f72d006d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d6766640 1 -- 172.21.15.64:0/3832199346 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72d00b3e80 con 0x7f72d00b55d0 2024-03-04T07:33:51.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.757+0000 7f72d5764640 1 --2- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d015ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.758+0000 7f72d6766640 1 --2- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 0x7f72d0071980 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f72c0009d90 tx=0x7f72c002bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9928b9e81cea1f5b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.758+0000 7f72d4f63640 1 -- 172.21.15.64:0/3832199346 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72c0039070 con 0x7f72d00b55d0 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.758+0000 7f72d4f63640 1 -- 172.21.15.64:0/3832199346 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72c002cc50 con 0x7f72d00b55d0 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.758+0000 7f72d4f63640 1 -- 172.21.15.64:0/3832199346 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72c0034440 con 0x7f72d00b55d0 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 -- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 msgr2=0x7f72d0071980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 --2- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 0x7f72d0071980 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f72c0009d90 tx=0x7f72c002bea0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 -- 172.21.15.64:0/3832199346 shutdown_connections 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 --2- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 0x7f72d0071980 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 --2- 172.21.15.64:0/3832199346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d015ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 -- 172.21.15.64:0/3832199346 >> 172.21.15.64:0/3832199346 conn(0x7f72d00b2720 msgr2=0x7f72d00b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 -- 172.21.15.64:0/3832199346 shutdown_connections 2024-03-04T07:33:51.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.759+0000 7f72d81c9640 1 -- 172.21.15.64:0/3832199346 wait complete. 2024-03-04T07:33:51.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.760+0000 7f72d81c9640 1 Processor -- start 2024-03-04T07:33:51.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.760+0000 7f72d81c9640 1 -- start start 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d81c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72d006e700 0x7f72d0154770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d81c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d00cfa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d81c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 0x7f72d00cff50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d81c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72d0072040 con 0x7f72d006e700 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d81c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72d0071ec0 con 0x7f72d00b55d0 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d81c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72d00721c0 con 0x7f72d0072660 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72d006e700 0x7f72d0154770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72d006e700 0x7f72d0154770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47790/0 (socket says 172.21.15.64:47790) 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 -- 172.21.15.64:0/2182474082 learned_addr learned my addr 172.21.15.64:0/2182474082 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d6766640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 0x7f72d00cff50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5764640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d00cfa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 -- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 msgr2=0x7f72d00cfa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d00cfa10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 -- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 msgr2=0x7f72d00cff50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 0x7f72d00cff50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 -- 172.21.15.64:0/2182474082 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72d0158d60 con 0x7f72d006e700 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.761+0000 7f72d5f65640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72d006e700 0x7f72d0154770 secure :-1 s=READY pgs=1566 cs=0 l=1 rev1=1 crypto rx=0x7f72c800c430 tx=0x7f72c800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.762+0000 7f72ce7fc640 1 -- 172.21.15.64:0/2182474082 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72c8017070 con 0x7f72d006e700 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.762+0000 7f72ce7fc640 1 -- 172.21.15.64:0/2182474082 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f72c8005ce0 con 0x7f72d006e700 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.762+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72d0159040 con 0x7f72d006e700 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.762+0000 7f72ce7fc640 1 -- 172.21.15.64:0/2182474082 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72c80072f0 con 0x7f72d006e700 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.762+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72d0159600 con 0x7f72d006e700 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.763+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f72d0067ff0 con 0x7f72d006e700 2024-03-04T07:33:51.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.763+0000 7f72ce7fc640 1 -- 172.21.15.64:0/2182474082 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f72c801a070 con 0x7f72d006e700 2024-03-04T07:33:51.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.763+0000 7f72ce7fc640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f72bc06b4d0 0x7f72bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:51.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.764+0000 7f72ce7fc640 1 -- 172.21.15.64:0/2182474082 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f72c8089570 con 0x7f72d006e700 2024-03-04T07:33:51.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.764+0000 7f72d5764640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f72bc06b4d0 0x7f72bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:51.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.764+0000 7f72d5764640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f72bc06b4d0 0x7f72bc06d990 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f72c4009560 tx=0x7f72c4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:51.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.768+0000 7f72ce7fc640 1 -- 172.21.15.64:0/2182474082 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f72c8057e30 con 0x7f72d006e700 2024-03-04T07:33:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.987+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f72d0072c30 con 0x7f72d006e700 2024-03-04T07:33:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.987+0000 7f72ce7fc640 1 -- 172.21.15.64:0/2182474082 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 84 v84) v1 ==== 94+0+4995 (secure 0 0 0) 0x7f72c805bed0 con 0x7f72d006e700 2024-03-04T07:33:51.987 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:51.987 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":84,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":6713,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6836/3217233400","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6834","nonce":3217233400},{"type":"v1","addr":"172.21.15.64:6836","nonce":3217233400}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77},{"gid":6716,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.64:6837/2523962873","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6835","nonce":2523962873},{"type":"v1","addr":"172.21.15.64:6837","nonce":2523962873}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":77}],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:46.932881+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:active","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[6728,8577]},"id":9}]} 2024-03-04T07:33:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 84 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f72bc06b4d0 msgr2=0x7f72bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f72bc06b4d0 0x7f72bc06d990 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f72c4009560 tx=0x7f72c4009190 comp rx=0 tx=0).stop 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72d006e700 msgr2=0x7f72d0154770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72d006e700 0x7f72d0154770 secure :-1 s=READY pgs=1566 cs=0 l=1 rev1=1 crypto rx=0x7f72c800c430 tx=0x7f72c800c8f0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 shutdown_connections 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f72bc06b4d0 0x7f72bc06d990 unknown :-1 s=CLOSED pgs=700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72d00b55d0 0x7f72d00cff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f72d0072660 0x7f72d00cfa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 --2- 172.21.15.64:0/2182474082 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72d006e700 0x7f72d0154770 unknown :-1 s=CLOSED pgs=1566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 >> 172.21.15.64:0/2182474082 conn(0x7f72d00b2720 msgr2=0x7f72d00b1f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:51.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 shutdown_connections 2024-03-04T07:33:51.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:51.989+0000 7f72d81c9640 1 -- 172.21.15.64:0/2182474082 wait complete. 2024-03-04T07:33:52.000 INFO:tasks.cephfs.test_strays:Ranks 0 and 1 are d and b 2024-03-04T07:33:52.000 DEBUG:tasks.ceph.mds.a:waiting for process to exit 2024-03-04T07:33:52.000 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:33:52.008 INFO:tasks.ceph.mds.a:Stopped 2024-03-04T07:33:52.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail a 2024-03-04T07:33:52.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.151+0000 7ff5c6c9a640 1 Processor -- start 2024-03-04T07:33:52.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.151+0000 7ff5c6c9a640 1 -- start start 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c6c9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c0071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c6c9a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff5c0071a80 0x7ff5c00b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c6c9a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff5c00b6f20 con 0x7ff5c015df70 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c6c9a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff5c00d3030 con 0x7ff5c0071a80 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c6c9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5c00d3310 con 0x7ff5c00d18c0 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c4a36640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff5c015df70 0x7ff5c00d27b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:60632/0 (socket says 172.21.15.64:60632) 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c4a36640 1 -- 172.21.15.64:0/749897878 learned_addr learned my addr 172.21.15.64:0/749897878 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5bffff640 1 --2- 172.21.15.64:0/749897878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c0071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5c5237640 1 --2- 172.21.15.64:0/749897878 >> v2:172.21.15.66:3300/0 conn(0x7ff5c0071a80 0x7ff5c00b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.152+0000 7ff5bf7fe640 1 -- 172.21.15.64:0/749897878 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1079803938 0 0) 0x7ff5c00b6f20 con 0x7ff5c015df70 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.153+0000 7ff5bf7fe640 1 -- 172.21.15.64:0/749897878 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff5a8003520 con 0x7ff5c015df70 2024-03-04T07:33:52.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.153+0000 7ff5bffff640 1 -- 172.21.15.64:0/749897878 >> v2:172.21.15.66:3300/0 conn(0x7ff5c0071a80 msgr2=0x7ff5c00b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.153+0000 7ff5bffff640 1 --2- 172.21.15.64:0/749897878 >> v2:172.21.15.66:3300/0 conn(0x7ff5c0071a80 0x7ff5c00b2700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.153+0000 7ff5bffff640 1 -- 172.21.15.64:0/749897878 >> v1:172.21.15.64:6789/0 conn(0x7ff5c015df70 legacy=0x7ff5c00d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.153+0000 7ff5bffff640 1 -- 172.21.15.64:0/749897878 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5c00d35f0 con 0x7ff5c00d18c0 2024-03-04T07:33:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.153+0000 7ff5bffff640 1 --2- 172.21.15.64:0/749897878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c0071540 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7ff5b40097a0 tx=0x7ff5b402b960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1e29267fc6990197 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5bf7fe640 1 -- 172.21.15.64:0/749897878 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5b4034070 con 0x7ff5c00d18c0 2024-03-04T07:33:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5bf7fe640 1 -- 172.21.15.64:0/749897878 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5b4030610 con 0x7ff5c00d18c0 2024-03-04T07:33:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5bf7fe640 1 -- 172.21.15.64:0/749897878 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5b40309c0 con 0x7ff5c00d18c0 2024-03-04T07:33:52.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/749897878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 msgr2=0x7ff5c0071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:52.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/749897878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c0071540 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7ff5b40097a0 tx=0x7ff5b402b960 comp rx=0 tx=0).stop 2024-03-04T07:33:52.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/749897878 shutdown_connections 2024-03-04T07:33:52.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/749897878 >> v2:172.21.15.66:3300/0 conn(0x7ff5c0071a80 0x7ff5c00b2700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:52.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/749897878 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c0071540 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:52.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/749897878 >> 172.21.15.64:0/749897878 conn(0x7ff5c00d3cf0 msgr2=0x7ff5c00d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:52.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/749897878 shutdown_connections 2024-03-04T07:33:52.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.154+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/749897878 wait complete. 2024-03-04T07:33:52.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.155+0000 7ff5c6c9a640 1 Processor -- start 2024-03-04T07:33:52.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.155+0000 7ff5c6c9a640 1 -- start start 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c6c9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5c0071a80 0x7ff5c00ce440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c6c9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c00b51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c6c9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff5c015df70 0x7ff5c00b56f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c6c9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5c015e4d0 con 0x7ff5c0071a80 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c6c9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5c015e350 con 0x7ff5c015df70 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c6c9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff5c00b2c40 con 0x7ff5c00d18c0 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c4a36640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5c0071a80 0x7ff5c00ce440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c4a36640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5c0071a80 0x7ff5c00ce440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47802/0 (socket says 172.21.15.64:47802) 2024-03-04T07:33:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c4a36640 1 -- 172.21.15.64:0/4031543693 learned_addr learned my addr 172.21.15.64:0/4031543693 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5bffff640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c00b51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.156+0000 7ff5c5237640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff5c015df70 0x7ff5c00b56f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c4a36640 1 -- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 msgr2=0x7ff5c00b51b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c4a36640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c00b51b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c4a36640 1 -- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff5c015df70 msgr2=0x7ff5c00b56f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c4a36640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff5c015df70 0x7ff5c00b56f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c4a36640 1 -- 172.21.15.64:0/4031543693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff5c00b5d90 con 0x7ff5c0071a80 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c4a36640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5c0071a80 0x7ff5c00ce440 secure :-1 s=READY pgs=1568 cs=0 l=1 rev1=1 crypto rx=0x7ff5b800caa0 tx=0x7ff5b800cf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5bd7fa640 1 -- 172.21.15.64:0/4031543693 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5b8017070 con 0x7ff5c0071a80 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5bd7fa640 1 -- 172.21.15.64:0/4031543693 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff5b800ac00 con 0x7ff5c0071a80 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5bd7fa640 1 -- 172.21.15.64:0/4031543693 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff5b8012400 con 0x7ff5c0071a80 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff5c014cc90 con 0x7ff5c0071a80 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.157+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff5c014d250 con 0x7ff5c0071a80 2024-03-04T07:33:52.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.158+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff5c0067ff0 con 0x7ff5c0071a80 2024-03-04T07:33:52.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.158+0000 7ff5bd7fa640 1 -- 172.21.15.64:0/4031543693 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff5b801a070 con 0x7ff5c0071a80 2024-03-04T07:33:52.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.159+0000 7ff5bd7fa640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff5ac06b4d0 0x7ff5ac06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:52.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.159+0000 7ff5bd7fa640 1 -- 172.21.15.64:0/4031543693 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(123..123 src has 1..123) v4 ==== 5709+0+0 (secure 0 0 0) 0x7ff5b8089fa0 con 0x7ff5c0071a80 2024-03-04T07:33:52.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.159+0000 7ff5bffff640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff5ac06b4d0 0x7ff5ac06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:52.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.159+0000 7ff5bffff640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff5ac06b4d0 0x7ff5ac06d990 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7ff5b4036000 tx=0x7ff5b403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:52.163 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.163+0000 7ff5bd7fa640 1 -- 172.21.15.64:0/4031543693 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff5b8058750 con 0x7ff5c0071a80 2024-03-04T07:33:52.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:52.417+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "a"} v 0) v1 -- 0x7ff5c014d420 con 0x7ff5c0071a80 2024-03-04T07:33:53.278 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.278+0000 7ff5bd7fa640 1 -- 172.21.15.64:0/4031543693 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "a"}]=0 failed mds gid 6716 v85) v1 ==== 95+0+0 (secure 0 0 0) 0x7ff5b805c7f0 con 0x7ff5c0071a80 2024-03-04T07:33:53.278 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 6716 2024-03-04T07:33:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff5ac06b4d0 msgr2=0x7ff5ac06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff5ac06b4d0 0x7ff5ac06d990 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7ff5b4036000 tx=0x7ff5b403b040 comp rx=0 tx=0).stop 2024-03-04T07:33:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5c0071a80 msgr2=0x7ff5c00ce440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5c0071a80 0x7ff5c00ce440 secure :-1 s=READY pgs=1568 cs=0 l=1 rev1=1 crypto rx=0x7ff5b800caa0 tx=0x7ff5b800cf60 comp rx=0 tx=0).stop 2024-03-04T07:33:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 shutdown_connections 2024-03-04T07:33:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff5ac06b4d0 0x7ff5ac06d990 unknown :-1 s=CLOSED pgs=701 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.281 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff5c015df70 0x7ff5c00b56f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff5c00d18c0 0x7ff5c00b51b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 --2- 172.21.15.64:0/4031543693 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff5c0071a80 0x7ff5c00ce440 unknown :-1 s=CLOSED pgs=1568 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 >> 172.21.15.64:0/4031543693 conn(0x7ff5c00d3cf0 msgr2=0x7ff5c015d1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:53.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 shutdown_connections 2024-03-04T07:33:53.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.281+0000 7ff5c6c9a640 1 -- 172.21.15.64:0/4031543693 wait complete. 2024-03-04T07:33:53.295 DEBUG:tasks.ceph.mds.c:waiting for process to exit 2024-03-04T07:33:53.295 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:33:53.305 INFO:tasks.ceph.mds.c:Stopped 2024-03-04T07:33:53.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mds fail c 2024-03-04T07:33:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.454+0000 7fa29715e640 1 Processor -- start 2024-03-04T07:33:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.454+0000 7fa29715e640 1 -- start start 2024-03-04T07:33:53.455 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.455+0000 7fa29715e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa2900d02e0 0x7fa2900cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.455+0000 7fa29715e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa2900b2320 con 0x7fa2900d06c0 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.455+0000 7fa29715e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa2900b0dd0 con 0x7fa2900b4260 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.455+0000 7fa29715e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa2900b0b80 con 0x7fa2900d02e0 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.455+0000 7fa294efa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa2900d02e0 0x7fa2900cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.455+0000 7fa2956fb640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa2900d06c0 0x7fa2900b1c10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42374/0 (socket says 172.21.15.64:42374) 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.455+0000 7fa2956fb640 1 -- 172.21.15.64:0/2730034706 learned_addr learned my addr 172.21.15.64:0/2730034706 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.456+0000 7fa294efa640 1 -- 172.21.15.64:0/2730034706 >> v1:172.21.15.66:6789/0 conn(0x7fa2900b4260 legacy=0x7fa2900b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.456+0000 7fa294efa640 1 -- 172.21.15.64:0/2730034706 >> v1:172.21.15.64:6789/0 conn(0x7fa2900d06c0 legacy=0x7fa2900b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.456+0000 7fa28f7fe640 1 -- 172.21.15.64:0/2730034706 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3551132569 0 0) 0x7fa2900b2320 con 0x7fa2900d06c0 2024-03-04T07:33:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.456+0000 7fa294efa640 1 -- 172.21.15.64:0/2730034706 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2900b10b0 con 0x7fa2900d02e0 2024-03-04T07:33:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.456+0000 7fa294efa640 1 --2- 172.21.15.64:0/2730034706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d02e0 0x7fa2900cfa00 secure :-1 s=READY pgs=1570 cs=0 l=1 rev1=1 crypto rx=0x7fa2800097a0 tx=0x7fa28002b9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8726bbffe368ae2d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.456+0000 7fa28f7fe640 1 -- 172.21.15.64:0/2730034706 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa28002f810 con 0x7fa2900d02e0 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa28f7fe640 1 -- 172.21.15.64:0/2730034706 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa28002f9b0 con 0x7fa2900d02e0 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa28f7fe640 1 -- 172.21.15.64:0/2730034706 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa28002fd80 con 0x7fa2900d02e0 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa29715e640 1 -- 172.21.15.64:0/2730034706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d02e0 msgr2=0x7fa2900cfa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa29715e640 1 --2- 172.21.15.64:0/2730034706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d02e0 0x7fa2900cfa00 secure :-1 s=READY pgs=1570 cs=0 l=1 rev1=1 crypto rx=0x7fa2800097a0 tx=0x7fa28002b9f0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa29715e640 1 -- 172.21.15.64:0/2730034706 shutdown_connections 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa29715e640 1 --2- 172.21.15.64:0/2730034706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d02e0 0x7fa2900cfa00 unknown :-1 s=CLOSED pgs=1570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa29715e640 1 -- 172.21.15.64:0/2730034706 >> 172.21.15.64:0/2730034706 conn(0x7fa2900b36d0 msgr2=0x7fa2900b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa29715e640 1 -- 172.21.15.64:0/2730034706 shutdown_connections 2024-03-04T07:33:53.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.457+0000 7fa29715e640 1 -- 172.21.15.64:0/2730034706 wait complete. 2024-03-04T07:33:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.458+0000 7fa29715e640 1 Processor -- start 2024-03-04T07:33:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.458+0000 7fa29715e640 1 -- start start 2024-03-04T07:33:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa29715e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2900b4260 0x7fa290146b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa29715e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2900d02e0 0x7fa290154cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:53.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa29715e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d06c0 0x7fa290142a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa29715e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2900b0360 con 0x7fa2900d06c0 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa29715e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2900aeec0 con 0x7fa2900d02e0 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa29715e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa2900b04e0 con 0x7fa2900b4260 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa2956fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d06c0 0x7fa290142a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa2956fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d06c0 0x7fa290142a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50910/0 (socket says 172.21.15.64:50910) 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa2956fb640 1 -- 172.21.15.64:0/685249725 learned_addr learned my addr 172.21.15.64:0/685249725 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa294efa640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2900b4260 0x7fa290146b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.459+0000 7fa28ffff640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2900d02e0 0x7fa290154cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa2956fb640 1 -- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2900b4260 msgr2=0x7fa290146b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa2956fb640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2900b4260 0x7fa290146b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa2956fb640 1 -- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2900d02e0 msgr2=0x7fa290154cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa2956fb640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2900d02e0 0x7fa290154cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa2956fb640 1 -- 172.21.15.64:0/685249725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2900c9680 con 0x7fa2900d06c0 2024-03-04T07:33:53.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa2956fb640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d06c0 0x7fa290142a90 secure :-1 s=READY pgs=1571 cs=0 l=1 rev1=1 crypto rx=0x7fa28800c7a0 tx=0x7fa28800cc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:53.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa28d7fa640 1 -- 172.21.15.64:0/685249725 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa288018070 con 0x7fa2900d06c0 2024-03-04T07:33:53.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa28d7fa640 1 -- 172.21.15.64:0/685249725 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa28800a640 con 0x7fa2900d06c0 2024-03-04T07:33:53.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa2900c9960 con 0x7fa2900d06c0 2024-03-04T07:33:53.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa28d7fa640 1 -- 172.21.15.64:0/685249725 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa28800aab0 con 0x7fa2900d06c0 2024-03-04T07:33:53.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.460+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa2900ca440 con 0x7fa2900d06c0 2024-03-04T07:33:53.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.461+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa290068700 con 0x7fa2900d06c0 2024-03-04T07:33:53.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.461+0000 7fa28d7fa640 1 -- 172.21.15.64:0/685249725 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa28801b070 con 0x7fa2900d06c0 2024-03-04T07:33:53.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.462+0000 7fa28d7fa640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa27c06b4d0 0x7fa27c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:53.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.462+0000 7fa28d7fa640 1 -- 172.21.15.64:0/685249725 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(124..124 src has 1..124) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa2880896f0 con 0x7fa2900d06c0 2024-03-04T07:33:53.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.462+0000 7fa294efa640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa27c06b4d0 0x7fa27c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:53.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.462+0000 7fa294efa640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa27c06b4d0 0x7fa27c06d990 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7fa280036000 tx=0x7fa280008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:53.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.466+0000 7fa28d7fa640 1 -- 172.21.15.64:0/685249725 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa288057fb0 con 0x7fa2900d06c0 2024-03-04T07:33:53.718 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:53.718+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "mds fail", "role_or_gid": "c"} v 0) v1 -- 0x7fa29006f4f0 con 0x7fa2900d06c0 2024-03-04T07:33:54.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.284+0000 7fa28d7fa640 1 -- 172.21.15.64:0/685249725 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "mds fail", "role_or_gid": "c"}]=0 failed mds gid 6713 v86) v1 ==== 95+0+0 (secure 0 0 0) 0x7fa28805c050 con 0x7fa2900d06c0 2024-03-04T07:33:54.284 INFO:teuthology.orchestra.run.smithi064.stderr:failed mds gid 6713 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa27c06b4d0 msgr2=0x7fa27c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa27c06b4d0 0x7fa27c06d990 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7fa280036000 tx=0x7fa280008030 comp rx=0 tx=0).stop 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d06c0 msgr2=0x7fa290142a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d06c0 0x7fa290142a90 secure :-1 s=READY pgs=1571 cs=0 l=1 rev1=1 crypto rx=0x7fa28800c7a0 tx=0x7fa28800cc60 comp rx=0 tx=0).stop 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 shutdown_connections 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa27c06b4d0 0x7fa27c06d990 unknown :-1 s=CLOSED pgs=702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa2900d06c0 0x7fa290142a90 unknown :-1 s=CLOSED pgs=1571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa2900d02e0 0x7fa290154cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 --2- 172.21.15.64:0/685249725 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa2900b4260 0x7fa290146b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 >> 172.21.15.64:0/685249725 conn(0x7fa2900b36d0 msgr2=0x7fa2900d1ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:54.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 shutdown_connections 2024-03-04T07:33:54.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.287+0000 7fa29715e640 1 -- 172.21.15.64:0/685249725 wait complete. 2024-03-04T07:33:54.300 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:54.300 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec bash -c ' 2024-03-04T07:33:54.300 DEBUG:teuthology.orchestra.run.smithi064:> mkdir pin 2024-03-04T07:33:54.301 DEBUG:teuthology.orchestra.run.smithi064:> touch pin/placeholder 2024-03-04T07:33:54.301 DEBUG:teuthology.orchestra.run.smithi064:> ') 2024-03-04T07:33:54.328 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:33:54.328 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec setfattr -n ceph.dir.pin -v 1 pin) 2024-03-04T07:33:54.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:54.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.533+0000 7f5a77cc2640 1 Processor -- start 2024-03-04T07:33:54.533 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.533+0000 7f5a77cc2640 1 -- start start 2024-03-04T07:33:54.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a77cc2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a70165a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:54.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a77cc2640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5a70071240 con 0x7f5a70158520 2024-03-04T07:33:54.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a77cc2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5a70071520 con 0x7f5a7006df30 2024-03-04T07:33:54.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a77cc2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a70071b70 con 0x7f5a700aee10 2024-03-04T07:33:54.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a75a5e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f5a70158520 0x7f5a7006d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42386/0 (socket says 172.21.15.64:42386) 2024-03-04T07:33:54.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a75a5e640 1 -- 172.21.15.64:0/2252955867 learned_addr learned my addr 172.21.15.64:0/2252955867 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a7625f640 1 --2- 172.21.15.64:0/2252955867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a70165a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a74a5c640 1 -- 172.21.15.64:0/2252955867 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1876808443 0 0) 0x7f5a70071240 con 0x7f5a70158520 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a74a5c640 1 -- 172.21.15.64:0/2252955867 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5a5c003520 con 0x7f5a70158520 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a74a5c640 1 -- 172.21.15.64:0/2252955867 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1520963920 0 0) 0x7f5a70071520 con 0x7f5a7006df30 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a74a5c640 1 -- 172.21.15.64:0/2252955867 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5a70071240 con 0x7f5a7006df30 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a7625f640 1 -- 172.21.15.64:0/2252955867 >> v1:172.21.15.66:6789/0 conn(0x7f5a7006df30 legacy=0x7f5a700ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a7625f640 1 -- 172.21.15.64:0/2252955867 >> v1:172.21.15.64:6789/0 conn(0x7f5a70158520 legacy=0x7f5a7006d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.534+0000 7f5a7625f640 1 -- 172.21.15.64:0/2252955867 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a70071800 con 0x7f5a700aee10 2024-03-04T07:33:54.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.535+0000 7f5a7625f640 1 --2- 172.21.15.64:0/2252955867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a70165a00 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f5a68009dc0 tx=0x7f5a6802bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4491194111dedcb8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.535+0000 7f5a74a5c640 1 -- 172.21.15.64:0/2252955867 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a68039070 con 0x7f5a700aee10 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.535+0000 7f5a74a5c640 1 -- 172.21.15.64:0/2252955867 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a6802ccd0 con 0x7f5a700aee10 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.535+0000 7f5a74a5c640 1 -- 172.21.15.64:0/2252955867 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a68034640 con 0x7f5a700aee10 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.536+0000 7f5a77cc2640 1 -- 172.21.15.64:0/2252955867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 msgr2=0x7f5a70165a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.536+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/2252955867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a70165a00 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f5a68009dc0 tx=0x7f5a6802bf20 comp rx=0 tx=0).stop 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.536+0000 7f5a77cc2640 1 -- 172.21.15.64:0/2252955867 shutdown_connections 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.536+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/2252955867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a70165a00 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.536+0000 7f5a77cc2640 1 -- 172.21.15.64:0/2252955867 >> 172.21.15.64:0/2252955867 conn(0x7f5a70066d00 msgr2=0x7f5a700670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.536+0000 7f5a77cc2640 1 -- 172.21.15.64:0/2252955867 shutdown_connections 2024-03-04T07:33:54.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.536+0000 7f5a77cc2640 1 -- 172.21.15.64:0/2252955867 wait complete. 2024-03-04T07:33:54.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.537+0000 7f5a77cc2640 1 Processor -- start 2024-03-04T07:33:54.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.537+0000 7f5a77cc2640 1 -- start start 2024-03-04T07:33:54.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a77cc2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a7006df30 0x7f5a701461d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:54.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a77cc2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a7006cdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:54.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a77cc2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a70158520 0x7f5a7006d310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:54.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a77cc2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a70158900 con 0x7f5a7006df30 2024-03-04T07:33:54.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a77cc2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a70072000 con 0x7f5a70158520 2024-03-04T07:33:54.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a77cc2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5a70158a80 con 0x7f5a700aee10 2024-03-04T07:33:54.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a70158520 0x7f5a7006d310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a70158520 0x7f5a7006d310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:53884/0 (socket says 172.21.15.64:53884) 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 -- 172.21.15.64:0/3095910342 learned_addr learned my addr 172.21.15.64:0/3095910342 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a75a5e640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a7006df30 0x7f5a701461d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7525d640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a7006cdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 -- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 msgr2=0x7f5a7006cdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a7006cdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 -- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a7006df30 msgr2=0x7f5a701461d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a7006df30 0x7f5a701461d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a7625f640 1 -- 172.21.15.64:0/3095910342 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a70154fd0 con 0x7f5a70158520 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.540+0000 7f5a75a5e640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a7006df30 0x7f5a701461d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.541+0000 7f5a7625f640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a70158520 0x7f5a7006d310 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f5a68009ee0 tx=0x7f5a68004a40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.541+0000 7f5a667fc640 1 -- 172.21.15.64:0/3095910342 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a68050070 con 0x7f5a70158520 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.541+0000 7f5a667fc640 1 -- 172.21.15.64:0/3095910342 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5a68041e10 con 0x7f5a70158520 2024-03-04T07:33:54.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.541+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5a701552b0 con 0x7f5a70158520 2024-03-04T07:33:54.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.541+0000 7f5a667fc640 1 -- 172.21.15.64:0/3095910342 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a68009450 con 0x7f5a70158520 2024-03-04T07:33:54.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.541+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5a70155870 con 0x7f5a70158520 2024-03-04T07:33:54.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.542+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5a70067ff0 con 0x7f5a70158520 2024-03-04T07:33:54.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.542+0000 7f5a667fc640 1 -- 172.21.15.64:0/3095910342 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5a68053070 con 0x7f5a70158520 2024-03-04T07:33:54.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.543+0000 7f5a667fc640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a5406b4d0 0x7f5a5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:54.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.543+0000 7f5a667fc640 1 -- 172.21.15.64:0/3095910342 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5a680b46d0 con 0x7f5a70158520 2024-03-04T07:33:54.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.543+0000 7f5a75a5e640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a5406b4d0 0x7f5a5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:54.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.543+0000 7f5a75a5e640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a5406b4d0 0x7f5a5406d990 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f5a6c007fd0 tx=0x7f5a6c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:54.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.547+0000 7f5a667fc640 1 -- 172.21.15.64:0/3095910342 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5a68046070 con 0x7f5a70158520 2024-03-04T07:33:54.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.769+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5a700b2060 con 0x7f5a70158520 2024-03-04T07:33:54.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.769+0000 7f5a667fc640 1 -- 172.21.15.64:0/3095910342 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 86 v86) v1 ==== 94+0+3414 (secure 0 0 0) 0x7f5a680835e0 con 0x7f5a70158520 2024-03-04T07:33:54.769 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:54.769 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":86,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:46.932881+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:active","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[6728,8577]},"id":9}]} 2024-03-04T07:33:54.769 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 86 2024-03-04T07:33:54.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a5406b4d0 msgr2=0x7f5a5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a5406b4d0 0x7f5a5406d990 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f5a6c007fd0 tx=0x7f5a6c009040 comp rx=0 tx=0).stop 2024-03-04T07:33:54.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a70158520 msgr2=0x7f5a7006d310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:54.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a70158520 0x7f5a7006d310 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f5a68009ee0 tx=0x7f5a68004a40 comp rx=0 tx=0).stop 2024-03-04T07:33:54.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 shutdown_connections 2024-03-04T07:33:54.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5a5406b4d0 0x7f5a5406d990 unknown :-1 s=CLOSED pgs=703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5a70158520 0x7f5a7006d310 unknown :-1 s=CLOSED pgs=553 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5a700aee10 0x7f5a7006cdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 --2- 172.21.15.64:0/3095910342 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5a7006df30 0x7f5a701461d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:54.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 >> 172.21.15.64:0/3095910342 conn(0x7f5a70066d00 msgr2=0x7f5a700b2280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:54.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 shutdown_connections 2024-03-04T07:33:54.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:54.771+0000 7f5a77cc2640 1 -- 172.21.15.64:0/3095910342 wait complete. 2024-03-04T07:33:54.782 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json get subtrees 2024-03-04T07:33:54.905 INFO:teuthology.orchestra.run.smithi066.stdout:[{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}}] 2024-03-04T07:33:54.905 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-04T07:33:54.906 INFO:tasks.cephfs.cephfs_test_case:[] =?= [('/pin', 1)] 2024-03-04T07:33:56.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:57.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.021+0000 7facdf881640 1 Processor -- start 2024-03-04T07:33:57.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.021+0000 7facdf881640 1 -- start start 2024-03-04T07:33:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdf881640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7facd815df70 0x7facd80d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdf881640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8071c50 0x7facd80b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdf881640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7facd80b6f20 con 0x7facd80d18c0 2024-03-04T07:33:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdf881640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7facd80d3030 con 0x7facd815df70 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdf881640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facd80d3310 con 0x7facd8071c50 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdde1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8071c50 0x7facd80b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdd61d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7facd815df70 0x7facd80d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdce1c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7facd80d18c0 0x7facd8071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:51170/0 (socket says 172.21.15.64:51170) 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdce1c640 1 -- 172.21.15.64:0/3275295686 learned_addr learned my addr 172.21.15.64:0/3275295686 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdde1e640 1 -- 172.21.15.64:0/3275295686 >> v2:172.21.15.66:3300/0 conn(0x7facd815df70 msgr2=0x7facd80d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdde1e640 1 --2- 172.21.15.64:0/3275295686 >> v2:172.21.15.66:3300/0 conn(0x7facd815df70 0x7facd80d27b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdde1e640 1 -- 172.21.15.64:0/3275295686 >> v1:172.21.15.66:6789/0 conn(0x7facd80d18c0 legacy=0x7facd8071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7faccffff640 1 -- 172.21.15.64:0/3275295686 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3152036357 0 0) 0x7facd80b6f20 con 0x7facd80d18c0 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdde1e640 1 -- 172.21.15.64:0/3275295686 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facd80d35f0 con 0x7facd8071c50 2024-03-04T07:33:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.022+0000 7facdd61d640 1 --2- 172.21.15.64:0/3275295686 >> v2:172.21.15.66:3300/0 conn(0x7facd815df70 0x7facd80d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.023+0000 7facdde1e640 1 --2- 172.21.15.64:0/3275295686 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8071c50 0x7facd80b2700 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7facd0009d90 tx=0x7facd002bf30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3f283a90d799e4ff server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.023+0000 7faccffff640 1 -- 172.21.15.64:0/3275295686 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facd0039070 con 0x7facd8071c50 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.023+0000 7faccffff640 1 -- 172.21.15.64:0/3275295686 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7facd002cce0 con 0x7facd8071c50 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.023+0000 7faccffff640 1 -- 172.21.15.64:0/3275295686 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facd0034660 con 0x7facd8071c50 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 -- 172.21.15.64:0/3275295686 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8071c50 msgr2=0x7facd80b2700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 --2- 172.21.15.64:0/3275295686 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8071c50 0x7facd80b2700 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7facd0009d90 tx=0x7facd002bf30 comp rx=0 tx=0).stop 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 -- 172.21.15.64:0/3275295686 shutdown_connections 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 --2- 172.21.15.64:0/3275295686 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd8071c50 0x7facd80b2700 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 --2- 172.21.15.64:0/3275295686 >> v2:172.21.15.66:3300/0 conn(0x7facd815df70 0x7facd80d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 -- 172.21.15.64:0/3275295686 >> 172.21.15.64:0/3275295686 conn(0x7facd80d3cf0 msgr2=0x7facd80d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:57.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 -- 172.21.15.64:0/3275295686 shutdown_connections 2024-03-04T07:33:57.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.024+0000 7facdf881640 1 -- 172.21.15.64:0/3275295686 wait complete. 2024-03-04T07:33:57.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.025+0000 7facdf881640 1 Processor -- start 2024-03-04T07:33:57.026 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.025+0000 7facdf881640 1 -- start start 2024-03-04T07:33:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdf881640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd8071c50 0x7facd80cadc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdf881640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80d18c0 0x7facd80acad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdf881640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd815df70 0x7facd80c9ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdf881640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facd815e350 con 0x7facd815df70 2024-03-04T07:33:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdf881640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facd80d2cf0 con 0x7facd8071c50 2024-03-04T07:33:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdf881640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facd815e4d0 con 0x7facd80d18c0 2024-03-04T07:33:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdde1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd815df70 0x7facd80c9ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdce1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80d18c0 0x7facd80acad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdd61d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd8071c50 0x7facd80cadc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdd61d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd8071c50 0x7facd80cadc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:53904/0 (socket says 172.21.15.64:53904) 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdd61d640 1 -- 172.21.15.64:0/1797908561 learned_addr learned my addr 172.21.15.64:0/1797908561 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdce1c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80d18c0 0x7facd80acad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:47848/0 (socket says 172.21.15.64:47848) 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdde1e640 1 -- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80d18c0 msgr2=0x7facd80acad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdde1e640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80d18c0 0x7facd80acad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdde1e640 1 -- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd8071c50 msgr2=0x7facd80cadc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdde1e640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd8071c50 0x7facd80cadc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdde1e640 1 -- 172.21.15.64:0/1797908561 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facd80cbcd0 con 0x7facd815df70 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdd61d640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd8071c50 0x7facd80cadc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.026+0000 7facdce1c640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80d18c0 0x7facd80acad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:33:57.029 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.027+0000 7facdde1e640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd815df70 0x7facd80c9ba0 secure :-1 s=READY pgs=1573 cs=0 l=1 rev1=1 crypto rx=0x7facd0033000 tx=0x7facd0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:57.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.027+0000 7faccdffb640 1 -- 172.21.15.64:0/1797908561 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facd0044020 con 0x7facd815df70 2024-03-04T07:33:57.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.027+0000 7faccdffb640 1 -- 172.21.15.64:0/1797908561 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7facd003a040 con 0x7facd815df70 2024-03-04T07:33:57.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.027+0000 7faccdffb640 1 -- 172.21.15.64:0/1797908561 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facd0039050 con 0x7facd815df70 2024-03-04T07:33:57.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.027+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7facd80cb430 con 0x7facd815df70 2024-03-04T07:33:57.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.027+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7facd80cb9f0 con 0x7facd815df70 2024-03-04T07:33:57.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.028+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7facd8067ff0 con 0x7facd815df70 2024-03-04T07:33:57.030 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.028+0000 7faccdffb640 1 -- 172.21.15.64:0/1797908561 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7facd0003bd0 con 0x7facd815df70 2024-03-04T07:33:57.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.029+0000 7faccdffb640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facbc06b4d0 0x7facbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:57.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.029+0000 7facdd61d640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facbc06b4d0 0x7facbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:57.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.029+0000 7faccdffb640 1 -- 172.21.15.64:0/1797908561 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7facd00ac520 con 0x7facd815df70 2024-03-04T07:33:57.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.030+0000 7facdd61d640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facbc06b4d0 0x7facbc06d990 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7facd80cad50 tx=0x7facd4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:57.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.034+0000 7faccdffb640 1 -- 172.21.15.64:0/1797908561 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7facd004e060 con 0x7facd815df70 2024-03-04T07:33:57.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.251+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7facd80b2570 con 0x7facd815df70 2024-03-04T07:33:57.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.252+0000 7faccdffb640 1 -- 172.21.15.64:0/1797908561 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 86 v86) v1 ==== 94+0+3414 (secure 0 0 0) 0x7facd007ade0 con 0x7facd815df70 2024-03-04T07:33:57.252 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:57.252 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":86,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:46.932881+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:active","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[6728,8577]},"id":9}]} 2024-03-04T07:33:57.252 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 86 2024-03-04T07:33:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facbc06b4d0 msgr2=0x7facbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facbc06b4d0 0x7facbc06d990 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7facd80cad50 tx=0x7facd4009190 comp rx=0 tx=0).stop 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd815df70 msgr2=0x7facd80c9ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd815df70 0x7facd80c9ba0 secure :-1 s=READY pgs=1573 cs=0 l=1 rev1=1 crypto rx=0x7facd0033000 tx=0x7facd0004a40 comp rx=0 tx=0).stop 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 shutdown_connections 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7facbc06b4d0 0x7facbc06d990 unknown :-1 s=CLOSED pgs=704 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7facd815df70 0x7facd80c9ba0 unknown :-1 s=CLOSED pgs=1573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7facd80d18c0 0x7facd80acad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 --2- 172.21.15.64:0/1797908561 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7facd8071c50 0x7facd80cadc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 >> 172.21.15.64:0/1797908561 conn(0x7facd80d3cf0 msgr2=0x7facd80b8290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 shutdown_connections 2024-03-04T07:33:57.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:57.254+0000 7facdf881640 1 -- 172.21.15.64:0/1797908561 wait complete. 2024-03-04T07:33:57.265 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json get subtrees 2024-03-04T07:33:57.387 INFO:teuthology.orchestra.run.smithi066.stdout:[{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}}] 2024-03-04T07:33:57.388 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-04T07:33:57.388 INFO:tasks.cephfs.cephfs_test_case:[] =?= [('/pin', 1)] 2024-03-04T07:33:59.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:33:59.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.504+0000 7f22f1202640 1 Processor -- start 2024-03-04T07:33:59.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.504+0000 7f22f1202640 1 -- start start 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.504+0000 7f22f1202640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f22ec15dbd0 0x7f22ec0ce920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.504+0000 7f22f1202640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f22ec0d2670 con 0x7f22ec0cee60 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.504+0000 7f22f1202640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f22ec0d2950 con 0x7f22ec15dfb0 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.504+0000 7f22f1202640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f22ec0d3280 con 0x7f22ec15dbd0 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22e3fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f22ec0cee60 0x7f22ec0d3710 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42388/0 (socket says 172.21.15.64:42388) 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22e3fff640 1 -- 172.21.15.64:0/2828538453 learned_addr learned my addr 172.21.15.64:0/2828538453 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ead9d640 1 --2- 172.21.15.64:0/2828538453 >> v2:172.21.15.66:3300/0 conn(0x7f22ec15dbd0 0x7f22ec0ce920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ea59c640 1 -- 172.21.15.64:0/2828538453 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3277293833 0 0) 0x7f22ec0d2670 con 0x7f22ec0cee60 2024-03-04T07:33:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ea59c640 1 -- 172.21.15.64:0/2828538453 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f22d8003520 con 0x7f22ec0cee60 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ea59c640 1 -- 172.21.15.64:0/2828538453 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3321757437 0 0) 0x7f22ec0d2950 con 0x7f22ec15dfb0 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ea59c640 1 -- 172.21.15.64:0/2828538453 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f22ec0d2670 con 0x7f22ec15dfb0 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ead9d640 1 -- 172.21.15.64:0/2828538453 >> v1:172.21.15.66:6789/0 conn(0x7f22ec15dfb0 legacy=0x7f22ec0d3e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ead9d640 1 -- 172.21.15.64:0/2828538453 >> v1:172.21.15.64:6789/0 conn(0x7f22ec0cee60 legacy=0x7f22ec0d3710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.505+0000 7f22ead9d640 1 -- 172.21.15.64:0/2828538453 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22ec0d2c30 con 0x7f22ec15dbd0 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.506+0000 7f22ead9d640 1 --2- 172.21.15.64:0/2828538453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 0x7f22ec0ce920 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f22dc0097a0 tx=0x7f22dc02b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cc1d6b400e66225c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.506+0000 7f22ea59c640 1 -- 172.21.15.64:0/2828538453 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22dc035020 con 0x7f22ec15dbd0 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.506+0000 7f22ea59c640 1 -- 172.21.15.64:0/2828538453 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f22dc031830 con 0x7f22ec15dbd0 2024-03-04T07:33:59.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.506+0000 7f22ea59c640 1 -- 172.21.15.64:0/2828538453 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22dc031be0 con 0x7f22ec15dbd0 2024-03-04T07:33:59.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.507+0000 7f22f1202640 1 -- 172.21.15.64:0/2828538453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 msgr2=0x7f22ec0ce920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:59.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.507+0000 7f22f1202640 1 --2- 172.21.15.64:0/2828538453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 0x7f22ec0ce920 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f22dc0097a0 tx=0x7f22dc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.507+0000 7f22f1202640 1 -- 172.21.15.64:0/2828538453 shutdown_connections 2024-03-04T07:33:59.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.507+0000 7f22f1202640 1 --2- 172.21.15.64:0/2828538453 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 0x7f22ec0ce920 unknown :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.507+0000 7f22f1202640 1 -- 172.21.15.64:0/2828538453 >> 172.21.15.64:0/2828538453 conn(0x7f22ec071960 msgr2=0x7f22ec071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.507+0000 7f22f1202640 1 -- 172.21.15.64:0/2828538453 shutdown_connections 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.507+0000 7f22f1202640 1 -- 172.21.15.64:0/2828538453 wait complete. 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.508+0000 7f22f1202640 1 Processor -- start 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.508+0000 7f22f1202640 1 -- start start 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22f1202640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22ec0cee60 0x7f22ec0b5d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22f1202640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 0x7f22ec155ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22f1202640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22ec15dfb0 0x7f22ec066d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22f1202640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22ec0b9980 con 0x7f22ec0cee60 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22f1202640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22ec0b7b80 con 0x7f22ec15dbd0 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22f1202640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f22ec168990 con 0x7f22ec15dfb0 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22ec0cee60 0x7f22ec0b5d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22ec0cee60 0x7f22ec0b5d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50940/0 (socket says 172.21.15.64:50940) 2024-03-04T07:33:59.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 -- 172.21.15.64:0/1784276731 learned_addr learned my addr 172.21.15.64:0/1784276731 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22e3fff640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 0x7f22ec155ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22eb59e640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22ec15dfb0 0x7f22ec066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 -- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22ec15dfb0 msgr2=0x7f22ec066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22ec15dfb0 0x7f22ec066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 -- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 msgr2=0x7f22ec155ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 0x7f22ec155ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.509+0000 7f22ead9d640 1 -- 172.21.15.64:0/1784276731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22ec0cd220 con 0x7f22ec0cee60 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.510+0000 7f22ead9d640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22ec0cee60 0x7f22ec0b5d30 secure :-1 s=READY pgs=1575 cs=0 l=1 rev1=1 crypto rx=0x7f22dc000c00 tx=0x7f22dc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.510+0000 7f22e37fe640 1 -- 172.21.15.64:0/1784276731 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22dc035040 con 0x7f22ec0cee60 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.510+0000 7f22e37fe640 1 -- 172.21.15.64:0/1784276731 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f22dc03b920 con 0x7f22ec0cee60 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.510+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f22ec0cd500 con 0x7f22ec0cee60 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.510+0000 7f22e37fe640 1 -- 172.21.15.64:0/1784276731 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22dc03bd10 con 0x7f22ec0cee60 2024-03-04T07:33:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.510+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f22ec0b3160 con 0x7f22ec0cee60 2024-03-04T07:33:59.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.511+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f22ec04f330 con 0x7f22ec0cee60 2024-03-04T07:33:59.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.511+0000 7f22e37fe640 1 -- 172.21.15.64:0/1784276731 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f22dc038020 con 0x7f22ec0cee60 2024-03-04T07:33:59.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.512+0000 7f22e37fe640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22cc06b4d0 0x7f22cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:33:59.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.512+0000 7f22e3fff640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22cc06b4d0 0x7f22cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:33:59.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.512+0000 7f22e37fe640 1 -- 172.21.15.64:0/1784276731 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f22dc0ac150 con 0x7f22ec0cee60 2024-03-04T07:33:59.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.512+0000 7f22e3fff640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22cc06b4d0 0x7f22cc06d990 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7f22ec0b6580 tx=0x7f22d4007d70 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:33:59.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.516+0000 7f22e37fe640 1 -- 172.21.15.64:0/1784276731 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f22dc07aa10 con 0x7f22ec0cee60 2024-03-04T07:33:59.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.737+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f22ec15e5f0 con 0x7f22ec0cee60 2024-03-04T07:33:59.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.737+0000 7f22e37fe640 1 -- 172.21.15.64:0/1784276731 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 86 v86) v1 ==== 94+0+3414 (secure 0 0 0) 0x7f22dc07eab0 con 0x7f22ec0cee60 2024-03-04T07:33:59.737 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:33:59.737 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":86,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:46.932881+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:active","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[6728,8577]},"id":9}]} 2024-03-04T07:33:59.738 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 86 2024-03-04T07:33:59.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22cc06b4d0 msgr2=0x7f22cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:59.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22cc06b4d0 0x7f22cc06d990 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7f22ec0b6580 tx=0x7f22d4007d70 comp rx=0 tx=0).stop 2024-03-04T07:33:59.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22ec0cee60 msgr2=0x7f22ec0b5d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:33:59.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22ec0cee60 0x7f22ec0b5d30 secure :-1 s=READY pgs=1575 cs=0 l=1 rev1=1 crypto rx=0x7f22dc000c00 tx=0x7f22dc0046c0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 shutdown_connections 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f22cc06b4d0 0x7f22cc06d990 unknown :-1 s=CLOSED pgs=705 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f22ec15dfb0 0x7f22ec066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f22ec15dbd0 0x7f22ec155ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 --2- 172.21.15.64:0/1784276731 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f22ec0cee60 0x7f22ec0b5d30 unknown :-1 s=CLOSED pgs=1575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 >> 172.21.15.64:0/1784276731 conn(0x7f22ec071960 msgr2=0x7f22ec0b27c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 shutdown_connections 2024-03-04T07:33:59.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:33:59.739+0000 7f22f1202640 1 -- 172.21.15.64:0/1784276731 wait complete. 2024-03-04T07:33:59.751 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json get subtrees 2024-03-04T07:33:59.899 INFO:teuthology.orchestra.run.smithi066.stdout:[{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}}] 2024-03-04T07:33:59.899 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-04T07:33:59.899 INFO:tasks.cephfs.cephfs_test_case:[] =?= [('/pin', 1)] 2024-03-04T07:34:01.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:02.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.015+0000 7f474ae40640 1 Processor -- start 2024-03-04T07:34:02.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.015+0000 7f474ae40640 1 -- start start 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.015+0000 7f474ae40640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f474406e700 0x7f474415ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.015+0000 7f474ae40640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f47440b55d0 0x7f4744071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.015+0000 7f474ae40640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f47440b4160 con 0x7f4744072660 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f474ae40640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f47440b5b40 con 0x7f474406e700 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f474ae40640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f47440b3ba0 con 0x7f47440b55d0 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f4748bdc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f474406e700 0x7f474415ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f4748bdc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f474406e700 0x7f474415ef80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50946/0 (socket says 172.21.15.64:50946) 2024-03-04T07:34:02.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f4748bdc640 1 -- 172.21.15.64:0/3312902634 learned_addr learned my addr 172.21.15.64:0/3312902634 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:02.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f47493dd640 1 --2- 172.21.15.64:0/3312902634 >> v2:172.21.15.66:3300/0 conn(0x7f47440b55d0 0x7f4744071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:02.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f4748bdc640 1 -- 172.21.15.64:0/3312902634 >> v2:172.21.15.66:3300/0 conn(0x7f47440b55d0 msgr2=0x7f4744071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:02.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f4748bdc640 1 --2- 172.21.15.64:0/3312902634 >> v2:172.21.15.66:3300/0 conn(0x7f47440b55d0 0x7f4744071980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f4748bdc640 1 -- 172.21.15.64:0/3312902634 >> v1:172.21.15.66:6789/0 conn(0x7f4744072660 legacy=0x7f474406d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:02.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.016+0000 7f4748bdc640 1 -- 172.21.15.64:0/3312902634 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47440b3e80 con 0x7f474406e700 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.017+0000 7f4748bdc640 1 --2- 172.21.15.64:0/3312902634 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f474406e700 0x7f474415ef80 secure :-1 s=READY pgs=1576 cs=0 l=1 rev1=1 crypto rx=0x7f4744071580 tx=0x7f473402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ff0c160fe8cc6860 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.017+0000 7f47437fe640 1 -- 172.21.15.64:0/3312902634 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4734034070 con 0x7f474406e700 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.017+0000 7f47437fe640 1 -- 172.21.15.64:0/3312902634 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4734030860 con 0x7f474406e700 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.017+0000 7f47437fe640 1 -- 172.21.15.64:0/3312902634 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4734030c30 con 0x7f474406e700 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 -- 172.21.15.64:0/3312902634 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f474406e700 msgr2=0x7f474415ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 --2- 172.21.15.64:0/3312902634 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f474406e700 0x7f474415ef80 secure :-1 s=READY pgs=1576 cs=0 l=1 rev1=1 crypto rx=0x7f4744071580 tx=0x7f473402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 -- 172.21.15.64:0/3312902634 shutdown_connections 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 --2- 172.21.15.64:0/3312902634 >> v2:172.21.15.66:3300/0 conn(0x7f47440b55d0 0x7f4744071980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 --2- 172.21.15.64:0/3312902634 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f474406e700 0x7f474415ef80 unknown :-1 s=CLOSED pgs=1576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 -- 172.21.15.64:0/3312902634 >> 172.21.15.64:0/3312902634 conn(0x7f47440b2720 msgr2=0x7f47440b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:02.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 -- 172.21.15.64:0/3312902634 shutdown_connections 2024-03-04T07:34:02.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.018+0000 7f474ae40640 1 -- 172.21.15.64:0/3312902634 wait complete. 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.019+0000 7f474ae40640 1 Processor -- start 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.019+0000 7f474ae40640 1 -- start start 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f474ae40640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f474406e700 0x7f47440af360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f474ae40640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4744072660 0x7f4744158bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f474ae40640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47440b55d0 0x7f4744159100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f474ae40640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4744072040 con 0x7f4744072660 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f474ae40640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4744071ec0 con 0x7f474406e700 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f474ae40640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47440721c0 con 0x7f47440b55d0 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4744072660 0x7f4744158bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:02.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4744072660 0x7f4744158bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50952/0 (socket says 172.21.15.64:50952) 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 -- 172.21.15.64:0/2452260866 learned_addr learned my addr 172.21.15.64:0/2452260866 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4748bdc640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f474406e700 0x7f47440af360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f47493dd640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47440b55d0 0x7f4744159100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 -- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47440b55d0 msgr2=0x7f4744159100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47440b55d0 0x7f4744159100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 -- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f474406e700 msgr2=0x7f47440af360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f474406e700 0x7f47440af360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 -- 172.21.15.64:0/2452260866 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47440afbb0 con 0x7f4744072660 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.020+0000 7f4743fff640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4744072660 0x7f4744158bc0 secure :-1 s=READY pgs=1577 cs=0 l=1 rev1=1 crypto rx=0x7f473800c440 tx=0x7f4738001e70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.021+0000 7f47417fa640 1 -- 172.21.15.64:0/2452260866 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47380052d0 con 0x7f4744072660 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.021+0000 7f47417fa640 1 -- 172.21.15.64:0/2452260866 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4738005470 con 0x7f4744072660 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.021+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47440afe90 con 0x7f4744072660 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.021+0000 7f47417fa640 1 -- 172.21.15.64:0/2452260866 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4738007260 con 0x7f4744072660 2024-03-04T07:34:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.021+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47440b0450 con 0x7f4744072660 2024-03-04T07:34:02.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.022+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4744074980 con 0x7f4744072660 2024-03-04T07:34:02.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.022+0000 7f47417fa640 1 -- 172.21.15.64:0/2452260866 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4738007400 con 0x7f4744072660 2024-03-04T07:34:02.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.023+0000 7f47417fa640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f473006b4d0 0x7f473006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:02.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.023+0000 7f47417fa640 1 -- 172.21.15.64:0/2452260866 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f4738089660 con 0x7f4744072660 2024-03-04T07:34:02.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.023+0000 7f4748bdc640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f473006b4d0 0x7f473006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:02.024 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.023+0000 7f4748bdc640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f473006b4d0 0x7f473006d990 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f4734036000 tx=0x7f4734008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:02.027 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.027+0000 7f47417fa640 1 -- 172.21.15.64:0/2452260866 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4744074980 con 0x7f4744072660 2024-03-04T07:34:02.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.244+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4744067ff0 con 0x7f4744072660 2024-03-04T07:34:02.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.244+0000 7f47417fa640 1 -- 172.21.15.64:0/2452260866 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 86 v86) v1 ==== 94+0+3414 (secure 0 0 0) 0x7f4738057e90 con 0x7f4744072660 2024-03-04T07:34:02.245 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:02.245 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":86,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":84,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:33:46.932881+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":2,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:active","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[6728,8577]},"id":9}]} 2024-03-04T07:34:02.245 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 86 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f473006b4d0 msgr2=0x7f473006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f473006b4d0 0x7f473006d990 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f4734036000 tx=0x7f4734008030 comp rx=0 tx=0).stop 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4744072660 msgr2=0x7f4744158bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4744072660 0x7f4744158bc0 secure :-1 s=READY pgs=1577 cs=0 l=1 rev1=1 crypto rx=0x7f473800c440 tx=0x7f4738001e70 comp rx=0 tx=0).stop 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 shutdown_connections 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f473006b4d0 0x7f473006d990 unknown :-1 s=CLOSED pgs=706 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47440b55d0 0x7f4744159100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4744072660 0x7f4744158bc0 unknown :-1 s=CLOSED pgs=1577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 --2- 172.21.15.64:0/2452260866 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f474406e700 0x7f47440af360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 >> 172.21.15.64:0/2452260866 conn(0x7f47440b2720 msgr2=0x7f474415d250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:02.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 shutdown_connections 2024-03-04T07:34:02.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:02.247+0000 7f474ae40640 1 -- 172.21.15.64:0/2452260866 wait complete. 2024-03-04T07:34:02.258 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json get subtrees 2024-03-04T07:34:02.387 INFO:teuthology.orchestra.run.smithi066.stdout:[{"is_auth":false,"auth_first":0,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"","dirfrag":"0x1","snapid_first":2,"projected_version":"0","version":"10","committing_version":"0","committed_version":"0","is_rep":true,"dir_auth":"0","states":[],"is_auth":false,"auth_state":{"replicas":{}},"replica_state":{"authority":[0,-2],"replica_nonce":1},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1},"nref":2}},{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":1,"dir":{"path":"/pin","dirfrag":"0x10000000000","snapid_first":2,"projected_version":"5","version":"5","committing_version":"0","committed_version":"0","is_rep":false,"dir_auth":"1","states":["auth","dirty","complete"],"is_auth":true,"auth_state":{"replicas":{"0":1}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"ptrwaiter":0,"child":1,"frozen":0,"subtree":1,"importing":0,"replicated":1,"dirty":1},"nref":4}},{"is_auth":true,"auth_first":1,"auth_second":-2,"export_pin":-1,"distributed_ephemeral_pin":false,"random_ephemeral_pin":false,"export_pin_target":-1,"dir":{"path":"~mds1","dirfrag":"0x101","snapid_first":2,"projected_version":"1","version":"1","committing_version":"1","committed_version":"1","is_rep":false,"dir_auth":"1","states":["auth","complete"],"is_auth":true,"auth_state":{"replicas":{}},"replica_state":{"authority":[1,-2],"replica_nonce":0},"auth_pins":0,"is_frozen":false,"is_freezing":false,"pins":{"child":1,"subtree":1,"dirty":0,"authpin":0},"nref":2}}] 2024-03-04T07:34:02.387 DEBUG:tasks.cephfs.filesystem:_json_asok output [ { "auth_first": 0, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "0", "committing_version": "0", "dir_auth": "0", "dirfrag": "0x1", "is_auth": false, "is_freezing": false, "is_frozen": false, "is_rep": true, "nref": 2, "path": "", "pins": { "child": 1, "subtree": 1 }, "projected_version": "0", "replica_state": { "authority": [ 0, -2 ], "replica_nonce": 1 }, "snapid_first": 2, "states": [], "version": "10" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": false, "random_ephemeral_pin": false }, { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": { "0": 1 } }, "committed_version": "0", "committing_version": "0", "dir_auth": "1", "dirfrag": "0x10000000000", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 4, "path": "/pin", "pins": { "child": 1, "dirty": 1, "frozen": 0, "importing": 0, "ptrwaiter": 0, "replicated": 1, "subtree": 1 }, "projected_version": "5", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "dirty", "complete" ], "version": "5" }, "distributed_ephemeral_pin": false, "export_pin": 1, "export_pin_target": 1, "is_auth": true, "random_ephemeral_pin": false }, { "auth_first": 1, "auth_second": -2, "dir": { "auth_pins": 0, "auth_state": { "replicas": {} }, "committed_version": "1", "committing_version": "1", "dir_auth": "1", "dirfrag": "0x101", "is_auth": true, "is_freezing": false, "is_frozen": false, "is_rep": false, "nref": 2, "path": "~mds1", "pins": { "authpin": 0, "child": 1, "dirty": 0, "subtree": 1 }, "projected_version": "1", "replica_state": { "authority": [ 1, -2 ], "replica_nonce": 0 }, "snapid_first": 2, "states": [ "auth", "complete" ], "version": "1" }, "distributed_ephemeral_pin": false, "export_pin": -1, "export_pin_target": -1, "is_auth": true, "random_ephemeral_pin": false } ] 2024-03-04T07:34:02.387 INFO:tasks.cephfs.cephfs_test_case:[('/pin', 1)] =?= [('/pin', 1)] 2024-03-04T07:34:02.387 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:34:02.388 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:34:02.402 DEBUG:teuthology.orchestra.run.smithi064:> stdin-killer -- python3 -c ' 2024-03-04T07:34:02.402 DEBUG:teuthology.orchestra.run.smithi064:> import fcntl 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> import sys 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> import time 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> import os 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> fcntl.fcntl(sys.stdin, fcntl.F_SETFL, fcntl.fcntl(sys.stdin, fcntl.F_GETFL) | os.O_NONBLOCK) 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> os.mkdir("/home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked") 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> fd = os.open("/home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked", os.O_RDONLY) 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> while True: 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> print("open_dir_background: keeping dir open", file=sys.stderr) 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> try: 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> if os.read(0, 4096) == b"": 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> break 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> except BlockingIOError: 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> pass 2024-03-04T07:34:02.403 DEBUG:teuthology.orchestra.run.smithi064:> time.sleep(2) 2024-03-04T07:34:02.404 DEBUG:teuthology.orchestra.run.smithi064:> ' 2024-03-04T07:34:02.445 DEBUG:teuthology.orchestra.run.smithi064:> stat /home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked 2024-03-04T07:34:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:34:02.461 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked': No such file or directory 2024-03-04T07:34:02.518 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:03.462 DEBUG:teuthology.orchestra.run.smithi064:> stat /home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked 2024-03-04T07:34:03.480 INFO:teuthology.orchestra.run.smithi064.stdout: File: /home/ubuntu/cephtest/mnt.0/pin/to-be-unlinked 2024-03-04T07:34:03.480 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 0 Blocks: 1 IO Block: 4096 directory 2024-03-04T07:34:03.480 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 29h/41d Inode: 2199023255552 Links: 2 2024-03-04T07:34:03.480 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0755/drwxr-xr-x) Uid: ( 1000/ ubuntu) Gid: ( 1301/ ubuntu) 2024-03-04T07:34:03.480 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:fusefs_t:s0 2024-03-04T07:34:03.480 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-03-04 07:34:02.511935100 +0000 2024-03-04T07:34:03.481 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-03-04 07:34:02.511935100 +0000 2024-03-04T07:34:03.481 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-03-04 07:34:02.511935100 +0000 2024-03-04T07:34:03.481 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2024-03-04T07:34:03.481 INFO:tasks.cephfs.mount:File pin/to-be-unlinked became visible from 0 after 1s 2024-03-04T07:34:03.481 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:34:03.481 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest/mnt.0 && exec rmdir pin/to-be-unlinked) 2024-03-04T07:34:03.542 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json perf dump mds_cache num_strays 2024-03-04T07:34:03.678 INFO:teuthology.orchestra.run.smithi066.stdout:{"mds_cache":{"num_strays":1}} 2024-03-04T07:34:03.678 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "num_strays": 1 } } 2024-03-04T07:34:03.679 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json perf dump mds_cache strays_created 2024-03-04T07:34:03.820 INFO:teuthology.orchestra.run.smithi066.stdout:{"mds_cache":{"strays_created":1}} 2024-03-04T07:34:03.820 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_created": 1 } } 2024-03-04T07:34:03.820 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json perf dump mds_cache strays_enqueued 2024-03-04T07:34:03.942 INFO:teuthology.orchestra.run.smithi066.stdout:{"mds_cache":{"strays_enqueued":0}} 2024-03-04T07:34:03.943 DEBUG:tasks.cephfs.filesystem:_json_asok output { "mds_cache": { "strays_enqueued": 0 } } 2024-03-04T07:34:03.944 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json flush journal 2024-03-04T07:34:04.108 INFO:teuthology.orchestra.run.smithi066.stdout:{"message":"","return_code":0} 2024-03-04T07:34:04.109 DEBUG:tasks.cephfs.filesystem:_json_asok output { "message": "", "return_code": 0 } 2024-03-04T07:34:04.109 DEBUG:teuthology.orchestra.run.smithi066:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-mds.b.asok --format=json cache drop 2024-03-04T07:34:04.234 INFO:teuthology.orchestra.run.smithi066.stdout:{"client_recall":{"return_code":0,"message":"(0) Success","recalled":0},"flush_journal":{"return_code":0,"message":""},"trim_cache":{"trimmed":0},"cache":{"pool":{"items":328,"bytes":65087}},"duration":0.0064810359999999999} 2024-03-04T07:34:04.235 DEBUG:tasks.cephfs.filesystem:_json_asok output { "cache": { "pool": { "bytes": 65087, "items": 328 } }, "client_recall": { "message": "(0) Success", "recalled": 0, "return_code": 0 }, "duration": 0.006481036, "flush_journal": { "message": "", "return_code": 0 }, "trim_cache": { "trimmed": 0 } } 2024-03-04T07:34:04.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 1 2024-03-04T07:34:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.353+0000 7f5ce3ced640 1 Processor -- start 2024-03-04T07:34:04.353 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.354+0000 7f5ce3ced640 1 -- start start 2024-03-04T07:34:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.354+0000 7f5ce3ced640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc1618e0 0x7f5cdc06b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.354+0000 7f5ce3ced640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5cdc0ce4f0 0x7f5cdc0ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.354+0000 7f5ce3ced640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5cdc0cee70 con 0x7f5cdc161d40 2024-03-04T07:34:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce3ced640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5cdc0cfce0 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce3ced640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5cdc15f730 con 0x7f5cdc1618e0 2024-03-04T07:34:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1a89640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc1618e0 0x7f5cdc06b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:04.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1288640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5cdc0ce4f0 0x7f5cdc0ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce228a640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f5cdc161d40 0x7f5cdc1621d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:53154/0 (socket says 172.21.15.64:53154) 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1288640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5cdc0ce4f0 0x7f5cdc0ce930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:51010/0 (socket says 172.21.15.64:51010) 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1a89640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc1618e0 0x7f5cdc06b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:35832/0 (socket says 172.21.15.64:35832) 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1288640 1 -- 172.21.15.64:0/717326651 learned_addr learned my addr 172.21.15.64:0/717326651 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1a89640 1 -- 172.21.15.64:0/717326651 >> v2:172.21.15.66:3300/0 conn(0x7f5cdc0ce4f0 msgr2=0x7f5cdc0ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1a89640 1 --2- 172.21.15.64:0/717326651 >> v2:172.21.15.66:3300/0 conn(0x7f5cdc0ce4f0 0x7f5cdc0ce930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1a89640 1 -- 172.21.15.64:0/717326651 >> v1:172.21.15.66:6789/0 conn(0x7f5cdc161d40 legacy=0x7f5cdc1621d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1a89640 1 -- 172.21.15.64:0/717326651 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5cdc15ebb0 con 0x7f5cdc1618e0 2024-03-04T07:34:04.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.355+0000 7f5ce1288640 1 --2- 172.21.15.64:0/717326651 >> v2:172.21.15.66:3300/0 conn(0x7f5cdc0ce4f0 0x7f5cdc0ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.356+0000 7f5ce1a89640 1 --2- 172.21.15.64:0/717326651 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc1618e0 0x7f5cdc06b130 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f5cd80097a0 tx=0x7f5cd802b960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a254fa96e8ce26b4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.356+0000 7f5ce0a87640 1 -- 172.21.15.64:0/717326651 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cd8034070 con 0x7f5cdc1618e0 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.356+0000 7f5ce0a87640 1 -- 172.21.15.64:0/717326651 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5cd8030610 con 0x7f5cdc1618e0 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.356+0000 7f5ce0a87640 1 -- 172.21.15.64:0/717326651 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cd80309e0 con 0x7f5cdc1618e0 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 -- 172.21.15.64:0/717326651 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc1618e0 msgr2=0x7f5cdc06b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/717326651 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc1618e0 0x7f5cdc06b130 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f5cd80097a0 tx=0x7f5cd802b960 comp rx=0 tx=0).stop 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 -- 172.21.15.64:0/717326651 shutdown_connections 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/717326651 >> v2:172.21.15.66:3300/0 conn(0x7f5cdc0ce4f0 0x7f5cdc0ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/717326651 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc1618e0 0x7f5cdc06b130 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 -- 172.21.15.64:0/717326651 >> 172.21.15.64:0/717326651 conn(0x7f5cdc0b3660 msgr2=0x7f5cdc0b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 -- 172.21.15.64:0/717326651 shutdown_connections 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.357+0000 7f5ce3ced640 1 -- 172.21.15.64:0/717326651 wait complete. 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.358+0000 7f5ce3ced640 1 Processor -- start 2024-03-04T07:34:04.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.358+0000 7f5ce3ced640 1 -- start start 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce3ced640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cdc0ce4f0 0x7f5cdc14f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce3ced640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc161d40 0x7f5cdc0b71f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce3ced640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cdc06fca0 0x7f5cdc070150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce3ced640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5cdc15f450 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce3ced640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5cdc15f170 con 0x7f5cdc06fca0 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce3ced640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5cdc06b670 con 0x7f5cdc161d40 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cdc0ce4f0 0x7f5cdc14f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cdc0ce4f0 0x7f5cdc14f030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35852/0 (socket says 172.21.15.64:35852) 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 -- 172.21.15.64:0/2974560394 learned_addr learned my addr 172.21.15.64:0/2974560394 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce228a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cdc06fca0 0x7f5cdc070150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:04.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1288640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc161d40 0x7f5cdc0b71f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 -- 172.21.15.64:0/2974560394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc161d40 msgr2=0x7f5cdc0b71f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc161d40 0x7f5cdc0b71f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 -- 172.21.15.64:0/2974560394 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cdc06fca0 msgr2=0x7f5cdc070150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cdc06fca0 0x7f5cdc070150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 -- 172.21.15.64:0/2974560394 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5cdc1f7e60 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1288640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc161d40 0x7f5cdc0b71f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.359+0000 7f5ce1a89640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cdc0ce4f0 0x7f5cdc14f030 secure :-1 s=READY pgs=1578 cs=0 l=1 rev1=1 crypto rx=0x7f5cd80098c0 tx=0x7f5cd80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.360+0000 7f5cd27fc640 1 -- 172.21.15.64:0/2974560394 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cd8034040 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.360+0000 7f5cd27fc640 1 -- 172.21.15.64:0/2974560394 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5cd8030410 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.360+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5cdc1f8140 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.360+0000 7f5cd27fc640 1 -- 172.21.15.64:0/2974560394 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cd8039c70 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.360+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5cdc1f8700 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.361+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5cdc09fc50 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.361+0000 7f5cd27fc640 1 -- 172.21.15.64:0/2974560394 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5cd8037070 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.361+0000 7f5cd27fc640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc006b4d0 0x7f5cc006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:04.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.362+0000 7f5ce1288640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc006b4d0 0x7f5cc006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:04.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.362+0000 7f5cd27fc640 1 -- 172.21.15.64:0/2974560394 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5cd80abf70 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.362+0000 7f5ce1288640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc006b4d0 0x7f5cc006d990 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f5ccc009630 tx=0x7f5ccc009240 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:04.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.366+0000 7f5cd27fc640 1 -- 172.21.15.64:0/2974560394 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5cd807a830 con 0x7f5cdc0ce4f0 2024-03-04T07:34:04.519 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:04.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:04.585+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "1"} v 0) v1 -- 0x7f5cdc160120 con 0x7f5cdc0ce4f0 2024-03-04T07:34:05.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.312+0000 7f5cd27fc640 1 -- 172.21.15.64:0/2974560394 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "1"}]=0 v88) v1 ==== 105+0+0 (secure 0 0 0) 0x7f5cd807e8d0 con 0x7f5cdc0ce4f0 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.314+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc006b4d0 msgr2=0x7f5cc006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.314+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc006b4d0 0x7f5cc006d990 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7f5ccc009630 tx=0x7f5ccc009240 comp rx=0 tx=0).stop 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.314+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cdc0ce4f0 msgr2=0x7f5cdc14f030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.314+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cdc0ce4f0 0x7f5cdc14f030 secure :-1 s=READY pgs=1578 cs=0 l=1 rev1=1 crypto rx=0x7f5cd80098c0 tx=0x7f5cd80046c0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 shutdown_connections 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5cc006b4d0 0x7f5cc006d990 unknown :-1 s=CLOSED pgs=707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5cdc06fca0 0x7f5cdc070150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5cdc161d40 0x7f5cdc0b71f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 --2- 172.21.15.64:0/2974560394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5cdc0ce4f0 0x7f5cdc14f030 unknown :-1 s=CLOSED pgs=1578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 >> 172.21.15.64:0/2974560394 conn(0x7f5cdc0b3660 msgr2=0x7f5cdc068e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:05.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 shutdown_connections 2024-03-04T07:34:05.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.315+0000 7f5ce3ced640 1 -- 172.21.15.64:0/2974560394 wait complete. 2024-03-04T07:34:05.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:05.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.431+0000 7f1c812b5640 1 Processor -- start 2024-03-04T07:34:05.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.431+0000 7f1c812b5640 1 -- start start 2024-03-04T07:34:05.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c812b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c7c0b0e90 0x7f1c7c06b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c812b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1c7c0ae9b0 0x7f1c7c0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c812b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1c7c0cee90 con 0x7f1c7c06b5c0 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c812b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1c7c0cfa00 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c812b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1c7c0cfc50 con 0x7f1c7c0ae9b0 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c7a7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c7c0b0e90 0x7f1c7c06b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c7a7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1c7c0b0e90 0x7f1c7c06b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35854/0 (socket says 172.21.15.64:35854) 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c7a7fc640 1 -- 172.21.15.64:0/2324457632 learned_addr learned my addr 172.21.15.64:0/2324457632 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.432+0000 7f1c7b7fe640 1 --2- 172.21.15.64:0/2324457632 >> v2:172.21.15.66:3300/0 conn(0x7f1c7c0ae9b0 0x7f1c7c0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.433+0000 7f1c7a7fc640 1 -- 172.21.15.64:0/2324457632 >> v2:172.21.15.66:3300/0 conn(0x7f1c7c0ae9b0 msgr2=0x7f1c7c0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.433+0000 7f1c7a7fc640 1 --2- 172.21.15.64:0/2324457632 >> v2:172.21.15.66:3300/0 conn(0x7f1c7c0ae9b0 0x7f1c7c0d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.433+0000 7f1c7a7fc640 1 -- 172.21.15.64:0/2324457632 >> v1:172.21.15.66:6789/0 conn(0x7f1c7c06b5c0 legacy=0x7f1c7c0b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.433+0000 7f1c7a7fc640 1 -- 172.21.15.64:0/2324457632 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c7c0cff30 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.433+0000 7f1c7a7fc640 1 --2- 172.21.15.64:0/2324457632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06b9a0 secure :-1 s=READY pgs=1579 cs=0 l=1 rev1=1 crypto rx=0x7f1c7c072170 tx=0x7f1c6402b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=43e3010f492a4791 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.433+0000 7f1c79ffb640 1 -- 172.21.15.64:0/2324457632 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c64035020 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c79ffb640 1 -- 172.21.15.64:0/2324457632 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1c64031870 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c79ffb640 1 -- 172.21.15.64:0/2324457632 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c64031c40 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 -- 172.21.15.64:0/2324457632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 msgr2=0x7f1c7c06b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 --2- 172.21.15.64:0/2324457632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06b9a0 secure :-1 s=READY pgs=1579 cs=0 l=1 rev1=1 crypto rx=0x7f1c7c072170 tx=0x7f1c6402b910 comp rx=0 tx=0).stop 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 -- 172.21.15.64:0/2324457632 shutdown_connections 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 --2- 172.21.15.64:0/2324457632 >> v2:172.21.15.66:3300/0 conn(0x7f1c7c0ae9b0 0x7f1c7c0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 --2- 172.21.15.64:0/2324457632 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06b9a0 unknown :-1 s=CLOSED pgs=1579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 -- 172.21.15.64:0/2324457632 >> 172.21.15.64:0/2324457632 conn(0x7f1c7c0b36d0 msgr2=0x7f1c7c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 -- 172.21.15.64:0/2324457632 shutdown_connections 2024-03-04T07:34:05.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.434+0000 7f1c812b5640 1 -- 172.21.15.64:0/2324457632 wait complete. 2024-03-04T07:34:05.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.435+0000 7f1c812b5640 1 Processor -- start 2024-03-04T07:34:05.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.435+0000 7f1c812b5640 1 -- start start 2024-03-04T07:34:05.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c812b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c7c06b5c0 0x7f1c7c1dd8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c812b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c7c0ae9b0 0x7f1c7c0cd700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c812b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06f690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c812b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c7c06c910 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c812b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c7c06c790 con 0x7f1c7c0ae9b0 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c812b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1c7c0aedc0 con 0x7f1c7c06b5c0 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c7b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06f690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c7b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06f690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35864/0 (socket says 172.21.15.64:35864) 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c7b7fe640 1 -- 172.21.15.64:0/194914451 learned_addr learned my addr 172.21.15.64:0/194914451 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c7affd640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c7c06b5c0 0x7f1c7c1dd8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.436+0000 7f1c7a7fc640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c7c0ae9b0 0x7f1c7c0cd700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c7b7fe640 1 -- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c7c06b5c0 msgr2=0x7f1c7c1dd8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c7b7fe640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c7c06b5c0 0x7f1c7c1dd8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c7b7fe640 1 -- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c7c0ae9b0 msgr2=0x7f1c7c0cd700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c7b7fe640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c7c0ae9b0 0x7f1c7c0cd700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c7b7fe640 1 -- 172.21.15.64:0/194914451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c7c0c5b30 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c7b7fe640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06f690 secure :-1 s=READY pgs=1580 cs=0 l=1 rev1=1 crypto rx=0x7f1c6c00e930 tx=0x7f1c6c00edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:05.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c737fe640 1 -- 172.21.15.64:0/194914451 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c6c017070 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c737fe640 1 -- 172.21.15.64:0/194914451 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1c6c00c980 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1c7c0c5e10 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c737fe640 1 -- 172.21.15.64:0/194914451 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c6c00cd70 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.437+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c7c0c63d0 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.438+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1c7c068700 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.438+0000 7f1c737fe640 1 -- 172.21.15.64:0/194914451 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1c6c01a070 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.439+0000 7f1c737fe640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c5c06b4d0 0x7f1c5c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:05.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.439+0000 7f1c737fe640 1 -- 172.21.15.64:0/194914451 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f1c6c08a490 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.439+0000 7f1c7affd640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c5c06b4d0 0x7f1c5c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:05.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.439+0000 7f1c7affd640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c5c06b4d0 0x7f1c5c06d990 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7f1c74001f10 tx=0x7f1c74009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:05.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.443+0000 7f1c737fe640 1 -- 172.21.15.64:0/194914451 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1c6c058e10 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.661+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1c7c1e9aa0 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.662+0000 7f1c737fe640 1 -- 172.21.15.64:0/194914451 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f1c6c05ceb0 con 0x7f1c7c0b0e90 2024-03-04T07:34:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:05.662 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:05.662 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c5c06b4d0 msgr2=0x7f1c5c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c5c06b4d0 0x7f1c5c06d990 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7f1c74001f10 tx=0x7f1c74009040 comp rx=0 tx=0).stop 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 msgr2=0x7f1c7c06f690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06f690 secure :-1 s=READY pgs=1580 cs=0 l=1 rev1=1 crypto rx=0x7f1c6c00e930 tx=0x7f1c6c00edf0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 shutdown_connections 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1c5c06b4d0 0x7f1c5c06d990 unknown :-1 s=CLOSED pgs=708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1c7c0b0e90 0x7f1c7c06f690 unknown :-1 s=CLOSED pgs=1580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1c7c0ae9b0 0x7f1c7c0cd700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 --2- 172.21.15.64:0/194914451 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1c7c06b5c0 0x7f1c7c1dd8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 >> 172.21.15.64:0/194914451 conn(0x7f1c7c0b36d0 msgr2=0x7f1c7c0b1350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 shutdown_connections 2024-03-04T07:34:05.664 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:05.664+0000 7f1c812b5640 1 -- 172.21.15.64:0/194914451 wait complete. 2024-03-04T07:34:05.876 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:05.876 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:06.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:06.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.990+0000 7f821f70c640 1 Processor -- start 2024-03-04T07:34:06.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.991+0000 7f821f70c640 1 -- start start 2024-03-04T07:34:06.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.991+0000 7f821f70c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f821815df00 0x7f82180d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:06.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.991+0000 7f821f70c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f82180b9a30 0x7f82180d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:06.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.991+0000 7f821f70c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f82180cea30 con 0x7f82180b95a0 2024-03-04T07:34:06.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821f70c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f82180d2af0 con 0x7f821815df00 2024-03-04T07:34:06.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821f70c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f82180cf3c0 con 0x7f82180b9a30 2024-03-04T07:34:06.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821d4a8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f821815df00 0x7f82180d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:06.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821d4a8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f821815df00 0x7f82180d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35868/0 (socket says 172.21.15.64:35868) 2024-03-04T07:34:06.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821dca9640 1 --2- 172.21.15.64:0/1665436705 >> v2:172.21.15.66:3300/0 conn(0x7f82180b9a30 0x7f82180d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:06.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821d4a8640 1 -- 172.21.15.64:0/1665436705 learned_addr learned my addr 172.21.15.64:0/1665436705 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:06.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821d4a8640 1 -- 172.21.15.64:0/1665436705 >> v2:172.21.15.66:3300/0 conn(0x7f82180b9a30 msgr2=0x7f82180d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:06.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f820ffff640 1 -- 172.21.15.64:0/1665436705 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3002203552 0 0) 0x7f82180cea30 con 0x7f82180b95a0 2024-03-04T07:34:06.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821d4a8640 1 --2- 172.21.15.64:0/1665436705 >> v2:172.21.15.66:3300/0 conn(0x7f82180b9a30 0x7f82180d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:06.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821d4a8640 1 -- 172.21.15.64:0/1665436705 >> v1:172.21.15.66:6789/0 conn(0x7f82180b95a0 legacy=0x7f82180b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:06.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.992+0000 7f821d4a8640 1 -- 172.21.15.64:0/1665436705 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82180cf0e0 con 0x7f821815df00 2024-03-04T07:34:06.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.993+0000 7f821d4a8640 1 --2- 172.21.15.64:0/1665436705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f821815df00 0x7f82180d25b0 secure :-1 s=READY pgs=1581 cs=0 l=1 rev1=1 crypto rx=0x7f82180d4d20 tx=0x7f821402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3d9337e9222b6875 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.993+0000 7f820ffff640 1 -- 172.21.15.64:0/1665436705 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8214034070 con 0x7f821815df00 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.993+0000 7f820ffff640 1 -- 172.21.15.64:0/1665436705 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8214030860 con 0x7f821815df00 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.993+0000 7f820ffff640 1 -- 172.21.15.64:0/1665436705 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8214030c30 con 0x7f821815df00 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.993+0000 7f821f70c640 1 -- 172.21.15.64:0/1665436705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f821815df00 msgr2=0x7f82180d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.993+0000 7f821f70c640 1 --2- 172.21.15.64:0/1665436705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f821815df00 0x7f82180d25b0 secure :-1 s=READY pgs=1581 cs=0 l=1 rev1=1 crypto rx=0x7f82180d4d20 tx=0x7f821402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.994+0000 7f821f70c640 1 -- 172.21.15.64:0/1665436705 shutdown_connections 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.994+0000 7f821f70c640 1 --2- 172.21.15.64:0/1665436705 >> v2:172.21.15.66:3300/0 conn(0x7f82180b9a30 0x7f82180d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.994+0000 7f821f70c640 1 --2- 172.21.15.64:0/1665436705 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f821815df00 0x7f82180d25b0 unknown :-1 s=CLOSED pgs=1581 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.994+0000 7f821f70c640 1 -- 172.21.15.64:0/1665436705 >> 172.21.15.64:0/1665436705 conn(0x7f8218071960 msgr2=0x7f8218071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.994+0000 7f821f70c640 1 -- 172.21.15.64:0/1665436705 shutdown_connections 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.994+0000 7f821f70c640 1 -- 172.21.15.64:0/1665436705 wait complete. 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.995+0000 7f821f70c640 1 Processor -- start 2024-03-04T07:34:06.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.995+0000 7f821f70c640 1 -- start start 2024-03-04T07:34:06.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821f70c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82180b95a0 0x7f82180c7840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:06.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821f70c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f821815df00 0x7f8218157cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:06.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821f70c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82180c8580 0x7f82180b49a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:06.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821f70c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82180cec70 con 0x7f82180b95a0 2024-03-04T07:34:06.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821f70c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82180ce750 con 0x7f82180c8580 2024-03-04T07:34:06.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821f70c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f82180cedf0 con 0x7f821815df00 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82180b95a0 0x7f82180c7840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82180b95a0 0x7f82180c7840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35880/0 (socket says 172.21.15.64:35880) 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 -- 172.21.15.64:0/3266121387 learned_addr learned my addr 172.21.15.64:0/3266121387 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821dca9640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82180c8580 0x7f82180b49a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821cca7640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f821815df00 0x7f8218157cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 -- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f821815df00 msgr2=0x7f8218157cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f821815df00 0x7f8218157cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 -- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82180c8580 msgr2=0x7f82180b49a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82180c8580 0x7f82180b49a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 -- 172.21.15.64:0/3266121387 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82180c90e0 con 0x7f82180b95a0 2024-03-04T07:34:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821d4a8640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82180b95a0 0x7f82180c7840 secure :-1 s=READY pgs=1582 cs=0 l=1 rev1=1 crypto rx=0x7f8214000c00 tx=0x7f82140046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:07.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f820dffb640 1 -- 172.21.15.64:0/3266121387 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8214034040 con 0x7f82180b95a0 2024-03-04T07:34:07.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f820dffb640 1 -- 172.21.15.64:0/3266121387 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f821403ab90 con 0x7f82180b95a0 2024-03-04T07:34:07.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f821815e2e0 con 0x7f82180b95a0 2024-03-04T07:34:07.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.996+0000 7f820dffb640 1 -- 172.21.15.64:0/3266121387 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8214004030 con 0x7f82180b95a0 2024-03-04T07:34:07.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.997+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f82180c8e00 con 0x7f82180b95a0 2024-03-04T07:34:07.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.998+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8218067ff0 con 0x7f82180b95a0 2024-03-04T07:34:07.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.998+0000 7f820dffb640 1 -- 172.21.15.64:0/3266121387 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f821404b060 con 0x7f82180b95a0 2024-03-04T07:34:07.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.999+0000 7f820dffb640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81fc06b4d0 0x7f81fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:07.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.999+0000 7f821cca7640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81fc06b4d0 0x7f81fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:07.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.999+0000 7f820dffb640 1 -- 172.21.15.64:0/3266121387 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f82140abeb0 con 0x7f82180b95a0 2024-03-04T07:34:07.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:06.999+0000 7f821cca7640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81fc06b4d0 0x7f81fc06d990 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7f82180b6260 tx=0x7f8208009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:07.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.003+0000 7f820dffb640 1 -- 172.21.15.64:0/3266121387 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f821403f070 con 0x7f82180b95a0 2024-03-04T07:34:07.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.221+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f82180c7d80 con 0x7f82180b95a0 2024-03-04T07:34:07.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.221+0000 7f820dffb640 1 -- 172.21.15.64:0/3266121387 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f821407a6e0 con 0x7f82180b95a0 2024-03-04T07:34:07.221 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:07.222 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:07.222 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:07.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81fc06b4d0 msgr2=0x7f81fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81fc06b4d0 0x7f81fc06d990 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7f82180b6260 tx=0x7f8208009040 comp rx=0 tx=0).stop 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82180b95a0 msgr2=0x7f82180c7840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82180b95a0 0x7f82180c7840 secure :-1 s=READY pgs=1582 cs=0 l=1 rev1=1 crypto rx=0x7f8214000c00 tx=0x7f82140046c0 comp rx=0 tx=0).stop 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 shutdown_connections 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81fc06b4d0 0x7f81fc06d990 unknown :-1 s=CLOSED pgs=709 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f82180c8580 0x7f82180b49a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f821815df00 0x7f8218157cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 --2- 172.21.15.64:0/3266121387 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f82180b95a0 0x7f82180c7840 unknown :-1 s=CLOSED pgs=1582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 >> 172.21.15.64:0/3266121387 conn(0x7f8218071960 msgr2=0x7f82181618b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 shutdown_connections 2024-03-04T07:34:07.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:07.223+0000 7f821f70c640 1 -- 172.21.15.64:0/3266121387 wait complete. 2024-03-04T07:34:07.436 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:07.436 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:08.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:08.523 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:08.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.551+0000 7f965e437640 1 Processor -- start 2024-03-04T07:34:08.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.551+0000 7f965e437640 1 -- start start 2024-03-04T07:34:08.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f965e437640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96580d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:08.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f965e437640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96580d18c0 0x7f9658071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f965e437640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f96580b6f20 con 0x7f9658071a80 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f965e437640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96580d3030 con 0x7f96580d18c0 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f965e437640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96580d3310 con 0x7f965815df70 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f96577fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96580d18c0 0x7f9658071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f965c9d4640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f9658071a80 0x7f96580b2700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:53192/0 (socket says 172.21.15.64:53192) 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f965c9d4640 1 -- 172.21.15.64:0/1031755719 learned_addr learned my addr 172.21.15.64:0/1031755719 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f96577fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96580d18c0 0x7f9658071540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35882/0 (socket says 172.21.15.64:35882) 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.552+0000 7f9657fff640 1 --2- 172.21.15.64:0/1031755719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96580d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.553+0000 7f96577fe640 1 -- 172.21.15.64:0/1031755719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 msgr2=0x7f96580d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.553+0000 7f96577fe640 1 --2- 172.21.15.64:0/1031755719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96580d27b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.553+0000 7f96577fe640 1 -- 172.21.15.64:0/1031755719 >> v1:172.21.15.64:6789/0 conn(0x7f9658071a80 legacy=0x7f96580b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.553+0000 7f96577fe640 1 -- 172.21.15.64:0/1031755719 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96580d35f0 con 0x7f96580d18c0 2024-03-04T07:34:08.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.553+0000 7f9657fff640 1 --2- 172.21.15.64:0/1031755719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96580d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.553+0000 7f96577fe640 1 --2- 172.21.15.64:0/1031755719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96580d18c0 0x7f9658071540 secure :-1 s=READY pgs=1584 cs=0 l=1 rev1=1 crypto rx=0x7f96400097a0 tx=0x7f964002b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6cda0a3b8a451f99 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f9656ffd640 1 -- 172.21.15.64:0/1031755719 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9640034070 con 0x7f96580d18c0 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f9656ffd640 1 -- 172.21.15.64:0/1031755719 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9640030610 con 0x7f96580d18c0 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f9656ffd640 1 -- 172.21.15.64:0/1031755719 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96400309e0 con 0x7f96580d18c0 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 -- 172.21.15.64:0/1031755719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96580d18c0 msgr2=0x7f9658071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 --2- 172.21.15.64:0/1031755719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96580d18c0 0x7f9658071540 secure :-1 s=READY pgs=1584 cs=0 l=1 rev1=1 crypto rx=0x7f96400097a0 tx=0x7f964002b960 comp rx=0 tx=0).stop 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 -- 172.21.15.64:0/1031755719 shutdown_connections 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 --2- 172.21.15.64:0/1031755719 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96580d18c0 0x7f9658071540 unknown :-1 s=CLOSED pgs=1584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 --2- 172.21.15.64:0/1031755719 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96580d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 -- 172.21.15.64:0/1031755719 >> 172.21.15.64:0/1031755719 conn(0x7f96580d3cf0 msgr2=0x7f96580d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 -- 172.21.15.64:0/1031755719 shutdown_connections 2024-03-04T07:34:08.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.554+0000 7f965e437640 1 -- 172.21.15.64:0/1031755719 wait complete. 2024-03-04T07:34:08.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.555+0000 7f965e437640 1 Processor -- start 2024-03-04T07:34:08.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.556+0000 7f965e437640 1 -- start start 2024-03-04T07:34:08.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.556+0000 7f965e437640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9658071a80 0x7f96580cb0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.556+0000 7f965e437640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96580d18c0 0x7f96581f4c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.556+0000 7f965e437640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96581f3ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.556+0000 7f965e437640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f965815e350 con 0x7f9658071a80 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.556+0000 7f965e437640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96580d2cf0 con 0x7f96580d18c0 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.556+0000 7f965e437640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f965815e4d0 con 0x7f965815df70 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9658071a80 0x7f96580cb0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9658071a80 0x7f96580cb0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35884/0 (socket says 172.21.15.64:35884) 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 -- 172.21.15.64:0/3658316900 learned_addr learned my addr 172.21.15.64:0/3658316900 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f965c9d4640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96581f3ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f96577fe640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96580d18c0 0x7f96581f4c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 -- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 msgr2=0x7f96581f3ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96581f3ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 -- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96580d18c0 msgr2=0x7f96581f4c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96580d18c0 0x7f96581f4c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 -- 172.21.15.64:0/3658316900 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96581f4310 con 0x7f9658071a80 2024-03-04T07:34:08.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9657fff640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9658071a80 0x7f96580cb0c0 secure :-1 s=READY pgs=1585 cs=0 l=1 rev1=1 crypto rx=0x7f964800e960 tx=0x7f964800ee20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:08.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9654ff9640 1 -- 172.21.15.64:0/3658316900 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9648017070 con 0x7f9658071a80 2024-03-04T07:34:08.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9654ff9640 1 -- 172.21.15.64:0/3658316900 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f964800ca00 con 0x7f9658071a80 2024-03-04T07:34:08.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96581f45f0 con 0x7f9658071a80 2024-03-04T07:34:08.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f9654ff9640 1 -- 172.21.15.64:0/3658316900 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f964800cdd0 con 0x7f9658071a80 2024-03-04T07:34:08.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.557+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96580cd9f0 con 0x7f9658071a80 2024-03-04T07:34:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.558+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9658067ff0 con 0x7f9658071a80 2024-03-04T07:34:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.559+0000 7f9654ff9640 1 -- 172.21.15.64:0/3658316900 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f964801a070 con 0x7f9658071a80 2024-03-04T07:34:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.559+0000 7f9654ff9640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963806b4d0 0x7f963806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.559+0000 7f9654ff9640 1 -- 172.21.15.64:0/3658316900 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f964808a430 con 0x7f9658071a80 2024-03-04T07:34:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.559+0000 7f96577fe640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963806b4d0 0x7f963806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.560+0000 7f96577fe640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963806b4d0 0x7f963806d990 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7f9640036000 tx=0x7f964003b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:08.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.564+0000 7f9654ff9640 1 -- 172.21.15.64:0/3658316900 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9648058be0 con 0x7f9658071a80 2024-03-04T07:34:08.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.784+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f96580b8b40 con 0x7f9658071a80 2024-03-04T07:34:08.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.785+0000 7f9654ff9640 1 -- 172.21.15.64:0/3658316900 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f964805cc80 con 0x7f9658071a80 2024-03-04T07:34:08.785 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:08.785 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:08.785 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:08.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.786+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963806b4d0 msgr2=0x7f963806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.786+0000 7f965e437640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963806b4d0 0x7f963806d990 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7f9640036000 tx=0x7f964003b040 comp rx=0 tx=0).stop 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.786+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9658071a80 msgr2=0x7f96580cb0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.786+0000 7f965e437640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9658071a80 0x7f96580cb0c0 secure :-1 s=READY pgs=1585 cs=0 l=1 rev1=1 crypto rx=0x7f964800e960 tx=0x7f964800ee20 comp rx=0 tx=0).stop 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 shutdown_connections 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f963806b4d0 0x7f963806d990 unknown :-1 s=CLOSED pgs=710 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f965815df70 0x7f96581f3ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96580d18c0 0x7f96581f4c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 --2- 172.21.15.64:0/3658316900 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9658071a80 0x7f96580cb0c0 unknown :-1 s=CLOSED pgs=1585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 >> 172.21.15.64:0/3658316900 conn(0x7f96580d3cf0 msgr2=0x7f96580b7c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 shutdown_connections 2024-03-04T07:34:08.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:08.787+0000 7f965e437640 1 -- 172.21.15.64:0/3658316900 wait complete. 2024-03-04T07:34:08.999 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:08.999 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:10.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:10.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.115+0000 7f1320c9a640 1 Processor -- start 2024-03-04T07:34:10.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.115+0000 7f1320c9a640 1 -- start start 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.116+0000 7f1320c9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c06d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.116+0000 7f1320c9a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f131c071240 con 0x7f131c06dd60 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.116+0000 7f1320c9a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f131c071520 con 0x7f131c0aee10 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.116+0000 7f1320c9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f131c06e1c0 con 0x7f131c158520 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.116+0000 7f1313fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f131c06dd60 0x7f131c0ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:53194/0 (socket says 172.21.15.64:53194) 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.116+0000 7f1313fff640 1 -- 172.21.15.64:0/2160980332 learned_addr learned my addr 172.21.15.64:0/2160980332 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.116+0000 7f131a59c640 1 --2- 172.21.15.64:0/2160980332 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c06d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.117+0000 7f1319d9b640 1 -- 172.21.15.64:0/2160980332 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2632735120 0 0) 0x7f131c071520 con 0x7f131c0aee10 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.117+0000 7f1319d9b640 1 -- 172.21.15.64:0/2160980332 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1308003520 con 0x7f131c0aee10 2024-03-04T07:34:10.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.117+0000 7f1319d9b640 1 -- 172.21.15.64:0/2160980332 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 855781872 0 0) 0x7f131c071240 con 0x7f131c06dd60 2024-03-04T07:34:10.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.117+0000 7f1319d9b640 1 -- 172.21.15.64:0/2160980332 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f131c071520 con 0x7f131c06dd60 2024-03-04T07:34:10.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.117+0000 7f131a59c640 1 -- 172.21.15.64:0/2160980332 >> v1:172.21.15.66:6789/0 conn(0x7f131c0aee10 legacy=0x7f131c0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:10.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.117+0000 7f131a59c640 1 -- 172.21.15.64:0/2160980332 >> v1:172.21.15.64:6789/0 conn(0x7f131c06dd60 legacy=0x7f131c0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:10.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.117+0000 7f131a59c640 1 -- 172.21.15.64:0/2160980332 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f131c071800 con 0x7f131c158520 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f131a59c640 1 --2- 172.21.15.64:0/2160980332 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c06d820 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f130c0097d0 tx=0x7f130c02b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e8930c06f9accfb9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1319d9b640 1 -- 172.21.15.64:0/2160980332 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f130c035020 con 0x7f131c158520 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1319d9b640 1 -- 172.21.15.64:0/2160980332 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f130c031890 con 0x7f131c158520 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1319d9b640 1 -- 172.21.15.64:0/2160980332 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f130c031c40 con 0x7f131c158520 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1320c9a640 1 -- 172.21.15.64:0/2160980332 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 msgr2=0x7f131c06d820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1320c9a640 1 --2- 172.21.15.64:0/2160980332 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c06d820 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f130c0097d0 tx=0x7f130c02b930 comp rx=0 tx=0).stop 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1320c9a640 1 -- 172.21.15.64:0/2160980332 shutdown_connections 2024-03-04T07:34:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1320c9a640 1 --2- 172.21.15.64:0/2160980332 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c06d820 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:10.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1320c9a640 1 -- 172.21.15.64:0/2160980332 >> 172.21.15.64:0/2160980332 conn(0x7f131c066d00 msgr2=0x7f131c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:10.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1320c9a640 1 -- 172.21.15.64:0/2160980332 shutdown_connections 2024-03-04T07:34:10.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.118+0000 7f1320c9a640 1 -- 172.21.15.64:0/2160980332 wait complete. 2024-03-04T07:34:10.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 Processor -- start 2024-03-04T07:34:10.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 -- start start 2024-03-04T07:34:10.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f131c06dd60 0x7f131c14f7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:10.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f131c0aee10 0x7f131c1559f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:10.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c0ccd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:10.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f131c158900 con 0x7f131c0aee10 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f131c071b70 con 0x7f131c06dd60 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.120+0000 7f1320c9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f131c158a80 con 0x7f131c158520 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f131c0aee10 0x7f131c1559f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f131c0aee10 0x7f131c1559f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35886/0 (socket says 172.21.15.64:35886) 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f131a59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f131c06dd60 0x7f131c14f7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 -- 172.21.15.64:0/3616169283 learned_addr learned my addr 172.21.15.64:0/3616169283 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f131ad9d640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c0ccd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 -- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 msgr2=0x7f131c0ccd40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c0ccd40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 -- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f131c06dd60 msgr2=0x7f131c14f7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f131c06dd60 0x7f131c14f7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 -- 172.21.15.64:0/3616169283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f131c0cd280 con 0x7f131c0aee10 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f131a59c640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f131c06dd60 0x7f131c14f7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1313fff640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f131c0aee10 0x7f131c1559f0 secure :-1 s=READY pgs=1587 cs=0 l=1 rev1=1 crypto rx=0x7f130400c440 tx=0x7f130400c900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:10.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1312ffd640 1 -- 172.21.15.64:0/3616169283 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1304017070 con 0x7f131c0aee10 2024-03-04T07:34:10.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1312ffd640 1 -- 172.21.15.64:0/3616169283 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1304005ce0 con 0x7f131c0aee10 2024-03-04T07:34:10.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f131c0cfac0 con 0x7f131c0aee10 2024-03-04T07:34:10.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1312ffd640 1 -- 172.21.15.64:0/3616169283 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13040072b0 con 0x7f131c0aee10 2024-03-04T07:34:10.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.121+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f131c15c4c0 con 0x7f131c0aee10 2024-03-04T07:34:10.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.123+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f131c067ff0 con 0x7f131c0aee10 2024-03-04T07:34:10.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.123+0000 7f1312ffd640 1 -- 172.21.15.64:0/3616169283 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f130401a070 con 0x7f131c0aee10 2024-03-04T07:34:10.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.123+0000 7f1312ffd640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f12fc06b4d0 0x7f12fc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:10.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.123+0000 7f131a59c640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f12fc06b4d0 0x7f12fc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:10.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.123+0000 7f1312ffd640 1 -- 172.21.15.64:0/3616169283 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f13040894e0 con 0x7f131c0aee10 2024-03-04T07:34:10.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.124+0000 7f131a59c640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f12fc06b4d0 0x7f12fc06d990 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f130c036000 tx=0x7f130c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:10.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.128+0000 7f1312ffd640 1 -- 172.21.15.64:0/3616169283 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1304057dd0 con 0x7f131c0aee10 2024-03-04T07:34:10.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.348+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f131c06c6f0 con 0x7f131c0aee10 2024-03-04T07:34:10.348 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.348+0000 7f1312ffd640 1 -- 172.21.15.64:0/3616169283 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f130405be70 con 0x7f131c0aee10 2024-03-04T07:34:10.348 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:10.348 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:10.349 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:10.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f12fc06b4d0 msgr2=0x7f12fc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:10.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f12fc06b4d0 0x7f12fc06d990 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f130c036000 tx=0x7f130c008030 comp rx=0 tx=0).stop 2024-03-04T07:34:10.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f131c0aee10 msgr2=0x7f131c1559f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:10.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f131c0aee10 0x7f131c1559f0 secure :-1 s=READY pgs=1587 cs=0 l=1 rev1=1 crypto rx=0x7f130400c440 tx=0x7f130400c900 comp rx=0 tx=0).stop 2024-03-04T07:34:10.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 shutdown_connections 2024-03-04T07:34:10.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f12fc06b4d0 0x7f12fc06d990 unknown :-1 s=CLOSED pgs=711 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:10.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f131c158520 0x7f131c0ccd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:10.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f131c0aee10 0x7f131c1559f0 unknown :-1 s=CLOSED pgs=1587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:10.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 --2- 172.21.15.64:0/3616169283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f131c06dd60 0x7f131c14f7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:10.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 >> 172.21.15.64:0/3616169283 conn(0x7f131c066d00 msgr2=0x7f131c06cef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:10.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 shutdown_connections 2024-03-04T07:34:10.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:10.350+0000 7f1320c9a640 1 -- 172.21.15.64:0/3616169283 wait complete. 2024-03-04T07:34:10.525 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:10.562 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:10.562 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:11.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:11.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.677+0000 7f3c916c3640 1 Processor -- start 2024-03-04T07:34:11.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.677+0000 7f3c916c3640 1 -- start start 2024-03-04T07:34:11.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c916c3640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3c8c0b2c50 0x7f3c8c0d2ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:11.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c916c3640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c8c0ce970 con 0x7f3c8c071c50 2024-03-04T07:34:11.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c916c3640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c8c0cef30 con 0x7f3c8c15e000 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c916c3640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c8c0cece0 con 0x7f3c8c0b2c50 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c8a7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3c8c071c50 0x7f3c8c0d25d0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:53200/0 (socket says 172.21.15.64:53200) 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c8a7fc640 1 -- 172.21.15.64:0/2495716123 learned_addr learned my addr 172.21.15.64:0/2495716123 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c8b7fe640 1 --2- 172.21.15.64:0/2495716123 >> v2:172.21.15.66:3300/0 conn(0x7f3c8c0b2c50 0x7f3c8c0d2ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c89ffb640 1 -- 172.21.15.64:0/2495716123 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2811454967 0 0) 0x7f3c8c0ce970 con 0x7f3c8c071c50 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c89ffb640 1 -- 172.21.15.64:0/2495716123 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3c74003520 con 0x7f3c8c071c50 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c8b7fe640 1 -- 172.21.15.64:0/2495716123 >> v1:172.21.15.66:6789/0 conn(0x7f3c8c15e000 legacy=0x7f3c8c071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c8b7fe640 1 -- 172.21.15.64:0/2495716123 >> v1:172.21.15.64:6789/0 conn(0x7f3c8c071c50 legacy=0x7f3c8c0d25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:11.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.678+0000 7f3c8b7fe640 1 -- 172.21.15.64:0/2495716123 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c8c0cf210 con 0x7f3c8c0b2c50 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.679+0000 7f3c8b7fe640 1 --2- 172.21.15.64:0/2495716123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0d2ce0 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f3c840097a0 tx=0x7f3c8402b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1916c3a8e6b4653d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.679+0000 7f3c89ffb640 1 -- 172.21.15.64:0/2495716123 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c84035020 con 0x7f3c8c0b2c50 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.679+0000 7f3c89ffb640 1 -- 172.21.15.64:0/2495716123 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c84031830 con 0x7f3c8c0b2c50 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.679+0000 7f3c89ffb640 1 -- 172.21.15.64:0/2495716123 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c84031be0 con 0x7f3c8c0b2c50 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.680+0000 7f3c916c3640 1 -- 172.21.15.64:0/2495716123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c0b2c50 msgr2=0x7f3c8c0d2ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.680+0000 7f3c916c3640 1 --2- 172.21.15.64:0/2495716123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0d2ce0 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f3c840097a0 tx=0x7f3c8402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.680+0000 7f3c916c3640 1 -- 172.21.15.64:0/2495716123 shutdown_connections 2024-03-04T07:34:11.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.680+0000 7f3c916c3640 1 --2- 172.21.15.64:0/2495716123 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0d2ce0 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.680+0000 7f3c916c3640 1 -- 172.21.15.64:0/2495716123 >> 172.21.15.64:0/2495716123 conn(0x7f3c8c0d3c80 msgr2=0x7f3c8c0d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:11.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.680+0000 7f3c916c3640 1 -- 172.21.15.64:0/2495716123 shutdown_connections 2024-03-04T07:34:11.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.680+0000 7f3c916c3640 1 -- 172.21.15.64:0/2495716123 wait complete. 2024-03-04T07:34:11.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.681+0000 7f3c916c3640 1 Processor -- start 2024-03-04T07:34:11.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.681+0000 7f3c916c3640 1 -- start start 2024-03-04T07:34:11.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c916c3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c071c50 0x7f3c8c152200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:11.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c916c3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0c8410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c916c3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c8c15e000 0x7f3c8c0c8950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c916c3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8c0b3320 con 0x7f3c8c0b2c50 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c916c3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8c0b31a0 con 0x7f3c8c071c50 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c916c3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c8c15e3e0 con 0x7f3c8c15e000 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c8affd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c071c50 0x7f3c8c152200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c8a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0c8410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c8affd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c071c50 0x7f3c8c152200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:51088/0 (socket says 172.21.15.64:51088) 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c8affd640 1 -- 172.21.15.64:0/1392169985 learned_addr learned my addr 172.21.15.64:0/1392169985 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.682+0000 7f3c8b7fe640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c8c15e000 0x7f3c8c0c8950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c8affd640 1 -- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c8c15e000 msgr2=0x7f3c8c0c8950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c8affd640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c8c15e000 0x7f3c8c0c8950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c8affd640 1 -- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c8c0b2c50 msgr2=0x7f3c8c0c8410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:11.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c8affd640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0c8410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c8affd640 1 -- 172.21.15.64:0/1392169985 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c8c0c8f30 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c8a7fc640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0c8410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c8affd640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c071c50 0x7f3c8c152200 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f3c7c00c790 tx=0x7f3c7c00cc50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c737fe640 1 -- 172.21.15.64:0/1392169985 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c7c016020 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c737fe640 1 -- 172.21.15.64:0/1392169985 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c7c006d50 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c8c0cadf0 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c737fe640 1 -- 172.21.15.64:0/1392169985 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c7c003bb0 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.683+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c8c0cb3b0 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.684+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c8c067ff0 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.685+0000 7f3c737fe640 1 -- 172.21.15.64:0/1392169985 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3c7c00a850 con 0x7f3c8c071c50 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.685+0000 7f3c737fe640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c7806b4d0 0x7f3c7806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:11.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.685+0000 7f3c737fe640 1 -- 172.21.15.64:0/1392169985 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3c7c089d50 con 0x7f3c8c071c50 2024-03-04T07:34:11.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.685+0000 7f3c8a7fc640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c7806b4d0 0x7f3c7806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:11.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.685+0000 7f3c8a7fc640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c7806b4d0 0x7f3c7806d990 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7f3c8c0ca1c0 tx=0x7f3c80007d30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:11.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.690+0000 7f3c737fe640 1 -- 172.21.15.64:0/1392169985 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3c7c08d030 con 0x7f3c8c071c50 2024-03-04T07:34:11.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.907+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3c8c0ca830 con 0x7f3c8c071c50 2024-03-04T07:34:11.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.908+0000 7f3c737fe640 1 -- 172.21.15.64:0/1392169985 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f3c7c020020 con 0x7f3c8c071c50 2024-03-04T07:34:11.908 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:11.908 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:11.908 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:11.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c7806b4d0 msgr2=0x7f3c7806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c7806b4d0 0x7f3c7806d990 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7f3c8c0ca1c0 tx=0x7f3c80007d30 comp rx=0 tx=0).stop 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c071c50 msgr2=0x7f3c8c152200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c071c50 0x7f3c8c152200 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f3c7c00c790 tx=0x7f3c7c00cc50 comp rx=0 tx=0).stop 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 shutdown_connections 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3c7806b4d0 0x7f3c7806d990 unknown :-1 s=CLOSED pgs=712 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c8c15e000 0x7f3c8c0c8950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c8c0b2c50 0x7f3c8c0c8410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 --2- 172.21.15.64:0/1392169985 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c8c071c50 0x7f3c8c152200 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 >> 172.21.15.64:0/1392169985 conn(0x7f3c8c0d3c80 msgr2=0x7f3c8c15f8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 shutdown_connections 2024-03-04T07:34:11.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:11.910+0000 7f3c916c3640 1 -- 172.21.15.64:0/1392169985 wait complete. 2024-03-04T07:34:12.122 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:12.123 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:12.528 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:13.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:13.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.238+0000 7f7ac795e640 1 Processor -- start 2024-03-04T07:34:13.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.238+0000 7f7ac795e640 1 -- start start 2024-03-04T07:34:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac795e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ac0071c50 0x7f7ac00d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac795e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7ac00b2c50 0x7f7ac00d2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac795e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7ac00ce970 con 0x7f7ac015e000 2024-03-04T07:34:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac795e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7ac00cece0 con 0x7f7ac0071c50 2024-03-04T07:34:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac795e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7ac00cef30 con 0x7f7ac00b2c50 2024-03-04T07:34:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac4ef9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ac0071c50 0x7f7ac00d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac4ef9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7ac0071c50 0x7f7ac00d25d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35908/0 (socket says 172.21.15.64:35908) 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/2739974698 learned_addr learned my addr 172.21.15.64:0/2739974698 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac5efb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7ac00b2c50 0x7f7ac00d2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ab7fff640 1 -- 172.21.15.64:0/2739974698 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1442981446 0 0) 0x7f7ac00ce970 con 0x7f7ac015e000 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/2739974698 >> v2:172.21.15.66:3300/0 conn(0x7f7ac00b2c50 msgr2=0x7f7ac00d2b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac4ef9640 1 --2- 172.21.15.64:0/2739974698 >> v2:172.21.15.66:3300/0 conn(0x7f7ac00b2c50 0x7f7ac00d2b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/2739974698 >> v1:172.21.15.66:6789/0 conn(0x7f7ac015e000 legacy=0x7f7ac0071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:13.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.239+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/2739974698 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ac00cf210 con 0x7f7ac0071c50 2024-03-04T07:34:13.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.240+0000 7f7ac4ef9640 1 --2- 172.21.15.64:0/2739974698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac0071c50 0x7f7ac00d25d0 secure :-1 s=READY pgs=1589 cs=0 l=1 rev1=1 crypto rx=0x7f7ac0162d50 tx=0x7f7ab002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=45d4882e7120772b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.240+0000 7f7ab7fff640 1 -- 172.21.15.64:0/2739974698 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ab0035020 con 0x7f7ac0071c50 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.240+0000 7f7ab7fff640 1 -- 172.21.15.64:0/2739974698 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ab0031870 con 0x7f7ac0071c50 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.240+0000 7f7ab7fff640 1 -- 172.21.15.64:0/2739974698 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ab0031c40 con 0x7f7ac0071c50 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 -- 172.21.15.64:0/2739974698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac0071c50 msgr2=0x7f7ac00d25d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 --2- 172.21.15.64:0/2739974698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac0071c50 0x7f7ac00d25d0 secure :-1 s=READY pgs=1589 cs=0 l=1 rev1=1 crypto rx=0x7f7ac0162d50 tx=0x7f7ab002b910 comp rx=0 tx=0).stop 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 -- 172.21.15.64:0/2739974698 shutdown_connections 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 --2- 172.21.15.64:0/2739974698 >> v2:172.21.15.66:3300/0 conn(0x7f7ac00b2c50 0x7f7ac00d2b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 --2- 172.21.15.64:0/2739974698 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac0071c50 0x7f7ac00d25d0 unknown :-1 s=CLOSED pgs=1589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 -- 172.21.15.64:0/2739974698 >> 172.21.15.64:0/2739974698 conn(0x7f7ac00d3c80 msgr2=0x7f7ac00d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 -- 172.21.15.64:0/2739974698 shutdown_connections 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.241+0000 7f7ac795e640 1 -- 172.21.15.64:0/2739974698 wait complete. 2024-03-04T07:34:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.242+0000 7f7ac795e640 1 Processor -- start 2024-03-04T07:34:13.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.242+0000 7f7ac795e640 1 -- start start 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac795e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ac0071c50 0x7f7ac01f78b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac795e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac00b2c50 0x7f7ac0154d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac795e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ac015e000 0x7f7ac0160890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac795e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ac00b3320 con 0x7f7ac00b2c50 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac795e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ac00b31a0 con 0x7f7ac0071c50 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac795e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7ac015e3e0 con 0x7f7ac015e000 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac00b2c50 0x7f7ac0154d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac00b2c50 0x7f7ac0154d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35914/0 (socket says 172.21.15.64:35914) 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/4246258544 learned_addr learned my addr 172.21.15.64:0/4246258544 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac56fa640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ac0071c50 0x7f7ac01f78b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac5efb640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ac015e000 0x7f7ac0160890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:13.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ac015e000 msgr2=0x7f7ac0160890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ac015e000 0x7f7ac0160890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ac0071c50 msgr2=0x7f7ac01f78b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ac0071c50 0x7f7ac01f78b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 -- 172.21.15.64:0/4246258544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ac01f7df0 con 0x7f7ac00b2c50 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ac4ef9640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac00b2c50 0x7f7ac0154d70 secure :-1 s=READY pgs=1590 cs=0 l=1 rev1=1 crypto rx=0x7f7ab002b950 tx=0x7f7ab00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.243+0000 7f7ab5ffb640 1 -- 172.21.15.64:0/4246258544 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ab0040070 con 0x7f7ac00b2c50 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.244+0000 7f7ab5ffb640 1 -- 172.21.15.64:0/4246258544 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7ab0030d00 con 0x7f7ac00b2c50 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.244+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7ac00cc260 con 0x7f7ac00b2c50 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.244+0000 7f7ab5ffb640 1 -- 172.21.15.64:0/4246258544 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ab0035040 con 0x7f7ac00b2c50 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.244+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ac00cc820 con 0x7f7ac00b2c50 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.245+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7ac0067ff0 con 0x7f7ac00b2c50 2024-03-04T07:34:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.245+0000 7f7ab5ffb640 1 -- 172.21.15.64:0/4246258544 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7ab004b060 con 0x7f7ac00b2c50 2024-03-04T07:34:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.246+0000 7f7ab5ffb640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7aa406b4d0 0x7f7aa406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.246+0000 7f7ab5ffb640 1 -- 172.21.15.64:0/4246258544 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f7ab00abd10 con 0x7f7ac00b2c50 2024-03-04T07:34:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.246+0000 7f7ac56fa640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7aa406b4d0 0x7f7aa406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.246+0000 7f7ac56fa640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7aa406b4d0 0x7f7aa406d990 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7f7abc000f60 tx=0x7f7abc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:13.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.250+0000 7f7ab5ffb640 1 -- 172.21.15.64:0/4246258544 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7ab007a5d0 con 0x7f7ac00b2c50 2024-03-04T07:34:13.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.471+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7ac015d370 con 0x7f7ac00b2c50 2024-03-04T07:34:13.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.472+0000 7f7ab5ffb640 1 -- 172.21.15.64:0/4246258544 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f7ab007e670 con 0x7f7ac00b2c50 2024-03-04T07:34:13.472 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:13.472 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:13.472 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:13.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7aa406b4d0 msgr2=0x7f7aa406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:13.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7aa406b4d0 0x7f7aa406d990 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7f7abc000f60 tx=0x7f7abc009040 comp rx=0 tx=0).stop 2024-03-04T07:34:13.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac00b2c50 msgr2=0x7f7ac0154d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:13.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac00b2c50 0x7f7ac0154d70 secure :-1 s=READY pgs=1590 cs=0 l=1 rev1=1 crypto rx=0x7f7ab002b950 tx=0x7f7ab00046c0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 shutdown_connections 2024-03-04T07:34:13.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7aa406b4d0 0x7f7aa406d990 unknown :-1 s=CLOSED pgs=713 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7ac015e000 0x7f7ac0160890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7ac00b2c50 0x7f7ac0154d70 unknown :-1 s=CLOSED pgs=1590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 --2- 172.21.15.64:0/4246258544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7ac0071c50 0x7f7ac01f78b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:13.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 >> 172.21.15.64:0/4246258544 conn(0x7f7ac00d3c80 msgr2=0x7f7ac015e960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:13.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 shutdown_connections 2024-03-04T07:34:13.475 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:13.474+0000 7f7ac795e640 1 -- 172.21.15.64:0/4246258544 wait complete. 2024-03-04T07:34:13.686 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:13.687 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:14.529 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:14.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:14.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.802+0000 7f218940e640 1 Processor -- start 2024-03-04T07:34:14.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.802+0000 7f218940e640 1 -- start start 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f218940e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f21840b1020 0x7f218406af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f218940e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21840cc0e0 con 0x7f218406ed10 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f218940e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f21840cc3c0 con 0x7f21840cd890 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f218940e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f21840d28b0 con 0x7f21840b1020 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f21827fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f218406ed10 0x7f21840cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:46322/0 (socket says 172.21.15.64:46322) 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f21827fc640 1 -- 172.21.15.64:0/220802623 learned_addr learned my addr 172.21.15.64:0/220802623 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2182ffd640 1 --2- 172.21.15.64:0/220802623 >> v2:172.21.15.66:3300/0 conn(0x7f21840b1020 0x7f218406af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:14.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2181ffb640 1 -- 172.21.15.64:0/220802623 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1389771692 0 0) 0x7f21840cc0e0 con 0x7f218406ed10 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2181ffb640 1 -- 172.21.15.64:0/220802623 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f216c003520 con 0x7f218406ed10 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2181ffb640 1 -- 172.21.15.64:0/220802623 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3779339441 0 0) 0x7f21840cc3c0 con 0x7f21840cd890 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2181ffb640 1 -- 172.21.15.64:0/220802623 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f21840cc0e0 con 0x7f21840cd890 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2182ffd640 1 -- 172.21.15.64:0/220802623 >> v1:172.21.15.66:6789/0 conn(0x7f21840cd890 legacy=0x7f218415d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2182ffd640 1 -- 172.21.15.64:0/220802623 >> v1:172.21.15.64:6789/0 conn(0x7f218406ed10 legacy=0x7f21840cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.803+0000 7f2182ffd640 1 -- 172.21.15.64:0/220802623 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21840cc6a0 con 0x7f21840b1020 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.804+0000 7f2182ffd640 1 --2- 172.21.15.64:0/220802623 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21840b1020 0x7f218406af20 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f2174009d90 tx=0x7f217402bf50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a98cbe6c88234bf8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.804+0000 7f2181ffb640 1 -- 172.21.15.64:0/220802623 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2174039070 con 0x7f21840b1020 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.804+0000 7f2181ffb640 1 -- 172.21.15.64:0/220802623 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f217402cd00 con 0x7f21840b1020 2024-03-04T07:34:14.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.804+0000 7f2181ffb640 1 -- 172.21.15.64:0/220802623 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2174034660 con 0x7f21840b1020 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.805+0000 7f218940e640 1 -- 172.21.15.64:0/220802623 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21840b1020 msgr2=0x7f218406af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.805+0000 7f218940e640 1 --2- 172.21.15.64:0/220802623 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21840b1020 0x7f218406af20 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f2174009d90 tx=0x7f217402bf50 comp rx=0 tx=0).stop 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.805+0000 7f218940e640 1 -- 172.21.15.64:0/220802623 shutdown_connections 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.805+0000 7f218940e640 1 --2- 172.21.15.64:0/220802623 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f21840b1020 0x7f218406af20 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.805+0000 7f218940e640 1 -- 172.21.15.64:0/220802623 >> 172.21.15.64:0/220802623 conn(0x7f21840d1a90 msgr2=0x7f21840d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.805+0000 7f218940e640 1 -- 172.21.15.64:0/220802623 shutdown_connections 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.805+0000 7f218940e640 1 -- 172.21.15.64:0/220802623 wait complete. 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.806+0000 7f218940e640 1 Processor -- start 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.806+0000 7f218940e640 1 -- start start 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f218940e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f218406ed10 0x7f21840b1e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:14.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f218940e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21840b1020 0x7f218415cd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f218940e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21840cd890 0x7f21840cad30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f218940e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21840ccc60 con 0x7f21840b1020 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f218940e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21840cf320 con 0x7f218406ed10 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f218940e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21840cdca0 con 0x7f21840cd890 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21840b1020 0x7f218415cd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f2182ffd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f218406ed10 0x7f21840b1e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21840b1020 0x7f218415cd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53878/0 (socket says 172.21.15.64:53878) 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 -- 172.21.15.64:0/1105844441 learned_addr learned my addr 172.21.15.64:0/1105844441 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21837fe640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21840cd890 0x7f21840cad30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 -- 172.21.15.64:0/1105844441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21840cd890 msgr2=0x7f21840cad30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21840cd890 0x7f21840cad30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 -- 172.21.15.64:0/1105844441 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f218406ed10 msgr2=0x7f21840b1e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f218406ed10 0x7f21840b1e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.807+0000 7f21827fc640 1 -- 172.21.15.64:0/1105844441 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21840b2350 con 0x7f21840b1020 2024-03-04T07:34:14.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.808+0000 7f21827fc640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21840b1020 0x7f218415cd90 secure :-1 s=READY pgs=1592 cs=0 l=1 rev1=1 crypto rx=0x7f217800c530 tx=0x7f217800c9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.808+0000 7f216b7fe640 1 -- 172.21.15.64:0/1105844441 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2178017070 con 0x7f21840b1020 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.808+0000 7f216b7fe640 1 -- 172.21.15.64:0/1105844441 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2178005ce0 con 0x7f21840b1020 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.808+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21840b25d0 con 0x7f21840b1020 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.808+0000 7f216b7fe640 1 -- 172.21.15.64:0/1105844441 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2178007270 con 0x7f21840b1020 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.808+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21840b4bc0 con 0x7f21840b1020 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.809+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f21840b0880 con 0x7f21840b1020 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.809+0000 7f216b7fe640 1 -- 172.21.15.64:0/1105844441 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f217801a070 con 0x7f21840b1020 2024-03-04T07:34:14.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.810+0000 7f216b7fe640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f217006b4d0 0x7f217006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:14.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.810+0000 7f216b7fe640 1 -- 172.21.15.64:0/1105844441 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f21780894b0 con 0x7f21840b1020 2024-03-04T07:34:14.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.810+0000 7f2182ffd640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f217006b4d0 0x7f217006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:14.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.810+0000 7f2182ffd640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f217006b4d0 0x7f217006d990 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f2174036000 tx=0x7f2174030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:14.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:14.814+0000 7f216b7fe640 1 -- 172.21.15.64:0/1105844441 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f21840b0880 con 0x7f21840b1020 2024-03-04T07:34:15.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.032+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2184067ff0 con 0x7f21840b1020 2024-03-04T07:34:15.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.032+0000 7f216b7fe640 1 -- 172.21.15.64:0/1105844441 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f21840b2350 con 0x7f21840b1020 2024-03-04T07:34:15.032 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:15.032 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:15.033 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:15.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f217006b4d0 msgr2=0x7f217006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:15.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f217006b4d0 0x7f217006d990 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f2174036000 tx=0x7f2174030000 comp rx=0 tx=0).stop 2024-03-04T07:34:15.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21840b1020 msgr2=0x7f218415cd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:15.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21840b1020 0x7f218415cd90 secure :-1 s=READY pgs=1592 cs=0 l=1 rev1=1 crypto rx=0x7f217800c530 tx=0x7f217800c9f0 comp rx=0 tx=0).stop 2024-03-04T07:34:15.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 shutdown_connections 2024-03-04T07:34:15.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f217006b4d0 0x7f217006d990 unknown :-1 s=CLOSED pgs=714 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:15.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f21840cd890 0x7f21840cad30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:15.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f21840b1020 0x7f218415cd90 unknown :-1 s=CLOSED pgs=1592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:15.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 --2- 172.21.15.64:0/1105844441 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f218406ed10 0x7f21840b1e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:15.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 >> 172.21.15.64:0/1105844441 conn(0x7f21840d1a90 msgr2=0x7f21840d0910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:15.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 shutdown_connections 2024-03-04T07:34:15.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:15.034+0000 7f218940e640 1 -- 172.21.15.64:0/1105844441 wait complete. 2024-03-04T07:34:15.247 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:15.247 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:16.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:16.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.363+0000 7f93589eb640 1 Processor -- start 2024-03-04T07:34:16.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.363+0000 7f93589eb640 1 -- start start 2024-03-04T07:34:16.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f93589eb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f93540b1d50 0x7f93540b21b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f93589eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935406d210 0x7f935406d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f93589eb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f935415e410 con 0x7f935415e030 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f93589eb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f93540b26f0 con 0x7f93540b1d50 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f93589eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9354162700 con 0x7f935406d210 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f935259c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f935415e030 0x7f935406ea30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:46338/0 (socket says 172.21.15.64:46338) 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f935259c640 1 -- 172.21.15.64:0/841646112 learned_addr learned my addr 172.21.15.64:0/841646112 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f9352d9d640 1 --2- 172.21.15.64:0/841646112 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935406d210 0x7f935406d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f934bfff640 1 --2- 172.21.15.64:0/841646112 >> v2:172.21.15.66:3300/0 conn(0x7f93540b1d50 0x7f93540b21b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f9351d9b640 1 -- 172.21.15.64:0/841646112 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3504767218 0 0) 0x7f9344001110 con 0x7f935415e030 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f9351d9b640 1 -- 172.21.15.64:0/841646112 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9340003520 con 0x7f935415e030 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f9352d9d640 1 -- 172.21.15.64:0/841646112 >> v2:172.21.15.66:3300/0 conn(0x7f93540b1d50 msgr2=0x7f93540b21b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f9352d9d640 1 --2- 172.21.15.64:0/841646112 >> v2:172.21.15.66:3300/0 conn(0x7f93540b1d50 0x7f93540b21b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f9352d9d640 1 -- 172.21.15.64:0/841646112 >> v1:172.21.15.64:6789/0 conn(0x7f935415e030 legacy=0x7f935406ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:16.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.364+0000 7f9352d9d640 1 -- 172.21.15.64:0/841646112 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f935406f140 con 0x7f935406d210 2024-03-04T07:34:16.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.365+0000 7f9352d9d640 1 --2- 172.21.15.64:0/841646112 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935406d210 0x7f935406d6c0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f934c009d90 tx=0x7f934c02bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cc91903e9ee70fe4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.365+0000 7f9351d9b640 1 -- 172.21.15.64:0/841646112 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f934c039070 con 0x7f935406d210 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.365+0000 7f9351d9b640 1 -- 172.21.15.64:0/841646112 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f934c02cc50 con 0x7f935406d210 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.365+0000 7f9351d9b640 1 -- 172.21.15.64:0/841646112 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f934c034440 con 0x7f935406d210 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 -- 172.21.15.64:0/841646112 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935406d210 msgr2=0x7f935406d6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 --2- 172.21.15.64:0/841646112 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935406d210 0x7f935406d6c0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f934c009d90 tx=0x7f934c02bea0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 -- 172.21.15.64:0/841646112 shutdown_connections 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 --2- 172.21.15.64:0/841646112 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935406d210 0x7f935406d6c0 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 --2- 172.21.15.64:0/841646112 >> v2:172.21.15.66:3300/0 conn(0x7f93540b1d50 0x7f93540b21b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 -- 172.21.15.64:0/841646112 >> 172.21.15.64:0/841646112 conn(0x7f93540b55e0 msgr2=0x7f93540b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:16.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 -- 172.21.15.64:0/841646112 shutdown_connections 2024-03-04T07:34:16.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.366+0000 7f93589eb640 1 -- 172.21.15.64:0/841646112 wait complete. 2024-03-04T07:34:16.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.367+0000 7f93589eb640 1 Processor -- start 2024-03-04T07:34:16.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.367+0000 7f93589eb640 1 -- start start 2024-03-04T07:34:16.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f93589eb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f935406d210 0x7f9354147600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:16.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f93589eb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93540b1d50 0x7f935415bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:16.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f93589eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935415e030 0x7f935415c800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:16.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f93589eb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93541620b0 con 0x7f93540b1d50 2024-03-04T07:34:16.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f93589eb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9354161dd0 con 0x7f935406d210 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f93589eb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9354162390 con 0x7f935415e030 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93540b1d50 0x7f935415bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f935259c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f935406d210 0x7f9354147600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93540b1d50 0x7f935415bde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53892/0 (socket says 172.21.15.64:53892) 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 -- 172.21.15.64:0/2262336479 learned_addr learned my addr 172.21.15.64:0/2262336479 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f9352d9d640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935415e030 0x7f935415c800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 -- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935415e030 msgr2=0x7f935415c800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935415e030 0x7f935415c800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 -- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f935406d210 msgr2=0x7f9354147600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f935406d210 0x7f9354147600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f934bfff640 1 -- 172.21.15.64:0/2262336479 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f935415c320 con 0x7f93540b1d50 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.368+0000 7f935259c640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f935406d210 0x7f9354147600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.369+0000 7f934bfff640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93540b1d50 0x7f935415bde0 secure :-1 s=READY pgs=1594 cs=0 l=1 rev1=1 crypto rx=0x7f933c00e9b0 tx=0x7f933c00ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:16.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.369+0000 7f934affd640 1 -- 172.21.15.64:0/2262336479 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f933c017070 con 0x7f93540b1d50 2024-03-04T07:34:16.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.369+0000 7f934affd640 1 -- 172.21.15.64:0/2262336479 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f933c00c9a0 con 0x7f93540b1d50 2024-03-04T07:34:16.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.369+0000 7f934affd640 1 -- 172.21.15.64:0/2262336479 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f933c00cd70 con 0x7f93540b1d50 2024-03-04T07:34:16.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.369+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f935415ad40 con 0x7f93540b1d50 2024-03-04T07:34:16.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.369+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f935415b300 con 0x7f93540b1d50 2024-03-04T07:34:16.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.370+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9354067ff0 con 0x7f93540b1d50 2024-03-04T07:34:16.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.370+0000 7f934affd640 1 -- 172.21.15.64:0/2262336479 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f933c01a070 con 0x7f93540b1d50 2024-03-04T07:34:16.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.371+0000 7f934affd640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f933406b4d0 0x7f933406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:16.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.371+0000 7f934affd640 1 -- 172.21.15.64:0/2262336479 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f933c08a180 con 0x7f93540b1d50 2024-03-04T07:34:16.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.371+0000 7f935259c640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f933406b4d0 0x7f933406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:16.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.371+0000 7f935259c640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f933406b4d0 0x7f933406d990 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f9344005880 tx=0x7f9344009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:16.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.375+0000 7f934affd640 1 -- 172.21.15.64:0/2262336479 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f933c0589b0 con 0x7f93540b1d50 2024-03-04T07:34:16.531 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:16.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.592+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f935406e210 con 0x7f93540b1d50 2024-03-04T07:34:16.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.592+0000 7f934affd640 1 -- 172.21.15.64:0/2262336479 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f933c05ca50 con 0x7f93540b1d50 2024-03-04T07:34:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:16.593 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:16.593 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:16.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f933406b4d0 msgr2=0x7f933406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:16.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f933406b4d0 0x7f933406d990 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f9344005880 tx=0x7f9344009040 comp rx=0 tx=0).stop 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93540b1d50 msgr2=0x7f935415bde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93540b1d50 0x7f935415bde0 secure :-1 s=READY pgs=1594 cs=0 l=1 rev1=1 crypto rx=0x7f933c00e9b0 tx=0x7f933c00ee70 comp rx=0 tx=0).stop 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 shutdown_connections 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f933406b4d0 0x7f933406d990 unknown :-1 s=CLOSED pgs=715 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f935415e030 0x7f935415c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f93540b1d50 0x7f935415bde0 unknown :-1 s=CLOSED pgs=1594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 --2- 172.21.15.64:0/2262336479 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f935406d210 0x7f9354147600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 >> 172.21.15.64:0/2262336479 conn(0x7f93540b55e0 msgr2=0x7f935406e730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 shutdown_connections 2024-03-04T07:34:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:16.594+0000 7f93589eb640 1 -- 172.21.15.64:0/2262336479 wait complete. 2024-03-04T07:34:16.807 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:16.807 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:17.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:17.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.925+0000 7f3ae5593640 1 Processor -- start 2024-03-04T07:34:17.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.925+0000 7f3ae5593640 1 -- start start 2024-03-04T07:34:17.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ae5593640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ae0071c50 0x7f3ae00d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:17.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ae5593640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00b2c50 0x7f3ae00d2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:17.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ae5593640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3ae00ce970 con 0x7f3ae015e000 2024-03-04T07:34:17.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ae5593640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3ae00cece0 con 0x7f3ae0071c50 2024-03-04T07:34:17.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ae5593640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae00cef30 con 0x7f3ae00b2c50 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ade7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3ae0071c50 0x7f3ae00d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3adeffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3ae015e000 0x7f3ae0071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:46346/0 (socket says 172.21.15.64:46346) 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3adeffd640 1 -- 172.21.15.64:0/976213473 learned_addr learned my addr 172.21.15.64:0/976213473 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3adf7fe640 1 --2- 172.21.15.64:0/976213473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00b2c50 0x7f3ae00d2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3addffb640 1 -- 172.21.15.64:0/976213473 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 316813709 0 0) 0x7f3ae00ce970 con 0x7f3ae015e000 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3addffb640 1 -- 172.21.15.64:0/976213473 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3ac8003520 con 0x7f3ae015e000 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ade7fc640 1 -- 172.21.15.64:0/976213473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00b2c50 msgr2=0x7f3ae00d2b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ade7fc640 1 --2- 172.21.15.64:0/976213473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00b2c50 0x7f3ae00d2b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ade7fc640 1 -- 172.21.15.64:0/976213473 >> v1:172.21.15.64:6789/0 conn(0x7f3ae015e000 legacy=0x7f3ae0071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:17.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.926+0000 7f3ade7fc640 1 -- 172.21.15.64:0/976213473 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ae00cf210 con 0x7f3ae0071c50 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.927+0000 7f3ade7fc640 1 --2- 172.21.15.64:0/976213473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0071c50 0x7f3ae00d25d0 secure :-1 s=READY pgs=1596 cs=0 l=1 rev1=1 crypto rx=0x7f3ad40097a0 tx=0x7f3ad402b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f82e279b5b9fd628 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.927+0000 7f3addffb640 1 -- 172.21.15.64:0/976213473 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad40315a0 con 0x7f3ae0071c50 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.927+0000 7f3addffb640 1 -- 172.21.15.64:0/976213473 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ad4037070 con 0x7f3ae0071c50 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.927+0000 7f3addffb640 1 -- 172.21.15.64:0/976213473 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad4031960 con 0x7f3ae0071c50 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 -- 172.21.15.64:0/976213473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0071c50 msgr2=0x7f3ae00d25d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 --2- 172.21.15.64:0/976213473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0071c50 0x7f3ae00d25d0 secure :-1 s=READY pgs=1596 cs=0 l=1 rev1=1 crypto rx=0x7f3ad40097a0 tx=0x7f3ad402b840 comp rx=0 tx=0).stop 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 -- 172.21.15.64:0/976213473 shutdown_connections 2024-03-04T07:34:17.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 --2- 172.21.15.64:0/976213473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae00b2c50 0x7f3ae00d2b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:17.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 --2- 172.21.15.64:0/976213473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae0071c50 0x7f3ae00d25d0 unknown :-1 s=CLOSED pgs=1596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:17.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 -- 172.21.15.64:0/976213473 >> 172.21.15.64:0/976213473 conn(0x7f3ae00d3c80 msgr2=0x7f3ae00d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:17.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 -- 172.21.15.64:0/976213473 shutdown_connections 2024-03-04T07:34:17.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.928+0000 7f3ae5593640 1 -- 172.21.15.64:0/976213473 wait complete. 2024-03-04T07:34:17.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.929+0000 7f3ae5593640 1 Processor -- start 2024-03-04T07:34:17.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.929+0000 7f3ae5593640 1 -- start start 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ae5593640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae0071c50 0x7f3ae00cd7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ae5593640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae00b2c50 0x7f3ae00cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ae5593640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae015e000 0x7f3ae00c8db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ae5593640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae00b3320 con 0x7f3ae00b2c50 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ae5593640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae00b31a0 con 0x7f3ae0071c50 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ae5593640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3ae015e3e0 con 0x7f3ae015e000 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae00b2c50 0x7f3ae00cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae00b2c50 0x7f3ae00cad50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53914/0 (socket says 172.21.15.64:53914) 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 -- 172.21.15.64:0/3286683797 learned_addr learned my addr 172.21.15.64:0/3286683797 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3adeffd640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae0071c50 0x7f3ae00cd7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3adf7fe640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae015e000 0x7f3ae00c8db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 -- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae015e000 msgr2=0x7f3ae00c8db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae015e000 0x7f3ae00c8db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 -- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae0071c50 msgr2=0x7f3ae00cd7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae0071c50 0x7f3ae00cd7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 -- 172.21.15.64:0/3286683797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ae01f7e10 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.930+0000 7f3ade7fc640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae00b2c50 0x7f3ae00cad50 secure :-1 s=READY pgs=1597 cs=0 l=1 rev1=1 crypto rx=0x7f3ad40098c0 tx=0x7f3ad40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.931+0000 7f3ac77fe640 1 -- 172.21.15.64:0/3286683797 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad4032630 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.931+0000 7f3ac77fe640 1 -- 172.21.15.64:0/3286683797 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ad40327d0 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.931+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3ae00cddc0 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.931+0000 7f3ac77fe640 1 -- 172.21.15.64:0/3286683797 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ad4032bc0 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.931+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3ae01f7050 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.932+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3ae0067ff0 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.932+0000 7f3ac77fe640 1 -- 172.21.15.64:0/3286683797 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3ad404d3f0 con 0x7f3ae00b2c50 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.933+0000 7f3ac77fe640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3acc06b520 0x7f3acc06d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.933+0000 7f3adeffd640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3acc06b520 0x7f3acc06d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:17.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.933+0000 7f3ac77fe640 1 -- 172.21.15.64:0/3286683797 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f3ad402f020 con 0x7f3ae00b2c50 2024-03-04T07:34:17.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.933+0000 7f3adeffd640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3acc06b520 0x7f3acc06d9e0 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7f3ad0005880 tx=0x7f3ad0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:17.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:17.937+0000 7f3ac77fe640 1 -- 172.21.15.64:0/3286683797 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3ad4040070 con 0x7f3ae00b2c50 2024-03-04T07:34:18.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.157+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3ae0071750 con 0x7f3ae00b2c50 2024-03-04T07:34:18.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.157+0000 7f3ac77fe640 1 -- 172.21.15.64:0/3286683797 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f3ad407a2e0 con 0x7f3ae00b2c50 2024-03-04T07:34:18.157 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:18.157 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:18.158 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:18.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3acc06b520 msgr2=0x7f3acc06d9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:18.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3acc06b520 0x7f3acc06d9e0 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7f3ad0005880 tx=0x7f3ad0009040 comp rx=0 tx=0).stop 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae00b2c50 msgr2=0x7f3ae00cad50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae00b2c50 0x7f3ae00cad50 secure :-1 s=READY pgs=1597 cs=0 l=1 rev1=1 crypto rx=0x7f3ad40098c0 tx=0x7f3ad40046c0 comp rx=0 tx=0).stop 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 shutdown_connections 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3acc06b520 0x7f3acc06d9e0 unknown :-1 s=CLOSED pgs=716 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3ae015e000 0x7f3ae00c8db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3ae00b2c50 0x7f3ae00cad50 unknown :-1 s=CLOSED pgs=1597 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 --2- 172.21.15.64:0/3286683797 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3ae0071c50 0x7f3ae00cd7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 >> 172.21.15.64:0/3286683797 conn(0x7f3ae00d3c80 msgr2=0x7f3ae01603c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 shutdown_connections 2024-03-04T07:34:18.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:18.159+0000 7f3ae5593640 1 -- 172.21.15.64:0/3286683797 wait complete. 2024-03-04T07:34:18.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:34:18.203 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:34:18.372 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:18.372 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:18.533 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:19.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:19.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.486+0000 7f0d78be9640 1 Processor -- start 2024-03-04T07:34:19.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.486+0000 7f0d78be9640 1 -- start start 2024-03-04T07:34:19.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.487+0000 7f0d78be9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0d74147930 0x7f0d74072530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:19.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.487+0000 7f0d78be9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d740adfc0 0x7f0d74161f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:19.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.487+0000 7f0d78be9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0d74072a70 con 0x7f0d74161ac0 2024-03-04T07:34:19.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.487+0000 7f0d78be9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0d74162460 con 0x7f0d74147930 2024-03-04T07:34:19.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.487+0000 7f0d78be9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d740ae450 con 0x7f0d740adfc0 2024-03-04T07:34:19.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d72d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d740adfc0 0x7f0d74161f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:19.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d7259c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0d74147930 0x7f0d74072530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:19.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d71d9b640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f0d74161ac0 0x7f0d740ad8b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:48762/0 (socket says 172.21.15.64:48762) 2024-03-04T07:34:19.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d71d9b640 1 -- 172.21.15.64:0/880229232 learned_addr learned my addr 172.21.15.64:0/880229232 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:19.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d72d9d640 1 -- 172.21.15.64:0/880229232 >> v2:172.21.15.66:3300/0 conn(0x7f0d74147930 msgr2=0x7f0d74072530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:19.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d72d9d640 1 --2- 172.21.15.64:0/880229232 >> v2:172.21.15.66:3300/0 conn(0x7f0d74147930 0x7f0d74072530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d72d9d640 1 -- 172.21.15.64:0/880229232 >> v1:172.21.15.66:6789/0 conn(0x7f0d74161ac0 legacy=0x7f0d740ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.488+0000 7f0d72d9d640 1 -- 172.21.15.64:0/880229232 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d7406dbe0 con 0x7f0d740adfc0 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.489+0000 7f0d72d9d640 1 --2- 172.21.15.64:0/880229232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d740adfc0 0x7f0d74161f20 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f0d6c009d90 tx=0x7f0d6c02bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a0525131775a3ccd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.489+0000 7f0d7159a640 1 -- 172.21.15.64:0/880229232 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d6c039070 con 0x7f0d740adfc0 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.489+0000 7f0d7159a640 1 -- 172.21.15.64:0/880229232 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d6c02cc50 con 0x7f0d740adfc0 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.489+0000 7f0d7159a640 1 -- 172.21.15.64:0/880229232 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d6c034440 con 0x7f0d740adfc0 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.489+0000 7f0d78be9640 1 -- 172.21.15.64:0/880229232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d740adfc0 msgr2=0x7f0d74161f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.489+0000 7f0d78be9640 1 --2- 172.21.15.64:0/880229232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d740adfc0 0x7f0d74161f20 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f0d6c009d90 tx=0x7f0d6c02bea0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.490+0000 7f0d78be9640 1 -- 172.21.15.64:0/880229232 shutdown_connections 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.490+0000 7f0d78be9640 1 --2- 172.21.15.64:0/880229232 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d740adfc0 0x7f0d74161f20 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.490+0000 7f0d78be9640 1 --2- 172.21.15.64:0/880229232 >> v2:172.21.15.66:3300/0 conn(0x7f0d74147930 0x7f0d74072530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.490+0000 7f0d78be9640 1 -- 172.21.15.64:0/880229232 >> 172.21.15.64:0/880229232 conn(0x7f0d7406cd20 msgr2=0x7f0d7406d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.490+0000 7f0d78be9640 1 -- 172.21.15.64:0/880229232 shutdown_connections 2024-03-04T07:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.490+0000 7f0d78be9640 1 -- 172.21.15.64:0/880229232 wait complete. 2024-03-04T07:34:19.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.491+0000 7f0d78be9640 1 Processor -- start 2024-03-04T07:34:19.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.491+0000 7f0d78be9640 1 -- start start 2024-03-04T07:34:19.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d78be9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d740adfc0 0x7f0d74143c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:19.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d78be9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d74147930 0x7f0d740cd270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:19.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d78be9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d74161ac0 0x7f0d7413d510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:19.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d78be9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d74072010 con 0x7f0d74161ac0 2024-03-04T07:34:19.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d78be9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d74071d30 con 0x7f0d740adfc0 2024-03-04T07:34:19.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d78be9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0d74147d10 con 0x7f0d74147930 2024-03-04T07:34:19.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d74161ac0 0x7f0d7413d510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d71d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d74147930 0x7f0d740cd270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d7259c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d740adfc0 0x7f0d74143c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d74161ac0 0x7f0d7413d510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53916/0 (socket says 172.21.15.64:53916) 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 -- 172.21.15.64:0/1998502075 learned_addr learned my addr 172.21.15.64:0/1998502075 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 -- 172.21.15.64:0/1998502075 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d74147930 msgr2=0x7f0d740cd270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d74147930 0x7f0d740cd270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 -- 172.21.15.64:0/1998502075 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d740adfc0 msgr2=0x7f0d74143c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d740adfc0 0x7f0d74143c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 -- 172.21.15.64:0/1998502075 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d7413e550 con 0x7f0d74161ac0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.492+0000 7f0d72d9d640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d74161ac0 0x7f0d7413d510 secure :-1 s=READY pgs=1598 cs=0 l=1 rev1=1 crypto rx=0x7f0d6c009d90 tx=0x7f0d6c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.493+0000 7f0d62ffd640 1 -- 172.21.15.64:0/1998502075 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d6c039050 con 0x7f0d74161ac0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.493+0000 7f0d62ffd640 1 -- 172.21.15.64:0/1998502075 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0d6c045070 con 0x7f0d74161ac0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.493+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d7413e830 con 0x7f0d74161ac0 2024-03-04T07:34:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.493+0000 7f0d62ffd640 1 -- 172.21.15.64:0/1998502075 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d6c009450 con 0x7f0d74161ac0 2024-03-04T07:34:19.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.493+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d7413edf0 con 0x7f0d74161ac0 2024-03-04T07:34:19.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.494+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d74067ff0 con 0x7f0d74161ac0 2024-03-04T07:34:19.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.494+0000 7f0d62ffd640 1 -- 172.21.15.64:0/1998502075 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0d6c03e610 con 0x7f0d74161ac0 2024-03-04T07:34:19.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.495+0000 7f0d62ffd640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d5406b4d0 0x7f0d5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:19.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.495+0000 7f0d62ffd640 1 -- 172.21.15.64:0/1998502075 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f0d6c0aca80 con 0x7f0d74161ac0 2024-03-04T07:34:19.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.495+0000 7f0d7259c640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d5406b4d0 0x7f0d5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:19.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.495+0000 7f0d7259c640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d5406b4d0 0x7f0d5406d990 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f0d64009b40 tx=0x7f0d64009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:19.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.499+0000 7f0d62ffd640 1 -- 172.21.15.64:0/1998502075 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0d6c07b340 con 0x7f0d74161ac0 2024-03-04T07:34:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.722+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0d740b31c0 con 0x7f0d74161ac0 2024-03-04T07:34:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.722+0000 7f0d62ffd640 1 -- 172.21.15.64:0/1998502075 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f0d6c07f3e0 con 0x7f0d74161ac0 2024-03-04T07:34:19.722 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:19.722 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:19.723 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d5406b4d0 msgr2=0x7f0d5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d5406b4d0 0x7f0d5406d990 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f0d64009b40 tx=0x7f0d64009190 comp rx=0 tx=0).stop 2024-03-04T07:34:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d74161ac0 msgr2=0x7f0d7413d510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d74161ac0 0x7f0d7413d510 secure :-1 s=READY pgs=1598 cs=0 l=1 rev1=1 crypto rx=0x7f0d6c009d90 tx=0x7f0d6c004a40 comp rx=0 tx=0).stop 2024-03-04T07:34:19.724 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 shutdown_connections 2024-03-04T07:34:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0d5406b4d0 0x7f0d5406d990 unknown :-1 s=CLOSED pgs=717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0d74161ac0 0x7f0d7413d510 unknown :-1 s=CLOSED pgs=1598 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0d74147930 0x7f0d740cd270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 --2- 172.21.15.64:0/1998502075 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0d740adfc0 0x7f0d74143c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 >> 172.21.15.64:0/1998502075 conn(0x7f0d7406cd20 msgr2=0x7f0d74144f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 shutdown_connections 2024-03-04T07:34:19.725 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:19.724+0000 7f0d78be9640 1 -- 172.21.15.64:0/1998502075 wait complete. 2024-03-04T07:34:19.936 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:19.937 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:20.535 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:20.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:21.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.051+0000 7fa49dfa0640 1 Processor -- start 2024-03-04T07:34:21.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.051+0000 7fa49dfa0640 1 -- start start 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49dfa0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa49815dd20 0x7fa4980d1bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49dfa0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa4980d1140 0x7fa4980b6c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49dfa0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa49815e100 con 0x7fa4980d2100 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49dfa0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4980d3190 con 0x7fa49815dd20 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49dfa0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa4980d2580 con 0x7fa4980d1140 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49cf9e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa49815dd20 0x7fa4980d1bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49cf9e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa49815dd20 0x7fa4980d1bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53930/0 (socket says 172.21.15.64:53930) 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49d79f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa4980d1140 0x7fa4980b6c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.052+0000 7fa49cf9e640 1 -- 172.21.15.64:0/482213099 learned_addr learned my addr 172.21.15.64:0/482213099 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.053+0000 7fa49cf9e640 1 -- 172.21.15.64:0/482213099 >> v2:172.21.15.66:3300/0 conn(0x7fa4980d1140 msgr2=0x7fa4980b6c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.053+0000 7fa49cf9e640 1 --2- 172.21.15.64:0/482213099 >> v2:172.21.15.66:3300/0 conn(0x7fa4980d1140 0x7fa4980b6c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.053+0000 7fa49cf9e640 1 -- 172.21.15.64:0/482213099 >> v1:172.21.15.66:6789/0 conn(0x7fa4980d2100 legacy=0x7fa4980d0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:21.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.053+0000 7fa49cf9e640 1 -- 172.21.15.64:0/482213099 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4980d2860 con 0x7fa49815dd20 2024-03-04T07:34:21.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.053+0000 7fa49d79f640 1 --2- 172.21.15.64:0/482213099 >> v2:172.21.15.66:3300/0 conn(0x7fa4980d1140 0x7fa4980b6c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:21.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.053+0000 7fa49cf9e640 1 --2- 172.21.15.64:0/482213099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49815dd20 0x7fa4980d1bc0 secure :-1 s=READY pgs=1599 cs=0 l=1 rev1=1 crypto rx=0x7fa4980cf3a0 tx=0x7fa48802b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b54f68220b4046ff server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:21.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.053+0000 7fa4977fe640 1 -- 172.21.15.64:0/482213099 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa488035020 con 0x7fa49815dd20 2024-03-04T07:34:21.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa4977fe640 1 -- 172.21.15.64:0/482213099 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa488031870 con 0x7fa49815dd20 2024-03-04T07:34:21.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa4977fe640 1 -- 172.21.15.64:0/482213099 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa488031c40 con 0x7fa49815dd20 2024-03-04T07:34:21.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 -- 172.21.15.64:0/482213099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49815dd20 msgr2=0x7fa4980d1bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:21.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/482213099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49815dd20 0x7fa4980d1bc0 secure :-1 s=READY pgs=1599 cs=0 l=1 rev1=1 crypto rx=0x7fa4980cf3a0 tx=0x7fa48802b910 comp rx=0 tx=0).stop 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 -- 172.21.15.64:0/482213099 shutdown_connections 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/482213099 >> v2:172.21.15.66:3300/0 conn(0x7fa4980d1140 0x7fa4980b6c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/482213099 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49815dd20 0x7fa4980d1bc0 unknown :-1 s=CLOSED pgs=1599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 -- 172.21.15.64:0/482213099 >> 172.21.15.64:0/482213099 conn(0x7fa498071960 msgr2=0x7fa498071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 -- 172.21.15.64:0/482213099 shutdown_connections 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.054+0000 7fa49dfa0640 1 -- 172.21.15.64:0/482213099 wait complete. 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.055+0000 7fa49dfa0640 1 Processor -- start 2024-03-04T07:34:21.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.055+0000 7fa49dfa0640 1 -- start start 2024-03-04T07:34:21.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49dfa0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4980d1140 0x7fa4981f7170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:21.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49dfa0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4980d2100 0x7fa4980cb630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:21.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49dfa0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49815dd20 0x7fa4980cbb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49dfa0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4980b9580 con 0x7fa4980d2100 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49dfa0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4980d2e20 con 0x7fa49815dd20 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49dfa0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4980b9700 con 0x7fa4980d1140 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa497fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4980d2100 0x7fa4980cb630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa497fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4980d2100 0x7fa4980cb630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53942/0 (socket says 172.21.15.64:53942) 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa497fff640 1 -- 172.21.15.64:0/2654531508 learned_addr learned my addr 172.21.15.64:0/2654531508 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49d79f640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49815dd20 0x7fa4980cbb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.056+0000 7fa49cf9e640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4980d1140 0x7fa4981f7170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa497fff640 1 -- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4980d1140 msgr2=0x7fa4981f7170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa497fff640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4980d1140 0x7fa4981f7170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa497fff640 1 -- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49815dd20 msgr2=0x7fa4980cbb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa497fff640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49815dd20 0x7fa4980cbb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa497fff640 1 -- 172.21.15.64:0/2654531508 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4980cc390 con 0x7fa4980d2100 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa497fff640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4980d2100 0x7fa4980cb630 secure :-1 s=READY pgs=1600 cs=0 l=1 rev1=1 crypto rx=0x7fa48c00c3a0 tx=0x7fa48c00c7a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa4957fa640 1 -- 172.21.15.64:0/2654531508 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa48c0052d0 con 0x7fa4980d2100 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa4957fa640 1 -- 172.21.15.64:0/2654531508 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa48c005490 con 0x7fa4980d2100 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa4957fa640 1 -- 172.21.15.64:0/2654531508 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa48c007260 con 0x7fa4980d2100 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4980cc670 con 0x7fa4980d2100 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.057+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4980ccc30 con 0x7fa4980d2100 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.058+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa49804f330 con 0x7fa4980d2100 2024-03-04T07:34:21.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.059+0000 7fa4957fa640 1 -- 172.21.15.64:0/2654531508 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa48c007400 con 0x7fa4980d2100 2024-03-04T07:34:21.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.059+0000 7fa4957fa640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa48406b4d0 0x7fa48406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:21.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.059+0000 7fa4957fa640 1 -- 172.21.15.64:0/2654531508 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fa48c089500 con 0x7fa4980d2100 2024-03-04T07:34:21.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.059+0000 7fa49cf9e640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa48406b4d0 0x7fa48406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:21.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.060+0000 7fa49cf9e640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa48406b4d0 0x7fa48406d990 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7fa488036000 tx=0x7fa48803c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:21.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.063+0000 7fa4957fa640 1 -- 172.21.15.64:0/2654531508 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa48c057dc0 con 0x7fa4980d2100 2024-03-04T07:34:21.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.282+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa4980b9000 con 0x7fa4980d2100 2024-03-04T07:34:21.282 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.282+0000 7fa4957fa640 1 -- 172.21.15.64:0/2654531508 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7fa48c05be60 con 0x7fa4980d2100 2024-03-04T07:34:21.282 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:21.282 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:21.283 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:21.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa48406b4d0 msgr2=0x7fa48406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:21.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa48406b4d0 0x7fa48406d990 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7fa488036000 tx=0x7fa48803c040 comp rx=0 tx=0).stop 2024-03-04T07:34:21.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4980d2100 msgr2=0x7fa4980cb630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:21.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4980d2100 0x7fa4980cb630 secure :-1 s=READY pgs=1600 cs=0 l=1 rev1=1 crypto rx=0x7fa48c00c3a0 tx=0x7fa48c00c7a0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 shutdown_connections 2024-03-04T07:34:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa48406b4d0 0x7fa48406d990 unknown :-1 s=CLOSED pgs=718 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49815dd20 0x7fa4980cbb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa4980d2100 0x7fa4980cb630 unknown :-1 s=CLOSED pgs=1600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 --2- 172.21.15.64:0/2654531508 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4980d1140 0x7fa4981f7170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 >> 172.21.15.64:0/2654531508 conn(0x7fa498071960 msgr2=0x7fa4980d36b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 shutdown_connections 2024-03-04T07:34:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:21.284+0000 7fa49dfa0640 1 -- 172.21.15.64:0/2654531508 wait complete. 2024-03-04T07:34:21.496 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:21.496 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:22.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:22.536 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:22.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 Processor -- start 2024-03-04T07:34:22.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 -- start start 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf840ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbf8406fe20 0x7fbf840ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fbf840d0880 0x7fbf840ced90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbf840d0dd0 con 0x7fbf8406fe20 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbf840d10b0 con 0x7fbf840d0880 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.612+0000 7fbf8942a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbf840cc0e0 con 0x7fbf8415e060 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbf8406fe20 0x7fbf840ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbf8406fe20 0x7fbf840ce850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53954/0 (socket says 172.21.15.64:53954) 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/4027290297 learned_addr learned my addr 172.21.15.64:0/4027290297 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf82ffd640 1 --2- 172.21.15.64:0/4027290297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf840ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:22.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf837fe640 1 --2- 172.21.15.64:0/4027290297 >> v2:172.21.15.66:3300/0 conn(0x7fbf840d0880 0x7fbf840ced90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:22.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/4027290297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 msgr2=0x7fbf840ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:22.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 --2- 172.21.15.64:0/4027290297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf840ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/4027290297 >> v2:172.21.15.66:3300/0 conn(0x7fbf840d0880 msgr2=0x7fbf840ced90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:22.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 --2- 172.21.15.64:0/4027290297 >> v2:172.21.15.66:3300/0 conn(0x7fbf840d0880 0x7fbf840ced90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.613+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/4027290297 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf840cc3c0 con 0x7fbf8406fe20 2024-03-04T07:34:22.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf7a7fc640 1 --2- 172.21.15.64:0/4027290297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf8406fe20 0x7fbf840ce850 secure :-1 s=READY pgs=1601 cs=0 l=1 rev1=1 crypto rx=0x7fbf84072a10 tx=0x7fbf6c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7e46947e1e6571d7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:22.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf827fc640 1 -- 172.21.15.64:0/4027290297 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf6c035020 con 0x7fbf8406fe20 2024-03-04T07:34:22.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf827fc640 1 -- 172.21.15.64:0/4027290297 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf6c031870 con 0x7fbf8406fe20 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf827fc640 1 -- 172.21.15.64:0/4027290297 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf6c031c40 con 0x7fbf8406fe20 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 -- 172.21.15.64:0/4027290297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf8406fe20 msgr2=0x7fbf840ce850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 --2- 172.21.15.64:0/4027290297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf8406fe20 0x7fbf840ce850 secure :-1 s=READY pgs=1601 cs=0 l=1 rev1=1 crypto rx=0x7fbf84072a10 tx=0x7fbf6c02b910 comp rx=0 tx=0).stop 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 -- 172.21.15.64:0/4027290297 shutdown_connections 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 --2- 172.21.15.64:0/4027290297 >> v2:172.21.15.66:3300/0 conn(0x7fbf840d0880 0x7fbf840ced90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 --2- 172.21.15.64:0/4027290297 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf8406fe20 0x7fbf840ce850 unknown :-1 s=CLOSED pgs=1601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 --2- 172.21.15.64:0/4027290297 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf840ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 -- 172.21.15.64:0/4027290297 >> 172.21.15.64:0/4027290297 conn(0x7fbf840d1a90 msgr2=0x7fbf840d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 -- 172.21.15.64:0/4027290297 shutdown_connections 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.614+0000 7fbf8942a640 1 -- 172.21.15.64:0/4027290297 wait complete. 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.616+0000 7fbf8942a640 1 Processor -- start 2024-03-04T07:34:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.616+0000 7fbf8942a640 1 -- start start 2024-03-04T07:34:22.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.616+0000 7fbf8942a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf8406fe20 0x7fbf841f7a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:22.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.616+0000 7fbf8942a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf840d0880 0x7fbf8414fb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:22.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.616+0000 7fbf8942a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf8414c5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:22.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf8942a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf8406ed30 con 0x7fbf840d0880 2024-03-04T07:34:22.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf8942a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf8415e440 con 0x7fbf8406fe20 2024-03-04T07:34:22.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf8942a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbf8406eeb0 con 0x7fbf8415e060 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf840d0880 0x7fbf8414fb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf840d0880 0x7fbf8414fb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:53964/0 (socket says 172.21.15.64:53964) 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/2466124708 learned_addr learned my addr 172.21.15.64:0/2466124708 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf82ffd640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf8406fe20 0x7fbf841f7a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf837fe640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf8414c5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 msgr2=0x7fbf8414c5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf8414c5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf8406fe20 msgr2=0x7fbf841f7a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf8406fe20 0x7fbf841f7a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 -- 172.21.15.64:0/2466124708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf84150090 con 0x7fbf840d0880 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7a7fc640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf840d0880 0x7fbf8414fb50 secure :-1 s=READY pgs=1602 cs=0 l=1 rev1=1 crypto rx=0x7fbf6c000c00 tx=0x7fbf6c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7bfff640 1 -- 172.21.15.64:0/2466124708 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf6c031e10 con 0x7fbf840d0880 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7bfff640 1 -- 172.21.15.64:0/2466124708 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbf6c03ad60 con 0x7fbf840d0880 2024-03-04T07:34:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbf840b49f0 con 0x7fbf840d0880 2024-03-04T07:34:22.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf7bfff640 1 -- 172.21.15.64:0/2466124708 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf6c044630 con 0x7fbf840d0880 2024-03-04T07:34:22.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.617+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbf841f7250 con 0x7fbf840d0880 2024-03-04T07:34:22.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.619+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbf841f7040 con 0x7fbf840d0880 2024-03-04T07:34:22.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.619+0000 7fbf7bfff640 1 -- 172.21.15.64:0/2466124708 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbf6c038020 con 0x7fbf840d0880 2024-03-04T07:34:22.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.619+0000 7fbf7bfff640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf6406b4d0 0x7fbf6406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:22.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.619+0000 7fbf7bfff640 1 -- 172.21.15.64:0/2466124708 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fbf6c0ac1e0 con 0x7fbf840d0880 2024-03-04T07:34:22.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.619+0000 7fbf82ffd640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf6406b4d0 0x7fbf6406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:22.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.620+0000 7fbf82ffd640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf6406b4d0 0x7fbf6406d990 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7fbf74009a20 tx=0x7fbf74009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:22.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.624+0000 7fbf7bfff640 1 -- 172.21.15.64:0/2466124708 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbf84150090 con 0x7fbf840d0880 2024-03-04T07:34:22.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.846+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbf84067ff0 con 0x7fbf840d0880 2024-03-04T07:34:22.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.846+0000 7fbf7bfff640 1 -- 172.21.15.64:0/2466124708 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7fbf6c03f070 con 0x7fbf840d0880 2024-03-04T07:34:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:22.847 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:22.847 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf6406b4d0 msgr2=0x7fbf6406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf6406b4d0 0x7fbf6406d990 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7fbf74009a20 tx=0x7fbf74009190 comp rx=0 tx=0).stop 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf840d0880 msgr2=0x7fbf8414fb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf840d0880 0x7fbf8414fb50 secure :-1 s=READY pgs=1602 cs=0 l=1 rev1=1 crypto rx=0x7fbf6c000c00 tx=0x7fbf6c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 shutdown_connections 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbf6406b4d0 0x7fbf6406d990 unknown :-1 s=CLOSED pgs=719 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbf8415e060 0x7fbf8414c5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbf840d0880 0x7fbf8414fb50 unknown :-1 s=CLOSED pgs=1602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.848+0000 7fbf8942a640 1 --2- 172.21.15.64:0/2466124708 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbf8406fe20 0x7fbf841f7a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.849+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 >> 172.21.15.64:0/2466124708 conn(0x7fbf840d1a90 msgr2=0x7fbf840b10e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.849+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 shutdown_connections 2024-03-04T07:34:22.849 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:22.849+0000 7fbf8942a640 1 -- 172.21.15.64:0/2466124708 wait complete. 2024-03-04T07:34:23.061 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:23.061 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:24.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:24.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.176+0000 7f373575e640 1 Processor -- start 2024-03-04T07:34:24.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.176+0000 7f373575e640 1 -- start start 2024-03-04T07:34:24.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f373575e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f373015e030 0x7f373006ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:24.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f373575e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373006ef70 0x7f37300b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:24.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f373575e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f373015e410 con 0x7f37300b2290 2024-03-04T07:34:24.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f373575e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3730162700 con 0x7f373015e030 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f373575e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3730161af0 con 0x7f373006ef70 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f372effd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f373015e030 0x7f373006ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f372effd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f373015e030 0x7f373006ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33250/0 (socket says 172.21.15.64:33250) 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f372effd640 1 -- 172.21.15.64:0/3340730431 learned_addr learned my addr 172.21.15.64:0/3340730431 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f372e7fc640 1 --2- 172.21.15.64:0/3340730431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373006ef70 0x7f37300b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f372effd640 1 -- 172.21.15.64:0/3340730431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373006ef70 msgr2=0x7f37300b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f372effd640 1 --2- 172.21.15.64:0/3340730431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373006ef70 0x7f37300b1d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.177+0000 7f372effd640 1 -- 172.21.15.64:0/3340730431 >> v1:172.21.15.66:6789/0 conn(0x7f37300b2290 legacy=0x7f37300d0040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.178+0000 7f372effd640 1 -- 172.21.15.64:0/3340730431 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3730161dd0 con 0x7f373015e030 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.178+0000 7f372dffb640 1 -- 172.21.15.64:0/3340730431 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3916019231 0 0) 0x7f3728001110 con 0x7f37300b2290 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.178+0000 7f372effd640 1 --2- 172.21.15.64:0/3340730431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373015e030 0x7f373006ea30 secure :-1 s=READY pgs=1603 cs=0 l=1 rev1=1 crypto rx=0x7f3720009e50 tx=0x7f372002bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=13b47b5f0d4d9c53 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.178+0000 7f372dffb640 1 -- 172.21.15.64:0/3340730431 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f372002cb00 con 0x7f373015e030 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f372dffb640 1 -- 172.21.15.64:0/3340730431 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f372002cca0 con 0x7f373015e030 2024-03-04T07:34:24.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f372dffb640 1 -- 172.21.15.64:0/3340730431 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3720034610 con 0x7f373015e030 2024-03-04T07:34:24.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 -- 172.21.15.64:0/3340730431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373015e030 msgr2=0x7f373006ea30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:24.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 --2- 172.21.15.64:0/3340730431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373015e030 0x7f373006ea30 secure :-1 s=READY pgs=1603 cs=0 l=1 rev1=1 crypto rx=0x7f3720009e50 tx=0x7f372002bf60 comp rx=0 tx=0).stop 2024-03-04T07:34:24.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 -- 172.21.15.64:0/3340730431 shutdown_connections 2024-03-04T07:34:24.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 --2- 172.21.15.64:0/3340730431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373006ef70 0x7f37300b1d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 --2- 172.21.15.64:0/3340730431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373015e030 0x7f373006ea30 unknown :-1 s=CLOSED pgs=1603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 -- 172.21.15.64:0/3340730431 >> 172.21.15.64:0/3340730431 conn(0x7f37300b55e0 msgr2=0x7f37300b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 -- 172.21.15.64:0/3340730431 shutdown_connections 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.179+0000 7f373575e640 1 -- 172.21.15.64:0/3340730431 wait complete. 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 Processor -- start 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 -- start start 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373006ef70 0x7f37300cfe90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37300b2290 0x7f373009d730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373015e030 0x7f37300aed90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f373006e220 con 0x7f373006ef70 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3730162390 con 0x7f37300b2290 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f373575e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f373006e3a0 con 0x7f373015e030 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f372effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373006ef70 0x7f37300cfe90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f372effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373006ef70 0x7f37300cfe90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33266/0 (socket says 172.21.15.64:33266) 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f372effd640 1 -- 172.21.15.64:0/2943875120 learned_addr learned my addr 172.21.15.64:0/2943875120 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:24.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f372e7fc640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37300b2290 0x7f373009d730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.181+0000 7f372f7fe640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373015e030 0x7f37300aed90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f372effd640 1 -- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373015e030 msgr2=0x7f37300aed90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f372effd640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373015e030 0x7f37300aed90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f372effd640 1 -- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37300b2290 msgr2=0x7f373009d730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f372effd640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37300b2290 0x7f373009d730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f372effd640 1 -- 172.21.15.64:0/2943875120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37300ad240 con 0x7f373006ef70 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f372effd640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373006ef70 0x7f37300cfe90 secure :-1 s=READY pgs=1604 cs=0 l=1 rev1=1 crypto rx=0x7f3720036040 tx=0x7f3720004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f37277fe640 1 -- 172.21.15.64:0/2943875120 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3720043020 con 0x7f373006ef70 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f37277fe640 1 -- 172.21.15.64:0/2943875120 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f372002ce50 con 0x7f373006ef70 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f37300ad520 con 0x7f373006ef70 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f37277fe640 1 -- 172.21.15.64:0/2943875120 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3720009510 con 0x7f373006ef70 2024-03-04T07:34:24.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.182+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f37300adae0 con 0x7f373006ef70 2024-03-04T07:34:24.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.183+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3730067ff0 con 0x7f373006ef70 2024-03-04T07:34:24.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.184+0000 7f37277fe640 1 -- 172.21.15.64:0/2943875120 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3720046020 con 0x7f373006ef70 2024-03-04T07:34:24.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.184+0000 7f37277fe640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f371006b500 0x7f371006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:24.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.184+0000 7f372e7fc640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f371006b500 0x7f371006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:24.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.184+0000 7f37277fe640 1 -- 172.21.15.64:0/2943875120 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f37200ac3a0 con 0x7f373006ef70 2024-03-04T07:34:24.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.185+0000 7f372e7fc640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f371006b500 0x7f371006d9c0 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7f373006ee00 tx=0x7f3718009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:24.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.189+0000 7f37277fe640 1 -- 172.21.15.64:0/2943875120 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f372004d070 con 0x7f373006ef70 2024-03-04T07:34:24.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.406+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f37300b2910 con 0x7f373006ef70 2024-03-04T07:34:24.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.406+0000 7f37277fe640 1 -- 172.21.15.64:0/2943875120 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f372007abe0 con 0x7f373006ef70 2024-03-04T07:34:24.406 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:24.406 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:24.407 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:24.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f371006b500 msgr2=0x7f371006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:24.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f371006b500 0x7f371006d9c0 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7f373006ee00 tx=0x7f3718009190 comp rx=0 tx=0).stop 2024-03-04T07:34:24.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373006ef70 msgr2=0x7f37300cfe90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:24.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373006ef70 0x7f37300cfe90 secure :-1 s=READY pgs=1604 cs=0 l=1 rev1=1 crypto rx=0x7f3720036040 tx=0x7f3720004a40 comp rx=0 tx=0).stop 2024-03-04T07:34:24.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 shutdown_connections 2024-03-04T07:34:24.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f371006b500 0x7f371006d9c0 unknown :-1 s=CLOSED pgs=720 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f373015e030 0x7f37300aed90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f37300b2290 0x7f373009d730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 --2- 172.21.15.64:0/2943875120 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f373006ef70 0x7f37300cfe90 unknown :-1 s=CLOSED pgs=1604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:24.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 >> 172.21.15.64:0/2943875120 conn(0x7f37300b55e0 msgr2=0x7f373015d2a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:24.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 shutdown_connections 2024-03-04T07:34:24.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:24.408+0000 7f373575e640 1 -- 172.21.15.64:0/2943875120 wait complete. 2024-03-04T07:34:24.538 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:24.620 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:24.620 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:25.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:25.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.736+0000 7f75ee719640 1 Processor -- start 2024-03-04T07:34:25.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.736+0000 7f75ee719640 1 -- start start 2024-03-04T07:34:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.736+0000 7f75ee719640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f75e80d0880 0x7f75e8168c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.736+0000 7f75ee719640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75e80cf130 con 0x7f75e815e060 2024-03-04T07:34:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.736+0000 7f75ee719640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f75e80d0dd0 con 0x7f75e806fe20 2024-03-04T07:34:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.736+0000 7f75ee719640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f75e80d10b0 con 0x7f75e80d0880 2024-03-04T07:34:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.737+0000 7f75eccb6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f75e80d0880 0x7f75e8168c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.737+0000 7f75e7fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f75e815e060 0x7f75e80ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41806/0 (socket says 172.21.15.64:41806) 2024-03-04T07:34:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.737+0000 7f75e7fff640 1 -- 172.21.15.64:0/108866096 learned_addr learned my addr 172.21.15.64:0/108866096 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:25.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.737+0000 7f75eccb6640 1 -- 172.21.15.64:0/108866096 >> v1:172.21.15.66:6789/0 conn(0x7f75e806fe20 legacy=0x7f75e80cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:25.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.737+0000 7f75eccb6640 1 -- 172.21.15.64:0/108866096 >> v1:172.21.15.64:6789/0 conn(0x7f75e815e060 legacy=0x7f75e80ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:25.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.737+0000 7f75e6ffd640 1 -- 172.21.15.64:0/108866096 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1598838645 0 0) 0x7f75e80d0dd0 con 0x7f75e806fe20 2024-03-04T07:34:25.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.737+0000 7f75eccb6640 1 -- 172.21.15.64:0/108866096 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75e80cc0e0 con 0x7f75e80d0880 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.738+0000 7f75eccb6640 1 --2- 172.21.15.64:0/108866096 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e8168c20 secure :-1 s=READY pgs=1606 cs=0 l=1 rev1=1 crypto rx=0x7f75e0009d90 tx=0x7f75e002bd80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=994ea96e9172ee3d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.738+0000 7f75e6ffd640 1 -- 172.21.15.64:0/108866096 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75e0039070 con 0x7f75e80d0880 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.738+0000 7f75e6ffd640 1 -- 172.21.15.64:0/108866096 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75e002cab0 con 0x7f75e80d0880 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.738+0000 7f75e6ffd640 1 -- 172.21.15.64:0/108866096 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75e0034440 con 0x7f75e80d0880 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.738+0000 7f75ee719640 1 -- 172.21.15.64:0/108866096 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 msgr2=0x7f75e8168c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.738+0000 7f75ee719640 1 --2- 172.21.15.64:0/108866096 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e8168c20 secure :-1 s=READY pgs=1606 cs=0 l=1 rev1=1 crypto rx=0x7f75e0009d90 tx=0x7f75e002bd80 comp rx=0 tx=0).stop 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.739+0000 7f75ee719640 1 -- 172.21.15.64:0/108866096 shutdown_connections 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.739+0000 7f75ee719640 1 --2- 172.21.15.64:0/108866096 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e8168c20 unknown :-1 s=CLOSED pgs=1606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.739+0000 7f75ee719640 1 -- 172.21.15.64:0/108866096 >> 172.21.15.64:0/108866096 conn(0x7f75e80d1a90 msgr2=0x7f75e80d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.739+0000 7f75ee719640 1 -- 172.21.15.64:0/108866096 shutdown_connections 2024-03-04T07:34:25.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.739+0000 7f75ee719640 1 -- 172.21.15.64:0/108866096 wait complete. 2024-03-04T07:34:25.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.740+0000 7f75ee719640 1 Processor -- start 2024-03-04T07:34:25.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.740+0000 7f75ee719640 1 -- start start 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75ee719640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75e806fe20 0x7f75e81f49e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75ee719640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e80ce470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75ee719640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75e815e060 0x7f75e8160a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75ee719640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75e815e440 con 0x7f75e80d0880 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75ee719640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75e80cc6a0 con 0x7f75e806fe20 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75ee719640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f75e80cdbd0 con 0x7f75e815e060 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e80ce470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:25.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e80ce470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33282/0 (socket says 172.21.15.64:33282) 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 -- 172.21.15.64:0/1549036283 learned_addr learned my addr 172.21.15.64:0/1549036283 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e7fff640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75e806fe20 0x7f75e81f49e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75eccb6640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75e815e060 0x7f75e8160a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 -- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75e815e060 msgr2=0x7f75e8160a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75e815e060 0x7f75e8160a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 -- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75e806fe20 msgr2=0x7f75e81f49e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75e806fe20 0x7f75e81f49e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 -- 172.21.15.64:0/1549036283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75e80c61a0 con 0x7f75e80d0880 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.741+0000 7f75e77fe640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e80ce470 secure :-1 s=READY pgs=1607 cs=0 l=1 rev1=1 crypto rx=0x7f75dc00c520 tx=0x7f75dc00c9e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.742+0000 7f75e4ff9640 1 -- 172.21.15.64:0/1549036283 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75dc017070 con 0x7f75e80d0880 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.742+0000 7f75e4ff9640 1 -- 172.21.15.64:0/1549036283 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75dc0035f0 con 0x7f75e80d0880 2024-03-04T07:34:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.742+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f75e80c6480 con 0x7f75e80d0880 2024-03-04T07:34:25.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.742+0000 7f75e4ff9640 1 -- 172.21.15.64:0/1549036283 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f75dc003a20 con 0x7f75e80d0880 2024-03-04T07:34:25.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.742+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f75e80c6a40 con 0x7f75e80d0880 2024-03-04T07:34:25.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.743+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f75e80c6c40 con 0x7f75e80d0880 2024-03-04T07:34:25.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.743+0000 7f75e4ff9640 1 -- 172.21.15.64:0/1549036283 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f75dc01a070 con 0x7f75e80d0880 2024-03-04T07:34:25.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.743+0000 7f75e4ff9640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f75d406b4d0 0x7f75d406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:25.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.744+0000 7f75e4ff9640 1 -- 172.21.15.64:0/1549036283 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f75dc0894a0 con 0x7f75e80d0880 2024-03-04T07:34:25.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.744+0000 7f75e7fff640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f75d406b4d0 0x7f75d406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:25.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.744+0000 7f75e7fff640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f75d406b4d0 0x7f75d406d990 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7f75d8000f60 tx=0x7f75d8007e20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:25.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.748+0000 7f75e4ff9640 1 -- 172.21.15.64:0/1549036283 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f75e80c6c40 con 0x7f75e80d0880 2024-03-04T07:34:25.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.968+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f75e8067ff0 con 0x7f75e80d0880 2024-03-04T07:34:25.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.969+0000 7f75e4ff9640 1 -- 172.21.15.64:0/1549036283 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f75dc057d60 con 0x7f75e80d0880 2024-03-04T07:34:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:25.969 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:25.969 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f75d406b4d0 msgr2=0x7f75d406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f75d406b4d0 0x7f75d406d990 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7f75d8000f60 tx=0x7f75d8007e20 comp rx=0 tx=0).stop 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 msgr2=0x7f75e80ce470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e80ce470 secure :-1 s=READY pgs=1607 cs=0 l=1 rev1=1 crypto rx=0x7f75dc00c520 tx=0x7f75dc00c9e0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 shutdown_connections 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f75d406b4d0 0x7f75d406d990 unknown :-1 s=CLOSED pgs=721 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f75e815e060 0x7f75e8160a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.971 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f75e80d0880 0x7f75e80ce470 unknown :-1 s=CLOSED pgs=1607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 --2- 172.21.15.64:0/1549036283 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f75e806fe20 0x7f75e81f49e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:25.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 >> 172.21.15.64:0/1549036283 conn(0x7f75e80d1a90 msgr2=0x7f75e80d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:25.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 shutdown_connections 2024-03-04T07:34:25.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:25.971+0000 7f75ee719640 1 -- 172.21.15.64:0/1549036283 wait complete. 2024-03-04T07:34:26.183 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:26.183 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:26.541 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:27.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:27.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.298+0000 7f83573c7640 1 Processor -- start 2024-03-04T07:34:27.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.299+0000 7f83573c7640 1 -- start start 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.299+0000 7f83573c7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f835006fe20 0x7f83500cea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.299+0000 7f83573c7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f83500d0880 0x7f83500ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.299+0000 7f83573c7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f83500d0dd0 con 0x7f835015e060 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.299+0000 7f83573c7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f83500d10b0 con 0x7f83500d0880 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.299+0000 7f83573c7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f83500cc0e0 con 0x7f835006fe20 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355964640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f83500d0880 0x7f83500ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355964640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f83500d0880 0x7f83500ccbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33290/0 (socket says 172.21.15.64:33290) 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355163640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f835015e060 0x7f83500ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41812/0 (socket says 172.21.15.64:41812) 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355964640 1 -- 172.21.15.64:0/3354775518 learned_addr learned my addr 172.21.15.64:0/3354775518 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8354962640 1 --2- 172.21.15.64:0/3354775518 >> v2:172.21.15.66:3300/0 conn(0x7f835006fe20 0x7f83500cea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355964640 1 -- 172.21.15.64:0/3354775518 >> v2:172.21.15.66:3300/0 conn(0x7f835006fe20 msgr2=0x7f83500cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:27.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355964640 1 --2- 172.21.15.64:0/3354775518 >> v2:172.21.15.66:3300/0 conn(0x7f835006fe20 0x7f83500cea20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355964640 1 -- 172.21.15.64:0/3354775518 >> v1:172.21.15.64:6789/0 conn(0x7f835015e060 legacy=0x7f83500ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:27.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f8355964640 1 -- 172.21.15.64:0/3354775518 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83500cc3c0 con 0x7f83500d0880 2024-03-04T07:34:27.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.300+0000 7f834ffff640 1 -- 172.21.15.64:0/3354775518 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2672958134 0 0) 0x7f83500d0dd0 con 0x7f835015e060 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f8355964640 1 --2- 172.21.15.64:0/3354775518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f83500d0880 0x7f83500ccbc0 secure :-1 s=READY pgs=1609 cs=0 l=1 rev1=1 crypto rx=0x7f8350068cd0 tx=0x7f834002bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7026ac28b548387d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f834ffff640 1 -- 172.21.15.64:0/3354775518 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f834002cb20 con 0x7f83500d0880 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f834ffff640 1 -- 172.21.15.64:0/3354775518 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f834002ccc0 con 0x7f83500d0880 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f834ffff640 1 -- 172.21.15.64:0/3354775518 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8340034610 con 0x7f83500d0880 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f83573c7640 1 -- 172.21.15.64:0/3354775518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f83500d0880 msgr2=0x7f83500ccbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f83573c7640 1 --2- 172.21.15.64:0/3354775518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f83500d0880 0x7f83500ccbc0 secure :-1 s=READY pgs=1609 cs=0 l=1 rev1=1 crypto rx=0x7f8350068cd0 tx=0x7f834002bf80 comp rx=0 tx=0).stop 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f83573c7640 1 -- 172.21.15.64:0/3354775518 shutdown_connections 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f83573c7640 1 --2- 172.21.15.64:0/3354775518 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f83500d0880 0x7f83500ccbc0 unknown :-1 s=CLOSED pgs=1609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f83573c7640 1 --2- 172.21.15.64:0/3354775518 >> v2:172.21.15.66:3300/0 conn(0x7f835006fe20 0x7f83500cea20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.301+0000 7f83573c7640 1 -- 172.21.15.64:0/3354775518 >> 172.21.15.64:0/3354775518 conn(0x7f83500d1a90 msgr2=0x7f83500d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:27.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.302+0000 7f83573c7640 1 -- 172.21.15.64:0/3354775518 shutdown_connections 2024-03-04T07:34:27.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.302+0000 7f83573c7640 1 -- 172.21.15.64:0/3354775518 wait complete. 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.303+0000 7f83573c7640 1 Processor -- start 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.303+0000 7f83573c7640 1 -- start start 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.303+0000 7f83573c7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f835006fe20 0x7f83501f77f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f83573c7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83500d0880 0x7f83500b48e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f83573c7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f835015e060 0x7f83500b69e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f83573c7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83500cef60 con 0x7f835015e060 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f83573c7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f835015e440 con 0x7f83500d0880 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f83573c7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f83500cf0e0 con 0x7f835006fe20 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f835015e060 0x7f83500b69e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f835015e060 0x7f83500b69e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33298/0 (socket says 172.21.15.64:33298) 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355163640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f835006fe20 0x7f83501f77f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8354962640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83500d0880 0x7f83500b48e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 -- 172.21.15.64:0/1319298570 learned_addr learned my addr 172.21.15.64:0/1319298570 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 -- 172.21.15.64:0/1319298570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f835006fe20 msgr2=0x7f83501f77f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f835006fe20 0x7f83501f77f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 -- 172.21.15.64:0/1319298570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83500d0880 msgr2=0x7f83500b48e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83500d0880 0x7f83500b48e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 -- 172.21.15.64:0/1319298570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83500b4e20 con 0x7f835015e060 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f8355964640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f835015e060 0x7f83500b69e0 secure :-1 s=READY pgs=1610 cs=0 l=1 rev1=1 crypto rx=0x7f8340036040 tx=0x7f8340004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f834dffb640 1 -- 172.21.15.64:0/1319298570 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8340043020 con 0x7f835015e060 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f834dffb640 1 -- 172.21.15.64:0/1319298570 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f834003e450 con 0x7f835015e060 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83500b50d0 con 0x7f835015e060 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.304+0000 7f834dffb640 1 -- 172.21.15.64:0/1319298570 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8340009510 con 0x7f835015e060 2024-03-04T07:34:27.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.305+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83500b5690 con 0x7f835015e060 2024-03-04T07:34:27.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.306+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f835006f980 con 0x7f835015e060 2024-03-04T07:34:27.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.306+0000 7f834dffb640 1 -- 172.21.15.64:0/1319298570 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8340046020 con 0x7f835015e060 2024-03-04T07:34:27.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.307+0000 7f834dffb640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f833c06b4d0 0x7f833c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:27.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.307+0000 7f8355163640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f833c06b4d0 0x7f833c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:27.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.307+0000 7f834dffb640 1 -- 172.21.15.64:0/1319298570 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f83400ac860 con 0x7f835015e060 2024-03-04T07:34:27.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.307+0000 7f8355163640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f833c06b4d0 0x7f833c06d990 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7f8348007fd0 tx=0x7f8348007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:27.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.311+0000 7f834dffb640 1 -- 172.21.15.64:0/1319298570 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f835006f980 con 0x7f835015e060 2024-03-04T07:34:27.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.531+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f83500d2010 con 0x7f835015e060 2024-03-04T07:34:27.531 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.532+0000 7f834dffb640 1 -- 172.21.15.64:0/1319298570 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f83400ac020 con 0x7f835015e060 2024-03-04T07:34:27.532 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:27.532 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:27.532 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f833c06b4d0 msgr2=0x7f833c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f833c06b4d0 0x7f833c06d990 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7f8348007fd0 tx=0x7f8348007de0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f835015e060 msgr2=0x7f83500b69e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f835015e060 0x7f83500b69e0 secure :-1 s=READY pgs=1610 cs=0 l=1 rev1=1 crypto rx=0x7f8340036040 tx=0x7f8340004a40 comp rx=0 tx=0).stop 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 shutdown_connections 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f833c06b4d0 0x7f833c06d990 unknown :-1 s=CLOSED pgs=722 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f835015e060 0x7f83500b69e0 unknown :-1 s=CLOSED pgs=1610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f83500d0880 0x7f83500b48e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 --2- 172.21.15.64:0/1319298570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f835006fe20 0x7f83501f77f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.533+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 >> 172.21.15.64:0/1319298570 conn(0x7f83500d1a90 msgr2=0x7f83500d12f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.534+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 shutdown_connections 2024-03-04T07:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:27.534+0000 7f83573c7640 1 -- 172.21.15.64:0/1319298570 wait complete. 2024-03-04T07:34:27.746 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:27.746 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:28.543 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:28.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:28.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 Processor -- start 2024-03-04T07:34:28.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 -- start start 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb6a415df00 0x7fb6a40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb6a40b9a30 0x7fb6a40b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb6a40cea30 con 0x7fb6a40b9a30 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb6a40d2af0 con 0x7fb6a415df00 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.861+0000 7fb6aa681640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6a40cf3c0 con 0x7fb6a40b95a0 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a8c1e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb6a40b9a30 0x7fb6a40b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a8c1e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb6a40b9a30 0x7fb6a40b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33306/0 (socket says 172.21.15.64:33306) 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a8c1e640 1 -- 172.21.15.64:0/112723584 learned_addr learned my addr 172.21.15.64:0/112723584 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a37fe640 1 --2- 172.21.15.64:0/112723584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:28.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a3fff640 1 --2- 172.21.15.64:0/112723584 >> v2:172.21.15.66:3300/0 conn(0x7fb6a415df00 0x7fb6a40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:28.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a37fe640 1 -- 172.21.15.64:0/112723584 >> v2:172.21.15.66:3300/0 conn(0x7fb6a415df00 msgr2=0x7fb6a40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:28.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a37fe640 1 --2- 172.21.15.64:0/112723584 >> v2:172.21.15.66:3300/0 conn(0x7fb6a415df00 0x7fb6a40d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:28.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a37fe640 1 -- 172.21.15.64:0/112723584 >> v2:172.21.15.64:3300/0 conn(0x7fb6a40b9a30 msgr2=0x7fb6a40b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:28.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a37fe640 1 --2- 172.21.15.64:0/112723584 >> v2:172.21.15.64:3300/0 conn(0x7fb6a40b9a30 0x7fb6a40b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:28.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a37fe640 1 -- 172.21.15.64:0/112723584 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6a40cf0e0 con 0x7fb6a40b95a0 2024-03-04T07:34:28.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.862+0000 7fb6a8c1e640 1 --2- 172.21.15.64:0/112723584 >> v2:172.21.15.64:3300/0 conn(0x7fb6a40b9a30 0x7fb6a40b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6a37fe640 1 --2- 172.21.15.64:0/112723584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a40b5700 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7fb6a40d11b0 tx=0x7fb69402b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a75e025a7c63b687 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:28.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6a2ffd640 1 -- 172.21.15.64:0/112723584 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6940315a0 con 0x7fb6a40b95a0 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6a2ffd640 1 -- 172.21.15.64:0/112723584 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb694037070 con 0x7fb6a40b95a0 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6a2ffd640 1 -- 172.21.15.64:0/112723584 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb694031960 con 0x7fb6a40b95a0 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6aa681640 1 -- 172.21.15.64:0/112723584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 msgr2=0x7fb6a40b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6aa681640 1 --2- 172.21.15.64:0/112723584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a40b5700 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7fb6a40d11b0 tx=0x7fb69402b840 comp rx=0 tx=0).stop 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6aa681640 1 -- 172.21.15.64:0/112723584 shutdown_connections 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6aa681640 1 --2- 172.21.15.64:0/112723584 >> v2:172.21.15.64:3300/0 conn(0x7fb6a40b9a30 0x7fb6a40b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6aa681640 1 --2- 172.21.15.64:0/112723584 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a40b5700 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6aa681640 1 --2- 172.21.15.64:0/112723584 >> v2:172.21.15.66:3300/0 conn(0x7fb6a415df00 0x7fb6a40d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.863+0000 7fb6aa681640 1 -- 172.21.15.64:0/112723584 >> 172.21.15.64:0/112723584 conn(0x7fb6a4071960 msgr2=0x7fb6a4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.864+0000 7fb6aa681640 1 -- 172.21.15.64:0/112723584 shutdown_connections 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.864+0000 7fb6aa681640 1 -- 172.21.15.64:0/112723584 wait complete. 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.865+0000 7fb6aa681640 1 Processor -- start 2024-03-04T07:34:28.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.865+0000 7fb6aa681640 1 -- start start 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6aa681640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a4154d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6aa681640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6a415df00 0x7fb6a40b3990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6aa681640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6a40b3ed0 0x7fb6a4155e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6aa681640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6a40cec70 con 0x7fb6a415df00 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6aa681640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6a40ce750 con 0x7fb6a40b3ed0 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6aa681640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb6a40cedf0 con 0x7fb6a40b95a0 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6a415df00 0x7fb6a40b3990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a3fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a4154d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6a415df00 0x7fb6a40b3990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33312/0 (socket says 172.21.15.64:33312) 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a37fe640 1 -- 172.21.15.64:0/920893361 learned_addr learned my addr 172.21.15.64:0/920893361 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:28.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a8c1e640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6a40b3ed0 0x7fb6a4155e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a3fff640 1 -- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6a40b3ed0 msgr2=0x7fb6a4155e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a3fff640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6a40b3ed0 0x7fb6a4155e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a3fff640 1 -- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6a415df00 msgr2=0x7fb6a40b3990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a3fff640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6a415df00 0x7fb6a40b3990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a3fff640 1 -- 172.21.15.64:0/920893361 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6a40b4490 con 0x7fb6a40b95a0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a37fe640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6a415df00 0x7fb6a40b3990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a3fff640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a4154d70 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fb68c00e8a0 tx=0x7fb68c00ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a0ff9640 1 -- 172.21.15.64:0/920893361 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb68c017070 con 0x7fb6a40b95a0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a0ff9640 1 -- 172.21.15.64:0/920893361 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb68c00c940 con 0x7fb6a40b95a0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6a40b4770 con 0x7fb6a40b95a0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.866+0000 7fb6a0ff9640 1 -- 172.21.15.64:0/920893361 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb68c009e60 con 0x7fb6a40b95a0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.867+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6a40b4d30 con 0x7fb6a40b95a0 2024-03-04T07:34:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.867+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb6a4067ff0 con 0x7fb6a40b95a0 2024-03-04T07:34:28.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.868+0000 7fb6a0ff9640 1 -- 172.21.15.64:0/920893361 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb68c01a070 con 0x7fb6a40b95a0 2024-03-04T07:34:28.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.868+0000 7fb6a0ff9640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb68406b4d0 0x7fb68406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:28.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.868+0000 7fb6a0ff9640 1 -- 172.21.15.64:0/920893361 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb68c08a2a0 con 0x7fb6a40b95a0 2024-03-04T07:34:28.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.869+0000 7fb6a37fe640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb68406b4d0 0x7fb68406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:28.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.869+0000 7fb6a37fe640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb68406b4d0 0x7fb68406d990 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7fb6a415c7c0 tx=0x7fb69403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:28.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:28.878+0000 7fb6a0ff9640 1 -- 172.21.15.64:0/920893361 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb68c058ac0 con 0x7fb6a40b95a0 2024-03-04T07:34:29.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.094+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb6a415c890 con 0x7fb6a40b95a0 2024-03-04T07:34:29.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.095+0000 7fb6a0ff9640 1 -- 172.21.15.64:0/920893361 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7fb68c05cb60 con 0x7fb6a40b95a0 2024-03-04T07:34:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:29.096 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:29.096 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.096+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb68406b4d0 msgr2=0x7fb68406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.096+0000 7fb6aa681640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb68406b4d0 0x7fb68406d990 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7fb6a415c7c0 tx=0x7fb69403d040 comp rx=0 tx=0).stop 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 msgr2=0x7fb6a4154d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a4154d70 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fb68c00e8a0 tx=0x7fb68c00ed60 comp rx=0 tx=0).stop 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 shutdown_connections 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb68406b4d0 0x7fb68406d990 unknown :-1 s=CLOSED pgs=723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb6a40b3ed0 0x7fb6a4155e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb6a415df00 0x7fb6a40b3990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 --2- 172.21.15.64:0/920893361 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb6a40b95a0 0x7fb6a4154d70 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 >> 172.21.15.64:0/920893361 conn(0x7fb6a4071960 msgr2=0x7fb6a415bbe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:29.098 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 shutdown_connections 2024-03-04T07:34:29.099 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:29.097+0000 7fb6aa681640 1 -- 172.21.15.64:0/920893361 wait complete. 2024-03-04T07:34:29.310 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:29.310 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:30.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:30.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.425+0000 7fba00cc9640 1 Processor -- start 2024-03-04T07:34:30.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.425+0000 7fba00cc9640 1 -- start start 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.426+0000 7fba00cc9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb9fc06d400 0x7fb9fc15cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.426+0000 7fba00cc9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb9fc0729b0 con 0x7fb9fc15bac0 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.426+0000 7fba00cc9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb9fc06c920 con 0x7fb9fc0aee10 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.426+0000 7fba00cc9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb9fc06cf70 con 0x7fb9fc06d400 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.426+0000 7fb9fa59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb9fc15bac0 0x7fb9fc0ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41818/0 (socket says 172.21.15.64:41818) 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.426+0000 7fb9fad9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb9fc06d400 0x7fb9fc15cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.426+0000 7fb9fa59c640 1 -- 172.21.15.64:0/456143493 learned_addr learned my addr 172.21.15.64:0/456143493 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.427+0000 7fb9fad9d640 1 -- 172.21.15.64:0/456143493 >> v1:172.21.15.66:6789/0 conn(0x7fb9fc0aee10 legacy=0x7fb9fc070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:30.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.427+0000 7fb9fad9d640 1 -- 172.21.15.64:0/456143493 >> v1:172.21.15.64:6789/0 conn(0x7fb9fc15bac0 legacy=0x7fb9fc0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:30.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.427+0000 7fb9f959a640 1 -- 172.21.15.64:0/456143493 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1143491144 0 0) 0x7fb9fc06c920 con 0x7fb9fc0aee10 2024-03-04T07:34:30.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.427+0000 7fb9fad9d640 1 -- 172.21.15.64:0/456143493 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9fc06cc00 con 0x7fb9fc06d400 2024-03-04T07:34:30.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fb9fad9d640 1 --2- 172.21.15.64:0/456143493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc06d400 0x7fb9fc15cbe0 secure :-1 s=READY pgs=1612 cs=0 l=1 rev1=1 crypto rx=0x7fb9f40097a0 tx=0x7fb9f402b790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eefa8b7f49d3e29 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:30.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fb9f959a640 1 -- 172.21.15.64:0/456143493 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb9f4037070 con 0x7fb9fc06d400 2024-03-04T07:34:30.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fb9f959a640 1 -- 172.21.15.64:0/456143493 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb9f4030710 con 0x7fb9fc06d400 2024-03-04T07:34:30.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fb9f959a640 1 -- 172.21.15.64:0/456143493 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb9f4030ae0 con 0x7fb9fc06d400 2024-03-04T07:34:30.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fba00cc9640 1 -- 172.21.15.64:0/456143493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc06d400 msgr2=0x7fb9fc15cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:30.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fba00cc9640 1 --2- 172.21.15.64:0/456143493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc06d400 0x7fb9fc15cbe0 secure :-1 s=READY pgs=1612 cs=0 l=1 rev1=1 crypto rx=0x7fb9f40097a0 tx=0x7fb9f402b790 comp rx=0 tx=0).stop 2024-03-04T07:34:30.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fba00cc9640 1 -- 172.21.15.64:0/456143493 shutdown_connections 2024-03-04T07:34:30.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fba00cc9640 1 --2- 172.21.15.64:0/456143493 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc06d400 0x7fb9fc15cbe0 unknown :-1 s=CLOSED pgs=1612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fba00cc9640 1 -- 172.21.15.64:0/456143493 >> 172.21.15.64:0/456143493 conn(0x7fb9fc066d00 msgr2=0x7fb9fc0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:30.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fba00cc9640 1 -- 172.21.15.64:0/456143493 shutdown_connections 2024-03-04T07:34:30.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.428+0000 7fba00cc9640 1 -- 172.21.15.64:0/456143493 wait complete. 2024-03-04T07:34:30.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 Processor -- start 2024-03-04T07:34:30.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 -- start start 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9fc06d400 0x7fb9fc0b4610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc0aee10 0x7fb9fc152000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb9fc15bac0 0x7fb9fc0ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9fc15c020 con 0x7fb9fc0aee10 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9fc15bea0 con 0x7fb9fc06d400 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.430+0000 7fba00cc9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb9fc072580 con 0x7fb9fc15bac0 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc0aee10 0x7fb9fc152000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc0aee10 0x7fb9fc152000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33322/0 (socket says 172.21.15.64:33322) 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 -- 172.21.15.64:0/515025775 learned_addr learned my addr 172.21.15.64:0/515025775 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9fa59c640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9fc06d400 0x7fb9fc0b4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:30.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9fad9d640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb9fc15bac0 0x7fb9fc0ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 -- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb9fc15bac0 msgr2=0x7fb9fc0ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb9fc15bac0 0x7fb9fc0ad8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 -- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9fc06d400 msgr2=0x7fb9fc0b4610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9fc06d400 0x7fb9fc0b4610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 -- 172.21.15.64:0/515025775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9fc158180 con 0x7fb9fc0aee10 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f9d9b640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc0aee10 0x7fb9fc152000 secure :-1 s=READY pgs=1613 cs=0 l=1 rev1=1 crypto rx=0x7fb9e400c4b0 tx=0x7fb9e400c970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f2ffd640 1 -- 172.21.15.64:0/515025775 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb9e4017070 con 0x7fb9fc0aee10 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f2ffd640 1 -- 172.21.15.64:0/515025775 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb9e4005480 con 0x7fb9fc0aee10 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb9fc0ce7e0 con 0x7fb9fc0aee10 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fb9f2ffd640 1 -- 172.21.15.64:0/515025775 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb9e4003830 con 0x7fb9fc0aee10 2024-03-04T07:34:30.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.431+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9fc0ceda0 con 0x7fb9fc0aee10 2024-03-04T07:34:30.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.432+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb9fc067ff0 con 0x7fb9fc0aee10 2024-03-04T07:34:30.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.433+0000 7fb9f2ffd640 1 -- 172.21.15.64:0/515025775 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb9e401a070 con 0x7fb9fc0aee10 2024-03-04T07:34:30.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.433+0000 7fb9f2ffd640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb9dc06b4d0 0x7fb9dc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:30.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.433+0000 7fb9f2ffd640 1 -- 172.21.15.64:0/515025775 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7fb9e4089450 con 0x7fb9fc0aee10 2024-03-04T07:34:30.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.433+0000 7fb9fa59c640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb9dc06b4d0 0x7fb9dc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:30.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.434+0000 7fb9fa59c640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb9dc06b4d0 0x7fb9dc06d990 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7fb9ec0012e0 tx=0x7fb9ec0017a0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:30.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.438+0000 7fb9f2ffd640 1 -- 172.21.15.64:0/515025775 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb9e4057d10 con 0x7fb9fc0aee10 2024-03-04T07:34:30.545 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:30.657 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.656+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb9fc06de50 con 0x7fb9fc0aee10 2024-03-04T07:34:30.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.657+0000 7fb9f2ffd640 1 -- 172.21.15.64:0/515025775 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7fb9e405bdb0 con 0x7fb9fc0aee10 2024-03-04T07:34:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:30.658 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:30.658 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb9dc06b4d0 msgr2=0x7fb9dc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb9dc06b4d0 0x7fb9dc06d990 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7fb9ec0012e0 tx=0x7fb9ec0017a0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc0aee10 msgr2=0x7fb9fc152000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc0aee10 0x7fb9fc152000 secure :-1 s=READY pgs=1613 cs=0 l=1 rev1=1 crypto rx=0x7fb9e400c4b0 tx=0x7fb9e400c970 comp rx=0 tx=0).stop 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 shutdown_connections 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb9dc06b4d0 0x7fb9dc06d990 unknown :-1 s=CLOSED pgs=724 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb9fc15bac0 0x7fb9fc0ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb9fc0aee10 0x7fb9fc152000 unknown :-1 s=CLOSED pgs=1613 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 --2- 172.21.15.64:0/515025775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb9fc06d400 0x7fb9fc0b4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:30.660 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 >> 172.21.15.64:0/515025775 conn(0x7fb9fc066d00 msgr2=0x7fb9fc0b16d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:30.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 shutdown_connections 2024-03-04T07:34:30.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:30.659+0000 7fba00cc9640 1 -- 172.21.15.64:0/515025775 wait complete. 2024-03-04T07:34:30.872 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:30.872 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:31.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:31.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.986+0000 7f63b10fb640 1 Processor -- start 2024-03-04T07:34:31.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.986+0000 7f63b10fb640 1 -- start start 2024-03-04T07:34:31.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63b10fb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f63ac071c50 0x7f63ac0d25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63b10fb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f63ac0b2c50 0x7f63ac0d2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63b10fb640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f63ac0ce970 con 0x7f63ac15e000 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63b10fb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63ac0cece0 con 0x7f63ac071c50 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63b10fb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f63ac0cef30 con 0x7f63ac0b2c50 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63aa59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f63ac071c50 0x7f63ac0d25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63aad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f63ac15e000 0x7f63ac071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41822/0 (socket says 172.21.15.64:41822) 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63aad9d640 1 -- 172.21.15.64:0/364803882 learned_addr learned my addr 172.21.15.64:0/364803882 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.987+0000 7f63ab59e640 1 --2- 172.21.15.64:0/364803882 >> v2:172.21.15.66:3300/0 conn(0x7f63ac0b2c50 0x7f63ac0d2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.988+0000 7f63aa59c640 1 -- 172.21.15.64:0/364803882 >> v2:172.21.15.66:3300/0 conn(0x7f63ac0b2c50 msgr2=0x7f63ac0d2b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.988+0000 7f63aa59c640 1 --2- 172.21.15.64:0/364803882 >> v2:172.21.15.66:3300/0 conn(0x7f63ac0b2c50 0x7f63ac0d2b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.988+0000 7f63aa59c640 1 -- 172.21.15.64:0/364803882 >> v1:172.21.15.64:6789/0 conn(0x7f63ac15e000 legacy=0x7f63ac071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:31.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.988+0000 7f63a9d9b640 1 -- 172.21.15.64:0/364803882 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 553087745 0 0) 0x7f63ac0ce970 con 0x7f63ac15e000 2024-03-04T07:34:31.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.988+0000 7f63aa59c640 1 -- 172.21.15.64:0/364803882 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63ac0cf210 con 0x7f63ac071c50 2024-03-04T07:34:31.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.988+0000 7f63aa59c640 1 --2- 172.21.15.64:0/364803882 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac0d25d0 secure :-1 s=READY pgs=1615 cs=0 l=1 rev1=1 crypto rx=0x7f63ac162d50 tx=0x7f63a002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f6a52bedd13d4bee server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:31.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63a9d9b640 1 -- 172.21.15.64:0/364803882 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a00315a0 con 0x7f63ac071c50 2024-03-04T07:34:31.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63a9d9b640 1 -- 172.21.15.64:0/364803882 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f63a0037070 con 0x7f63ac071c50 2024-03-04T07:34:31.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63a9d9b640 1 -- 172.21.15.64:0/364803882 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a0031960 con 0x7f63ac071c50 2024-03-04T07:34:31.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 -- 172.21.15.64:0/364803882 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 msgr2=0x7f63ac0d25d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:31.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 --2- 172.21.15.64:0/364803882 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac0d25d0 secure :-1 s=READY pgs=1615 cs=0 l=1 rev1=1 crypto rx=0x7f63ac162d50 tx=0x7f63a002b840 comp rx=0 tx=0).stop 2024-03-04T07:34:31.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 -- 172.21.15.64:0/364803882 shutdown_connections 2024-03-04T07:34:31.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 --2- 172.21.15.64:0/364803882 >> v2:172.21.15.66:3300/0 conn(0x7f63ac0b2c50 0x7f63ac0d2b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:31.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 --2- 172.21.15.64:0/364803882 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac0d25d0 unknown :-1 s=CLOSED pgs=1615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:31.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 -- 172.21.15.64:0/364803882 >> 172.21.15.64:0/364803882 conn(0x7f63ac0d3c80 msgr2=0x7f63ac0d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:31.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 -- 172.21.15.64:0/364803882 shutdown_connections 2024-03-04T07:34:31.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.989+0000 7f63b10fb640 1 -- 172.21.15.64:0/364803882 wait complete. 2024-03-04T07:34:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 Processor -- start 2024-03-04T07:34:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 -- start start 2024-03-04T07:34:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac157f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63ac0b2c50 0x7f63ac0b1a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63ac15e000 0x7f63ac0b1f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63ac0b3320 con 0x7f63ac071c50 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63ac0b31a0 con 0x7f63ac15e000 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.991+0000 7f63b10fb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f63ac15e3e0 con 0x7f63ac0b2c50 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac157f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac157f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:33352/0 (socket says 172.21.15.64:33352) 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 -- 172.21.15.64:0/468440009 learned_addr learned my addr 172.21.15.64:0/468440009 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63ab59e640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63ac15e000 0x7f63ac0b1f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aa59c640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63ac0b2c50 0x7f63ac0b1a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 -- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63ac0b2c50 msgr2=0x7f63ac0b1a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63ac0b2c50 0x7f63ac0b1a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 -- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63ac15e000 msgr2=0x7f63ac0b1f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63ac15e000 0x7f63ac0b1f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 -- 172.21.15.64:0/468440009 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63ac1f4290 con 0x7f63ac071c50 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63ab59e640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63ac15e000 0x7f63ac0b1f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63aad9d640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac157f10 secure :-1 s=READY pgs=1616 cs=0 l=1 rev1=1 crypto rx=0x7f63a400c430 tx=0x7f63a400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63937fe640 1 -- 172.21.15.64:0/468440009 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a40074e0 con 0x7f63ac071c50 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63937fe640 1 -- 172.21.15.64:0/468440009 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f63a40047e0 con 0x7f63ac071c50 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63ac1f4570 con 0x7f63ac071c50 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63937fe640 1 -- 172.21.15.64:0/468440009 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a4004bd0 con 0x7f63ac071c50 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.992+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63ac1f4b30 con 0x7f63ac071c50 2024-03-04T07:34:31.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.993+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f63ac0722f0 con 0x7f63ac071c50 2024-03-04T07:34:31.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.994+0000 7f63937fe640 1 -- 172.21.15.64:0/468440009 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f63a4011070 con 0x7f63ac071c50 2024-03-04T07:34:31.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.994+0000 7f63937fe640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f639806b4d0 0x7f639806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:31.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.994+0000 7f63937fe640 1 -- 172.21.15.64:0/468440009 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f63a4089ca0 con 0x7f63ac071c50 2024-03-04T07:34:31.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.994+0000 7f63aa59c640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f639806b4d0 0x7f639806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:31.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.995+0000 7f63aa59c640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f639806b4d0 0x7f639806d990 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f63a0036000 tx=0x7f63a0008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:31.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:31.999+0000 7f63937fe640 1 -- 172.21.15.64:0/468440009 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f63ac0722f0 con 0x7f63ac071c50 2024-03-04T07:34:32.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.220+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f63ac067ff0 con 0x7f63ac071c50 2024-03-04T07:34:32.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.221+0000 7f63937fe640 1 -- 172.21.15.64:0/468440009 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 89 v89) v1 ==== 94+0+3412 (secure 0 0 0) 0x7f63a4058560 con 0x7f63ac071c50 2024-03-04T07:34:32.222 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:32.222 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":89,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":89,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:05.315418+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:32.222 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 89 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f639806b4d0 msgr2=0x7f639806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f639806b4d0 0x7f639806d990 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f63a0036000 tx=0x7f63a0008030 comp rx=0 tx=0).stop 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 msgr2=0x7f63ac157f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac157f10 secure :-1 s=READY pgs=1616 cs=0 l=1 rev1=1 crypto rx=0x7f63a400c430 tx=0x7f63a400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 shutdown_connections 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f639806b4d0 0x7f639806d990 unknown :-1 s=CLOSED pgs=725 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f63ac15e000 0x7f63ac0b1f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:32.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f63ac0b2c50 0x7f63ac0b1a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:32.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 --2- 172.21.15.64:0/468440009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f63ac071c50 0x7f63ac157f10 unknown :-1 s=CLOSED pgs=1616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:32.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 >> 172.21.15.64:0/468440009 conn(0x7f63ac0d3c80 msgr2=0x7f63ac160ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:32.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 shutdown_connections 2024-03-04T07:34:32.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:32.223+0000 7f63b10fb640 1 -- 172.21.15.64:0/468440009 wait complete. 2024-03-04T07:34:32.436 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 89, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:05.315418+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:32.437 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:32.547 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:33.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:33.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.551+0000 7f5f858c2640 1 Processor -- start 2024-03-04T07:34:33.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.551+0000 7f5f858c2640 1 -- start start 2024-03-04T07:34:33.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f858c2640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5f8015bac0 0x7f5f8015cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f858c2640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5f800aee10 0x7f5f80070870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f858c2640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f5f800729b0 con 0x7f5f8006d400 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f858c2640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5f8006cf70 con 0x7f5f8015bac0 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f858c2640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f5f8006c920 con 0x7f5f800aee10 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f7effd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5f8015bac0 0x7f5f8015cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f7effd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f5f8015bac0 0x7f5f8015cbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35140/0 (socket says 172.21.15.64:35140) 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f7effd640 1 -- 172.21.15.64:0/4097263009 learned_addr learned my addr 172.21.15.64:0/4097263009 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.552+0000 7f5f7f7fe640 1 --2- 172.21.15.64:0/4097263009 >> v2:172.21.15.66:3300/0 conn(0x7f5f800aee10 0x7f5f80070870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.553+0000 7f5f7effd640 1 -- 172.21.15.64:0/4097263009 >> v2:172.21.15.66:3300/0 conn(0x7f5f800aee10 msgr2=0x7f5f80070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.553+0000 7f5f7effd640 1 --2- 172.21.15.64:0/4097263009 >> v2:172.21.15.66:3300/0 conn(0x7f5f800aee10 0x7f5f80070870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.553+0000 7f5f7effd640 1 -- 172.21.15.64:0/4097263009 >> v1:172.21.15.66:6789/0 conn(0x7f5f8006d400 legacy=0x7f5f800ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:33.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.553+0000 7f5f7effd640 1 -- 172.21.15.64:0/4097263009 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f8006cc00 con 0x7f5f8015bac0 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.553+0000 7f5f7effd640 1 --2- 172.21.15.64:0/4097263009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f8015cbe0 secure :-1 s=READY pgs=1617 cs=0 l=1 rev1=1 crypto rx=0x7f5f8006d0f0 tx=0x7f5f7002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=648fd7962ce7c2c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.553+0000 7f5f7dffb640 1 -- 172.21.15.64:0/4097263009 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f70034070 con 0x7f5f8015bac0 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f7dffb640 1 -- 172.21.15.64:0/4097263009 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f70030860 con 0x7f5f8015bac0 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f7dffb640 1 -- 172.21.15.64:0/4097263009 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f70030c30 con 0x7f5f8015bac0 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 -- 172.21.15.64:0/4097263009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 msgr2=0x7f5f8015cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 --2- 172.21.15.64:0/4097263009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f8015cbe0 secure :-1 s=READY pgs=1617 cs=0 l=1 rev1=1 crypto rx=0x7f5f8006d0f0 tx=0x7f5f7002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 -- 172.21.15.64:0/4097263009 shutdown_connections 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 --2- 172.21.15.64:0/4097263009 >> v2:172.21.15.66:3300/0 conn(0x7f5f800aee10 0x7f5f80070870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 --2- 172.21.15.64:0/4097263009 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f8015cbe0 unknown :-1 s=CLOSED pgs=1617 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 -- 172.21.15.64:0/4097263009 >> 172.21.15.64:0/4097263009 conn(0x7f5f80066d00 msgr2=0x7f5f800670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 -- 172.21.15.64:0/4097263009 shutdown_connections 2024-03-04T07:34:33.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.554+0000 7f5f858c2640 1 -- 172.21.15.64:0/4097263009 wait complete. 2024-03-04T07:34:33.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.555+0000 7f5f858c2640 1 Processor -- start 2024-03-04T07:34:33.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.556+0000 7f5f858c2640 1 -- start start 2024-03-04T07:34:33.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.556+0000 7f5f858c2640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f8006d400 0x7f5f801477f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:33.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.556+0000 7f5f858c2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f800aee10 0x7f5f80151d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:33.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.556+0000 7f5f858c2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f800ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.556+0000 7f5f858c2640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f8015c020 con 0x7f5f8015bac0 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.556+0000 7f5f858c2640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f8015bea0 con 0x7f5f8006d400 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.556+0000 7f5f858c2640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5f80072580 con 0x7f5f800aee10 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f800ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7effd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f8006d400 0x7f5f801477f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f800ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35144/0 (socket says 172.21.15.64:35144) 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 -- 172.21.15.64:0/1501427147 learned_addr learned my addr 172.21.15.64:0/1501427147 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7e7fc640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f800aee10 0x7f5f80151d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 -- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f800aee10 msgr2=0x7f5f80151d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f800aee10 0x7f5f80151d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 -- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f8006d400 msgr2=0x7f5f801477f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f8006d400 0x7f5f801477f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 -- 172.21.15.64:0/1501427147 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f80148200 con 0x7f5f8015bac0 2024-03-04T07:34:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7effd640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f8006d400 0x7f5f801477f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:33.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.557+0000 7f5f7f7fe640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f800ad8b0 secure :-1 s=READY pgs=1618 cs=0 l=1 rev1=1 crypto rx=0x7f5f7800efc0 tx=0x7f5f7800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:33.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.559+0000 7f5f77fff640 1 -- 172.21.15.64:0/1501427147 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f78018020 con 0x7f5f8015bac0 2024-03-04T07:34:33.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.559+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5f801484e0 con 0x7f5f8015bac0 2024-03-04T07:34:33.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.559+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5f80148aa0 con 0x7f5f8015bac0 2024-03-04T07:34:33.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.559+0000 7f5f77fff640 1 -- 172.21.15.64:0/1501427147 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5f78014480 con 0x7f5f8015bac0 2024-03-04T07:34:33.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.559+0000 7f5f77fff640 1 -- 172.21.15.64:0/1501427147 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5f78014830 con 0x7f5f8015bac0 2024-03-04T07:34:33.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.560+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5f80067ff0 con 0x7f5f8015bac0 2024-03-04T07:34:33.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.560+0000 7f5f77fff640 1 -- 172.21.15.64:0/1501427147 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5f78005d10 con 0x7f5f8015bac0 2024-03-04T07:34:33.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.561+0000 7f5f77fff640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f6006b500 0x7f5f6006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:33.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.561+0000 7f5f77fff640 1 -- 172.21.15.64:0/1501427147 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f5f7808ab30 con 0x7f5f8015bac0 2024-03-04T07:34:33.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.561+0000 7f5f7effd640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f6006b500 0x7f5f6006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:33.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.561+0000 7f5f7effd640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f6006b500 0x7f5f6006d9c0 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f5f70036000 tx=0x7f5f70008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:33.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.566+0000 7f5f77fff640 1 -- 172.21.15.64:0/1501427147 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5f78022030 con 0x7f5f8015bac0 2024-03-04T07:34:33.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.787+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5f80156c40 con 0x7f5f8015bac0 2024-03-04T07:34:33.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.788+0000 7f5f77fff640 1 -- 172.21.15.64:0/1501427147 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f5f78010040 con 0x7f5f8015bac0 2024-03-04T07:34:33.789 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:33.789 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:33.789 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f6006b500 msgr2=0x7f5f6006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f6006b500 0x7f5f6006d9c0 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f5f70036000 tx=0x7f5f70008030 comp rx=0 tx=0).stop 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 msgr2=0x7f5f800ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f800ad8b0 secure :-1 s=READY pgs=1618 cs=0 l=1 rev1=1 crypto rx=0x7f5f7800efc0 tx=0x7f5f7800c490 comp rx=0 tx=0).stop 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 shutdown_connections 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5f6006b500 0x7f5f6006d9c0 unknown :-1 s=CLOSED pgs=726 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5f8015bac0 0x7f5f800ad8b0 unknown :-1 s=CLOSED pgs=1618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f5f800aee10 0x7f5f80151d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 --2- 172.21.15.64:0/1501427147 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5f8006d400 0x7f5f801477f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:33.791 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 >> 172.21.15.64:0/1501427147 conn(0x7f5f80066d00 msgr2=0x7f5f80155290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:33.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 shutdown_connections 2024-03-04T07:34:33.792 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:33.790+0000 7f5f858c2640 1 -- 172.21.15.64:0/1501427147 wait complete. 2024-03-04T07:34:34.003 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:34.003 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:34.549 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:35.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:35.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.117+0000 7f2b610df640 1 Processor -- start 2024-03-04T07:34:35.118 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.117+0000 7f2b610df640 1 -- start start 2024-03-04T07:34:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b610df640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2b5c15a370 0x7f2b5c0af370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b610df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b610df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2b5c15a750 con 0x7f2b5c0b2290 2024-03-04T07:34:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b610df640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2b5c0af8b0 con 0x7f2b5c15a370 2024-03-04T07:34:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b610df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2b5c0afae0 con 0x7f2b5c06ed90 2024-03-04T07:34:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5b59e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2b5c0b2290 0x7f2b5c06d720 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59644/0 (socket says 172.21.15.64:59644) 2024-03-04T07:34:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5b59e640 1 -- 172.21.15.64:0/2131192533 learned_addr learned my addr 172.21.15.64:0/2131192533 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5ad9d640 1 --2- 172.21.15.64:0/2131192533 >> v2:172.21.15.66:3300/0 conn(0x7f2b5c15a370 0x7f2b5c0af370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5a59c640 1 --2- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b59d9b640 1 -- 172.21.15.64:0/2131192533 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 347036728 0 0) 0x7f2b5c15a750 con 0x7f2b5c0b2290 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b59d9b640 1 -- 172.21.15.64:0/2131192533 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2b44003520 con 0x7f2b5c0b2290 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5ad9d640 1 -- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c06ed90 msgr2=0x7f2b5c0b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5ad9d640 1 --2- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5ad9d640 1 -- 172.21.15.64:0/2131192533 >> v1:172.21.15.64:6789/0 conn(0x7f2b5c0b2290 legacy=0x7f2b5c06d720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5ad9d640 1 -- 172.21.15.64:0/2131192533 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b5c0ae0b0 con 0x7f2b5c15a370 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.118+0000 7f2b5a59c640 1 --2- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.119+0000 7f2b5ad9d640 1 --2- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c15a370 0x7f2b5c0af370 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7f2b4c0097a0 tx=0x7f2b4c02b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=caf9c275fab7f8cd server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.119+0000 7f2b59d9b640 1 -- 172.21.15.64:0/2131192533 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b4c0315a0 con 0x7f2b5c15a370 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.119+0000 7f2b59d9b640 1 -- 172.21.15.64:0/2131192533 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2b4c037070 con 0x7f2b5c15a370 2024-03-04T07:34:35.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.119+0000 7f2b59d9b640 1 -- 172.21.15.64:0/2131192533 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b4c031940 con 0x7f2b5c15a370 2024-03-04T07:34:35.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 -- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c15a370 msgr2=0x7f2b5c0af370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:35.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 --2- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c15a370 0x7f2b5c0af370 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7f2b4c0097a0 tx=0x7f2b4c02b840 comp rx=0 tx=0).stop 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 -- 172.21.15.64:0/2131192533 shutdown_connections 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 --2- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 --2- 172.21.15.64:0/2131192533 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c15a370 0x7f2b5c0af370 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 -- 172.21.15.64:0/2131192533 >> 172.21.15.64:0/2131192533 conn(0x7f2b5c0b55e0 msgr2=0x7f2b5c0b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 -- 172.21.15.64:0/2131192533 shutdown_connections 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.120+0000 7f2b610df640 1 -- 172.21.15.64:0/2131192533 wait complete. 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.121+0000 7f2b610df640 1 Processor -- start 2024-03-04T07:34:35.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b610df640 1 -- start start 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b610df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b610df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c0b2290 0x7f2b5c14d490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b610df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c15a370 0x7f2b5c1554e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b610df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b5c15e250 con 0x7f2b5c06ed90 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b610df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2b5c0b26a0 con 0x7f2b5c0b2290 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b610df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2b5c15e530 con 0x7f2b5c15a370 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b5ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b5ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35158/0 (socket says 172.21.15.64:35158) 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b5ad9d640 1 -- 172.21.15.64:0/2246060321 learned_addr learned my addr 172.21.15.64:0/2246060321 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b5a59c640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c0b2290 0x7f2b5c14d490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.122+0000 7f2b5b59e640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c15a370 0x7f2b5c1554e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:35.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b5a59c640 1 -- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c15a370 msgr2=0x7f2b5c1554e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b5a59c640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c15a370 0x7f2b5c1554e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b5a59c640 1 -- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b5c06ed90 msgr2=0x7f2b5c0b1850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b5a59c640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b5a59c640 1 -- 172.21.15.64:0/2246060321 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2b5c14ad70 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b5ad9d640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b5a59c640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c0b2290 0x7f2b5c14d490 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f2b5000e900 tx=0x7f2b5000edc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b437fe640 1 -- 172.21.15.64:0/2246060321 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b50017070 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b437fe640 1 -- 172.21.15.64:0/2246060321 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2b5000ca00 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2b5c14b050 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b437fe640 1 -- 172.21.15.64:0/2246060321 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2b5000cdf0 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.123+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2b5c14b610 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.124+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2b5c067ff0 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.125+0000 7f2b437fe640 1 -- 172.21.15.64:0/2246060321 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2b5001a070 con 0x7f2b5c0b2290 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.125+0000 7f2b437fe640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b4806b4d0 0x7f2b4806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:35.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.125+0000 7f2b437fe640 1 -- 172.21.15.64:0/2246060321 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(125..125 src has 1..125) v4 ==== 5709+0+0 (secure 0 0 0) 0x7f2b5008a330 con 0x7f2b5c0b2290 2024-03-04T07:34:35.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.125+0000 7f2b5ad9d640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b4806b4d0 0x7f2b4806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:35.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.126+0000 7f2b5ad9d640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b4806b4d0 0x7f2b4806d990 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f2b5c1f7440 tx=0x7f2b4c03d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:35.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.135+0000 7f2b437fe640 1 -- 172.21.15.64:0/2246060321 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2b50058bf0 con 0x7f2b5c0b2290 2024-03-04T07:34:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.360+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2b5c15dc40 con 0x7f2b5c0b2290 2024-03-04T07:34:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.360+0000 7f2b437fe640 1 -- 172.21.15.64:0/2246060321 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f2b5005cc90 con 0x7f2b5c0b2290 2024-03-04T07:34:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:35.361 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:35.362 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.362+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b4806b4d0 msgr2=0x7f2b4806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:35.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.362+0000 7f2b610df640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b4806b4d0 0x7f2b4806d990 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f2b5c1f7440 tx=0x7f2b4c03d040 comp rx=0 tx=0).stop 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.362+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c0b2290 msgr2=0x7f2b5c14d490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.362+0000 7f2b610df640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c0b2290 0x7f2b5c14d490 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f2b5000e900 tx=0x7f2b5000edc0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 shutdown_connections 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2b4806b4d0 0x7f2b4806d990 unknown :-1 s=CLOSED pgs=727 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2b5c15a370 0x7f2b5c1554e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2b5c0b2290 0x7f2b5c14d490 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 --2- 172.21.15.64:0/2246060321 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2b5c06ed90 0x7f2b5c0b1850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 >> 172.21.15.64:0/2246060321 conn(0x7f2b5c0b55e0 msgr2=0x7f2b5c15a750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 shutdown_connections 2024-03-04T07:34:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:35.363+0000 7f2b610df640 1 -- 172.21.15.64:0/2246060321 wait complete. 2024-03-04T07:34:35.575 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:35.576 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:36.551 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:36.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:36.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.689+0000 7fc0bd0df640 1 Processor -- start 2024-03-04T07:34:36.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.690+0000 7fc0bd0df640 1 -- start start 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.690+0000 7fc0bd0df640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc0b8071540 0x7fc0b80b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.690+0000 7fc0bd0df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc0b80ce610 con 0x7fc0b80b2b50 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.690+0000 7fc0bd0df640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc0b80d2920 con 0x7fc0b80d3400 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.690+0000 7fc0bd0df640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc0b80d2f70 con 0x7fc0b8071540 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.690+0000 7fc0b759e640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc0b80b2b50 0x7fc0b80b6e20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59656/0 (socket says 172.21.15.64:59656) 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.690+0000 7fc0b759e640 1 -- 172.21.15.64:0/2526792377 learned_addr learned my addr 172.21.15.64:0/2526792377 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.691+0000 7fc0b6d9d640 1 --2- 172.21.15.64:0/2526792377 >> v2:172.21.15.66:3300/0 conn(0x7fc0b8071540 0x7fc0b80b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.691+0000 7fc0b5d9b640 1 -- 172.21.15.64:0/2526792377 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 130486364 0 0) 0x7fc0b80ce610 con 0x7fc0b80b2b50 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.691+0000 7fc0b5d9b640 1 -- 172.21.15.64:0/2526792377 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc0a4003520 con 0x7fc0b80b2b50 2024-03-04T07:34:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.691+0000 7fc0b6d9d640 1 -- 172.21.15.64:0/2526792377 >> v1:172.21.15.66:6789/0 conn(0x7fc0b80d3400 legacy=0x7fc0b8071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:36.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.691+0000 7fc0b6d9d640 1 -- 172.21.15.64:0/2526792377 >> v1:172.21.15.64:6789/0 conn(0x7fc0b80b2b50 legacy=0x7fc0b80b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:36.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.691+0000 7fc0b6d9d640 1 -- 172.21.15.64:0/2526792377 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0b80d2c00 con 0x7fc0b8071540 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0b6d9d640 1 --2- 172.21.15.64:0/2526792377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b8071540 0x7fc0b80b30a0 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fc0a80097a0 tx=0x7fc0a802b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=20a62c0ef894cc7a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0b5d9b640 1 -- 172.21.15.64:0/2526792377 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0a8034070 con 0x7fc0b8071540 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0b5d9b640 1 -- 172.21.15.64:0/2526792377 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc0a8030610 con 0x7fc0b8071540 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0b5d9b640 1 -- 172.21.15.64:0/2526792377 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0a80309c0 con 0x7fc0b8071540 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0bd0df640 1 -- 172.21.15.64:0/2526792377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b8071540 msgr2=0x7fc0b80b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/2526792377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b8071540 0x7fc0b80b30a0 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fc0a80097a0 tx=0x7fc0a802b960 comp rx=0 tx=0).stop 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0bd0df640 1 -- 172.21.15.64:0/2526792377 shutdown_connections 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/2526792377 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b8071540 0x7fc0b80b30a0 unknown :-1 s=CLOSED pgs=577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0bd0df640 1 -- 172.21.15.64:0/2526792377 >> 172.21.15.64:0/2526792377 conn(0x7fc0b80d3ca0 msgr2=0x7fc0b80d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:36.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0bd0df640 1 -- 172.21.15.64:0/2526792377 shutdown_connections 2024-03-04T07:34:36.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.692+0000 7fc0bd0df640 1 -- 172.21.15.64:0/2526792377 wait complete. 2024-03-04T07:34:36.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.694+0000 7fc0bd0df640 1 Processor -- start 2024-03-04T07:34:36.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.694+0000 7fc0bd0df640 1 -- start start 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0bd0df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0b8071540 0x7fc0b80aaa20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0bd0df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b80b2b50 0x7fc0b81f3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0bd0df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc0b80d3400 0x7fc0b81f3cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0bd0df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0b815e370 con 0x7fc0b80d3400 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0bd0df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0b80d4be0 con 0x7fc0b80b2b50 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0bd0df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc0b80ab4e0 con 0x7fc0b8071540 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc0b80d3400 0x7fc0b81f3cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc0b80d3400 0x7fc0b81f3cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35162/0 (socket says 172.21.15.64:35162) 2024-03-04T07:34:36.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 -- 172.21.15.64:0/3408025726 learned_addr learned my addr 172.21.15.64:0/3408025726 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b659c640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b80b2b50 0x7fc0b81f3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b6d9d640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0b8071540 0x7fc0b80aaa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 -- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0b8071540 msgr2=0x7fc0b80aaa20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0b8071540 0x7fc0b80aaa20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 -- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b80b2b50 msgr2=0x7fc0b81f3770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b80b2b50 0x7fc0b81f3770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 -- 172.21.15.64:0/3408025726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0b80ab7c0 con 0x7fc0b80d3400 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b659c640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b80b2b50 0x7fc0b81f3770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.695+0000 7fc0b759e640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc0b80d3400 0x7fc0b81f3cb0 secure :-1 s=READY pgs=1621 cs=0 l=1 rev1=1 crypto rx=0x7fc0b000cb70 tx=0x7fc0b000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.696+0000 7fc0af7fe640 1 -- 172.21.15.64:0/3408025726 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0b0017070 con 0x7fc0b80d3400 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.696+0000 7fc0af7fe640 1 -- 172.21.15.64:0/3408025726 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc0b000ad40 con 0x7fc0b80d3400 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.696+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0b80abaa0 con 0x7fc0b80d3400 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.696+0000 7fc0af7fe640 1 -- 172.21.15.64:0/3408025726 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0b0012620 con 0x7fc0b80d3400 2024-03-04T07:34:36.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.696+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0b80ac060 con 0x7fc0b80d3400 2024-03-04T07:34:36.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.697+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc0b8068700 con 0x7fc0b80d3400 2024-03-04T07:34:36.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.697+0000 7fc0af7fe640 1 -- 172.21.15.64:0/3408025726 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc0b001a070 con 0x7fc0b80d3400 2024-03-04T07:34:36.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.698+0000 7fc0af7fe640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc09806b4d0 0x7fc09806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:36.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.698+0000 7fc0b6d9d640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc09806b4d0 0x7fc09806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:36.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.698+0000 7fc0af7fe640 1 -- 172.21.15.64:0/3408025726 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc0b008a010 con 0x7fc0b80d3400 2024-03-04T07:34:36.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.698+0000 7fc0b6d9d640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc09806b4d0 0x7fc09806d990 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7fc0a8036000 tx=0x7fc0a8008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:36.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.702+0000 7fc0af7fe640 1 -- 172.21.15.64:0/3408025726 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc0b0058800 con 0x7fc0b80d3400 2024-03-04T07:34:36.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.920+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc0b80b9a60 con 0x7fc0b80d3400 2024-03-04T07:34:36.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.921+0000 7fc0af7fe640 1 -- 172.21.15.64:0/3408025726 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fc0b005c8a0 con 0x7fc0b80d3400 2024-03-04T07:34:36.922 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:36.922 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:36.922 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.922+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc09806b4d0 msgr2=0x7fc09806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.922+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc09806b4d0 0x7fc09806d990 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7fc0a8036000 tx=0x7fc0a8008030 comp rx=0 tx=0).stop 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.922+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc0b80d3400 msgr2=0x7fc0b81f3cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.922+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc0b80d3400 0x7fc0b81f3cb0 secure :-1 s=READY pgs=1621 cs=0 l=1 rev1=1 crypto rx=0x7fc0b000cb70 tx=0x7fc0b000a430 comp rx=0 tx=0).stop 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 shutdown_connections 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc09806b4d0 0x7fc09806d990 unknown :-1 s=CLOSED pgs=728 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc0b80d3400 0x7fc0b81f3cb0 unknown :-1 s=CLOSED pgs=1621 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc0b80b2b50 0x7fc0b81f3770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:36.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 --2- 172.21.15.64:0/3408025726 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc0b8071540 0x7fc0b80aaa20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:36.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 >> 172.21.15.64:0/3408025726 conn(0x7fc0b80d3ca0 msgr2=0x7fc0b80b8da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:36.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 shutdown_connections 2024-03-04T07:34:36.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:36.923+0000 7fc0bd0df640 1 -- 172.21.15.64:0/3408025726 wait complete. 2024-03-04T07:34:37.136 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:37.136 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:38.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:38.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.251+0000 7f4f5fd55640 1 Processor -- start 2024-03-04T07:34:38.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.251+0000 7f4f5fd55640 1 -- start start 2024-03-04T07:34:38.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5fd55640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f5806ed10 0x7f4f5806af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:38.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5fd55640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4f580cc0e0 con 0x7f4f580b1020 2024-03-04T07:34:38.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5fd55640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4f580cc3c0 con 0x7f4f580cd890 2024-03-04T07:34:38.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5fd55640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f580d28b0 con 0x7f4f5806ed10 2024-03-04T07:34:38.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5d2f0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f5806ed10 0x7f4f5806af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:38.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5daf1640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4f580b1020 0x7f4f580cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59660/0 (socket says 172.21.15.64:59660) 2024-03-04T07:34:38.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5daf1640 1 -- 172.21.15.64:0/2716075132 learned_addr learned my addr 172.21.15.64:0/2716075132 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:38.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5d2f0640 1 -- 172.21.15.64:0/2716075132 >> v1:172.21.15.66:6789/0 conn(0x7f4f580cd890 legacy=0x7f4f5815d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:38.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5d2f0640 1 -- 172.21.15.64:0/2716075132 >> v1:172.21.15.64:6789/0 conn(0x7f4f580b1020 legacy=0x7f4f580cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:38.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5d2f0640 1 -- 172.21.15.64:0/2716075132 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f580cc6a0 con 0x7f4f5806ed10 2024-03-04T07:34:38.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.252+0000 7f4f5caef640 1 -- 172.21.15.64:0/2716075132 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2910288394 0 0) 0x7f4f580cc0e0 con 0x7f4f580b1020 2024-03-04T07:34:38.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.253+0000 7f4f5d2f0640 1 --2- 172.21.15.64:0/2716075132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f5806ed10 0x7f4f5806af20 secure :-1 s=READY pgs=1623 cs=0 l=1 rev1=1 crypto rx=0x7f4f480097a0 tx=0x7f4f4802b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4c90b3cfadd87d9d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:38.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.253+0000 7f4f5caef640 1 -- 172.21.15.64:0/2716075132 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f48034070 con 0x7f4f5806ed10 2024-03-04T07:34:38.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.253+0000 7f4f5caef640 1 -- 172.21.15.64:0/2716075132 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f48030610 con 0x7f4f5806ed10 2024-03-04T07:34:38.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.253+0000 7f4f5caef640 1 -- 172.21.15.64:0/2716075132 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f480309e0 con 0x7f4f5806ed10 2024-03-04T07:34:38.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.254+0000 7f4f5fd55640 1 -- 172.21.15.64:0/2716075132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f5806ed10 msgr2=0x7f4f5806af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:38.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.254+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/2716075132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f5806ed10 0x7f4f5806af20 secure :-1 s=READY pgs=1623 cs=0 l=1 rev1=1 crypto rx=0x7f4f480097a0 tx=0x7f4f4802b960 comp rx=0 tx=0).stop 2024-03-04T07:34:38.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.254+0000 7f4f5fd55640 1 -- 172.21.15.64:0/2716075132 shutdown_connections 2024-03-04T07:34:38.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.254+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/2716075132 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f5806ed10 0x7f4f5806af20 unknown :-1 s=CLOSED pgs=1623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.254+0000 7f4f5fd55640 1 -- 172.21.15.64:0/2716075132 >> 172.21.15.64:0/2716075132 conn(0x7f4f580d1a90 msgr2=0x7f4f580d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:38.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.254+0000 7f4f5fd55640 1 -- 172.21.15.64:0/2716075132 shutdown_connections 2024-03-04T07:34:38.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.254+0000 7f4f5fd55640 1 -- 172.21.15.64:0/2716075132 wait complete. 2024-03-04T07:34:38.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.255+0000 7f4f5fd55640 1 Processor -- start 2024-03-04T07:34:38.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.255+0000 7f4f5fd55640 1 -- start start 2024-03-04T07:34:38.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5fd55640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f5806ed10 0x7f4f580c9bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:38.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5fd55640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f580b1020 0x7f4f580cb420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:38.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5fd55640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f580cd890 0x7f4f580b2d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5fd55640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f580ccc60 con 0x7f4f580b1020 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5fd55640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f580cf320 con 0x7f4f5806ed10 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5fd55640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f580cdca0 con 0x7f4f580cd890 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f580b1020 0x7f4f580cb420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f580b1020 0x7f4f580cb420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35182/0 (socket says 172.21.15.64:35182) 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 -- 172.21.15.64:0/425966512 learned_addr learned my addr 172.21.15.64:0/425966512 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5daf1640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f5806ed10 0x7f4f580c9bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5e2f2640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f580cd890 0x7f4f580b2d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 -- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f580cd890 msgr2=0x7f4f580b2d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f580cd890 0x7f4f580b2d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 -- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f5806ed10 msgr2=0x7f4f580c9bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f5806ed10 0x7f4f580c9bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 -- 172.21.15.64:0/425966512 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f580cb960 con 0x7f4f580b1020 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.256+0000 7f4f5d2f0640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f580b1020 0x7f4f580cb420 secure :-1 s=READY pgs=1624 cs=0 l=1 rev1=1 crypto rx=0x7f4f48009890 tx=0x7f4f480046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:38.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.257+0000 7f4f4e7fc640 1 -- 172.21.15.64:0/425966512 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f48034040 con 0x7f4f580b1020 2024-03-04T07:34:38.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.257+0000 7f4f4e7fc640 1 -- 172.21.15.64:0/425966512 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f48039a10 con 0x7f4f580b1020 2024-03-04T07:34:38.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.257+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f580ca1b0 con 0x7f4f580b1020 2024-03-04T07:34:38.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.257+0000 7f4f4e7fc640 1 -- 172.21.15.64:0/425966512 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f48039de0 con 0x7f4f580b1020 2024-03-04T07:34:38.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.257+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f580ca770 con 0x7f4f580b1020 2024-03-04T07:34:38.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.258+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f58067ff0 con 0x7f4f580b1020 2024-03-04T07:34:38.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.258+0000 7f4f4e7fc640 1 -- 172.21.15.64:0/425966512 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4f48037070 con 0x7f4f580b1020 2024-03-04T07:34:38.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.259+0000 7f4f4e7fc640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f3c06b4d0 0x7f4f3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:38.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.259+0000 7f4f4e7fc640 1 -- 172.21.15.64:0/425966512 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4f480ac210 con 0x7f4f580b1020 2024-03-04T07:34:38.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.259+0000 7f4f5daf1640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f3c06b4d0 0x7f4f3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:38.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.259+0000 7f4f5daf1640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f3c06b4d0 0x7f4f3c06d990 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f4f54007f70 tx=0x7f4f54007f00 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:38.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.263+0000 7f4f4e7fc640 1 -- 172.21.15.64:0/425966512 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4f4807aa00 con 0x7f4f580b1020 2024-03-04T07:34:38.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.484+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4f581614f0 con 0x7f4f580b1020 2024-03-04T07:34:38.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.484+0000 7f4f4e7fc640 1 -- 172.21.15.64:0/425966512 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4f4807eaa0 con 0x7f4f580b1020 2024-03-04T07:34:38.485 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:38.485 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:38.486 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f3c06b4d0 msgr2=0x7f4f3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f3c06b4d0 0x7f4f3c06d990 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f4f54007f70 tx=0x7f4f54007f00 comp rx=0 tx=0).stop 2024-03-04T07:34:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f580b1020 msgr2=0x7f4f580cb420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f580b1020 0x7f4f580cb420 secure :-1 s=READY pgs=1624 cs=0 l=1 rev1=1 crypto rx=0x7f4f48009890 tx=0x7f4f480046c0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 shutdown_connections 2024-03-04T07:34:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f3c06b4d0 0x7f4f3c06d990 unknown :-1 s=CLOSED pgs=729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f580cd890 0x7f4f580b2d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f580b1020 0x7f4f580cb420 unknown :-1 s=CLOSED pgs=1624 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 --2- 172.21.15.64:0/425966512 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f5806ed10 0x7f4f580c9bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:38.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 >> 172.21.15.64:0/425966512 conn(0x7f4f580d1a90 msgr2=0x7f4f5806f5e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:38.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 shutdown_connections 2024-03-04T07:34:38.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:38.486+0000 7f4f5fd55640 1 -- 172.21.15.64:0/425966512 wait complete. 2024-03-04T07:34:38.553 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:38.699 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:38.699 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:39.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.814+0000 7f4df15a0640 1 Processor -- start 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.814+0000 7f4df15a0640 1 -- start start 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4df15a0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4dec15dd20 0x7f4dec0d1bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4df15a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0d0a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4df15a0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4dec0d1390 con 0x7f4dec0d0f70 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4df15a0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4dec0d3190 con 0x7f4dec15dd20 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4df15a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4dec15e100 con 0x7f4dec0d2100 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4df0d9f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4dec0d0f70 0x7f4dec0d3a00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59670/0 (socket says 172.21.15.64:59670) 2024-03-04T07:34:39.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4df0d9f640 1 -- 172.21.15.64:0/2452454843 learned_addr learned my addr 172.21.15.64:0/2452454843 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4debfff640 1 --2- 172.21.15.64:0/2452454843 >> v2:172.21.15.66:3300/0 conn(0x7f4dec15dd20 0x7f4dec0d1bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4deb7fe640 1 --2- 172.21.15.64:0/2452454843 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0d0a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4deaffd640 1 -- 172.21.15.64:0/2452454843 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3694069592 0 0) 0x7f4dec0d1390 con 0x7f4dec0d0f70 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4deaffd640 1 -- 172.21.15.64:0/2452454843 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4dd8003520 con 0x7f4dec0d0f70 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4deb7fe640 1 -- 172.21.15.64:0/2452454843 >> v2:172.21.15.66:3300/0 conn(0x7f4dec15dd20 msgr2=0x7f4dec0d1bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4deb7fe640 1 --2- 172.21.15.64:0/2452454843 >> v2:172.21.15.66:3300/0 conn(0x7f4dec15dd20 0x7f4dec0d1bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4deb7fe640 1 -- 172.21.15.64:0/2452454843 >> v1:172.21.15.64:6789/0 conn(0x7f4dec0d0f70 legacy=0x7f4dec0d3a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:39.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4deb7fe640 1 -- 172.21.15.64:0/2452454843 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4dec0b7c50 con 0x7f4dec0d2100 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.815+0000 7f4debfff640 1 --2- 172.21.15.64:0/2452454843 >> v2:172.21.15.66:3300/0 conn(0x7f4dec15dd20 0x7f4dec0d1bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.816+0000 7f4deb7fe640 1 --2- 172.21.15.64:0/2452454843 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0d0a30 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f4dd40097a0 tx=0x7f4dd402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1df76df4c6ca8836 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.816+0000 7f4deaffd640 1 -- 172.21.15.64:0/2452454843 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd4035020 con 0x7f4dec0d2100 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.816+0000 7f4deaffd640 1 -- 172.21.15.64:0/2452454843 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4dd4031830 con 0x7f4dec0d2100 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.816+0000 7f4deaffd640 1 -- 172.21.15.64:0/2452454843 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd4031c00 con 0x7f4dec0d2100 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 -- 172.21.15.64:0/2452454843 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 msgr2=0x7f4dec0d0a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2452454843 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0d0a30 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f4dd40097a0 tx=0x7f4dd402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 -- 172.21.15.64:0/2452454843 shutdown_connections 2024-03-04T07:34:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2452454843 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0d0a30 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2452454843 >> v2:172.21.15.66:3300/0 conn(0x7f4dec15dd20 0x7f4dec0d1bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 -- 172.21.15.64:0/2452454843 >> 172.21.15.64:0/2452454843 conn(0x7f4dec071960 msgr2=0x7f4dec071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 -- 172.21.15.64:0/2452454843 shutdown_connections 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.817+0000 7f4df15a0640 1 -- 172.21.15.64:0/2452454843 wait complete. 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.818+0000 7f4df15a0640 1 Processor -- start 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.818+0000 7f4df15a0640 1 -- start start 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df15a0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4dec0d0f70 0x7f4dec0c8b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df15a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0ccd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df15a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4dec15dd20 0x7f4dec0cd280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df15a0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dec0d2b40 con 0x7f4dec15dd20 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df15a0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dec0d2860 con 0x7f4dec0d0f70 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df15a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4dec0d2e20 con 0x7f4dec0d2100 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4dec15dd20 0x7f4dec0cd280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:39.827 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4dec15dd20 0x7f4dec0cd280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35186/0 (socket says 172.21.15.64:35186) 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 -- 172.21.15.64:0/2700957631 learned_addr learned my addr 172.21.15.64:0/2700957631 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4deb7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0ccd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4debfff640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4dec0d0f70 0x7f4dec0c8b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 -- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 msgr2=0x7f4dec0ccd40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0ccd40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 -- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4dec0d0f70 msgr2=0x7f4dec0c8b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4dec0d0f70 0x7f4dec0c8b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4df0d9f640 1 -- 172.21.15.64:0/2700957631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4dec0c7ad0 con 0x7f4dec15dd20 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4deb7fe640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0ccd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.819+0000 7f4debfff640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4dec0d0f70 0x7f4dec0c8b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.820+0000 7f4df0d9f640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4dec15dd20 0x7f4dec0cd280 secure :-1 s=READY pgs=1626 cs=0 l=1 rev1=1 crypto rx=0x7f4de400cb40 tx=0x7f4de400a060 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:39.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.820+0000 7f4de8ff9640 1 -- 172.21.15.64:0/2700957631 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4de4017070 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.820+0000 7f4de8ff9640 1 -- 172.21.15.64:0/2700957631 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4de400ac40 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.820+0000 7f4de8ff9640 1 -- 172.21.15.64:0/2700957631 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4de4012620 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.820+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4dec0c7db0 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.820+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4dec0c8370 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.821+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4dec04f330 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.821+0000 7f4de8ff9640 1 -- 172.21.15.64:0/2700957631 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4de401a070 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.822+0000 7f4de8ff9640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4dcc06b4d0 0x7f4dcc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.822+0000 7f4debfff640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4dcc06b4d0 0x7f4dcc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.822+0000 7f4de8ff9640 1 -- 172.21.15.64:0/2700957631 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4de408a110 con 0x7f4dec15dd20 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.822+0000 7f4debfff640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4dcc06b4d0 0x7f4dcc06d990 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f4ddc009560 tx=0x7f4ddc009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:39.826+0000 7f4de8ff9640 1 -- 172.21.15.64:0/2700957631 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4de40587d0 con 0x7f4dec15dd20 2024-03-04T07:34:40.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.052+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4dec0b4070 con 0x7f4dec15dd20 2024-03-04T07:34:40.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.052+0000 7f4de8ff9640 1 -- 172.21.15.64:0/2700957631 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4de405c870 con 0x7f4dec15dd20 2024-03-04T07:34:40.054 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:40.054 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:40.055 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4dcc06b4d0 msgr2=0x7f4dcc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4dcc06b4d0 0x7f4dcc06d990 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f4ddc009560 tx=0x7f4ddc009190 comp rx=0 tx=0).stop 2024-03-04T07:34:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4dec15dd20 msgr2=0x7f4dec0cd280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4dec15dd20 0x7f4dec0cd280 secure :-1 s=READY pgs=1626 cs=0 l=1 rev1=1 crypto rx=0x7f4de400cb40 tx=0x7f4de400a060 comp rx=0 tx=0).stop 2024-03-04T07:34:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 shutdown_connections 2024-03-04T07:34:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4dcc06b4d0 0x7f4dcc06d990 unknown :-1 s=CLOSED pgs=730 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4dec15dd20 0x7f4dec0cd280 unknown :-1 s=CLOSED pgs=1626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4dec0d2100 0x7f4dec0ccd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 --2- 172.21.15.64:0/2700957631 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4dec0d0f70 0x7f4dec0c8b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 >> 172.21.15.64:0/2700957631 conn(0x7f4dec071960 msgr2=0x7f4dec0b7e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 shutdown_connections 2024-03-04T07:34:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:40.055+0000 7f4df15a0640 1 -- 172.21.15.64:0/2700957631 wait complete. 2024-03-04T07:34:40.268 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:40.269 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:40.555 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:41.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:41.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.383+0000 7f35d47c7640 1 Processor -- start 2024-03-04T07:34:41.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.383+0000 7f35d47c7640 1 -- start start 2024-03-04T07:34:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d47c7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f35cc157b80 0x7f35cc15b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d47c7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f35cc157f60 0x7f35cc15b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d47c7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f35cc06c7b0 con 0x7f35cc0cbab0 2024-03-04T07:34:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d47c7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35cc06ca90 con 0x7f35cc157f60 2024-03-04T07:34:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d47c7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f35cc06cd70 con 0x7f35cc157b80 2024-03-04T07:34:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d1d62640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f35cc157f60 0x7f35cc15b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:41.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d1d62640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f35cc157f60 0x7f35cc15b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35188/0 (socket says 172.21.15.64:35188) 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d1d62640 1 -- 172.21.15.64:0/1793993721 learned_addr learned my addr 172.21.15.64:0/1793993721 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d2563640 1 --2- 172.21.15.64:0/1793993721 >> v2:172.21.15.66:3300/0 conn(0x7f35cc157b80 0x7f35cc15b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d1d62640 1 -- 172.21.15.64:0/1793993721 >> v2:172.21.15.66:3300/0 conn(0x7f35cc157b80 msgr2=0x7f35cc15b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d1d62640 1 --2- 172.21.15.64:0/1793993721 >> v2:172.21.15.66:3300/0 conn(0x7f35cc157b80 0x7f35cc15b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d1d62640 1 -- 172.21.15.64:0/1793993721 >> v1:172.21.15.66:6789/0 conn(0x7f35cc0cbab0 legacy=0x7f35cc070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.384+0000 7f35d1d62640 1 -- 172.21.15.64:0/1793993721 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35cc06d050 con 0x7f35cc157f60 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.385+0000 7f35d1d62640 1 --2- 172.21.15.64:0/1793993721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc157f60 0x7f35cc15b9a0 secure :-1 s=READY pgs=1627 cs=0 l=1 rev1=1 crypto rx=0x7f35cc06ed60 tx=0x7f35bc02e170 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dfdce232bd9cf1dc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.385+0000 7f35d1561640 1 -- 172.21.15.64:0/1793993721 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35bc03b070 con 0x7f35cc157f60 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.385+0000 7f35d1561640 1 -- 172.21.15.64:0/1793993721 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35bc036440 con 0x7f35cc157f60 2024-03-04T07:34:41.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.385+0000 7f35d1561640 1 -- 172.21.15.64:0/1793993721 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35bc036810 con 0x7f35cc157f60 2024-03-04T07:34:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 -- 172.21.15.64:0/1793993721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc157f60 msgr2=0x7f35cc15b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1793993721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc157f60 0x7f35cc15b9a0 secure :-1 s=READY pgs=1627 cs=0 l=1 rev1=1 crypto rx=0x7f35cc06ed60 tx=0x7f35bc02e170 comp rx=0 tx=0).stop 2024-03-04T07:34:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 -- 172.21.15.64:0/1793993721 shutdown_connections 2024-03-04T07:34:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1793993721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc157f60 0x7f35cc15b9a0 unknown :-1 s=CLOSED pgs=1627 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1793993721 >> v2:172.21.15.66:3300/0 conn(0x7f35cc157b80 0x7f35cc15b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 -- 172.21.15.64:0/1793993721 >> 172.21.15.64:0/1793993721 conn(0x7f35cc066d00 msgr2=0x7f35cc0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 -- 172.21.15.64:0/1793993721 shutdown_connections 2024-03-04T07:34:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.386+0000 7f35d47c7640 1 -- 172.21.15.64:0/1793993721 wait complete. 2024-03-04T07:34:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.387+0000 7f35d47c7640 1 Processor -- start 2024-03-04T07:34:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.387+0000 7f35d47c7640 1 -- start start 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d47c7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc0cbab0 0x7f35cc14cb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d47c7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35cc157b80 0x7f35cc0b38f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d47c7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35cc157f60 0x7f35cc151bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d47c7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35cc0cc1d0 con 0x7f35cc0cbab0 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d47c7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35cc0cc050 con 0x7f35cc157f60 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d47c7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f35cc071250 con 0x7f35cc157b80 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d2563640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc0cbab0 0x7f35cc14cb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d2563640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc0cbab0 0x7f35cc14cb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35198/0 (socket says 172.21.15.64:35198) 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d2563640 1 -- 172.21.15.64:0/1020794431 learned_addr learned my addr 172.21.15.64:0/1020794431 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d1d62640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35cc157b80 0x7f35cc0b38f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.388+0000 7f35d2d64640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35cc157f60 0x7f35cc151bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d2563640 1 -- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35cc157b80 msgr2=0x7f35cc0b38f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d2563640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35cc157b80 0x7f35cc0b38f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d2563640 1 -- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35cc157f60 msgr2=0x7f35cc151bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d2563640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35cc157f60 0x7f35cc151bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d2563640 1 -- 172.21.15.64:0/1020794431 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35cc0afe90 con 0x7f35cc0cbab0 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d2563640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc0cbab0 0x7f35cc14cb80 secure :-1 s=READY pgs=1628 cs=0 l=1 rev1=1 crypto rx=0x7f35c800e9b0 tx=0x7f35c800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35c2ffd640 1 -- 172.21.15.64:0/1020794431 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35c8017070 con 0x7f35cc0cbab0 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35c2ffd640 1 -- 172.21.15.64:0/1020794431 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f35c800c9a0 con 0x7f35cc0cbab0 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35cc0b0170 con 0x7f35cc0cbab0 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35c2ffd640 1 -- 172.21.15.64:0/1020794431 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f35c800cd90 con 0x7f35cc0cbab0 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.389+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35cc14d0c0 con 0x7f35cc0cbab0 2024-03-04T07:34:41.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.390+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f35cc0aef10 con 0x7f35cc0cbab0 2024-03-04T07:34:41.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.390+0000 7f35c2ffd640 1 -- 172.21.15.64:0/1020794431 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f35c801a070 con 0x7f35cc0cbab0 2024-03-04T07:34:41.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.391+0000 7f35c2ffd640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35b006b4d0 0x7f35b006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:41.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.391+0000 7f35c2ffd640 1 -- 172.21.15.64:0/1020794431 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f35c808a4e0 con 0x7f35cc0cbab0 2024-03-04T07:34:41.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.391+0000 7f35d1d62640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35b006b4d0 0x7f35b006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:41.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.391+0000 7f35d1d62640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35b006b4d0 0x7f35b006d990 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f35bc038000 tx=0x7f35bc032000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:41.396 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.395+0000 7f35c2ffd640 1 -- 172.21.15.64:0/1020794431 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f35cc0aef10 con 0x7f35cc0cbab0 2024-03-04T07:34:41.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.619+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f35cc067ff0 con 0x7f35cc0cbab0 2024-03-04T07:34:41.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.619+0000 7f35c2ffd640 1 -- 172.21.15.64:0/1020794431 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f35c8058d60 con 0x7f35cc0cbab0 2024-03-04T07:34:41.621 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:41.621 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:41.621 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:41.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35b006b4d0 msgr2=0x7f35b006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:41.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35b006b4d0 0x7f35b006d990 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f35bc038000 tx=0x7f35bc032000 comp rx=0 tx=0).stop 2024-03-04T07:34:41.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc0cbab0 msgr2=0x7f35cc14cb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:41.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc0cbab0 0x7f35cc14cb80 secure :-1 s=READY pgs=1628 cs=0 l=1 rev1=1 crypto rx=0x7f35c800e9b0 tx=0x7f35c800ee70 comp rx=0 tx=0).stop 2024-03-04T07:34:41.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 shutdown_connections 2024-03-04T07:34:41.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f35b006b4d0 0x7f35b006d990 unknown :-1 s=CLOSED pgs=731 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f35cc157f60 0x7f35cc151bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f35cc157b80 0x7f35cc0b38f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 --2- 172.21.15.64:0/1020794431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f35cc0cbab0 0x7f35cc14cb80 unknown :-1 s=CLOSED pgs=1628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:41.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 >> 172.21.15.64:0/1020794431 conn(0x7f35cc066d00 msgr2=0x7f35cc154870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:41.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 shutdown_connections 2024-03-04T07:34:41.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:41.622+0000 7f35d47c7640 1 -- 172.21.15.64:0/1020794431 wait complete. 2024-03-04T07:34:41.835 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:41.835 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:42.558 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:42.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:42.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.949+0000 7f56e7684640 1 Processor -- start 2024-03-04T07:34:42.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.950+0000 7f56e7684640 1 -- start start 2024-03-04T07:34:42.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.950+0000 7f56e7684640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:42.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.950+0000 7f56e7684640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f56e00cea30 con 0x7f56e00b9a30 2024-03-04T07:34:42.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.950+0000 7f56e7684640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f56e00d2af0 con 0x7f56e00b95a0 2024-03-04T07:34:42.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.950+0000 7f56e7684640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f56e00cf3c0 con 0x7f56e015df00 2024-03-04T07:34:42.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56e5c21640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f56e00b9a30 0x7f56e00d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59678/0 (socket says 172.21.15.64:59678) 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56e5c21640 1 -- 172.21.15.64:0/1943721904 learned_addr learned my addr 172.21.15.64:0/1943721904 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56e5420640 1 --2- 172.21.15.64:0/1943721904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56dffff640 1 -- 172.21.15.64:0/1943721904 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3749037901 0 0) 0x7f56e00d2af0 con 0x7f56e00b95a0 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56dffff640 1 -- 172.21.15.64:0/1943721904 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f56c8003520 con 0x7f56e00b95a0 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56dffff640 1 -- 172.21.15.64:0/1943721904 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4232699599 0 0) 0x7f56e00cea30 con 0x7f56e00b9a30 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56dffff640 1 -- 172.21.15.64:0/1943721904 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f56e00d2af0 con 0x7f56e00b9a30 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56e5420640 1 -- 172.21.15.64:0/1943721904 >> v1:172.21.15.66:6789/0 conn(0x7f56e00b95a0 legacy=0x7f56e00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56e5420640 1 -- 172.21.15.64:0/1943721904 >> v1:172.21.15.64:6789/0 conn(0x7f56e00b9a30 legacy=0x7f56e00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:42.953 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.951+0000 7f56e5420640 1 -- 172.21.15.64:0/1943721904 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56e00cf0e0 con 0x7f56e015df00 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.952+0000 7f56e5420640 1 --2- 172.21.15.64:0/1943721904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00d25b0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f56d8009dc0 tx=0x7f56d802bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dcf2dfd826c5398 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.952+0000 7f56dffff640 1 -- 172.21.15.64:0/1943721904 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56d8039070 con 0x7f56e015df00 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.952+0000 7f56dffff640 1 -- 172.21.15.64:0/1943721904 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f56d802ccd0 con 0x7f56e015df00 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.952+0000 7f56dffff640 1 -- 172.21.15.64:0/1943721904 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56d8034640 con 0x7f56e015df00 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.953+0000 7f56e7684640 1 -- 172.21.15.64:0/1943721904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 msgr2=0x7f56e00d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.953+0000 7f56e7684640 1 --2- 172.21.15.64:0/1943721904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00d25b0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f56d8009dc0 tx=0x7f56d802bf20 comp rx=0 tx=0).stop 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.953+0000 7f56e7684640 1 -- 172.21.15.64:0/1943721904 shutdown_connections 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.953+0000 7f56e7684640 1 --2- 172.21.15.64:0/1943721904 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00d25b0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.953+0000 7f56e7684640 1 -- 172.21.15.64:0/1943721904 >> 172.21.15.64:0/1943721904 conn(0x7f56e0071960 msgr2=0x7f56e0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.953+0000 7f56e7684640 1 -- 172.21.15.64:0/1943721904 shutdown_connections 2024-03-04T07:34:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.953+0000 7f56e7684640 1 -- 172.21.15.64:0/1943721904 wait complete. 2024-03-04T07:34:42.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.954+0000 7f56e7684640 1 Processor -- start 2024-03-04T07:34:42.956 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.954+0000 7f56e7684640 1 -- start start 2024-03-04T07:34:42.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e7684640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56e00b95a0 0x7f56e01490e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:42.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e7684640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00cc1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:42.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e7684640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56e0156660 0x7f56e015e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:42.958 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e7684640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56e00cec70 con 0x7f56e0156660 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e7684640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56e00ce750 con 0x7f56e00b95a0 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e7684640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f56e00cedf0 con 0x7f56e015df00 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56e0156660 0x7f56e015e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56e0156660 0x7f56e015e2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:35204/0 (socket says 172.21.15.64:35204) 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 -- 172.21.15.64:0/467732473 learned_addr learned my addr 172.21.15.64:0/467732473 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5420640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56e00b95a0 0x7f56e01490e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e4c1f640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00cc1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 -- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 msgr2=0x7f56e00cc1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00cc1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 -- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56e00b95a0 msgr2=0x7f56e01490e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56e00b95a0 0x7f56e01490e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 -- 172.21.15.64:0/467732473 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56e00cc710 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56e5c21640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56e0156660 0x7f56e015e2e0 secure :-1 s=READY pgs=1630 cs=0 l=1 rev1=1 crypto rx=0x7f56d000c3c0 tx=0x7f56d000c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56ddffb640 1 -- 172.21.15.64:0/467732473 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56d0017070 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.955+0000 7f56ddffb640 1 -- 172.21.15.64:0/467732473 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f56d0005ce0 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.956+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56e00cb130 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.956+0000 7f56ddffb640 1 -- 172.21.15.64:0/467732473 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56d0007310 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.956+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56e00cb6f0 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.957+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56e0067ff0 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.957+0000 7f56ddffb640 1 -- 172.21.15.64:0/467732473 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f56d001a070 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.957+0000 7f56ddffb640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56cc06b4d0 0x7f56cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.957+0000 7f56ddffb640 1 -- 172.21.15.64:0/467732473 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f56d00895e0 con 0x7f56e0156660 2024-03-04T07:34:42.960 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.957+0000 7f56e5420640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56cc06b4d0 0x7f56cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:42.961 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.958+0000 7f56e5420640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56cc06b4d0 0x7f56cc06d990 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f56d8036000 tx=0x7f56d8030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:42.962 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:42.962+0000 7f56ddffb640 1 -- 172.21.15.64:0/467732473 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f56d0057d40 con 0x7f56e0156660 2024-03-04T07:34:43.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.180+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f56e00d3f40 con 0x7f56e0156660 2024-03-04T07:34:43.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.181+0000 7f56ddffb640 1 -- 172.21.15.64:0/467732473 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f56d005bde0 con 0x7f56e0156660 2024-03-04T07:34:43.182 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:43.182 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:43.182 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56cc06b4d0 msgr2=0x7f56cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56cc06b4d0 0x7f56cc06d990 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f56d8036000 tx=0x7f56d8030000 comp rx=0 tx=0).stop 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56e0156660 msgr2=0x7f56e015e2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56e0156660 0x7f56e015e2e0 secure :-1 s=READY pgs=1630 cs=0 l=1 rev1=1 crypto rx=0x7f56d000c3c0 tx=0x7f56d000c880 comp rx=0 tx=0).stop 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 shutdown_connections 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f56cc06b4d0 0x7f56cc06d990 unknown :-1 s=CLOSED pgs=732 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f56e0156660 0x7f56e015e2e0 unknown :-1 s=CLOSED pgs=1630 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f56e015df00 0x7f56e00cc1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 --2- 172.21.15.64:0/467732473 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f56e00b95a0 0x7f56e01490e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:43.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 >> 172.21.15.64:0/467732473 conn(0x7f56e0071960 msgr2=0x7f56e015d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:43.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 shutdown_connections 2024-03-04T07:34:43.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:43.183+0000 7f56e7684640 1 -- 172.21.15.64:0/467732473 wait complete. 2024-03-04T07:34:43.397 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:43.397 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:44.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:44.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.512+0000 7fc3dc1dd640 1 Processor -- start 2024-03-04T07:34:44.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.512+0000 7fc3dc1dd640 1 -- start start 2024-03-04T07:34:44.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.513+0000 7fc3dc1dd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc3d415df70 0x7fc3d40d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.513+0000 7fc3dc1dd640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc3d40b6f20 con 0x7fc3d40d18c0 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.513+0000 7fc3dc1dd640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc3d40d3030 con 0x7fc3d4071c50 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.513+0000 7fc3dc1dd640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc3d40d3310 con 0x7fc3d415df70 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.513+0000 7fc3d9f79640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc3d415df70 0x7fc3d40d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.513+0000 7fc3d9778640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc3d40d18c0 0x7fc3d4071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54136/0 (socket says 172.21.15.64:54136) 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.513+0000 7fc3d9778640 1 -- 172.21.15.64:0/2765819142 learned_addr learned my addr 172.21.15.64:0/2765819142 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.514+0000 7fc3d9f79640 1 -- 172.21.15.64:0/2765819142 >> v1:172.21.15.66:6789/0 conn(0x7fc3d4071c50 legacy=0x7fc3d40b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.514+0000 7fc3d9f79640 1 -- 172.21.15.64:0/2765819142 >> v1:172.21.15.64:6789/0 conn(0x7fc3d40d18c0 legacy=0x7fc3d4071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.514+0000 7fc3d8f77640 1 -- 172.21.15.64:0/2765819142 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2577756137 0 0) 0x7fc3d40d3030 con 0x7fc3d4071c50 2024-03-04T07:34:44.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.514+0000 7fc3d9f79640 1 -- 172.21.15.64:0/2765819142 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3d40d35f0 con 0x7fc3d415df70 2024-03-04T07:34:44.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.514+0000 7fc3d9f79640 1 --2- 172.21.15.64:0/2765819142 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d415df70 0x7fc3d40d27b0 secure :-1 s=READY pgs=1632 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc0097a0 tx=0x7fc3cc02b820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6e3d824957a848d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:44.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3d8f77640 1 -- 172.21.15.64:0/2765819142 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3cc038070 con 0x7fc3d415df70 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3d8f77640 1 -- 172.21.15.64:0/2765819142 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3cc031700 con 0x7fc3d415df70 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3d8f77640 1 -- 172.21.15.64:0/2765819142 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3cc031ad0 con 0x7fc3d415df70 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/2765819142 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d415df70 msgr2=0x7fc3d40d27b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/2765819142 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d415df70 0x7fc3d40d27b0 secure :-1 s=READY pgs=1632 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc0097a0 tx=0x7fc3cc02b820 comp rx=0 tx=0).stop 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/2765819142 shutdown_connections 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/2765819142 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d415df70 0x7fc3d40d27b0 unknown :-1 s=CLOSED pgs=1632 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/2765819142 >> 172.21.15.64:0/2765819142 conn(0x7fc3d40d3cf0 msgr2=0x7fc3d40d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/2765819142 shutdown_connections 2024-03-04T07:34:44.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.515+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/2765819142 wait complete. 2024-03-04T07:34:44.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 Processor -- start 2024-03-04T07:34:44.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 -- start start 2024-03-04T07:34:44.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3d4071c50 0x7fc3d40c8810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:44.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d40d18c0 0x7fc3d40c9e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:44.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3d415df70 0x7fc3d4157f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3d415e350 con 0x7fc3d40d18c0 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3d40d2cf0 con 0x7fc3d4071c50 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3dc1dd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc3d415e4d0 con 0x7fc3d415df70 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3d9778640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d40d18c0 0x7fc3d40c9e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3d9778640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d40d18c0 0x7fc3d40c9e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47016/0 (socket says 172.21.15.64:47016) 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3d9778640 1 -- 172.21.15.64:0/3939376659 learned_addr learned my addr 172.21.15.64:0/3939376659 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3d9f79640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3d4071c50 0x7fc3d40c8810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.517+0000 7fc3da77a640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3d415df70 0x7fc3d4157f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d9778640 1 -- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3d415df70 msgr2=0x7fc3d4157f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d9778640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3d415df70 0x7fc3d4157f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d9778640 1 -- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3d4071c50 msgr2=0x7fc3d40c8810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d9778640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3d4071c50 0x7fc3d40c8810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d9778640 1 -- 172.21.15.64:0/3939376659 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3d40c8df0 con 0x7fc3d40d18c0 2024-03-04T07:34:44.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d9778640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d40d18c0 0x7fc3d40c9e90 secure :-1 s=READY pgs=1633 cs=0 l=1 rev1=1 crypto rx=0x7fc3c800c780 tx=0x7fc3c800cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:44.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d27fc640 1 -- 172.21.15.64:0/3939376659 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3c8018070 con 0x7fc3d40d18c0 2024-03-04T07:34:44.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d27fc640 1 -- 172.21.15.64:0/3939376659 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3c80072d0 con 0x7fc3d40d18c0 2024-03-04T07:34:44.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3d40c9130 con 0x7fc3d40d18c0 2024-03-04T07:34:44.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3d27fc640 1 -- 172.21.15.64:0/3939376659 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3c80047e0 con 0x7fc3d40d18c0 2024-03-04T07:34:44.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.518+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3d40c9970 con 0x7fc3d40d18c0 2024-03-04T07:34:44.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.519+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc3d4067ff0 con 0x7fc3d40d18c0 2024-03-04T07:34:44.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.520+0000 7fc3d27fc640 1 -- 172.21.15.64:0/3939376659 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc3c801b070 con 0x7fc3d40d18c0 2024-03-04T07:34:44.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.520+0000 7fc3d27fc640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3c006b4d0 0x7fc3c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:44.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.520+0000 7fc3d27fc640 1 -- 172.21.15.64:0/3939376659 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc3c80895b0 con 0x7fc3d40d18c0 2024-03-04T07:34:44.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.520+0000 7fc3d9f79640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3c006b4d0 0x7fc3c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:44.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.521+0000 7fc3d9f79640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3c006b4d0 0x7fc3c006d990 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc035000 tx=0x7fc3cc008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:44.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.525+0000 7fc3d27fc640 1 -- 172.21.15.64:0/3939376659 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc3d40c9130 con 0x7fc3d40d18c0 2024-03-04T07:34:44.559 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:44.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.744+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc3d40c9310 con 0x7fc3d40d18c0 2024-03-04T07:34:44.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.744+0000 7fc3d27fc640 1 -- 172.21.15.64:0/3939376659 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fc3c8006e20 con 0x7fc3d40d18c0 2024-03-04T07:34:44.744 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:44.744 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:44.745 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:44.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3c006b4d0 msgr2=0x7fc3c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:44.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3c006b4d0 0x7fc3c006d990 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc035000 tx=0x7fc3cc008030 comp rx=0 tx=0).stop 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d40d18c0 msgr2=0x7fc3d40c9e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d40d18c0 0x7fc3d40c9e90 secure :-1 s=READY pgs=1633 cs=0 l=1 rev1=1 crypto rx=0x7fc3c800c780 tx=0x7fc3c800cc40 comp rx=0 tx=0).stop 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 shutdown_connections 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3c006b4d0 0x7fc3c006d990 unknown :-1 s=CLOSED pgs=733 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3d415df70 0x7fc3d4157f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3d40d18c0 0x7fc3d40c9e90 unknown :-1 s=CLOSED pgs=1633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 --2- 172.21.15.64:0/3939376659 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc3d4071c50 0x7fc3d40c8810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 >> 172.21.15.64:0/3939376659 conn(0x7fc3d40d3cf0 msgr2=0x7fc3d415fe40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 shutdown_connections 2024-03-04T07:34:44.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:44.746+0000 7fc3dc1dd640 1 -- 172.21.15.64:0/3939376659 wait complete. 2024-03-04T07:34:44.958 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:44.959 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:45.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:46.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.075+0000 7f395f85c640 1 Processor -- start 2024-03-04T07:34:46.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.075+0000 7f395f85c640 1 -- start start 2024-03-04T07:34:46.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395f85c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f39580d3400 0x7f39580b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:46.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395f85c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39580b2b50 0x7f39580b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395f85c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f39580ce610 con 0x7f3958071540 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395f85c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f39580d2f70 con 0x7f39580d3400 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395f85c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39580d4be0 con 0x7f39580b2b50 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395d5f8640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3958071540 0x7f3958071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54152/0 (socket says 172.21.15.64:54152) 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395d5f8640 1 -- 172.21.15.64:0/4273956658 learned_addr learned my addr 172.21.15.64:0/4273956658 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395ddf9640 1 --2- 172.21.15.64:0/4273956658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39580b2b50 0x7f39580b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395cdf7640 1 --2- 172.21.15.64:0/4273956658 >> v2:172.21.15.66:3300/0 conn(0x7f39580d3400 0x7f39580b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f394ffff640 1 -- 172.21.15.64:0/4273956658 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 296798670 0 0) 0x7f39580ce610 con 0x7f3958071540 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f394ffff640 1 -- 172.21.15.64:0/4273956658 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3944003520 con 0x7f3958071540 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395ddf9640 1 -- 172.21.15.64:0/4273956658 >> v2:172.21.15.66:3300/0 conn(0x7f39580d3400 msgr2=0x7f39580b30a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395ddf9640 1 --2- 172.21.15.64:0/4273956658 >> v2:172.21.15.66:3300/0 conn(0x7f39580d3400 0x7f39580b30a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395ddf9640 1 -- 172.21.15.64:0/4273956658 >> v1:172.21.15.64:6789/0 conn(0x7f3958071540 legacy=0x7f3958071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:46.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.076+0000 7f395ddf9640 1 -- 172.21.15.64:0/4273956658 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39580d2920 con 0x7f39580b2b50 2024-03-04T07:34:46.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.077+0000 7f395cdf7640 1 --2- 172.21.15.64:0/4273956658 >> v2:172.21.15.66:3300/0 conn(0x7f39580d3400 0x7f39580b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.077+0000 7f395ddf9640 1 --2- 172.21.15.64:0/4273956658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39580b2b50 0x7f39580b6e20 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f3950009d90 tx=0x7f395002bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6edc8d736824b7d9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.077+0000 7f394ffff640 1 -- 172.21.15.64:0/4273956658 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3950039070 con 0x7f39580b2b50 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f394ffff640 1 -- 172.21.15.64:0/4273956658 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f395002cc50 con 0x7f39580b2b50 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f394ffff640 1 -- 172.21.15.64:0/4273956658 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3950034440 con 0x7f39580b2b50 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 -- 172.21.15.64:0/4273956658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39580b2b50 msgr2=0x7f39580b6e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 --2- 172.21.15.64:0/4273956658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39580b2b50 0x7f39580b6e20 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f3950009d90 tx=0x7f395002bea0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 -- 172.21.15.64:0/4273956658 shutdown_connections 2024-03-04T07:34:46.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 --2- 172.21.15.64:0/4273956658 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f39580b2b50 0x7f39580b6e20 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 --2- 172.21.15.64:0/4273956658 >> v2:172.21.15.66:3300/0 conn(0x7f39580d3400 0x7f39580b30a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 -- 172.21.15.64:0/4273956658 >> 172.21.15.64:0/4273956658 conn(0x7f39580d3ca0 msgr2=0x7f39580d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:46.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 -- 172.21.15.64:0/4273956658 shutdown_connections 2024-03-04T07:34:46.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.078+0000 7f395f85c640 1 -- 172.21.15.64:0/4273956658 wait complete. 2024-03-04T07:34:46.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.079+0000 7f395f85c640 1 Processor -- start 2024-03-04T07:34:46.080 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.079+0000 7f395f85c640 1 -- start start 2024-03-04T07:34:46.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395f85c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3958071540 0x7f3958159b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:46.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395f85c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39580b2b50 0x7f39580b4d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:46.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395f85c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39580d3400 0x7f39581f3de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:46.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395f85c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f395815e370 con 0x7f39580b2b50 2024-03-04T07:34:46.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395f85c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39580ce8f0 con 0x7f39580d3400 2024-03-04T07:34:46.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395f85c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f39581f6fe0 con 0x7f3958071540 2024-03-04T07:34:46.082 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395cdf7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39580b2b50 0x7f39580b4d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395cdf7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39580b2b50 0x7f39580b4d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47028/0 (socket says 172.21.15.64:47028) 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395cdf7640 1 -- 172.21.15.64:0/3277086421 learned_addr learned my addr 172.21.15.64:0/3277086421 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395d5f8640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3958071540 0x7f3958159b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.080+0000 7f395ddf9640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39580d3400 0x7f39581f3de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395cdf7640 1 -- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3958071540 msgr2=0x7f3958159b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395cdf7640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3958071540 0x7f3958159b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395cdf7640 1 -- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39580d3400 msgr2=0x7f39581f3de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395cdf7640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39580d3400 0x7f39581f3de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395cdf7640 1 -- 172.21.15.64:0/3277086421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39580b57a0 con 0x7f39580b2b50 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395d5f8640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3958071540 0x7f3958159b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395cdf7640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39580b2b50 0x7f39580b4d70 secure :-1 s=READY pgs=1635 cs=0 l=1 rev1=1 crypto rx=0x7f394800e910 tx=0x7f394800edd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:46.083 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f394dffb640 1 -- 172.21.15.64:0/3277086421 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3948017070 con 0x7f39580b2b50 2024-03-04T07:34:46.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f394dffb640 1 -- 172.21.15.64:0/3277086421 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f394800ca00 con 0x7f39580b2b50 2024-03-04T07:34:46.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39580b5a80 con 0x7f39580b2b50 2024-03-04T07:34:46.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f394dffb640 1 -- 172.21.15.64:0/3277086421 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f394800cdb0 con 0x7f39580b2b50 2024-03-04T07:34:46.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.081+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39580b6320 con 0x7f39580b2b50 2024-03-04T07:34:46.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.082+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3958068700 con 0x7f39580b2b50 2024-03-04T07:34:46.084 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.083+0000 7f394dffb640 1 -- 172.21.15.64:0/3277086421 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f394801a070 con 0x7f39580b2b50 2024-03-04T07:34:46.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.083+0000 7f394dffb640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393c06b4d0 0x7f393c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:46.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.083+0000 7f394dffb640 1 -- 172.21.15.64:0/3277086421 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f394808a3e0 con 0x7f39580b2b50 2024-03-04T07:34:46.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.083+0000 7f395d5f8640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393c06b4d0 0x7f393c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:46.085 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.084+0000 7f395d5f8640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393c06b4d0 0x7f393c06d990 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7f3954005880 tx=0x7f3954009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:46.087 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.087+0000 7f394dffb640 1 -- 172.21.15.64:0/3277086421 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3948058bd0 con 0x7f39580b2b50 2024-03-04T07:34:46.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.311+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f39580b5c50 con 0x7f39580b2b50 2024-03-04T07:34:46.311 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.311+0000 7f394dffb640 1 -- 172.21.15.64:0/3277086421 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f394805cc70 con 0x7f39580b2b50 2024-03-04T07:34:46.311 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:46.312 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:46.312 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.313+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393c06b4d0 msgr2=0x7f393c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393c06b4d0 0x7f393c06d990 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7f3954005880 tx=0x7f3954009040 comp rx=0 tx=0).stop 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39580b2b50 msgr2=0x7f39580b4d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39580b2b50 0x7f39580b4d70 secure :-1 s=READY pgs=1635 cs=0 l=1 rev1=1 crypto rx=0x7f394800e910 tx=0x7f394800edd0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 shutdown_connections 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f393c06b4d0 0x7f393c06d990 unknown :-1 s=CLOSED pgs=734 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f39580d3400 0x7f39581f3de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f39580b2b50 0x7f39580b4d70 unknown :-1 s=CLOSED pgs=1635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 --2- 172.21.15.64:0/3277086421 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3958071540 0x7f3958159b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 >> 172.21.15.64:0/3277086421 conn(0x7f39580d3ca0 msgr2=0x7f395815ef30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 shutdown_connections 2024-03-04T07:34:46.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:46.314+0000 7f395f85c640 1 -- 172.21.15.64:0/3277086421 wait complete. 2024-03-04T07:34:46.526 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:46.526 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:46.561 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:47.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:47.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.642+0000 7f3d16cb1640 1 Processor -- start 2024-03-04T07:34:47.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.642+0000 7f3d16cb1640 1 -- start start 2024-03-04T07:34:47.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d16cb1640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3d1006e700 0x7f3d1015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:47.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d16cb1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3d100b55d0 0x7f3d10071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:47.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d16cb1640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3d100b4160 con 0x7f3d10072660 2024-03-04T07:34:47.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d16cb1640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d100b5b40 con 0x7f3d100b55d0 2024-03-04T07:34:47.643 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d16cb1640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3d100b3ba0 con 0x7f3d1006e700 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d1524e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3d100b55d0 0x7f3d10071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d0ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3d10072660 0x7f3d1006d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54164/0 (socket says 172.21.15.64:54164) 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d0ffff640 1 -- 172.21.15.64:0/4189409268 learned_addr learned my addr 172.21.15.64:0/4189409268 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d14a4d640 1 --2- 172.21.15.64:0/4189409268 >> v2:172.21.15.66:3300/0 conn(0x7f3d1006e700 0x7f3d1015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d1524e640 1 -- 172.21.15.64:0/4189409268 >> v2:172.21.15.66:3300/0 conn(0x7f3d1006e700 msgr2=0x7f3d1015ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d1524e640 1 --2- 172.21.15.64:0/4189409268 >> v2:172.21.15.66:3300/0 conn(0x7f3d1006e700 0x7f3d1015ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d1524e640 1 -- 172.21.15.64:0/4189409268 >> v1:172.21.15.64:6789/0 conn(0x7f3d10072660 legacy=0x7f3d1006d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:47.644 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.643+0000 7f3d1524e640 1 -- 172.21.15.64:0/4189409268 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d100b3e80 con 0x7f3d100b55d0 2024-03-04T07:34:47.645 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.644+0000 7f3d1524e640 1 --2- 172.21.15.64:0/4189409268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d100b55d0 0x7f3d10071980 secure :-1 s=READY pgs=1637 cs=0 l=1 rev1=1 crypto rx=0x7f3d1006bf60 tx=0x7f3d0802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4d80dd63588b3f5c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.644+0000 7f3d0f7fe640 1 -- 172.21.15.64:0/4189409268 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d08039070 con 0x7f3d100b55d0 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.644+0000 7f3d0f7fe640 1 -- 172.21.15.64:0/4189409268 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d0802cc80 con 0x7f3d100b55d0 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.644+0000 7f3d0f7fe640 1 -- 172.21.15.64:0/4189409268 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d08034660 con 0x7f3d100b55d0 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 -- 172.21.15.64:0/4189409268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d100b55d0 msgr2=0x7f3d10071980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/4189409268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d100b55d0 0x7f3d10071980 secure :-1 s=READY pgs=1637 cs=0 l=1 rev1=1 crypto rx=0x7f3d1006bf60 tx=0x7f3d0802bed0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 -- 172.21.15.64:0/4189409268 shutdown_connections 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/4189409268 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d100b55d0 0x7f3d10071980 unknown :-1 s=CLOSED pgs=1637 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/4189409268 >> v2:172.21.15.66:3300/0 conn(0x7f3d1006e700 0x7f3d1015ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 -- 172.21.15.64:0/4189409268 >> 172.21.15.64:0/4189409268 conn(0x7f3d100b2720 msgr2=0x7f3d100b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 -- 172.21.15.64:0/4189409268 shutdown_connections 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.645+0000 7f3d16cb1640 1 -- 172.21.15.64:0/4189409268 wait complete. 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.646+0000 7f3d16cb1640 1 Processor -- start 2024-03-04T07:34:47.646 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.646+0000 7f3d16cb1640 1 -- start start 2024-03-04T07:34:47.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d16cb1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d1006e700 0x7f3d10144790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:47.648 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d16cb1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3d10072660 0x7f3d100cd420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d16cb1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3d100b55d0 0x7f3d100cf0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d16cb1640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d10072040 con 0x7f3d1006e700 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d16cb1640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d10071ec0 con 0x7f3d10072660 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d16cb1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3d100721c0 con 0x7f3d100b55d0 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d1006e700 0x7f3d10144790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d1006e700 0x7f3d10144790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47054/0 (socket says 172.21.15.64:47054) 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 -- 172.21.15.64:0/3097401062 learned_addr learned my addr 172.21.15.64:0/3097401062 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d1524e640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3d100b55d0 0x7f3d100cf0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d0ffff640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3d10072660 0x7f3d100cd420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 -- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3d100b55d0 msgr2=0x7f3d100cf0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3d100b55d0 0x7f3d100cf0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 -- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3d10072660 msgr2=0x7f3d100cd420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3d10072660 0x7f3d100cd420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.649 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 -- 172.21.15.64:0/3097401062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d100cdd90 con 0x7f3d1006e700 2024-03-04T07:34:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.647+0000 7f3d14a4d640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d1006e700 0x7f3d10144790 secure :-1 s=READY pgs=1638 cs=0 l=1 rev1=1 crypto rx=0x7f3d0400e9e0 tx=0x7f3d0400eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.648+0000 7f3d0d7fa640 1 -- 172.21.15.64:0/3097401062 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d04017070 con 0x7f3d1006e700 2024-03-04T07:34:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.648+0000 7f3d0d7fa640 1 -- 172.21.15.64:0/3097401062 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3d0400c9d0 con 0x7f3d1006e700 2024-03-04T07:34:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.648+0000 7f3d0d7fa640 1 -- 172.21.15.64:0/3097401062 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d0400cdc0 con 0x7f3d1006e700 2024-03-04T07:34:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.648+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d100ce070 con 0x7f3d1006e700 2024-03-04T07:34:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.648+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d100cd960 con 0x7f3d1006e700 2024-03-04T07:34:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.649+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d10067ff0 con 0x7f3d1006e700 2024-03-04T07:34:47.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.649+0000 7f3d0d7fa640 1 -- 172.21.15.64:0/3097401062 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3d0401a070 con 0x7f3d1006e700 2024-03-04T07:34:47.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.649+0000 7f3d0d7fa640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cf406b500 0x7f3cf406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:47.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.650+0000 7f3d0d7fa640 1 -- 172.21.15.64:0/3097401062 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3d0408a150 con 0x7f3d1006e700 2024-03-04T07:34:47.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.650+0000 7f3d0ffff640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cf406b500 0x7f3cf406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:47.651 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.650+0000 7f3d0ffff640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cf406b500 0x7f3cf406d9c0 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7f3d101581a0 tx=0x7f3cfc007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:47.653 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.654+0000 7f3d0d7fa640 1 -- 172.21.15.64:0/3097401062 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3d040589d0 con 0x7f3d1006e700 2024-03-04T07:34:47.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.875+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3d10072c30 con 0x7f3d1006e700 2024-03-04T07:34:47.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.875+0000 7f3d0d7fa640 1 -- 172.21.15.64:0/3097401062 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f3d0405ca70 con 0x7f3d1006e700 2024-03-04T07:34:47.875 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:47.876 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:47.876 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:47.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cf406b500 msgr2=0x7f3cf406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:47.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cf406b500 0x7f3cf406d9c0 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7f3d101581a0 tx=0x7f3cfc007e80 comp rx=0 tx=0).stop 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d1006e700 msgr2=0x7f3d10144790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d1006e700 0x7f3d10144790 secure :-1 s=READY pgs=1638 cs=0 l=1 rev1=1 crypto rx=0x7f3d0400e9e0 tx=0x7f3d0400eea0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 shutdown_connections 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3cf406b500 0x7f3cf406d9c0 unknown :-1 s=CLOSED pgs=735 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3d100b55d0 0x7f3d100cf0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3d10072660 0x7f3d100cd420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 --2- 172.21.15.64:0/3097401062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3d1006e700 0x7f3d10144790 unknown :-1 s=CLOSED pgs=1638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 >> 172.21.15.64:0/3097401062 conn(0x7f3d100b2720 msgr2=0x7f3d100b2b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 shutdown_connections 2024-03-04T07:34:47.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:47.877+0000 7f3d16cb1640 1 -- 172.21.15.64:0/3097401062 wait complete. 2024-03-04T07:34:48.090 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:48.091 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:48.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:34:48.267 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:34:48.563 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:49.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:49.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.206+0000 7f0140b81640 1 Processor -- start 2024-03-04T07:34:49.206 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.206+0000 7f0140b81640 1 -- start start 2024-03-04T07:34:49.207 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0140b81640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f013c0d18c0 0x7f013c071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0140b81640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f013c0b6f20 con 0x7f013c071a80 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0140b81640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f013c0d3030 con 0x7f013c15df70 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0140b81640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f013c071eb0 con 0x7f013c0d18c0 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f013ad9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f013c071a80 0x7f013c0b2700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54174/0 (socket says 172.21.15.64:54174) 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f013ad9d640 1 -- 172.21.15.64:0/698012709 learned_addr learned my addr 172.21.15.64:0/698012709 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0139d9b640 1 --2- 172.21.15.64:0/698012709 >> v2:172.21.15.66:3300/0 conn(0x7f013c0d18c0 0x7f013c071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f013959a640 1 -- 172.21.15.64:0/698012709 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 872313890 0 0) 0x7f013c0d3030 con 0x7f013c15df70 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f013959a640 1 -- 172.21.15.64:0/698012709 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f0124003520 con 0x7f013c15df70 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f013959a640 1 -- 172.21.15.64:0/698012709 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4188016352 0 0) 0x7f013c0b6f20 con 0x7f013c071a80 2024-03-04T07:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0139d9b640 1 -- 172.21.15.64:0/698012709 >> v1:172.21.15.66:6789/0 conn(0x7f013c15df70 legacy=0x7f013c0d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:49.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0139d9b640 1 -- 172.21.15.64:0/698012709 >> v1:172.21.15.64:6789/0 conn(0x7f013c071a80 legacy=0x7f013c0b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:49.209 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.207+0000 7f0139d9b640 1 -- 172.21.15.64:0/698012709 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f013c0d3310 con 0x7f013c0d18c0 2024-03-04T07:34:49.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.208+0000 7f0139d9b640 1 --2- 172.21.15.64:0/698012709 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c0d18c0 0x7f013c071540 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f01300097a0 tx=0x7f013002b9f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b264b8229b59a5a6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:49.210 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.208+0000 7f013959a640 1 -- 172.21.15.64:0/698012709 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f013002f810 con 0x7f013c0d18c0 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.208+0000 7f013959a640 1 -- 172.21.15.64:0/698012709 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f013002f9b0 con 0x7f013c0d18c0 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.208+0000 7f013959a640 1 -- 172.21.15.64:0/698012709 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f013002fd80 con 0x7f013c0d18c0 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.209+0000 7f0140b81640 1 -- 172.21.15.64:0/698012709 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c0d18c0 msgr2=0x7f013c071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.209+0000 7f0140b81640 1 --2- 172.21.15.64:0/698012709 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c0d18c0 0x7f013c071540 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f01300097a0 tx=0x7f013002b9f0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.209+0000 7f0140b81640 1 -- 172.21.15.64:0/698012709 shutdown_connections 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.209+0000 7f0140b81640 1 --2- 172.21.15.64:0/698012709 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c0d18c0 0x7f013c071540 unknown :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.209+0000 7f0140b81640 1 -- 172.21.15.64:0/698012709 >> 172.21.15.64:0/698012709 conn(0x7f013c0d3cf0 msgr2=0x7f013c0d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.209+0000 7f0140b81640 1 -- 172.21.15.64:0/698012709 shutdown_connections 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.209+0000 7f0140b81640 1 -- 172.21.15.64:0/698012709 wait complete. 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.210+0000 7f0140b81640 1 Processor -- start 2024-03-04T07:34:49.211 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.210+0000 7f0140b81640 1 -- start start 2024-03-04T07:34:49.213 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0140b81640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c071a80 0x7f013c1f65e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0140b81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013c0d18c0 0x7f013c0c8810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0140b81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f013c15df70 0x7f013c0c8d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0140b81640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f013c15e350 con 0x7f013c0d18c0 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0140b81640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f013c0d38d0 con 0x7f013c071a80 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0140b81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f013c15e4d0 con 0x7f013c15df70 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0139d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013c0d18c0 0x7f013c0c8810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f013a59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c071a80 0x7f013c1f65e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0139d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013c0d18c0 0x7f013c0c8810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47058/0 (socket says 172.21.15.64:47058) 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f0139d9b640 1 -- 172.21.15.64:0/416124490 learned_addr learned my addr 172.21.15.64:0/416124490 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.211+0000 7f013ad9d640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f013c15df70 0x7f013c0c8d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f013a59c640 1 -- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f013c15df70 msgr2=0x7f013c0c8d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f013a59c640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f013c15df70 0x7f013c0c8d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f013a59c640 1 -- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013c0d18c0 msgr2=0x7f013c0c8810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f013a59c640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013c0d18c0 0x7f013c0c8810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f013a59c640 1 -- 172.21.15.64:0/416124490 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f013c0c9420 con 0x7f013c071a80 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f0139d9b640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013c0d18c0 0x7f013c0c8810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f013a59c640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c071a80 0x7f013c1f65e0 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f012c00c3c0 tx=0x7f012c001e70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f012affd640 1 -- 172.21.15.64:0/416124490 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f012c017070 con 0x7f013c071a80 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f012affd640 1 -- 172.21.15.64:0/416124490 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f012c0052d0 con 0x7f013c071a80 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f013c0c9700 con 0x7f013c071a80 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f012affd640 1 -- 172.21.15.64:0/416124490 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f012c003830 con 0x7f013c071a80 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.212+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f013c0c9cc0 con 0x7f013c071a80 2024-03-04T07:34:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.213+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f013c0cfc60 con 0x7f013c071a80 2024-03-04T07:34:49.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.213+0000 7f012affd640 1 -- 172.21.15.64:0/416124490 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f012c01a070 con 0x7f013c071a80 2024-03-04T07:34:49.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.214+0000 7f012affd640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011c06b4d0 0x7f011c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:49.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.214+0000 7f012affd640 1 -- 172.21.15.64:0/416124490 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f012c089430 con 0x7f013c071a80 2024-03-04T07:34:49.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.214+0000 7f0139d9b640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011c06b4d0 0x7f011c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:49.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.214+0000 7f0139d9b640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011c06b4d0 0x7f011c06d990 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7f013002ba30 tx=0x7f0130008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:49.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.219+0000 7f012affd640 1 -- 172.21.15.64:0/416124490 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f013c0cfc60 con 0x7f013c071a80 2024-03-04T07:34:49.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.439+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f013c067ff0 con 0x7f013c071a80 2024-03-04T07:34:49.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.439+0000 7f012affd640 1 -- 172.21.15.64:0/416124490 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f012c057c20 con 0x7f013c071a80 2024-03-04T07:34:49.439 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:49.440 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:49.440 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011c06b4d0 msgr2=0x7f011c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011c06b4d0 0x7f011c06d990 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7f013002ba30 tx=0x7f0130008030 comp rx=0 tx=0).stop 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c071a80 msgr2=0x7f013c1f65e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c071a80 0x7f013c1f65e0 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f012c00c3c0 tx=0x7f012c001e70 comp rx=0 tx=0).stop 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 shutdown_connections 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f011c06b4d0 0x7f011c06d990 unknown :-1 s=CLOSED pgs=736 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f013c15df70 0x7f013c0c8d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f013c0d18c0 0x7f013c0c8810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 --2- 172.21.15.64:0/416124490 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f013c071a80 0x7f013c1f65e0 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:49.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 >> 172.21.15.64:0/416124490 conn(0x7f013c0d3cf0 msgr2=0x7f013c160f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:49.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 shutdown_connections 2024-03-04T07:34:49.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:49.442+0000 7f0140b81640 1 -- 172.21.15.64:0/416124490 wait complete. 2024-03-04T07:34:49.654 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:49.654 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:50.566 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:50.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:50.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.769+0000 7f72991f7640 1 Processor -- start 2024-03-04T07:34:50.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.769+0000 7f72991f7640 1 -- start start 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.769+0000 7f72991f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f72940d0880 0x7f7294168c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.769+0000 7f72991f7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72940cf130 con 0x7f729415e060 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.769+0000 7f72991f7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72940d0dd0 con 0x7f729406fe20 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.769+0000 7f72991f7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f72940d10b0 con 0x7f72940d0880 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f7292d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f729415e060 0x7f72940ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54190/0 (socket says 172.21.15.64:54190) 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f7292d9d640 1 -- 172.21.15.64:0/3104323881 learned_addr learned my addr 172.21.15.64:0/3104323881 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f729359e640 1 --2- 172.21.15.64:0/3104323881 >> v2:172.21.15.66:3300/0 conn(0x7f72940d0880 0x7f7294168c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f7291d9b640 1 -- 172.21.15.64:0/3104323881 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3115668580 0 0) 0x7f72940cf130 con 0x7f729415e060 2024-03-04T07:34:50.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f7291d9b640 1 -- 172.21.15.64:0/3104323881 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f727c003520 con 0x7f729415e060 2024-03-04T07:34:50.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f7291d9b640 1 -- 172.21.15.64:0/3104323881 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3732292511 0 0) 0x7f72940d0dd0 con 0x7f729406fe20 2024-03-04T07:34:50.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f7291d9b640 1 -- 172.21.15.64:0/3104323881 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f72940cf130 con 0x7f729406fe20 2024-03-04T07:34:50.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f729359e640 1 -- 172.21.15.64:0/3104323881 >> v1:172.21.15.66:6789/0 conn(0x7f729406fe20 legacy=0x7f72940cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:50.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f729359e640 1 -- 172.21.15.64:0/3104323881 >> v1:172.21.15.64:6789/0 conn(0x7f729415e060 legacy=0x7f72940ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:50.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.770+0000 7f729359e640 1 -- 172.21.15.64:0/3104323881 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72940cc0e0 con 0x7f72940d0880 2024-03-04T07:34:50.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.771+0000 7f729359e640 1 --2- 172.21.15.64:0/3104323881 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f7294168c20 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7f728c009d90 tx=0x7f728c02bf50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8b830cab635fc6b9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:50.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.771+0000 7f7291d9b640 1 -- 172.21.15.64:0/3104323881 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f728c039070 con 0x7f72940d0880 2024-03-04T07:34:50.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.771+0000 7f7291d9b640 1 -- 172.21.15.64:0/3104323881 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f728c02cd00 con 0x7f72940d0880 2024-03-04T07:34:50.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.771+0000 7f7291d9b640 1 -- 172.21.15.64:0/3104323881 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f728c034660 con 0x7f72940d0880 2024-03-04T07:34:50.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.771+0000 7f72991f7640 1 -- 172.21.15.64:0/3104323881 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 msgr2=0x7f7294168c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:50.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.771+0000 7f72991f7640 1 --2- 172.21.15.64:0/3104323881 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f7294168c20 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7f728c009d90 tx=0x7f728c02bf50 comp rx=0 tx=0).stop 2024-03-04T07:34:50.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.772+0000 7f72991f7640 1 -- 172.21.15.64:0/3104323881 shutdown_connections 2024-03-04T07:34:50.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.772+0000 7f72991f7640 1 --2- 172.21.15.64:0/3104323881 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f7294168c20 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:50.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.772+0000 7f72991f7640 1 -- 172.21.15.64:0/3104323881 >> 172.21.15.64:0/3104323881 conn(0x7f72940d1a90 msgr2=0x7f72940d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:50.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.772+0000 7f72991f7640 1 -- 172.21.15.64:0/3104323881 shutdown_connections 2024-03-04T07:34:50.773 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.772+0000 7f72991f7640 1 -- 172.21.15.64:0/3104323881 wait complete. 2024-03-04T07:34:50.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.773+0000 7f72991f7640 1 Processor -- start 2024-03-04T07:34:50.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.773+0000 7f72991f7640 1 -- start start 2024-03-04T07:34:50.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f72991f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f729406fe20 0x7f72941f4570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f72991f7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f729414e2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f72991f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f729415e060 0x7f72941f70a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f72991f7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f729415e440 con 0x7f729415e060 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f72991f7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72940cc6a0 con 0x7f72940d0880 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f72991f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72940cdbd0 con 0x7f729406fe20 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f729415e060 0x7f72941f70a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f7292d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f729406fe20 0x7f72941f4570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f729415e060 0x7f72941f70a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47070/0 (socket says 172.21.15.64:47070) 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 -- 172.21.15.64:0/2600722596 learned_addr learned my addr 172.21.15.64:0/2600722596 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729259c640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f729414e2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 -- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f729406fe20 msgr2=0x7f72941f4570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f729406fe20 0x7f72941f4570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:50.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 -- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 msgr2=0x7f729414e2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f729414e2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729359e640 1 -- 172.21.15.64:0/2600722596 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f729414e7e0 con 0x7f729415e060 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f7292d9d640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f729406fe20 0x7f72941f4570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.774+0000 7f729259c640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f729414e2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.775+0000 7f729359e640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f729415e060 0x7f72941f70a0 secure :-1 s=READY pgs=1641 cs=0 l=1 rev1=1 crypto rx=0x7f728c009d90 tx=0x7f728c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.775+0000 7f727b7fe640 1 -- 172.21.15.64:0/2600722596 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f728c039050 con 0x7f729415e060 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.775+0000 7f727b7fe640 1 -- 172.21.15.64:0/2600722596 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f728c045070 con 0x7f729415e060 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.775+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72941f3ab0 con 0x7f729415e060 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.775+0000 7f727b7fe640 1 -- 172.21.15.64:0/2600722596 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f728c009450 con 0x7f729415e060 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.775+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72941f7e00 con 0x7f729415e060 2024-03-04T07:34:50.777 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.776+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7294067ff0 con 0x7f729415e060 2024-03-04T07:34:50.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.776+0000 7f727b7fe640 1 -- 172.21.15.64:0/2600722596 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f728c03e850 con 0x7f729415e060 2024-03-04T07:34:50.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.777+0000 7f727b7fe640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f728006b4d0 0x7f728006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:50.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.777+0000 7f7292d9d640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f728006b4d0 0x7f728006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:50.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.777+0000 7f727b7fe640 1 -- 172.21.15.64:0/2600722596 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f728c0acff0 con 0x7f729415e060 2024-03-04T07:34:50.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.777+0000 7f7292d9d640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f728006b4d0 0x7f728006d990 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7f7284000f60 tx=0x7f7284007d70 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:50.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:50.781+0000 7f727b7fe640 1 -- 172.21.15.64:0/2600722596 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f728c07b8b0 con 0x7f729415e060 2024-03-04T07:34:51.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.000+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f72940d1390 con 0x7f729415e060 2024-03-04T07:34:51.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.000+0000 7f727b7fe640 1 -- 172.21.15.64:0/2600722596 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f728c07f950 con 0x7f729415e060 2024-03-04T07:34:51.000 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:51.000 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:51.001 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:51.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f728006b4d0 msgr2=0x7f728006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:51.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f728006b4d0 0x7f728006d990 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7f7284000f60 tx=0x7f7284007d70 comp rx=0 tx=0).stop 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f729415e060 msgr2=0x7f72941f70a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f729415e060 0x7f72941f70a0 secure :-1 s=READY pgs=1641 cs=0 l=1 rev1=1 crypto rx=0x7f728c009d90 tx=0x7f728c004a40 comp rx=0 tx=0).stop 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 shutdown_connections 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f728006b4d0 0x7f728006d990 unknown :-1 s=CLOSED pgs=737 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f729415e060 0x7f72941f70a0 unknown :-1 s=CLOSED pgs=1641 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72940d0880 0x7f729414e2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 --2- 172.21.15.64:0/2600722596 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f729406fe20 0x7f72941f4570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 >> 172.21.15.64:0/2600722596 conn(0x7f72940d1a90 msgr2=0x7f729406eb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 shutdown_connections 2024-03-04T07:34:51.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:51.002+0000 7f72991f7640 1 -- 172.21.15.64:0/2600722596 wait complete. 2024-03-04T07:34:51.214 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:51.215 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:52.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:52.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.329+0000 7fbb10da0640 1 Processor -- start 2024-03-04T07:34:52.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.329+0000 7fbb10da0640 1 -- start start 2024-03-04T07:34:52.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb10da0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fbb0c15dd20 0x7fbb0c0d1bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:52.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb10da0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d2100 0x7fbb0c0d0a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:52.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb10da0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbb0c0d0f70 0x7fbb0c0d3a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:52.330 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb10da0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbb0c0d3190 con 0x7fbb0c0d0f70 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb10da0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbb0c0d1390 con 0x7fbb0c15dd20 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb10da0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbb0c15e100 con 0x7fbb0c0d2100 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbb0c0d0f70 0x7fbb0c0d3a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbb0c0d0f70 0x7fbb0c0d3a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47078/0 (socket says 172.21.15.64:47078) 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 -- 172.21.15.64:0/2246716022 learned_addr learned my addr 172.21.15.64:0/2246716022 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0affd640 1 --2- 172.21.15.64:0/2246716022 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d2100 0x7fbb0c0d0a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0b7fe640 1 --2- 172.21.15.64:0/2246716022 >> v2:172.21.15.66:3300/0 conn(0x7fbb0c15dd20 0x7fbb0c0d1bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 -- 172.21.15.64:0/2246716022 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d2100 msgr2=0x7fbb0c0d0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 --2- 172.21.15.64:0/2246716022 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d2100 0x7fbb0c0d0a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 -- 172.21.15.64:0/2246716022 >> v2:172.21.15.66:3300/0 conn(0x7fbb0c15dd20 msgr2=0x7fbb0c0d1bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 --2- 172.21.15.64:0/2246716022 >> v2:172.21.15.66:3300/0 conn(0x7fbb0c15dd20 0x7fbb0c0d1bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.331 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.330+0000 7fbb0bfff640 1 -- 172.21.15.64:0/2246716022 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb0c0d2580 con 0x7fbb0c0d0f70 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.331+0000 7fbb0bfff640 1 --2- 172.21.15.64:0/2246716022 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d0f70 0x7fbb0c0d3a00 secure :-1 s=READY pgs=1642 cs=0 l=1 rev1=1 crypto rx=0x7fbb0c06b320 tx=0x7fbb0402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c0743f231e0d521 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.331+0000 7fbb0a7fc640 1 -- 172.21.15.64:0/2246716022 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb0402ca70 con 0x7fbb0c0d0f70 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.331+0000 7fbb0a7fc640 1 -- 172.21.15.64:0/2246716022 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbb0402cc10 con 0x7fbb0c0d0f70 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.331+0000 7fbb0a7fc640 1 -- 172.21.15.64:0/2246716022 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb040343f0 con 0x7fbb0c0d0f70 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 -- 172.21.15.64:0/2246716022 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d0f70 msgr2=0x7fbb0c0d3a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2246716022 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d0f70 0x7fbb0c0d3a00 secure :-1 s=READY pgs=1642 cs=0 l=1 rev1=1 crypto rx=0x7fbb0c06b320 tx=0x7fbb0402bed0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 -- 172.21.15.64:0/2246716022 shutdown_connections 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2246716022 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d0f70 0x7fbb0c0d3a00 unknown :-1 s=CLOSED pgs=1642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2246716022 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d2100 0x7fbb0c0d0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2246716022 >> v2:172.21.15.66:3300/0 conn(0x7fbb0c15dd20 0x7fbb0c0d1bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 -- 172.21.15.64:0/2246716022 >> 172.21.15.64:0/2246716022 conn(0x7fbb0c071960 msgr2=0x7fbb0c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 -- 172.21.15.64:0/2246716022 shutdown_connections 2024-03-04T07:34:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.332+0000 7fbb10da0640 1 -- 172.21.15.64:0/2246716022 wait complete. 2024-03-04T07:34:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.333+0000 7fbb10da0640 1 Processor -- start 2024-03-04T07:34:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.333+0000 7fbb10da0640 1 -- start start 2024-03-04T07:34:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb10da0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d0f70 0x7fbb0c1f7730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb10da0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d2100 0x7fbb0c15f280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb10da0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb0c15dd20 0x7fbb0c15f7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb10da0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb0c0d2e20 con 0x7fbb0c0d2100 2024-03-04T07:34:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb10da0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb0c0d2b40 con 0x7fbb0c15dd20 2024-03-04T07:34:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb10da0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbb0c0b9580 con 0x7fbb0c0d0f70 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb0affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d2100 0x7fbb0c15f280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb0affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d2100 0x7fbb0c15f280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:47082/0 (socket says 172.21.15.64:47082) 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb0affd640 1 -- 172.21.15.64:0/2067867909 learned_addr learned my addr 172.21.15.64:0/2067867909 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb0bfff640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb0c15dd20 0x7fbb0c15f7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.334+0000 7fbb0b7fe640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d0f70 0x7fbb0c1f7730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb0affd640 1 -- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d0f70 msgr2=0x7fbb0c1f7730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb0affd640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d0f70 0x7fbb0c1f7730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb0affd640 1 -- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb0c15dd20 msgr2=0x7fbb0c15f7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb0affd640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb0c15dd20 0x7fbb0c15f7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb0affd640 1 -- 172.21.15.64:0/2067867909 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb0c1f3860 con 0x7fbb0c0d2100 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb0affd640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d2100 0x7fbb0c15f280 secure :-1 s=READY pgs=1643 cs=0 l=1 rev1=1 crypto rx=0x7fbb0000e9b0 tx=0x7fbb0000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbaf3fff640 1 -- 172.21.15.64:0/2067867909 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb00017070 con 0x7fbb0c0d2100 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbaf3fff640 1 -- 172.21.15.64:0/2067867909 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbb0000c9a0 con 0x7fbb0c0d2100 2024-03-04T07:34:52.337 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb0c1f4890 con 0x7fbb0c0d2100 2024-03-04T07:34:52.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbaf3fff640 1 -- 172.21.15.64:0/2067867909 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb0000cd90 con 0x7fbb0c0d2100 2024-03-04T07:34:52.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.335+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb0c1f3db0 con 0x7fbb0c0d2100 2024-03-04T07:34:52.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.336+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbb0c04f330 con 0x7fbb0c0d2100 2024-03-04T07:34:52.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.336+0000 7fbaf3fff640 1 -- 172.21.15.64:0/2067867909 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbb0001a070 con 0x7fbb0c0d2100 2024-03-04T07:34:52.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.337+0000 7fbaf3fff640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbaf806b500 0x7fbaf806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:52.338 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.337+0000 7fbaf3fff640 1 -- 172.21.15.64:0/2067867909 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbb0008a1e0 con 0x7fbb0c0d2100 2024-03-04T07:34:52.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.337+0000 7fbb0b7fe640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbaf806b500 0x7fbaf806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:52.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.337+0000 7fbb0b7fe640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbaf806b500 0x7fbaf806d9c0 secure :-1 s=READY pgs=738 cs=0 l=1 rev1=1 crypto rx=0x7fbb0c0c8690 tx=0x7fbafc009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:52.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.341+0000 7fbaf3fff640 1 -- 172.21.15.64:0/2067867909 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbb000589d0 con 0x7fbb0c0d2100 2024-03-04T07:34:52.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.560+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbb0c06b600 con 0x7fbb0c0d2100 2024-03-04T07:34:52.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.561+0000 7fbaf3fff640 1 -- 172.21.15.64:0/2067867909 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fbb0005ca70 con 0x7fbb0c0d2100 2024-03-04T07:34:52.561 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:52.561 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:52.561 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:52.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbaf806b500 msgr2=0x7fbaf806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:52.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbaf806b500 0x7fbaf806d9c0 secure :-1 s=READY pgs=738 cs=0 l=1 rev1=1 crypto rx=0x7fbb0c0c8690 tx=0x7fbafc009190 comp rx=0 tx=0).stop 2024-03-04T07:34:52.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d2100 msgr2=0x7fbb0c15f280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:52.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d2100 0x7fbb0c15f280 secure :-1 s=READY pgs=1643 cs=0 l=1 rev1=1 crypto rx=0x7fbb0000e9b0 tx=0x7fbb0000ee70 comp rx=0 tx=0).stop 2024-03-04T07:34:52.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 shutdown_connections 2024-03-04T07:34:52.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbaf806b500 0x7fbaf806d9c0 unknown :-1 s=CLOSED pgs=738 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbb0c15dd20 0x7fbb0c15f7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbb0c0d2100 0x7fbb0c15f280 unknown :-1 s=CLOSED pgs=1643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 --2- 172.21.15.64:0/2067867909 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbb0c0d0f70 0x7fbb0c1f7730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:52.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 >> 172.21.15.64:0/2067867909 conn(0x7fbb0c071960 msgr2=0x7fbb0c0b8540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:52.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 shutdown_connections 2024-03-04T07:34:52.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:52.563+0000 7fbb10da0640 1 -- 172.21.15.64:0/2067867909 wait complete. 2024-03-04T07:34:52.567 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:52.775 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:52.775 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:53.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:53.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36bac81640 1 Processor -- start 2024-03-04T07:34:53.889 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36bac81640 1 -- start start 2024-03-04T07:34:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36bac81640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f36b415df00 0x7f36b40d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36bac81640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f36b40b9a30 0x7f36b40d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36bac81640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36b40cea30 con 0x7f36b40b95a0 2024-03-04T07:34:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36bac81640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f36b40d2af0 con 0x7f36b415df00 2024-03-04T07:34:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36bac81640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f36b40cf3c0 con 0x7f36b40b9a30 2024-03-04T07:34:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.890+0000 7f36b8a1d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f36b415df00 0x7f36b40d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.891+0000 7f36b8a1d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f36b415df00 0x7f36b40d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37158/0 (socket says 172.21.15.64:37158) 2024-03-04T07:34:53.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.891+0000 7f36b8a1d640 1 -- 172.21.15.64:0/4105955020 learned_addr learned my addr 172.21.15.64:0/4105955020 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:53.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.891+0000 7f36b921e640 1 --2- 172.21.15.64:0/4105955020 >> v2:172.21.15.66:3300/0 conn(0x7f36b40b9a30 0x7f36b40d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:53.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.891+0000 7f36b8a1d640 1 -- 172.21.15.64:0/4105955020 >> v2:172.21.15.66:3300/0 conn(0x7f36b40b9a30 msgr2=0x7f36b40d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:53.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.891+0000 7f36b8a1d640 1 --2- 172.21.15.64:0/4105955020 >> v2:172.21.15.66:3300/0 conn(0x7f36b40b9a30 0x7f36b40d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:53.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.891+0000 7f36b8a1d640 1 -- 172.21.15.64:0/4105955020 >> v1:172.21.15.66:6789/0 conn(0x7f36b40b95a0 legacy=0x7f36b40b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:53.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.891+0000 7f36b8a1d640 1 -- 172.21.15.64:0/4105955020 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36b40cf0e0 con 0x7f36b415df00 2024-03-04T07:34:53.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36b8a1d640 1 --2- 172.21.15.64:0/4105955020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b415df00 0x7f36b40d25b0 secure :-1 s=READY pgs=1644 cs=0 l=1 rev1=1 crypto rx=0x7f36b40d4d20 tx=0x7f36ac02bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=650fbc5943efaca2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:53.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36b37fe640 1 -- 172.21.15.64:0/4105955020 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36ac02cb20 con 0x7f36b415df00 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36b37fe640 1 -- 172.21.15.64:0/4105955020 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36ac02ccc0 con 0x7f36b415df00 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36b37fe640 1 -- 172.21.15.64:0/4105955020 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36ac034610 con 0x7f36b415df00 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 -- 172.21.15.64:0/4105955020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b415df00 msgr2=0x7f36b40d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 --2- 172.21.15.64:0/4105955020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b415df00 0x7f36b40d25b0 secure :-1 s=READY pgs=1644 cs=0 l=1 rev1=1 crypto rx=0x7f36b40d4d20 tx=0x7f36ac02bf80 comp rx=0 tx=0).stop 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 -- 172.21.15.64:0/4105955020 shutdown_connections 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 --2- 172.21.15.64:0/4105955020 >> v2:172.21.15.66:3300/0 conn(0x7f36b40b9a30 0x7f36b40d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 --2- 172.21.15.64:0/4105955020 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b415df00 0x7f36b40d25b0 unknown :-1 s=CLOSED pgs=1644 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 -- 172.21.15.64:0/4105955020 >> 172.21.15.64:0/4105955020 conn(0x7f36b4071960 msgr2=0x7f36b4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 -- 172.21.15.64:0/4105955020 shutdown_connections 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.892+0000 7f36bac81640 1 -- 172.21.15.64:0/4105955020 wait complete. 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.894+0000 7f36bac81640 1 Processor -- start 2024-03-04T07:34:53.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.894+0000 7f36bac81640 1 -- start start 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36bac81640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36b40b95a0 0x7f36b415f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36bac81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36b415df00 0x7f36b4154f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36bac81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b40cd8d0 0x7f36b415f6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36bac81640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36b40cec70 con 0x7f36b40cd8d0 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36bac81640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36b40ce750 con 0x7f36b40b95a0 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36bac81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36b40cedf0 con 0x7f36b415df00 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b40cd8d0 0x7f36b415f6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b40cd8d0 0x7f36b415f6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37172/0 (socket says 172.21.15.64:37172) 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b8a1d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36b40b95a0 0x7f36b415f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 -- 172.21.15.64:0/1681464261 learned_addr learned my addr 172.21.15.64:0/1681464261 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b3fff640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36b415df00 0x7f36b4154f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 -- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36b415df00 msgr2=0x7f36b4154f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36b415df00 0x7f36b4154f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:53.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 -- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36b40b95a0 msgr2=0x7f36b415f010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36b40b95a0 0x7f36b415f010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 -- 172.21.15.64:0/1681464261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36b41554b0 con 0x7f36b40cd8d0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b8a1d640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36b40b95a0 0x7f36b415f010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b921e640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b40cd8d0 0x7f36b415f6c0 secure :-1 s=READY pgs=1645 cs=0 l=1 rev1=1 crypto rx=0x7f36a400e930 tx=0x7f36a400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b17fa640 1 -- 172.21.15.64:0/1681464261 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36a4017070 con 0x7f36b40cd8d0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b17fa640 1 -- 172.21.15.64:0/1681464261 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36a400c980 con 0x7f36b40cd8d0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36b4160030 con 0x7f36b40cd8d0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.895+0000 7f36b17fa640 1 -- 172.21.15.64:0/1681464261 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36a400cd70 con 0x7f36b40cd8d0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.896+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f36b415e810 con 0x7f36b40cd8d0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.896+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f36b4067ff0 con 0x7f36b40cd8d0 2024-03-04T07:34:53.898 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.897+0000 7f36b17fa640 1 -- 172.21.15.64:0/1681464261 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f36a401a070 con 0x7f36b40cd8d0 2024-03-04T07:34:53.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.897+0000 7f36b17fa640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36a006b4d0 0x7f36a006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:53.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.898+0000 7f36b17fa640 1 -- 172.21.15.64:0/1681464261 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f36a408a4d0 con 0x7f36b40cd8d0 2024-03-04T07:34:53.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.898+0000 7f36b8a1d640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36a006b4d0 0x7f36a006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:53.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.898+0000 7f36b8a1d640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36a006b4d0 0x7f36a006d990 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f36ac035000 tx=0x7f36ac030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:53.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:53.902+0000 7f36b17fa640 1 -- 172.21.15.64:0/1681464261 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f36a4058e10 con 0x7f36b40cd8d0 2024-03-04T07:34:54.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.123+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f36b4162690 con 0x7f36b40cd8d0 2024-03-04T07:34:54.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.123+0000 7f36b17fa640 1 -- 172.21.15.64:0/1681464261 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f36a405ceb0 con 0x7f36b40cd8d0 2024-03-04T07:34:54.123 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:54.123 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:54.124 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36a006b4d0 msgr2=0x7f36a006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36a006b4d0 0x7f36a006d990 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f36ac035000 tx=0x7f36ac030000 comp rx=0 tx=0).stop 2024-03-04T07:34:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b40cd8d0 msgr2=0x7f36b415f6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b40cd8d0 0x7f36b415f6c0 secure :-1 s=READY pgs=1645 cs=0 l=1 rev1=1 crypto rx=0x7f36a400e930 tx=0x7f36a400edf0 comp rx=0 tx=0).stop 2024-03-04T07:34:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 shutdown_connections 2024-03-04T07:34:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36a006b4d0 0x7f36a006d990 unknown :-1 s=CLOSED pgs=739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36b40cd8d0 0x7f36b415f6c0 unknown :-1 s=CLOSED pgs=1645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:54.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36b415df00 0x7f36b4154f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:54.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 --2- 172.21.15.64:0/1681464261 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36b40b95a0 0x7f36b415f010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:54.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 >> 172.21.15.64:0/1681464261 conn(0x7f36b4071960 msgr2=0x7f36b40b7870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:54.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 shutdown_connections 2024-03-04T07:34:54.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:54.125+0000 7f36bac81640 1 -- 172.21.15.64:0/1681464261 wait complete. 2024-03-04T07:34:54.337 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:54.337 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:54.568 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:55.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:55.452 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.453+0000 7f19aa1f7640 1 Processor -- start 2024-03-04T07:34:55.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.453+0000 7f19aa1f7640 1 -- start start 2024-03-04T07:34:55.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.453+0000 7f19aa1f7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f19a40b95a0 0x7f19a40b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:55.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.453+0000 7f19aa1f7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f19a40b9a10 0x7f19a40b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:55.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.453+0000 7f19aa1f7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f19a40cea30 con 0x7f19a415df00 2024-03-04T07:34:55.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.453+0000 7f19aa1f7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f19a40cf3c0 con 0x7f19a40b9a10 2024-03-04T07:34:55.456 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.453+0000 7f19aa1f7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f19a40cf0e0 con 0x7f19a40b95a0 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f19a40b9a10 0x7f19a40b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a37fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f19a415df00 0x7f19a40d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54780/0 (socket says 172.21.15.64:54780) 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a37fe640 1 -- 172.21.15.64:0/1560403426 learned_addr learned my addr 172.21.15.64:0/1560403426 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a2ffd640 1 --2- 172.21.15.64:0/1560403426 >> v2:172.21.15.66:3300/0 conn(0x7f19a40b95a0 0x7f19a40b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a3fff640 1 -- 172.21.15.64:0/1560403426 >> v2:172.21.15.66:3300/0 conn(0x7f19a40b95a0 msgr2=0x7f19a40b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a3fff640 1 --2- 172.21.15.64:0/1560403426 >> v2:172.21.15.66:3300/0 conn(0x7f19a40b95a0 0x7f19a40b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a3fff640 1 -- 172.21.15.64:0/1560403426 >> v1:172.21.15.64:6789/0 conn(0x7f19a415df00 legacy=0x7f19a40d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a3fff640 1 -- 172.21.15.64:0/1560403426 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19a40ce470 con 0x7f19a40b9a10 2024-03-04T07:34:55.457 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.454+0000 7f19a27fc640 1 -- 172.21.15.64:0/1560403426 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1090083057 0 0) 0x7f19a40cea30 con 0x7f19a415df00 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19a3fff640 1 --2- 172.21.15.64:0/1560403426 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b5c40 secure :-1 s=READY pgs=1647 cs=0 l=1 rev1=1 crypto rx=0x7f19a40d1760 tx=0x7f199c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=12510e4445bb81be server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19a27fc640 1 -- 172.21.15.64:0/1560403426 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f199c039070 con 0x7f19a40b9a10 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19a27fc640 1 -- 172.21.15.64:0/1560403426 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f199c02cc80 con 0x7f19a40b9a10 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19a27fc640 1 -- 172.21.15.64:0/1560403426 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f199c034660 con 0x7f19a40b9a10 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19aa1f7640 1 -- 172.21.15.64:0/1560403426 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 msgr2=0x7f19a40b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/1560403426 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b5c40 secure :-1 s=READY pgs=1647 cs=0 l=1 rev1=1 crypto rx=0x7f19a40d1760 tx=0x7f199c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19aa1f7640 1 -- 172.21.15.64:0/1560403426 shutdown_connections 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/1560403426 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b5c40 unknown :-1 s=CLOSED pgs=1647 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/1560403426 >> v2:172.21.15.66:3300/0 conn(0x7f19a40b95a0 0x7f19a40b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.455+0000 7f19aa1f7640 1 -- 172.21.15.64:0/1560403426 >> 172.21.15.64:0/1560403426 conn(0x7f19a4071960 msgr2=0x7f19a4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:55.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.456+0000 7f19aa1f7640 1 -- 172.21.15.64:0/1560403426 shutdown_connections 2024-03-04T07:34:55.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.456+0000 7f19aa1f7640 1 -- 172.21.15.64:0/1560403426 wait complete. 2024-03-04T07:34:55.462 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.457+0000 7f19aa1f7640 1 Processor -- start 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.457+0000 7f19aa1f7640 1 -- start start 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19aa1f7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19a40b95a0 0x7f19a415ef90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19aa1f7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b75e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19aa1f7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19a415df00 0x7f19a40b7b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19aa1f7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19a40cedf0 con 0x7f19a40b9a10 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19aa1f7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19a40cec70 con 0x7f19a415df00 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19aa1f7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f19a40d3220 con 0x7f19a40b95a0 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b75e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b75e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37188/0 (socket says 172.21.15.64:37188) 2024-03-04T07:34:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 -- 172.21.15.64:0/2116854077 learned_addr learned my addr 172.21.15.64:0/2116854077 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a3fff640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19a415df00 0x7f19a40b7b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a37fe640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19a40b95a0 0x7f19a415ef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 -- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19a40b95a0 msgr2=0x7f19a415ef90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19a40b95a0 0x7f19a415ef90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 -- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19a415df00 msgr2=0x7f19a40b7b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19a415df00 0x7f19a40b7b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 -- 172.21.15.64:0/2116854077 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19a40cc420 con 0x7f19a40b9a10 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19a2ffd640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b75e0 secure :-1 s=READY pgs=1648 cs=0 l=1 rev1=1 crypto rx=0x7f199400e9e0 tx=0x7f199400eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f199bfff640 1 -- 172.21.15.64:0/2116854077 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1994017070 con 0x7f19a40b9a10 2024-03-04T07:34:55.464 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f199bfff640 1 -- 172.21.15.64:0/2116854077 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f199400c9d0 con 0x7f19a40b9a10 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f199bfff640 1 -- 172.21.15.64:0/2116854077 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f199400cdc0 con 0x7f19a40b9a10 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.458+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f19a415f7e0 con 0x7f19a40b9a10 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.459+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f19a40cc6e0 con 0x7f19a40b9a10 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.460+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f19a4067ff0 con 0x7f19a40b9a10 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.460+0000 7f199bfff640 1 -- 172.21.15.64:0/2116854077 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f199401a070 con 0x7f19a40b9a10 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.460+0000 7f199bfff640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f198406b4d0 0x7f198406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.460+0000 7f199bfff640 1 -- 172.21.15.64:0/2116854077 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f199408a1d0 con 0x7f19a40b9a10 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.460+0000 7f19a37fe640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f198406b4d0 0x7f198406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:55.465 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.461+0000 7f19a37fe640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f198406b4d0 0x7f198406d990 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f198c007fd0 tx=0x7f198c007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:55.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.465+0000 7f199bfff640 1 -- 172.21.15.64:0/2116854077 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f19940589c0 con 0x7f19a40b9a10 2024-03-04T07:34:55.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.685+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f19a40c7530 con 0x7f19a40b9a10 2024-03-04T07:34:55.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.686+0000 7f199bfff640 1 -- 172.21.15.64:0/2116854077 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f199405ca60 con 0x7f19a40b9a10 2024-03-04T07:34:55.686 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:55.686 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:55.686 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:55.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f198406b4d0 msgr2=0x7f198406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:55.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f198406b4d0 0x7f198406d990 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f198c007fd0 tx=0x7f198c007de0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 msgr2=0x7f19a40b75e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:55.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b75e0 secure :-1 s=READY pgs=1648 cs=0 l=1 rev1=1 crypto rx=0x7f199400e9e0 tx=0x7f199400eea0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 shutdown_connections 2024-03-04T07:34:55.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f198406b4d0 0x7f198406d990 unknown :-1 s=CLOSED pgs=740 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f19a415df00 0x7f19a40b7b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f19a40b9a10 0x7f19a40b75e0 unknown :-1 s=CLOSED pgs=1648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 --2- 172.21.15.64:0/2116854077 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f19a40b95a0 0x7f19a415ef90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:55.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 >> 172.21.15.64:0/2116854077 conn(0x7f19a4071960 msgr2=0x7f19a4071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:55.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 shutdown_connections 2024-03-04T07:34:55.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:55.688+0000 7f19aa1f7640 1 -- 172.21.15.64:0/2116854077 wait complete. 2024-03-04T07:34:55.900 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:55.900 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:56.568 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:56.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:57.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.012+0000 7f273b666640 1 Processor -- start 2024-03-04T07:34:57.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.012+0000 7f273b666640 1 -- start start 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.013+0000 7f273b666640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.013+0000 7f273b666640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f27340b95a0 0x7f27340b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.013+0000 7f273b666640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f27340cea30 con 0x7f27340b9a30 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.013+0000 7f273b666640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f27340d2af0 con 0x7f27340b95a0 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.013+0000 7f273b666640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f27340cf3c0 con 0x7f273415df00 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2738c01640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f27340b95a0 0x7f27340b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2739c03640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f27340b9a30 0x7f27340d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54782/0 (socket says 172.21.15.64:54782) 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2739402640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:57.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2739c03640 1 -- 172.21.15.64:0/3160947867 learned_addr learned my addr 172.21.15.64:0/3160947867 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:57.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2738c01640 1 -- 172.21.15.64:0/3160947867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 msgr2=0x7f27340d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:34:57.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2738c01640 1 --2- 172.21.15.64:0/3160947867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2738c01640 1 -- 172.21.15.64:0/3160947867 >> v1:172.21.15.64:6789/0 conn(0x7f27340b9a30 legacy=0x7f27340d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:57.015 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.014+0000 7f2738c01640 1 -- 172.21.15.64:0/3160947867 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27340cf0e0 con 0x7f27340b95a0 2024-03-04T07:34:57.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.015+0000 7f2738c01640 1 --2- 172.21.15.64:0/3160947867 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27340b5700 secure :-1 s=READY pgs=1650 cs=0 l=1 rev1=1 crypto rx=0x7f27280097a0 tx=0x7f272802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2bee897bc387d03 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:57.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.015+0000 7f2733fff640 1 -- 172.21.15.64:0/3160947867 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2728035020 con 0x7f27340b95a0 2024-03-04T07:34:57.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.015+0000 7f2733fff640 1 -- 172.21.15.64:0/3160947867 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2728031830 con 0x7f27340b95a0 2024-03-04T07:34:57.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.015+0000 7f2733fff640 1 -- 172.21.15.64:0/3160947867 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2728031c00 con 0x7f27340b95a0 2024-03-04T07:34:57.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.015+0000 7f273b666640 1 -- 172.21.15.64:0/3160947867 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 msgr2=0x7f27340b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:57.016 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.015+0000 7f273b666640 1 --2- 172.21.15.64:0/3160947867 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27340b5700 secure :-1 s=READY pgs=1650 cs=0 l=1 rev1=1 crypto rx=0x7f27280097a0 tx=0x7f272802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.016+0000 7f273b666640 1 -- 172.21.15.64:0/3160947867 shutdown_connections 2024-03-04T07:34:57.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.016+0000 7f273b666640 1 --2- 172.21.15.64:0/3160947867 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27340b5700 unknown :-1 s=CLOSED pgs=1650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.016+0000 7f273b666640 1 --2- 172.21.15.64:0/3160947867 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.016+0000 7f273b666640 1 -- 172.21.15.64:0/3160947867 >> 172.21.15.64:0/3160947867 conn(0x7f2734071960 msgr2=0x7f2734071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:57.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.016+0000 7f273b666640 1 -- 172.21.15.64:0/3160947867 shutdown_connections 2024-03-04T07:34:57.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.016+0000 7f273b666640 1 -- 172.21.15.64:0/3160947867 wait complete. 2024-03-04T07:34:57.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.017+0000 7f273b666640 1 Processor -- start 2024-03-04T07:34:57.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.017+0000 7f273b666640 1 -- start start 2024-03-04T07:34:57.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f273b666640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27341f7cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:57.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f273b666640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340b5400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:57.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f273b666640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27341f8230 0x7f273415f260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:57.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f273b666640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27340cec70 con 0x7f27340b95a0 2024-03-04T07:34:57.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f273b666640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27340ce750 con 0x7f27341f8230 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f273b666640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f27340cedf0 con 0x7f273415df00 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27341f7cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27341f7cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37212/0 (socket says 172.21.15.64:37212) 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 -- 172.21.15.64:0/4243660124 learned_addr learned my addr 172.21.15.64:0/4243660124 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2738c01640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340b5400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739c03640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27341f8230 0x7f273415f260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 -- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 msgr2=0x7f27340b5400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340b5400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 -- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27341f8230 msgr2=0x7f273415f260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27341f8230 0x7f273415f260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 -- 172.21.15.64:0/4243660124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27341f8740 con 0x7f27340b95a0 2024-03-04T07:34:57.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2739402640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27341f7cf0 secure :-1 s=READY pgs=1651 cs=0 l=1 rev1=1 crypto rx=0x7f272400efc0 tx=0x7f272400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:57.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2731ffb640 1 -- 172.21.15.64:0/4243660124 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2724018020 con 0x7f27340b95a0 2024-03-04T07:34:57.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2731ffb640 1 -- 172.21.15.64:0/4243660124 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2724014480 con 0x7f27340b95a0 2024-03-04T07:34:57.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f2731ffb640 1 -- 172.21.15.64:0/4243660124 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2724014850 con 0x7f27340b95a0 2024-03-04T07:34:57.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.018+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f27341f8a20 con 0x7f27340b95a0 2024-03-04T07:34:57.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.019+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f27340b33e0 con 0x7f27340b95a0 2024-03-04T07:34:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.020+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f27340b68a0 con 0x7f27340b95a0 2024-03-04T07:34:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.020+0000 7f2731ffb640 1 -- 172.21.15.64:0/4243660124 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2724005d10 con 0x7f27340b95a0 2024-03-04T07:34:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.020+0000 7f2731ffb640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f272006b4d0 0x7f272006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.021+0000 7f2731ffb640 1 -- 172.21.15.64:0/4243660124 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f272408abe0 con 0x7f27340b95a0 2024-03-04T07:34:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.021+0000 7f2738c01640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f272006b4d0 0x7f272006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:57.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.021+0000 7f2738c01640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f272006b4d0 0x7f272006d990 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f2728036000 tx=0x7f2728008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:57.025 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.025+0000 7f2731ffb640 1 -- 172.21.15.64:0/4243660124 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f27340b68a0 con 0x7f27340b95a0 2024-03-04T07:34:57.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.244+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2734067ff0 con 0x7f27340b95a0 2024-03-04T07:34:57.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.244+0000 7f2731ffb640 1 -- 172.21.15.64:0/4243660124 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f2724022070 con 0x7f27340b95a0 2024-03-04T07:34:57.244 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:57.245 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:57.245 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f272006b4d0 msgr2=0x7f272006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f272006b4d0 0x7f272006d990 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f2728036000 tx=0x7f2728008030 comp rx=0 tx=0).stop 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 msgr2=0x7f27341f7cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27341f7cf0 secure :-1 s=READY pgs=1651 cs=0 l=1 rev1=1 crypto rx=0x7f272400efc0 tx=0x7f272400c490 comp rx=0 tx=0).stop 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 shutdown_connections 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f272006b4d0 0x7f272006d990 unknown :-1 s=CLOSED pgs=741 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27341f8230 0x7f273415f260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f273415df00 0x7f27340b5400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 --2- 172.21.15.64:0/4243660124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27340b95a0 0x7f27341f7cf0 unknown :-1 s=CLOSED pgs=1651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:57.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 >> 172.21.15.64:0/4243660124 conn(0x7f2734071960 msgr2=0x7f27340d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:57.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 shutdown_connections 2024-03-04T07:34:57.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:57.247+0000 7f273b666640 1 -- 172.21.15.64:0/4243660124 wait complete. 2024-03-04T07:34:57.459 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:57.459 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:34:58.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:34:58.571 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:34:58.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.574+0000 7ff019b65640 1 Processor -- start 2024-03-04T07:34:58.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.574+0000 7ff019b65640 1 -- start start 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff019b65640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff01415e060 0x7ff0140ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff019b65640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0140d0dd0 con 0x7ff01406fe20 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff019b65640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0140d10b0 con 0x7ff0140d0880 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff019b65640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0140cc0e0 con 0x7ff01415e060 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff012ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff01406fe20 0x7ff0140ce850 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54798/0 (socket says 172.21.15.64:54798) 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff012ffd640 1 -- 172.21.15.64:0/1271857643 learned_addr learned my addr 172.21.15.64:0/1271857643 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff0137fe640 1 --2- 172.21.15.64:0/1271857643 >> v2:172.21.15.66:3300/0 conn(0x7ff01415e060 0x7ff0140ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff0127fc640 1 -- 172.21.15.64:0/1271857643 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 887675542 0 0) 0x7ff0140d0dd0 con 0x7ff01406fe20 2024-03-04T07:34:58.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff0127fc640 1 -- 172.21.15.64:0/1271857643 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7feffc003520 con 0x7ff01406fe20 2024-03-04T07:34:58.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff0137fe640 1 -- 172.21.15.64:0/1271857643 >> v1:172.21.15.66:6789/0 conn(0x7ff0140d0880 legacy=0x7ff0140ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:58.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.575+0000 7ff0137fe640 1 -- 172.21.15.64:0/1271857643 >> v1:172.21.15.64:6789/0 conn(0x7ff01406fe20 legacy=0x7ff0140ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:58.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.576+0000 7ff0137fe640 1 -- 172.21.15.64:0/1271857643 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0140cc3c0 con 0x7ff01415e060 2024-03-04T07:34:58.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.576+0000 7ff0137fe640 1 --2- 172.21.15.64:0/1271857643 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff01415e060 0x7ff0140ce310 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7ff0040097a0 tx=0x7ff00402b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9b83987acb136f9e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:58.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.576+0000 7ff0127fc640 1 -- 172.21.15.64:0/1271857643 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff004034070 con 0x7ff01415e060 2024-03-04T07:34:58.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.576+0000 7ff0127fc640 1 -- 172.21.15.64:0/1271857643 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff004030610 con 0x7ff01415e060 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.576+0000 7ff0127fc640 1 -- 172.21.15.64:0/1271857643 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0040309c0 con 0x7ff01415e060 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.577+0000 7ff019b65640 1 -- 172.21.15.64:0/1271857643 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff01415e060 msgr2=0x7ff0140ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.577+0000 7ff019b65640 1 --2- 172.21.15.64:0/1271857643 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff01415e060 0x7ff0140ce310 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7ff0040097a0 tx=0x7ff00402b960 comp rx=0 tx=0).stop 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.577+0000 7ff019b65640 1 -- 172.21.15.64:0/1271857643 shutdown_connections 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.577+0000 7ff019b65640 1 --2- 172.21.15.64:0/1271857643 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff01415e060 0x7ff0140ce310 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.577+0000 7ff019b65640 1 -- 172.21.15.64:0/1271857643 >> 172.21.15.64:0/1271857643 conn(0x7ff0140d1a90 msgr2=0x7ff0140d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.577+0000 7ff019b65640 1 -- 172.21.15.64:0/1271857643 shutdown_connections 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.577+0000 7ff019b65640 1 -- 172.21.15.64:0/1271857643 wait complete. 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.578+0000 7ff019b65640 1 Processor -- start 2024-03-04T07:34:58.579 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.578+0000 7ff019b65640 1 -- start start 2024-03-04T07:34:58.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff019b65640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff01406fe20 0x7ff014157e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:58.582 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff019b65640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0140d0880 0x7ff0141f49a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff019b65640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff01415e060 0x7ff0141f7450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff019b65640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0140cef60 con 0x7ff01406fe20 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff019b65640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff01415e440 con 0x7ff0140d0880 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff019b65640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0140cf0e0 con 0x7ff01415e060 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff0137fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff01406fe20 0x7ff014157e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff0137fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff01406fe20 0x7ff014157e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37214/0 (socket says 172.21.15.64:37214) 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff0137fe640 1 -- 172.21.15.64:0/3466081348 learned_addr learned my addr 172.21.15.64:0/3466081348 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff012ffd640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0140d0880 0x7ff0141f49a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.579+0000 7ff013fff640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff01415e060 0x7ff0141f7450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff0137fe640 1 -- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff01415e060 msgr2=0x7ff0141f7450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff0137fe640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff01415e060 0x7ff0141f7450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.583 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff0137fe640 1 -- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0140d0880 msgr2=0x7ff0141f49a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff0137fe640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0140d0880 0x7ff0141f49a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff0137fe640 1 -- 172.21.15.64:0/3466081348 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0141583a0 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff012ffd640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0140d0880 0x7ff0141f49a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff0137fe640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff01406fe20 0x7ff014157e60 secure :-1 s=READY pgs=1653 cs=0 l=1 rev1=1 crypto rx=0x7ff004000c00 tx=0x7ff0040046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7feffbfff640 1 -- 172.21.15.64:0/3466081348 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff004047070 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7feffbfff640 1 -- 172.21.15.64:0/3466081348 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff004030410 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7feffbfff640 1 -- 172.21.15.64:0/3466081348 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff004034040 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff0141f3ee0 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.580+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff0141f8110 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.581+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff014067ff0 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.582+0000 7feffbfff640 1 -- 172.21.15.64:0/3466081348 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff00404a070 con 0x7ff01406fe20 2024-03-04T07:34:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.582+0000 7feffbfff640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff00006b4d0 0x7ff00006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:34:58.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.582+0000 7feffbfff640 1 -- 172.21.15.64:0/3466081348 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff0040aba80 con 0x7ff01406fe20 2024-03-04T07:34:58.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.582+0000 7ff012ffd640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff00006b4d0 0x7ff00006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:34:58.585 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.583+0000 7ff012ffd640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff00006b4d0 0x7ff00006d990 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7ff0140cbb60 tx=0x7ff008007d30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:34:58.586 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.587+0000 7feffbfff640 1 -- 172.21.15.64:0/3466081348 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff00407a300 con 0x7ff01406fe20 2024-03-04T07:34:58.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.808+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff0140d2260 con 0x7ff01406fe20 2024-03-04T07:34:58.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.808+0000 7feffbfff640 1 -- 172.21.15.64:0/3466081348 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7ff00407e3a0 con 0x7ff01406fe20 2024-03-04T07:34:58.809 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:34:58.809 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:34:58.809 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff00006b4d0 msgr2=0x7ff00006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff00006b4d0 0x7ff00006d990 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7ff0140cbb60 tx=0x7ff008007d30 comp rx=0 tx=0).stop 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff01406fe20 msgr2=0x7ff014157e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff01406fe20 0x7ff014157e60 secure :-1 s=READY pgs=1653 cs=0 l=1 rev1=1 crypto rx=0x7ff004000c00 tx=0x7ff0040046c0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 shutdown_connections 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff00006b4d0 0x7ff00006d990 unknown :-1 s=CLOSED pgs=742 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff01415e060 0x7ff0141f7450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0140d0880 0x7ff0141f49a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 --2- 172.21.15.64:0/3466081348 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff01406fe20 0x7ff014157e60 unknown :-1 s=CLOSED pgs=1653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 >> 172.21.15.64:0/3466081348 conn(0x7ff0140d1a90 msgr2=0x7ff0140b2300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 shutdown_connections 2024-03-04T07:34:58.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:34:58.810+0000 7ff019b65640 1 -- 172.21.15.64:0/3466081348 wait complete. 2024-03-04T07:34:59.023 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:34:59.023 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:00.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:00.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.138+0000 7fa9a7471640 1 Processor -- start 2024-03-04T07:35:00.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.139+0000 7fa9a7471640 1 -- start start 2024-03-04T07:35:00.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.139+0000 7fa9a7471640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa9a00b1020 0x7fa9a006af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:00.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.139+0000 7fa9a7471640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa9a006ed10 0x7fa9a00cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:00.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.139+0000 7fa9a7471640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a00cd6c0 0x7fa9a00cdad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:00.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.139+0000 7fa9a7471640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa9a00d28b0 con 0x7fa9a006ed10 2024-03-04T07:35:00.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.139+0000 7fa9a7471640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa9a00cc0e0 con 0x7fa9a00b1020 2024-03-04T07:35:00.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.139+0000 7fa9a7471640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a00cc3c0 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa9a006ed10 0x7fa9a00cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fa9a006ed10 0x7fa9a00cd180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37222/0 (socket says 172.21.15.64:37222) 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 -- 172.21.15.64:0/398432670 learned_addr learned my addr 172.21.15.64:0/398432670 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a520d640 1 --2- 172.21.15.64:0/398432670 >> v2:172.21.15.66:3300/0 conn(0x7fa9a00b1020 0x7fa9a006af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a5a0e640 1 --2- 172.21.15.64:0/398432670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a00cd6c0 0x7fa9a00cdad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 -- 172.21.15.64:0/398432670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a00cd6c0 msgr2=0x7fa9a00cdad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 --2- 172.21.15.64:0/398432670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a00cd6c0 0x7fa9a00cdad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 -- 172.21.15.64:0/398432670 >> v2:172.21.15.66:3300/0 conn(0x7fa9a00b1020 msgr2=0x7fa9a006af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 --2- 172.21.15.64:0/398432670 >> v2:172.21.15.66:3300/0 conn(0x7fa9a00b1020 0x7fa9a006af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.140+0000 7fa9a4a0c640 1 -- 172.21.15.64:0/398432670 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9a00cc6a0 con 0x7fa9a006ed10 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a4a0c640 1 --2- 172.21.15.64:0/398432670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a006ed10 0x7fa9a00cd180 secure :-1 s=READY pgs=1654 cs=0 l=1 rev1=1 crypto rx=0x7fa9a0070d50 tx=0x7fa99002b900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4b7daaf7a0d16613 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa997fff640 1 -- 172.21.15.64:0/398432670 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa990035020 con 0x7fa9a006ed10 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa997fff640 1 -- 172.21.15.64:0/398432670 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa99002d040 con 0x7fa9a006ed10 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa997fff640 1 -- 172.21.15.64:0/398432670 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa990031a80 con 0x7fa9a006ed10 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a7471640 1 -- 172.21.15.64:0/398432670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a006ed10 msgr2=0x7fa9a00cd180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a7471640 1 --2- 172.21.15.64:0/398432670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a006ed10 0x7fa9a00cd180 secure :-1 s=READY pgs=1654 cs=0 l=1 rev1=1 crypto rx=0x7fa9a0070d50 tx=0x7fa99002b900 comp rx=0 tx=0).stop 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a7471640 1 -- 172.21.15.64:0/398432670 shutdown_connections 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a7471640 1 --2- 172.21.15.64:0/398432670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a00cd6c0 0x7fa9a00cdad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a7471640 1 --2- 172.21.15.64:0/398432670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a006ed10 0x7fa9a00cd180 unknown :-1 s=CLOSED pgs=1654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a7471640 1 --2- 172.21.15.64:0/398432670 >> v2:172.21.15.66:3300/0 conn(0x7fa9a00b1020 0x7fa9a006af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.141+0000 7fa9a7471640 1 -- 172.21.15.64:0/398432670 >> 172.21.15.64:0/398432670 conn(0x7fa9a00d1a90 msgr2=0x7fa9a00d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:00.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.142+0000 7fa9a7471640 1 -- 172.21.15.64:0/398432670 shutdown_connections 2024-03-04T07:35:00.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.142+0000 7fa9a7471640 1 -- 172.21.15.64:0/398432670 wait complete. 2024-03-04T07:35:00.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.143+0000 7fa9a7471640 1 Processor -- start 2024-03-04T07:35:00.144 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.143+0000 7fa9a7471640 1 -- start start 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a7471640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a006ed10 0x7fa9a01f8170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a7471640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9a00b1020 0x7fa9a0153b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a7471640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a00cd6c0 0x7fa9a00b48e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a7471640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a015d700 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a7471640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a00ccc60 con 0x7fa9a00b1020 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a7471640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a015d880 con 0x7fa9a006ed10 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a00cd6c0 0x7fa9a00b48e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a520d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a006ed10 0x7fa9a01f8170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a00cd6c0 0x7fa9a00b48e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37236/0 (socket says 172.21.15.64:37236) 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 -- 172.21.15.64:0/976558355 learned_addr learned my addr 172.21.15.64:0/976558355 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a4a0c640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9a00b1020 0x7fa9a0153b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 -- 172.21.15.64:0/976558355 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a006ed10 msgr2=0x7fa9a01f8170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a006ed10 0x7fa9a01f8170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 -- 172.21.15.64:0/976558355 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9a00b1020 msgr2=0x7fa9a0153b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9a00b1020 0x7fa9a0153b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 -- 172.21.15.64:0/976558355 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9a00b5730 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.144+0000 7fa9a5a0e640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a00cd6c0 0x7fa9a00b48e0 secure :-1 s=READY pgs=1655 cs=0 l=1 rev1=1 crypto rx=0x7fa998009d30 tx=0x7fa99800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.145+0000 7fa995ffb640 1 -- 172.21.15.64:0/976558355 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa998018020 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.145+0000 7fa995ffb640 1 -- 172.21.15.64:0/976558355 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa998014480 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.145+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa9a00b5450 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.145+0000 7fa995ffb640 1 -- 172.21.15.64:0/976558355 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa998014870 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.145+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9a01f7970 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.146+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa9a0067ff0 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.147 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.146+0000 7fa995ffb640 1 -- 172.21.15.64:0/976558355 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa998005d10 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.147+0000 7fa995ffb640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa98406b500 0x7fa98406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.147+0000 7fa995ffb640 1 -- 172.21.15.64:0/976558355 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa99808aa30 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.147+0000 7fa9a520d640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa98406b500 0x7fa98406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.147+0000 7fa9a520d640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa98406b500 0x7fa98406d9c0 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7fa99c009b40 tx=0x7fa99c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:00.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.151+0000 7fa995ffb640 1 -- 172.21.15.64:0/976558355 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa998022070 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.369+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa9a00b4370 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.369+0000 7fa995ffb640 1 -- 172.21.15.64:0/976558355 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fa998010040 con 0x7fa9a00cd6c0 2024-03-04T07:35:00.370 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:00.370 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:00.370 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa98406b500 msgr2=0x7fa98406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa98406b500 0x7fa98406d9c0 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7fa99c009b40 tx=0x7fa99c009190 comp rx=0 tx=0).stop 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a00cd6c0 msgr2=0x7fa9a00b48e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a00cd6c0 0x7fa9a00b48e0 secure :-1 s=READY pgs=1655 cs=0 l=1 rev1=1 crypto rx=0x7fa998009d30 tx=0x7fa99800c490 comp rx=0 tx=0).stop 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 shutdown_connections 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa98406b500 0x7fa98406d9c0 unknown :-1 s=CLOSED pgs=743 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa9a00cd6c0 0x7fa9a00b48e0 unknown :-1 s=CLOSED pgs=1655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa9a00b1020 0x7fa9a0153b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 --2- 172.21.15.64:0/976558355 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa9a006ed10 0x7fa9a01f8170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 >> 172.21.15.64:0/976558355 conn(0x7fa9a00d1a90 msgr2=0x7fa9a00ce9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:00.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.371+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 shutdown_connections 2024-03-04T07:35:00.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:00.372+0000 7fa9a7471640 1 -- 172.21.15.64:0/976558355 wait complete. 2024-03-04T07:35:00.572 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:00.584 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:00.584 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:01.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:01.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.699+0000 7f73a415c640 1 Processor -- start 2024-03-04T07:35:01.699 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.699+0000 7f73a415c640 1 -- start start 2024-03-04T07:35:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a415c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c0aee10 0x7f739c15cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a415c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f739c06d400 0x7f739c070870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a415c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f739c0729b0 con 0x7f739c15bac0 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a415c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f739c06cf70 con 0x7f739c06d400 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a415c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f739c06c920 con 0x7f739c0aee10 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a26f9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f739c06d400 0x7f739c070870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a1ef8640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f739c15bac0 0x7f739c0ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54812/0 (socket says 172.21.15.64:54812) 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a1ef8640 1 -- 172.21.15.64:0/3081970210 learned_addr learned my addr 172.21.15.64:0/3081970210 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.700+0000 7f73a16f7640 1 --2- 172.21.15.64:0/3081970210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c0aee10 0x7f739c15cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.701+0000 7f73a26f9640 1 -- 172.21.15.64:0/3081970210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c0aee10 msgr2=0x7f739c15cbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.701+0000 7f73a26f9640 1 --2- 172.21.15.64:0/3081970210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c0aee10 0x7f739c15cbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.701+0000 7f73a26f9640 1 -- 172.21.15.64:0/3081970210 >> v1:172.21.15.64:6789/0 conn(0x7f739c15bac0 legacy=0x7f739c0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:01.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.701+0000 7f73a26f9640 1 -- 172.21.15.64:0/3081970210 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f739c06cc00 con 0x7f739c06d400 2024-03-04T07:35:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.701+0000 7f73a0ef6640 1 -- 172.21.15.64:0/3081970210 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 928874580 0 0) 0x7f739c0729b0 con 0x7f739c15bac0 2024-03-04T07:35:01.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.701+0000 7f73a26f9640 1 --2- 172.21.15.64:0/3081970210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c070870 secure :-1 s=READY pgs=1657 cs=0 l=1 rev1=1 crypto rx=0x7f73980097a0 tx=0x7f739802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e754ffc15d757036 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:01.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a0ef6640 1 -- 172.21.15.64:0/3081970210 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7398035020 con 0x7f739c06d400 2024-03-04T07:35:01.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a0ef6640 1 -- 172.21.15.64:0/3081970210 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7398031830 con 0x7f739c06d400 2024-03-04T07:35:01.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a0ef6640 1 -- 172.21.15.64:0/3081970210 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7398031c00 con 0x7f739c06d400 2024-03-04T07:35:01.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 -- 172.21.15.64:0/3081970210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 msgr2=0x7f739c070870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:01.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 --2- 172.21.15.64:0/3081970210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c070870 secure :-1 s=READY pgs=1657 cs=0 l=1 rev1=1 crypto rx=0x7f73980097a0 tx=0x7f739802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 -- 172.21.15.64:0/3081970210 shutdown_connections 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 --2- 172.21.15.64:0/3081970210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c070870 unknown :-1 s=CLOSED pgs=1657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 --2- 172.21.15.64:0/3081970210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c0aee10 0x7f739c15cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 -- 172.21.15.64:0/3081970210 >> 172.21.15.64:0/3081970210 conn(0x7f739c066d00 msgr2=0x7f739c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 -- 172.21.15.64:0/3081970210 shutdown_connections 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.702+0000 7f73a415c640 1 -- 172.21.15.64:0/3081970210 wait complete. 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.703+0000 7f73a415c640 1 Processor -- start 2024-03-04T07:35:01.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.703+0000 7f73a415c640 1 -- start start 2024-03-04T07:35:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a415c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c0adb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a415c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f739c0aee10 0x7f739c0b29c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a415c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c15bac0 0x7f739c156ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a415c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f739c15c020 con 0x7f739c06d400 2024-03-04T07:35:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a415c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f739c15bea0 con 0x7f739c0aee10 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a415c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f739c072580 con 0x7f739c15bac0 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a1ef8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c0adb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a1ef8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c0adb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37252/0 (socket says 172.21.15.64:37252) 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a1ef8640 1 -- 172.21.15.64:0/501022898 learned_addr learned my addr 172.21.15.64:0/501022898 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a26f9640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c15bac0 0x7f739c156ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.704+0000 7f73a16f7640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f739c0aee10 0x7f739c0b29c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a1ef8640 1 -- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c15bac0 msgr2=0x7f739c156ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a1ef8640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c15bac0 0x7f739c156ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a1ef8640 1 -- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f739c0aee10 msgr2=0x7f739c0b29c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a1ef8640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f739c0aee10 0x7f739c0b29c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a1ef8640 1 -- 172.21.15.64:0/501022898 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f739c0b3330 con 0x7f739c06d400 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a1ef8640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c0adb20 secure :-1 s=READY pgs=1658 cs=0 l=1 rev1=1 crypto rx=0x7f739400c890 tx=0x7f739400cd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73927fc640 1 -- 172.21.15.64:0/501022898 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7394018070 con 0x7f739c06d400 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73927fc640 1 -- 172.21.15.64:0/501022898 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f739400a640 con 0x7f739c06d400 2024-03-04T07:35:01.707 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f739c0b3610 con 0x7f739c06d400 2024-03-04T07:35:01.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73927fc640 1 -- 172.21.15.64:0/501022898 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f739400aa30 con 0x7f739c06d400 2024-03-04T07:35:01.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.705+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f739c0ae060 con 0x7f739c06d400 2024-03-04T07:35:01.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.706+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f739c067ff0 con 0x7f739c06d400 2024-03-04T07:35:01.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.707+0000 7f73927fc640 1 -- 172.21.15.64:0/501022898 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f739401b070 con 0x7f739c06d400 2024-03-04T07:35:01.708 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.707+0000 7f73927fc640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738006b4d0 0x7f738006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:01.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.707+0000 7f73927fc640 1 -- 172.21.15.64:0/501022898 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f73940897f0 con 0x7f739c06d400 2024-03-04T07:35:01.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.707+0000 7f73a16f7640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738006b4d0 0x7f738006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:01.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.708+0000 7f73a16f7640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738006b4d0 0x7f738006d990 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f739c067200 tx=0x7f738c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:01.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.711+0000 7f73927fc640 1 -- 172.21.15.64:0/501022898 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7394058070 con 0x7f739c06d400 2024-03-04T07:35:01.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.930+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f739c1e42f0 con 0x7f739c06d400 2024-03-04T07:35:01.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.930+0000 7f73927fc640 1 -- 172.21.15.64:0/501022898 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f739405c110 con 0x7f739c06d400 2024-03-04T07:35:01.930 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:01.930 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:01.931 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:01.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738006b4d0 msgr2=0x7f738006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:01.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738006b4d0 0x7f738006d990 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f739c067200 tx=0x7f738c009190 comp rx=0 tx=0).stop 2024-03-04T07:35:01.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 msgr2=0x7f739c0adb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:01.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c0adb20 secure :-1 s=READY pgs=1658 cs=0 l=1 rev1=1 crypto rx=0x7f739400c890 tx=0x7f739400cd50 comp rx=0 tx=0).stop 2024-03-04T07:35:01.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 shutdown_connections 2024-03-04T07:35:01.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f738006b4d0 0x7f738006d990 unknown :-1 s=CLOSED pgs=744 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f739c15bac0 0x7f739c156ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f739c0aee10 0x7f739c0b29c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 --2- 172.21.15.64:0/501022898 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f739c06d400 0x7f739c0adb20 unknown :-1 s=CLOSED pgs=1658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:01.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 >> 172.21.15.64:0/501022898 conn(0x7f739c066d00 msgr2=0x7f739c15e360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:01.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 shutdown_connections 2024-03-04T07:35:01.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:01.932+0000 7f73a415c640 1 -- 172.21.15.64:0/501022898 wait complete. 2024-03-04T07:35:02.144 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:02.144 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:02.575 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:03.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:03.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.259+0000 7fdac1b1e640 1 Processor -- start 2024-03-04T07:35:03.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.259+0000 7fdac1b1e640 1 -- start start 2024-03-04T07:35:03.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdac1b1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc0d18c0 0x7fdabc071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:03.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdac1b1e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdabc0b6f20 con 0x7fdabc15df70 2024-03-04T07:35:03.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdac1b1e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fdabc0d3030 con 0x7fdabc071a80 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdac1b1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdabc0d3310 con 0x7fdabc0d18c0 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdabaffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc0d18c0 0x7fdabc071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdabb7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fdabc15df70 0x7fdabc0d27b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:54822/0 (socket says 172.21.15.64:54822) 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdabbfff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fdabc071a80 0x7fdabc0b2700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:52624/0 (socket says 172.21.15.64:52624) 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdabb7fe640 1 -- 172.21.15.64:0/4031968630 learned_addr learned my addr 172.21.15.64:0/4031968630 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdaba7fc640 1 -- 172.21.15.64:0/4031968630 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2752355148 0 0) 0x7fdabc0b6f20 con 0x7fdabc15df70 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.260+0000 7fdaba7fc640 1 -- 172.21.15.64:0/4031968630 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fdaa4003520 con 0x7fdabc15df70 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.261+0000 7fdabaffd640 1 -- 172.21.15.64:0/4031968630 >> v1:172.21.15.66:6789/0 conn(0x7fdabc071a80 legacy=0x7fdabc0b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.261+0000 7fdaba7fc640 1 -- 172.21.15.64:0/4031968630 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1388420141 0 0) 0x7fdabc0d3030 con 0x7fdabc071a80 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.261+0000 7fdabaffd640 1 -- 172.21.15.64:0/4031968630 >> v1:172.21.15.64:6789/0 conn(0x7fdabc15df70 legacy=0x7fdabc0d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:03.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.261+0000 7fdabaffd640 1 -- 172.21.15.64:0/4031968630 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdabc0d35f0 con 0x7fdabc0d18c0 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.261+0000 7fdabaffd640 1 --2- 172.21.15.64:0/4031968630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc0d18c0 0x7fdabc071540 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fdab00097d0 tx=0x7fdab002b9d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e506b43ea22fd2a0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.261+0000 7fdaba7fc640 1 -- 172.21.15.64:0/4031968630 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdab0034070 con 0x7fdabc0d18c0 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdaba7fc640 1 -- 172.21.15.64:0/4031968630 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdab0030890 con 0x7fdabc0d18c0 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdaba7fc640 1 -- 172.21.15.64:0/4031968630 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdab0030c60 con 0x7fdabc0d18c0 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdac1b1e640 1 -- 172.21.15.64:0/4031968630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc0d18c0 msgr2=0x7fdabc071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/4031968630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc0d18c0 0x7fdabc071540 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fdab00097d0 tx=0x7fdab002b9d0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdac1b1e640 1 -- 172.21.15.64:0/4031968630 shutdown_connections 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/4031968630 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc0d18c0 0x7fdabc071540 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdac1b1e640 1 -- 172.21.15.64:0/4031968630 >> 172.21.15.64:0/4031968630 conn(0x7fdabc0d3cf0 msgr2=0x7fdabc0d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdac1b1e640 1 -- 172.21.15.64:0/4031968630 shutdown_connections 2024-03-04T07:35:03.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.262+0000 7fdac1b1e640 1 -- 172.21.15.64:0/4031968630 wait complete. 2024-03-04T07:35:03.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.263+0000 7fdac1b1e640 1 Processor -- start 2024-03-04T07:35:03.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdac1b1e640 1 -- start start 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdac1b1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc071a80 0x7fdabc14d190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdac1b1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdabc0d18c0 0x7fdabc1503e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdac1b1e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdabc15df70 0x7fdabc0cca60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdac1b1e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdabc15e4d0 con 0x7fdabc0d18c0 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdac1b1e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdabc15e350 con 0x7fdabc15df70 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdac1b1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fdabc0b8410 con 0x7fdabc071a80 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdabaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdabc0d18c0 0x7fdabc1503e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdabb7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc071a80 0x7fdabc14d190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdabbfff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdabc15df70 0x7fdabc0cca60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdabaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdabc0d18c0 0x7fdabc1503e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37260/0 (socket says 172.21.15.64:37260) 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.264+0000 7fdabaffd640 1 -- 172.21.15.64:0/1421057711 learned_addr learned my addr 172.21.15.64:0/1421057711 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdabaffd640 1 -- 172.21.15.64:0/1421057711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc071a80 msgr2=0x7fdabc14d190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdabaffd640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc071a80 0x7fdabc14d190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.267 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdabaffd640 1 -- 172.21.15.64:0/1421057711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdabc15df70 msgr2=0x7fdabc0cca60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdabaffd640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdabc15df70 0x7fdabc0cca60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdabaffd640 1 -- 172.21.15.64:0/1421057711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdabc150920 con 0x7fdabc0d18c0 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdabbfff640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdabc15df70 0x7fdabc0cca60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdabaffd640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdabc0d18c0 0x7fdabc1503e0 secure :-1 s=READY pgs=1660 cs=0 l=1 rev1=1 crypto rx=0x7fdab00097d0 tx=0x7fdab00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdaa3fff640 1 -- 172.21.15.64:0/1421057711 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdab0034040 con 0x7fdabc0d18c0 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdaa3fff640 1 -- 172.21.15.64:0/1421057711 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fdab0030690 con 0x7fdabc0d18c0 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdabc14dcb0 con 0x7fdabc0d18c0 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdaa3fff640 1 -- 172.21.15.64:0/1421057711 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdab0039d20 con 0x7fdabc0d18c0 2024-03-04T07:35:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.265+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdabc14e270 con 0x7fdabc0d18c0 2024-03-04T07:35:03.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.266+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdabc067ff0 con 0x7fdabc0d18c0 2024-03-04T07:35:03.269 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.267+0000 7fdaa3fff640 1 -- 172.21.15.64:0/1421057711 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fdab004b060 con 0x7fdabc0d18c0 2024-03-04T07:35:03.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.267+0000 7fdaa3fff640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaa806b500 0x7fdaa806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:03.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.267+0000 7fdabb7fe640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaa806b500 0x7fdaa806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:03.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.267+0000 7fdaa3fff640 1 -- 172.21.15.64:0/1421057711 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fdab00abf10 con 0x7fdabc0d18c0 2024-03-04T07:35:03.270 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.268+0000 7fdabb7fe640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaa806b500 0x7fdaa806d9c0 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7fdaac005880 tx=0x7fdaac009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:03.271 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.272+0000 7fdaa3fff640 1 -- 172.21.15.64:0/1421057711 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fdabc150920 con 0x7fdabc0d18c0 2024-03-04T07:35:03.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.488+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fdabc0d2ac0 con 0x7fdabc0d18c0 2024-03-04T07:35:03.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.489+0000 7fdaa3fff640 1 -- 172.21.15.64:0/1421057711 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fdab007a830 con 0x7fdabc0d18c0 2024-03-04T07:35:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:03.489 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:03.490 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaa806b500 msgr2=0x7fdaa806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaa806b500 0x7fdaa806d9c0 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7fdaac005880 tx=0x7fdaac009040 comp rx=0 tx=0).stop 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdabc0d18c0 msgr2=0x7fdabc1503e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdabc0d18c0 0x7fdabc1503e0 secure :-1 s=READY pgs=1660 cs=0 l=1 rev1=1 crypto rx=0x7fdab00097d0 tx=0x7fdab00046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 shutdown_connections 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fdaa806b500 0x7fdaa806d9c0 unknown :-1 s=CLOSED pgs=745 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fdabc15df70 0x7fdabc0cca60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fdabc0d18c0 0x7fdabc1503e0 unknown :-1 s=CLOSED pgs=1660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 --2- 172.21.15.64:0/1421057711 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fdabc071a80 0x7fdabc14d190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:03.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.490+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 >> 172.21.15.64:0/1421057711 conn(0x7fdabc0d3cf0 msgr2=0x7fdabc0d1d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:03.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.491+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 shutdown_connections 2024-03-04T07:35:03.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:03.491+0000 7fdac1b1e640 1 -- 172.21.15.64:0/1421057711 wait complete. 2024-03-04T07:35:03.703 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:03.703 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:04.577 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:04.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:04.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.817+0000 7f2837a5c640 1 Processor -- start 2024-03-04T07:35:04.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.817+0000 7f2837a5c640 1 -- start start 2024-03-04T07:35:04.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.818+0000 7f2837a5c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f283006e700 0x7f283015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:04.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.818+0000 7f2837a5c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28300b4160 con 0x7f28300b55d0 2024-03-04T07:35:04.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.818+0000 7f2837a5c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28300b5b40 con 0x7f2830072660 2024-03-04T07:35:04.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.818+0000 7f2837a5c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f28300b3ba0 con 0x7f283006e700 2024-03-04T07:35:04.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.818+0000 7f2835ff9640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f28300b55d0 0x7f2830071980 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40280/0 (socket says 172.21.15.64:40280) 2024-03-04T07:35:04.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.818+0000 7f2835ff9640 1 -- 172.21.15.64:0/1344253581 learned_addr learned my addr 172.21.15.64:0/1344253581 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:04.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.818+0000 7f28357f8640 1 --2- 172.21.15.64:0/1344253581 >> v2:172.21.15.66:3300/0 conn(0x7f283006e700 0x7f283015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:04.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.819+0000 7f2827fff640 1 -- 172.21.15.64:0/1344253581 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3188014903 0 0) 0x7f28300b4160 con 0x7f28300b55d0 2024-03-04T07:35:04.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.819+0000 7f2827fff640 1 -- 172.21.15.64:0/1344253581 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f281c003520 con 0x7f28300b55d0 2024-03-04T07:35:04.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.819+0000 7f2827fff640 1 -- 172.21.15.64:0/1344253581 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 525760365 0 0) 0x7f28300b5b40 con 0x7f2830072660 2024-03-04T07:35:04.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.819+0000 7f2827fff640 1 -- 172.21.15.64:0/1344253581 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f28300b4160 con 0x7f2830072660 2024-03-04T07:35:04.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.819+0000 7f28357f8640 1 -- 172.21.15.64:0/1344253581 >> v1:172.21.15.66:6789/0 conn(0x7f2830072660 legacy=0x7f283006d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:04.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.819+0000 7f28357f8640 1 -- 172.21.15.64:0/1344253581 >> v1:172.21.15.64:6789/0 conn(0x7f28300b55d0 legacy=0x7f2830071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:04.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.819+0000 7f28357f8640 1 -- 172.21.15.64:0/1344253581 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28300b3e80 con 0x7f283006e700 2024-03-04T07:35:04.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f28357f8640 1 --2- 172.21.15.64:0/1344253581 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f283006e700 0x7f283015ef80 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f28280097a0 tx=0x7f282802b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5224425ecc732751 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:04.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2827fff640 1 -- 172.21.15.64:0/1344253581 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2828034070 con 0x7f283006e700 2024-03-04T07:35:04.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2827fff640 1 -- 172.21.15.64:0/1344253581 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2828030610 con 0x7f283006e700 2024-03-04T07:35:04.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2827fff640 1 -- 172.21.15.64:0/1344253581 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28280309e0 con 0x7f283006e700 2024-03-04T07:35:04.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2837a5c640 1 -- 172.21.15.64:0/1344253581 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f283006e700 msgr2=0x7f283015ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:04.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2837a5c640 1 --2- 172.21.15.64:0/1344253581 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f283006e700 0x7f283015ef80 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f28280097a0 tx=0x7f282802b960 comp rx=0 tx=0).stop 2024-03-04T07:35:04.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2837a5c640 1 -- 172.21.15.64:0/1344253581 shutdown_connections 2024-03-04T07:35:04.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2837a5c640 1 --2- 172.21.15.64:0/1344253581 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f283006e700 0x7f283015ef80 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:04.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.820+0000 7f2837a5c640 1 -- 172.21.15.64:0/1344253581 >> 172.21.15.64:0/1344253581 conn(0x7f28300b2720 msgr2=0x7f28300b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:04.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.821+0000 7f2837a5c640 1 -- 172.21.15.64:0/1344253581 shutdown_connections 2024-03-04T07:35:04.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.821+0000 7f2837a5c640 1 -- 172.21.15.64:0/1344253581 wait complete. 2024-03-04T07:35:04.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.822+0000 7f2837a5c640 1 Processor -- start 2024-03-04T07:35:04.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.822+0000 7f2837a5c640 1 -- start start 2024-03-04T07:35:04.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.822+0000 7f2837a5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f283006e700 0x7f283014c0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:04.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.822+0000 7f2837a5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2830072660 0x7f28300ae290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:04.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.822+0000 7f2837a5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28300b55d0 0x7f28301f6f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:04.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.822+0000 7f2837a5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2830072210 con 0x7f2830072660 2024-03-04T07:35:04.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2837a5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2830072090 con 0x7f28300b55d0 2024-03-04T07:35:04.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2837a5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f28300670e0 con 0x7f283006e700 2024-03-04T07:35:04.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2830072660 0x7f28300ae290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2830072660 0x7f28300ae290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38336/0 (socket says 172.21.15.64:38336) 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 -- 172.21.15.64:0/3971239213 learned_addr learned my addr 172.21.15.64:0/3971239213 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2835ff9640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28300b55d0 0x7f28301f6f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f28357f8640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f283006e700 0x7f283014c0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 -- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f283006e700 msgr2=0x7f283014c0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f283006e700 0x7f283014c0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 -- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28300b55d0 msgr2=0x7f28301f6f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28300b55d0 0x7f28301f6f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:04.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 -- 172.21.15.64:0/3971239213 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28300aed10 con 0x7f2830072660 2024-03-04T07:35:04.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2834ff7640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2830072660 0x7f28300ae290 secure :-1 s=READY pgs=1662 cs=0 l=1 rev1=1 crypto rx=0x7f282000c400 tx=0x7f282000c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:04.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2825ffb640 1 -- 172.21.15.64:0/3971239213 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2820017070 con 0x7f2830072660 2024-03-04T07:35:04.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2825ffb640 1 -- 172.21.15.64:0/3971239213 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f28200052d0 con 0x7f2830072660 2024-03-04T07:35:04.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2825ffb640 1 -- 172.21.15.64:0/3971239213 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2820003810 con 0x7f2830072660 2024-03-04T07:35:04.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.823+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28300aeff0 con 0x7f2830072660 2024-03-04T07:35:04.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.824+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28300cd5c0 con 0x7f2830072660 2024-03-04T07:35:04.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.825+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2830067ff0 con 0x7f2830072660 2024-03-04T07:35:04.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.825+0000 7f2825ffb640 1 -- 172.21.15.64:0/3971239213 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f282001a070 con 0x7f2830072660 2024-03-04T07:35:04.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.825+0000 7f2825ffb640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281406b4d0 0x7f281406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:04.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.825+0000 7f2825ffb640 1 -- 172.21.15.64:0/3971239213 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f2820089520 con 0x7f2830072660 2024-03-04T07:35:04.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.825+0000 7f28357f8640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281406b4d0 0x7f281406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:04.833 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.826+0000 7f28357f8640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281406b4d0 0x7f281406d990 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f2828036000 tx=0x7f282803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:04.834 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:04.830+0000 7f2825ffb640 1 -- 172.21.15.64:0/3971239213 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2820057da0 con 0x7f2830072660 2024-03-04T07:35:05.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.051+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f283015be20 con 0x7f2830072660 2024-03-04T07:35:05.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.052+0000 7f2825ffb640 1 -- 172.21.15.64:0/3971239213 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f282005be40 con 0x7f2830072660 2024-03-04T07:35:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:05.052 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:05.053 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281406b4d0 msgr2=0x7f281406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281406b4d0 0x7f281406d990 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f2828036000 tx=0x7f282803b040 comp rx=0 tx=0).stop 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2830072660 msgr2=0x7f28300ae290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2830072660 0x7f28300ae290 secure :-1 s=READY pgs=1662 cs=0 l=1 rev1=1 crypto rx=0x7f282000c400 tx=0x7f282000c8c0 comp rx=0 tx=0).stop 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 shutdown_connections 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f281406b4d0 0x7f281406d990 unknown :-1 s=CLOSED pgs=746 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f28300b55d0 0x7f28301f6f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2830072660 0x7f28300ae290 unknown :-1 s=CLOSED pgs=1662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:05.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 --2- 172.21.15.64:0/3971239213 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f283006e700 0x7f283014c0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:05.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 >> 172.21.15.64:0/3971239213 conn(0x7f28300b2720 msgr2=0x7f2830157f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:05.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 shutdown_connections 2024-03-04T07:35:05.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:05.054+0000 7f2837a5c640 1 -- 172.21.15.64:0/3971239213 wait complete. 2024-03-04T07:35:05.266 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:05.266 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:06.267 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:06.380 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.381+0000 7f0a0f237640 1 Processor -- start 2024-03-04T07:35:06.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.381+0000 7f0a0f237640 1 -- start start 2024-03-04T07:35:06.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.381+0000 7f0a0f237640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0a0806df30 0x7f0a080ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.381+0000 7f0a0f237640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a080b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0f237640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0a08071240 con 0x7f0a08158520 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0f237640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0a080af0f0 con 0x7f0a0806df30 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0f237640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0a08071b70 con 0x7f0a080aec40 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0cfd3640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f0a08158520 0x7f0a0806d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40290/0 (socket says 172.21.15.64:40290) 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0cfd3640 1 -- 172.21.15.64:0/3817460262 learned_addr learned my addr 172.21.15.64:0/3817460262 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0d7d4640 1 --2- 172.21.15.64:0/3817460262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a080b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a07fff640 1 --2- 172.21.15.64:0/3817460262 >> v2:172.21.15.66:3300/0 conn(0x7f0a0806df30 0x7f0a080ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a077fe640 1 -- 172.21.15.64:0/3817460262 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1132109970 0 0) 0x7f0a08071240 con 0x7f0a08158520 2024-03-04T07:35:06.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a077fe640 1 -- 172.21.15.64:0/3817460262 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f09f8003520 con 0x7f0a08158520 2024-03-04T07:35:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0d7d4640 1 -- 172.21.15.64:0/3817460262 >> v2:172.21.15.66:3300/0 conn(0x7f0a0806df30 msgr2=0x7f0a080ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0d7d4640 1 --2- 172.21.15.64:0/3817460262 >> v2:172.21.15.66:3300/0 conn(0x7f0a0806df30 0x7f0a080ae700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0d7d4640 1 -- 172.21.15.64:0/3817460262 >> v1:172.21.15.64:6789/0 conn(0x7f0a08158520 legacy=0x7f0a0806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.382+0000 7f0a0d7d4640 1 -- 172.21.15.64:0/3817460262 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a08071520 con 0x7f0a080aec40 2024-03-04T07:35:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.383+0000 7f0a0d7d4640 1 --2- 172.21.15.64:0/3817460262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a080b3640 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0a00009d90 tx=0x7f0a0002bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f28a06d5bd6946c8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.383+0000 7f0a077fe640 1 -- 172.21.15.64:0/3817460262 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a00039070 con 0x7f0a080aec40 2024-03-04T07:35:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.383+0000 7f0a077fe640 1 -- 172.21.15.64:0/3817460262 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a0002cc50 con 0x7f0a080aec40 2024-03-04T07:35:06.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.383+0000 7f0a077fe640 1 -- 172.21.15.64:0/3817460262 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a00034440 con 0x7f0a080aec40 2024-03-04T07:35:06.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 -- 172.21.15.64:0/3817460262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 msgr2=0x7f0a080b3640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:06.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 --2- 172.21.15.64:0/3817460262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a080b3640 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0a00009d90 tx=0x7f0a0002bea0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 -- 172.21.15.64:0/3817460262 shutdown_connections 2024-03-04T07:35:06.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 --2- 172.21.15.64:0/3817460262 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a080b3640 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 --2- 172.21.15.64:0/3817460262 >> v2:172.21.15.66:3300/0 conn(0x7f0a0806df30 0x7f0a080ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 -- 172.21.15.64:0/3817460262 >> 172.21.15.64:0/3817460262 conn(0x7f0a08066d00 msgr2=0x7f0a080670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 -- 172.21.15.64:0/3817460262 shutdown_connections 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.384+0000 7f0a0f237640 1 -- 172.21.15.64:0/3817460262 wait complete. 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.385+0000 7f0a0f237640 1 Processor -- start 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.385+0000 7f0a0f237640 1 -- start start 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0f237640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a0806df30 0x7f0a08152c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0f237640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a08155ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0f237640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a08158520 0x7f0a08156010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:06.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0f237640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a08158900 con 0x7f0a08158520 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0f237640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a08072000 con 0x7f0a0806df30 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0f237640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0a08158a80 con 0x7f0a080aec40 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a08158520 0x7f0a08156010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0cfd3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a0806df30 0x7f0a08152c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a08158520 0x7f0a08156010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38346/0 (socket says 172.21.15.64:38346) 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 -- 172.21.15.64:0/4082723173 learned_addr learned my addr 172.21.15.64:0/4082723173 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a07fff640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a08155ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 -- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 msgr2=0x7f0a08155ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a08155ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 -- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a0806df30 msgr2=0x7f0a08152c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a0806df30 0x7f0a08152c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 -- 172.21.15.64:0/4082723173 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a081f4a50 con 0x7f0a08158520 2024-03-04T07:35:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0cfd3640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a0806df30 0x7f0a08152c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.386+0000 7f0a0d7d4640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a08158520 0x7f0a08156010 secure :-1 s=READY pgs=1664 cs=0 l=1 rev1=1 crypto rx=0x7f0a00009d90 tx=0x7f0a00004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.387+0000 7f0a057fa640 1 -- 172.21.15.64:0/4082723173 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a00039050 con 0x7f0a08158520 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.387+0000 7f0a057fa640 1 -- 172.21.15.64:0/4082723173 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0a00045070 con 0x7f0a08158520 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.387+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0a080ccd50 con 0x7f0a08158520 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.387+0000 7f0a057fa640 1 -- 172.21.15.64:0/4082723173 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a00009450 con 0x7f0a08158520 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.387+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a080cd310 con 0x7f0a08158520 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.388+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0a08067ff0 con 0x7f0a08158520 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.388+0000 7f0a057fa640 1 -- 172.21.15.64:0/4082723173 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0a0003e610 con 0x7f0a08158520 2024-03-04T07:35:06.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.389+0000 7f0a057fa640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09ec06b4d0 0x7f09ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:06.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.389+0000 7f0a0cfd3640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09ec06b4d0 0x7f09ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:06.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.389+0000 7f0a057fa640 1 -- 172.21.15.64:0/4082723173 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0a000acae0 con 0x7f0a08158520 2024-03-04T07:35:06.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.389+0000 7f0a0cfd3640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09ec06b4d0 0x7f09ec06d990 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7f09fc005880 tx=0x7f09fc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:06.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.393+0000 7f0a057fa640 1 -- 172.21.15.64:0/4082723173 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0a0007b390 con 0x7f0a08158520 2024-03-04T07:35:06.579 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:06.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.611+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0a080b31f0 con 0x7f0a08158520 2024-03-04T07:35:06.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.612+0000 7f0a057fa640 1 -- 172.21.15.64:0/4082723173 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f0a0007f430 con 0x7f0a08158520 2024-03-04T07:35:06.612 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:06.612 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:06.613 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09ec06b4d0 msgr2=0x7f09ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09ec06b4d0 0x7f09ec06d990 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7f09fc005880 tx=0x7f09fc009040 comp rx=0 tx=0).stop 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a08158520 msgr2=0x7f0a08156010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a08158520 0x7f0a08156010 secure :-1 s=READY pgs=1664 cs=0 l=1 rev1=1 crypto rx=0x7f0a00009d90 tx=0x7f0a00004a40 comp rx=0 tx=0).stop 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 shutdown_connections 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f09ec06b4d0 0x7f09ec06d990 unknown :-1 s=CLOSED pgs=747 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0a08158520 0x7f0a08156010 unknown :-1 s=CLOSED pgs=1664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0a080aec40 0x7f0a08155ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 --2- 172.21.15.64:0/4082723173 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0a0806df30 0x7f0a08152c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 >> 172.21.15.64:0/4082723173 conn(0x7f0a08066d00 msgr2=0x7f0a08158c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:06.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 shutdown_connections 2024-03-04T07:35:06.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:06.614+0000 7f0a0f237640 1 -- 172.21.15.64:0/4082723173 wait complete. 2024-03-04T07:35:06.826 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:06.827 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:07.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:07.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.943+0000 7f4ca1ceb640 1 Processor -- start 2024-03-04T07:35:07.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.943+0000 7f4ca1ceb640 1 -- start start 2024-03-04T07:35:07.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4ca1ceb640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4c9c15e030 0x7f4c9c06ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:07.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4ca1ceb640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4c9c06ef70 0x7f4c9c0b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:07.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4ca1ceb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c0d0040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:07.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4ca1ceb640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4c9c162700 con 0x7f4c9c15e030 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4ca1ceb640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4c9c15e410 con 0x7f4c9c06ef70 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4ca1ceb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4c9c161af0 con 0x7f4c9c0b2290 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4c9c15e030 0x7f4c9c06ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4c9c15e030 0x7f4c9c06ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38360/0 (socket says 172.21.15.64:38360) 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3443102394 learned_addr learned my addr 172.21.15.64:0/3443102394 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9affd640 1 --2- 172.21.15.64:0/3443102394 >> v2:172.21.15.66:3300/0 conn(0x7f4c9c06ef70 0x7f4c9c0b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9bfff640 1 --2- 172.21.15.64:0/3443102394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c0d0040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3443102394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 msgr2=0x7f4c9c0d0040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 --2- 172.21.15.64:0/3443102394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c0d0040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3443102394 >> v2:172.21.15.66:3300/0 conn(0x7f4c9c06ef70 msgr2=0x7f4c9c0b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 --2- 172.21.15.64:0/3443102394 >> v2:172.21.15.66:3300/0 conn(0x7f4c9c06ef70 0x7f4c9c0b1d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:07.945 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.944+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3443102394 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c9c161dd0 con 0x7f4c9c15e030 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.945+0000 7f4c9b7fe640 1 --2- 172.21.15.64:0/3443102394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c15e030 0x7f4c9c06ea30 secure :-1 s=READY pgs=1665 cs=0 l=1 rev1=1 crypto rx=0x7f4c9c0720a0 tx=0x7f4c8c02b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=14ad5f3c31d9d3da server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.945+0000 7f4c9a7fc640 1 -- 172.21.15.64:0/3443102394 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c8c035020 con 0x7f4c9c15e030 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.945+0000 7f4c9a7fc640 1 -- 172.21.15.64:0/3443102394 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4c8c031870 con 0x7f4c9c15e030 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.945+0000 7f4c9a7fc640 1 -- 172.21.15.64:0/3443102394 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c8c031c40 con 0x7f4c9c15e030 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3443102394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c15e030 msgr2=0x7f4c9c06ea30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3443102394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c15e030 0x7f4c9c06ea30 secure :-1 s=READY pgs=1665 cs=0 l=1 rev1=1 crypto rx=0x7f4c9c0720a0 tx=0x7f4c8c02b910 comp rx=0 tx=0).stop 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3443102394 shutdown_connections 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3443102394 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c0d0040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3443102394 >> v2:172.21.15.66:3300/0 conn(0x7f4c9c06ef70 0x7f4c9c0b1d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3443102394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c15e030 0x7f4c9c06ea30 unknown :-1 s=CLOSED pgs=1665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:07.947 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3443102394 >> 172.21.15.64:0/3443102394 conn(0x7f4c9c0b55e0 msgr2=0x7f4c9c0b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:07.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3443102394 shutdown_connections 2024-03-04T07:35:07.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.946+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3443102394 wait complete. 2024-03-04T07:35:07.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.947+0000 7f4ca1ceb640 1 Processor -- start 2024-03-04T07:35:07.948 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.947+0000 7f4ca1ceb640 1 -- start start 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4ca1ceb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c06ef70 0x7f4c9c146d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4ca1ceb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c1574b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4ca1ceb640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c9c15e030 0x7f4c9c1579f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4ca1ceb640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c9c0b31e0 con 0x7f4c9c06ef70 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4ca1ceb640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c9c162390 con 0x7f4c9c15e030 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4ca1ceb640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4c9c0b3360 con 0x7f4c9c0b2290 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c06ef70 0x7f4c9c146d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c06ef70 0x7f4c9c146d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38366/0 (socket says 172.21.15.64:38366) 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3225615525 learned_addr learned my addr 172.21.15.64:0/3225615525 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:07.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9bfff640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c9c15e030 0x7f4c9c1579f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9affd640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c1574b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 msgr2=0x7f4c9c1574b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c1574b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c9c15e030 msgr2=0x7f4c9c1579f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c9c15e030 0x7f4c9c1579f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.948+0000 7f4c9b7fe640 1 -- 172.21.15.64:0/3225615525 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c9c15b790 con 0x7f4c9c06ef70 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.949+0000 7f4c9b7fe640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c06ef70 0x7f4c9c146d60 secure :-1 s=READY pgs=1666 cs=0 l=1 rev1=1 crypto rx=0x7f4c8c02b950 tx=0x7f4c8c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.949+0000 7f4c83fff640 1 -- 172.21.15.64:0/3225615525 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c8c040070 con 0x7f4c9c06ef70 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.949+0000 7f4c83fff640 1 -- 172.21.15.64:0/3225615525 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4c8c031e10 con 0x7f4c9c06ef70 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.949+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4c9c1e7cc0 con 0x7f4c9c06ef70 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.949+0000 7f4c83fff640 1 -- 172.21.15.64:0/3225615525 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c8c035040 con 0x7f4c9c06ef70 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.949+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4c9c15ba30 con 0x7f4c9c06ef70 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.950+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4c9c067ff0 con 0x7f4c9c06ef70 2024-03-04T07:35:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.950+0000 7f4c83fff640 1 -- 172.21.15.64:0/3225615525 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4c8c04b060 con 0x7f4c9c06ef70 2024-03-04T07:35:07.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.951+0000 7f4c83fff640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c8806b4d0 0x7f4c8806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:07.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.951+0000 7f4c83fff640 1 -- 172.21.15.64:0/3225615525 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4c8c0abd00 con 0x7f4c9c06ef70 2024-03-04T07:35:07.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.951+0000 7f4c9affd640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c8806b4d0 0x7f4c8806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:07.952 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.951+0000 7f4c9affd640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c8806b4d0 0x7f4c8806d990 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f4c9c1590b0 tx=0x7f4c90009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:07.955 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:07.955+0000 7f4c83fff640 1 -- 172.21.15.64:0/3225615525 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4c8c07a580 con 0x7f4c9c06ef70 2024-03-04T07:35:08.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.175+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4c9c0ad140 con 0x7f4c9c06ef70 2024-03-04T07:35:08.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.175+0000 7f4c83fff640 1 -- 172.21.15.64:0/3225615525 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4c8c07e620 con 0x7f4c9c06ef70 2024-03-04T07:35:08.175 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:08.175 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:08.176 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:08.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c8806b4d0 msgr2=0x7f4c8806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:08.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c8806b4d0 0x7f4c8806d990 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f4c9c1590b0 tx=0x7f4c90009190 comp rx=0 tx=0).stop 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c06ef70 msgr2=0x7f4c9c146d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c06ef70 0x7f4c9c146d60 secure :-1 s=READY pgs=1666 cs=0 l=1 rev1=1 crypto rx=0x7f4c8c02b950 tx=0x7f4c8c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 shutdown_connections 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4c8806b4d0 0x7f4c8806d990 unknown :-1 s=CLOSED pgs=748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4c9c15e030 0x7f4c9c1579f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4c9c0b2290 0x7f4c9c1574b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 --2- 172.21.15.64:0/3225615525 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4c9c06ef70 0x7f4c9c146d60 unknown :-1 s=CLOSED pgs=1666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 >> 172.21.15.64:0/3225615525 conn(0x7f4c9c0b55e0 msgr2=0x7f4c9c0b16f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 shutdown_connections 2024-03-04T07:35:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:08.177+0000 7f4ca1ceb640 1 -- 172.21.15.64:0/3225615525 wait complete. 2024-03-04T07:35:08.389 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:08.390 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:08.580 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:09.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:09.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.505+0000 7ff933e29640 1 Processor -- start 2024-03-04T07:35:09.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.505+0000 7ff933e29640 1 -- start start 2024-03-04T07:35:09.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff933e29640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c158520 0x7ff92c06d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:09.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff933e29640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff92c06dd60 0x7ff92c0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:09.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff933e29640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff92c0af080 con 0x7ff92c0aec40 2024-03-04T07:35:09.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff933e29640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff92c06e1c0 con 0x7ff92c06dd60 2024-03-04T07:35:09.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff933e29640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff92c071b70 con 0x7ff92c158520 2024-03-04T07:35:09.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff9313c4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff92c06dd60 0x7ff92c0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff9313c4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff92c06dd60 0x7ff92c0ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38368/0 (socket says 172.21.15.64:38368) 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff9313c4640 1 -- 172.21.15.64:0/2148918541 learned_addr learned my addr 172.21.15.64:0/2148918541 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff931bc5640 1 --2- 172.21.15.64:0/2148918541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c158520 0x7ff92c06d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff9313c4640 1 -- 172.21.15.64:0/2148918541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c158520 msgr2=0x7ff92c06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff9313c4640 1 --2- 172.21.15.64:0/2148918541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c158520 0x7ff92c06d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff9313c4640 1 -- 172.21.15.64:0/2148918541 >> v1:172.21.15.66:6789/0 conn(0x7ff92c0aec40 legacy=0x7ff92c15e670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff9313c4640 1 -- 172.21.15.64:0/2148918541 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff92c071240 con 0x7ff92c06dd60 2024-03-04T07:35:09.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.506+0000 7ff930bc3640 1 -- 172.21.15.64:0/2148918541 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1051776755 0 0) 0x7ff92c0af080 con 0x7ff92c0aec40 2024-03-04T07:35:09.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.507+0000 7ff9313c4640 1 --2- 172.21.15.64:0/2148918541 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c06dd60 0x7ff92c0ae700 secure :-1 s=READY pgs=1667 cs=0 l=1 rev1=1 crypto rx=0x7ff91c009870 tx=0x7ff91c02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dee42296aba8b9c5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:09.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.507+0000 7ff930bc3640 1 -- 172.21.15.64:0/2148918541 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff91c034070 con 0x7ff92c06dd60 2024-03-04T07:35:09.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.507+0000 7ff930bc3640 1 -- 172.21.15.64:0/2148918541 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff91c030840 con 0x7ff92c06dd60 2024-03-04T07:35:09.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.507+0000 7ff930bc3640 1 -- 172.21.15.64:0/2148918541 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff91c030c10 con 0x7ff92c06dd60 2024-03-04T07:35:09.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 -- 172.21.15.64:0/2148918541 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c06dd60 msgr2=0x7ff92c0ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 --2- 172.21.15.64:0/2148918541 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c06dd60 0x7ff92c0ae700 secure :-1 s=READY pgs=1667 cs=0 l=1 rev1=1 crypto rx=0x7ff91c009870 tx=0x7ff91c02b980 comp rx=0 tx=0).stop 2024-03-04T07:35:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 -- 172.21.15.64:0/2148918541 shutdown_connections 2024-03-04T07:35:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 --2- 172.21.15.64:0/2148918541 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c06dd60 0x7ff92c0ae700 unknown :-1 s=CLOSED pgs=1667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 --2- 172.21.15.64:0/2148918541 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c158520 0x7ff92c06d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 -- 172.21.15.64:0/2148918541 >> 172.21.15.64:0/2148918541 conn(0x7ff92c066d00 msgr2=0x7ff92c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 -- 172.21.15.64:0/2148918541 shutdown_connections 2024-03-04T07:35:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.508+0000 7ff933e29640 1 -- 172.21.15.64:0/2148918541 wait complete. 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.509+0000 7ff933e29640 1 Processor -- start 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.509+0000 7ff933e29640 1 -- start start 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff933e29640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c06dd60 0x7ff92c15b800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff933e29640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c0aec40 0x7ff92c13f160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff933e29640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff92c158520 0x7ff92c13f6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff933e29640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff92c0b4650 con 0x7ff92c0aec40 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff933e29640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff92c071800 con 0x7ff92c158520 2024-03-04T07:35:09.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff933e29640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff92c072000 con 0x7ff92c06dd60 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c0aec40 0x7ff92c13f160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c0aec40 0x7ff92c13f160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38382/0 (socket says 172.21.15.64:38382) 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 -- 172.21.15.64:0/2071701553 learned_addr learned my addr 172.21.15.64:0/2071701553 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff931bc5640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c06dd60 0x7ff92c15b800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9323c6640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff92c158520 0x7ff92c13f6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 -- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c06dd60 msgr2=0x7ff92c15b800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c06dd60 0x7ff92c15b800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 -- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff92c158520 msgr2=0x7ff92c13f6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff92c158520 0x7ff92c13f6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 -- 172.21.15.64:0/2071701553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff92c13fcd0 con 0x7ff92c0aec40 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.510+0000 7ff9313c4640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c0aec40 0x7ff92c13f160 secure :-1 s=READY pgs=1668 cs=0 l=1 rev1=1 crypto rx=0x7ff91c02b9c0 tx=0x7ff91c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.511+0000 7ff9227fc640 1 -- 172.21.15.64:0/2071701553 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff91c048070 con 0x7ff92c0aec40 2024-03-04T07:35:09.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.511+0000 7ff9227fc640 1 -- 172.21.15.64:0/2071701553 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff91c030640 con 0x7ff92c0aec40 2024-03-04T07:35:09.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.511+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff92c140010 con 0x7ff92c0aec40 2024-03-04T07:35:09.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.511+0000 7ff9227fc640 1 -- 172.21.15.64:0/2071701553 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff91c034040 con 0x7ff92c0aec40 2024-03-04T07:35:09.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.511+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff92c13d690 con 0x7ff92c0aec40 2024-03-04T07:35:09.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.512+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff92c067ff0 con 0x7ff92c0aec40 2024-03-04T07:35:09.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.512+0000 7ff9227fc640 1 -- 172.21.15.64:0/2071701553 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff91c04b070 con 0x7ff92c0aec40 2024-03-04T07:35:09.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.513+0000 7ff9227fc640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff91006b4d0 0x7ff91006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:09.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.513+0000 7ff9227fc640 1 -- 172.21.15.64:0/2071701553 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff91c0abfe0 con 0x7ff92c0aec40 2024-03-04T07:35:09.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.513+0000 7ff931bc5640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff91006b4d0 0x7ff91006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:09.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.513+0000 7ff931bc5640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff91006b4d0 0x7ff91006d990 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7ff928011340 tx=0x7ff92800b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:09.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.517+0000 7ff9227fc640 1 -- 172.21.15.64:0/2071701553 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff92c140010 con 0x7ff92c0aec40 2024-03-04T07:35:09.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.738+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff92c0b31f0 con 0x7ff92c0aec40 2024-03-04T07:35:09.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.738+0000 7ff9227fc640 1 -- 172.21.15.64:0/2071701553 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7ff91c030810 con 0x7ff92c0aec40 2024-03-04T07:35:09.738 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:09.738 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:09.739 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff91006b4d0 msgr2=0x7ff91006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff91006b4d0 0x7ff91006d990 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7ff928011340 tx=0x7ff92800b040 comp rx=0 tx=0).stop 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c0aec40 msgr2=0x7ff92c13f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c0aec40 0x7ff92c13f160 secure :-1 s=READY pgs=1668 cs=0 l=1 rev1=1 crypto rx=0x7ff91c02b9c0 tx=0x7ff91c004730 comp rx=0 tx=0).stop 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 shutdown_connections 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff91006b4d0 0x7ff91006d990 unknown :-1 s=CLOSED pgs=749 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff92c158520 0x7ff92c13f6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff92c0aec40 0x7ff92c13f160 unknown :-1 s=CLOSED pgs=1668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 --2- 172.21.15.64:0/2071701553 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff92c06dd60 0x7ff92c15b800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.740+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 >> 172.21.15.64:0/2071701553 conn(0x7ff92c066d00 msgr2=0x7ff92c0b3d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.741+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 shutdown_connections 2024-03-04T07:35:09.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:09.741+0000 7ff933e29640 1 -- 172.21.15.64:0/2071701553 wait complete. 2024-03-04T07:35:09.953 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:09.953 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:10.583 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:10.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:11.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.068+0000 7f4b9eb13640 1 Processor -- start 2024-03-04T07:35:11.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.068+0000 7f4b9eb13640 1 -- start start 2024-03-04T07:35:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.068+0000 7f4b9eb13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b980b95a0 0x7f4b980b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9eb13640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4b980b9a10 0x7f4b980b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9eb13640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4b980cea30 con 0x7f4b9815df00 2024-03-04T07:35:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9eb13640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4b980cf3c0 con 0x7f4b980b9a10 2024-03-04T07:35:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9eb13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b980cf0e0 con 0x7f4b980b95a0 2024-03-04T07:35:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9d0b0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4b980b9a10 0x7f4b980b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9c8af640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4b9815df00 0x7f4b980d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40292/0 (socket says 172.21.15.64:40292) 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9c8af640 1 -- 172.21.15.64:0/2531479889 learned_addr learned my addr 172.21.15.64:0/2531479889 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b97fff640 1 --2- 172.21.15.64:0/2531479889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b980b95a0 0x7f4b980b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9d0b0640 1 -- 172.21.15.64:0/2531479889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b980b95a0 msgr2=0x7f4b980b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9d0b0640 1 --2- 172.21.15.64:0/2531479889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b980b95a0 0x7f4b980b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9d0b0640 1 -- 172.21.15.64:0/2531479889 >> v1:172.21.15.64:6789/0 conn(0x7f4b9815df00 legacy=0x7f4b980d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b9d0b0640 1 -- 172.21.15.64:0/2531479889 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b980ce470 con 0x7f4b980b9a10 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b977fe640 1 -- 172.21.15.64:0/2531479889 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2118193905 0 0) 0x7f4b980cea30 con 0x7f4b9815df00 2024-03-04T07:35:11.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.069+0000 7f4b97fff640 1 --2- 172.21.15.64:0/2531479889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b980b95a0 0x7f4b980b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.070+0000 7f4b9d0b0640 1 --2- 172.21.15.64:0/2531479889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980b5c40 secure :-1 s=READY pgs=1670 cs=0 l=1 rev1=1 crypto rx=0x7f4b90009d90 tx=0x7f4b9002bea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b4b4cb734061b070 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.070+0000 7f4b977fe640 1 -- 172.21.15.64:0/2531479889 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b90039070 con 0x7f4b980b9a10 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.070+0000 7f4b977fe640 1 -- 172.21.15.64:0/2531479889 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b9002cc50 con 0x7f4b980b9a10 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.070+0000 7f4b977fe640 1 -- 172.21.15.64:0/2531479889 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b90034440 con 0x7f4b980b9a10 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 -- 172.21.15.64:0/2531479889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 msgr2=0x7f4b980b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/2531479889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980b5c40 secure :-1 s=READY pgs=1670 cs=0 l=1 rev1=1 crypto rx=0x7f4b90009d90 tx=0x7f4b9002bea0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 -- 172.21.15.64:0/2531479889 shutdown_connections 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/2531479889 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980b5c40 unknown :-1 s=CLOSED pgs=1670 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/2531479889 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b980b95a0 0x7f4b980b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 -- 172.21.15.64:0/2531479889 >> 172.21.15.64:0/2531479889 conn(0x7f4b98071960 msgr2=0x7f4b98071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:11.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 -- 172.21.15.64:0/2531479889 shutdown_connections 2024-03-04T07:35:11.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.071+0000 7f4b9eb13640 1 -- 172.21.15.64:0/2531479889 wait complete. 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.072+0000 7f4b9eb13640 1 Processor -- start 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.072+0000 7f4b9eb13640 1 -- start start 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9eb13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b980b95a0 0x7f4b98152100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9eb13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980c9740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9eb13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b9815df00 0x7f4b980ccf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9eb13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b980cedf0 con 0x7f4b980b9a10 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9eb13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b980cec70 con 0x7f4b980b95a0 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9eb13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4b980d3220 con 0x7f4b9815df00 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b97fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980c9740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b97fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980c9740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38404/0 (socket says 172.21.15.64:38404) 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b97fff640 1 -- 172.21.15.64:0/4212203953 learned_addr learned my addr 172.21.15.64:0/4212203953 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9d0b0640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b9815df00 0x7f4b980ccf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:11.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9c8af640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b980b95a0 0x7f4b98152100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9d0b0640 1 -- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b980b95a0 msgr2=0x7f4b98152100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9d0b0640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b980b95a0 0x7f4b98152100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9d0b0640 1 -- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 msgr2=0x7f4b980c9740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9d0b0640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980c9740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9d0b0640 1 -- 172.21.15.64:0/4212203953 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b98152640 con 0x7f4b9815df00 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b97fff640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980c9740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.073+0000 7f4b9d0b0640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b9815df00 0x7f4b980ccf20 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f4b980b6870 tx=0x7f4b90004a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.074+0000 7f4b957fa640 1 -- 172.21.15.64:0/4212203953 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b90039050 con 0x7f4b9815df00 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.074+0000 7f4b957fa640 1 -- 172.21.15.64:0/4212203953 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b90045070 con 0x7f4b9815df00 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.074+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b980ca1e0 con 0x7f4b9815df00 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.074+0000 7f4b957fa640 1 -- 172.21.15.64:0/4212203953 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b90009450 con 0x7f4b9815df00 2024-03-04T07:35:11.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.074+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b981f50e0 con 0x7f4b9815df00 2024-03-04T07:35:11.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.075+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b980cab20 con 0x7f4b9815df00 2024-03-04T07:35:11.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.075+0000 7f4b957fa640 1 -- 172.21.15.64:0/4212203953 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4b9003e610 con 0x7f4b9815df00 2024-03-04T07:35:11.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.075+0000 7f4b957fa640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7c06b4d0 0x7f4b7c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:11.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.075+0000 7f4b957fa640 1 -- 172.21.15.64:0/4212203953 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4b900aca00 con 0x7f4b9815df00 2024-03-04T07:35:11.076 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.076+0000 7f4b9c8af640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7c06b4d0 0x7f4b7c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:11.077 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.076+0000 7f4b9c8af640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7c06b4d0 0x7f4b7c06d990 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7f4b8c007fd0 tx=0x7f4b8c007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:11.079 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.080+0000 7f4b957fa640 1 -- 172.21.15.64:0/4212203953 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4b980cab20 con 0x7f4b9815df00 2024-03-04T07:35:11.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.301+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4b981f4ed0 con 0x7f4b9815df00 2024-03-04T07:35:11.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.301+0000 7f4b957fa640 1 -- 172.21.15.64:0/4212203953 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4b98152640 con 0x7f4b9815df00 2024-03-04T07:35:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:11.302 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:11.302 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7c06b4d0 msgr2=0x7f4b7c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7c06b4d0 0x7f4b7c06d990 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7f4b8c007fd0 tx=0x7f4b8c007de0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b9815df00 msgr2=0x7f4b980ccf20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b9815df00 0x7f4b980ccf20 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f4b980b6870 tx=0x7f4b90004a40 comp rx=0 tx=0).stop 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 shutdown_connections 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b7c06b4d0 0x7f4b7c06d990 unknown :-1 s=CLOSED pgs=750 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4b9815df00 0x7f4b980ccf20 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4b980b9a10 0x7f4b980c9740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 --2- 172.21.15.64:0/4212203953 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4b980b95a0 0x7f4b98152100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.303+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 >> 172.21.15.64:0/4212203953 conn(0x7f4b98071960 msgr2=0x7f4b980b5920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.304+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 shutdown_connections 2024-03-04T07:35:11.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:11.304+0000 7f4b9eb13640 1 -- 172.21.15.64:0/4212203953 wait complete. 2024-03-04T07:35:11.515 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:11.516 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:12.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:12.583 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.630+0000 7ffa86f3a640 1 Processor -- start 2024-03-04T07:35:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.630+0000 7ffa86f3a640 1 -- start start 2024-03-04T07:35:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa86f3a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072090 0x7ffa8015ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa86f3a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffa800b4160 con 0x7ffa8006e700 2024-03-04T07:35:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa86f3a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ffa800b5b40 con 0x7ffa80072660 2024-03-04T07:35:12.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa86f3a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffa800b3ba0 con 0x7ffa80072090 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa84cd6640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ffa8006e700 0x7ffa8006d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40308/0 (socket says 172.21.15.64:40308) 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa84cd6640 1 -- 172.21.15.64:0/2178713366 learned_addr learned my addr 172.21.15.64:0/2178713366 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa854d7640 1 --2- 172.21.15.64:0/2178713366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072090 0x7ffa8015ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.631+0000 7ffa7f7fe640 1 -- 172.21.15.64:0/2178713366 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3689612667 0 0) 0x7ffa800b4160 con 0x7ffa8006e700 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.632+0000 7ffa7f7fe640 1 -- 172.21.15.64:0/2178713366 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ffa68003520 con 0x7ffa8006e700 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.632+0000 7ffa7f7fe640 1 -- 172.21.15.64:0/2178713366 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3436622178 0 0) 0x7ffa800b5b40 con 0x7ffa80072660 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.632+0000 7ffa854d7640 1 -- 172.21.15.64:0/2178713366 >> v1:172.21.15.66:6789/0 conn(0x7ffa80072660 legacy=0x7ffa80071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.632+0000 7ffa854d7640 1 -- 172.21.15.64:0/2178713366 >> v1:172.21.15.64:6789/0 conn(0x7ffa8006e700 legacy=0x7ffa8006d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:12.632 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.632+0000 7ffa854d7640 1 -- 172.21.15.64:0/2178713366 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffa800b3e80 con 0x7ffa80072090 2024-03-04T07:35:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.632+0000 7ffa854d7640 1 --2- 172.21.15.64:0/2178713366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072090 0x7ffa8015ef80 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7ffa700097d0 tx=0x7ffa7002b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d46b8e0470c6c52e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa7f7fe640 1 -- 172.21.15.64:0/2178713366 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa70035020 con 0x7ffa80072090 2024-03-04T07:35:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa7f7fe640 1 -- 172.21.15.64:0/2178713366 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffa70031890 con 0x7ffa80072090 2024-03-04T07:35:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa7f7fe640 1 -- 172.21.15.64:0/2178713366 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa70031c60 con 0x7ffa80072090 2024-03-04T07:35:12.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2178713366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072090 msgr2=0x7ffa8015ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2178713366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072090 0x7ffa8015ef80 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7ffa700097d0 tx=0x7ffa7002b930 comp rx=0 tx=0).stop 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2178713366 shutdown_connections 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2178713366 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072090 0x7ffa8015ef80 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2178713366 >> 172.21.15.64:0/2178713366 conn(0x7ffa800b2720 msgr2=0x7ffa800b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2178713366 shutdown_connections 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.633+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2178713366 wait complete. 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.634+0000 7ffa86f3a640 1 Processor -- start 2024-03-04T07:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.634+0000 7ffa86f3a640 1 -- start start 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa86f3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa8006e700 0x7ffa8014a5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa86f3a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa80072090 0x7ffa8015ab30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa86f3a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072660 0x7ffa80161860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa86f3a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffa800b5750 con 0x7ffa8006e700 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa86f3a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffa800b55d0 con 0x7ffa80072090 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa86f3a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ffa800b58d0 con 0x7ffa80072660 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa84cd6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa8006e700 0x7ffa8014a5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa84cd6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa8006e700 0x7ffa8014a5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38414/0 (socket says 172.21.15.64:38414) 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa84cd6640 1 -- 172.21.15.64:0/2810155962 learned_addr learned my addr 172.21.15.64:0/2810155962 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa854d7640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072660 0x7ffa80161860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:12.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.635+0000 7ffa7ffff640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa80072090 0x7ffa8015ab30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa84cd6640 1 -- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072660 msgr2=0x7ffa80161860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa84cd6640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072660 0x7ffa80161860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa84cd6640 1 -- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa80072090 msgr2=0x7ffa8015ab30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa84cd6640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa80072090 0x7ffa8015ab30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa84cd6640 1 -- 172.21.15.64:0/2810155962 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffa8014ab10 con 0x7ffa8006e700 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa854d7640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072660 0x7ffa80161860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa84cd6640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa8006e700 0x7ffa8014a5d0 secure :-1 s=READY pgs=1672 cs=0 l=1 rev1=1 crypto rx=0x7ffa7800cb70 tx=0x7ffa7800a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa7d7fa640 1 -- 172.21.15.64:0/2810155962 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa78017070 con 0x7ffa8006e700 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa7d7fa640 1 -- 172.21.15.64:0/2810155962 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ffa7800ad40 con 0x7ffa8006e700 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffa800cd550 con 0x7ffa8006e700 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa7d7fa640 1 -- 172.21.15.64:0/2810155962 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffa78012620 con 0x7ffa8006e700 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.636+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffa800cdb10 con 0x7ffa8006e700 2024-03-04T07:35:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.637+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffa80067ff0 con 0x7ffa8006e700 2024-03-04T07:35:12.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.638+0000 7ffa7d7fa640 1 -- 172.21.15.64:0/2810155962 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ffa7801a070 con 0x7ffa8006e700 2024-03-04T07:35:12.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.638+0000 7ffa7d7fa640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa6c06b4d0 0x7ffa6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:12.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.638+0000 7ffa7d7fa640 1 -- 172.21.15.64:0/2810155962 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ffa78089f70 con 0x7ffa8006e700 2024-03-04T07:35:12.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.638+0000 7ffa7ffff640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa6c06b4d0 0x7ffa6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:12.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.639+0000 7ffa7ffff640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa6c06b4d0 0x7ffa6c06d990 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7ffa801623e0 tx=0x7ffa74009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:12.642 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.643+0000 7ffa7d7fa640 1 -- 172.21.15.64:0/2810155962 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ffa780587f0 con 0x7ffa8006e700 2024-03-04T07:35:12.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.862+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ffa80072c30 con 0x7ffa8006e700 2024-03-04T07:35:12.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.863+0000 7ffa7d7fa640 1 -- 172.21.15.64:0/2810155962 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7ffa7805c890 con 0x7ffa8006e700 2024-03-04T07:35:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:12.863 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:12.863 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.864+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa6c06b4d0 msgr2=0x7ffa6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.864+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa6c06b4d0 0x7ffa6c06d990 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7ffa801623e0 tx=0x7ffa74009040 comp rx=0 tx=0).stop 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.864+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa8006e700 msgr2=0x7ffa8014a5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.864+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa8006e700 0x7ffa8014a5d0 secure :-1 s=READY pgs=1672 cs=0 l=1 rev1=1 crypto rx=0x7ffa7800cb70 tx=0x7ffa7800a430 comp rx=0 tx=0).stop 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 shutdown_connections 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ffa6c06b4d0 0x7ffa6c06d990 unknown :-1 s=CLOSED pgs=751 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ffa80072660 0x7ffa80161860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ffa80072090 0x7ffa8015ab30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 --2- 172.21.15.64:0/2810155962 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ffa8006e700 0x7ffa8014a5d0 unknown :-1 s=CLOSED pgs=1672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:12.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 >> 172.21.15.64:0/2810155962 conn(0x7ffa800b2720 msgr2=0x7ffa80071980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:12.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 shutdown_connections 2024-03-04T07:35:12.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:12.865+0000 7ffa86f3a640 1 -- 172.21.15.64:0/2810155962 wait complete. 2024-03-04T07:35:13.077 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:13.077 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:14.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:14.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.192+0000 7feb9a00f640 1 Processor -- start 2024-03-04T07:35:14.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.192+0000 7feb9a00f640 1 -- start start 2024-03-04T07:35:14.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb9a00f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feb9415df70 0x7feb940d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:14.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb9a00f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feb940b6f20 con 0x7feb94071c50 2024-03-04T07:35:14.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb9a00f640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feb940d3030 con 0x7feb940d18c0 2024-03-04T07:35:14.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb9a00f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feb940d3310 con 0x7feb9415df70 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb93fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7feb94071c50 0x7feb940b2700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59530/0 (socket says 172.21.15.64:59530) 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb93fff640 1 -- 172.21.15.64:0/1710077217 learned_addr learned my addr 172.21.15.64:0/1710077217 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb937fe640 1 --2- 172.21.15.64:0/1710077217 >> v2:172.21.15.66:3300/0 conn(0x7feb9415df70 0x7feb940d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb927fc640 1 -- 172.21.15.64:0/1710077217 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1293947872 0 0) 0x7feb940b6f20 con 0x7feb94071c50 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb927fc640 1 -- 172.21.15.64:0/1710077217 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7feb7c003520 con 0x7feb94071c50 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb927fc640 1 -- 172.21.15.64:0/1710077217 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 477867599 0 0) 0x7feb940d3030 con 0x7feb940d18c0 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb927fc640 1 -- 172.21.15.64:0/1710077217 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7feb940b6f20 con 0x7feb940d18c0 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb937fe640 1 -- 172.21.15.64:0/1710077217 >> v1:172.21.15.66:6789/0 conn(0x7feb940d18c0 legacy=0x7feb94071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb937fe640 1 -- 172.21.15.64:0/1710077217 >> v1:172.21.15.64:6789/0 conn(0x7feb94071c50 legacy=0x7feb940b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:14.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.193+0000 7feb937fe640 1 -- 172.21.15.64:0/1710077217 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feb940d35f0 con 0x7feb9415df70 2024-03-04T07:35:14.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.194+0000 7feb937fe640 1 --2- 172.21.15.64:0/1710077217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940d27b0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7feb84009d90 tx=0x7feb8402bfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=97191af7061096ee server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:14.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.194+0000 7feb927fc640 1 -- 172.21.15.64:0/1710077217 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb84039070 con 0x7feb9415df70 2024-03-04T07:35:14.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.194+0000 7feb927fc640 1 -- 172.21.15.64:0/1710077217 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feb8402cd80 con 0x7feb9415df70 2024-03-04T07:35:14.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.194+0000 7feb927fc640 1 -- 172.21.15.64:0/1710077217 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb84034640 con 0x7feb9415df70 2024-03-04T07:35:14.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.195+0000 7feb9a00f640 1 -- 172.21.15.64:0/1710077217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 msgr2=0x7feb940d27b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.195+0000 7feb9a00f640 1 --2- 172.21.15.64:0/1710077217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940d27b0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7feb84009d90 tx=0x7feb8402bfd0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.195+0000 7feb9a00f640 1 -- 172.21.15.64:0/1710077217 shutdown_connections 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.195+0000 7feb9a00f640 1 --2- 172.21.15.64:0/1710077217 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940d27b0 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.195+0000 7feb9a00f640 1 -- 172.21.15.64:0/1710077217 >> 172.21.15.64:0/1710077217 conn(0x7feb940d3cf0 msgr2=0x7feb940d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.195+0000 7feb9a00f640 1 -- 172.21.15.64:0/1710077217 shutdown_connections 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.195+0000 7feb9a00f640 1 -- 172.21.15.64:0/1710077217 wait complete. 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.196+0000 7feb9a00f640 1 Processor -- start 2024-03-04T07:35:14.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.196+0000 7feb9a00f640 1 -- start start 2024-03-04T07:35:14.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb9a00f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feb94071c50 0x7feb94150a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:14.198 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb9a00f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feb940d18c0 0x7feb940c9c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb9a00f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940b1aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb9a00f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feb9415e350 con 0x7feb940d18c0 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb9a00f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feb940d2cf0 con 0x7feb9415df70 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb9a00f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feb9415e4d0 con 0x7feb94071c50 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb92ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feb940d18c0 0x7feb940c9c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb93fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940b1aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb92ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feb940d18c0 0x7feb940c9c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:41996/0 (socket says 172.21.15.64:41996) 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb93fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940b1aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:47484/0 (socket says 172.21.15.64:47484) 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb92ffd640 1 -- 172.21.15.64:0/2138977087 learned_addr learned my addr 172.21.15.64:0/2138977087 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.197+0000 7feb937fe640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feb94071c50 0x7feb94150a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb93fff640 1 -- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feb94071c50 msgr2=0x7feb94150a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb93fff640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feb94071c50 0x7feb94150a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb93fff640 1 -- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feb940d18c0 msgr2=0x7feb940c9c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb93fff640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feb940d18c0 0x7feb940c9c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb93fff640 1 -- 172.21.15.64:0/2138977087 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feb94150fc0 con 0x7feb9415df70 2024-03-04T07:35:14.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb92ffd640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feb940d18c0 0x7feb940c9c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:14.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb93fff640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940b1aa0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7feb940b68e0 tx=0x7feb8c00c890 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:14.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb7bfff640 1 -- 172.21.15.64:0/2138977087 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb8c017070 con 0x7feb9415df70 2024-03-04T07:35:14.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb7bfff640 1 -- 172.21.15.64:0/2138977087 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feb8c005ce0 con 0x7feb9415df70 2024-03-04T07:35:14.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feb940ccfd0 con 0x7feb9415df70 2024-03-04T07:35:14.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb7bfff640 1 -- 172.21.15.64:0/2138977087 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb8c0072d0 con 0x7feb9415df70 2024-03-04T07:35:14.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.198+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feb940cd590 con 0x7feb9415df70 2024-03-04T07:35:14.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.199+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feb94067ff0 con 0x7feb9415df70 2024-03-04T07:35:14.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.199+0000 7feb7bfff640 1 -- 172.21.15.64:0/2138977087 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feb8c01a070 con 0x7feb9415df70 2024-03-04T07:35:14.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.200+0000 7feb7bfff640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feb8006b500 0x7feb8006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:14.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.200+0000 7feb7bfff640 1 -- 172.21.15.64:0/2138977087 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7feb8c0894c0 con 0x7feb9415df70 2024-03-04T07:35:14.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.200+0000 7feb937fe640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feb8006b500 0x7feb8006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:14.201 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.200+0000 7feb937fe640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feb8006b500 0x7feb8006d9c0 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7feb84036000 tx=0x7feb84030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:14.204 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.205+0000 7feb7bfff640 1 -- 172.21.15.64:0/2138977087 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feb8c007e20 con 0x7feb9415df70 2024-03-04T07:35:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.425+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7feb940b3290 con 0x7feb9415df70 2024-03-04T07:35:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.425+0000 7feb7bfff640 1 -- 172.21.15.64:0/2138977087 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7feb8c057c10 con 0x7feb9415df70 2024-03-04T07:35:14.425 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:14.425 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feb8006b500 msgr2=0x7feb8006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feb8006b500 0x7feb8006d9c0 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7feb84036000 tx=0x7feb84030000 comp rx=0 tx=0).stop 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 msgr2=0x7feb940b1aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940b1aa0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7feb940b68e0 tx=0x7feb8c00c890 comp rx=0 tx=0).stop 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 shutdown_connections 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feb8006b500 0x7feb8006d9c0 unknown :-1 s=CLOSED pgs=752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feb9415df70 0x7feb940b1aa0 unknown :-1 s=CLOSED pgs=597 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feb940d18c0 0x7feb940c9c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 --2- 172.21.15.64:0/2138977087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feb94071c50 0x7feb94150a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 >> 172.21.15.64:0/2138977087 conn(0x7feb940d3cf0 msgr2=0x7feb94160490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 shutdown_connections 2024-03-04T07:35:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:14.427+0000 7feb9a00f640 1 -- 172.21.15.64:0/2138977087 wait complete. 2024-03-04T07:35:14.586 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:14.639 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:14.639 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:15.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:15.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.753+0000 7f5a039bc640 1 Processor -- start 2024-03-04T07:35:15.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.754+0000 7f5a039bc640 1 -- start start 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.754+0000 7f5a039bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc15dca0 0x7f59fc06fd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.754+0000 7f5a039bc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f59fc15e080 0x7f59fc066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.754+0000 7f5a039bc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f59fc161c20 con 0x7f59fc0b00c0 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.754+0000 7f5a039bc640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f59fc1619d0 con 0x7f59fc15e080 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.754+0000 7f5a039bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f59fc161f00 con 0x7f59fc15dca0 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a01f59640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f59fc0b00c0 0x7f59fc0719b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59542/0 (socket says 172.21.15.64:59542) 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a00f57640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f59fc15e080 0x7f59fc066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a01f59640 1 -- 172.21.15.64:0/176444670 learned_addr learned my addr 172.21.15.64:0/176444670 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:15.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a01758640 1 --2- 172.21.15.64:0/176444670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc15dca0 0x7f59fc06fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:15.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f59f3fff640 1 -- 172.21.15.64:0/176444670 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2617725837 0 0) 0x7f59fc161c20 con 0x7f59fc0b00c0 2024-03-04T07:35:15.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f59f3fff640 1 -- 172.21.15.64:0/176444670 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f59e8003520 con 0x7f59fc0b00c0 2024-03-04T07:35:15.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a01758640 1 -- 172.21.15.64:0/176444670 >> v2:172.21.15.64:3300/0 conn(0x7f59fc15e080 msgr2=0x7f59fc066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:15.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a01758640 1 --2- 172.21.15.64:0/176444670 >> v2:172.21.15.64:3300/0 conn(0x7f59fc15e080 0x7f59fc066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a01758640 1 -- 172.21.15.64:0/176444670 >> v1:172.21.15.64:6789/0 conn(0x7f59fc0b00c0 legacy=0x7f59fc0719b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:15.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a01758640 1 -- 172.21.15.64:0/176444670 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59fc1621e0 con 0x7f59fc15dca0 2024-03-04T07:35:15.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.755+0000 7f5a00f57640 1 --2- 172.21.15.64:0/176444670 >> v2:172.21.15.64:3300/0 conn(0x7f59fc15e080 0x7f59fc066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:15.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.756+0000 7f5a01758640 1 --2- 172.21.15.64:0/176444670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc15dca0 0x7f59fc06fd10 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f59f80097a0 tx=0x7f59f802b8b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=69d58ce65ed662ef server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:15.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.756+0000 7f59f3fff640 1 -- 172.21.15.64:0/176444670 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f59f8035020 con 0x7f59fc15dca0 2024-03-04T07:35:15.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.756+0000 7f59f3fff640 1 -- 172.21.15.64:0/176444670 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f59f8031810 con 0x7f59fc15dca0 2024-03-04T07:35:15.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.756+0000 7f59f3fff640 1 -- 172.21.15.64:0/176444670 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f59f8031bc0 con 0x7f59fc15dca0 2024-03-04T07:35:15.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.756+0000 7f5a039bc640 1 -- 172.21.15.64:0/176444670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc15dca0 msgr2=0x7f59fc06fd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:15.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.756+0000 7f5a039bc640 1 --2- 172.21.15.64:0/176444670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc15dca0 0x7f59fc06fd10 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f59f80097a0 tx=0x7f59f802b8b0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.757+0000 7f5a039bc640 1 -- 172.21.15.64:0/176444670 shutdown_connections 2024-03-04T07:35:15.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.757+0000 7f5a039bc640 1 --2- 172.21.15.64:0/176444670 >> v2:172.21.15.64:3300/0 conn(0x7f59fc15e080 0x7f59fc066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.757+0000 7f5a039bc640 1 --2- 172.21.15.64:0/176444670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc15dca0 0x7f59fc06fd10 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.757+0000 7f5a039bc640 1 -- 172.21.15.64:0/176444670 >> 172.21.15.64:0/176444670 conn(0x7f59fc0d0990 msgr2=0x7f59fc0d0d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:15.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.757+0000 7f5a039bc640 1 -- 172.21.15.64:0/176444670 shutdown_connections 2024-03-04T07:35:15.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.757+0000 7f5a039bc640 1 -- 172.21.15.64:0/176444670 wait complete. 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.758+0000 7f5a039bc640 1 Processor -- start 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.758+0000 7f5a039bc640 1 -- start start 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.758+0000 7f5a039bc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc0b00c0 0x7f59fc15ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.758+0000 7f5a039bc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f59fc15dca0 0x7f59fc151970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a039bc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f59fc150230 0x7f59fc0cde90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a039bc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59fc0703d0 con 0x7f59fc150230 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a039bc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59fc070250 con 0x7f59fc15dca0 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a039bc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f59fc0b4cb0 con 0x7f59fc0b00c0 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f59fc150230 0x7f59fc0cde90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01758640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc0b00c0 0x7f59fc15ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f59fc150230 0x7f59fc0cde90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42018/0 (socket says 172.21.15.64:42018) 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 -- 172.21.15.64:0/3851941733 learned_addr learned my addr 172.21.15.64:0/3851941733 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a00f57640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f59fc15dca0 0x7f59fc151970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 -- 172.21.15.64:0/3851941733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc0b00c0 msgr2=0x7f59fc15ac60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc0b00c0 0x7f59fc15ac60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 -- 172.21.15.64:0/3851941733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f59fc15dca0 msgr2=0x7f59fc151970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f59fc15dca0 0x7f59fc151970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 -- 172.21.15.64:0/3851941733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59fc150710 con 0x7f59fc150230 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a01f59640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f59fc150230 0x7f59fc0cde90 secure :-1 s=READY pgs=1675 cs=0 l=1 rev1=1 crypto rx=0x7f59f400cbb0 tx=0x7f59f400a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f59f1ffb640 1 -- 172.21.15.64:0/3851941733 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f59f4017070 con 0x7f59fc150230 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f59f1ffb640 1 -- 172.21.15.64:0/3851941733 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f59f400ad40 con 0x7f59fc150230 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f59fc1509f0 con 0x7f59fc150230 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.759+0000 7f59f1ffb640 1 -- 172.21.15.64:0/3851941733 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f59f4012600 con 0x7f59fc150230 2024-03-04T07:35:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.760+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f59fc150fb0 con 0x7f59fc150230 2024-03-04T07:35:15.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.761+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f59fc1f6e30 con 0x7f59fc150230 2024-03-04T07:35:15.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.761+0000 7f59f1ffb640 1 -- 172.21.15.64:0/3851941733 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f59f401a070 con 0x7f59fc150230 2024-03-04T07:35:15.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.761+0000 7f59f1ffb640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59e006b4d0 0x7f59e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:15.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.761+0000 7f59f1ffb640 1 -- 172.21.15.64:0/3851941733 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f59f4089ee0 con 0x7f59fc150230 2024-03-04T07:35:15.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.761+0000 7f5a01758640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59e006b4d0 0x7f59e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:15.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.762+0000 7f5a01758640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59e006b4d0 0x7f59e006d990 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7f59f8036000 tx=0x7f59f8008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:15.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.766+0000 7f59f1ffb640 1 -- 172.21.15.64:0/3851941733 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f59fc1f6e30 con 0x7f59fc150230 2024-03-04T07:35:15.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.985+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f59fc0af340 con 0x7f59fc150230 2024-03-04T07:35:15.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.985+0000 7f59f1ffb640 1 -- 172.21.15.64:0/3851941733 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f59f4058760 con 0x7f59fc150230 2024-03-04T07:35:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:15.986 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:15.986 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:15.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59e006b4d0 msgr2=0x7f59e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:15.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59e006b4d0 0x7f59e006d990 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7f59f8036000 tx=0x7f59f8008030 comp rx=0 tx=0).stop 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f59fc150230 msgr2=0x7f59fc0cde90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f59fc150230 0x7f59fc0cde90 secure :-1 s=READY pgs=1675 cs=0 l=1 rev1=1 crypto rx=0x7f59f400cbb0 tx=0x7f59f400a430 comp rx=0 tx=0).stop 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 shutdown_connections 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f59e006b4d0 0x7f59e006d990 unknown :-1 s=CLOSED pgs=753 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f59fc150230 0x7f59fc0cde90 unknown :-1 s=CLOSED pgs=1675 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f59fc15dca0 0x7f59fc151970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 --2- 172.21.15.64:0/3851941733 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f59fc0b00c0 0x7f59fc15ac60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 >> 172.21.15.64:0/3851941733 conn(0x7f59fc0d0990 msgr2=0x7f59fc15d470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 shutdown_connections 2024-03-04T07:35:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:15.987+0000 7f5a039bc640 1 -- 172.21.15.64:0/3851941733 wait complete. 2024-03-04T07:35:16.199 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:16.199 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:16.588 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:17.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:17.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.314+0000 7f4bad113640 1 Processor -- start 2024-03-04T07:35:17.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.314+0000 7f4bad113640 1 -- start start 2024-03-04T07:35:17.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4bad113640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80b4e70 0x7f4ba8161ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:17.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4bad113640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4ba8162000 0x7f4ba80ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:17.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4bad113640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4ba806dbe0 con 0x7f4ba8147930 2024-03-04T07:35:17.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4bad113640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4ba8071a50 con 0x7f4ba8162000 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4bad113640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba8071d30 con 0x7f4ba80b4e70 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4ba8162000 0x7f4ba80ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba759e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4ba8162000 0x7f4ba80ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42022/0 (socket says 172.21.15.64:42022) 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba759e640 1 -- 172.21.15.64:0/3698742201 learned_addr learned my addr 172.21.15.64:0/3698742201 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba659c640 1 --2- 172.21.15.64:0/3698742201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80b4e70 0x7f4ba8161ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba759e640 1 -- 172.21.15.64:0/3698742201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80b4e70 msgr2=0x7f4ba8161ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba759e640 1 --2- 172.21.15.64:0/3698742201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80b4e70 0x7f4ba8161ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba759e640 1 -- 172.21.15.64:0/3698742201 >> v1:172.21.15.64:6789/0 conn(0x7f4ba8147930 legacy=0x7f4ba8072530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba759e640 1 -- 172.21.15.64:0/3698742201 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ba8072010 con 0x7f4ba8162000 2024-03-04T07:35:17.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.315+0000 7f4ba5d9b640 1 -- 172.21.15.64:0/3698742201 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4102435613 0 0) 0x7f4ba806dbe0 con 0x7f4ba8147930 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.316+0000 7f4ba759e640 1 --2- 172.21.15.64:0/3698742201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba8162000 0x7f4ba80ad8b0 secure :-1 s=READY pgs=1677 cs=0 l=1 rev1=1 crypto rx=0x7f4b98009e50 tx=0x7f4b9802bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f52a61e18e5a7d4c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.316+0000 7f4ba5d9b640 1 -- 172.21.15.64:0/3698742201 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b9802cb00 con 0x7f4ba8162000 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.316+0000 7f4ba5d9b640 1 -- 172.21.15.64:0/3698742201 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4b9802cca0 con 0x7f4ba8162000 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.316+0000 7f4ba5d9b640 1 -- 172.21.15.64:0/3698742201 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b98034610 con 0x7f4ba8162000 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 -- 172.21.15.64:0/3698742201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba8162000 msgr2=0x7f4ba80ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 --2- 172.21.15.64:0/3698742201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba8162000 0x7f4ba80ad8b0 secure :-1 s=READY pgs=1677 cs=0 l=1 rev1=1 crypto rx=0x7f4b98009e50 tx=0x7f4b9802bf60 comp rx=0 tx=0).stop 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 -- 172.21.15.64:0/3698742201 shutdown_connections 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 --2- 172.21.15.64:0/3698742201 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba8162000 0x7f4ba80ad8b0 unknown :-1 s=CLOSED pgs=1677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 --2- 172.21.15.64:0/3698742201 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba80b4e70 0x7f4ba8161ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 -- 172.21.15.64:0/3698742201 >> 172.21.15.64:0/3698742201 conn(0x7f4ba806cd20 msgr2=0x7f4ba806d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 -- 172.21.15.64:0/3698742201 shutdown_connections 2024-03-04T07:35:17.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.317+0000 7f4bad113640 1 -- 172.21.15.64:0/3698742201 wait complete. 2024-03-04T07:35:17.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.318+0000 7f4bad113640 1 Processor -- start 2024-03-04T07:35:17.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4bad113640 1 -- start start 2024-03-04T07:35:17.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4bad113640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4e70 0x7f4ba813f710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:17.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4bad113640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba8147930 0x7f4ba81e82d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:17.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4bad113640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba8162000 0x7f4ba8140160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4bad113640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba806d840 con 0x7f4ba80b4e70 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4bad113640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba80b52b0 con 0x7f4ba8162000 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4bad113640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4ba806d9c0 con 0x7f4ba8147930 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4ba6d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4e70 0x7f4ba813f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4ba6d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4e70 0x7f4ba813f710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42036/0 (socket says 172.21.15.64:42036) 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4ba6d9d640 1 -- 172.21.15.64:0/2510175420 learned_addr learned my addr 172.21.15.64:0/2510175420 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4ba659c640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba8147930 0x7f4ba81e82d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.319+0000 7f4ba759e640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba8162000 0x7f4ba8140160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4ba6d9d640 1 -- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba8147930 msgr2=0x7f4ba81e82d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4ba6d9d640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba8147930 0x7f4ba81e82d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4ba6d9d640 1 -- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba8162000 msgr2=0x7f4ba8140160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4ba6d9d640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba8162000 0x7f4ba8140160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4ba6d9d640 1 -- 172.21.15.64:0/2510175420 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ba80cd2e0 con 0x7f4ba80b4e70 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4ba659c640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba8147930 0x7f4ba81e82d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4ba6d9d640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4e70 0x7f4ba813f710 secure :-1 s=READY pgs=1678 cs=0 l=1 rev1=1 crypto rx=0x7f4ba000c4b0 tx=0x7f4ba000c970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4b9f7fe640 1 -- 172.21.15.64:0/2510175420 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ba0017070 con 0x7f4ba80b4e70 2024-03-04T07:35:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4b9f7fe640 1 -- 172.21.15.64:0/2510175420 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4ba0005480 con 0x7f4ba80b4e70 2024-03-04T07:35:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ba81e8a00 con 0x7f4ba80b4e70 2024-03-04T07:35:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4b9f7fe640 1 -- 172.21.15.64:0/2510175420 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ba0003830 con 0x7f4ba80b4e70 2024-03-04T07:35:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.320+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ba81e8fc0 con 0x7f4ba80b4e70 2024-03-04T07:35:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.321+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ba8067ff0 con 0x7f4ba80b4e70 2024-03-04T07:35:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.322+0000 7f4b9f7fe640 1 -- 172.21.15.64:0/2510175420 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4ba001a070 con 0x7f4ba80b4e70 2024-03-04T07:35:17.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.322+0000 7f4b9f7fe640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b8806b4d0 0x7f4b8806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:17.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.322+0000 7f4ba659c640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b8806b4d0 0x7f4b8806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:17.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.322+0000 7f4b9f7fe640 1 -- 172.21.15.64:0/2510175420 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4ba0089540 con 0x7f4ba80b4e70 2024-03-04T07:35:17.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.323+0000 7f4ba659c640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b8806b4d0 0x7f4b8806d990 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f4ba80ae500 tx=0x7f4b90009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:17.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.327+0000 7f4b9f7fe640 1 -- 172.21.15.64:0/2510175420 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4ba0057d30 con 0x7f4ba80b4e70 2024-03-04T07:35:17.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.544+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4ba80b31c0 con 0x7f4ba80b4e70 2024-03-04T07:35:17.544 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.544+0000 7f4b9f7fe640 1 -- 172.21.15.64:0/2510175420 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4ba005bdd0 con 0x7f4ba80b4e70 2024-03-04T07:35:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:17.545 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:17.545 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b8806b4d0 msgr2=0x7f4b8806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b8806b4d0 0x7f4b8806d990 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f4ba80ae500 tx=0x7f4b90009190 comp rx=0 tx=0).stop 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4e70 msgr2=0x7f4ba813f710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4e70 0x7f4ba813f710 secure :-1 s=READY pgs=1678 cs=0 l=1 rev1=1 crypto rx=0x7f4ba000c4b0 tx=0x7f4ba000c970 comp rx=0 tx=0).stop 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 shutdown_connections 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4b8806b4d0 0x7f4b8806d990 unknown :-1 s=CLOSED pgs=754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4ba8162000 0x7f4ba8140160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4ba8147930 0x7f4ba81e82d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 --2- 172.21.15.64:0/2510175420 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4ba80b4e70 0x7f4ba813f710 unknown :-1 s=CLOSED pgs=1678 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:17.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 >> 172.21.15.64:0/2510175420 conn(0x7f4ba806cd20 msgr2=0x7f4ba80aee20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:17.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 shutdown_connections 2024-03-04T07:35:17.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:17.546+0000 7f4bad113640 1 -- 172.21.15.64:0/2510175420 wait complete. 2024-03-04T07:35:17.760 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:17.761 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:18.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:35:18.332 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:35:18.590 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:18.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:18.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.875+0000 7f4d7b066640 1 Processor -- start 2024-03-04T07:35:18.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.875+0000 7f4d7b066640 1 -- start start 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d7b066640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d7406e700 0x7f4d7415ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d7b066640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4d740b4160 con 0x7f4d74072660 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d7b066640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4d740b5b40 con 0x7f4d740b55d0 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d7b066640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d740b3ba0 con 0x7f4d7406e700 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d73fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4d74072660 0x7f4d7406d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59566/0 (socket says 172.21.15.64:59566) 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d73fff640 1 -- 172.21.15.64:0/499962865 learned_addr learned my addr 172.21.15.64:0/499962865 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d78e02640 1 --2- 172.21.15.64:0/499962865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d7406e700 0x7f4d7415ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d737fe640 1 -- 172.21.15.64:0/499962865 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 896844431 0 0) 0x7f4d740b4160 con 0x7f4d74072660 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d737fe640 1 -- 172.21.15.64:0/499962865 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4d64003520 con 0x7f4d74072660 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d737fe640 1 -- 172.21.15.64:0/499962865 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 591564433 0 0) 0x7f4d740b5b40 con 0x7f4d740b55d0 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.876+0000 7f4d737fe640 1 -- 172.21.15.64:0/499962865 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4d740b4160 con 0x7f4d740b55d0 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.877+0000 7f4d78e02640 1 -- 172.21.15.64:0/499962865 >> v1:172.21.15.66:6789/0 conn(0x7f4d740b55d0 legacy=0x7f4d74071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.877+0000 7f4d78e02640 1 -- 172.21.15.64:0/499962865 >> v1:172.21.15.64:6789/0 conn(0x7f4d74072660 legacy=0x7f4d7406d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:18.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.877+0000 7f4d78e02640 1 -- 172.21.15.64:0/499962865 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d740b3e80 con 0x7f4d7406e700 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.877+0000 7f4d78e02640 1 --2- 172.21.15.64:0/499962865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d7406e700 0x7f4d7415ef80 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f4d680097d0 tx=0x7f4d6802b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=43aa33f327c9a93f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d737fe640 1 -- 172.21.15.64:0/499962865 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d68035020 con 0x7f4d7406e700 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d737fe640 1 -- 172.21.15.64:0/499962865 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d68031890 con 0x7f4d7406e700 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d737fe640 1 -- 172.21.15.64:0/499962865 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d68031c40 con 0x7f4d7406e700 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d7b066640 1 -- 172.21.15.64:0/499962865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d7406e700 msgr2=0x7f4d7415ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d7b066640 1 --2- 172.21.15.64:0/499962865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d7406e700 0x7f4d7415ef80 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f4d680097d0 tx=0x7f4d6802b930 comp rx=0 tx=0).stop 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d7b066640 1 -- 172.21.15.64:0/499962865 shutdown_connections 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d7b066640 1 --2- 172.21.15.64:0/499962865 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d7406e700 0x7f4d7415ef80 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d7b066640 1 -- 172.21.15.64:0/499962865 >> 172.21.15.64:0/499962865 conn(0x7f4d740b2720 msgr2=0x7f4d740b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d7b066640 1 -- 172.21.15.64:0/499962865 shutdown_connections 2024-03-04T07:35:18.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.878+0000 7f4d7b066640 1 -- 172.21.15.64:0/499962865 wait complete. 2024-03-04T07:35:18.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.879+0000 7f4d7b066640 1 Processor -- start 2024-03-04T07:35:18.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.879+0000 7f4d7b066640 1 -- start start 2024-03-04T07:35:18.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d7b066640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d7406e700 0x7f4d741442d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d7b066640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d74072660 0x7f4d74146690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d7b066640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d740b55d0 0x7f4d74146bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d7b066640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d74072210 con 0x7f4d7406e700 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d7b066640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d74072090 con 0x7f4d740b55d0 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d7b066640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4d740670e0 con 0x7f4d74072660 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d78e02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d7406e700 0x7f4d741442d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.880+0000 7f4d78e02640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d7406e700 0x7f4d741442d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42042/0 (socket says 172.21.15.64:42042) 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d78e02640 1 -- 172.21.15.64:0/2291474437 learned_addr learned my addr 172.21.15.64:0/2291474437 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d73fff640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d74072660 0x7f4d74146690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d79603640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d740b55d0 0x7f4d74146bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d78e02640 1 -- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d74072660 msgr2=0x7f4d74146690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d78e02640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d74072660 0x7f4d74146690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d78e02640 1 -- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d740b55d0 msgr2=0x7f4d74146bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:18.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d78e02640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d740b55d0 0x7f4d74146bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:18.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d78e02640 1 -- 172.21.15.64:0/2291474437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d7415a890 con 0x7f4d7406e700 2024-03-04T07:35:18.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d78e02640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d7406e700 0x7f4d741442d0 secure :-1 s=READY pgs=1680 cs=0 l=1 rev1=1 crypto rx=0x7f4d68000c00 tx=0x7f4d680046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:18.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d717fa640 1 -- 172.21.15.64:0/2291474437 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d68035040 con 0x7f4d7406e700 2024-03-04T07:35:18.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d717fa640 1 -- 172.21.15.64:0/2291474437 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4d68030d20 con 0x7f4d7406e700 2024-03-04T07:35:18.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d717fa640 1 -- 172.21.15.64:0/2291474437 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d68006e10 con 0x7f4d7406e700 2024-03-04T07:35:18.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d740ced10 con 0x7f4d7406e700 2024-03-04T07:35:18.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.881+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4d740cf7f0 con 0x7f4d7406e700 2024-03-04T07:35:18.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.883+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4d740ceb00 con 0x7f4d7406e700 2024-03-04T07:35:18.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.883+0000 7f4d717fa640 1 -- 172.21.15.64:0/2291474437 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4d68038020 con 0x7f4d7406e700 2024-03-04T07:35:18.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.883+0000 7f4d717fa640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5806b4d0 0x7f4d5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:18.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.883+0000 7f4d717fa640 1 -- 172.21.15.64:0/2291474437 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4d680ac7c0 con 0x7f4d7406e700 2024-03-04T07:35:18.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.883+0000 7f4d73fff640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5806b4d0 0x7f4d5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:18.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.884+0000 7f4d73fff640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5806b4d0 0x7f4d5806d990 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f4d74067b20 tx=0x7f4d60007d70 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:18.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:18.888+0000 7f4d717fa640 1 -- 172.21.15.64:0/2291474437 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4d6802d040 con 0x7f4d7406e700 2024-03-04T07:35:19.104 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.104+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4d74067ff0 con 0x7f4d7406e700 2024-03-04T07:35:19.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.105+0000 7f4d717fa640 1 -- 172.21.15.64:0/2291474437 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4d6807af30 con 0x7f4d7406e700 2024-03-04T07:35:19.105 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:19.105 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:19.105 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5806b4d0 msgr2=0x7f4d5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5806b4d0 0x7f4d5806d990 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f4d74067b20 tx=0x7f4d60007d70 comp rx=0 tx=0).stop 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d7406e700 msgr2=0x7f4d741442d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d7406e700 0x7f4d741442d0 secure :-1 s=READY pgs=1680 cs=0 l=1 rev1=1 crypto rx=0x7f4d68000c00 tx=0x7f4d680046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 shutdown_connections 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4d5806b4d0 0x7f4d5806d990 unknown :-1 s=CLOSED pgs=755 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4d740b55d0 0x7f4d74146bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4d74072660 0x7f4d74146690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:19.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 --2- 172.21.15.64:0/2291474437 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4d7406e700 0x7f4d741442d0 unknown :-1 s=CLOSED pgs=1680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:19.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 >> 172.21.15.64:0/2291474437 conn(0x7f4d740b2720 msgr2=0x7f4d740b1ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:19.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 shutdown_connections 2024-03-04T07:35:19.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:19.107+0000 7f4d7b066640 1 -- 172.21.15.64:0/2291474437 wait complete. 2024-03-04T07:35:19.319 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:19.319 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:20.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:20.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.434+0000 7efdc444e640 1 Processor -- start 2024-03-04T07:35:20.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.434+0000 7efdc444e640 1 -- start start 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc444e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efdbc145780 0x7efdbc0d0470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc444e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc066d00 0x7efdbc0671b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc444e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efdbc0cf3d0 con 0x7efdbc0ceed0 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc444e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efdbc072c30 con 0x7efdbc145780 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc444e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efdbc0cf6b0 con 0x7efdbc066d00 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc21ea640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efdbc145780 0x7efdbc0d0470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc29eb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc066d00 0x7efdbc0671b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc19e9640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7efdbc0ceed0 0x7efdbc072520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:33848/0 (socket says 172.21.15.64:33848) 2024-03-04T07:35:20.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc19e9640 1 -- 172.21.15.64:0/3184193295 learned_addr learned my addr 172.21.15.64:0/3184193295 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc21ea640 1 -- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc066d00 msgr2=0x7efdbc0671b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc21ea640 1 --2- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc066d00 0x7efdbc0671b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc21ea640 1 -- 172.21.15.64:0/3184193295 >> v1:172.21.15.66:6789/0 conn(0x7efdbc0ceed0 legacy=0x7efdbc072520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc21ea640 1 -- 172.21.15.64:0/3184193295 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efdbc0cf990 con 0x7efdbc145780 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.435+0000 7efdc11e8640 1 -- 172.21.15.64:0/3184193295 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 779757037 0 0) 0x7efdbc0cf3d0 con 0x7efdbc0ceed0 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.436+0000 7efdc29eb640 1 --2- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc066d00 0x7efdbc0671b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.436+0000 7efdc21ea640 1 --2- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 0x7efdbc0d0470 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7efdb80097a0 tx=0x7efdb802b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=678c807fd46015d0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:20.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.436+0000 7efdc11e8640 1 -- 172.21.15.64:0/3184193295 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdb80315a0 con 0x7efdbc145780 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.436+0000 7efdc11e8640 1 -- 172.21.15.64:0/3184193295 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efdb8037070 con 0x7efdbc145780 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.436+0000 7efdc11e8640 1 -- 172.21.15.64:0/3184193295 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdb8031960 con 0x7efdbc145780 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 -- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 msgr2=0x7efdbc0d0470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 --2- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 0x7efdbc0d0470 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7efdb80097a0 tx=0x7efdb802b840 comp rx=0 tx=0).stop 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 -- 172.21.15.64:0/3184193295 shutdown_connections 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 --2- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc066d00 0x7efdbc0671b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 --2- 172.21.15.64:0/3184193295 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 0x7efdbc0d0470 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 -- 172.21.15.64:0/3184193295 >> 172.21.15.64:0/3184193295 conn(0x7efdbc0b2720 msgr2=0x7efdbc0b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 -- 172.21.15.64:0/3184193295 shutdown_connections 2024-03-04T07:35:20.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.437+0000 7efdc444e640 1 -- 172.21.15.64:0/3184193295 wait complete. 2024-03-04T07:35:20.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.438+0000 7efdc444e640 1 Processor -- start 2024-03-04T07:35:20.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.438+0000 7efdc444e640 1 -- start start 2024-03-04T07:35:20.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc444e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdbc066d00 0x7efdbc1f1860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc444e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc0ceed0 0x7efdbc13dc80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc444e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 0x7efdbc1f4280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc444e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdbc06db60 con 0x7efdbc066d00 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc444e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdbc06d9e0 con 0x7efdbc145780 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc444e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efdbc06dce0 con 0x7efdbc0ceed0 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc21ea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdbc066d00 0x7efdbc1f1860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc21ea640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdbc066d00 0x7efdbc1f1860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42058/0 (socket says 172.21.15.64:42058) 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc21ea640 1 -- 172.21.15.64:0/93815535 learned_addr learned my addr 172.21.15.64:0/93815535 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc19e9640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc0ceed0 0x7efdbc13dc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.439+0000 7efdc29eb640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 0x7efdbc1f4280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc21ea640 1 -- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc0ceed0 msgr2=0x7efdbc13dc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc21ea640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc0ceed0 0x7efdbc13dc80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc21ea640 1 -- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 msgr2=0x7efdbc1f4280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc21ea640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 0x7efdbc1f4280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc21ea640 1 -- 172.21.15.64:0/93815535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efdbc13fc40 con 0x7efdbc066d00 2024-03-04T07:35:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc21ea640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdbc066d00 0x7efdbc1f1860 secure :-1 s=READY pgs=1681 cs=0 l=1 rev1=1 crypto rx=0x7efdb8034040 tx=0x7efdb80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdb2ffd640 1 -- 172.21.15.64:0/93815535 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdb8032630 con 0x7efdbc066d00 2024-03-04T07:35:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdb2ffd640 1 -- 172.21.15.64:0/93815535 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efdb80327f0 con 0x7efdbc066d00 2024-03-04T07:35:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efdbc1408b0 con 0x7efdbc066d00 2024-03-04T07:35:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdb2ffd640 1 -- 172.21.15.64:0/93815535 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdb8032be0 con 0x7efdbc066d00 2024-03-04T07:35:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.440+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efdbc140e70 con 0x7efdbc066d00 2024-03-04T07:35:20.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.441+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efdbc067ff0 con 0x7efdbc066d00 2024-03-04T07:35:20.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.442+0000 7efdb2ffd640 1 -- 172.21.15.64:0/93815535 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efdb803b3f0 con 0x7efdbc066d00 2024-03-04T07:35:20.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.442+0000 7efdb2ffd640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efda006b4d0 0x7efda006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:20.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.442+0000 7efdc19e9640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efda006b4d0 0x7efda006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:20.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.442+0000 7efdb2ffd640 1 -- 172.21.15.64:0/93815535 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7efdb802f030 con 0x7efdbc066d00 2024-03-04T07:35:20.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.443+0000 7efdc19e9640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efda006b4d0 0x7efda006d990 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7efdac007f70 tx=0x7efdac007f00 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:20.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.447+0000 7efdb2ffd640 1 -- 172.21.15.64:0/93815535 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efdb8040070 con 0x7efdbc066d00 2024-03-04T07:35:20.591 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:20.665 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.666+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7efdbc06eae0 con 0x7efdbc066d00 2024-03-04T07:35:20.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.666+0000 7efdb2ffd640 1 -- 172.21.15.64:0/93815535 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7efdb807ace0 con 0x7efdbc066d00 2024-03-04T07:35:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:20.666 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:20.667 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efda006b4d0 msgr2=0x7efda006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efda006b4d0 0x7efda006d990 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7efdac007f70 tx=0x7efdac007f00 comp rx=0 tx=0).stop 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdbc066d00 msgr2=0x7efdbc1f1860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdbc066d00 0x7efdbc1f1860 secure :-1 s=READY pgs=1681 cs=0 l=1 rev1=1 crypto rx=0x7efdb8034040 tx=0x7efdb80046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 shutdown_connections 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efda006b4d0 0x7efda006d990 unknown :-1 s=CLOSED pgs=756 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efdbc145780 0x7efdbc1f4280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efdbc0ceed0 0x7efdbc13dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 --2- 172.21.15.64:0/93815535 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efdbc066d00 0x7efdbc1f1860 unknown :-1 s=CLOSED pgs=1681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.668+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 >> 172.21.15.64:0/93815535 conn(0x7efdbc0b2720 msgr2=0x7efdbc06f000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.669+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 shutdown_connections 2024-03-04T07:35:20.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:20.669+0000 7efdc444e640 1 -- 172.21.15.64:0/93815535 wait complete. 2024-03-04T07:35:20.881 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:20.881 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:21.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:21.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.996+0000 7f286861c640 1 Processor -- start 2024-03-04T07:35:21.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.996+0000 7f286861c640 1 -- start start 2024-03-04T07:35:21.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f286861c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f286015df70 0x7f28600d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:21.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f286861c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2860071c50 0x7f28600b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:21.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f286861c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f28600b6f20 con 0x7f28600d18c0 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f286861c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f28600d3030 con 0x7f286015df70 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f286861c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f28600d3310 con 0x7f2860071c50 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f28663b8640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f286015df70 0x7f28600d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f2865bb7640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f28600d18c0 0x7f2860071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:59580/0 (socket says 172.21.15.64:59580) 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f2865bb7640 1 -- 172.21.15.64:0/771864253 learned_addr learned my addr 172.21.15.64:0/771864253 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.997+0000 7f2866bb9640 1 --2- 172.21.15.64:0/771864253 >> v2:172.21.15.66:3300/0 conn(0x7f2860071c50 0x7f28600b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.998+0000 7f28663b8640 1 -- 172.21.15.64:0/771864253 >> v2:172.21.15.66:3300/0 conn(0x7f2860071c50 msgr2=0x7f28600b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.998+0000 7f28663b8640 1 --2- 172.21.15.64:0/771864253 >> v2:172.21.15.66:3300/0 conn(0x7f2860071c50 0x7f28600b2700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.998+0000 7f28663b8640 1 -- 172.21.15.64:0/771864253 >> v1:172.21.15.64:6789/0 conn(0x7f28600d18c0 legacy=0x7f2860071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.998+0000 7f28663b8640 1 -- 172.21.15.64:0/771864253 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28600d35f0 con 0x7f286015df70 2024-03-04T07:35:22.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.998+0000 7f28663b8640 1 --2- 172.21.15.64:0/771864253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f28600d27b0 secure :-1 s=READY pgs=1683 cs=0 l=1 rev1=1 crypto rx=0x7f286008d130 tx=0x7f285002bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8acf62273f2eae33 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:22.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.998+0000 7f28653b6640 1 -- 172.21.15.64:0/771864253 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2850039070 con 0x7f286015df70 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f28653b6640 1 -- 172.21.15.64:0/771864253 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f285002cd00 con 0x7f286015df70 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f28653b6640 1 -- 172.21.15.64:0/771864253 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2850034660 con 0x7f286015df70 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 -- 172.21.15.64:0/771864253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 msgr2=0x7f28600d27b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 --2- 172.21.15.64:0/771864253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f28600d27b0 secure :-1 s=READY pgs=1683 cs=0 l=1 rev1=1 crypto rx=0x7f286008d130 tx=0x7f285002bf50 comp rx=0 tx=0).stop 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 -- 172.21.15.64:0/771864253 shutdown_connections 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 --2- 172.21.15.64:0/771864253 >> v2:172.21.15.66:3300/0 conn(0x7f2860071c50 0x7f28600b2700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 --2- 172.21.15.64:0/771864253 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f28600d27b0 unknown :-1 s=CLOSED pgs=1683 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 -- 172.21.15.64:0/771864253 >> 172.21.15.64:0/771864253 conn(0x7f28600d3cf0 msgr2=0x7f28600d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 -- 172.21.15.64:0/771864253 shutdown_connections 2024-03-04T07:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:21.999+0000 7f286861c640 1 -- 172.21.15.64:0/771864253 wait complete. 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.000+0000 7f286861c640 1 Processor -- start 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.000+0000 7f286861c640 1 -- start start 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f286861c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2860071c50 0x7f28600cbed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f286861c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28600d18c0 0x7f28600cc410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f286861c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f2860158d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f286861c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f286015e350 con 0x7f286015df70 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f286861c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28600d2cf0 con 0x7f2860071c50 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f286861c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f286015e4d0 con 0x7f28600d18c0 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f2866bb9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f2860158d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f2866bb9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f2860158d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42080/0 (socket says 172.21.15.64:42080) 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f2866bb9640 1 -- 172.21.15.64:0/622032364 learned_addr learned my addr 172.21.15.64:0/622032364 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:22.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f2865bb7640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28600d18c0 0x7f28600cc410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.001+0000 7f28663b8640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2860071c50 0x7f28600cbed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f2866bb9640 1 -- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28600d18c0 msgr2=0x7f28600cc410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f2866bb9640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28600d18c0 0x7f28600cc410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f2866bb9640 1 -- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2860071c50 msgr2=0x7f28600cbed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f2866bb9640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2860071c50 0x7f28600cbed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f2866bb9640 1 -- 172.21.15.64:0/622032364 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28600ccc50 con 0x7f286015df70 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f2866bb9640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f2860158d20 secure :-1 s=READY pgs=1684 cs=0 l=1 rev1=1 crypto rx=0x7f285800e850 tx=0x7f285800ed10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f285effd640 1 -- 172.21.15.64:0/622032364 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2858017070 con 0x7f286015df70 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f285effd640 1 -- 172.21.15.64:0/622032364 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f285800c940 con 0x7f286015df70 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28600ccf30 con 0x7f286015df70 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f285effd640 1 -- 172.21.15.64:0/622032364 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f285800cd30 con 0x7f286015df70 2024-03-04T07:35:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.002+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28600cd4f0 con 0x7f286015df70 2024-03-04T07:35:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.003+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2860067ff0 con 0x7f286015df70 2024-03-04T07:35:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.003+0000 7f285effd640 1 -- 172.21.15.64:0/622032364 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f285801a070 con 0x7f286015df70 2024-03-04T07:35:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.004+0000 7f285effd640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f284c06b4d0 0x7f284c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.004+0000 7f285effd640 1 -- 172.21.15.64:0/622032364 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f285808a450 con 0x7f286015df70 2024-03-04T07:35:22.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.004+0000 7f28663b8640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f284c06b4d0 0x7f284c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:22.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.004+0000 7f28663b8640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f284c06b4d0 0x7f284c06d990 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f2850036000 tx=0x7f2850030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:22.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.008+0000 7f285effd640 1 -- 172.21.15.64:0/622032364 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2858058cd0 con 0x7f286015df70 2024-03-04T07:35:22.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.224+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f28600cf010 con 0x7f286015df70 2024-03-04T07:35:22.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.225+0000 7f285effd640 1 -- 172.21.15.64:0/622032364 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f285805cd70 con 0x7f286015df70 2024-03-04T07:35:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:22.226 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:22.226 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:22.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f284c06b4d0 msgr2=0x7f284c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:22.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f284c06b4d0 0x7f284c06d990 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f2850036000 tx=0x7f2850030000 comp rx=0 tx=0).stop 2024-03-04T07:35:22.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 msgr2=0x7f2860158d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:22.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f2860158d20 secure :-1 s=READY pgs=1684 cs=0 l=1 rev1=1 crypto rx=0x7f285800e850 tx=0x7f285800ed10 comp rx=0 tx=0).stop 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 shutdown_connections 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f284c06b4d0 0x7f284c06d990 unknown :-1 s=CLOSED pgs=757 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f286015df70 0x7f2860158d20 unknown :-1 s=CLOSED pgs=1684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f28600d18c0 0x7f28600cc410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 --2- 172.21.15.64:0/622032364 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2860071c50 0x7f28600cbed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 >> 172.21.15.64:0/622032364 conn(0x7f28600d3cf0 msgr2=0x7f28600b5f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 shutdown_connections 2024-03-04T07:35:22.229 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:22.227+0000 7f286861c640 1 -- 172.21.15.64:0/622032364 wait complete. 2024-03-04T07:35:22.440 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:22.440 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:22.593 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:23.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.554+0000 7f1e8a02a640 1 Processor -- start 2024-03-04T07:35:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.554+0000 7f1e8a02a640 1 -- start start 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e8a02a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e840d0880 0x7f1e84168c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e8a02a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1e840cf130 con 0x7f1e8406fe20 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e8a02a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1e840d0dd0 con 0x7f1e8415e060 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e8a02a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1e840d10b0 con 0x7f1e840d0880 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e82ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1e8406fe20 0x7f1e840cea20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47956/0 (socket says 172.21.15.64:47956) 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e82ffd640 1 -- 172.21.15.64:0/3074615750 learned_addr learned my addr 172.21.15.64:0/3074615750 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e83fff640 1 --2- 172.21.15.64:0/3074615750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e840d0880 0x7f1e84168c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:23.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e827fc640 1 -- 172.21.15.64:0/3074615750 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 20204952 0 0) 0x7f1e840cf130 con 0x7f1e8406fe20 2024-03-04T07:35:23.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e827fc640 1 -- 172.21.15.64:0/3074615750 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1e6c003520 con 0x7f1e8406fe20 2024-03-04T07:35:23.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e827fc640 1 -- 172.21.15.64:0/3074615750 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1322186578 0 0) 0x7f1e840d0dd0 con 0x7f1e8415e060 2024-03-04T07:35:23.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e83fff640 1 -- 172.21.15.64:0/3074615750 >> v1:172.21.15.66:6789/0 conn(0x7f1e8415e060 legacy=0x7f1e840ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:23.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e83fff640 1 -- 172.21.15.64:0/3074615750 >> v1:172.21.15.64:6789/0 conn(0x7f1e8406fe20 legacy=0x7f1e840cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:23.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.555+0000 7f1e83fff640 1 -- 172.21.15.64:0/3074615750 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1e840cc0e0 con 0x7f1e840d0880 2024-03-04T07:35:23.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.556+0000 7f1e83fff640 1 --2- 172.21.15.64:0/3074615750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e840d0880 0x7f1e84168c20 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f1e7c009dc0 tx=0x7f1e7c02bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f6cf3853e1bd1395 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:23.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.556+0000 7f1e827fc640 1 -- 172.21.15.64:0/3074615750 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e7c039070 con 0x7f1e840d0880 2024-03-04T07:35:23.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.556+0000 7f1e827fc640 1 -- 172.21.15.64:0/3074615750 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1e7c02ccd0 con 0x7f1e840d0880 2024-03-04T07:35:23.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.556+0000 7f1e827fc640 1 -- 172.21.15.64:0/3074615750 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e7c034640 con 0x7f1e840d0880 2024-03-04T07:35:23.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.557+0000 7f1e8a02a640 1 -- 172.21.15.64:0/3074615750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e840d0880 msgr2=0x7f1e84168c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:23.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.557+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/3074615750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e840d0880 0x7f1e84168c20 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f1e7c009dc0 tx=0x7f1e7c02bf20 comp rx=0 tx=0).stop 2024-03-04T07:35:23.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.557+0000 7f1e8a02a640 1 -- 172.21.15.64:0/3074615750 shutdown_connections 2024-03-04T07:35:23.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.557+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/3074615750 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e840d0880 0x7f1e84168c20 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:23.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.557+0000 7f1e8a02a640 1 -- 172.21.15.64:0/3074615750 >> 172.21.15.64:0/3074615750 conn(0x7f1e840d1a90 msgr2=0x7f1e840d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:23.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.557+0000 7f1e8a02a640 1 -- 172.21.15.64:0/3074615750 shutdown_connections 2024-03-04T07:35:23.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.557+0000 7f1e8a02a640 1 -- 172.21.15.64:0/3074615750 wait complete. 2024-03-04T07:35:23.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.558+0000 7f1e8a02a640 1 Processor -- start 2024-03-04T07:35:23.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.558+0000 7f1e8a02a640 1 -- start start 2024-03-04T07:35:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e8a02a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1e8406fe20 0x7f1e841f4200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e8a02a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1e840d0880 0x7f1e84158000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e8a02a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e8415e060 0x7f1e841e8510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e8a02a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1e8415e440 con 0x7f1e8406fe20 2024-03-04T07:35:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e8a02a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1e840cc6a0 con 0x7f1e840d0880 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e8a02a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1e840cdbd0 con 0x7f1e8415e060 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1e8406fe20 0x7f1e841f4200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e82ffd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1e840d0880 0x7f1e84158000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1e8406fe20 0x7f1e841f4200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43896/0 (socket says 172.21.15.64:43896) 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e837fe640 1 -- 172.21.15.64:0/2944211074 learned_addr learned my addr 172.21.15.64:0/2944211074 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.559+0000 7f1e83fff640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e8415e060 0x7f1e841e8510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e83fff640 1 -- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1e840d0880 msgr2=0x7f1e84158000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e83fff640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1e840d0880 0x7f1e84158000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e83fff640 1 -- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1e8406fe20 msgr2=0x7f1e841f4200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e83fff640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1e8406fe20 0x7f1e841f4200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e83fff640 1 -- 172.21.15.64:0/2944211074 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1e841f7bb0 con 0x7f1e8415e060 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e837fe640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1e8406fe20 0x7f1e841f4200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e83fff640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e8415e060 0x7f1e841e8510 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f1e84072320 tx=0x7f1e7c004a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e6bfff640 1 -- 172.21.15.64:0/2944211074 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e7c039050 con 0x7f1e8415e060 2024-03-04T07:35:23.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e6bfff640 1 -- 172.21.15.64:0/2944211074 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1e7c03a040 con 0x7f1e8415e060 2024-03-04T07:35:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e6bfff640 1 -- 172.21.15.64:0/2944211074 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1e7c03e610 con 0x7f1e8415e060 2024-03-04T07:35:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1e840c8170 con 0x7f1e8415e060 2024-03-04T07:35:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.560+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1e841f70d0 con 0x7f1e8415e060 2024-03-04T07:35:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.561+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1e840b0c20 con 0x7f1e8415e060 2024-03-04T07:35:23.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.561+0000 7f1e6bfff640 1 -- 172.21.15.64:0/2944211074 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1e7c047020 con 0x7f1e8415e060 2024-03-04T07:35:23.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.562+0000 7f1e6bfff640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1e7006b4d0 0x7f1e7006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:23.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.562+0000 7f1e6bfff640 1 -- 172.21.15.64:0/2944211074 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1e7c0ac4f0 con 0x7f1e8415e060 2024-03-04T07:35:23.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.562+0000 7f1e837fe640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1e7006b4d0 0x7f1e7006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:23.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.562+0000 7f1e837fe640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1e7006b4d0 0x7f1e7006d990 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7f1e74000f60 tx=0x7f1e74009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:23.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.566+0000 7f1e6bfff640 1 -- 172.21.15.64:0/2944211074 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1e840b0c20 con 0x7f1e8415e060 2024-03-04T07:35:23.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.785+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1e84067ff0 con 0x7f1e8415e060 2024-03-04T07:35:23.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.785+0000 7f1e6bfff640 1 -- 172.21.15.64:0/2944211074 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f1e7c07acd0 con 0x7f1e8415e060 2024-03-04T07:35:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:23.787 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:23.787 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:23.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1e7006b4d0 msgr2=0x7f1e7006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:23.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1e7006b4d0 0x7f1e7006d990 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7f1e74000f60 tx=0x7f1e74009040 comp rx=0 tx=0).stop 2024-03-04T07:35:23.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e8415e060 msgr2=0x7f1e841e8510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:23.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e8415e060 0x7f1e841e8510 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f1e84072320 tx=0x7f1e7c004a40 comp rx=0 tx=0).stop 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 shutdown_connections 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1e7006b4d0 0x7f1e7006d990 unknown :-1 s=CLOSED pgs=758 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1e8415e060 0x7f1e841e8510 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1e840d0880 0x7f1e84158000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 --2- 172.21.15.64:0/2944211074 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1e8406fe20 0x7f1e841f4200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 >> 172.21.15.64:0/2944211074 conn(0x7f1e840d1a90 msgr2=0x7f1e8415d320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 shutdown_connections 2024-03-04T07:35:23.789 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:23.787+0000 7f1e8a02a640 1 -- 172.21.15.64:0/2944211074 wait complete. 2024-03-04T07:35:24.000 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:24.000 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:24.595 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:25.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:25.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.114+0000 7f29ef516640 1 Processor -- start 2024-03-04T07:35:25.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.114+0000 7f29ef516640 1 -- start start 2024-03-04T07:35:25.116 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ef516640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e815bac0 0x7f29e815cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ef516640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f29e80aee10 0x7f29e8070870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ef516640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29e80729b0 con 0x7f29e806d400 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ef516640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f29e806cf70 con 0x7f29e80aee10 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ef516640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29e806c920 con 0x7f29e815bac0 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29edab3640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f29e80aee10 0x7f29e8070870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ed2b2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e815bac0 0x7f29e815cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ecab1640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f29e806d400 0x7f29e80ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38848/0 (socket says 172.21.15.64:38848) 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ecab1640 1 -- 172.21.15.64:0/192337460 learned_addr learned my addr 172.21.15.64:0/192337460 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29edab3640 1 -- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e815bac0 msgr2=0x7f29e815cbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29edab3640 1 --2- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e815bac0 0x7f29e815cbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29edab3640 1 -- 172.21.15.64:0/192337460 >> v1:172.21.15.66:6789/0 conn(0x7f29e806d400 legacy=0x7f29e80ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29edab3640 1 -- 172.21.15.64:0/192337460 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29e806cc00 con 0x7f29e80aee10 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29dffff640 1 -- 172.21.15.64:0/192337460 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1013170357 0 0) 0x7f29e80729b0 con 0x7f29e806d400 2024-03-04T07:35:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.115+0000 7f29ed2b2640 1 --2- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e815bac0 0x7f29e815cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.116+0000 7f29edab3640 1 --2- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 0x7f29e8070870 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f29e0009d90 tx=0x7f29e002bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7e6d228064358492 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.116+0000 7f29dffff640 1 -- 172.21.15.64:0/192337460 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29e0039070 con 0x7f29e80aee10 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.116+0000 7f29dffff640 1 -- 172.21.15.64:0/192337460 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29e002cc50 con 0x7f29e80aee10 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.116+0000 7f29dffff640 1 -- 172.21.15.64:0/192337460 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29e0034440 con 0x7f29e80aee10 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 -- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 msgr2=0x7f29e8070870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 --2- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 0x7f29e8070870 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f29e0009d90 tx=0x7f29e002bea0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 -- 172.21.15.64:0/192337460 shutdown_connections 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 --2- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 0x7f29e8070870 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 --2- 172.21.15.64:0/192337460 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e815bac0 0x7f29e815cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 -- 172.21.15.64:0/192337460 >> 172.21.15.64:0/192337460 conn(0x7f29e8066d00 msgr2=0x7f29e80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 -- 172.21.15.64:0/192337460 shutdown_connections 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.117+0000 7f29ef516640 1 -- 172.21.15.64:0/192337460 wait complete. 2024-03-04T07:35:25.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.118+0000 7f29ef516640 1 Processor -- start 2024-03-04T07:35:25.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.118+0000 7f29ef516640 1 -- start start 2024-03-04T07:35:25.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ef516640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e806d400 0x7f29e81f4110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ef516640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 0x7f29e81f6ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ef516640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29e815bac0 0x7f29e80b1580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ef516640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29e815c020 con 0x7f29e815bac0 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ef516640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29e815bea0 con 0x7f29e80aee10 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ef516640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29e8072580 con 0x7f29e806d400 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29edab3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29e815bac0 0x7f29e80b1580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ed2b2640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e806d400 0x7f29e81f4110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29edab3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29e815bac0 0x7f29e80b1580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43910/0 (socket says 172.21.15.64:43910) 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29edab3640 1 -- 172.21.15.64:0/1836098114 learned_addr learned my addr 172.21.15.64:0/1836098114 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ed2b2640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e806d400 0x7f29e81f4110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:56126/0 (socket says 172.21.15.64:56126) 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ecab1640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 0x7f29e81f6ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ed2b2640 1 -- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 msgr2=0x7f29e81f6ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ed2b2640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 0x7f29e81f6ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ed2b2640 1 -- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29e815bac0 msgr2=0x7f29e80b1580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ed2b2640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29e815bac0 0x7f29e80b1580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29ed2b2640 1 -- 172.21.15.64:0/1836098114 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29e81f7410 con 0x7f29e806d400 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.119+0000 7f29edab3640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29e815bac0 0x7f29e80b1580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.120+0000 7f29ed2b2640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e806d400 0x7f29e81f4110 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f29e400e900 tx=0x7f29e400edc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.120+0000 7f29ddffb640 1 -- 172.21.15.64:0/1836098114 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29e4017070 con 0x7f29e806d400 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.120+0000 7f29ddffb640 1 -- 172.21.15.64:0/1836098114 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29e400ca00 con 0x7f29e806d400 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.120+0000 7f29ddffb640 1 -- 172.21.15.64:0/1836098114 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29e400cdf0 con 0x7f29e806d400 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.120+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29e8158000 con 0x7f29e806d400 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.120+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f29e81585c0 con 0x7f29e806d400 2024-03-04T07:35:25.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.121+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f29e8067ff0 con 0x7f29e806d400 2024-03-04T07:35:25.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.121+0000 7f29ddffb640 1 -- 172.21.15.64:0/1836098114 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f29e401a070 con 0x7f29e806d400 2024-03-04T07:35:25.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.122+0000 7f29ddffb640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29cc06b4d0 0x7f29cc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:25.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.122+0000 7f29ddffb640 1 -- 172.21.15.64:0/1836098114 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f29e408a0f0 con 0x7f29e806d400 2024-03-04T07:35:25.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.122+0000 7f29ecab1640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29cc06b4d0 0x7f29cc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:25.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.122+0000 7f29ecab1640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29cc06b4d0 0x7f29cc06d990 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f29e80b4120 tx=0x7f29d8007de0 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:25.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.126+0000 7f29ddffb640 1 -- 172.21.15.64:0/1836098114 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f29e4058850 con 0x7f29e806d400 2024-03-04T07:35:25.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.345+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f29e806de50 con 0x7f29e806d400 2024-03-04T07:35:25.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.346+0000 7f29ddffb640 1 -- 172.21.15.64:0/1836098114 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f29e405c8f0 con 0x7f29e806d400 2024-03-04T07:35:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:25.347 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:25.347 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:25.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29cc06b4d0 msgr2=0x7f29cc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:25.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29cc06b4d0 0x7f29cc06d990 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f29e80b4120 tx=0x7f29d8007de0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e806d400 msgr2=0x7f29e81f4110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:25.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e806d400 0x7f29e81f4110 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f29e400e900 tx=0x7f29e400edc0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 shutdown_connections 2024-03-04T07:35:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f29cc06b4d0 0x7f29cc06d990 unknown :-1 s=CLOSED pgs=759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29e815bac0 0x7f29e80b1580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29e80aee10 0x7f29e81f6ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 --2- 172.21.15.64:0/1836098114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29e806d400 0x7f29e81f4110 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 >> 172.21.15.64:0/1836098114 conn(0x7f29e8066d00 msgr2=0x7f29e815cd40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 shutdown_connections 2024-03-04T07:35:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:25.348+0000 7f29ef516640 1 -- 172.21.15.64:0/1836098114 wait complete. 2024-03-04T07:35:25.561 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:25.561 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:26.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:26.596 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:26.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.675+0000 7f60104c9640 1 Processor -- start 2024-03-04T07:35:26.676 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.675+0000 7f60104c9640 1 -- start start 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.676+0000 7f60104c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f600815df00 0x7f60080d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.676+0000 7f60104c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f60080cea30 con 0x7f60080b95a0 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.676+0000 7f60104c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f60080d2af0 con 0x7f60080b9a30 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.676+0000 7f60104c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f60080cf3c0 con 0x7f600815df00 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600da64640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f60080b95a0 0x7f60080b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47972/0 (socket says 172.21.15.64:47972) 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600da64640 1 -- 172.21.15.64:0/3586248393 learned_addr learned my addr 172.21.15.64:0/3586248393 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600e265640 1 --2- 172.21.15.64:0/3586248393 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f600815df00 0x7f60080d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600d263640 1 -- 172.21.15.64:0/3586248393 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2722226513 0 0) 0x7f60080d2af0 con 0x7f60080b9a30 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600d263640 1 -- 172.21.15.64:0/3586248393 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5ff4003520 con 0x7f60080b9a30 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600e265640 1 -- 172.21.15.64:0/3586248393 >> v1:172.21.15.66:6789/0 conn(0x7f60080b9a30 legacy=0x7f60080d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600e265640 1 -- 172.21.15.64:0/3586248393 >> v1:172.21.15.64:6789/0 conn(0x7f60080b95a0 legacy=0x7f60080b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:26.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.677+0000 7f600e265640 1 -- 172.21.15.64:0/3586248393 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60080cf0e0 con 0x7f600815df00 2024-03-04T07:35:26.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.678+0000 7f600e265640 1 --2- 172.21.15.64:0/3586248393 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f600815df00 0x7f60080d25b0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f60040097d0 tx=0x7f600402b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bdc63ad1cb69468e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:26.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f600d263640 1 -- 172.21.15.64:0/3586248393 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6004035020 con 0x7f600815df00 2024-03-04T07:35:26.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f600d263640 1 -- 172.21.15.64:0/3586248393 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6004031890 con 0x7f600815df00 2024-03-04T07:35:26.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f600d263640 1 -- 172.21.15.64:0/3586248393 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6004031c40 con 0x7f600815df00 2024-03-04T07:35:26.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f60104c9640 1 -- 172.21.15.64:0/3586248393 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f600815df00 msgr2=0x7f60080d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:26.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f60104c9640 1 --2- 172.21.15.64:0/3586248393 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f600815df00 0x7f60080d25b0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f60040097d0 tx=0x7f600402b930 comp rx=0 tx=0).stop 2024-03-04T07:35:26.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f60104c9640 1 -- 172.21.15.64:0/3586248393 shutdown_connections 2024-03-04T07:35:26.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f60104c9640 1 --2- 172.21.15.64:0/3586248393 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f600815df00 0x7f60080d25b0 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:26.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f60104c9640 1 -- 172.21.15.64:0/3586248393 >> 172.21.15.64:0/3586248393 conn(0x7f6008071960 msgr2=0x7f6008071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:26.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f60104c9640 1 -- 172.21.15.64:0/3586248393 shutdown_connections 2024-03-04T07:35:26.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.681+0000 7f60104c9640 1 -- 172.21.15.64:0/3586248393 wait complete. 2024-03-04T07:35:26.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 Processor -- start 2024-03-04T07:35:26.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 -- start start 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60080b95a0 0x7f60080b5110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f600815df00 0x7f60080cc6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60080ccbe0 0x7f6008157dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60080cec70 con 0x7f600815df00 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60080ce750 con 0x7f60080ccbe0 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.683+0000 7f60104c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f60080cedf0 con 0x7f60080b95a0 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f600815df00 0x7f60080cc6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f600815df00 0x7f60080cc6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43924/0 (socket says 172.21.15.64:43924) 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 -- 172.21.15.64:0/337946861 learned_addr learned my addr 172.21.15.64:0/337946861 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600e265640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60080b95a0 0x7f60080b5110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:26.686 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600ea66640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60080ccbe0 0x7f6008157dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 -- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60080b95a0 msgr2=0x7f60080b5110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60080b95a0 0x7f60080b5110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 -- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60080ccbe0 msgr2=0x7f6008157dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60080ccbe0 0x7f6008157dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 -- 172.21.15.64:0/337946861 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60080cdab0 con 0x7f600815df00 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600ea66640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60080ccbe0 0x7f6008157dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600e265640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60080b95a0 0x7f60080b5110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f600da64640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f600815df00 0x7f60080cc6a0 secure :-1 s=READY pgs=1687 cs=0 l=1 rev1=1 crypto rx=0x7f5ff800c4a0 tx=0x7f5ff800c960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f5ffeffd640 1 -- 172.21.15.64:0/337946861 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ff8017070 con 0x7f600815df00 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f5ffeffd640 1 -- 172.21.15.64:0/337946861 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5ff8005ce0 con 0x7f600815df00 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f60080c7930 con 0x7f600815df00 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f5ffeffd640 1 -- 172.21.15.64:0/337946861 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ff8007270 con 0x7f600815df00 2024-03-04T07:35:26.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.684+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f60080c7ef0 con 0x7f600815df00 2024-03-04T07:35:26.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.685+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6008067ff0 con 0x7f600815df00 2024-03-04T07:35:26.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.686+0000 7f5ffeffd640 1 -- 172.21.15.64:0/337946861 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f5ff801a070 con 0x7f600815df00 2024-03-04T07:35:26.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.686+0000 7f5ffeffd640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fec06b4d0 0x7f5fec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:26.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.686+0000 7f5ffeffd640 1 -- 172.21.15.64:0/337946861 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f5ff8089380 con 0x7f600815df00 2024-03-04T07:35:26.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.686+0000 7f600e265640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fec06b4d0 0x7f5fec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:26.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.687+0000 7f600e265640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fec06b4d0 0x7f5fec06d990 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f6004036000 tx=0x7f6004008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:26.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.691+0000 7f5ffeffd640 1 -- 172.21.15.64:0/337946861 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5ff8057c00 con 0x7f600815df00 2024-03-04T07:35:26.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.909+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f60080b3390 con 0x7f600815df00 2024-03-04T07:35:26.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.909+0000 7f5ffeffd640 1 -- 172.21.15.64:0/337946861 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f5ff805bca0 con 0x7f600815df00 2024-03-04T07:35:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:26.910 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:26.911 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:26.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fec06b4d0 msgr2=0x7f5fec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:26.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fec06b4d0 0x7f5fec06d990 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f6004036000 tx=0x7f6004008030 comp rx=0 tx=0).stop 2024-03-04T07:35:26.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f600815df00 msgr2=0x7f60080cc6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:26.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f600815df00 0x7f60080cc6a0 secure :-1 s=READY pgs=1687 cs=0 l=1 rev1=1 crypto rx=0x7f5ff800c4a0 tx=0x7f5ff800c960 comp rx=0 tx=0).stop 2024-03-04T07:35:26.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 shutdown_connections 2024-03-04T07:35:26.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f5fec06b4d0 0x7f5fec06d990 unknown :-1 s=CLOSED pgs=760 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:26.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f60080ccbe0 0x7f6008157dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:26.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f600815df00 0x7f60080cc6a0 unknown :-1 s=CLOSED pgs=1687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:26.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 --2- 172.21.15.64:0/337946861 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f60080b95a0 0x7f60080b5110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:26.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 >> 172.21.15.64:0/337946861 conn(0x7f6008071960 msgr2=0x7f60081618b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:26.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 shutdown_connections 2024-03-04T07:35:26.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:26.911+0000 7f60104c9640 1 -- 172.21.15.64:0/337946861 wait complete. 2024-03-04T07:35:27.124 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:27.124 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:28.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.238+0000 7f4616dbc640 1 Processor -- start 2024-03-04T07:35:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.238+0000 7f4616dbc640 1 -- start start 2024-03-04T07:35:28.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4616dbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4616dbc640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4610071a80 0x7f46100b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4616dbc640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f46100b6f20 con 0x7f461015df70 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4616dbc640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f46100d3030 con 0x7f4610071a80 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4616dbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46100d3310 con 0x7f46100d18c0 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4614b58640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f461015df70 0x7f46100d27b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:47988/0 (socket says 172.21.15.64:47988) 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4614b58640 1 -- 172.21.15.64:0/107148327 learned_addr learned my addr 172.21.15.64:0/107148327 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f460ffff640 1 --2- 172.21.15.64:0/107148327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f4615359640 1 --2- 172.21.15.64:0/107148327 >> v2:172.21.15.66:3300/0 conn(0x7f4610071a80 0x7f46100b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.239+0000 7f460f7fe640 1 -- 172.21.15.64:0/107148327 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3749687241 0 0) 0x7f46100b6f20 con 0x7f461015df70 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.240+0000 7f460f7fe640 1 -- 172.21.15.64:0/107148327 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f45fc003520 con 0x7f461015df70 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.240+0000 7f460ffff640 1 -- 172.21.15.64:0/107148327 >> v2:172.21.15.66:3300/0 conn(0x7f4610071a80 msgr2=0x7f46100b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.240+0000 7f460ffff640 1 --2- 172.21.15.64:0/107148327 >> v2:172.21.15.66:3300/0 conn(0x7f4610071a80 0x7f46100b2700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.240+0000 7f460ffff640 1 -- 172.21.15.64:0/107148327 >> v1:172.21.15.64:6789/0 conn(0x7f461015df70 legacy=0x7f46100d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.240+0000 7f460ffff640 1 -- 172.21.15.64:0/107148327 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46100d35f0 con 0x7f46100d18c0 2024-03-04T07:35:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.240+0000 7f4615359640 1 --2- 172.21.15.64:0/107148327 >> v2:172.21.15.66:3300/0 conn(0x7f4610071a80 0x7f46100b2700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:28.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.240+0000 7f460ffff640 1 --2- 172.21.15.64:0/107148327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610071540 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f45f80097a0 tx=0x7f45f802b960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=47047fbed4acb802 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f460f7fe640 1 -- 172.21.15.64:0/107148327 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45f8034070 con 0x7f46100d18c0 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f460f7fe640 1 -- 172.21.15.64:0/107148327 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f45f8030610 con 0x7f46100d18c0 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f460f7fe640 1 -- 172.21.15.64:0/107148327 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45f80309c0 con 0x7f46100d18c0 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 -- 172.21.15.64:0/107148327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 msgr2=0x7f4610071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 --2- 172.21.15.64:0/107148327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610071540 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f45f80097a0 tx=0x7f45f802b960 comp rx=0 tx=0).stop 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 -- 172.21.15.64:0/107148327 shutdown_connections 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 --2- 172.21.15.64:0/107148327 >> v2:172.21.15.66:3300/0 conn(0x7f4610071a80 0x7f46100b2700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 --2- 172.21.15.64:0/107148327 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610071540 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 -- 172.21.15.64:0/107148327 >> 172.21.15.64:0/107148327 conn(0x7f46100d3cf0 msgr2=0x7f46100d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:28.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 -- 172.21.15.64:0/107148327 shutdown_connections 2024-03-04T07:35:28.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.241+0000 7f4616dbc640 1 -- 172.21.15.64:0/107148327 wait complete. 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.242+0000 7f4616dbc640 1 Processor -- start 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.242+0000 7f4616dbc640 1 -- start start 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4616dbc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4610071a80 0x7f4610154d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4616dbc640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610152f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4616dbc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f461015df70 0x7f46101555d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4616dbc640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f461015e4d0 con 0x7f4610071a80 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4616dbc640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f461015e350 con 0x7f461015df70 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4616dbc640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f46100b2c40 con 0x7f46100d18c0 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4614b58640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4610071a80 0x7f4610154d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4614b58640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4610071a80 0x7f4610154d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43934/0 (socket says 172.21.15.64:43934) 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4614b58640 1 -- 172.21.15.64:0/2936613385 learned_addr learned my addr 172.21.15.64:0/2936613385 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f460ffff640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610152f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.243+0000 7f4615359640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f461015df70 0x7f46101555d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:28.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4614b58640 1 -- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 msgr2=0x7f4610152f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4614b58640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610152f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4614b58640 1 -- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f461015df70 msgr2=0x7f46101555d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4614b58640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f461015df70 0x7f46101555d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4614b58640 1 -- 172.21.15.64:0/2936613385 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4610153cf0 con 0x7f4610071a80 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4614b58640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4610071a80 0x7f4610154d70 secure :-1 s=READY pgs=1689 cs=0 l=1 rev1=1 crypto rx=0x7f460800cad0 tx=0x7f460800ced0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f460d7fa640 1 -- 172.21.15.64:0/2936613385 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4608017070 con 0x7f4610071a80 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f460d7fa640 1 -- 172.21.15.64:0/2936613385 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f460800ab70 con 0x7f4610071a80 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4610153fb0 con 0x7f4610071a80 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f460d7fa640 1 -- 172.21.15.64:0/2936613385 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4608012400 con 0x7f4610071a80 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.244+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4610154570 con 0x7f4610071a80 2024-03-04T07:35:28.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.245+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f461004f3d0 con 0x7f4610071a80 2024-03-04T07:35:28.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.246+0000 7f460d7fa640 1 -- 172.21.15.64:0/2936613385 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f460801a070 con 0x7f4610071a80 2024-03-04T07:35:28.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.246+0000 7f460d7fa640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f45f006b4d0 0x7f45f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:28.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.246+0000 7f460ffff640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f45f006b4d0 0x7f45f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:28.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.246+0000 7f460d7fa640 1 -- 172.21.15.64:0/2936613385 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4608089ea0 con 0x7f4610071a80 2024-03-04T07:35:28.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.247+0000 7f460ffff640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f45f006b4d0 0x7f45f006d990 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f45f8036000 tx=0x7f45f803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:28.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.251+0000 7f460d7fa640 1 -- 172.21.15.64:0/2936613385 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f461004f3d0 con 0x7f4610071a80 2024-03-04T07:35:28.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.470+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4610153490 con 0x7f4610071a80 2024-03-04T07:35:28.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.470+0000 7f460d7fa640 1 -- 172.21.15.64:0/2936613385 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4608058690 con 0x7f4610071a80 2024-03-04T07:35:28.472 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:28.472 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:28.472 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f45f006b4d0 msgr2=0x7f45f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f45f006b4d0 0x7f45f006d990 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f45f8036000 tx=0x7f45f803b040 comp rx=0 tx=0).stop 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4610071a80 msgr2=0x7f4610154d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4610071a80 0x7f4610154d70 secure :-1 s=READY pgs=1689 cs=0 l=1 rev1=1 crypto rx=0x7f460800cad0 tx=0x7f460800ced0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 shutdown_connections 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f45f006b4d0 0x7f45f006d990 unknown :-1 s=CLOSED pgs=761 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f461015df70 0x7f46101555d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f46100d18c0 0x7f4610152f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 --2- 172.21.15.64:0/2936613385 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4610071a80 0x7f4610154d70 unknown :-1 s=CLOSED pgs=1689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 >> 172.21.15.64:0/2936613385 conn(0x7f46100d3cf0 msgr2=0x7f46100b7ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 shutdown_connections 2024-03-04T07:35:28.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:28.472+0000 7f4616dbc640 1 -- 172.21.15.64:0/2936613385 wait complete. 2024-03-04T07:35:28.599 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:28.685 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:28.685 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:29.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:29.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.799+0000 7fc311f1e640 1 Processor -- start 2024-03-04T07:35:29.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.799+0000 7fc311f1e640 1 -- start start 2024-03-04T07:35:29.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.799+0000 7fc311f1e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc30c06df30 0x7fc30c0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:29.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.799+0000 7fc311f1e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc30c0aec40 0x7fc30c0b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:29.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc311f1e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc30c071240 con 0x7fc30c158520 2024-03-04T07:35:29.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc311f1e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc30c0af0f0 con 0x7fc30c06df30 2024-03-04T07:35:29.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc311f1e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc30c071b70 con 0x7fc30c0aec40 2024-03-04T07:35:29.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc30c06df30 0x7fc30c0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:29.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30affd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fc30c06df30 0x7fc30c0ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43946/0 (socket says 172.21.15.64:43946) 2024-03-04T07:35:29.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30affd640 1 -- 172.21.15.64:0/3371390360 learned_addr learned my addr 172.21.15.64:0/3371390360 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:29.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30bfff640 1 --2- 172.21.15.64:0/3371390360 >> v2:172.21.15.66:3300/0 conn(0x7fc30c0aec40 0x7fc30c0b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:29.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30affd640 1 -- 172.21.15.64:0/3371390360 >> v2:172.21.15.66:3300/0 conn(0x7fc30c0aec40 msgr2=0x7fc30c0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:29.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30affd640 1 --2- 172.21.15.64:0/3371390360 >> v2:172.21.15.66:3300/0 conn(0x7fc30c0aec40 0x7fc30c0b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:29.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30affd640 1 -- 172.21.15.64:0/3371390360 >> v1:172.21.15.66:6789/0 conn(0x7fc30c158520 legacy=0x7fc30c06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:29.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.800+0000 7fc30affd640 1 -- 172.21.15.64:0/3371390360 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc30c071520 con 0x7fc30c06df30 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.801+0000 7fc30affd640 1 --2- 172.21.15.64:0/3371390360 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c06df30 0x7fc30c0ae700 secure :-1 s=READY pgs=1690 cs=0 l=1 rev1=1 crypto rx=0x7fc300009870 tx=0x7fc30002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d3bac90048dcc551 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.801+0000 7fc30a7fc640 1 -- 172.21.15.64:0/3371390360 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc300034070 con 0x7fc30c06df30 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.801+0000 7fc30a7fc640 1 -- 172.21.15.64:0/3371390360 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc300030860 con 0x7fc30c06df30 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.801+0000 7fc30a7fc640 1 -- 172.21.15.64:0/3371390360 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc300030c30 con 0x7fc30c06df30 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.801+0000 7fc311f1e640 1 -- 172.21.15.64:0/3371390360 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c06df30 msgr2=0x7fc30c0ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.801+0000 7fc311f1e640 1 --2- 172.21.15.64:0/3371390360 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c06df30 0x7fc30c0ae700 secure :-1 s=READY pgs=1690 cs=0 l=1 rev1=1 crypto rx=0x7fc300009870 tx=0x7fc30002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.802+0000 7fc311f1e640 1 -- 172.21.15.64:0/3371390360 shutdown_connections 2024-03-04T07:35:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.802+0000 7fc311f1e640 1 --2- 172.21.15.64:0/3371390360 >> v2:172.21.15.66:3300/0 conn(0x7fc30c0aec40 0x7fc30c0b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:29.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.802+0000 7fc311f1e640 1 --2- 172.21.15.64:0/3371390360 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c06df30 0x7fc30c0ae700 unknown :-1 s=CLOSED pgs=1690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:29.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.802+0000 7fc311f1e640 1 -- 172.21.15.64:0/3371390360 >> 172.21.15.64:0/3371390360 conn(0x7fc30c066d00 msgr2=0x7fc30c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:29.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.802+0000 7fc311f1e640 1 -- 172.21.15.64:0/3371390360 shutdown_connections 2024-03-04T07:35:29.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.802+0000 7fc311f1e640 1 -- 172.21.15.64:0/3371390360 wait complete. 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.803+0000 7fc311f1e640 1 Processor -- start 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.803+0000 7fc311f1e640 1 -- start start 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc311f1e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc30c06df30 0x7fc30c140e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc311f1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c0aec40 0x7fc30c151b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc311f1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc30c158520 0x7fc30c14f7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc311f1e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc30c158900 con 0x7fc30c0aec40 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc311f1e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc30c072000 con 0x7fc30c06df30 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc311f1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc30c158a80 con 0x7fc30c158520 2024-03-04T07:35:29.806 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c0aec40 0x7fc30c151b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c0aec40 0x7fc30c151b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43950/0 (socket says 172.21.15.64:43950) 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 -- 172.21.15.64:0/264879034 learned_addr learned my addr 172.21.15.64:0/264879034 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30b7fe640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc30c06df30 0x7fc30c140e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30bfff640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc30c158520 0x7fc30c14f7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 -- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc30c158520 msgr2=0x7fc30c14f7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc30c158520 0x7fc30c14f7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 -- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc30c06df30 msgr2=0x7fc30c140e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc30c06df30 0x7fc30c140e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 -- 172.21.15.64:0/264879034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc30c0af3f0 con 0x7fc30c0aec40 2024-03-04T07:35:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.804+0000 7fc30affd640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c0aec40 0x7fc30c151b90 secure :-1 s=READY pgs=1691 cs=0 l=1 rev1=1 crypto rx=0x7fc300000c00 tx=0x7fc300004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.805+0000 7fc2f3fff640 1 -- 172.21.15.64:0/264879034 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc300048070 con 0x7fc30c0aec40 2024-03-04T07:35:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.805+0000 7fc2f3fff640 1 -- 172.21.15.64:0/264879034 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc300030660 con 0x7fc30c0aec40 2024-03-04T07:35:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.805+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc30c0af6d0 con 0x7fc30c0aec40 2024-03-04T07:35:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.805+0000 7fc2f3fff640 1 -- 172.21.15.64:0/264879034 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc300034040 con 0x7fc30c0aec40 2024-03-04T07:35:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.805+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc30c0afc90 con 0x7fc30c0aec40 2024-03-04T07:35:29.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.806+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc30c067ff0 con 0x7fc30c0aec40 2024-03-04T07:35:29.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.806+0000 7fc2f3fff640 1 -- 172.21.15.64:0/264879034 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc30004b070 con 0x7fc30c0aec40 2024-03-04T07:35:29.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.806+0000 7fc2f3fff640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc2f806b4d0 0x7fc2f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:29.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.807+0000 7fc2f3fff640 1 -- 172.21.15.64:0/264879034 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc3000ac040 con 0x7fc30c0aec40 2024-03-04T07:35:29.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.807+0000 7fc30b7fe640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc2f806b4d0 0x7fc2f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:29.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.807+0000 7fc30b7fe640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc2f806b4d0 0x7fc2f806d990 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7fc2fc0016e0 tx=0x7fc2fc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:29.812 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:29.811+0000 7fc2f3fff640 1 -- 172.21.15.64:0/264879034 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc30007a950 con 0x7fc30c0aec40 2024-03-04T07:35:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.032+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc30c141370 con 0x7fc30c0aec40 2024-03-04T07:35:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.033+0000 7fc2f3fff640 1 -- 172.21.15.64:0/264879034 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fc30007e9f0 con 0x7fc30c0aec40 2024-03-04T07:35:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:30.034 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:30.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc2f806b4d0 msgr2=0x7fc2f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc2f806b4d0 0x7fc2f806d990 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7fc2fc0016e0 tx=0x7fc2fc009040 comp rx=0 tx=0).stop 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c0aec40 msgr2=0x7fc30c151b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c0aec40 0x7fc30c151b90 secure :-1 s=READY pgs=1691 cs=0 l=1 rev1=1 crypto rx=0x7fc300000c00 tx=0x7fc300004730 comp rx=0 tx=0).stop 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 shutdown_connections 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc2f806b4d0 0x7fc2f806d990 unknown :-1 s=CLOSED pgs=762 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc30c158520 0x7fc30c14f7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc30c0aec40 0x7fc30c151b90 unknown :-1 s=CLOSED pgs=1691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:30.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 --2- 172.21.15.64:0/264879034 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc30c06df30 0x7fc30c140e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 >> 172.21.15.64:0/264879034 conn(0x7fc30c066d00 msgr2=0x7fc30c0b3640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 shutdown_connections 2024-03-04T07:35:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:30.035+0000 7fc311f1e640 1 -- 172.21.15.64:0/264879034 wait complete. 2024-03-04T07:35:30.248 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:30.248 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:30.600 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:31.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:31.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.362+0000 7fe49bb3e640 1 Processor -- start 2024-03-04T07:35:31.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.362+0000 7fe49bb3e640 1 -- start start 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe49bb3e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe49415e060 0x7fe4940ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe49bb3e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe49406fe20 0x7fe4940ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe49bb3e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe4940d0dd0 con 0x7fe4940d0880 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe49bb3e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe4940d10b0 con 0x7fe49415e060 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe49bb3e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4940cc0e0 con 0x7fe49406fe20 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe4990d9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe49406fe20 0x7fe4940ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe4998da640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe49415e060 0x7fe4940ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe4990d9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe49406fe20 0x7fe4940ce850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:56198/0 (socket says 172.21.15.64:56198) 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe4990d9640 1 -- 172.21.15.64:0/3946017530 learned_addr learned my addr 172.21.15.64:0/3946017530 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.363+0000 7fe49a0db640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fe4940d0880 0x7fe49406ed30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38870/0 (socket says 172.21.15.64:38870) 2024-03-04T07:35:31.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.364+0000 7fe4998da640 1 -- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe49406fe20 msgr2=0x7fe4940ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:31.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.364+0000 7fe4998da640 1 --2- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe49406fe20 0x7fe4940ce850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.364+0000 7fe4998da640 1 -- 172.21.15.64:0/3946017530 >> v1:172.21.15.66:6789/0 conn(0x7fe4940d0880 legacy=0x7fe49406ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:31.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.364+0000 7fe4998da640 1 -- 172.21.15.64:0/3946017530 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4940cc3c0 con 0x7fe49415e060 2024-03-04T07:35:31.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.364+0000 7fe4988d8640 1 -- 172.21.15.64:0/3946017530 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1414590600 0 0) 0x7fe4940d0dd0 con 0x7fe4940d0880 2024-03-04T07:35:31.366 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.364+0000 7fe4990d9640 1 --2- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe49406fe20 0x7fe4940ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.364+0000 7fe4998da640 1 --2- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce310 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7fe484009d90 tx=0x7fe48402bea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2db65f9f89a6f4cb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe4988d8640 1 -- 172.21.15.64:0/3946017530 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe484039070 con 0x7fe49415e060 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe4988d8640 1 -- 172.21.15.64:0/3946017530 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe48402cc50 con 0x7fe49415e060 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe4988d8640 1 -- 172.21.15.64:0/3946017530 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe484034440 con 0x7fe49415e060 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe49bb3e640 1 -- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 msgr2=0x7fe4940ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce310 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7fe484009d90 tx=0x7fe48402bea0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe49bb3e640 1 -- 172.21.15.64:0/3946017530 shutdown_connections 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe49406fe20 0x7fe4940ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/3946017530 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce310 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe49bb3e640 1 -- 172.21.15.64:0/3946017530 >> 172.21.15.64:0/3946017530 conn(0x7fe4940d1a90 msgr2=0x7fe4940d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:31.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.365+0000 7fe49bb3e640 1 -- 172.21.15.64:0/3946017530 shutdown_connections 2024-03-04T07:35:31.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.366+0000 7fe49bb3e640 1 -- 172.21.15.64:0/3946017530 wait complete. 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 Processor -- start 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 -- start start 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe49406fe20 0x7fe4941f3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4940d0880 0x7fe4940b2de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4940cdbd0 con 0x7fe49406fe20 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe49415e440 con 0x7fe49415e060 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.367+0000 7fe49bb3e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe4940cdd50 con 0x7fe4940d0880 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4998da640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe49406fe20 0x7fe4941f3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4940d0880 0x7fe4940b2de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:31.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe49a0db640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4998da640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe49406fe20 0x7fe4941f3680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43956/0 (socket says 172.21.15.64:43956) 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4998da640 1 -- 172.21.15.64:0/1373938292 learned_addr learned my addr 172.21.15.64:0/1373938292 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4940d0880 0x7fe4940b2de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:56206/0 (socket says 172.21.15.64:56206) 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 -- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 msgr2=0x7fe4940ce530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 -- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe49406fe20 msgr2=0x7fe4941f3680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe49406fe20 0x7fe4941f3680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 -- 172.21.15.64:0/1373938292 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4940b3320 con 0x7fe4940d0880 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe49a0db640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4990d9640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4940d0880 0x7fe4940b2de0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fe494070270 tx=0x7fe48800ecf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4927fc640 1 -- 172.21.15.64:0/1373938292 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe488004030 con 0x7fe4940d0880 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4927fc640 1 -- 172.21.15.64:0/1373938292 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe4880041d0 con 0x7fe4940d0880 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4940b1d40 con 0x7fe4940d0880 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe4927fc640 1 -- 172.21.15.64:0/1373938292 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe48800c940 con 0x7fe4940d0880 2024-03-04T07:35:31.371 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.368+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4940b2300 con 0x7fe4940d0880 2024-03-04T07:35:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.370+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe494067ff0 con 0x7fe4940d0880 2024-03-04T07:35:31.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.370+0000 7fe4927fc640 1 -- 172.21.15.64:0/1373938292 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe48800cae0 con 0x7fe4940d0880 2024-03-04T07:35:31.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.370+0000 7fe4927fc640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe48006b500 0x7fe48006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:31.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.370+0000 7fe4927fc640 1 -- 172.21.15.64:0/1373938292 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fe48808a450 con 0x7fe4940d0880 2024-03-04T07:35:31.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.370+0000 7fe4998da640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe48006b500 0x7fe48006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:31.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.371+0000 7fe4998da640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe48006b500 0x7fe48006d9c0 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7fe484000f80 tx=0x7fe484030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:31.375 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.375+0000 7fe4927fc640 1 -- 172.21.15.64:0/1373938292 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe488017020 con 0x7fe4940d0880 2024-03-04T07:35:31.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.593+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe4940d1750 con 0x7fe4940d0880 2024-03-04T07:35:31.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.594+0000 7fe4927fc640 1 -- 172.21.15.64:0/1373938292 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fe488058ba0 con 0x7fe4940d0880 2024-03-04T07:35:31.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:31.595 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:31.596 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:31.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe48006b500 msgr2=0x7fe48006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe48006b500 0x7fe48006d9c0 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7fe484000f80 tx=0x7fe484030000 comp rx=0 tx=0).stop 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4940d0880 msgr2=0x7fe4940b2de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4940d0880 0x7fe4940b2de0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fe494070270 tx=0x7fe48800ecf0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 shutdown_connections 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe48006b500 0x7fe48006d9c0 unknown :-1 s=CLOSED pgs=763 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe49415e060 0x7fe4940ce530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe4940d0880 0x7fe4940b2de0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 --2- 172.21.15.64:0/1373938292 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe49406fe20 0x7fe4941f3680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 >> 172.21.15.64:0/1373938292 conn(0x7fe4940d1a90 msgr2=0x7fe4940ccec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 shutdown_connections 2024-03-04T07:35:31.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:31.596+0000 7fe49bb3e640 1 -- 172.21.15.64:0/1373938292 wait complete. 2024-03-04T07:35:31.810 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:31.810 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:32.603 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:32.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:32.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.924+0000 7feef2681640 1 Processor -- start 2024-03-04T07:35:32.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.924+0000 7feef2681640 1 -- start start 2024-03-04T07:35:32.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feef2681640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feeec15bea0 0x7feeec0cd330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:32.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feef2681640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec0cf660 0x7feeec0cfb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:32.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feef2681640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7feeec0cf270 con 0x7feeec0d2580 2024-03-04T07:35:32.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feef2681640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7feeec15c280 con 0x7feeec15bea0 2024-03-04T07:35:32.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feef2681640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feeec0cc850 con 0x7feeec0cf660 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feeebfff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7feeec15bea0 0x7feeec0cd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feeeb7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec0cf660 0x7feeec0cfb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feef0c1e640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7feeec0d2580 0x7feeec06eb00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38882/0 (socket says 172.21.15.64:38882) 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feef0c1e640 1 -- 172.21.15.64:0/1787807003 learned_addr learned my addr 172.21.15.64:0/1787807003 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feeeb7fe640 1 -- 172.21.15.64:0/1787807003 >> v2:172.21.15.66:3300/0 conn(0x7feeec15bea0 msgr2=0x7feeec0cd330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feeeb7fe640 1 --2- 172.21.15.64:0/1787807003 >> v2:172.21.15.66:3300/0 conn(0x7feeec15bea0 0x7feeec0cd330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feeeb7fe640 1 -- 172.21.15.64:0/1787807003 >> v1:172.21.15.66:6789/0 conn(0x7feeec0d2580 legacy=0x7feeec06eb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feeeb7fe640 1 -- 172.21.15.64:0/1787807003 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feeec0ccb30 con 0x7feeec0cf660 2024-03-04T07:35:32.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.925+0000 7feeebfff640 1 --2- 172.21.15.64:0/1787807003 >> v2:172.21.15.66:3300/0 conn(0x7feeec15bea0 0x7feeec0cd330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.926+0000 7feeeb7fe640 1 --2- 172.21.15.64:0/1787807003 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec0cf660 0x7feeec0cfb10 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7feee00097d0 tx=0x7feee002b810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eb0fc67e8525df51 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.926+0000 7feeeaffd640 1 -- 172.21.15.64:0/1787807003 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feee0031770 con 0x7feeec0cf660 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.926+0000 7feeeaffd640 1 -- 172.21.15.64:0/1787807003 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feee0037070 con 0x7feeec0cf660 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.926+0000 7feeeaffd640 1 -- 172.21.15.64:0/1787807003 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feee0031b30 con 0x7feeec0cf660 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 -- 172.21.15.64:0/1787807003 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec0cf660 msgr2=0x7feeec0cfb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 --2- 172.21.15.64:0/1787807003 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec0cf660 0x7feeec0cfb10 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7feee00097d0 tx=0x7feee002b810 comp rx=0 tx=0).stop 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 -- 172.21.15.64:0/1787807003 shutdown_connections 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 --2- 172.21.15.64:0/1787807003 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec0cf660 0x7feeec0cfb10 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 --2- 172.21.15.64:0/1787807003 >> v2:172.21.15.66:3300/0 conn(0x7feeec15bea0 0x7feeec0cd330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 -- 172.21.15.64:0/1787807003 >> 172.21.15.64:0/1787807003 conn(0x7feeec0d1a90 msgr2=0x7feeec0d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 -- 172.21.15.64:0/1787807003 shutdown_connections 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.927+0000 7feef2681640 1 -- 172.21.15.64:0/1787807003 wait complete. 2024-03-04T07:35:32.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.928+0000 7feef2681640 1 Processor -- start 2024-03-04T07:35:32.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.928+0000 7feef2681640 1 -- start start 2024-03-04T07:35:32.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef2681640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeec0cf660 0x7feeec06c6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:32.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef2681640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeec0d2580 0x7feeec06cbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef2681640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec15bea0 0x7feeec0ca150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef2681640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feeec0cc500 con 0x7feeec0d2580 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef2681640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feeec0cd870 con 0x7feeec0cf660 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef2681640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feeec0ceeb0 con 0x7feeec15bea0 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeeb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeec0d2580 0x7feeec06cbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeebfff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeec0cf660 0x7feeec06c6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef0c1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec15bea0 0x7feeec0ca150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeebfff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeec0cf660 0x7feeec06c6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:40396/0 (socket says 172.21.15.64:40396) 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeebfff640 1 -- 172.21.15.64:0/893649590 learned_addr learned my addr 172.21.15.64:0/893649590 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeeb7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeec0d2580 0x7feeec06cbf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:43958/0 (socket says 172.21.15.64:43958) 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeeb7fe640 1 -- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec15bea0 msgr2=0x7feeec0ca150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeeb7fe640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec15bea0 0x7feeec0ca150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeeb7fe640 1 -- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeec0cf660 msgr2=0x7feeec06c6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:32.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeeb7fe640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeec0cf660 0x7feeec06c6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeeb7fe640 1 -- 172.21.15.64:0/893649590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feeec1f3bd0 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feef0c1e640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec15bea0 0x7feeec0ca150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.929+0000 7feeebfff640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeec0cf660 0x7feeec06c6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.930+0000 7feeeb7fe640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeec0d2580 0x7feeec06cbf0 secure :-1 s=READY pgs=1692 cs=0 l=1 rev1=1 crypto rx=0x7feee0033000 tx=0x7feee00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.930+0000 7feee8ff9640 1 -- 172.21.15.64:0/893649590 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feee0045600 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.930+0000 7feee8ff9640 1 -- 172.21.15.64:0/893649590 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7feee00457c0 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.930+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feeec1f3e40 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.930+0000 7feee8ff9640 1 -- 172.21.15.64:0/893649590 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feee0045bb0 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.930+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feeec1f46e0 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.931+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feeec1f4010 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.931+0000 7feee8ff9640 1 -- 172.21.15.64:0/893649590 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7feee0031070 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.932+0000 7feee8ff9640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feed806b4d0 0x7feed806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.932+0000 7feee8ff9640 1 -- 172.21.15.64:0/893649590 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7feee00abd50 con 0x7feeec0d2580 2024-03-04T07:35:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.932+0000 7feeebfff640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feed806b4d0 0x7feed806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:32.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.932+0000 7feeebfff640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feed806b4d0 0x7feed806d990 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7feedc009560 tx=0x7feedc009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:32.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:32.936+0000 7feee8ff9640 1 -- 172.21.15.64:0/893649590 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7feee00af030 con 0x7feeec0d2580 2024-03-04T07:35:33.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.154+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7feeec067ff0 con 0x7feeec0d2580 2024-03-04T07:35:33.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.154+0000 7feee8ff9640 1 -- 172.21.15.64:0/893649590 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7feee0040070 con 0x7feeec0d2580 2024-03-04T07:35:33.156 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:33.156 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:33.156 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:33.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feed806b4d0 msgr2=0x7feed806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feed806b4d0 0x7feed806d990 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7feedc009560 tx=0x7feedc009190 comp rx=0 tx=0).stop 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeec0d2580 msgr2=0x7feeec06cbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeec0d2580 0x7feeec06cbf0 secure :-1 s=READY pgs=1692 cs=0 l=1 rev1=1 crypto rx=0x7feee0033000 tx=0x7feee00046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 shutdown_connections 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7feed806b4d0 0x7feed806d990 unknown :-1 s=CLOSED pgs=764 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7feeec15bea0 0x7feeec0ca150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7feeec0d2580 0x7feeec06cbf0 unknown :-1 s=CLOSED pgs=1692 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 --2- 172.21.15.64:0/893649590 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7feeec0cf660 0x7feeec06c6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 >> 172.21.15.64:0/893649590 conn(0x7feeec0d1a90 msgr2=0x7feeec0b1560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 shutdown_connections 2024-03-04T07:35:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:33.156+0000 7feef2681640 1 -- 172.21.15.64:0/893649590 wait complete. 2024-03-04T07:35:33.369 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:33.370 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:34.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:34.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.483+0000 7f166cbb5640 1 Processor -- start 2024-03-04T07:35:34.485 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.483+0000 7f166cbb5640 1 -- start start 2024-03-04T07:35:34.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.484+0000 7f166cbb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680b0e90 0x7f166806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:34.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.484+0000 7f166cbb5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f16680ae9b0 0x7f16680d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:34.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.484+0000 7f166cbb5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f16680cee90 con 0x7f166806b5c0 2024-03-04T07:35:34.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.484+0000 7f166cbb5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f16680cfa00 con 0x7f16680ae9b0 2024-03-04T07:35:34.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.484+0000 7f166cbb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16680cfc50 con 0x7f16680b0e90 2024-03-04T07:35:34.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.484+0000 7f1665d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680b0e90 0x7f166806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f1666d9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f16680ae9b0 0x7f16680d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f166659c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f166806b5c0 0x7f16680b2e00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:60338/0 (socket says 172.21.15.64:60338) 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f166659c640 1 -- 172.21.15.64:0/3010900084 learned_addr learned my addr 172.21.15.64:0/3010900084 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f166559a640 1 -- 172.21.15.64:0/3010900084 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 30934675 0 0) 0x7f16680cee90 con 0x7f166806b5c0 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f166559a640 1 -- 172.21.15.64:0/3010900084 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1654003520 con 0x7f166806b5c0 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f1666d9d640 1 -- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680b0e90 msgr2=0x7f166806b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f1666d9d640 1 --2- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680b0e90 0x7f166806b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f1666d9d640 1 -- 172.21.15.64:0/3010900084 >> v1:172.21.15.66:6789/0 conn(0x7f166806b5c0 legacy=0x7f16680b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:34.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f1666d9d640 1 -- 172.21.15.64:0/3010900084 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16680cff30 con 0x7f16680ae9b0 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.485+0000 7f1665d9b640 1 --2- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680b0e90 0x7f166806b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f1666d9d640 1 --2- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16680ae9b0 0x7f16680d03a0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f16600097a0 tx=0x7f166002b820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fd0b5c7e75a14dc3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166559a640 1 -- 172.21.15.64:0/3010900084 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1660031780 con 0x7f16680ae9b0 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166559a640 1 -- 172.21.15.64:0/3010900084 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1660037070 con 0x7f16680ae9b0 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166559a640 1 -- 172.21.15.64:0/3010900084 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1660031b40 con 0x7f16680ae9b0 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166cbb5640 1 -- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16680ae9b0 msgr2=0x7f16680d03a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16680ae9b0 0x7f16680d03a0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f16600097a0 tx=0x7f166002b820 comp rx=0 tx=0).stop 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166cbb5640 1 -- 172.21.15.64:0/3010900084 shutdown_connections 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f16680ae9b0 0x7f16680d03a0 unknown :-1 s=CLOSED pgs=610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.486+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3010900084 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680b0e90 0x7f166806b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.487+0000 7f166cbb5640 1 -- 172.21.15.64:0/3010900084 >> 172.21.15.64:0/3010900084 conn(0x7f16680b36d0 msgr2=0x7f16680b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.487+0000 7f166cbb5640 1 -- 172.21.15.64:0/3010900084 shutdown_connections 2024-03-04T07:35:34.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.487+0000 7f166cbb5640 1 -- 172.21.15.64:0/3010900084 wait complete. 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.488+0000 7f166cbb5640 1 Processor -- start 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.488+0000 7f166cbb5640 1 -- start start 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166cbb5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f166806b5c0 0x7f16680cc940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166cbb5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680ae9b0 0x7f16680c5630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166cbb5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16680b0e90 0x7f16680c5b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166cbb5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f166806c910 con 0x7f16680b0e90 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166cbb5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f166806c790 con 0x7f166806b5c0 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166cbb5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16680aee20 con 0x7f16680ae9b0 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1666d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16680b0e90 0x7f16680c5b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1665d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680ae9b0 0x7f16680c5630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166659c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f166806b5c0 0x7f16680cc940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1666d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16680b0e90 0x7f16680c5b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40124/0 (socket says 172.21.15.64:40124) 2024-03-04T07:35:34.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1666d9d640 1 -- 172.21.15.64:0/3867739687 learned_addr learned my addr 172.21.15.64:0/3867739687 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1665d9b640 1 -- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f166806b5c0 msgr2=0x7f16680cc940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1665d9b640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f166806b5c0 0x7f16680cc940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1665d9b640 1 -- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16680b0e90 msgr2=0x7f16680c5b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1665d9b640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16680b0e90 0x7f16680c5b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1665d9b640 1 -- 172.21.15.64:0/3867739687 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16680cd320 con 0x7f16680ae9b0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1666d9d640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16680b0e90 0x7f16680c5b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f166659c640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f166806b5c0 0x7f16680cc940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.489+0000 7f1665d9b640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680ae9b0 0x7f16680c5630 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f165000e930 tx=0x7f165000edf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.490+0000 7f165effd640 1 -- 172.21.15.64:0/3867739687 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1650017070 con 0x7f16680ae9b0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.490+0000 7f165effd640 1 -- 172.21.15.64:0/3867739687 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f165000ca00 con 0x7f16680ae9b0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.490+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16680cd600 con 0x7f16680ae9b0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.490+0000 7f165effd640 1 -- 172.21.15.64:0/3867739687 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f165000cdd0 con 0x7f16680ae9b0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.490+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16681d2e70 con 0x7f16680ae9b0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.491+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1668068700 con 0x7f16680ae9b0 2024-03-04T07:35:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.491+0000 7f165effd640 1 -- 172.21.15.64:0/3867739687 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f165001a070 con 0x7f16680ae9b0 2024-03-04T07:35:34.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.491+0000 7f165effd640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164806b4d0 0x7f164806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:34.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.492+0000 7f166659c640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164806b4d0 0x7f164806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:34.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.492+0000 7f165effd640 1 -- 172.21.15.64:0/3867739687 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f165008a320 con 0x7f16680ae9b0 2024-03-04T07:35:34.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.492+0000 7f166659c640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164806b4d0 0x7f164806d990 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7f1658002230 tx=0x7f1658009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:34.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.496+0000 7f165effd640 1 -- 172.21.15.64:0/3867739687 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1650058ba0 con 0x7f16680ae9b0 2024-03-04T07:35:34.605 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:34.719 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.719+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f16680afd70 con 0x7f16680ae9b0 2024-03-04T07:35:34.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.719+0000 7f165effd640 1 -- 172.21.15.64:0/3867739687 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f165005cc40 con 0x7f16680ae9b0 2024-03-04T07:35:34.720 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:34.720 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:34.721 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164806b4d0 msgr2=0x7f164806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164806b4d0 0x7f164806d990 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7f1658002230 tx=0x7f1658009040 comp rx=0 tx=0).stop 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680ae9b0 msgr2=0x7f16680c5630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680ae9b0 0x7f16680c5630 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f165000e930 tx=0x7f165000edf0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 shutdown_connections 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f164806b4d0 0x7f164806d990 unknown :-1 s=CLOSED pgs=765 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f16680b0e90 0x7f16680c5b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f16680ae9b0 0x7f16680c5630 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 --2- 172.21.15.64:0/3867739687 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f166806b5c0 0x7f16680cc940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 >> 172.21.15.64:0/3867739687 conn(0x7f16680b36d0 msgr2=0x7f16680b1350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 shutdown_connections 2024-03-04T07:35:34.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:34.721+0000 7f166cbb5640 1 -- 172.21.15.64:0/3867739687 wait complete. 2024-03-04T07:35:34.934 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:34.935 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:35.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:36.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.049+0000 7f724eea3640 1 Processor -- start 2024-03-04T07:35:36.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.049+0000 7f724eea3640 1 -- start start 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.049+0000 7f724eea3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f72480b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.049+0000 7f724eea3640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72480ce610 con 0x7f72480d3400 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.049+0000 7f724eea3640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f72480d2920 con 0x7f72480b2b50 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.049+0000 7f724eea3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72480d2f70 con 0x7f7248071540 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f7247fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f72480d3400 0x7f7248071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:49966/0 (socket says 172.21.15.64:49966) 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f724cc3f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f72480b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f7247fff640 1 -- 172.21.15.64:0/3511588127 learned_addr learned my addr 172.21.15.64:0/3511588127 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f724d440640 1 --1- 172.21.15.64:0/3511588127 >> v1:172.21.15.66:6789/0 conn(0x7f72480b2b50 0x7f72480b6e20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:60352/0 (socket says 172.21.15.64:60352) 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f72477fe640 1 -- 172.21.15.64:0/3511588127 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 54398492 0 0) 0x7f72480d2920 con 0x7f72480b2b50 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f724cc3f640 1 -- 172.21.15.64:0/3511588127 >> v1:172.21.15.66:6789/0 conn(0x7f72480b2b50 legacy=0x7f72480b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:36.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f724cc3f640 1 -- 172.21.15.64:0/3511588127 >> v1:172.21.15.64:6789/0 conn(0x7f72480d3400 legacy=0x7f7248071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:36.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.050+0000 7f724cc3f640 1 -- 172.21.15.64:0/3511588127 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72480d2c00 con 0x7f7248071540 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f724cc3f640 1 --2- 172.21.15.64:0/3511588127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f72480b30a0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f723c0097d0 tx=0x7f723c02b770 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3d753765bf3bf7d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f72477fe640 1 -- 172.21.15.64:0/3511588127 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723c037070 con 0x7f7248071540 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f72477fe640 1 -- 172.21.15.64:0/3511588127 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f723c030710 con 0x7f7248071540 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f72477fe640 1 -- 172.21.15.64:0/3511588127 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723c030ae0 con 0x7f7248071540 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f724eea3640 1 -- 172.21.15.64:0/3511588127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 msgr2=0x7f72480b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f724eea3640 1 --2- 172.21.15.64:0/3511588127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f72480b30a0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f723c0097d0 tx=0x7f723c02b770 comp rx=0 tx=0).stop 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f724eea3640 1 -- 172.21.15.64:0/3511588127 shutdown_connections 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f724eea3640 1 --2- 172.21.15.64:0/3511588127 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f72480b30a0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.051+0000 7f724eea3640 1 -- 172.21.15.64:0/3511588127 >> 172.21.15.64:0/3511588127 conn(0x7f72480d3ca0 msgr2=0x7f72480d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:36.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.052+0000 7f724eea3640 1 -- 172.21.15.64:0/3511588127 shutdown_connections 2024-03-04T07:35:36.054 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.052+0000 7f724eea3640 1 -- 172.21.15.64:0/3511588127 wait complete. 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.053+0000 7f724eea3640 1 Processor -- start 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.053+0000 7f724eea3640 1 -- start start 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724eea3640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f724815ab60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724eea3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72480b2b50 0x7f72481f72c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724eea3640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72480d3400 0x7f72481f7800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724eea3640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f724815e370 con 0x7f72480b2b50 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724eea3640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72480d4be0 con 0x7f72480d3400 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724eea3640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f72480cb8a0 con 0x7f7248071540 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f7247fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72480b2b50 0x7f72481f72c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f7247fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72480b2b50 0x7f72481f72c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40126/0 (socket says 172.21.15.64:40126) 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f7247fff640 1 -- 172.21.15.64:0/3831128924 learned_addr learned my addr 172.21.15.64:0/3831128924 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:36.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724cc3f640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f724815ab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724d440640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72480d3400 0x7f72481f7800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724cc3f640 1 -- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72480d3400 msgr2=0x7f72481f7800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724cc3f640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72480d3400 0x7f72481f7800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724cc3f640 1 -- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72480b2b50 msgr2=0x7f72481f72c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724cc3f640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72480b2b50 0x7f72481f72c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724cc3f640 1 -- 172.21.15.64:0/3831128924 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72480cbb40 con 0x7f7248071540 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f7247fff640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72480b2b50 0x7f72481f72c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724cc3f640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f724815ab60 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f723c032000 tx=0x7f723c0046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f72457fa640 1 -- 172.21.15.64:0/3831128924 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723c042070 con 0x7f7248071540 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f72457fa640 1 -- 172.21.15.64:0/3831128924 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f723c030e90 con 0x7f7248071540 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.054+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72480cbe20 con 0x7f7248071540 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.055+0000 7f72457fa640 1 -- 172.21.15.64:0/3831128924 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723c037040 con 0x7f7248071540 2024-03-04T07:35:36.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.055+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72480cc3e0 con 0x7f7248071540 2024-03-04T07:35:36.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.056+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7248068700 con 0x7f7248071540 2024-03-04T07:35:36.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.056+0000 7f72457fa640 1 -- 172.21.15.64:0/3831128924 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f723c045070 con 0x7f7248071540 2024-03-04T07:35:36.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.056+0000 7f72457fa640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f722c06b4d0 0x7f722c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:36.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.056+0000 7f72457fa640 1 -- 172.21.15.64:0/3831128924 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f723c0ac130 con 0x7f7248071540 2024-03-04T07:35:36.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.056+0000 7f7247fff640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f722c06b4d0 0x7f722c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:36.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.057+0000 7f7247fff640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f722c06b4d0 0x7f722c06d990 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7f724815a550 tx=0x7f7234007e90 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:36.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.061+0000 7f72457fa640 1 -- 172.21.15.64:0/3831128924 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f723c07a9b0 con 0x7f7248071540 2024-03-04T07:35:36.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.282+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f72480b8bc0 con 0x7f7248071540 2024-03-04T07:35:36.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.284+0000 7f72457fa640 1 -- 172.21.15.64:0/3831128924 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f723c07ea50 con 0x7f7248071540 2024-03-04T07:35:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:36.284 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:36.284 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f722c06b4d0 msgr2=0x7f722c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f722c06b4d0 0x7f722c06d990 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7f724815a550 tx=0x7f7234007e90 comp rx=0 tx=0).stop 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 msgr2=0x7f724815ab60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f724815ab60 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f723c032000 tx=0x7f723c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 shutdown_connections 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f722c06b4d0 0x7f722c06d990 unknown :-1 s=CLOSED pgs=766 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f72480d3400 0x7f72481f7800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f72480b2b50 0x7f72481f72c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 --2- 172.21.15.64:0/3831128924 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7248071540 0x7f724815ab60 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 >> 172.21.15.64:0/3831128924 conn(0x7f72480d3ca0 msgr2=0x7f724815fbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 shutdown_connections 2024-03-04T07:35:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:36.285+0000 7f724eea3640 1 -- 172.21.15.64:0/3831128924 wait complete. 2024-03-04T07:35:36.498 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:36.498 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:36.607 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:37.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.612+0000 7f7e3570f640 1 Processor -- start 2024-03-04T07:35:37.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.612+0000 7f7e3570f640 1 -- start start 2024-03-04T07:35:37.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e3570f640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7e3015df00 0x7f7e300d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e3570f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b9a30 0x7f7e300d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e3570f640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7e300cea30 con 0x7f7e300b95a0 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e3570f640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7e300d2af0 con 0x7f7e3015df00 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e3570f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e300cf3c0 con 0x7f7e300b9a30 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2e7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7e300b95a0 0x7f7e300b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:49970/0 (socket says 172.21.15.64:49970) 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2e7fc640 1 -- 172.21.15.64:0/854300017 learned_addr learned my addr 172.21.15.64:0/854300017 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2f7fe640 1 --2- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b9a30 0x7f7e300d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2effd640 1 --2- 172.21.15.64:0/854300017 >> v2:172.21.15.66:3300/0 conn(0x7f7e3015df00 0x7f7e300d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2dffb640 1 -- 172.21.15.64:0/854300017 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3677683230 0 0) 0x7f7e300cea30 con 0x7f7e300b95a0 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2dffb640 1 -- 172.21.15.64:0/854300017 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7e1c003520 con 0x7f7e300b95a0 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2effd640 1 -- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b9a30 msgr2=0x7f7e300d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2effd640 1 --2- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b9a30 0x7f7e300d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2effd640 1 -- 172.21.15.64:0/854300017 >> v1:172.21.15.64:6789/0 conn(0x7f7e300b95a0 legacy=0x7f7e300b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2effd640 1 -- 172.21.15.64:0/854300017 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e300cf0e0 con 0x7f7e3015df00 2024-03-04T07:35:37.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.613+0000 7f7e2f7fe640 1 --2- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b9a30 0x7f7e300d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.614+0000 7f7e2effd640 1 --2- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e3015df00 0x7f7e300d25b0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f7e280097a0 tx=0x7f7e2802b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1f578ec1dd5cebe1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.614+0000 7f7e2dffb640 1 -- 172.21.15.64:0/854300017 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e28035020 con 0x7f7e3015df00 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.614+0000 7f7e2dffb640 1 -- 172.21.15.64:0/854300017 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e28031830 con 0x7f7e3015df00 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.614+0000 7f7e2dffb640 1 -- 172.21.15.64:0/854300017 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e28031be0 con 0x7f7e3015df00 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 -- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e3015df00 msgr2=0x7f7e300d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 --2- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e3015df00 0x7f7e300d25b0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f7e280097a0 tx=0x7f7e2802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 -- 172.21.15.64:0/854300017 shutdown_connections 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 --2- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b9a30 0x7f7e300d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 --2- 172.21.15.64:0/854300017 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e3015df00 0x7f7e300d25b0 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 -- 172.21.15.64:0/854300017 >> 172.21.15.64:0/854300017 conn(0x7f7e30071960 msgr2=0x7f7e30071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:37.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 -- 172.21.15.64:0/854300017 shutdown_connections 2024-03-04T07:35:37.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.615+0000 7f7e3570f640 1 -- 172.21.15.64:0/854300017 wait complete. 2024-03-04T07:35:37.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.616+0000 7f7e3570f640 1 Processor -- start 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.616+0000 7f7e3570f640 1 -- start start 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e3570f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e300b95a0 0x7f7e3014e1e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e3570f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e3015df00 0x7f7e300b7e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e3570f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b83d0 0x7f7e301603b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e3570f640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e300cec70 con 0x7f7e3015df00 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e3570f640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e300ce750 con 0x7f7e300b95a0 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e3570f640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7e300cedf0 con 0x7f7e300b83d0 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e2e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e3015df00 0x7f7e300b7e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e2e7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e3015df00 0x7f7e300b7e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40128/0 (socket says 172.21.15.64:40128) 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e2e7fc640 1 -- 172.21.15.64:0/3953307040 learned_addr learned my addr 172.21.15.64:0/3953307040 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e2f7fe640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b83d0 0x7f7e301603b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.617+0000 7f7e2effd640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e300b95a0 0x7f7e3014e1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2e7fc640 1 -- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b83d0 msgr2=0x7f7e301603b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:37.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2e7fc640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b83d0 0x7f7e301603b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2e7fc640 1 -- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e300b95a0 msgr2=0x7f7e3014e1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2e7fc640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e300b95a0 0x7f7e3014e1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2e7fc640 1 -- 172.21.15.64:0/3953307040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e301610a0 con 0x7f7e3015df00 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2f7fe640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b83d0 0x7f7e301603b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2effd640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e300b95a0 0x7f7e3014e1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e2e7fc640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e3015df00 0x7f7e300b7e90 secure :-1 s=READY pgs=1695 cs=0 l=1 rev1=1 crypto rx=0x7f7e1800c390 tx=0x7f7e1800c790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e277fe640 1 -- 172.21.15.64:0/3953307040 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e18017070 con 0x7f7e3015df00 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e277fe640 1 -- 172.21.15.64:0/3953307040 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7e18005ce0 con 0x7f7e3015df00 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7e3014eb20 con 0x7f7e3015df00 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e277fe640 1 -- 172.21.15.64:0/3953307040 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e180072f0 con 0x7f7e3015df00 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.618+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e3014f0e0 con 0x7f7e3015df00 2024-03-04T07:35:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.619+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7e301614c0 con 0x7f7e3015df00 2024-03-04T07:35:37.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.619+0000 7f7e277fe640 1 -- 172.21.15.64:0/3953307040 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7e1801a070 con 0x7f7e3015df00 2024-03-04T07:35:37.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.620+0000 7f7e277fe640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e1006b4d0 0x7f7e1006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:37.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.620+0000 7f7e2effd640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e1006b4d0 0x7f7e1006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:37.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.620+0000 7f7e277fe640 1 -- 172.21.15.64:0/3953307040 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f7e18089450 con 0x7f7e3015df00 2024-03-04T07:35:37.621 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.621+0000 7f7e2effd640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e1006b4d0 0x7f7e1006d990 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f7e28036000 tx=0x7f7e28008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:37.624 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.625+0000 7f7e277fe640 1 -- 172.21.15.64:0/3953307040 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7e301614c0 con 0x7f7e3015df00 2024-03-04T07:35:37.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.842+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7e30067ff0 con 0x7f7e3015df00 2024-03-04T07:35:37.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.843+0000 7f7e277fe640 1 -- 172.21.15.64:0/3953307040 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f7e18057cd0 con 0x7f7e3015df00 2024-03-04T07:35:37.843 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:37.843 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:37.843 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.844+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e1006b4d0 msgr2=0x7f7e1006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.844+0000 7f7e3570f640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e1006b4d0 0x7f7e1006d990 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f7e28036000 tx=0x7f7e28008030 comp rx=0 tx=0).stop 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.844+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e3015df00 msgr2=0x7f7e300b7e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.844+0000 7f7e3570f640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e3015df00 0x7f7e300b7e90 secure :-1 s=READY pgs=1695 cs=0 l=1 rev1=1 crypto rx=0x7f7e1800c390 tx=0x7f7e1800c790 comp rx=0 tx=0).stop 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 shutdown_connections 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7e1006b4d0 0x7f7e1006d990 unknown :-1 s=CLOSED pgs=767 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7e300b83d0 0x7f7e301603b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7e3015df00 0x7f7e300b7e90 unknown :-1 s=CLOSED pgs=1695 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 --2- 172.21.15.64:0/3953307040 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7e300b95a0 0x7f7e3014e1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 >> 172.21.15.64:0/3953307040 conn(0x7f7e30071960 msgr2=0x7f7e300b3bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 shutdown_connections 2024-03-04T07:35:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:37.845+0000 7f7e3570f640 1 -- 172.21.15.64:0/3953307040 wait complete. 2024-03-04T07:35:38.057 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:38.057 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:38.609 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:39.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:39.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.172+0000 7fc5a89e9640 1 Processor -- start 2024-03-04T07:35:39.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.172+0000 7fc5a89e9640 1 -- start start 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a89e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40b1020 0x7fc5a406af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a89e9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc5a406ed10 0x7fc5a40cd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a89e9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc5a40cc0e0 con 0x7fc5a40cd6c0 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a89e9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc5a40d28b0 con 0x7fc5a406ed10 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a89e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5a40cf320 con 0x7fc5a40b1020 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a1d9b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc5a406ed10 0x7fc5a40cd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a259c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40b1020 0x7fc5a406af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a259c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40b1020 0x7fc5a406af20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:55786/0 (socket says 172.21.15.64:55786) 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a2d9d640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fc5a40cd6c0 0x7fc5a415d700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:60356/0 (socket says 172.21.15.64:60356) 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a1d9b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc5a406ed10 0x7fc5a40cd180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:50088/0 (socket says 172.21.15.64:50088) 2024-03-04T07:35:39.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.173+0000 7fc5a259c640 1 -- 172.21.15.64:0/3288226838 learned_addr learned my addr 172.21.15.64:0/3288226838 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:39.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.174+0000 7fc5a1d9b640 1 -- 172.21.15.64:0/3288226838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40b1020 msgr2=0x7fc5a406af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:39.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.174+0000 7fc5a1d9b640 1 --2- 172.21.15.64:0/3288226838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40b1020 0x7fc5a406af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.174+0000 7fc5a1d9b640 1 -- 172.21.15.64:0/3288226838 >> v1:172.21.15.66:6789/0 conn(0x7fc5a40cd6c0 legacy=0x7fc5a415d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:39.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.174+0000 7fc5a1d9b640 1 -- 172.21.15.64:0/3288226838 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5a40cc3c0 con 0x7fc5a406ed10 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.174+0000 7fc5a1d9b640 1 --2- 172.21.15.64:0/3288226838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a406ed10 0x7fc5a40cd180 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7fc5980097a0 tx=0x7fc59802b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7962ab4b142ea96f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a159a640 1 -- 172.21.15.64:0/3288226838 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc598035020 con 0x7fc5a406ed10 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a159a640 1 -- 172.21.15.64:0/3288226838 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc598031830 con 0x7fc5a406ed10 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a159a640 1 -- 172.21.15.64:0/3288226838 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc598031c00 con 0x7fc5a406ed10 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 -- 172.21.15.64:0/3288226838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a406ed10 msgr2=0x7fc5a40cd180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/3288226838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a406ed10 0x7fc5a40cd180 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7fc5980097a0 tx=0x7fc59802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 -- 172.21.15.64:0/3288226838 shutdown_connections 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/3288226838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a406ed10 0x7fc5a40cd180 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/3288226838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40b1020 0x7fc5a406af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 -- 172.21.15.64:0/3288226838 >> 172.21.15.64:0/3288226838 conn(0x7fc5a40d1a90 msgr2=0x7fc5a40d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:39.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 -- 172.21.15.64:0/3288226838 shutdown_connections 2024-03-04T07:35:39.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.175+0000 7fc5a89e9640 1 -- 172.21.15.64:0/3288226838 wait complete. 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 Processor -- start 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 -- start start 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5a406ed10 0x7fc5a4158e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a40b1020 0x7fc5a40b6cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40cd6c0 0x7fc5a40b7200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5a40ccc60 con 0x7fc5a406ed10 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5a40cc980 con 0x7fc5a40b1020 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a89e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc5a40cdaf0 con 0x7fc5a40cd6c0 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.177+0000 7fc5a259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5a406ed10 0x7fc5a4158e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a1d9b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a40b1020 0x7fc5a40b6cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a2d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40cd6c0 0x7fc5a40b7200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a259c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5a406ed10 0x7fc5a4158e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40132/0 (socket says 172.21.15.64:40132) 2024-03-04T07:35:39.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a259c640 1 -- 172.21.15.64:0/92911952 learned_addr learned my addr 172.21.15.64:0/92911952 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a1d9b640 1 -- 172.21.15.64:0/92911952 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40cd6c0 msgr2=0x7fc5a40b7200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a1d9b640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40cd6c0 0x7fc5a40b7200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a1d9b640 1 -- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5a406ed10 msgr2=0x7fc5a4158e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a1d9b640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5a406ed10 0x7fc5a4158e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a1d9b640 1 -- 172.21.15.64:0/92911952 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5a40b5840 con 0x7fc5a40b1020 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a259c640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5a406ed10 0x7fc5a4158e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a1d9b640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a40b1020 0x7fc5a40b6cc0 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7fc598033000 tx=0x7fc5980046c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc592ffd640 1 -- 172.21.15.64:0/92911952 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc598035040 con 0x7fc5a40b1020 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc592ffd640 1 -- 172.21.15.64:0/92911952 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc59803f070 con 0x7fc5a40b1020 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5a40cb9b0 con 0x7fc5a40b1020 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc592ffd640 1 -- 172.21.15.64:0/92911952 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc59803bb60 con 0x7fc5a40b1020 2024-03-04T07:35:39.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.178+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5a40c9d50 con 0x7fc5a40b1020 2024-03-04T07:35:39.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.179+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5a4067ff0 con 0x7fc5a40b1020 2024-03-04T07:35:39.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.180+0000 7fc592ffd640 1 -- 172.21.15.64:0/92911952 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc598038020 con 0x7fc5a40b1020 2024-03-04T07:35:39.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.180+0000 7fc592ffd640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc58406b500 0x7fc58406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:39.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.180+0000 7fc5a259c640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc58406b500 0x7fc58406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:39.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.180+0000 7fc592ffd640 1 -- 172.21.15.64:0/92911952 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc5980abcc0 con 0x7fc5a40b1020 2024-03-04T07:35:39.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.181+0000 7fc5a259c640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc58406b500 0x7fc58406d9c0 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7fc5940094a0 tx=0x7fc594009240 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:39.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.184+0000 7fc592ffd640 1 -- 172.21.15.64:0/92911952 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc598031800 con 0x7fc5a40b1020 2024-03-04T07:35:39.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.403+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc5a406f2f0 con 0x7fc5a40b1020 2024-03-04T07:35:39.403 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.403+0000 7fc592ffd640 1 -- 172.21.15.64:0/92911952 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fc59807a4a0 con 0x7fc5a40b1020 2024-03-04T07:35:39.403 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:39.404 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:39.404 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc58406b500 msgr2=0x7fc58406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc58406b500 0x7fc58406d9c0 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7fc5940094a0 tx=0x7fc594009240 comp rx=0 tx=0).stop 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a40b1020 msgr2=0x7fc5a40b6cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a40b1020 0x7fc5a40b6cc0 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7fc598033000 tx=0x7fc5980046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 shutdown_connections 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc58406b500 0x7fc58406d9c0 unknown :-1 s=CLOSED pgs=768 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc5a40cd6c0 0x7fc5a40b7200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc5a40b1020 0x7fc5a40b6cc0 unknown :-1 s=CLOSED pgs=615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 --2- 172.21.15.64:0/92911952 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc5a406ed10 0x7fc5a4158e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 >> 172.21.15.64:0/92911952 conn(0x7fc5a40d1a90 msgr2=0x7fc5a40d1310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 shutdown_connections 2024-03-04T07:35:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:39.405+0000 7fc5a89e9640 1 -- 172.21.15.64:0/92911952 wait complete. 2024-03-04T07:35:39.617 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:39.617 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:40.611 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:40.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:40.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.733+0000 7f44bd2a4640 1 Processor -- start 2024-03-04T07:35:40.733 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.733+0000 7f44bd2a4640 1 -- start start 2024-03-04T07:35:40.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.734+0000 7f44bd2a4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f44b806b5c0 0x7f44b80b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:40.734 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.734+0000 7f44bd2a4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f44b80b3340 con 0x7f44b80ae9b0 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.734+0000 7f44bd2a4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f44b80cee90 con 0x7f44b80b0e90 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.734+0000 7f44bd2a4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f44b80cfa00 con 0x7f44b806b5c0 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.734+0000 7f44b6ffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f44b806b5c0 0x7f44b80b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.734+0000 7f44b77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f44b80ae9b0 0x7f44b80d03a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:49984/0 (socket says 172.21.15.64:49984) 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.734+0000 7f44b77fe640 1 -- 172.21.15.64:0/3243552812 learned_addr learned my addr 172.21.15.64:0/3243552812 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.735+0000 7f44b6ffd640 1 -- 172.21.15.64:0/3243552812 >> v1:172.21.15.66:6789/0 conn(0x7f44b80b0e90 legacy=0x7f44b806b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.735+0000 7f44b5ffb640 1 -- 172.21.15.64:0/3243552812 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3608422420 0 0) 0x7f44b80cee90 con 0x7f44b80b0e90 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.735+0000 7f44b6ffd640 1 -- 172.21.15.64:0/3243552812 >> v1:172.21.15.64:6789/0 conn(0x7f44b80ae9b0 legacy=0x7f44b80d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:40.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.735+0000 7f44b6ffd640 1 -- 172.21.15.64:0/3243552812 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44b80cfc50 con 0x7f44b806b5c0 2024-03-04T07:35:40.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.735+0000 7f44b6ffd640 1 --2- 172.21.15.64:0/3243552812 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80b2e00 secure :-1 s=READY pgs=1697 cs=0 l=1 rev1=1 crypto rx=0x7f44a8009d90 tx=0x7f44a802bcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4a6c017e47fd3b5b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:40.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44b5ffb640 1 -- 172.21.15.64:0/3243552812 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44a8039070 con 0x7f44b806b5c0 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44b5ffb640 1 -- 172.21.15.64:0/3243552812 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f44a802ca20 con 0x7f44b806b5c0 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44b5ffb640 1 -- 172.21.15.64:0/3243552812 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44a802cdf0 con 0x7f44b806b5c0 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44bd2a4640 1 -- 172.21.15.64:0/3243552812 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 msgr2=0x7f44b80b2e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/3243552812 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80b2e00 secure :-1 s=READY pgs=1697 cs=0 l=1 rev1=1 crypto rx=0x7f44a8009d90 tx=0x7f44a802bcf0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44bd2a4640 1 -- 172.21.15.64:0/3243552812 shutdown_connections 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/3243552812 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80b2e00 unknown :-1 s=CLOSED pgs=1697 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44bd2a4640 1 -- 172.21.15.64:0/3243552812 >> 172.21.15.64:0/3243552812 conn(0x7f44b80b36d0 msgr2=0x7f44b80b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44bd2a4640 1 -- 172.21.15.64:0/3243552812 shutdown_connections 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.736+0000 7f44bd2a4640 1 -- 172.21.15.64:0/3243552812 wait complete. 2024-03-04T07:35:40.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.737+0000 7f44bd2a4640 1 Processor -- start 2024-03-04T07:35:40.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44bd2a4640 1 -- start start 2024-03-04T07:35:40.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44bd2a4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80cb4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:40.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44bd2a4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f44b80ae9b0 0x7f44b80aedc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:40.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44bd2a4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f44b80b0e90 0x7f44b80af300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:40.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44bd2a4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44b806c910 con 0x7f44b806b5c0 2024-03-04T07:35:40.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44bd2a4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44b806c790 con 0x7f44b80b0e90 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44bd2a4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f44b80d0ab0 con 0x7f44b80ae9b0 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44b6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80cb4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44b6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80cb4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40144/0 (socket says 172.21.15.64:40144) 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44b6ffd640 1 -- 172.21.15.64:0/502052062 learned_addr learned my addr 172.21.15.64:0/502052062 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44b77fe640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f44b80b0e90 0x7f44b80af300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.738+0000 7f44b67fc640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f44b80ae9b0 0x7f44b80aedc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44b6ffd640 1 -- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f44b80ae9b0 msgr2=0x7f44b80aedc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44b6ffd640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f44b80ae9b0 0x7f44b80aedc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44b6ffd640 1 -- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f44b80b0e90 msgr2=0x7f44b80af300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44b6ffd640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f44b80b0e90 0x7f44b80af300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44b6ffd640 1 -- 172.21.15.64:0/502052062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44b814cb00 con 0x7f44b806b5c0 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44b6ffd640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80cb4e0 secure :-1 s=READY pgs=1698 cs=0 l=1 rev1=1 crypto rx=0x7f44a8037040 tx=0x7f44a8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44af7fe640 1 -- 172.21.15.64:0/502052062 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44a8044020 con 0x7f44b806b5c0 2024-03-04T07:35:40.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44af7fe640 1 -- 172.21.15.64:0/502052062 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f44a8034d30 con 0x7f44b806b5c0 2024-03-04T07:35:40.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44af7fe640 1 -- 172.21.15.64:0/502052062 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44a8039040 con 0x7f44b806b5c0 2024-03-04T07:35:40.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44b814cde0 con 0x7f44b806b5c0 2024-03-04T07:35:40.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.739+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44b814d3a0 con 0x7f44b806b5c0 2024-03-04T07:35:40.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.740+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f44b8068700 con 0x7f44b806b5c0 2024-03-04T07:35:40.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.741+0000 7f44af7fe640 1 -- 172.21.15.64:0/502052062 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f44a8047020 con 0x7f44b806b5c0 2024-03-04T07:35:40.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.741+0000 7f44af7fe640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f449806b500 0x7f449806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:40.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.741+0000 7f44af7fe640 1 -- 172.21.15.64:0/502052062 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f44a80ac690 con 0x7f44b806b5c0 2024-03-04T07:35:40.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.741+0000 7f44b67fc640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f449806b500 0x7f449806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:40.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.742+0000 7f44b67fc640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f449806b500 0x7f449806d9c0 secure :-1 s=READY pgs=769 cs=0 l=1 rev1=1 crypto rx=0x7f44b81f70d0 tx=0x7f44a0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:40.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.746+0000 7f44af7fe640 1 -- 172.21.15.64:0/502052062 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f44a804e070 con 0x7f44b806b5c0 2024-03-04T07:35:40.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.965+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f44b80708f0 con 0x7f44b806b5c0 2024-03-04T07:35:40.965 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.965+0000 7f44af7fe640 1 -- 172.21.15.64:0/502052062 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f44a807ae80 con 0x7f44b806b5c0 2024-03-04T07:35:40.965 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:40.966 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:40.966 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f449806b500 msgr2=0x7f449806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f449806b500 0x7f449806d9c0 secure :-1 s=READY pgs=769 cs=0 l=1 rev1=1 crypto rx=0x7f44b81f70d0 tx=0x7f44a0009040 comp rx=0 tx=0).stop 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 msgr2=0x7f44b80cb4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80cb4e0 secure :-1 s=READY pgs=1698 cs=0 l=1 rev1=1 crypto rx=0x7f44a8037040 tx=0x7f44a8004a40 comp rx=0 tx=0).stop 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 shutdown_connections 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f449806b500 0x7f449806d9c0 unknown :-1 s=CLOSED pgs=769 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f44b80b0e90 0x7f44b80af300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f44b80ae9b0 0x7f44b80aedc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 --2- 172.21.15.64:0/502052062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f44b806b5c0 0x7f44b80cb4e0 unknown :-1 s=CLOSED pgs=1698 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 >> 172.21.15.64:0/502052062 conn(0x7f44b80b36d0 msgr2=0x7f44b80b2670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 shutdown_connections 2024-03-04T07:35:40.968 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:40.968+0000 7f44bd2a4640 1 -- 172.21.15.64:0/502052062 wait complete. 2024-03-04T07:35:41.179 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:41.180 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:42.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:42.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.294+0000 7f010a35e640 1 Processor -- start 2024-03-04T07:35:42.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.294+0000 7f010a35e640 1 -- start start 2024-03-04T07:35:42.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f010a35e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f01041451a0 0x7f01040b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:42.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f010a35e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f01040b3c70 con 0x7f01041456f0 2024-03-04T07:35:42.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f010a35e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f01040afad0 con 0x7f010415bde0 2024-03-04T07:35:42.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f010a35e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f01040b4840 con 0x7f01041451a0 2024-03-04T07:35:42.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f01088fb640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f01041456f0 0x7f010415c1c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50000/0 (socket says 172.21.15.64:50000) 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f01088fb640 1 -- 172.21.15.64:0/3624869841 learned_addr learned my addr 172.21.15.64:0/3624869841 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f01037fe640 1 --2- 172.21.15.64:0/3624869841 >> v2:172.21.15.66:3300/0 conn(0x7f01041451a0 0x7f01040b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f0102ffd640 1 -- 172.21.15.64:0/3624869841 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1700012021 0 0) 0x7f01040b3c70 con 0x7f01041456f0 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f0102ffd640 1 -- 172.21.15.64:0/3624869841 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f00ec003520 con 0x7f01041456f0 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f0102ffd640 1 -- 172.21.15.64:0/3624869841 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4261528036 0 0) 0x7f01040afad0 con 0x7f010415bde0 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f0102ffd640 1 -- 172.21.15.64:0/3624869841 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f01040b3c70 con 0x7f010415bde0 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f01037fe640 1 -- 172.21.15.64:0/3624869841 >> v1:172.21.15.66:6789/0 conn(0x7f010415bde0 legacy=0x7f01040b4cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f01037fe640 1 -- 172.21.15.64:0/3624869841 >> v1:172.21.15.64:6789/0 conn(0x7f01041456f0 legacy=0x7f010415c1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:42.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.295+0000 7f01037fe640 1 -- 172.21.15.64:0/3624869841 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01040b3f10 con 0x7f01041451a0 2024-03-04T07:35:42.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.296+0000 7f01037fe640 1 --2- 172.21.15.64:0/3624869841 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041451a0 0x7f01040b1d50 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f00f40097d0 tx=0x7f00f402b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b85e6ed31e04cc4c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:42.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.296+0000 7f0102ffd640 1 -- 172.21.15.64:0/3624869841 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00f4034070 con 0x7f01041451a0 2024-03-04T07:35:42.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f0102ffd640 1 -- 172.21.15.64:0/3624869841 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f00f4030610 con 0x7f01041451a0 2024-03-04T07:35:42.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f0102ffd640 1 -- 172.21.15.64:0/3624869841 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00f40309e0 con 0x7f01041451a0 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f010a35e640 1 -- 172.21.15.64:0/3624869841 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041451a0 msgr2=0x7f01040b1d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f010a35e640 1 --2- 172.21.15.64:0/3624869841 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041451a0 0x7f01040b1d50 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f00f40097d0 tx=0x7f00f402b960 comp rx=0 tx=0).stop 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f010a35e640 1 -- 172.21.15.64:0/3624869841 shutdown_connections 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f010a35e640 1 --2- 172.21.15.64:0/3624869841 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041451a0 0x7f01040b1d50 unknown :-1 s=CLOSED pgs=618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f010a35e640 1 -- 172.21.15.64:0/3624869841 >> 172.21.15.64:0/3624869841 conn(0x7f01040b55e0 msgr2=0x7f01040b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f010a35e640 1 -- 172.21.15.64:0/3624869841 shutdown_connections 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.297+0000 7f010a35e640 1 -- 172.21.15.64:0/3624869841 wait complete. 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.298+0000 7f010a35e640 1 Processor -- start 2024-03-04T07:35:42.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f010a35e640 1 -- start start 2024-03-04T07:35:42.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f010a35e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01041451a0 0x7f01041f4900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:42.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f010a35e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041456f0 0x7f01041f7d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:42.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f010a35e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f010415bde0 0x7f010414f600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:42.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f010a35e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01040b44d0 con 0x7f01041451a0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f010a35e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01040b53e0 con 0x7f01041456f0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f010a35e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f01040b2290 con 0x7f010415bde0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f0103fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01041451a0 0x7f01041f4900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f01037fe640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041456f0 0x7f01041f7d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f0103fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01041451a0 0x7f01041f4900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:40154/0 (socket says 172.21.15.64:40154) 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f0103fff640 1 -- 172.21.15.64:0/1315662775 learned_addr learned my addr 172.21.15.64:0/1315662775 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.299+0000 7f01088fb640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f010415bde0 0x7f010414f600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0103fff640 1 -- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f010415bde0 msgr2=0x7f010414f600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0103fff640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f010415bde0 0x7f010414f600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0103fff640 1 -- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041456f0 msgr2=0x7f01041f7d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0103fff640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041456f0 0x7f01041f7d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0103fff640 1 -- 172.21.15.64:0/1315662775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0104155c80 con 0x7f01041451a0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f01088fb640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f010415bde0 0x7f010414f600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0103fff640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01041451a0 0x7f01041f4900 secure :-1 s=READY pgs=1700 cs=0 l=1 rev1=1 crypto rx=0x7f00f800c430 tx=0x7f00f800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0100ff9640 1 -- 172.21.15.64:0/1315662775 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00f8017070 con 0x7f01041451a0 2024-03-04T07:35:42.301 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0100ff9640 1 -- 172.21.15.64:0/1315662775 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f00f80052f0 con 0x7f01041451a0 2024-03-04T07:35:42.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01041f7580 con 0x7f01041451a0 2024-03-04T07:35:42.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f0100ff9640 1 -- 172.21.15.64:0/1315662775 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00f8003830 con 0x7f01041451a0 2024-03-04T07:35:42.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.300+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f01041556c0 con 0x7f01041451a0 2024-03-04T07:35:42.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.301+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0104067ff0 con 0x7f01041451a0 2024-03-04T07:35:42.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.302+0000 7f0100ff9640 1 -- 172.21.15.64:0/1315662775 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f00f801a070 con 0x7f01041451a0 2024-03-04T07:35:42.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.302+0000 7f0100ff9640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f00f006b4d0 0x7f00f006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:42.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.302+0000 7f01037fe640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f00f006b4d0 0x7f00f006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:42.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.302+0000 7f0100ff9640 1 -- 172.21.15.64:0/1315662775 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f00f8089500 con 0x7f01041451a0 2024-03-04T07:35:42.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.303+0000 7f01037fe640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f00f006b4d0 0x7f00f006d990 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f00f4036000 tx=0x7f00f403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:42.306 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.307+0000 7f0100ff9640 1 -- 172.21.15.64:0/1315662775 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f00f8057d80 con 0x7f01041451a0 2024-03-04T07:35:42.524 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.525+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f01040b1890 con 0x7f01041451a0 2024-03-04T07:35:42.525 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.525+0000 7f0100ff9640 1 -- 172.21.15.64:0/1315662775 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f00f805e0d0 con 0x7f01041451a0 2024-03-04T07:35:42.525 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:42.525 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:42.527 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:42.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f00f006b4d0 msgr2=0x7f00f006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:42.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f00f006b4d0 0x7f00f006d990 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f00f4036000 tx=0x7f00f403b040 comp rx=0 tx=0).stop 2024-03-04T07:35:42.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01041451a0 msgr2=0x7f01041f4900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:42.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01041451a0 0x7f01041f4900 secure :-1 s=READY pgs=1700 cs=0 l=1 rev1=1 crypto rx=0x7f00f800c430 tx=0x7f00f800c8f0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 shutdown_connections 2024-03-04T07:35:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f00f006b4d0 0x7f00f006d990 unknown :-1 s=CLOSED pgs=770 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f010415bde0 0x7f010414f600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f01041456f0 0x7f01041f7d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 --2- 172.21.15.64:0/1315662775 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f01041451a0 0x7f01041f4900 unknown :-1 s=CLOSED pgs=1700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 >> 172.21.15.64:0/1315662775 conn(0x7f01040b55e0 msgr2=0x7f01040b2dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 shutdown_connections 2024-03-04T07:35:42.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:42.527+0000 7f010a35e640 1 -- 172.21.15.64:0/1315662775 wait complete. 2024-03-04T07:35:42.613 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:42.739 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:42.739 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:43.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.855+0000 7f3c12bd7640 1 Processor -- start 2024-03-04T07:35:43.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.855+0000 7f3c12bd7640 1 -- start start 2024-03-04T07:35:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c12bd7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3c0c15e000 0x7f3c0c071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:43.856 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c12bd7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3c0c0d35e0 0x7f3c0c071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c12bd7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3c0c0ce970 con 0x7f3c0c0b2c50 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c12bd7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c0c0cece0 con 0x7f3c0c15e000 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c12bd7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3c0c0cef30 con 0x7f3c0c0d35e0 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c10973640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3c0c15e000 0x7f3c0c071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c0bfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3c0c0b2c50 0x7f3c0c161cb0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40014/0 (socket says 172.21.15.64:40014) 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c0bfff640 1 -- 172.21.15.64:0/1162370283 learned_addr learned my addr 172.21.15.64:0/1162370283 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c11174640 1 --2- 172.21.15.64:0/1162370283 >> v2:172.21.15.66:3300/0 conn(0x7f3c0c0d35e0 0x7f3c0c071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c10973640 1 -- 172.21.15.64:0/1162370283 >> v2:172.21.15.66:3300/0 conn(0x7f3c0c0d35e0 msgr2=0x7f3c0c071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c10973640 1 --2- 172.21.15.64:0/1162370283 >> v2:172.21.15.66:3300/0 conn(0x7f3c0c0d35e0 0x7f3c0c071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c10973640 1 -- 172.21.15.64:0/1162370283 >> v1:172.21.15.64:6789/0 conn(0x7f3c0c0b2c50 legacy=0x7f3c0c161cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c0b7fe640 1 -- 172.21.15.64:0/1162370283 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4100628318 0 0) 0x7f3c0c0ce970 con 0x7f3c0c0b2c50 2024-03-04T07:35:43.857 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.856+0000 7f3c10973640 1 -- 172.21.15.64:0/1162370283 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c0c0cf210 con 0x7f3c0c15e000 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.857+0000 7f3c10973640 1 --2- 172.21.15.64:0/1162370283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c15e000 0x7f3c0c071540 secure :-1 s=READY pgs=1702 cs=0 l=1 rev1=1 crypto rx=0x7f3c0c073780 tx=0x7f3c0002b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a59bd7a79e69c3fb server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.857+0000 7f3c0b7fe640 1 -- 172.21.15.64:0/1162370283 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c000315a0 con 0x7f3c0c15e000 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.857+0000 7f3c0b7fe640 1 -- 172.21.15.64:0/1162370283 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c00037070 con 0x7f3c0c15e000 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.857+0000 7f3c0b7fe640 1 -- 172.21.15.64:0/1162370283 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c00031960 con 0x7f3c0c15e000 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 -- 172.21.15.64:0/1162370283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c15e000 msgr2=0x7f3c0c071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/1162370283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c15e000 0x7f3c0c071540 secure :-1 s=READY pgs=1702 cs=0 l=1 rev1=1 crypto rx=0x7f3c0c073780 tx=0x7f3c0002b840 comp rx=0 tx=0).stop 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 -- 172.21.15.64:0/1162370283 shutdown_connections 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/1162370283 >> v2:172.21.15.66:3300/0 conn(0x7f3c0c0d35e0 0x7f3c0c071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/1162370283 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c15e000 0x7f3c0c071540 unknown :-1 s=CLOSED pgs=1702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 -- 172.21.15.64:0/1162370283 >> 172.21.15.64:0/1162370283 conn(0x7f3c0c0d3c80 msgr2=0x7f3c0c0d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 -- 172.21.15.64:0/1162370283 shutdown_connections 2024-03-04T07:35:43.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.858+0000 7f3c12bd7640 1 -- 172.21.15.64:0/1162370283 wait complete. 2024-03-04T07:35:43.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.859+0000 7f3c12bd7640 1 Processor -- start 2024-03-04T07:35:43.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.859+0000 7f3c12bd7640 1 -- start start 2024-03-04T07:35:43.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c12bd7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c0b2c50 0x7f3c0c0cb760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c12bd7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c0c0d35e0 0x7f3c0c1f3100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c12bd7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c0c15e000 0x7f3c0c15ba20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c12bd7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c0c0b3320 con 0x7f3c0c0b2c50 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c12bd7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c0c0b31a0 con 0x7f3c0c15e000 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c12bd7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3c0c15e3e0 con 0x7f3c0c0d35e0 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c10973640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c0b2c50 0x7f3c0c0cb760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c10973640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c0b2c50 0x7f3c0c0cb760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37622/0 (socket says 172.21.15.64:37622) 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c10973640 1 -- 172.21.15.64:0/2990183005 learned_addr learned my addr 172.21.15.64:0/2990183005 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c11174640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c0c15e000 0x7f3c0c15ba20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c0bfff640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c0c0d35e0 0x7f3c0c1f3100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c11174640 1 -- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c0c0d35e0 msgr2=0x7f3c0c1f3100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c11174640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c0c0d35e0 0x7f3c0c1f3100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c11174640 1 -- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c0b2c50 msgr2=0x7f3c0c0cb760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c11174640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c0b2c50 0x7f3c0c0cb760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c11174640 1 -- 172.21.15.64:0/2990183005 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c0c1f7220 con 0x7f3c0c15e000 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.860+0000 7f3c10973640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c0b2c50 0x7f3c0c0cb760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.861+0000 7f3c11174640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c0c15e000 0x7f3c0c15ba20 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f3c0400eb70 tx=0x7f3c0400c490 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.861+0000 7f3c097fa640 1 -- 172.21.15.64:0/2990183005 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c04018020 con 0x7f3c0c15e000 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.861+0000 7f3c097fa640 1 -- 172.21.15.64:0/2990183005 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3c0400cd20 con 0x7f3c0c15e000 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.861+0000 7f3c097fa640 1 -- 172.21.15.64:0/2990183005 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c040146c0 con 0x7f3c0c15e000 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.861+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c0c1f7500 con 0x7f3c0c15e000 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.861+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c0c1f7ac0 con 0x7f3c0c15e000 2024-03-04T07:35:43.863 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.862+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c0c067ff0 con 0x7f3c0c15e000 2024-03-04T07:35:43.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.862+0000 7f3c097fa640 1 -- 172.21.15.64:0/2990183005 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3c04005d10 con 0x7f3c0c15e000 2024-03-04T07:35:43.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.863+0000 7f3c097fa640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bf006b4d0 0x7f3bf006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:43.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.863+0000 7f3c097fa640 1 -- 172.21.15.64:0/2990183005 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3c0408ae50 con 0x7f3c0c15e000 2024-03-04T07:35:43.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.863+0000 7f3c10973640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bf006b4d0 0x7f3bf006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:43.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.863+0000 7f3c10973640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bf006b4d0 0x7f3bf006d990 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7f3c00000c00 tx=0x7f3c0003d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:43.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:43.872+0000 7f3c097fa640 1 -- 172.21.15.64:0/2990183005 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3c04022030 con 0x7f3c0c15e000 2024-03-04T07:35:44.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.092+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3c0c0b7a60 con 0x7f3c0c15e000 2024-03-04T07:35:44.092 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.093+0000 7f3c097fa640 1 -- 172.21.15.64:0/2990183005 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f3c04010040 con 0x7f3c0c15e000 2024-03-04T07:35:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:44.093 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:44.093 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bf006b4d0 msgr2=0x7f3bf006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bf006b4d0 0x7f3bf006d990 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7f3c00000c00 tx=0x7f3c0003d040 comp rx=0 tx=0).stop 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c0c15e000 msgr2=0x7f3c0c15ba20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c0c15e000 0x7f3c0c15ba20 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f3c0400eb70 tx=0x7f3c0400c490 comp rx=0 tx=0).stop 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 shutdown_connections 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bf006b4d0 0x7f3bf006d990 unknown :-1 s=CLOSED pgs=771 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3c0c15e000 0x7f3c0c15ba20 unknown :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:44.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3c0c0d35e0 0x7f3c0c1f3100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:44.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 --2- 172.21.15.64:0/2990183005 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3c0c0b2c50 0x7f3c0c0cb760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:44.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 >> 172.21.15.64:0/2990183005 conn(0x7f3c0c0d3c80 msgr2=0x7f3c0c0b6f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:44.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 shutdown_connections 2024-03-04T07:35:44.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:44.095+0000 7f3c12bd7640 1 -- 172.21.15.64:0/2990183005 wait complete. 2024-03-04T07:35:44.307 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:44.307 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:44.615 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:45.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:45.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.421+0000 7f4384c5c640 1 Processor -- start 2024-03-04T07:35:45.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.421+0000 7f4384c5c640 1 -- start start 2024-03-04T07:35:45.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f4384c5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:45.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f4384c5c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f43800b95a0 0x7f43800b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:45.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f4384c5c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f43800b9a30 0x7f43800b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:45.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f4384c5c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43800cea30 con 0x7f43800b9a30 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f4384c5c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43800d2af0 con 0x7f43800b95a0 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f4384c5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43800cf3c0 con 0x7f438015df00 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f43800b9a30 0x7f43800b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f43800b9a30 0x7f43800b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37638/0 (socket says 172.21.15.64:37638) 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 -- 172.21.15.64:0/2778366228 learned_addr learned my addr 172.21.15.64:0/2778366228 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437e59c640 1 --2- 172.21.15.64:0/2778366228 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437dd9b640 1 --2- 172.21.15.64:0/2778366228 >> v2:172.21.15.66:3300/0 conn(0x7f43800b95a0 0x7f43800b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 -- 172.21.15.64:0/2778366228 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 msgr2=0x7f43800d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 --2- 172.21.15.64:0/2778366228 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 -- 172.21.15.64:0/2778366228 >> v2:172.21.15.66:3300/0 conn(0x7f43800b95a0 msgr2=0x7f43800b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 --2- 172.21.15.64:0/2778366228 >> v2:172.21.15.66:3300/0 conn(0x7f43800b95a0 0x7f43800b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437ed9d640 1 -- 172.21.15.64:0/2778366228 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43800cf0e0 con 0x7f43800b9a30 2024-03-04T07:35:45.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.422+0000 7f437dd9b640 1 --2- 172.21.15.64:0/2778366228 >> v2:172.21.15.66:3300/0 conn(0x7f43800b95a0 0x7f43800b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.423+0000 7f437ed9d640 1 --2- 172.21.15.64:0/2778366228 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b9a30 0x7f43800b5c40 secure :-1 s=READY pgs=1703 cs=0 l=1 rev1=1 crypto rx=0x7f438008cbc0 tx=0x7f437802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6bb7ba8ca79388d9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.423+0000 7f437d59a640 1 -- 172.21.15.64:0/2778366228 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f437802ca70 con 0x7f43800b9a30 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.423+0000 7f437d59a640 1 -- 172.21.15.64:0/2778366228 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f437802cc10 con 0x7f43800b9a30 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.423+0000 7f437d59a640 1 -- 172.21.15.64:0/2778366228 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43780343f0 con 0x7f43800b9a30 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 -- 172.21.15.64:0/2778366228 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b9a30 msgr2=0x7f43800b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 --2- 172.21.15.64:0/2778366228 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b9a30 0x7f43800b5c40 secure :-1 s=READY pgs=1703 cs=0 l=1 rev1=1 crypto rx=0x7f438008cbc0 tx=0x7f437802bed0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 -- 172.21.15.64:0/2778366228 shutdown_connections 2024-03-04T07:35:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 --2- 172.21.15.64:0/2778366228 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b9a30 0x7f43800b5c40 unknown :-1 s=CLOSED pgs=1703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 --2- 172.21.15.64:0/2778366228 >> v2:172.21.15.66:3300/0 conn(0x7f43800b95a0 0x7f43800b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 --2- 172.21.15.64:0/2778366228 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 -- 172.21.15.64:0/2778366228 >> 172.21.15.64:0/2778366228 conn(0x7f4380071960 msgr2=0x7f4380071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 -- 172.21.15.64:0/2778366228 shutdown_connections 2024-03-04T07:35:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.424+0000 7f4384c5c640 1 -- 172.21.15.64:0/2778366228 wait complete. 2024-03-04T07:35:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.425+0000 7f4384c5c640 1 Processor -- start 2024-03-04T07:35:45.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.425+0000 7f4384c5c640 1 -- start start 2024-03-04T07:35:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f4384c5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b95a0 0x7f438014d1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f4384c5c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800c77b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f4384c5c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4380149cc0 0x7f438015f330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f4384c5c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43800cec70 con 0x7f43800b95a0 2024-03-04T07:35:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f4384c5c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43800ce750 con 0x7f4380149cc0 2024-03-04T07:35:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f4384c5c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43800cedf0 con 0x7f438015df00 2024-03-04T07:35:45.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f437e59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b95a0 0x7f438014d1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f437e59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b95a0 0x7f438014d1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37642/0 (socket says 172.21.15.64:37642) 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f437e59c640 1 -- 172.21.15.64:0/1178305349 learned_addr learned my addr 172.21.15.64:0/1178305349 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f437ed9d640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4380149cc0 0x7f438015f330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.426+0000 7f437dd9b640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800c77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f437e59c640 1 -- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 msgr2=0x7f43800c77b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f437e59c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800c77b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f437e59c640 1 -- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4380149cc0 msgr2=0x7f438015f330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f437e59c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4380149cc0 0x7f438015f330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f437e59c640 1 -- 172.21.15.64:0/1178305349 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43800c7cf0 con 0x7f43800b95a0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f437e59c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b95a0 0x7f438014d1b0 secure :-1 s=READY pgs=1704 cs=0 l=1 rev1=1 crypto rx=0x7f437000e9b0 tx=0x7f437000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f436effd640 1 -- 172.21.15.64:0/1178305349 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4370017070 con 0x7f43800b95a0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f436effd640 1 -- 172.21.15.64:0/1178305349 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f437000c9a0 con 0x7f43800b95a0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f436effd640 1 -- 172.21.15.64:0/1178305349 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f437000cd90 con 0x7f43800b95a0 2024-03-04T07:35:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f43800b5b30 con 0x7f43800b95a0 2024-03-04T07:35:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.427+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f43800b3000 con 0x7f43800b95a0 2024-03-04T07:35:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.428+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4380067ff0 con 0x7f43800b95a0 2024-03-04T07:35:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.428+0000 7f436effd640 1 -- 172.21.15.64:0/1178305349 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f437001a070 con 0x7f43800b95a0 2024-03-04T07:35:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.429+0000 7f436effd640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f436006b4d0 0x7f436006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.429+0000 7f436effd640 1 -- 172.21.15.64:0/1178305349 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f437008a0d0 con 0x7f43800b95a0 2024-03-04T07:35:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.429+0000 7f437dd9b640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f436006b4d0 0x7f436006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:45.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.429+0000 7f437dd9b640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f436006b4d0 0x7f436006d990 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f4374009560 tx=0x7f4374009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:45.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.433+0000 7f436effd640 1 -- 172.21.15.64:0/1178305349 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f43700588c0 con 0x7f43800b95a0 2024-03-04T07:35:45.655 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.655+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f438006b780 con 0x7f43800b95a0 2024-03-04T07:35:45.656 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.656+0000 7f436effd640 1 -- 172.21.15.64:0/1178305349 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f437005c960 con 0x7f43800b95a0 2024-03-04T07:35:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:45.656 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:45.656 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:45.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f436006b4d0 msgr2=0x7f436006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:45.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f436006b4d0 0x7f436006d990 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f4374009560 tx=0x7f4374009190 comp rx=0 tx=0).stop 2024-03-04T07:35:45.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b95a0 msgr2=0x7f438014d1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:45.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b95a0 0x7f438014d1b0 secure :-1 s=READY pgs=1704 cs=0 l=1 rev1=1 crypto rx=0x7f437000e9b0 tx=0x7f437000ee70 comp rx=0 tx=0).stop 2024-03-04T07:35:45.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 shutdown_connections 2024-03-04T07:35:45.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f436006b4d0 0x7f436006d990 unknown :-1 s=CLOSED pgs=772 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4380149cc0 0x7f438015f330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f438015df00 0x7f43800c77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 --2- 172.21.15.64:0/1178305349 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43800b95a0 0x7f438014d1b0 unknown :-1 s=CLOSED pgs=1704 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:45.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 >> 172.21.15.64:0/1178305349 conn(0x7f4380071960 msgr2=0x7f4380071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:45.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 shutdown_connections 2024-03-04T07:35:45.659 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:45.658+0000 7f4384c5c640 1 -- 172.21.15.64:0/1178305349 wait complete. 2024-03-04T07:35:45.870 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:45.870 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:46.617 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:46.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:46.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.985+0000 7ff4e8993640 1 Processor -- start 2024-03-04T07:35:46.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.985+0000 7ff4e8993640 1 -- start start 2024-03-04T07:35:46.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e8993640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40d3400 0x7ff4e40b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:46.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e8993640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff4e40b2b50 0x7ff4e40b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:46.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e8993640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff4e40ce610 con 0x7ff4e4071540 2024-03-04T07:35:46.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e8993640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff4e40d2f70 con 0x7ff4e40b2b50 2024-03-04T07:35:46.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e8993640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4e40d4be0 con 0x7ff4e40d3400 2024-03-04T07:35:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e2d9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff4e40b2b50 0x7ff4e40b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e1d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40d3400 0x7ff4e40b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e259c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7ff4e4071540 0x7ff4e4071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:48714/0 (socket says 172.21.15.64:48714) 2024-03-04T07:35:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.986+0000 7ff4e259c640 1 -- 172.21.15.64:0/348739877 learned_addr learned my addr 172.21.15.64:0/348739877 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.987+0000 7ff4e1d9b640 1 -- 172.21.15.64:0/348739877 >> v2:172.21.15.66:3300/0 conn(0x7ff4e40b2b50 msgr2=0x7ff4e40b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.987+0000 7ff4e1d9b640 1 --2- 172.21.15.64:0/348739877 >> v2:172.21.15.66:3300/0 conn(0x7ff4e40b2b50 0x7ff4e40b6e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.987+0000 7ff4e1d9b640 1 -- 172.21.15.64:0/348739877 >> v1:172.21.15.66:6789/0 conn(0x7ff4e4071540 legacy=0x7ff4e4071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.987+0000 7ff4e1d9b640 1 -- 172.21.15.64:0/348739877 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4e40d2920 con 0x7ff4e40d3400 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.987+0000 7ff4e2d9d640 1 --2- 172.21.15.64:0/348739877 >> v2:172.21.15.66:3300/0 conn(0x7ff4e40b2b50 0x7ff4e40b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e1d9b640 1 --2- 172.21.15.64:0/348739877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40d3400 0x7ff4e40b30a0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff4cc0097a0 tx=0x7ff4cc02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2e1af2689dd63e4c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e159a640 1 -- 172.21.15.64:0/348739877 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4cc035020 con 0x7ff4e40d3400 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e159a640 1 -- 172.21.15.64:0/348739877 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4cc031830 con 0x7ff4e40d3400 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e159a640 1 -- 172.21.15.64:0/348739877 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4cc031c00 con 0x7ff4e40d3400 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e8993640 1 -- 172.21.15.64:0/348739877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40d3400 msgr2=0x7ff4e40b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e8993640 1 --2- 172.21.15.64:0/348739877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40d3400 0x7ff4e40b30a0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff4cc0097a0 tx=0x7ff4cc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e8993640 1 -- 172.21.15.64:0/348739877 shutdown_connections 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e8993640 1 --2- 172.21.15.64:0/348739877 >> v2:172.21.15.66:3300/0 conn(0x7ff4e40b2b50 0x7ff4e40b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e8993640 1 --2- 172.21.15.64:0/348739877 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40d3400 0x7ff4e40b30a0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.988+0000 7ff4e8993640 1 -- 172.21.15.64:0/348739877 >> 172.21.15.64:0/348739877 conn(0x7ff4e40d3ca0 msgr2=0x7ff4e40d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.989+0000 7ff4e8993640 1 -- 172.21.15.64:0/348739877 shutdown_connections 2024-03-04T07:35:46.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.989+0000 7ff4e8993640 1 -- 172.21.15.64:0/348739877 wait complete. 2024-03-04T07:35:46.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.990+0000 7ff4e8993640 1 Processor -- start 2024-03-04T07:35:46.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.990+0000 7ff4e8993640 1 -- start start 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e8993640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4e4071540 0x7ff4e4157c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e8993640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40b2b50 0x7ff4e41573e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e8993640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4e40d3400 0x7ff4e40b7de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e8993640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4e415e370 con 0x7ff4e40d3400 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e8993640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4e40ce8f0 con 0x7ff4e4071540 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e8993640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff4e40b5b60 con 0x7ff4e40b2b50 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4e40d3400 0x7ff4e40b7de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e1d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40b2b50 0x7ff4e41573e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4e40d3400 0x7ff4e40b7de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37654/0 (socket says 172.21.15.64:37654) 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 -- 172.21.15.64:0/1265317391 learned_addr learned my addr 172.21.15.64:0/1265317391 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e259c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4e4071540 0x7ff4e4157c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 -- 172.21.15.64:0/1265317391 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40b2b50 msgr2=0x7ff4e41573e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40b2b50 0x7ff4e41573e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 -- 172.21.15.64:0/1265317391 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4e4071540 msgr2=0x7ff4e4157c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4e4071540 0x7ff4e4157c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 -- 172.21.15.64:0/1265317391 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4e40b5e40 con 0x7ff4e40d3400 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.991+0000 7ff4e2d9d640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4e40d3400 0x7ff4e40b7de0 secure :-1 s=READY pgs=1705 cs=0 l=1 rev1=1 crypto rx=0x7ff4dc00e930 tx=0x7ff4dc00edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.992+0000 7ff4daffd640 1 -- 172.21.15.64:0/1265317391 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4dc017070 con 0x7ff4e40d3400 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.992+0000 7ff4daffd640 1 -- 172.21.15.64:0/1265317391 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff4dc00ca00 con 0x7ff4e40d3400 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.992+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4e40b6120 con 0x7ff4e40d3400 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.992+0000 7ff4daffd640 1 -- 172.21.15.64:0/1265317391 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4dc00cdd0 con 0x7ff4e40d3400 2024-03-04T07:35:46.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.992+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4e40b6400 con 0x7ff4e40d3400 2024-03-04T07:35:46.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.993+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4e4068700 con 0x7ff4e40d3400 2024-03-04T07:35:46.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.993+0000 7ff4daffd640 1 -- 172.21.15.64:0/1265317391 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff4dc013470 con 0x7ff4e40d3400 2024-03-04T07:35:46.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.993+0000 7ff4daffd640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4c406b500 0x7ff4c406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:46.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.993+0000 7ff4daffd640 1 -- 172.21.15.64:0/1265317391 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff4dc089ff0 con 0x7ff4e40d3400 2024-03-04T07:35:46.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.994+0000 7ff4e259c640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4c406b500 0x7ff4c406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:46.994 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.994+0000 7ff4e259c640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4c406b500 0x7ff4c406d9c0 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7ff4d40012e0 tx=0x7ff4d4001a30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:46.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:46.998+0000 7ff4daffd640 1 -- 172.21.15.64:0/1265317391 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff4dc058930 con 0x7ff4e40d3400 2024-03-04T07:35:47.215 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.215+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff4e4160e30 con 0x7ff4e40d3400 2024-03-04T07:35:47.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.216+0000 7ff4daffd640 1 -- 172.21.15.64:0/1265317391 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7ff4dc05c9d0 con 0x7ff4e40d3400 2024-03-04T07:35:47.216 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:47.216 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:47.216 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4c406b500 msgr2=0x7ff4c406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4c406b500 0x7ff4c406d9c0 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7ff4d40012e0 tx=0x7ff4d4001a30 comp rx=0 tx=0).stop 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4e40d3400 msgr2=0x7ff4e40b7de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4e40d3400 0x7ff4e40b7de0 secure :-1 s=READY pgs=1705 cs=0 l=1 rev1=1 crypto rx=0x7ff4dc00e930 tx=0x7ff4dc00edf0 comp rx=0 tx=0).stop 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 shutdown_connections 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff4c406b500 0x7ff4c406d9c0 unknown :-1 s=CLOSED pgs=773 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff4e40d3400 0x7ff4e40b7de0 unknown :-1 s=CLOSED pgs=1705 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff4e40b2b50 0x7ff4e41573e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:47.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 --2- 172.21.15.64:0/1265317391 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff4e4071540 0x7ff4e4157c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:47.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 >> 172.21.15.64:0/1265317391 conn(0x7ff4e40d3ca0 msgr2=0x7ff4e41601d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:47.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 shutdown_connections 2024-03-04T07:35:47.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:47.218+0000 7ff4e8993640 1 -- 172.21.15.64:0/1265317391 wait complete. 2024-03-04T07:35:47.430 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:47.430 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:48.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:35:48.397 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:35:48.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:48.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.548+0000 7f5524834640 1 Processor -- start 2024-03-04T07:35:48.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.548+0000 7f5524834640 1 -- start start 2024-03-04T07:35:48.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.549+0000 7f5524834640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:48.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.549+0000 7f5524834640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55200cea30 con 0x7f55200b9a30 2024-03-04T07:35:48.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.549+0000 7f5524834640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f55200d2af0 con 0x7f55200b95a0 2024-03-04T07:35:48.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.549+0000 7f5524834640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55200cf3c0 con 0x7f552015df00 2024-03-04T07:35:48.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551ed9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f55200b9a30 0x7f55200d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40018/0 (socket says 172.21.15.64:40018) 2024-03-04T07:35:48.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551ed9d640 1 -- 172.21.15.64:0/2899976222 learned_addr learned my addr 172.21.15.64:0/2899976222 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551e59c640 1 --2- 172.21.15.64:0/2899976222 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551d59a640 1 -- 172.21.15.64:0/2899976222 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3033608309 0 0) 0x7f55200cea30 con 0x7f55200b9a30 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551d59a640 1 -- 172.21.15.64:0/2899976222 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5508003520 con 0x7f55200b9a30 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551d59a640 1 -- 172.21.15.64:0/2899976222 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3362419367 0 0) 0x7f55200d2af0 con 0x7f55200b95a0 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551d59a640 1 -- 172.21.15.64:0/2899976222 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f55200cea30 con 0x7f55200b95a0 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551e59c640 1 -- 172.21.15.64:0/2899976222 >> v1:172.21.15.66:6789/0 conn(0x7f55200b95a0 legacy=0x7f55200b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551e59c640 1 -- 172.21.15.64:0/2899976222 >> v1:172.21.15.64:6789/0 conn(0x7f55200b9a30 legacy=0x7f55200d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:48.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.550+0000 7f551e59c640 1 -- 172.21.15.64:0/2899976222 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55200cf0e0 con 0x7f552015df00 2024-03-04T07:35:48.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.551+0000 7f551e59c640 1 --2- 172.21.15.64:0/2899976222 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200d25b0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f5510009dc0 tx=0x7f551002bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d056e1beaea522c2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:48.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.551+0000 7f551d59a640 1 -- 172.21.15.64:0/2899976222 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5510039070 con 0x7f552015df00 2024-03-04T07:35:48.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.551+0000 7f551d59a640 1 -- 172.21.15.64:0/2899976222 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f551002ccd0 con 0x7f552015df00 2024-03-04T07:35:48.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.551+0000 7f551d59a640 1 -- 172.21.15.64:0/2899976222 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5510034640 con 0x7f552015df00 2024-03-04T07:35:48.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.551+0000 7f5524834640 1 -- 172.21.15.64:0/2899976222 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 msgr2=0x7f55200d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:48.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.551+0000 7f5524834640 1 --2- 172.21.15.64:0/2899976222 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200d25b0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f5510009dc0 tx=0x7f551002bf20 comp rx=0 tx=0).stop 2024-03-04T07:35:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.552+0000 7f5524834640 1 -- 172.21.15.64:0/2899976222 shutdown_connections 2024-03-04T07:35:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.552+0000 7f5524834640 1 --2- 172.21.15.64:0/2899976222 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200d25b0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.552+0000 7f5524834640 1 -- 172.21.15.64:0/2899976222 >> 172.21.15.64:0/2899976222 conn(0x7f5520071960 msgr2=0x7f5520071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.552+0000 7f5524834640 1 -- 172.21.15.64:0/2899976222 shutdown_connections 2024-03-04T07:35:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.552+0000 7f5524834640 1 -- 172.21.15.64:0/2899976222 wait complete. 2024-03-04T07:35:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.553+0000 7f5524834640 1 Processor -- start 2024-03-04T07:35:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.553+0000 7f5524834640 1 -- start start 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f5524834640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55200b95a0 0x7f55201554c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f5524834640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200cd560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f5524834640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55200c9020 0x7f55200b54e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f5524834640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55200cec70 con 0x7f55200c9020 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f5524834640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55200ce750 con 0x7f55200b95a0 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f5524834640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f55200cedf0 con 0x7f552015df00 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55200c9020 0x7f55200b54e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55200c9020 0x7f55200b54e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37656/0 (socket says 172.21.15.64:37656) 2024-03-04T07:35:48.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 -- 172.21.15.64:0/532931080 learned_addr learned my addr 172.21.15.64:0/532931080 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551e59c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55200b95a0 0x7f55201554c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551dd9b640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200cd560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 -- 172.21.15.64:0/532931080 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 msgr2=0x7f55200cd560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200cd560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 -- 172.21.15.64:0/532931080 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55200b95a0 msgr2=0x7f55201554c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55200b95a0 0x7f55201554c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 -- 172.21.15.64:0/532931080 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5520155a00 con 0x7f55200c9020 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.554+0000 7f551ed9d640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55200c9020 0x7f55200b54e0 secure :-1 s=READY pgs=1707 cs=0 l=1 rev1=1 crypto rx=0x7f551800c400 tx=0x7f551800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.555+0000 7f550effd640 1 -- 172.21.15.64:0/532931080 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5518017070 con 0x7f55200c9020 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.555+0000 7f550effd640 1 -- 172.21.15.64:0/532931080 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5518005d00 con 0x7f55200c9020 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.555+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f55200c7f80 con 0x7f55200c9020 2024-03-04T07:35:48.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.555+0000 7f550effd640 1 -- 172.21.15.64:0/532931080 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5518007310 con 0x7f55200c9020 2024-03-04T07:35:48.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.555+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55200c8540 con 0x7f55200c9020 2024-03-04T07:35:48.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.556+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5520067ff0 con 0x7f55200c9020 2024-03-04T07:35:48.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.556+0000 7f550effd640 1 -- 172.21.15.64:0/532931080 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f551801a070 con 0x7f55200c9020 2024-03-04T07:35:48.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.556+0000 7f550effd640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f550006b4d0 0x7f550006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:48.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.557+0000 7f550effd640 1 -- 172.21.15.64:0/532931080 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f5518089680 con 0x7f55200c9020 2024-03-04T07:35:48.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.557+0000 7f551e59c640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f550006b4d0 0x7f550006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:48.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.557+0000 7f551e59c640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f550006b4d0 0x7f550006d990 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f5510036000 tx=0x7f5510030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:48.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.561+0000 7f550effd640 1 -- 172.21.15.64:0/532931080 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f5518057df0 con 0x7f55200c9020 2024-03-04T07:35:48.619 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:48.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.783+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f552006b780 con 0x7f55200c9020 2024-03-04T07:35:48.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.783+0000 7f550effd640 1 -- 172.21.15.64:0/532931080 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f551805be90 con 0x7f55200c9020 2024-03-04T07:35:48.784 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:48.784 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:48.784 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f550006b4d0 msgr2=0x7f550006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f550006b4d0 0x7f550006d990 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f5510036000 tx=0x7f5510030000 comp rx=0 tx=0).stop 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55200c9020 msgr2=0x7f55200b54e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55200c9020 0x7f55200b54e0 secure :-1 s=READY pgs=1707 cs=0 l=1 rev1=1 crypto rx=0x7f551800c400 tx=0x7f551800c8c0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 shutdown_connections 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f550006b4d0 0x7f550006d990 unknown :-1 s=CLOSED pgs=774 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f55200c9020 0x7f55200b54e0 unknown :-1 s=CLOSED pgs=1707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.786 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f552015df00 0x7f55200cd560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 --2- 172.21.15.64:0/532931080 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f55200b95a0 0x7f55201554c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:48.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 >> 172.21.15.64:0/532931080 conn(0x7f5520071960 msgr2=0x7f55200d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:48.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 shutdown_connections 2024-03-04T07:35:48.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:48.786+0000 7f5524834640 1 -- 172.21.15.64:0/532931080 wait complete. 2024-03-04T07:35:48.998 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:48.998 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:49.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:50.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.113+0000 7f4bd573e640 1 Processor -- start 2024-03-04T07:35:50.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.113+0000 7f4bd573e640 1 -- start start 2024-03-04T07:35:50.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.114+0000 7f4bd573e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4bd00d3400 0x7f4bd00b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.114+0000 7f4bd573e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4bd00ce610 con 0x7f4bd0071540 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.114+0000 7f4bd573e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4bd00d2920 con 0x7f4bd00b2b50 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.114+0000 7f4bd573e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4bd00d2f70 con 0x7f4bd00d3400 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.114+0000 7f4bce7fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4bd00d3400 0x7f4bd00b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.114+0000 7f4bceffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4bd0071540 0x7f4bd0071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40034/0 (socket says 172.21.15.64:40034) 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.114+0000 7f4bceffd640 1 -- 172.21.15.64:0/2656240431 learned_addr learned my addr 172.21.15.64:0/2656240431 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.115+0000 7f4bce7fc640 1 -- 172.21.15.64:0/2656240431 >> v1:172.21.15.66:6789/0 conn(0x7f4bd00b2b50 legacy=0x7f4bd00b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.115+0000 7f4bce7fc640 1 -- 172.21.15.64:0/2656240431 >> v1:172.21.15.64:6789/0 conn(0x7f4bd0071540 legacy=0x7f4bd0071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.115+0000 7f4bcdffb640 1 -- 172.21.15.64:0/2656240431 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3561662088 0 0) 0x7f4bd00d2920 con 0x7f4bd00b2b50 2024-03-04T07:35:50.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.115+0000 7f4bce7fc640 1 -- 172.21.15.64:0/2656240431 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4bd00d2c00 con 0x7f4bd00d3400 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.115+0000 7f4bce7fc640 1 --2- 172.21.15.64:0/2656240431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00d3400 0x7f4bd00b30a0 secure :-1 s=READY pgs=1709 cs=0 l=1 rev1=1 crypto rx=0x7f4bb80097a0 tx=0x7f4bb802b790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=21e4698f2e0b5666 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bcdffb640 1 -- 172.21.15.64:0/2656240431 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb8037070 con 0x7f4bd00d3400 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bcdffb640 1 -- 172.21.15.64:0/2656240431 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4bb8030710 con 0x7f4bd00d3400 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bcdffb640 1 -- 172.21.15.64:0/2656240431 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb8030ae0 con 0x7f4bd00d3400 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bd573e640 1 -- 172.21.15.64:0/2656240431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00d3400 msgr2=0x7f4bd00b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bd573e640 1 --2- 172.21.15.64:0/2656240431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00d3400 0x7f4bd00b30a0 secure :-1 s=READY pgs=1709 cs=0 l=1 rev1=1 crypto rx=0x7f4bb80097a0 tx=0x7f4bb802b790 comp rx=0 tx=0).stop 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bd573e640 1 -- 172.21.15.64:0/2656240431 shutdown_connections 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bd573e640 1 --2- 172.21.15.64:0/2656240431 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00d3400 0x7f4bd00b30a0 unknown :-1 s=CLOSED pgs=1709 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bd573e640 1 -- 172.21.15.64:0/2656240431 >> 172.21.15.64:0/2656240431 conn(0x7f4bd00d3ca0 msgr2=0x7f4bd00d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bd573e640 1 -- 172.21.15.64:0/2656240431 shutdown_connections 2024-03-04T07:35:50.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.116+0000 7f4bd573e640 1 -- 172.21.15.64:0/2656240431 wait complete. 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 Processor -- start 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 -- start start 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bd0071540 0x7f4bd00b52c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00b2b50 0x7f4bd0158740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bd00d3400 0x7f4bd0157270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4bd015e370 con 0x7f4bd00b2b50 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4bd00d4be0 con 0x7f4bd0071540 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bd573e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4bd00b65c0 con 0x7f4bd00d3400 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bce7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00b2b50 0x7f4bd0158740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:50.120 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bce7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00b2b50 0x7f4bd0158740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37676/0 (socket says 172.21.15.64:37676) 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bce7fc640 1 -- 172.21.15.64:0/1296100379 learned_addr learned my addr 172.21.15.64:0/1296100379 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bceffd640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bd0071540 0x7f4bd00b52c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.118+0000 7f4bcf7fe640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bd00d3400 0x7f4bd0157270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bce7fc640 1 -- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bd00d3400 msgr2=0x7f4bd0157270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bce7fc640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bd00d3400 0x7f4bd0157270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bce7fc640 1 -- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bd0071540 msgr2=0x7f4bd00b52c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bce7fc640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bd0071540 0x7f4bd00b52c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bce7fc640 1 -- 172.21.15.64:0/1296100379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4bd01577b0 con 0x7f4bd00b2b50 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bce7fc640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00b2b50 0x7f4bd0158740 secure :-1 s=READY pgs=1710 cs=0 l=1 rev1=1 crypto rx=0x7f4bb8000c00 tx=0x7f4bb80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bc77fe640 1 -- 172.21.15.64:0/1296100379 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb8042070 con 0x7f4bd00b2b50 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bc77fe640 1 -- 172.21.15.64:0/1296100379 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4bb8032a40 con 0x7f4bd00b2b50 2024-03-04T07:35:50.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bc77fe640 1 -- 172.21.15.64:0/1296100379 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb8037040 con 0x7f4bd00b2b50 2024-03-04T07:35:50.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4bd0158e20 con 0x7f4bd00b2b50 2024-03-04T07:35:50.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.119+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4bd01593e0 con 0x7f4bd00b2b50 2024-03-04T07:35:50.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.120+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4bd0068700 con 0x7f4bd00b2b50 2024-03-04T07:35:50.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.121+0000 7f4bc77fe640 1 -- 172.21.15.64:0/1296100379 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4bb8045070 con 0x7f4bd00b2b50 2024-03-04T07:35:50.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.121+0000 7f4bc77fe640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4bb006b500 0x7f4bb006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:50.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.121+0000 7f4bc77fe640 1 -- 172.21.15.64:0/1296100379 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4bb80ac1a0 con 0x7f4bd00b2b50 2024-03-04T07:35:50.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.121+0000 7f4bceffd640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4bb006b500 0x7f4bb006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:50.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.122+0000 7f4bceffd640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4bb006b500 0x7f4bb006d9c0 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f4bc8000f60 tx=0x7f4bc8007e20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:50.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.125+0000 7f4bc77fe640 1 -- 172.21.15.64:0/1296100379 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4bd01577b0 con 0x7f4bd00b2b50 2024-03-04T07:35:50.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.343+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4bd00c7030 con 0x7f4bd00b2b50 2024-03-04T07:35:50.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.344+0000 7f4bc77fe640 1 -- 172.21.15.64:0/1296100379 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4bb807ab40 con 0x7f4bd00b2b50 2024-03-04T07:35:50.344 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:50.344 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:50.344 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:50.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4bb006b500 msgr2=0x7f4bb006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:50.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4bb006b500 0x7f4bb006d9c0 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f4bc8000f60 tx=0x7f4bc8007e20 comp rx=0 tx=0).stop 2024-03-04T07:35:50.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00b2b50 msgr2=0x7f4bd0158740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:50.346 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00b2b50 0x7f4bd0158740 secure :-1 s=READY pgs=1710 cs=0 l=1 rev1=1 crypto rx=0x7f4bb8000c00 tx=0x7f4bb80046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 shutdown_connections 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4bb006b500 0x7f4bb006d9c0 unknown :-1 s=CLOSED pgs=775 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bd00d3400 0x7f4bd0157270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bd00b2b50 0x7f4bd0158740 unknown :-1 s=CLOSED pgs=1710 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 --2- 172.21.15.64:0/1296100379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bd0071540 0x7f4bd00b52c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 >> 172.21.15.64:0/1296100379 conn(0x7f4bd00d3ca0 msgr2=0x7f4bd00ceb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 shutdown_connections 2024-03-04T07:35:50.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:50.346+0000 7f4bd573e640 1 -- 172.21.15.64:0/1296100379 wait complete. 2024-03-04T07:35:50.558 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:50.558 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:50.621 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:51.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.673+0000 7f67c595c640 1 Processor -- start 2024-03-04T07:35:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.673+0000 7f67c595c640 1 -- start start 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.674+0000 7f67c595c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f67c00d3400 0x7f67c00b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.674+0000 7f67c595c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67c00ce610 con 0x7f67c00b2b50 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.674+0000 7f67c595c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f67c00d2920 con 0x7f67c0071540 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.674+0000 7f67c595c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f67c00d2f70 con 0x7f67c00d3400 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.674+0000 7f67bf7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f67c00b2b50 0x7f67c00b6e20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:40040/0 (socket says 172.21.15.64:40040) 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.674+0000 7f67bf7fe640 1 -- 172.21.15.64:0/2116317672 learned_addr learned my addr 172.21.15.64:0/2116317672 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.674+0000 7f67be7fc640 1 --2- 172.21.15.64:0/2116317672 >> v2:172.21.15.66:3300/0 conn(0x7f67c00d3400 0x7f67c00b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.675+0000 7f67bdffb640 1 -- 172.21.15.64:0/2116317672 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2665984191 0 0) 0x7f67c00ce610 con 0x7f67c00b2b50 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.675+0000 7f67bdffb640 1 -- 172.21.15.64:0/2116317672 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f67ac003520 con 0x7f67c00b2b50 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.675+0000 7f67be7fc640 1 -- 172.21.15.64:0/2116317672 >> v1:172.21.15.66:6789/0 conn(0x7f67c0071540 legacy=0x7f67c0071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.675+0000 7f67be7fc640 1 -- 172.21.15.64:0/2116317672 >> v1:172.21.15.64:6789/0 conn(0x7f67c00b2b50 legacy=0x7f67c00b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:51.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.675+0000 7f67be7fc640 1 -- 172.21.15.64:0/2116317672 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67c00d2c00 con 0x7f67c00d3400 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67be7fc640 1 --2- 172.21.15.64:0/2116317672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c00b30a0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f67a80097a0 tx=0x7f67a802b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b9ee50d23241e042 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67bdffb640 1 -- 172.21.15.64:0/2116317672 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67a8034070 con 0x7f67c00d3400 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67bdffb640 1 -- 172.21.15.64:0/2116317672 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f67a8030610 con 0x7f67c00d3400 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67bdffb640 1 -- 172.21.15.64:0/2116317672 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67a80309c0 con 0x7f67c00d3400 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67c595c640 1 -- 172.21.15.64:0/2116317672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 msgr2=0x7f67c00b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67c595c640 1 --2- 172.21.15.64:0/2116317672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c00b30a0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f67a80097a0 tx=0x7f67a802b960 comp rx=0 tx=0).stop 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67c595c640 1 -- 172.21.15.64:0/2116317672 shutdown_connections 2024-03-04T07:35:51.677 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67c595c640 1 --2- 172.21.15.64:0/2116317672 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c00b30a0 unknown :-1 s=CLOSED pgs=624 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67c595c640 1 -- 172.21.15.64:0/2116317672 >> 172.21.15.64:0/2116317672 conn(0x7f67c00d3ca0 msgr2=0x7f67c00d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:51.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67c595c640 1 -- 172.21.15.64:0/2116317672 shutdown_connections 2024-03-04T07:35:51.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.676+0000 7f67c595c640 1 -- 172.21.15.64:0/2116317672 wait complete. 2024-03-04T07:35:51.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.678+0000 7f67c595c640 1 Processor -- start 2024-03-04T07:35:51.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.678+0000 7f67c595c640 1 -- start start 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67c595c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c0071540 0x7f67c014c7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67c595c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c00b2b50 0x7f67c01f6d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67c595c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c01f72c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67c595c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67c015e370 con 0x7f67c00b2b50 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67c595c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f67c00d4be0 con 0x7f67c00d3400 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67c595c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f67c014d990 con 0x7f67c0071540 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c00b2b50 0x7f67c01f6d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c00b2b50 0x7f67c01f6d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37688/0 (socket says 172.21.15.64:37688) 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 -- 172.21.15.64:0/3951645101 learned_addr learned my addr 172.21.15.64:0/3951645101 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67bf7fe640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c01f72c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67beffd640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c0071540 0x7f67c014c7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 -- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c0071540 msgr2=0x7f67c014c7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c0071540 0x7f67c014c7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 -- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 msgr2=0x7f67c01f72c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c01f72c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 -- 172.21.15.64:0/3951645101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67c00c9d20 con 0x7f67c00b2b50 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67bf7fe640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c01f72c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67be7fc640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c00b2b50 0x7f67c01f6d80 secure :-1 s=READY pgs=1712 cs=0 l=1 rev1=1 crypto rx=0x7f67a8000c00 tx=0x7f67a80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67b7fff640 1 -- 172.21.15.64:0/3951645101 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67a8047070 con 0x7f67c00b2b50 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67b7fff640 1 -- 172.21.15.64:0/3951645101 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f67a8030410 con 0x7f67c00b2b50 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f67c014d340 con 0x7f67c00b2b50 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.679+0000 7f67b7fff640 1 -- 172.21.15.64:0/3951645101 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f67a8034040 con 0x7f67c00b2b50 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.680+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f67c014cd20 con 0x7f67c00b2b50 2024-03-04T07:35:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.681+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f67c0068700 con 0x7f67c00b2b50 2024-03-04T07:35:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.681+0000 7f67b7fff640 1 -- 172.21.15.64:0/3951645101 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f67a804a070 con 0x7f67c00b2b50 2024-03-04T07:35:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.681+0000 7f67b7fff640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67a006b4d0 0x7f67a006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:51.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.682+0000 7f67b7fff640 1 -- 172.21.15.64:0/3951645101 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f67a80abb10 con 0x7f67c00b2b50 2024-03-04T07:35:51.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.682+0000 7f67beffd640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67a006b4d0 0x7f67a006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:51.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.682+0000 7f67beffd640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67a006b4d0 0x7f67a006d990 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f67b0001f10 tx=0x7f67b0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:51.685 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.686+0000 7f67b7fff640 1 -- 172.21.15.64:0/3951645101 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f67a80305e0 con 0x7f67c00b2b50 2024-03-04T07:35:51.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.909+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f67c015ed00 con 0x7f67c00b2b50 2024-03-04T07:35:51.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.909+0000 7f67b7fff640 1 -- 172.21.15.64:0/3951645101 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f67a807a300 con 0x7f67c00b2b50 2024-03-04T07:35:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:51.910 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:51.910 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:51.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67a006b4d0 msgr2=0x7f67a006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:51.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67a006b4d0 0x7f67a006d990 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f67b0001f10 tx=0x7f67b0009040 comp rx=0 tx=0).stop 2024-03-04T07:35:51.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c00b2b50 msgr2=0x7f67c01f6d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:51.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c00b2b50 0x7f67c01f6d80 secure :-1 s=READY pgs=1712 cs=0 l=1 rev1=1 crypto rx=0x7f67a8000c00 tx=0x7f67a80046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 shutdown_connections 2024-03-04T07:35:51.912 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f67a006b4d0 0x7f67a006d990 unknown :-1 s=CLOSED pgs=776 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f67c00d3400 0x7f67c01f72c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f67c00b2b50 0x7f67c01f6d80 unknown :-1 s=CLOSED pgs=1712 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 --2- 172.21.15.64:0/3951645101 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f67c0071540 0x7f67c014c7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:51.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 >> 172.21.15.64:0/3951645101 conn(0x7f67c00d3ca0 msgr2=0x7f67c015f0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:51.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 shutdown_connections 2024-03-04T07:35:51.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:51.912+0000 7f67c595c640 1 -- 172.21.15.64:0/3951645101 wait complete. 2024-03-04T07:35:52.124 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:52.124 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:52.624 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:53.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:53.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.239+0000 7f2eeed47640 1 Processor -- start 2024-03-04T07:35:53.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.239+0000 7f2eeed47640 1 -- start start 2024-03-04T07:35:53.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eeed47640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:53.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eeed47640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:53.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eeed47640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2ee80cea30 con 0x7f2ee80b9a30 2024-03-04T07:35:53.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eeed47640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2ee80d2af0 con 0x7f2ee815df00 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eeed47640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2ee80cf3c0 con 0x7f2ee80b95a0 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eecae3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eecae3640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37692/0 (socket says 172.21.15.64:37692) 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eecae3640 1 -- 172.21.15.64:0/2481983886 learned_addr learned my addr 172.21.15.64:0/2481983886 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2481983886 >> v2:172.21.15.66:3300/0 conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eecae3640 1 -- 172.21.15.64:0/2481983886 >> v2:172.21.15.66:3300/0 conn(0x7f2ee80b95a0 msgr2=0x7f2ee80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eecae3640 1 --2- 172.21.15.64:0/2481983886 >> v2:172.21.15.66:3300/0 conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eecae3640 1 -- 172.21.15.64:0/2481983886 >> v1:172.21.15.66:6789/0 conn(0x7f2ee80b9a30 legacy=0x7f2ee80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:53.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.240+0000 7f2eecae3640 1 -- 172.21.15.64:0/2481983886 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ee80cf0e0 con 0x7f2ee815df00 2024-03-04T07:35:53.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.241+0000 7f2eecae3640 1 --2- 172.21.15.64:0/2481983886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 0x7f2ee80d25b0 secure :-1 s=READY pgs=1713 cs=0 l=1 rev1=1 crypto rx=0x7f2ee80d1760 tx=0x7f2edc02b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=248ac0ac90b4a025 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:53.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.241+0000 7f2ee77fe640 1 -- 172.21.15.64:0/2481983886 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2edc034070 con 0x7f2ee815df00 2024-03-04T07:35:53.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.241+0000 7f2ee77fe640 1 -- 172.21.15.64:0/2481983886 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2edc030860 con 0x7f2ee815df00 2024-03-04T07:35:53.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.241+0000 7f2ee77fe640 1 -- 172.21.15.64:0/2481983886 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2edc030c30 con 0x7f2ee815df00 2024-03-04T07:35:53.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 -- 172.21.15.64:0/2481983886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 msgr2=0x7f2ee80d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:53.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2481983886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 0x7f2ee80d25b0 secure :-1 s=READY pgs=1713 cs=0 l=1 rev1=1 crypto rx=0x7f2ee80d1760 tx=0x7f2edc02b9a0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 -- 172.21.15.64:0/2481983886 shutdown_connections 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2481983886 >> v2:172.21.15.66:3300/0 conn(0x7f2ee80b95a0 0x7f2ee80b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2481983886 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee815df00 0x7f2ee80d25b0 unknown :-1 s=CLOSED pgs=1713 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 -- 172.21.15.64:0/2481983886 >> 172.21.15.64:0/2481983886 conn(0x7f2ee8071960 msgr2=0x7f2ee8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 -- 172.21.15.64:0/2481983886 shutdown_connections 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.242+0000 7f2eeed47640 1 -- 172.21.15.64:0/2481983886 wait complete. 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.243+0000 7f2eeed47640 1 Processor -- start 2024-03-04T07:35:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.243+0000 7f2eeed47640 1 -- start start 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eeed47640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b95a0 0x7f2ee80cd6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eeed47640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee815df00 0x7f2ee80c8d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eeed47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80cb760 0x7f2ee80b4a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eeed47640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ee80cec70 con 0x7f2ee80b95a0 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eeed47640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ee80ce750 con 0x7f2ee815df00 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eeed47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ee80cedf0 con 0x7f2ee80cb760 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eecae3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b95a0 0x7f2ee80cd6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eecae3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b95a0 0x7f2ee80cd6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37702/0 (socket says 172.21.15.64:37702) 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eecae3640 1 -- 172.21.15.64:0/2808602383 learned_addr learned my addr 172.21.15.64:0/2808602383 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:53.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee815df00 0x7f2ee80c8d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.244+0000 7f2eed2e4640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80cb760 0x7f2ee80b4a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eecae3640 1 -- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80cb760 msgr2=0x7f2ee80b4a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eecae3640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80cb760 0x7f2ee80b4a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eecae3640 1 -- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee815df00 msgr2=0x7f2ee80c8d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eecae3640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee815df00 0x7f2ee80c8d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eecae3640 1 -- 172.21.15.64:0/2808602383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ee80c9260 con 0x7f2ee80b95a0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eecae3640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b95a0 0x7f2ee80cd6a0 secure :-1 s=READY pgs=1714 cs=0 l=1 rev1=1 crypto rx=0x7f2edc000c00 tx=0x7f2edc0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2ee57fa640 1 -- 172.21.15.64:0/2808602383 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2edc034040 con 0x7f2ee80b95a0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2ee57fa640 1 -- 172.21.15.64:0/2808602383 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2edc039d90 con 0x7f2ee80b95a0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ee80ccf30 con 0x7f2ee80b95a0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2ee57fa640 1 -- 172.21.15.64:0/2808602383 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2edc043610 con 0x7f2ee80b95a0 2024-03-04T07:35:53.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.245+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ee80c7f60 con 0x7f2ee80b95a0 2024-03-04T07:35:53.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.246+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ee8067ff0 con 0x7f2ee80b95a0 2024-03-04T07:35:53.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.246+0000 7f2ee57fa640 1 -- 172.21.15.64:0/2808602383 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2edc037070 con 0x7f2ee80b95a0 2024-03-04T07:35:53.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.247+0000 7f2ee57fa640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ecc06b4d0 0x7f2ecc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:53.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.247+0000 7f2ee57fa640 1 -- 172.21.15.64:0/2808602383 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f2edc0ac1f0 con 0x7f2ee80b95a0 2024-03-04T07:35:53.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.247+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ecc06b4d0 0x7f2ecc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:53.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.247+0000 7f2ee7fff640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ecc06b4d0 0x7f2ecc06d990 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f2ee81610b0 tx=0x7f2ed4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:53.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.251+0000 7f2ee57fa640 1 -- 172.21.15.64:0/2808602383 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2ee80c9260 con 0x7f2ee80b95a0 2024-03-04T07:35:53.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.471+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2ee80d30e0 con 0x7f2ee80b95a0 2024-03-04T07:35:53.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.471+0000 7f2ee57fa640 1 -- 172.21.15.64:0/2808602383 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f2edc03e070 con 0x7f2ee80b95a0 2024-03-04T07:35:53.471 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:53.472 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:53.472 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:53.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ecc06b4d0 msgr2=0x7f2ecc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:53.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ecc06b4d0 0x7f2ecc06d990 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f2ee81610b0 tx=0x7f2ed4009190 comp rx=0 tx=0).stop 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b95a0 msgr2=0x7f2ee80cd6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b95a0 0x7f2ee80cd6a0 secure :-1 s=READY pgs=1714 cs=0 l=1 rev1=1 crypto rx=0x7f2edc000c00 tx=0x7f2edc0046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 shutdown_connections 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2ecc06b4d0 0x7f2ecc06d990 unknown :-1 s=CLOSED pgs=777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2ee80cb760 0x7f2ee80b4a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2ee815df00 0x7f2ee80c8d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 --2- 172.21.15.64:0/2808602383 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2ee80b95a0 0x7f2ee80cd6a0 unknown :-1 s=CLOSED pgs=1714 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 >> 172.21.15.64:0/2808602383 conn(0x7f2ee8071960 msgr2=0x7f2ee815bb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 shutdown_connections 2024-03-04T07:35:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:53.473+0000 7f2eeed47640 1 -- 172.21.15.64:0/2808602383 wait complete. 2024-03-04T07:35:53.685 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:53.686 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:54.626 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:54.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:54.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.801+0000 7f616ce88640 1 Processor -- start 2024-03-04T07:35:54.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.801+0000 7f616ce88640 1 -- start start 2024-03-04T07:35:54.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.802+0000 7f616ce88640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f616815df70 0x7f61680d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:54.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.802+0000 7f616ce88640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f61680b6f20 con 0x7f61680d18c0 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.802+0000 7f616ce88640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f61680d3030 con 0x7f6168071c50 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.802+0000 7f616ce88640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f61680d3310 con 0x7f616815df70 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.802+0000 7f616659c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f616815df70 0x7f61680d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.802+0000 7f6165d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f61680d18c0 0x7f6168071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41376/0 (socket says 172.21.15.64:41376) 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.802+0000 7f6165d9b640 1 -- 172.21.15.64:0/2393205126 learned_addr learned my addr 172.21.15.64:0/2393205126 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.803+0000 7f616559a640 1 -- 172.21.15.64:0/2393205126 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3532733109 0 0) 0x7f61680b6f20 con 0x7f61680d18c0 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.803+0000 7f616559a640 1 -- 172.21.15.64:0/2393205126 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6154003520 con 0x7f61680d18c0 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.803+0000 7f616559a640 1 -- 172.21.15.64:0/2393205126 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 518459312 0 0) 0x7f61680d3030 con 0x7f6168071c50 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.803+0000 7f616659c640 1 -- 172.21.15.64:0/2393205126 >> v1:172.21.15.66:6789/0 conn(0x7f6168071c50 legacy=0x7f61680b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.803+0000 7f616659c640 1 -- 172.21.15.64:0/2393205126 >> v1:172.21.15.64:6789/0 conn(0x7f61680d18c0 legacy=0x7f6168071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:54.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.803+0000 7f616659c640 1 -- 172.21.15.64:0/2393205126 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61680d35f0 con 0x7f616815df70 2024-03-04T07:35:54.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.803+0000 7f616659c640 1 --2- 172.21.15.64:0/2393205126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61680d27b0 secure :-1 s=READY pgs=1716 cs=0 l=1 rev1=1 crypto rx=0x7f6158009d90 tx=0x7f615802bfd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e47801e9f3af0821 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616559a640 1 -- 172.21.15.64:0/2393205126 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6158039070 con 0x7f616815df70 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616559a640 1 -- 172.21.15.64:0/2393205126 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f615802cd80 con 0x7f616815df70 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616559a640 1 -- 172.21.15.64:0/2393205126 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6158034640 con 0x7f616815df70 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616ce88640 1 -- 172.21.15.64:0/2393205126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 msgr2=0x7f61680d27b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616ce88640 1 --2- 172.21.15.64:0/2393205126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61680d27b0 secure :-1 s=READY pgs=1716 cs=0 l=1 rev1=1 crypto rx=0x7f6158009d90 tx=0x7f615802bfd0 comp rx=0 tx=0).stop 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616ce88640 1 -- 172.21.15.64:0/2393205126 shutdown_connections 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616ce88640 1 --2- 172.21.15.64:0/2393205126 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61680d27b0 unknown :-1 s=CLOSED pgs=1716 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616ce88640 1 -- 172.21.15.64:0/2393205126 >> 172.21.15.64:0/2393205126 conn(0x7f61680d3cf0 msgr2=0x7f61680d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616ce88640 1 -- 172.21.15.64:0/2393205126 shutdown_connections 2024-03-04T07:35:54.805 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.804+0000 7f616ce88640 1 -- 172.21.15.64:0/2393205126 wait complete. 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 Processor -- start 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 -- start start 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6168071c50 0x7f61680ccf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680d18c0 0x7f61680cd4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61681f3c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f616815e350 con 0x7f616815df70 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61680d2cf0 con 0x7f6168071c50 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.806+0000 7f616ce88640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f616815e4d0 con 0x7f61680d18c0 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f616659c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6168071c50 0x7f61680ccf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:54.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61681f3c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f616659c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6168071c50 0x7f61680ccf90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:59528/0 (socket says 172.21.15.64:59528) 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61681f3c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56074/0 (socket says 172.21.15.64:56074) 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f616659c640 1 -- 172.21.15.64:0/2192146670 learned_addr learned my addr 172.21.15.64:0/2192146670 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6165d9b640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680d18c0 0x7f61680cd4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 -- 172.21.15.64:0/2192146670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680d18c0 msgr2=0x7f61680cd4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680d18c0 0x7f61680cd4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 -- 172.21.15.64:0/2192146670 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6168071c50 msgr2=0x7f61680ccf90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6168071c50 0x7f61680ccf90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 -- 172.21.15.64:0/2192146670 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61680b55b0 con 0x7f616815df70 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f6166d9d640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61681f3c80 secure :-1 s=READY pgs=1717 cs=0 l=1 rev1=1 crypto rx=0x7f616000c3e0 tx=0x7f616000c8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f615effd640 1 -- 172.21.15.64:0/2192146670 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6160017070 con 0x7f616815df70 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f615effd640 1 -- 172.21.15.64:0/2192146670 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f61600035f0 con 0x7f616815df70 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61680b5890 con 0x7f616815df70 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f615effd640 1 -- 172.21.15.64:0/2192146670 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61600039e0 con 0x7f616815df70 2024-03-04T07:35:54.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.807+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61680b5e50 con 0x7f616815df70 2024-03-04T07:35:54.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.808+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f616815bb20 con 0x7f616815df70 2024-03-04T07:35:54.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.809+0000 7f615effd640 1 -- 172.21.15.64:0/2192146670 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f616001a070 con 0x7f616815df70 2024-03-04T07:35:54.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.809+0000 7f615effd640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:54.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.809+0000 7f615effd640 1 -- 172.21.15.64:0/2192146670 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f61600894f0 con 0x7f616815df70 2024-03-04T07:35:54.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.809+0000 7f616659c640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:54.811 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.810+0000 7f616659c640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f6158000f80 tx=0x7f6158030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:54.813 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:54.814+0000 7f615effd640 1 -- 172.21.15.64:0/2192146670 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f616815bb20 con 0x7f616815df70 2024-03-04T07:35:55.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.032+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f61680d1e20 con 0x7f616815df70 2024-03-04T07:35:55.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.032+0000 7f615effd640 1 -- 172.21.15.64:0/2192146670 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f6160057ce0 con 0x7f616815df70 2024-03-04T07:35:55.032 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:55.032 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 msgr2=0x7f614806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f6158000f80 tx=0x7f6158030000 comp rx=0 tx=0).stop 2024-03-04T07:35:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 msgr2=0x7f61681f3c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61681f3c80 secure :-1 s=READY pgs=1717 cs=0 l=1 rev1=1 crypto rx=0x7f616000c3e0 tx=0x7f616000c8a0 comp rx=0 tx=0).stop 2024-03-04T07:35:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 shutdown_connections 2024-03-04T07:35:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f614806b4d0 0x7f614806d990 unknown :-1 s=CLOSED pgs=778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:55.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f616815df70 0x7f61681f3c80 unknown :-1 s=CLOSED pgs=1717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61680d18c0 0x7f61680cd4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 --2- 172.21.15.64:0/2192146670 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6168071c50 0x7f61680ccf90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 >> 172.21.15.64:0/2192146670 conn(0x7f61680d3cf0 msgr2=0x7f61680b8200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 shutdown_connections 2024-03-04T07:35:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:55.034+0000 7f616ce88640 1 -- 172.21.15.64:0/2192146670 wait complete. 2024-03-04T07:35:55.246 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:55.246 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:56.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:56.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.360+0000 7f47e244e640 1 Processor -- start 2024-03-04T07:35:56.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.360+0000 7f47e244e640 1 -- start start 2024-03-04T07:35:56.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47e244e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc158520 0x7f47dc06d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47e244e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f47dc06dd60 0x7f47dc0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47e244e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f47dc0af080 con 0x7f47dc0aec40 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47e244e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f47dc06e1c0 con 0x7f47dc06dd60 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47e244e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47dc071b70 con 0x7f47dc158520 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47db7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f47dc06dd60 0x7f47dc0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47db7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f47dc06dd60 0x7f47dc0ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56090/0 (socket says 172.21.15.64:56090) 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47dbfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc158520 0x7f47dc06d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47db7fe640 1 -- 172.21.15.64:0/697635461 learned_addr learned my addr 172.21.15.64:0/697635461 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47e09eb640 1 --1- 172.21.15.64:0/697635461 >> v1:172.21.15.66:6789/0 conn(0x7f47dc0aec40 0x7f47dc15e670 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:44570/0 (socket says 172.21.15.64:44570) 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47db7fe640 1 -- 172.21.15.64:0/697635461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc158520 msgr2=0x7f47dc06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:56.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47db7fe640 1 --2- 172.21.15.64:0/697635461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc158520 0x7f47dc06d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47db7fe640 1 -- 172.21.15.64:0/697635461 >> v1:172.21.15.66:6789/0 conn(0x7f47dc0aec40 legacy=0x7f47dc15e670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:56.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47db7fe640 1 -- 172.21.15.64:0/697635461 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47dc071240 con 0x7f47dc06dd60 2024-03-04T07:35:56.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.361+0000 7f47daffd640 1 -- 172.21.15.64:0/697635461 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 218154774 0 0) 0x7f47dc0af080 con 0x7f47dc0aec40 2024-03-04T07:35:56.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.362+0000 7f47db7fe640 1 --2- 172.21.15.64:0/697635461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc06dd60 0x7f47dc0ae700 secure :-1 s=READY pgs=1718 cs=0 l=1 rev1=1 crypto rx=0x7f47cc009870 tx=0x7f47cc02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1236396083094bba server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:56.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.362+0000 7f47daffd640 1 -- 172.21.15.64:0/697635461 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47cc034070 con 0x7f47dc06dd60 2024-03-04T07:35:56.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.362+0000 7f47daffd640 1 -- 172.21.15.64:0/697635461 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f47cc030840 con 0x7f47dc06dd60 2024-03-04T07:35:56.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.362+0000 7f47daffd640 1 -- 172.21.15.64:0/697635461 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47cc030c10 con 0x7f47dc06dd60 2024-03-04T07:35:56.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 -- 172.21.15.64:0/697635461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc06dd60 msgr2=0x7f47dc0ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:56.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 --2- 172.21.15.64:0/697635461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc06dd60 0x7f47dc0ae700 secure :-1 s=READY pgs=1718 cs=0 l=1 rev1=1 crypto rx=0x7f47cc009870 tx=0x7f47cc02b980 comp rx=0 tx=0).stop 2024-03-04T07:35:56.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 -- 172.21.15.64:0/697635461 shutdown_connections 2024-03-04T07:35:56.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 --2- 172.21.15.64:0/697635461 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc06dd60 0x7f47dc0ae700 unknown :-1 s=CLOSED pgs=1718 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 --2- 172.21.15.64:0/697635461 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc158520 0x7f47dc06d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 -- 172.21.15.64:0/697635461 >> 172.21.15.64:0/697635461 conn(0x7f47dc066d00 msgr2=0x7f47dc0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:56.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 -- 172.21.15.64:0/697635461 shutdown_connections 2024-03-04T07:35:56.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.363+0000 7f47e244e640 1 -- 172.21.15.64:0/697635461 wait complete. 2024-03-04T07:35:56.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.364+0000 7f47e244e640 1 Processor -- start 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e244e640 1 -- start start 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e244e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc06dd60 0x7f47dc151b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e244e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc0aec40 0x7f47dc140040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e244e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47dc158520 0x7f47dc15d660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e244e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47dc0b4650 con 0x7f47dc0aec40 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e244e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47dc071800 con 0x7f47dc158520 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e244e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f47dc072000 con 0x7f47dc06dd60 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47db7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc0aec40 0x7f47dc140040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47db7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc0aec40 0x7f47dc140040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56100/0 (socket says 172.21.15.64:56100) 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47db7fe640 1 -- 172.21.15.64:0/3757996763 learned_addr learned my addr 172.21.15.64:0/3757996763 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47dbfff640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc06dd60 0x7f47dc151b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.365+0000 7f47e09eb640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47dc158520 0x7f47dc15d660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:56.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47db7fe640 1 -- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc06dd60 msgr2=0x7f47dc151b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47db7fe640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc06dd60 0x7f47dc151b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47db7fe640 1 -- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47dc158520 msgr2=0x7f47dc15d660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47db7fe640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47dc158520 0x7f47dc15d660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47db7fe640 1 -- 172.21.15.64:0/3757996763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47dc15c8a0 con 0x7f47dc0aec40 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47db7fe640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc0aec40 0x7f47dc140040 secure :-1 s=READY pgs=1719 cs=0 l=1 rev1=1 crypto rx=0x7f47cc000c00 tx=0x7f47cc004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47d8ff9640 1 -- 172.21.15.64:0/3757996763 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47cc048070 con 0x7f47dc0aec40 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47d8ff9640 1 -- 172.21.15.64:0/3757996763 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f47cc030640 con 0x7f47dc0aec40 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47dc15cb80 con 0x7f47dc0aec40 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47d8ff9640 1 -- 172.21.15.64:0/3757996763 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47cc034040 con 0x7f47dc0aec40 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.366+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47dc140580 con 0x7f47dc0aec40 2024-03-04T07:35:56.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.367+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f47dc067ff0 con 0x7f47dc0aec40 2024-03-04T07:35:56.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.367+0000 7f47d8ff9640 1 -- 172.21.15.64:0/3757996763 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f47cc04b070 con 0x7f47dc0aec40 2024-03-04T07:35:56.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.368+0000 7f47d8ff9640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47c806b4d0 0x7f47c806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:56.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.368+0000 7f47d8ff9640 1 -- 172.21.15.64:0/3757996763 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f47cc0ac010 con 0x7f47dc0aec40 2024-03-04T07:35:56.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.368+0000 7f47dbfff640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47c806b4d0 0x7f47c806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:56.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.368+0000 7f47dbfff640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47c806b4d0 0x7f47c806d990 secure :-1 s=READY pgs=779 cs=0 l=1 rev1=1 crypto rx=0x7f47d0009aa0 tx=0x7f47d0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:56.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.372+0000 7f47d8ff9640 1 -- 172.21.15.64:0/3757996763 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f47cc07a800 con 0x7f47dc0aec40 2024-03-04T07:35:56.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.595+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f47dc0b31f0 con 0x7f47dc0aec40 2024-03-04T07:35:56.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.595+0000 7f47d8ff9640 1 -- 172.21.15.64:0/3757996763 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f47cc07e8a0 con 0x7f47dc0aec40 2024-03-04T07:35:56.595 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:56.595 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:56.596 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:56.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47c806b4d0 msgr2=0x7f47c806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:56.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47c806b4d0 0x7f47c806d990 secure :-1 s=READY pgs=779 cs=0 l=1 rev1=1 crypto rx=0x7f47d0009aa0 tx=0x7f47d0009190 comp rx=0 tx=0).stop 2024-03-04T07:35:56.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc0aec40 msgr2=0x7f47dc140040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:56.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc0aec40 0x7f47dc140040 secure :-1 s=READY pgs=1719 cs=0 l=1 rev1=1 crypto rx=0x7f47cc000c00 tx=0x7f47cc004730 comp rx=0 tx=0).stop 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 shutdown_connections 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f47c806b4d0 0x7f47c806d990 unknown :-1 s=CLOSED pgs=779 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f47dc158520 0x7f47dc15d660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f47dc0aec40 0x7f47dc140040 unknown :-1 s=CLOSED pgs=1719 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 --2- 172.21.15.64:0/3757996763 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f47dc06dd60 0x7f47dc151b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 >> 172.21.15.64:0/3757996763 conn(0x7f47dc066d00 msgr2=0x7f47dc06c6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 shutdown_connections 2024-03-04T07:35:56.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:56.597+0000 7f47e244e640 1 -- 172.21.15.64:0/3757996763 wait complete. 2024-03-04T07:35:56.627 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:56.810 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:56.810 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:57.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:57.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.925+0000 7f81f7fff640 1 Processor -- start 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.925+0000 7f81f7fff640 1 -- start start 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.926+0000 7f81f7fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f81f80ce9f0 0x7f81f80d0f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.926+0000 7f81f7fff640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81f80cfea0 con 0x7f81f81620d0 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.926+0000 7f81f7fff640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f81f80d0180 con 0x7f81f806bb40 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.926+0000 7f81f7fff640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f81f80d0460 con 0x7f81f80ce9f0 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.926+0000 7f81f77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f81f81620d0 0x7f81f815f7c0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41390/0 (socket says 172.21.15.64:41390) 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.926+0000 7f81f77fe640 1 -- 172.21.15.64:0/3684637053 learned_addr learned my addr 172.21.15.64:0/3684637053 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.926+0000 7f81f6ffd640 1 --2- 172.21.15.64:0/3684637053 >> v2:172.21.15.66:3300/0 conn(0x7f81f80ce9f0 0x7f81f80d0f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.927+0000 7f81f5ffb640 1 -- 172.21.15.64:0/3684637053 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2468778391 0 0) 0x7f81f80d0180 con 0x7f81f806bb40 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.927+0000 7f81f5ffb640 1 -- 172.21.15.64:0/3684637053 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f81e4003520 con 0x7f81f806bb40 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.927+0000 7f81f5ffb640 1 -- 172.21.15.64:0/3684637053 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 583839969 0 0) 0x7f81f80cfea0 con 0x7f81f81620d0 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.927+0000 7f81f6ffd640 1 -- 172.21.15.64:0/3684637053 >> v1:172.21.15.66:6789/0 conn(0x7f81f806bb40 legacy=0x7f81f81619c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.927+0000 7f81f6ffd640 1 -- 172.21.15.64:0/3684637053 >> v1:172.21.15.64:6789/0 conn(0x7f81f81620d0 legacy=0x7f81f815f7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:57.927 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.927+0000 7f81f6ffd640 1 -- 172.21.15.64:0/3684637053 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81f80d0740 con 0x7f81f80ce9f0 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.927+0000 7f81f6ffd640 1 --2- 172.21.15.64:0/3684637053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80d0f40 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f81f00097a0 tx=0x7f81f002b9f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d7a7d7e039ffb618 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f5ffb640 1 -- 172.21.15.64:0/3684637053 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81f002f810 con 0x7f81f80ce9f0 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f5ffb640 1 -- 172.21.15.64:0/3684637053 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81f002f9b0 con 0x7f81f80ce9f0 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f5ffb640 1 -- 172.21.15.64:0/3684637053 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81f002fd80 con 0x7f81f80ce9f0 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f7fff640 1 -- 172.21.15.64:0/3684637053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 msgr2=0x7f81f80d0f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f7fff640 1 --2- 172.21.15.64:0/3684637053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80d0f40 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f81f00097a0 tx=0x7f81f002b9f0 comp rx=0 tx=0).stop 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f7fff640 1 -- 172.21.15.64:0/3684637053 shutdown_connections 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f7fff640 1 --2- 172.21.15.64:0/3684637053 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80d0f40 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f7fff640 1 -- 172.21.15.64:0/3684637053 >> 172.21.15.64:0/3684637053 conn(0x7f81f80b3430 msgr2=0x7f81f80b3840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f7fff640 1 -- 172.21.15.64:0/3684637053 shutdown_connections 2024-03-04T07:35:57.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.928+0000 7f81f7fff640 1 -- 172.21.15.64:0/3684637053 wait complete. 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.929+0000 7f81f7fff640 1 Processor -- start 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f7fff640 1 -- start start 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f7fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81f806bb40 0x7f81f8145700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f7fff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80c89d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f7fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81f81620d0 0x7f81f80c8f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f7fff640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81f806bf80 con 0x7f81f806bb40 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f7fff640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81f8162720 con 0x7f81f80ce9f0 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f7fff640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f81f806c100 con 0x7f81f81620d0 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81f806bb40 0x7f81f8145700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81f806bb40 0x7f81f8145700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56108/0 (socket says 172.21.15.64:56108) 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f6ffd640 1 -- 172.21.15.64:0/2774901454 learned_addr learned my addr 172.21.15.64:0/2774901454 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f67fc640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80c89d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:57.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.930+0000 7f81f77fe640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81f81620d0 0x7f81f80c8f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f6ffd640 1 -- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81f81620d0 msgr2=0x7f81f80c8f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f6ffd640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81f81620d0 0x7f81f80c8f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f6ffd640 1 -- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 msgr2=0x7f81f80c89d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f6ffd640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80c89d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f6ffd640 1 -- 172.21.15.64:0/2774901454 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81f8145c40 con 0x7f81f806bb40 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f67fc640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80c89d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f6ffd640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81f806bb40 0x7f81f8145700 secure :-1 s=READY pgs=1721 cs=0 l=1 rev1=1 crypto rx=0x7f81f00098c0 tx=0x7f81f0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81effff640 1 -- 172.21.15.64:0/2774901454 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81f0047070 con 0x7f81f806bb40 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81effff640 1 -- 172.21.15.64:0/2774901454 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f81f0037810 con 0x7f81f806bb40 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81effff640 1 -- 172.21.15.64:0/2774901454 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81f0037be0 con 0x7f81f806bb40 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81f80cb750 con 0x7f81f806bb40 2024-03-04T07:35:57.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.931+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81f81e8ae0 con 0x7f81f806bb40 2024-03-04T07:35:57.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.932+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81f804f330 con 0x7f81f806bb40 2024-03-04T07:35:57.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.933+0000 7f81effff640 1 -- 172.21.15.64:0/2774901454 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f81f004a070 con 0x7f81f806bb40 2024-03-04T07:35:57.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.933+0000 7f81effff640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81d806b4d0 0x7f81d806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:57.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.933+0000 7f81effff640 1 -- 172.21.15.64:0/2774901454 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f81f00abe50 con 0x7f81f806bb40 2024-03-04T07:35:57.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.933+0000 7f81f67fc640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81d806b4d0 0x7f81d806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:57.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.934+0000 7f81f67fc640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81d806b4d0 0x7f81d806d990 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7f81f81ea450 tx=0x7f81e0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:57.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:57.938+0000 7f81effff640 1 -- 172.21.15.64:0/2774901454 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f81f8145c40 con 0x7f81f806bb40 2024-03-04T07:35:58.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.158+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f81f8072b10 con 0x7f81f806bb40 2024-03-04T07:35:58.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.158+0000 7f81effff640 1 -- 172.21.15.64:0/2774901454 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f81f0033030 con 0x7f81f806bb40 2024-03-04T07:35:58.158 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:58.159 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:58.159 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81d806b4d0 msgr2=0x7f81d806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81d806b4d0 0x7f81d806d990 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7f81f81ea450 tx=0x7f81e0009040 comp rx=0 tx=0).stop 2024-03-04T07:35:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81f806bb40 msgr2=0x7f81f8145700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81f806bb40 0x7f81f8145700 secure :-1 s=READY pgs=1721 cs=0 l=1 rev1=1 crypto rx=0x7f81f00098c0 tx=0x7f81f0004730 comp rx=0 tx=0).stop 2024-03-04T07:35:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 shutdown_connections 2024-03-04T07:35:58.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f81d806b4d0 0x7f81d806d990 unknown :-1 s=CLOSED pgs=780 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:58.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f81f81620d0 0x7f81f80c8f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:58.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81f80ce9f0 0x7f81f80c89d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:58.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 --2- 172.21.15.64:0/2774901454 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81f806bb40 0x7f81f8145700 unknown :-1 s=CLOSED pgs=1721 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:58.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 >> 172.21.15.64:0/2774901454 conn(0x7f81f80b3430 msgr2=0x7f81f80ccbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:58.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 shutdown_connections 2024-03-04T07:35:58.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:58.160+0000 7f81f7fff640 1 -- 172.21.15.64:0/2774901454 wait complete. 2024-03-04T07:35:58.372 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:58.372 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:35:58.630 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:35:59.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:35:59.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.487+0000 7f4bcc8df640 1 Processor -- start 2024-03-04T07:35:59.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.487+0000 7f4bcc8df640 1 -- start start 2024-03-04T07:35:59.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bcc8df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc815e060 0x7f4bc80ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:59.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bcc8df640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f4bc806fe20 0x7f4bc80ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bcc8df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4bc80d0dd0 con 0x7f4bc80d0880 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bcc8df640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4bc80d10b0 con 0x7f4bc806fe20 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bcc8df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4bc80cc0e0 con 0x7f4bc815e060 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc6d9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f4bc80d0880 0x7f4bc806ed30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41394/0 (socket says 172.21.15.64:41394) 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc6d9d640 1 -- 172.21.15.64:0/3902365946 learned_addr learned my addr 172.21.15.64:0/3902365946 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc659c640 1 --2- 172.21.15.64:0/3902365946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc815e060 0x7f4bc80ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc5d9b640 1 --2- 172.21.15.64:0/3902365946 >> v2:172.21.15.66:3300/0 conn(0x7f4bc806fe20 0x7f4bc80ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:59.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc559a640 1 -- 172.21.15.64:0/3902365946 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2733190623 0 0) 0x7f4bc80d0dd0 con 0x7f4bc80d0880 2024-03-04T07:35:59.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc559a640 1 -- 172.21.15.64:0/3902365946 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4bb0003520 con 0x7f4bc80d0880 2024-03-04T07:35:59.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc659c640 1 -- 172.21.15.64:0/3902365946 >> v2:172.21.15.66:3300/0 conn(0x7f4bc806fe20 msgr2=0x7f4bc80ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:35:59.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc659c640 1 --2- 172.21.15.64:0/3902365946 >> v2:172.21.15.66:3300/0 conn(0x7f4bc806fe20 0x7f4bc80ce850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc659c640 1 -- 172.21.15.64:0/3902365946 >> v1:172.21.15.64:6789/0 conn(0x7f4bc80d0880 legacy=0x7f4bc806ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:59.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.488+0000 7f4bc659c640 1 -- 172.21.15.64:0/3902365946 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4bc80cc3c0 con 0x7f4bc815e060 2024-03-04T07:35:59.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.489+0000 7f4bc5d9b640 1 --2- 172.21.15.64:0/3902365946 >> v2:172.21.15.66:3300/0 conn(0x7f4bc806fe20 0x7f4bc80ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:59.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.489+0000 7f4bc659c640 1 --2- 172.21.15.64:0/3902365946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc815e060 0x7f4bc80ce310 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f4bb80097a0 tx=0x7f4bb802b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c3ae66f3cf1bb618 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:59.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.489+0000 7f4bc559a640 1 -- 172.21.15.64:0/3902365946 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb8035020 con 0x7f4bc815e060 2024-03-04T07:35:59.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bc559a640 1 -- 172.21.15.64:0/3902365946 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4bb8031830 con 0x7f4bc815e060 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bc559a640 1 -- 172.21.15.64:0/3902365946 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb8031be0 con 0x7f4bc815e060 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 -- 172.21.15.64:0/3902365946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc815e060 msgr2=0x7f4bc80ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/3902365946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc815e060 0x7f4bc80ce310 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f4bb80097a0 tx=0x7f4bb802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 -- 172.21.15.64:0/3902365946 shutdown_connections 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/3902365946 >> v2:172.21.15.66:3300/0 conn(0x7f4bc806fe20 0x7f4bc80ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/3902365946 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc815e060 0x7f4bc80ce310 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 -- 172.21.15.64:0/3902365946 >> 172.21.15.64:0/3902365946 conn(0x7f4bc80d1a90 msgr2=0x7f4bc80d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 -- 172.21.15.64:0/3902365946 shutdown_connections 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.490+0000 7f4bcc8df640 1 -- 172.21.15.64:0/3902365946 wait complete. 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 Processor -- start 2024-03-04T07:35:59.492 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 -- start start 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bc806fe20 0x7f4bc8152f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc80d0880 0x7f4bc81f3c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bc815e060 0x7f4bc81f41c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4bc80cdbd0 con 0x7f4bc806fe20 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4bc815e440 con 0x7f4bc815e060 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bcc8df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4bc80cdd50 con 0x7f4bc80d0880 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bc659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bc806fe20 0x7f4bc8152f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bc659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bc806fe20 0x7f4bc8152f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56118/0 (socket says 172.21.15.64:56118) 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bc659c640 1 -- 172.21.15.64:0/2945877537 learned_addr learned my addr 172.21.15.64:0/2945877537 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:35:59.495 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bc6d9d640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bc815e060 0x7f4bc81f41c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.492+0000 7f4bc5d9b640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc80d0880 0x7f4bc81f3c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bc659c640 1 -- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc80d0880 msgr2=0x7f4bc81f3c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bc659c640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc80d0880 0x7f4bc81f3c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bc659c640 1 -- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bc815e060 msgr2=0x7f4bc81f41c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bc659c640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bc815e060 0x7f4bc81f41c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bc659c640 1 -- 172.21.15.64:0/2945877537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4bc80b4b80 con 0x7f4bc806fe20 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bc6d9d640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bc815e060 0x7f4bc81f41c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bc659c640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bc806fe20 0x7f4bc8152f40 secure :-1 s=READY pgs=1723 cs=0 l=1 rev1=1 crypto rx=0x7f4bb802b910 tx=0x7f4bb80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bb6ffd640 1 -- 172.21.15.64:0/2945877537 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb8035040 con 0x7f4bc806fe20 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bb6ffd640 1 -- 172.21.15.64:0/2945877537 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4bb803b840 con 0x7f4bc806fe20 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4bc81f47f0 con 0x7f4bc806fe20 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bb6ffd640 1 -- 172.21.15.64:0/2945877537 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4bb803bc10 con 0x7f4bc806fe20 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.493+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4bc815f510 con 0x7f4bc806fe20 2024-03-04T07:35:59.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.494+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4bc81f4be0 con 0x7f4bc806fe20 2024-03-04T07:35:59.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.495+0000 7f4bb6ffd640 1 -- 172.21.15.64:0/2945877537 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4bb8038020 con 0x7f4bc806fe20 2024-03-04T07:35:59.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.495+0000 7f4bb6ffd640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ba806b4d0 0x7f4ba806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:35:59.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.495+0000 7f4bb6ffd640 1 -- 172.21.15.64:0/2945877537 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4bb80ac200 con 0x7f4bc806fe20 2024-03-04T07:35:59.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.495+0000 7f4bc5d9b640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ba806b4d0 0x7f4ba806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:35:59.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.496+0000 7f4bc5d9b640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ba806b4d0 0x7f4ba806d990 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f4bbc009560 tx=0x7f4bbc009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:35:59.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.500+0000 7f4bb6ffd640 1 -- 172.21.15.64:0/2945877537 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4bb807a9f0 con 0x7f4bc806fe20 2024-03-04T07:35:59.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.719+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4bc80ce330 con 0x7f4bc806fe20 2024-03-04T07:35:59.720 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.720+0000 7f4bb6ffd640 1 -- 172.21.15.64:0/2945877537 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f4bb807ea90 con 0x7f4bc806fe20 2024-03-04T07:35:59.720 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:35:59.720 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:35:59.721 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:35:59.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ba806b4d0 msgr2=0x7f4ba806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:59.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ba806b4d0 0x7f4ba806d990 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f4bbc009560 tx=0x7f4bbc009190 comp rx=0 tx=0).stop 2024-03-04T07:35:59.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bc806fe20 msgr2=0x7f4bc8152f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:35:59.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bc806fe20 0x7f4bc8152f40 secure :-1 s=READY pgs=1723 cs=0 l=1 rev1=1 crypto rx=0x7f4bb802b910 tx=0x7f4bb80046c0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 shutdown_connections 2024-03-04T07:35:59.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4ba806b4d0 0x7f4ba806d990 unknown :-1 s=CLOSED pgs=781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.722 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4bc815e060 0x7f4bc81f41c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4bc80d0880 0x7f4bc81f3c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 --2- 172.21.15.64:0/2945877537 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4bc806fe20 0x7f4bc8152f40 unknown :-1 s=CLOSED pgs=1723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:35:59.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 >> 172.21.15.64:0/2945877537 conn(0x7f4bc80d1a90 msgr2=0x7f4bc815d070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:35:59.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 shutdown_connections 2024-03-04T07:35:59.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:35:59.722+0000 7f4bcc8df640 1 -- 172.21.15.64:0/2945877537 wait complete. 2024-03-04T07:35:59.934 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:35:59.934 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:00.632 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:00.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:01.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.048+0000 7f96b5963640 1 Processor -- start 2024-03-04T07:36:01.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96b5963640 1 -- start start 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96b5963640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96b015df00 0x7f96b00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96b5963640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f96b00b95a0 0x7f96b00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96b5963640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00b9a30 0x7f96b00b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96b5963640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96b00cea30 con 0x7f96b015df00 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96b5963640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f96b00d2af0 con 0x7f96b00b95a0 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96b5963640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96b00cf3c0 con 0x7f96b00b9a30 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.049+0000 7f96aeffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96b015df00 0x7f96b00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96aeffd640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f96b015df00 0x7f96b00d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56126/0 (socket says 172.21.15.64:56126) 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96aeffd640 1 -- 172.21.15.64:0/247843892 learned_addr learned my addr 172.21.15.64:0/247843892 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96af7fe640 1 --2- 172.21.15.64:0/247843892 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00b9a30 0x7f96b00b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96ae7fc640 1 --2- 172.21.15.64:0/247843892 >> v2:172.21.15.66:3300/0 conn(0x7f96b00b95a0 0x7f96b00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96aeffd640 1 -- 172.21.15.64:0/247843892 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00b9a30 msgr2=0x7f96b00b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:01.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96aeffd640 1 --2- 172.21.15.64:0/247843892 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00b9a30 0x7f96b00b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96aeffd640 1 -- 172.21.15.64:0/247843892 >> v2:172.21.15.66:3300/0 conn(0x7f96b00b95a0 msgr2=0x7f96b00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:01.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96aeffd640 1 --2- 172.21.15.64:0/247843892 >> v2:172.21.15.66:3300/0 conn(0x7f96b00b95a0 0x7f96b00b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.050+0000 7f96aeffd640 1 -- 172.21.15.64:0/247843892 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96b00cf0e0 con 0x7f96b015df00 2024-03-04T07:36:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96aeffd640 1 --2- 172.21.15.64:0/247843892 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b00d25b0 secure :-1 s=READY pgs=1724 cs=0 l=1 rev1=1 crypto rx=0x7f96b00d1760 tx=0x7f96a002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8155563f0dcf7b6b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96adffb640 1 -- 172.21.15.64:0/247843892 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96a002ca70 con 0x7f96b015df00 2024-03-04T07:36:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96adffb640 1 -- 172.21.15.64:0/247843892 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f96a002cc10 con 0x7f96b015df00 2024-03-04T07:36:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96adffb640 1 -- 172.21.15.64:0/247843892 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96a00343f0 con 0x7f96b015df00 2024-03-04T07:36:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 -- 172.21.15.64:0/247843892 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 msgr2=0x7f96b00d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 --2- 172.21.15.64:0/247843892 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b00d25b0 secure :-1 s=READY pgs=1724 cs=0 l=1 rev1=1 crypto rx=0x7f96b00d1760 tx=0x7f96a002bed0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 -- 172.21.15.64:0/247843892 shutdown_connections 2024-03-04T07:36:01.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 --2- 172.21.15.64:0/247843892 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00b9a30 0x7f96b00b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 --2- 172.21.15.64:0/247843892 >> v2:172.21.15.66:3300/0 conn(0x7f96b00b95a0 0x7f96b00b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 --2- 172.21.15.64:0/247843892 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b00d25b0 unknown :-1 s=CLOSED pgs=1724 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 -- 172.21.15.64:0/247843892 >> 172.21.15.64:0/247843892 conn(0x7f96b0071960 msgr2=0x7f96b0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:01.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 -- 172.21.15.64:0/247843892 shutdown_connections 2024-03-04T07:36:01.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.051+0000 7f96b5963640 1 -- 172.21.15.64:0/247843892 wait complete. 2024-03-04T07:36:01.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.052+0000 7f96b5963640 1 Processor -- start 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.053+0000 7f96b5963640 1 -- start start 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.053+0000 7f96b5963640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96b00b95a0 0x7f96b015e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.053+0000 7f96b5963640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b0156560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.053+0000 7f96b5963640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00c96b0 0x7f96b00c89d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.053+0000 7f96b5963640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96b00cec70 con 0x7f96b015df00 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.053+0000 7f96b5963640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96b00ce750 con 0x7f96b00b95a0 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.053+0000 7f96b5963640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96b00cedf0 con 0x7f96b00c96b0 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b0156560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b0156560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56136/0 (socket says 172.21.15.64:56136) 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 -- 172.21.15.64:0/623321113 learned_addr learned my addr 172.21.15.64:0/623321113 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96af7fe640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00c96b0 0x7f96b00c89d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96aeffd640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96b00b95a0 0x7f96b015e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 -- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00c96b0 msgr2=0x7f96b00c89d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00c96b0 0x7f96b00c89d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 -- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96b00b95a0 msgr2=0x7f96b015e2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96b00b95a0 0x7f96b015e2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 -- 172.21.15.64:0/623321113 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96b015e8c0 con 0x7f96b015df00 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96ae7fc640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b0156560 secure :-1 s=READY pgs=1725 cs=0 l=1 rev1=1 crypto rx=0x7f969800e9b0 tx=0x7f969800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96a7fff640 1 -- 172.21.15.64:0/623321113 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9698017070 con 0x7f96b015df00 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96a7fff640 1 -- 172.21.15.64:0/623321113 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f969800c9a0 con 0x7f96b015df00 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96a7fff640 1 -- 172.21.15.64:0/623321113 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f969800cd90 con 0x7f96b015df00 2024-03-04T07:36:01.056 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96b015eba0 con 0x7f96b015df00 2024-03-04T07:36:01.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.054+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96b015f160 con 0x7f96b015df00 2024-03-04T07:36:01.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.055+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f96b0067ff0 con 0x7f96b015df00 2024-03-04T07:36:01.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.056+0000 7f96a7fff640 1 -- 172.21.15.64:0/623321113 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f969801a070 con 0x7f96b015df00 2024-03-04T07:36:01.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.056+0000 7f96a7fff640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b4d0 0x7f969006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:01.057 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.056+0000 7f96a7fff640 1 -- 172.21.15.64:0/623321113 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f969808a4e0 con 0x7f96b015df00 2024-03-04T07:36:01.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.056+0000 7f96aeffd640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b4d0 0x7f969006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:01.058 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.057+0000 7f96aeffd640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b4d0 0x7f969006d990 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7f96a0035000 tx=0x7f96a0030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:01.060 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.060+0000 7f96a7fff640 1 -- 172.21.15.64:0/623321113 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9698058d60 con 0x7f96b015df00 2024-03-04T07:36:01.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.280+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f96b00d3d30 con 0x7f96b015df00 2024-03-04T07:36:01.280 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.281+0000 7f96a7fff640 1 -- 172.21.15.64:0/623321113 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f969805ce00 con 0x7f96b015df00 2024-03-04T07:36:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:01.281 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:01.281 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b4d0 msgr2=0x7f969006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b4d0 0x7f969006d990 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7f96a0035000 tx=0x7f96a0030000 comp rx=0 tx=0).stop 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 msgr2=0x7f96b0156560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b0156560 secure :-1 s=READY pgs=1725 cs=0 l=1 rev1=1 crypto rx=0x7f969800e9b0 tx=0x7f969800ee70 comp rx=0 tx=0).stop 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 shutdown_connections 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f969006b4d0 0x7f969006d990 unknown :-1 s=CLOSED pgs=782 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f96b00c96b0 0x7f96b00c89d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f96b015df00 0x7f96b0156560 unknown :-1 s=CLOSED pgs=1725 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.283 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 --2- 172.21.15.64:0/623321113 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f96b00b95a0 0x7f96b015e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 >> 172.21.15.64:0/623321113 conn(0x7f96b0071960 msgr2=0x7f96b00d2d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 shutdown_connections 2024-03-04T07:36:01.284 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:01.283+0000 7f96b5963640 1 -- 172.21.15.64:0/623321113 wait complete. 2024-03-04T07:36:01.495 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:01.495 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:02.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:02.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.610+0000 7fcc29437640 1 Processor -- start 2024-03-04T07:36:02.610 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.610+0000 7fcc29437640 1 -- start start 2024-03-04T07:36:02.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc29437640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc24071c50 0x7fcc240b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc29437640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc240b6f20 con 0x7fcc240d18c0 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc29437640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcc240d3030 con 0x7fcc2415df70 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc29437640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc240d3310 con 0x7fcc24071c50 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc237fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc24071c50 0x7fcc240b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc227fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcc240d18c0 0x7fcc24071540 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:41408/0 (socket says 172.21.15.64:41408) 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc237fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc24071c50 0x7fcc240b2700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56146/0 (socket says 172.21.15.64:56146) 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.611+0000 7fcc227fc640 1 -- 172.21.15.64:0/738791946 learned_addr learned my addr 172.21.15.64:0/738791946 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.612+0000 7fcc237fe640 1 -- 172.21.15.64:0/738791946 >> v1:172.21.15.66:6789/0 conn(0x7fcc2415df70 legacy=0x7fcc240d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.612+0000 7fcc237fe640 1 -- 172.21.15.64:0/738791946 >> v1:172.21.15.64:6789/0 conn(0x7fcc240d18c0 legacy=0x7fcc24071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.612+0000 7fcc237fe640 1 -- 172.21.15.64:0/738791946 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc240d35f0 con 0x7fcc24071c50 2024-03-04T07:36:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.612+0000 7fcc21ffb640 1 -- 172.21.15.64:0/738791946 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 300714601 0 0) 0x7fcc240b6f20 con 0x7fcc240d18c0 2024-03-04T07:36:02.613 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.612+0000 7fcc237fe640 1 --2- 172.21.15.64:0/738791946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc24071c50 0x7fcc240b2700 secure :-1 s=READY pgs=1727 cs=0 l=1 rev1=1 crypto rx=0x7fcc1c0097a0 tx=0x7fcc1c02b9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd2b016ff02e2dba server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc21ffb640 1 -- 172.21.15.64:0/738791946 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1c02f810 con 0x7fcc24071c50 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc21ffb640 1 -- 172.21.15.64:0/738791946 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc1c02f9b0 con 0x7fcc24071c50 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc21ffb640 1 -- 172.21.15.64:0/738791946 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1c02fd80 con 0x7fcc24071c50 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc29437640 1 -- 172.21.15.64:0/738791946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc24071c50 msgr2=0x7fcc240b2700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc29437640 1 --2- 172.21.15.64:0/738791946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc24071c50 0x7fcc240b2700 secure :-1 s=READY pgs=1727 cs=0 l=1 rev1=1 crypto rx=0x7fcc1c0097a0 tx=0x7fcc1c02b9f0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc29437640 1 -- 172.21.15.64:0/738791946 shutdown_connections 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc29437640 1 --2- 172.21.15.64:0/738791946 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc24071c50 0x7fcc240b2700 unknown :-1 s=CLOSED pgs=1727 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc29437640 1 -- 172.21.15.64:0/738791946 >> 172.21.15.64:0/738791946 conn(0x7fcc240d3cf0 msgr2=0x7fcc240d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc29437640 1 -- 172.21.15.64:0/738791946 shutdown_connections 2024-03-04T07:36:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.613+0000 7fcc29437640 1 -- 172.21.15.64:0/738791946 wait complete. 2024-03-04T07:36:02.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 Processor -- start 2024-03-04T07:36:02.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 -- start start 2024-03-04T07:36:02.616 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc24071c50 0x7fcc240cdae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc240d18c0 0x7fcc2415af90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc2415df70 0x7fcc2415b4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc2415e4d0 con 0x7fcc2415df70 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc2415e350 con 0x7fcc24071c50 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.615+0000 7fcc29437640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc240b8410 con 0x7fcc240d18c0 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc2415df70 0x7fcc2415b4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc2415df70 0x7fcc2415b4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:56160/0 (socket says 172.21.15.64:56160) 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc22ffd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc24071c50 0x7fcc240cdae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 -- 172.21.15.64:0/1796095992 learned_addr learned my addr 172.21.15.64:0/1796095992 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc227fc640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc240d18c0 0x7fcc2415af90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 -- 172.21.15.64:0/1796095992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc240d18c0 msgr2=0x7fcc2415af90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc240d18c0 0x7fcc2415af90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 -- 172.21.15.64:0/1796095992 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc24071c50 msgr2=0x7fcc240cdae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:02.617 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc24071c50 0x7fcc240cdae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 -- 172.21.15.64:0/1796095992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc240c95b0 con 0x7fcc2415df70 2024-03-04T07:36:02.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc237fe640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc2415df70 0x7fcc2415b4d0 secure :-1 s=READY pgs=1728 cs=0 l=1 rev1=1 crypto rx=0x7fcc1c0098c0 tx=0x7fcc1c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:02.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc1b7fe640 1 -- 172.21.15.64:0/1796095992 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1c047070 con 0x7fcc2415df70 2024-03-04T07:36:02.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc1b7fe640 1 -- 172.21.15.64:0/1796095992 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc1c037bc0 con 0x7fcc2415df70 2024-03-04T07:36:02.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc240c9890 con 0x7fcc2415df70 2024-03-04T07:36:02.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc1b7fe640 1 -- 172.21.15.64:0/1796095992 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc1c02fe60 con 0x7fcc2415df70 2024-03-04T07:36:02.618 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.616+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc240c9e50 con 0x7fcc2415df70 2024-03-04T07:36:02.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.617+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc240b62e0 con 0x7fcc2415df70 2024-03-04T07:36:02.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.618+0000 7fcc1b7fe640 1 -- 172.21.15.64:0/1796095992 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcc1c04a070 con 0x7fcc2415df70 2024-03-04T07:36:02.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.618+0000 7fcc1b7fe640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc0406b500 0x7fcc0406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:02.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.618+0000 7fcc22ffd640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc0406b500 0x7fcc0406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:02.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.618+0000 7fcc1b7fe640 1 -- 172.21.15.64:0/1796095992 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fcc1c0abef0 con 0x7fcc2415df70 2024-03-04T07:36:02.619 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.619+0000 7fcc22ffd640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc0406b500 0x7fcc0406d9c0 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fcc140012e0 tx=0x7fcc14009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:02.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.623+0000 7fcc1b7fe640 1 -- 172.21.15.64:0/1796095992 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcc240b62e0 con 0x7fcc2415df70 2024-03-04T07:36:02.633 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:02.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.841+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fcc240ca1c0 con 0x7fcc2415df70 2024-03-04T07:36:02.841 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.841+0000 7fcc1b7fe640 1 -- 172.21.15.64:0/1796095992 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7fcc1c033030 con 0x7fcc2415df70 2024-03-04T07:36:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:02.842 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:02.842 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:36:02.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc0406b500 msgr2=0x7fcc0406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:02.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc0406b500 0x7fcc0406d9c0 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fcc140012e0 tx=0x7fcc14009040 comp rx=0 tx=0).stop 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc2415df70 msgr2=0x7fcc2415b4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc2415df70 0x7fcc2415b4d0 secure :-1 s=READY pgs=1728 cs=0 l=1 rev1=1 crypto rx=0x7fcc1c0098c0 tx=0x7fcc1c004730 comp rx=0 tx=0).stop 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 shutdown_connections 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc0406b500 0x7fcc0406d9c0 unknown :-1 s=CLOSED pgs=783 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc2415df70 0x7fcc2415b4d0 unknown :-1 s=CLOSED pgs=1728 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc240d18c0 0x7fcc2415af90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 --2- 172.21.15.64:0/1796095992 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc24071c50 0x7fcc240cdae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 >> 172.21.15.64:0/1796095992 conn(0x7fcc240d3cf0 msgr2=0x7fcc24161050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 shutdown_connections 2024-03-04T07:36:02.844 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:02.843+0000 7fcc29437640 1 -- 172.21.15.64:0/1796095992 wait complete. 2024-03-04T07:36:03.056 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:03.056 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:04.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:04.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.170+0000 7f242b0b5640 1 Processor -- start 2024-03-04T07:36:04.170 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.170+0000 7f242b0b5640 1 -- start start 2024-03-04T07:36:04.171 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f242b0b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f242415df70 0x7f24240d27b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f242b0b5640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f24240d18c0 0x7f2424071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f242b0b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f24240b6f20 con 0x7f2424071a80 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f242b0b5640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f24240d3030 con 0x7f24240d18c0 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f242b0b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f24240d3310 con 0x7f242415df70 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f2423fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f24240d18c0 0x7f2424071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f2423fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f24240d18c0 0x7f2424071540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50878/0 (socket says 172.21.15.64:50878) 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f2423fff640 1 -- 172.21.15.64:0/177190558 learned_addr learned my addr 172.21.15.64:0/177190558 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f2428e51640 1 --2- 172.21.15.64:0/177190558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f242415df70 0x7f24240d27b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.171+0000 7f24237fe640 1 -- 172.21.15.64:0/177190558 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1662833014 0 0) 0x7f24240b6f20 con 0x7f2424071a80 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.172+0000 7f2423fff640 1 -- 172.21.15.64:0/177190558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f242415df70 msgr2=0x7f24240d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.172+0000 7f2423fff640 1 --2- 172.21.15.64:0/177190558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f242415df70 0x7f24240d27b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.172+0000 7f2423fff640 1 -- 172.21.15.64:0/177190558 >> v1:172.21.15.66:6789/0 conn(0x7f2424071a80 legacy=0x7f24240b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.172+0000 7f2423fff640 1 -- 172.21.15.64:0/177190558 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f24240d35f0 con 0x7f24240d18c0 2024-03-04T07:36:04.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.172+0000 7f2428e51640 1 --2- 172.21.15.64:0/177190558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f242415df70 0x7f24240d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.172+0000 7f2423fff640 1 --2- 172.21.15.64:0/177190558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24240d18c0 0x7f2424071540 secure :-1 s=READY pgs=1729 cs=0 l=1 rev1=1 crypto rx=0x7f2414009e50 tx=0x7f241402bff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a6c5a661596bd6d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.172+0000 7f24237fe640 1 -- 172.21.15.64:0/177190558 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f241402cb90 con 0x7f24240d18c0 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f24237fe640 1 -- 172.21.15.64:0/177190558 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f241402cd30 con 0x7f24240d18c0 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f24237fe640 1 -- 172.21.15.64:0/177190558 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2414034610 con 0x7f24240d18c0 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 -- 172.21.15.64:0/177190558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24240d18c0 msgr2=0x7f2424071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 --2- 172.21.15.64:0/177190558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24240d18c0 0x7f2424071540 secure :-1 s=READY pgs=1729 cs=0 l=1 rev1=1 crypto rx=0x7f2414009e50 tx=0x7f241402bff0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 -- 172.21.15.64:0/177190558 shutdown_connections 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 --2- 172.21.15.64:0/177190558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f24240d18c0 0x7f2424071540 unknown :-1 s=CLOSED pgs=1729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 --2- 172.21.15.64:0/177190558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f242415df70 0x7f24240d27b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 -- 172.21.15.64:0/177190558 >> 172.21.15.64:0/177190558 conn(0x7f24240d3cf0 msgr2=0x7f24240d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 -- 172.21.15.64:0/177190558 shutdown_connections 2024-03-04T07:36:04.174 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.173+0000 7f242b0b5640 1 -- 172.21.15.64:0/177190558 wait complete. 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.174+0000 7f242b0b5640 1 Processor -- start 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.174+0000 7f242b0b5640 1 -- start start 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.175+0000 7f242b0b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2424071a80 0x7f2424159cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.175+0000 7f242b0b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24240d18c0 0x7f242415a200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.175+0000 7f242b0b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f242415df70 0x7f24241f77b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.175+0000 7f242b0b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f242415e350 con 0x7f242415df70 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.175+0000 7f242b0b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f24240d2cf0 con 0x7f2424071a80 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.175+0000 7f242b0b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f242415e4d0 con 0x7f24240d18c0 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.175+0000 7f2429652640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f242415df70 0x7f24241f77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2428e51640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2424071a80 0x7f2424159cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2423fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24240d18c0 0x7f242415a200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2428e51640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2424071a80 0x7f2424159cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:32864/0 (socket says 172.21.15.64:32864) 2024-03-04T07:36:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2429652640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f242415df70 0x7f24241f77b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50884/0 (socket says 172.21.15.64:50884) 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2428e51640 1 -- 172.21.15.64:0/335645425 learned_addr learned my addr 172.21.15.64:0/335645425 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2429652640 1 -- 172.21.15.64:0/335645425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24240d18c0 msgr2=0x7f242415a200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2429652640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24240d18c0 0x7f242415a200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2429652640 1 -- 172.21.15.64:0/335645425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2424071a80 msgr2=0x7f2424159cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2429652640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2424071a80 0x7f2424159cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2429652640 1 -- 172.21.15.64:0/335645425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f24240cd8b0 con 0x7f242415df70 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2428e51640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2424071a80 0x7f2424159cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f2429652640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f242415df70 0x7f24241f77b0 secure :-1 s=READY pgs=1730 cs=0 l=1 rev1=1 crypto rx=0x7f240c00c3e0 tx=0x7f240c00c8a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f24217fa640 1 -- 172.21.15.64:0/335645425 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f240c017070 con 0x7f242415df70 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f24217fa640 1 -- 172.21.15.64:0/335645425 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f240c0035f0 con 0x7f242415df70 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f24240cdb90 con 0x7f242415df70 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f24217fa640 1 -- 172.21.15.64:0/335645425 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f240c0039e0 con 0x7f242415df70 2024-03-04T07:36:04.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.176+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f24240c7ce0 con 0x7f242415df70 2024-03-04T07:36:04.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.177+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2424067ff0 con 0x7f242415df70 2024-03-04T07:36:04.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.178+0000 7f24217fa640 1 -- 172.21.15.64:0/335645425 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f240c01a070 con 0x7f242415df70 2024-03-04T07:36:04.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.178+0000 7f24217fa640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f240406b4d0 0x7f240406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:04.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.178+0000 7f24217fa640 1 -- 172.21.15.64:0/335645425 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f240c089450 con 0x7f242415df70 2024-03-04T07:36:04.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.178+0000 7f2428e51640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f240406b4d0 0x7f240406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:04.180 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.179+0000 7f2428e51640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f240406b4d0 0x7f240406d990 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7f241c009560 tx=0x7f241c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:04.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.183+0000 7f24217fa640 1 -- 172.21.15.64:0/335645425 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f240c057d00 con 0x7f242415df70 2024-03-04T07:36:04.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.405+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2424072290 con 0x7f242415df70 2024-03-04T07:36:04.405 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.405+0000 7f24217fa640 1 -- 172.21.15.64:0/335645425 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f240c05bda0 con 0x7f242415df70 2024-03-04T07:36:04.406 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:04.406 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:04.406 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:36:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f240406b4d0 msgr2=0x7f240406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f240406b4d0 0x7f240406d990 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7f241c009560 tx=0x7f241c009190 comp rx=0 tx=0).stop 2024-03-04T07:36:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f242415df70 msgr2=0x7f24241f77b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f242415df70 0x7f24241f77b0 secure :-1 s=READY pgs=1730 cs=0 l=1 rev1=1 crypto rx=0x7f240c00c3e0 tx=0x7f240c00c8a0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 shutdown_connections 2024-03-04T07:36:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f240406b4d0 0x7f240406d990 unknown :-1 s=CLOSED pgs=784 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f242415df70 0x7f24241f77b0 unknown :-1 s=CLOSED pgs=1730 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f24240d18c0 0x7f242415a200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 --2- 172.21.15.64:0/335645425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2424071a80 0x7f2424159cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:04.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 >> 172.21.15.64:0/335645425 conn(0x7f24240d3cf0 msgr2=0x7f24240b5f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:04.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 shutdown_connections 2024-03-04T07:36:04.409 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:04.408+0000 7f242b0b5640 1 -- 172.21.15.64:0/335645425 wait complete. 2024-03-04T07:36:04.620 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:04.620 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:04.636 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:05.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:05.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.736+0000 7f95e2a64640 1 Processor -- start 2024-03-04T07:36:05.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.736+0000 7f95e2a64640 1 -- start start 2024-03-04T07:36:05.737 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e2a64640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f95dc06e700 0x7f95dc15ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e2a64640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc0b55d0 0x7f95dc071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e2a64640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f95dc0b4160 con 0x7f95dc072660 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e2a64640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f95dc0b5b40 con 0x7f95dc06e700 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e2a64640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95dc0b3ba0 con 0x7f95dc0b55d0 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95dbfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f95dc072660 0x7f95dc06d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45368/0 (socket says 172.21.15.64:45368) 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95dbfff640 1 -- 172.21.15.64:0/1856563673 learned_addr learned my addr 172.21.15.64:0/1856563673 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e0800640 1 --2- 172.21.15.64:0/1856563673 >> v2:172.21.15.66:3300/0 conn(0x7f95dc06e700 0x7f95dc15ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e1001640 1 --2- 172.21.15.64:0/1856563673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc0b55d0 0x7f95dc071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95db7fe640 1 -- 172.21.15.64:0/1856563673 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1632783702 0 0) 0x7f95dc0b4160 con 0x7f95dc072660 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95db7fe640 1 -- 172.21.15.64:0/1856563673 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f95cc003520 con 0x7f95dc072660 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e1001640 1 -- 172.21.15.64:0/1856563673 >> v2:172.21.15.66:3300/0 conn(0x7f95dc06e700 msgr2=0x7f95dc15ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.737+0000 7f95e1001640 1 --2- 172.21.15.64:0/1856563673 >> v2:172.21.15.66:3300/0 conn(0x7f95dc06e700 0x7f95dc15ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.738+0000 7f95e1001640 1 -- 172.21.15.64:0/1856563673 >> v1:172.21.15.64:6789/0 conn(0x7f95dc072660 legacy=0x7f95dc06d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:05.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.738+0000 7f95e1001640 1 -- 172.21.15.64:0/1856563673 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95dc0b3e80 con 0x7f95dc0b55d0 2024-03-04T07:36:05.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.738+0000 7f95e0800640 1 --2- 172.21.15.64:0/1856563673 >> v2:172.21.15.66:3300/0 conn(0x7f95dc06e700 0x7f95dc15ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:05.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.738+0000 7f95e1001640 1 --2- 172.21.15.64:0/1856563673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc0b55d0 0x7f95dc071980 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f95d4009d90 tx=0x7f95d402bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e821f8dbfc2e43d7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:05.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.738+0000 7f95db7fe640 1 -- 172.21.15.64:0/1856563673 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95d4039070 con 0x7f95dc0b55d0 2024-03-04T07:36:05.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95db7fe640 1 -- 172.21.15.64:0/1856563673 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95d402cc50 con 0x7f95dc0b55d0 2024-03-04T07:36:05.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95db7fe640 1 -- 172.21.15.64:0/1856563673 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95d4034440 con 0x7f95dc0b55d0 2024-03-04T07:36:05.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 -- 172.21.15.64:0/1856563673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc0b55d0 msgr2=0x7f95dc071980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:05.740 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 --2- 172.21.15.64:0/1856563673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc0b55d0 0x7f95dc071980 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f95d4009d90 tx=0x7f95d402bea0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 -- 172.21.15.64:0/1856563673 shutdown_connections 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 --2- 172.21.15.64:0/1856563673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc0b55d0 0x7f95dc071980 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 --2- 172.21.15.64:0/1856563673 >> v2:172.21.15.66:3300/0 conn(0x7f95dc06e700 0x7f95dc15ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 -- 172.21.15.64:0/1856563673 >> 172.21.15.64:0/1856563673 conn(0x7f95dc0b2720 msgr2=0x7f95dc0b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 -- 172.21.15.64:0/1856563673 shutdown_connections 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.739+0000 7f95e2a64640 1 -- 172.21.15.64:0/1856563673 wait complete. 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.740+0000 7f95e2a64640 1 Processor -- start 2024-03-04T07:36:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.741+0000 7f95e2a64640 1 -- start start 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.741+0000 7f95e2a64640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc06e700 0x7f95dc157a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.741+0000 7f95e2a64640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95dc072660 0x7f95dc0b0830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.741+0000 7f95e2a64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95dc0b55d0 0x7f95dc15ad90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.741+0000 7f95e2a64640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95dc072040 con 0x7f95dc0b55d0 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.741+0000 7f95e2a64640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95dc071ec0 con 0x7f95dc072660 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.741+0000 7f95e2a64640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95dc0721c0 con 0x7f95dc06e700 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95e1001640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95dc0b55d0 0x7f95dc15ad90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95dbfff640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95dc072660 0x7f95dc0b0830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95e1001640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95dc0b55d0 0x7f95dc15ad90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50900/0 (socket says 172.21.15.64:50900) 2024-03-04T07:36:05.743 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95e1001640 1 -- 172.21.15.64:0/3772497612 learned_addr learned my addr 172.21.15.64:0/3772497612 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95e0800640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc06e700 0x7f95dc157a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95dbfff640 1 -- 172.21.15.64:0/3772497612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc06e700 msgr2=0x7f95dc157a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95dbfff640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc06e700 0x7f95dc157a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95dbfff640 1 -- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95dc0b55d0 msgr2=0x7f95dc15ad90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95dbfff640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95dc0b55d0 0x7f95dc15ad90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95dbfff640 1 -- 172.21.15.64:0/3772497612 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95dc157fc0 con 0x7f95dc072660 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95e1001640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95dc0b55d0 0x7f95dc15ad90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95dbfff640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95dc072660 0x7f95dc0b0830 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f95c800c400 tx=0x7f95c800c8c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95d97fa640 1 -- 172.21.15.64:0/3772497612 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95c8017070 con 0x7f95dc072660 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95d97fa640 1 -- 172.21.15.64:0/3772497612 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95c8005ce0 con 0x7f95dc072660 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95dc0b1280 con 0x7f95dc072660 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95d97fa640 1 -- 172.21.15.64:0/3772497612 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95c80072f0 con 0x7f95dc072660 2024-03-04T07:36:05.744 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.742+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95dc0cecc0 con 0x7f95dc072660 2024-03-04T07:36:05.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.743+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f95dc0b33b0 con 0x7f95dc072660 2024-03-04T07:36:05.745 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.744+0000 7f95d97fa640 1 -- 172.21.15.64:0/3772497612 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f95c801a070 con 0x7f95dc072660 2024-03-04T07:36:05.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.744+0000 7f95d97fa640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f95c006b4d0 0x7f95c006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:05.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.744+0000 7f95d97fa640 1 -- 172.21.15.64:0/3772497612 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f95c8089510 con 0x7f95dc072660 2024-03-04T07:36:05.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.744+0000 7f95e0800640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f95c006b4d0 0x7f95c006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:05.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.745+0000 7f95e0800640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f95c006b4d0 0x7f95c006d990 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7f95dc0ce790 tx=0x7f95d0009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:05.748 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.749+0000 7f95d97fa640 1 -- 172.21.15.64:0/3772497612 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f95dc0b33b0 con 0x7f95dc072660 2024-03-04T07:36:05.969 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.970+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f95dc067ff0 con 0x7f95dc072660 2024-03-04T07:36:05.970 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.970+0000 7f95d97fa640 1 -- 172.21.15.64:0/3772497612 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f95c8087e20 con 0x7f95dc072660 2024-03-04T07:36:05.970 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:05.970 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:05.971 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:36:05.972 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f95c006b4d0 msgr2=0x7f95c006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f95c006b4d0 0x7f95c006d990 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7f95dc0ce790 tx=0x7f95d0009190 comp rx=0 tx=0).stop 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95dc072660 msgr2=0x7f95dc0b0830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95dc072660 0x7f95dc0b0830 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f95c800c400 tx=0x7f95c800c8c0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 shutdown_connections 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f95c006b4d0 0x7f95c006d990 unknown :-1 s=CLOSED pgs=785 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f95dc0b55d0 0x7f95dc15ad90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95dc072660 0x7f95dc0b0830 unknown :-1 s=CLOSED pgs=632 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 --2- 172.21.15.64:0/3772497612 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f95dc06e700 0x7f95dc157a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.972+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 >> 172.21.15.64:0/3772497612 conn(0x7f95dc0b2720 msgr2=0x7f95dc0b2b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.973+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 shutdown_connections 2024-03-04T07:36:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:05.973+0000 7f95e2a64640 1 -- 172.21.15.64:0/3772497612 wait complete. 2024-03-04T07:36:06.184 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:06.185 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:06.638 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:07.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:07.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.300+0000 7f1fced5e640 1 Processor -- start 2024-03-04T07:36:07.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.300+0000 7f1fced5e640 1 -- start start 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fced5e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1fc8157b80 0x7f1fc815b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fced5e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1fc8157f60 0x7f1fc815b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fced5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8070b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fced5e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1fc806c7b0 con 0x7f1fc8157b80 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fced5e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1fc806ca90 con 0x7f1fc8157f60 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fced5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1fc806cd70 con 0x7f1fc80cbab0 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fccafa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1fc8157b80 0x7f1fc815b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fccafa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1fc8157b80 0x7f1fc815b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50912/0 (socket says 172.21.15.64:50912) 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fccafa640 1 -- 172.21.15.64:0/4264315258 learned_addr learned my addr 172.21.15.64:0/4264315258 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fc7fff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1fc8157f60 0x7f1fc815b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fcd2fb640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8070b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:07.302 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fccafa640 1 -- 172.21.15.64:0/4264315258 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 msgr2=0x7f1fc8070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:07.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fccafa640 1 --2- 172.21.15.64:0/4264315258 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8070b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.301+0000 7f1fccafa640 1 -- 172.21.15.64:0/4264315258 >> v2:172.21.15.66:3300/0 conn(0x7f1fc8157f60 msgr2=0x7f1fc815b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:07.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.302+0000 7f1fccafa640 1 --2- 172.21.15.64:0/4264315258 >> v2:172.21.15.66:3300/0 conn(0x7f1fc8157f60 0x7f1fc815b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.302+0000 7f1fccafa640 1 -- 172.21.15.64:0/4264315258 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1fc806d050 con 0x7f1fc8157b80 2024-03-04T07:36:07.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.302+0000 7f1fc7fff640 1 --2- 172.21.15.64:0/4264315258 >> v2:172.21.15.66:3300/0 conn(0x7f1fc8157f60 0x7f1fc815b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:07.303 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.302+0000 7f1fccafa640 1 --2- 172.21.15.64:0/4264315258 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc8157b80 0x7f1fc815b260 secure :-1 s=READY pgs=1732 cs=0 l=1 rev1=1 crypto rx=0x7f1fc815a7e0 tx=0x7f1fc002bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8d83823633a7a24f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:07.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.302+0000 7f1fc77fe640 1 -- 172.21.15.64:0/4264315258 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fc002ca70 con 0x7f1fc8157b80 2024-03-04T07:36:07.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.302+0000 7f1fc77fe640 1 -- 172.21.15.64:0/4264315258 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1fc002cc10 con 0x7f1fc8157b80 2024-03-04T07:36:07.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.302+0000 7f1fc77fe640 1 -- 172.21.15.64:0/4264315258 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fc00343f0 con 0x7f1fc8157b80 2024-03-04T07:36:07.304 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 -- 172.21.15.64:0/4264315258 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc8157b80 msgr2=0x7f1fc815b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 --2- 172.21.15.64:0/4264315258 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc8157b80 0x7f1fc815b260 secure :-1 s=READY pgs=1732 cs=0 l=1 rev1=1 crypto rx=0x7f1fc815a7e0 tx=0x7f1fc002bed0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 -- 172.21.15.64:0/4264315258 shutdown_connections 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 --2- 172.21.15.64:0/4264315258 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8070b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 --2- 172.21.15.64:0/4264315258 >> v2:172.21.15.66:3300/0 conn(0x7f1fc8157f60 0x7f1fc815b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 --2- 172.21.15.64:0/4264315258 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc8157b80 0x7f1fc815b260 unknown :-1 s=CLOSED pgs=1732 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 -- 172.21.15.64:0/4264315258 >> 172.21.15.64:0/4264315258 conn(0x7f1fc8066d00 msgr2=0x7f1fc80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 -- 172.21.15.64:0/4264315258 shutdown_connections 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.303+0000 7f1fced5e640 1 -- 172.21.15.64:0/4264315258 wait complete. 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.304+0000 7f1fced5e640 1 Processor -- start 2024-03-04T07:36:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.304+0000 7f1fced5e640 1 -- start start 2024-03-04T07:36:07.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fced5e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8150410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:07.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fced5e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fc8157f60 0x7f1fc80ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:07.307 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fced5e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc80addf0 0x7f1fc81536d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fced5e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1fc80cc1d0 con 0x7f1fc80addf0 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fced5e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1fc80cc050 con 0x7f1fc8157f60 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fced5e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1fc8071080 con 0x7f1fc80cbab0 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fcd2fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc80addf0 0x7f1fc81536d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fcd2fb640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc80addf0 0x7f1fc81536d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50914/0 (socket says 172.21.15.64:50914) 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fccafa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8150410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fcd2fb640 1 -- 172.21.15.64:0/896453450 learned_addr learned my addr 172.21.15.64:0/896453450 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.305+0000 7f1fc7fff640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fc8157f60 0x7f1fc80ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fcd2fb640 1 -- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 msgr2=0x7f1fc8150410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fcd2fb640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8150410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fcd2fb640 1 -- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fc8157f60 msgr2=0x7f1fc80ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fcd2fb640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fc8157f60 0x7f1fc80ad8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.308 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fcd2fb640 1 -- 172.21.15.64:0/896453450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1fc8154420 con 0x7f1fc80addf0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fccafa640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8150410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fcd2fb640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc80addf0 0x7f1fc81536d0 secure :-1 s=READY pgs=1733 cs=0 l=1 rev1=1 crypto rx=0x7f1fb80089e0 tx=0x7f1fb8008ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fc57fa640 1 -- 172.21.15.64:0/896453450 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fb801b070 con 0x7f1fc80addf0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fc57fa640 1 -- 172.21.15.64:0/896453450 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1fb8009cc0 con 0x7f1fc80addf0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fc57fa640 1 -- 172.21.15.64:0/896453450 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fb80166a0 con 0x7f1fc80addf0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1fc8154700 con 0x7f1fc80addf0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.306+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1fc8154cc0 con 0x7f1fc80addf0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.307+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1fc8067ff0 con 0x7f1fc80addf0 2024-03-04T07:36:07.309 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.308+0000 7f1fc57fa640 1 -- 172.21.15.64:0/896453450 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1fb801e070 con 0x7f1fc80addf0 2024-03-04T07:36:07.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.308+0000 7f1fc57fa640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1fa806b4d0 0x7f1fa806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:07.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.308+0000 7f1fccafa640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1fa806b4d0 0x7f1fa806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:07.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.308+0000 7f1fc57fa640 1 -- 172.21.15.64:0/896453450 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1fb8089d30 con 0x7f1fc80addf0 2024-03-04T07:36:07.310 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.309+0000 7f1fccafa640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1fa806b4d0 0x7f1fa806d990 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7f1fc0035000 tx=0x7f1fc0030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:07.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.313+0000 7f1fc57fa640 1 -- 172.21.15.64:0/896453450 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1fb8058520 con 0x7f1fc80addf0 2024-03-04T07:36:07.534 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.534+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1fc80cd670 con 0x7f1fc80addf0 2024-03-04T07:36:07.535 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.535+0000 7f1fc57fa640 1 -- 172.21.15.64:0/896453450 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 90 v90) v1 ==== 94+0+3460 (secure 0 0 0) 0x7f1fb805c5c0 con 0x7f1fc80addf0 2024-03-04T07:36:07.535 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:07.535 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":90,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":90,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:34:33.556697+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:07.535 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 90 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1fa806b4d0 msgr2=0x7f1fa806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1fa806b4d0 0x7f1fa806d990 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7f1fc0035000 tx=0x7f1fc0030000 comp rx=0 tx=0).stop 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc80addf0 msgr2=0x7f1fc81536d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc80addf0 0x7f1fc81536d0 secure :-1 s=READY pgs=1733 cs=0 l=1 rev1=1 crypto rx=0x7f1fb80089e0 tx=0x7f1fb8008ad0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 shutdown_connections 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1fa806b4d0 0x7f1fa806d990 unknown :-1 s=CLOSED pgs=786 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fc80addf0 0x7f1fc81536d0 unknown :-1 s=CLOSED pgs=1733 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fc8157f60 0x7f1fc80ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 --2- 172.21.15.64:0/896453450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fc80cbab0 0x7f1fc8150410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:07.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 >> 172.21.15.64:0/896453450 conn(0x7f1fc8066d00 msgr2=0x7f1fc815a850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:07.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 shutdown_connections 2024-03-04T07:36:07.538 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:07.537+0000 7f1fced5e640 1 -- 172.21.15.64:0/896453450 wait complete. 2024-03-04T07:36:07.749 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 90, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:34:33.556697+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [1], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:07.749 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:08.640 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:08.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:08.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.864+0000 7f09057f6640 1 Processor -- start 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.864+0000 7f09057f6640 1 -- start start 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f09057f6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f09000d18c0 0x7f0900071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f09057f6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0900071a80 0x7f09000b2700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f09057f6640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f09000b6f20 con 0x7f090015df70 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f09057f6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0900071eb0 con 0x7f0900071a80 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f09057f6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f09000d3030 con 0x7f09000d18c0 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08ff7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0900071a80 0x7f09000b2700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08ff7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0900071a80 0x7f09000b2700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50918/0 (socket says 172.21.15.64:50918) 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08ff7fe640 1 -- 172.21.15.64:0/3027085399 learned_addr learned my addr 172.21.15.64:0/3027085399 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08fe7fc640 1 --2- 172.21.15.64:0/3027085399 >> v2:172.21.15.66:3300/0 conn(0x7f09000d18c0 0x7f0900071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08ff7fe640 1 -- 172.21.15.64:0/3027085399 >> v2:172.21.15.66:3300/0 conn(0x7f09000d18c0 msgr2=0x7f0900071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08ff7fe640 1 --2- 172.21.15.64:0/3027085399 >> v2:172.21.15.66:3300/0 conn(0x7f09000d18c0 0x7f0900071540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:08.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08ff7fe640 1 -- 172.21.15.64:0/3027085399 >> v1:172.21.15.66:6789/0 conn(0x7f090015df70 legacy=0x7f09000d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:08.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08fdffb640 1 -- 172.21.15.64:0/3027085399 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 854565558 0 0) 0x7f09000b6f20 con 0x7f090015df70 2024-03-04T07:36:08.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.865+0000 7f08ff7fe640 1 -- 172.21.15.64:0/3027085399 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09000d3310 con 0x7f0900071a80 2024-03-04T07:36:08.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.866+0000 7f08ff7fe640 1 --2- 172.21.15.64:0/3027085399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09000b2700 secure :-1 s=READY pgs=1734 cs=0 l=1 rev1=1 crypto rx=0x7f0900161b30 tx=0x7f08f802ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3e18d1c2a6842e5d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:08.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.866+0000 7f08fdffb640 1 -- 172.21.15.64:0/3027085399 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08f802f850 con 0x7f0900071a80 2024-03-04T07:36:08.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.866+0000 7f08fdffb640 1 -- 172.21.15.64:0/3027085399 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08f802f9f0 con 0x7f0900071a80 2024-03-04T07:36:08.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.866+0000 7f08fdffb640 1 -- 172.21.15.64:0/3027085399 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08f802fdc0 con 0x7f0900071a80 2024-03-04T07:36:08.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 -- 172.21.15.64:0/3027085399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 msgr2=0x7f09000b2700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:08.868 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 --2- 172.21.15.64:0/3027085399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09000b2700 secure :-1 s=READY pgs=1734 cs=0 l=1 rev1=1 crypto rx=0x7f0900161b30 tx=0x7f08f802ba30 comp rx=0 tx=0).stop 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 -- 172.21.15.64:0/3027085399 shutdown_connections 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 --2- 172.21.15.64:0/3027085399 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09000b2700 unknown :-1 s=CLOSED pgs=1734 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 --2- 172.21.15.64:0/3027085399 >> v2:172.21.15.66:3300/0 conn(0x7f09000d18c0 0x7f0900071540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 -- 172.21.15.64:0/3027085399 >> 172.21.15.64:0/3027085399 conn(0x7f09000d3cf0 msgr2=0x7f09000d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 -- 172.21.15.64:0/3027085399 shutdown_connections 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.867+0000 7f09057f6640 1 -- 172.21.15.64:0/3027085399 wait complete. 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.868+0000 7f09057f6640 1 Processor -- start 2024-03-04T07:36:08.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.868+0000 7f09057f6640 1 -- start start 2024-03-04T07:36:08.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f09057f6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09001511d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:08.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f09057f6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09000d18c0 0x7f09000c89c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f09057f6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f090015df70 0x7f09000cc770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f09057f6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f090015e350 con 0x7f0900071a80 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f09057f6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09000d38d0 con 0x7f090015df70 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f09057f6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f090015e4d0 con 0x7f09000d18c0 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09001511d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09001511d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50930/0 (socket says 172.21.15.64:50930) 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 -- 172.21.15.64:0/3101901475 learned_addr learned my addr 172.21.15.64:0/3101901475 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08fe7fc640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09000d18c0 0x7f09000c89c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08ff7fe640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f090015df70 0x7f09000cc770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 -- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09000d18c0 msgr2=0x7f09000c89c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09000d18c0 0x7f09000c89c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 -- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f090015df70 msgr2=0x7f09000cc770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:08.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f090015df70 0x7f09000cc770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.869+0000 7f08feffd640 1 -- 172.21.15.64:0/3101901475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f09000c7960 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.870+0000 7f08feffd640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09001511d0 secure :-1 s=READY pgs=1735 cs=0 l=1 rev1=1 crypto rx=0x7f08f000c7f0 tx=0x7f08f000ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.870+0000 7f08f77fe640 1 -- 172.21.15.64:0/3101901475 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08f0017070 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.870+0000 7f08f77fe640 1 -- 172.21.15.64:0/3101901475 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08f0003da0 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.870+0000 7f08f77fe640 1 -- 172.21.15.64:0/3101901475 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08f0005290 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.870+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f09000c7c00 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.870+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f09000c81c0 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.871+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0900067ff0 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.871+0000 7f08f77fe640 1 -- 172.21.15.64:0/3101901475 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f08f001a070 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.872+0000 7f08f77fe640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08e006b4d0 0x7f08e006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.872+0000 7f08f77fe640 1 -- 172.21.15.64:0/3101901475 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f08f0089860 con 0x7f0900071a80 2024-03-04T07:36:08.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.872+0000 7f08fe7fc640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08e006b4d0 0x7f08e006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:08.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.872+0000 7f08fe7fc640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08e006b4d0 0x7f08e006d990 secure :-1 s=READY pgs=787 cs=0 l=1 rev1=1 crypto rx=0x7f09000c7740 tx=0x7f08e8009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:08.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:08.876+0000 7f08f77fe640 1 -- 172.21.15.64:0/3101901475 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f08f0058050 con 0x7f0900071a80 2024-03-04T07:36:09.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.097+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f09000b2930 con 0x7f0900071a80 2024-03-04T07:36:09.097 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.097+0000 7f08f77fe640 1 -- 172.21.15.64:0/3101901475 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f08f005c0f0 con 0x7f0900071a80 2024-03-04T07:36:09.098 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:09.098 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:09.098 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:09.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08e006b4d0 msgr2=0x7f08e006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:09.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08e006b4d0 0x7f08e006d990 secure :-1 s=READY pgs=787 cs=0 l=1 rev1=1 crypto rx=0x7f09000c7740 tx=0x7f08e8009190 comp rx=0 tx=0).stop 2024-03-04T07:36:09.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 msgr2=0x7f09001511d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:09.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09001511d0 secure :-1 s=READY pgs=1735 cs=0 l=1 rev1=1 crypto rx=0x7f08f000c7f0 tx=0x7f08f000ccb0 comp rx=0 tx=0).stop 2024-03-04T07:36:09.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 shutdown_connections 2024-03-04T07:36:09.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f08e006b4d0 0x7f08e006d990 unknown :-1 s=CLOSED pgs=787 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:09.100 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f090015df70 0x7f09000cc770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:09.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f09000d18c0 0x7f09000c89c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:09.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 --2- 172.21.15.64:0/3101901475 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0900071a80 0x7f09001511d0 unknown :-1 s=CLOSED pgs=1735 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:09.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.099+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 >> 172.21.15.64:0/3101901475 conn(0x7f09000d3cf0 msgr2=0x7f090015ae60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:09.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.100+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 shutdown_connections 2024-03-04T07:36:09.101 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:09.100+0000 7f09057f6640 1 -- 172.21.15.64:0/3101901475 wait complete. 2024-03-04T07:36:09.312 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:09.312 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:10.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:10.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.427+0000 7f2d8d837640 1 Processor -- start 2024-03-04T07:36:10.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.427+0000 7f2d8d837640 1 -- start start 2024-03-04T07:36:10.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.427+0000 7f2d8d837640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2d8815e060 0x7f2d880ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:10.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.427+0000 7f2d8d837640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d880d0dd0 con 0x7f2d8806fe20 2024-03-04T07:36:10.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.427+0000 7f2d8d837640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f2d880d10b0 con 0x7f2d880d0880 2024-03-04T07:36:10.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.427+0000 7f2d8d837640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d880cc0e0 con 0x7f2d8815e060 2024-03-04T07:36:10.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d867fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f2d8806fe20 0x7f2d880ce850 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:45380/0 (socket says 172.21.15.64:45380) 2024-03-04T07:36:10.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d867fc640 1 -- 172.21.15.64:0/1448867763 learned_addr learned my addr 172.21.15.64:0/1448867763 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d86ffd640 1 --2- 172.21.15.64:0/1448867763 >> v2:172.21.15.66:3300/0 conn(0x7f2d8815e060 0x7f2d880ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d85ffb640 1 -- 172.21.15.64:0/1448867763 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1741573715 0 0) 0x7f2d880d0dd0 con 0x7f2d8806fe20 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d85ffb640 1 -- 172.21.15.64:0/1448867763 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d70003520 con 0x7f2d8806fe20 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d85ffb640 1 -- 172.21.15.64:0/1448867763 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1898517058 0 0) 0x7f2d880d10b0 con 0x7f2d880d0880 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d85ffb640 1 -- 172.21.15.64:0/1448867763 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2d880d0dd0 con 0x7f2d880d0880 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d86ffd640 1 -- 172.21.15.64:0/1448867763 >> v1:172.21.15.66:6789/0 conn(0x7f2d880d0880 legacy=0x7f2d880ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d86ffd640 1 -- 172.21.15.64:0/1448867763 >> v1:172.21.15.64:6789/0 conn(0x7f2d8806fe20 legacy=0x7f2d880ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:10.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.428+0000 7f2d86ffd640 1 -- 172.21.15.64:0/1448867763 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d880cc3c0 con 0x7f2d8815e060 2024-03-04T07:36:10.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.429+0000 7f2d86ffd640 1 --2- 172.21.15.64:0/1448867763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 0x7f2d880ce310 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f2d780097a0 tx=0x7f2d7802b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=32fd46edd8830266 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:10.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.429+0000 7f2d85ffb640 1 -- 172.21.15.64:0/1448867763 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d78034070 con 0x7f2d8815e060 2024-03-04T07:36:10.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.429+0000 7f2d85ffb640 1 -- 172.21.15.64:0/1448867763 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d78030610 con 0x7f2d8815e060 2024-03-04T07:36:10.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.429+0000 7f2d85ffb640 1 -- 172.21.15.64:0/1448867763 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d780309c0 con 0x7f2d8815e060 2024-03-04T07:36:10.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.430+0000 7f2d8d837640 1 -- 172.21.15.64:0/1448867763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 msgr2=0x7f2d880ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:10.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.430+0000 7f2d8d837640 1 --2- 172.21.15.64:0/1448867763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 0x7f2d880ce310 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f2d780097a0 tx=0x7f2d7802b960 comp rx=0 tx=0).stop 2024-03-04T07:36:10.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.430+0000 7f2d8d837640 1 -- 172.21.15.64:0/1448867763 shutdown_connections 2024-03-04T07:36:10.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.430+0000 7f2d8d837640 1 --2- 172.21.15.64:0/1448867763 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 0x7f2d880ce310 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.430+0000 7f2d8d837640 1 -- 172.21.15.64:0/1448867763 >> 172.21.15.64:0/1448867763 conn(0x7f2d880d1a90 msgr2=0x7f2d880d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:10.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.430+0000 7f2d8d837640 1 -- 172.21.15.64:0/1448867763 shutdown_connections 2024-03-04T07:36:10.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.430+0000 7f2d8d837640 1 -- 172.21.15.64:0/1448867763 wait complete. 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.431+0000 7f2d8d837640 1 Processor -- start 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.431+0000 7f2d8d837640 1 -- start start 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d8d837640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d8806fe20 0x7f2d8815b7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d8d837640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d880d0880 0x7f2d881e5670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d8d837640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 0x7f2d880b5260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d8d837640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d880cef60 con 0x7f2d8806fe20 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d8d837640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d8815e440 con 0x7f2d8815e060 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d8d837640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d880cf0e0 con 0x7f2d880d0880 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d8806fe20 0x7f2d8815b7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d8806fe20 0x7f2d8815b7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50932/0 (socket says 172.21.15.64:50932) 2024-03-04T07:36:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 -- 172.21.15.64:0/3319005151 learned_addr learned my addr 172.21.15.64:0/3319005151 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d867fc640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d880d0880 0x7f2d881e5670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d877fe640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 0x7f2d880b5260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 -- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d880d0880 msgr2=0x7f2d881e5670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d880d0880 0x7f2d881e5670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 -- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 msgr2=0x7f2d880b5260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 0x7f2d880b5260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 -- 172.21.15.64:0/3319005151 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d881e60c0 con 0x7f2d8806fe20 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.432+0000 7f2d86ffd640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d8806fe20 0x7f2d8815b7a0 secure :-1 s=READY pgs=1737 cs=0 l=1 rev1=1 crypto rx=0x7f2d78000c00 tx=0x7f2d780046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.433+0000 7f2d6ffff640 1 -- 172.21.15.64:0/3319005151 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d78034040 con 0x7f2d8806fe20 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.433+0000 7f2d6ffff640 1 -- 172.21.15.64:0/3319005151 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2d7803aaa0 con 0x7f2d8806fe20 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.433+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d881e63a0 con 0x7f2d8806fe20 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.433+0000 7f2d6ffff640 1 -- 172.21.15.64:0/3319005151 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d78006e10 con 0x7f2d8806fe20 2024-03-04T07:36:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.433+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d881e6960 con 0x7f2d8806fe20 2024-03-04T07:36:10.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.434+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d881e5420 con 0x7f2d8806fe20 2024-03-04T07:36:10.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.434+0000 7f2d6ffff640 1 -- 172.21.15.64:0/3319005151 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d7804b060 con 0x7f2d8806fe20 2024-03-04T07:36:10.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.434+0000 7f2d6ffff640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7406b4d0 0x7f2d7406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:10.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.435+0000 7f2d6ffff640 1 -- 172.21.15.64:0/3319005151 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f2d780abe70 con 0x7f2d8806fe20 2024-03-04T07:36:10.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.435+0000 7f2d867fc640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7406b4d0 0x7f2d7406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:10.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.435+0000 7f2d867fc640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7406b4d0 0x7f2d7406d990 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7f2d881f7940 tx=0x7f2d7c007d70 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:10.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.439+0000 7f2d6ffff640 1 -- 172.21.15.64:0/3319005151 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d881e5420 con 0x7f2d8806fe20 2024-03-04T07:36:10.641 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:10.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.658+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2d88067ff0 con 0x7f2d8806fe20 2024-03-04T07:36:10.658 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.658+0000 7f2d6ffff640 1 -- 172.21.15.64:0/3319005151 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f2d7803f070 con 0x7f2d8806fe20 2024-03-04T07:36:10.659 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:10.659 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:10.659 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7406b4d0 msgr2=0x7f2d7406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7406b4d0 0x7f2d7406d990 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7f2d881f7940 tx=0x7f2d7c007d70 comp rx=0 tx=0).stop 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d8806fe20 msgr2=0x7f2d8815b7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d8806fe20 0x7f2d8815b7a0 secure :-1 s=READY pgs=1737 cs=0 l=1 rev1=1 crypto rx=0x7f2d78000c00 tx=0x7f2d780046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 shutdown_connections 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d7406b4d0 0x7f2d7406d990 unknown :-1 s=CLOSED pgs=788 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d8815e060 0x7f2d880b5260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d880d0880 0x7f2d881e5670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 --2- 172.21.15.64:0/3319005151 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d8806fe20 0x7f2d8815b7a0 unknown :-1 s=CLOSED pgs=1737 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 >> 172.21.15.64:0/3319005151 conn(0x7f2d880d1a90 msgr2=0x7f2d881e32d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:10.661 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 shutdown_connections 2024-03-04T07:36:10.662 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:10.660+0000 7f2d8d837640 1 -- 172.21.15.64:0/3319005151 wait complete. 2024-03-04T07:36:10.873 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:10.873 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:11.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:11.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.988+0000 7fbc32fae640 1 Processor -- start 2024-03-04T07:36:11.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.988+0000 7fbc32fae640 1 -- start start 2024-03-04T07:36:11.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.989+0000 7fbc32fae640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbc2c0b0e90 0x7fbc2c06b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:11.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.989+0000 7fbc32fae640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.989+0000 7fbc32fae640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbc2c0cee90 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.989+0000 7fbc32fae640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbc2c0cfa00 con 0x7fbc2c0b0e90 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.989+0000 7fbc32fae640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbc2c0cfc50 con 0x7fbc2c0ae9b0 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.989+0000 7fbc2bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbc2c0b0e90 0x7fbc2c06b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.989+0000 7fbc2bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbc2c0b0e90 0x7fbc2c06b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50936/0 (socket says 172.21.15.64:50936) 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.990+0000 7fbc2bfff640 1 -- 172.21.15.64:0/2910452388 learned_addr learned my addr 172.21.15.64:0/2910452388 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.990+0000 7fbc3154b640 1 --2- 172.21.15.64:0/2910452388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.990+0000 7fbc2b7fe640 1 -- 172.21.15.64:0/2910452388 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1519077603 0 0) 0x7fbc2c0cee90 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.990+0000 7fbc2bfff640 1 -- 172.21.15.64:0/2910452388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 msgr2=0x7fbc2c0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.990+0000 7fbc2bfff640 1 --2- 172.21.15.64:0/2910452388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.990+0000 7fbc2bfff640 1 -- 172.21.15.64:0/2910452388 >> v1:172.21.15.66:6789/0 conn(0x7fbc2c06b5c0 legacy=0x7fbc2c0b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.990+0000 7fbc2bfff640 1 -- 172.21.15.64:0/2910452388 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc2c0cff30 con 0x7fbc2c0b0e90 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc2bfff640 1 --2- 172.21.15.64:0/2910452388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c0b0e90 0x7fbc2c06b9a0 secure :-1 s=READY pgs=1738 cs=0 l=1 rev1=1 crypto rx=0x7fbc20009870 tx=0x7fbc2002b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a8f6a59c0e0d6d99 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc2b7fe640 1 -- 172.21.15.64:0/2910452388 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc20035020 con 0x7fbc2c0b0e90 2024-03-04T07:36:11.991 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc2b7fe640 1 -- 172.21.15.64:0/2910452388 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc20031640 con 0x7fbc2c0b0e90 2024-03-04T07:36:11.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc2b7fe640 1 -- 172.21.15.64:0/2910452388 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc2002d040 con 0x7fbc2c0b0e90 2024-03-04T07:36:11.992 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 -- 172.21.15.64:0/2910452388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c0b0e90 msgr2=0x7fbc2c06b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 --2- 172.21.15.64:0/2910452388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c0b0e90 0x7fbc2c06b9a0 secure :-1 s=READY pgs=1738 cs=0 l=1 rev1=1 crypto rx=0x7fbc20009870 tx=0x7fbc2002b8f0 comp rx=0 tx=0).stop 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 -- 172.21.15.64:0/2910452388 shutdown_connections 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 --2- 172.21.15.64:0/2910452388 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 --2- 172.21.15.64:0/2910452388 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c0b0e90 0x7fbc2c06b9a0 unknown :-1 s=CLOSED pgs=1738 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 -- 172.21.15.64:0/2910452388 >> 172.21.15.64:0/2910452388 conn(0x7fbc2c0b36d0 msgr2=0x7fbc2c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 -- 172.21.15.64:0/2910452388 shutdown_connections 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.991+0000 7fbc32fae640 1 -- 172.21.15.64:0/2910452388 wait complete. 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.993+0000 7fbc32fae640 1 Processor -- start 2024-03-04T07:36:11.993 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.993+0000 7fbc32fae640 1 -- start start 2024-03-04T07:36:11.995 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c06b5c0 0x7fbc2c1d5ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0c92c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc2c0b0e90 0x7fbc2c0c9800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc2c06c910 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc2c06c790 con 0x7fbc2c0b0e90 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbc2c0aee00 con 0x7fbc2c0ae9b0 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c06b5c0 0x7fbc2c1d5ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c06b5c0 0x7fbc2c1d5ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:50950/0 (socket says 172.21.15.64:50950) 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 -- 172.21.15.64:0/1695587267 learned_addr learned my addr 172.21.15.64:0/1695587267 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc2bfff640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0c92c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc3154b640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc2c0b0e90 0x7fbc2c0c9800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 -- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 msgr2=0x7fbc2c0c92c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0c92c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:11.996 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 -- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc2c0b0e90 msgr2=0x7fbc2c0c9800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc2c0b0e90 0x7fbc2c0c9800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 -- 172.21.15.64:0/1695587267 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc2c1d61e0 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc30d4a640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c06b5c0 0x7fbc2c1d5ca0 secure :-1 s=READY pgs=1739 cs=0 l=1 rev1=1 crypto rx=0x7fbc2400c890 tx=0x7fbc2400cd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc297fa640 1 -- 172.21.15.64:0/1695587267 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc24017070 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc297fa640 1 -- 172.21.15.64:0/1695587267 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbc24003dc0 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc297fa640 1 -- 172.21.15.64:0/1695587267 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc240052b0 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc2c1e46d0 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.994+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc2c0c64a0 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.995+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc2c068700 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.997 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.996+0000 7fbc297fa640 1 -- 172.21.15.64:0/1695587267 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbc2401a070 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.996+0000 7fbc297fa640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:11.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.996+0000 7fbc297fa640 1 -- 172.21.15.64:0/1695587267 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbc24089900 con 0x7fbc2c06b5c0 2024-03-04T07:36:11.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.996+0000 7fbc2bfff640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:11.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:11.997+0000 7fbc2bfff640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 secure :-1 s=READY pgs=789 cs=0 l=1 rev1=1 crypto rx=0x7fbc20036000 tx=0x7fbc2003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:12.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.001+0000 7fbc297fa640 1 -- 172.21.15.64:0/1695587267 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbc2c1d61e0 con 0x7fbc2c06b5c0 2024-03-04T07:36:12.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.218+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbc2c0b1ad0 con 0x7fbc2c06b5c0 2024-03-04T07:36:12.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.218+0000 7fbc297fa640 1 -- 172.21.15.64:0/1695587267 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fbc240580f0 con 0x7fbc2c06b5c0 2024-03-04T07:36:12.219 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:12.219 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:12.219 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:12.220 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.220+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 msgr2=0x7fbc1806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.220+0000 7fbc32fae640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 secure :-1 s=READY pgs=789 cs=0 l=1 rev1=1 crypto rx=0x7fbc20036000 tx=0x7fbc2003c040 comp rx=0 tx=0).stop 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.220+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c06b5c0 msgr2=0x7fbc2c1d5ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.220+0000 7fbc32fae640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c06b5c0 0x7fbc2c1d5ca0 secure :-1 s=READY pgs=1739 cs=0 l=1 rev1=1 crypto rx=0x7fbc2400c890 tx=0x7fbc2400cd50 comp rx=0 tx=0).stop 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 shutdown_connections 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbc1806b4d0 0x7fbc1806d990 unknown :-1 s=CLOSED pgs=789 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbc2c0b0e90 0x7fbc2c0c9800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbc2c0ae9b0 0x7fbc2c0c92c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 --2- 172.21.15.64:0/1695587267 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbc2c06b5c0 0x7fbc2c1d5ca0 unknown :-1 s=CLOSED pgs=1739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 >> 172.21.15.64:0/1695587267 conn(0x7fbc2c0b36d0 msgr2=0x7fbc2c1e51d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 shutdown_connections 2024-03-04T07:36:12.221 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:12.221+0000 7fbc32fae640 1 -- 172.21.15.64:0/1695587267 wait complete. 2024-03-04T07:36:12.432 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:12.433 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:12.643 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:13.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:13.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.547+0000 7f1770081640 1 Processor -- start 2024-03-04T07:36:13.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.547+0000 7f1770081640 1 -- start start 2024-03-04T07:36:13.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f1770081640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f176806fe20 0x7f17680cea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:13.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f1770081640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17680d0dd0 con 0x7f176815e060 2024-03-04T07:36:13.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f1770081640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17680d10b0 con 0x7f17680d0880 2024-03-04T07:36:13.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f1770081640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17680cc0e0 con 0x7f176806fe20 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176de1d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f176815e060 0x7f17680ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35220/0 (socket says 172.21.15.64:35220) 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176de1d640 1 -- 172.21.15.64:0/247179281 learned_addr learned my addr 172.21.15.64:0/247179281 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176d61c640 1 --2- 172.21.15.64:0/247179281 >> v2:172.21.15.66:3300/0 conn(0x7f176806fe20 0x7f17680cea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176ce1b640 1 -- 172.21.15.64:0/247179281 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1883385184 0 0) 0x7f17680d0dd0 con 0x7f176815e060 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176ce1b640 1 -- 172.21.15.64:0/247179281 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1754003520 con 0x7f176815e060 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176ce1b640 1 -- 172.21.15.64:0/247179281 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 595339505 0 0) 0x7f17680d10b0 con 0x7f17680d0880 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176ce1b640 1 -- 172.21.15.64:0/247179281 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f17680d0dd0 con 0x7f17680d0880 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176d61c640 1 -- 172.21.15.64:0/247179281 >> v1:172.21.15.66:6789/0 conn(0x7f17680d0880 legacy=0x7f17680ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176d61c640 1 -- 172.21.15.64:0/247179281 >> v1:172.21.15.64:6789/0 conn(0x7f176815e060 legacy=0x7f17680ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:13.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.548+0000 7f176d61c640 1 -- 172.21.15.64:0/247179281 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17680cc3c0 con 0x7f176806fe20 2024-03-04T07:36:13.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.549+0000 7f176d61c640 1 --2- 172.21.15.64:0/247179281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f176806fe20 0x7f17680cea20 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f17580097a0 tx=0x7f175802b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c910d616bc6ec4bc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:13.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.549+0000 7f176ce1b640 1 -- 172.21.15.64:0/247179281 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1758034070 con 0x7f176806fe20 2024-03-04T07:36:13.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.549+0000 7f176ce1b640 1 -- 172.21.15.64:0/247179281 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1758030610 con 0x7f176806fe20 2024-03-04T07:36:13.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.549+0000 7f176ce1b640 1 -- 172.21.15.64:0/247179281 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17580309c0 con 0x7f176806fe20 2024-03-04T07:36:13.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.550+0000 7f1770081640 1 -- 172.21.15.64:0/247179281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f176806fe20 msgr2=0x7f17680cea20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:13.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.550+0000 7f1770081640 1 --2- 172.21.15.64:0/247179281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f176806fe20 0x7f17680cea20 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f17580097a0 tx=0x7f175802b960 comp rx=0 tx=0).stop 2024-03-04T07:36:13.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.550+0000 7f1770081640 1 -- 172.21.15.64:0/247179281 shutdown_connections 2024-03-04T07:36:13.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.550+0000 7f1770081640 1 --2- 172.21.15.64:0/247179281 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f176806fe20 0x7f17680cea20 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.550+0000 7f1770081640 1 -- 172.21.15.64:0/247179281 >> 172.21.15.64:0/247179281 conn(0x7f17680d1a90 msgr2=0x7f17680d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:13.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.550+0000 7f1770081640 1 -- 172.21.15.64:0/247179281 shutdown_connections 2024-03-04T07:36:13.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.550+0000 7f1770081640 1 -- 172.21.15.64:0/247179281 wait complete. 2024-03-04T07:36:13.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.551+0000 7f1770081640 1 Processor -- start 2024-03-04T07:36:13.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.551+0000 7f1770081640 1 -- start start 2024-03-04T07:36:13.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f1770081640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f176806fe20 0x7f176814dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:13.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f1770081640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17680d0880 0x7f17680c8b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:13.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f1770081640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f176815e060 0x7f176814e820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:13.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f1770081640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17680cef60 con 0x7f176806fe20 2024-03-04T07:36:13.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f1770081640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f176815e440 con 0x7f17680d0880 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f1770081640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17680cf0e0 con 0x7f176815e060 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f176806fe20 0x7f176814dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f176806fe20 0x7f176814dff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34830/0 (socket says 172.21.15.64:34830) 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 -- 172.21.15.64:0/2084571838 learned_addr learned my addr 172.21.15.64:0/2084571838 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176d61c640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17680d0880 0x7f17680c8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176e61e640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f176815e060 0x7f176814e820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 -- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f176815e060 msgr2=0x7f176814e820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f176815e060 0x7f176814e820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 -- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17680d0880 msgr2=0x7f17680c8b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17680d0880 0x7f17680c8b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 -- 172.21.15.64:0/2084571838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17680c6990 con 0x7f176806fe20 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176d61c640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17680d0880 0x7f17680c8b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.552+0000 7f176de1d640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f176806fe20 0x7f176814dff0 secure :-1 s=READY pgs=1741 cs=0 l=1 rev1=1 crypto rx=0x7f176400c410 tx=0x7f176400c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.553+0000 7f175e7fc640 1 -- 172.21.15.64:0/2084571838 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1764017070 con 0x7f176806fe20 2024-03-04T07:36:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.553+0000 7f175e7fc640 1 -- 172.21.15.64:0/2084571838 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1764005ce0 con 0x7f176806fe20 2024-03-04T07:36:13.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.553+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17680c6c70 con 0x7f176806fe20 2024-03-04T07:36:13.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.553+0000 7f175e7fc640 1 -- 172.21.15.64:0/2084571838 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17640072d0 con 0x7f176806fe20 2024-03-04T07:36:13.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.553+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17680c7230 con 0x7f176806fe20 2024-03-04T07:36:13.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.554+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1768067ff0 con 0x7f176806fe20 2024-03-04T07:36:13.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.554+0000 7f175e7fc640 1 -- 172.21.15.64:0/2084571838 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f176401a070 con 0x7f176806fe20 2024-03-04T07:36:13.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.554+0000 7f175e7fc640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f174c06b4d0 0x7f174c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:13.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.554+0000 7f175e7fc640 1 -- 172.21.15.64:0/2084571838 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1764089540 con 0x7f176806fe20 2024-03-04T07:36:13.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.555+0000 7f176d61c640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f174c06b4d0 0x7f174c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:13.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.555+0000 7f176d61c640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f174c06b4d0 0x7f174c06d990 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7f1758036000 tx=0x7f175803b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:13.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.559+0000 7f175e7fc640 1 -- 172.21.15.64:0/2084571838 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1764057d30 con 0x7f176806fe20 2024-03-04T07:36:13.775 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.776+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f17680d1530 con 0x7f176806fe20 2024-03-04T07:36:13.776 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.776+0000 7f175e7fc640 1 -- 172.21.15.64:0/2084571838 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f176405bdd0 con 0x7f176806fe20 2024-03-04T07:36:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:13.776 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:13.777 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:13.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f174c06b4d0 msgr2=0x7f174c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:13.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f174c06b4d0 0x7f174c06d990 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7f1758036000 tx=0x7f175803b040 comp rx=0 tx=0).stop 2024-03-04T07:36:13.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f176806fe20 msgr2=0x7f176814dff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:13.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f176806fe20 0x7f176814dff0 secure :-1 s=READY pgs=1741 cs=0 l=1 rev1=1 crypto rx=0x7f176400c410 tx=0x7f176400c8d0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 shutdown_connections 2024-03-04T07:36:13.778 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f174c06b4d0 0x7f174c06d990 unknown :-1 s=CLOSED pgs=790 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f176815e060 0x7f176814e820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17680d0880 0x7f17680c8b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 --2- 172.21.15.64:0/2084571838 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f176806fe20 0x7f176814dff0 unknown :-1 s=CLOSED pgs=1741 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:13.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 >> 172.21.15.64:0/2084571838 conn(0x7f17680d1a90 msgr2=0x7f176806bd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:13.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 shutdown_connections 2024-03-04T07:36:13.779 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:13.778+0000 7f1770081640 1 -- 172.21.15.64:0/2084571838 wait complete. 2024-03-04T07:36:13.990 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:13.990 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:14.645 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:14.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:15.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.104+0000 7ff0d981c640 1 Processor -- start 2024-03-04T07:36:15.105 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.104+0000 7ff0d981c640 1 -- start start 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d981c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff0d40d02e0 0x7ff0d40cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d981c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0d40b4260 0x7ff0d40b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d981c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff0d40b0dd0 con 0x7ff0d40d06c0 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d981c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0d40b0b80 con 0x7ff0d40b4260 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d981c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff0d40aeec0 con 0x7ff0d40d02e0 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d27fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0d40b4260 0x7ff0d40b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d27fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff0d40b4260 0x7ff0d40b1500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34832/0 (socket says 172.21.15.64:34832) 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d2ffd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff0d40d02e0 0x7ff0d40cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.105+0000 7ff0d27fc640 1 -- 172.21.15.64:0/1516329407 learned_addr learned my addr 172.21.15.64:0/1516329407 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.106+0000 7ff0d27fc640 1 -- 172.21.15.64:0/1516329407 >> v2:172.21.15.66:3300/0 conn(0x7ff0d40d02e0 msgr2=0x7ff0d40cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.106+0000 7ff0d27fc640 1 --2- 172.21.15.64:0/1516329407 >> v2:172.21.15.66:3300/0 conn(0x7ff0d40d02e0 0x7ff0d40cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.107 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.106+0000 7ff0d27fc640 1 -- 172.21.15.64:0/1516329407 >> v1:172.21.15.66:6789/0 conn(0x7ff0d40d06c0 legacy=0x7ff0d40b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:15.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.106+0000 7ff0d27fc640 1 -- 172.21.15.64:0/1516329407 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0d40b10b0 con 0x7ff0d40b4260 2024-03-04T07:36:15.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.106+0000 7ff0d27fc640 1 --2- 172.21.15.64:0/1516329407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40b4260 0x7ff0d40b1500 secure :-1 s=READY pgs=1742 cs=0 l=1 rev1=1 crypto rx=0x7ff0d4071b00 tx=0x7ff0bc02ba30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f15dd382c85ef696 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:15.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d1ffb640 1 -- 172.21.15.64:0/1516329407 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0bc02f850 con 0x7ff0d40b4260 2024-03-04T07:36:15.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d1ffb640 1 -- 172.21.15.64:0/1516329407 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0bc02f9f0 con 0x7ff0d40b4260 2024-03-04T07:36:15.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d1ffb640 1 -- 172.21.15.64:0/1516329407 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0bc02fdc0 con 0x7ff0d40b4260 2024-03-04T07:36:15.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 -- 172.21.15.64:0/1516329407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40b4260 msgr2=0x7ff0d40b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:15.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 --2- 172.21.15.64:0/1516329407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40b4260 0x7ff0d40b1500 secure :-1 s=READY pgs=1742 cs=0 l=1 rev1=1 crypto rx=0x7ff0d4071b00 tx=0x7ff0bc02ba30 comp rx=0 tx=0).stop 2024-03-04T07:36:15.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 -- 172.21.15.64:0/1516329407 shutdown_connections 2024-03-04T07:36:15.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 --2- 172.21.15.64:0/1516329407 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40b4260 0x7ff0d40b1500 unknown :-1 s=CLOSED pgs=1742 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 --2- 172.21.15.64:0/1516329407 >> v2:172.21.15.66:3300/0 conn(0x7ff0d40d02e0 0x7ff0d40cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 -- 172.21.15.64:0/1516329407 >> 172.21.15.64:0/1516329407 conn(0x7ff0d40b36d0 msgr2=0x7ff0d40b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:15.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 -- 172.21.15.64:0/1516329407 shutdown_connections 2024-03-04T07:36:15.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.107+0000 7ff0d981c640 1 -- 172.21.15.64:0/1516329407 wait complete. 2024-03-04T07:36:15.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.108+0000 7ff0d981c640 1 Processor -- start 2024-03-04T07:36:15.110 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d981c640 1 -- start start 2024-03-04T07:36:15.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d981c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0d40b4260 0x7ff0d41511b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:15.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d981c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0d40d02e0 0x7ff0d415d770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:15.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d981c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40d06c0 0x7ff0d415dcb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:15.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d981c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0d40b22d0 con 0x7ff0d40d06c0 2024-03-04T07:36:15.112 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d981c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff0d40b2150 con 0x7ff0d40d02e0 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d981c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff0d40af350 con 0x7ff0d40b4260 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40d06c0 0x7ff0d415dcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d27fc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0d40d02e0 0x7ff0d415d770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d37fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40d06c0 0x7ff0d415dcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34834/0 (socket says 172.21.15.64:34834) 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d2ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0d40b4260 0x7ff0d41511b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d27fc640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0d40d02e0 0x7ff0d415d770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:34574/0 (socket says 172.21.15.64:34574) 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.109+0000 7ff0d37fe640 1 -- 172.21.15.64:0/3953090955 learned_addr learned my addr 172.21.15.64:0/3953090955 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d37fe640 1 -- 172.21.15.64:0/3953090955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0d40b4260 msgr2=0x7ff0d41511b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d37fe640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0d40b4260 0x7ff0d41511b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d37fe640 1 -- 172.21.15.64:0/3953090955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0d40d02e0 msgr2=0x7ff0d415d770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d37fe640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0d40d02e0 0x7ff0d415d770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d37fe640 1 -- 172.21.15.64:0/3953090955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff0d41f1c60 con 0x7ff0d40d06c0 2024-03-04T07:36:15.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d27fc640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0d40d02e0 0x7ff0d415d770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d37fe640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40d06c0 0x7ff0d415dcb0 secure :-1 s=READY pgs=1743 cs=0 l=1 rev1=1 crypto rx=0x7ff0c400c6b0 tx=0x7ff0c400cb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0cbfff640 1 -- 172.21.15.64:0/3953090955 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0c4003b70 con 0x7ff0d40d06c0 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0cbfff640 1 -- 172.21.15.64:0/3953090955 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff0c4003d30 con 0x7ff0d40d06c0 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff0d41556c0 con 0x7ff0d40d06c0 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0cbfff640 1 -- 172.21.15.64:0/3953090955 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff0c4005310 con 0x7ff0d40d06c0 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.110+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff0d41559a0 con 0x7ff0d40d06c0 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.111+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff0d4068700 con 0x7ff0d40d06c0 2024-03-04T07:36:15.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.112+0000 7ff0cbfff640 1 -- 172.21.15.64:0/3953090955 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff0c40054b0 con 0x7ff0d40d06c0 2024-03-04T07:36:15.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.112+0000 7ff0cbfff640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0b406b4d0 0x7ff0b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:15.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.112+0000 7ff0cbfff640 1 -- 172.21.15.64:0/3953090955 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff0c4089f60 con 0x7ff0d40d06c0 2024-03-04T07:36:15.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.112+0000 7ff0d2ffd640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0b406b4d0 0x7ff0b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:15.115 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.113+0000 7ff0d2ffd640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0b406b4d0 0x7ff0b406d990 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7ff0cc009b40 tx=0x7ff0cc009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:15.117 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.117+0000 7ff0cbfff640 1 -- 172.21.15.64:0/3953090955 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff0c4058750 con 0x7ff0d40d06c0 2024-03-04T07:36:15.340 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.339+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff0d415c150 con 0x7ff0d40d06c0 2024-03-04T07:36:15.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.340+0000 7ff0cbfff640 1 -- 172.21.15.64:0/3953090955 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7ff0c405c7f0 con 0x7ff0d40d06c0 2024-03-04T07:36:15.341 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:15.341 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:15.342 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:15.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0b406b4d0 msgr2=0x7ff0b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:15.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0b406b4d0 0x7ff0b406d990 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7ff0cc009b40 tx=0x7ff0cc009190 comp rx=0 tx=0).stop 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40d06c0 msgr2=0x7ff0d415dcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40d06c0 0x7ff0d415dcb0 secure :-1 s=READY pgs=1743 cs=0 l=1 rev1=1 crypto rx=0x7ff0c400c6b0 tx=0x7ff0c400cb70 comp rx=0 tx=0).stop 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 shutdown_connections 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff0b406b4d0 0x7ff0b406d990 unknown :-1 s=CLOSED pgs=791 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff0d40d06c0 0x7ff0d415dcb0 unknown :-1 s=CLOSED pgs=1743 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff0d40d02e0 0x7ff0d415d770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 --2- 172.21.15.64:0/3953090955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff0d40b4260 0x7ff0d41511b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 >> 172.21.15.64:0/3953090955 conn(0x7ff0d40b36d0 msgr2=0x7ff0d40cd310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 shutdown_connections 2024-03-04T07:36:15.344 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:15.342+0000 7ff0d981c640 1 -- 172.21.15.64:0/3953090955 wait complete. 2024-03-04T07:36:15.555 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:15.555 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:16.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:16.647 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.669+0000 7fcf77393640 1 Processor -- start 2024-03-04T07:36:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.669+0000 7fcf77393640 1 -- start start 2024-03-04T07:36:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf77393640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcf7015e060 0x7fcf700ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf77393640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcf7006fe20 0x7fcf700ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf77393640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcf700d0dd0 con 0x7fcf700d0880 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf77393640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcf700d10b0 con 0x7fcf7006fe20 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf77393640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcf700cc0e0 con 0x7fcf7015e060 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf7492e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcf7006fe20 0x7fcf700ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf75930640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fcf700d0880 0x7fcf7006ed30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35232/0 (socket says 172.21.15.64:35232) 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf75930640 1 -- 172.21.15.64:0/1971501362 learned_addr learned my addr 172.21.15.64:0/1971501362 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf7512f640 1 --2- 172.21.15.64:0/1971501362 >> v2:172.21.15.66:3300/0 conn(0x7fcf7015e060 0x7fcf700ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf7492e640 1 -- 172.21.15.64:0/1971501362 >> v2:172.21.15.66:3300/0 conn(0x7fcf7015e060 msgr2=0x7fcf700ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf7492e640 1 --2- 172.21.15.64:0/1971501362 >> v2:172.21.15.66:3300/0 conn(0x7fcf7015e060 0x7fcf700ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf7492e640 1 -- 172.21.15.64:0/1971501362 >> v1:172.21.15.64:6789/0 conn(0x7fcf700d0880 legacy=0x7fcf7006ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf67fff640 1 -- 172.21.15.64:0/1971501362 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2622590730 0 0) 0x7fcf700d0dd0 con 0x7fcf700d0880 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.670+0000 7fcf7492e640 1 -- 172.21.15.64:0/1971501362 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcf700cc3c0 con 0x7fcf7006fe20 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.671+0000 7fcf7492e640 1 --2- 172.21.15.64:0/1971501362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf7006fe20 0x7fcf700ce850 secure :-1 s=READY pgs=1745 cs=0 l=1 rev1=1 crypto rx=0x7fcf700722f0 tx=0x7fcf6002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b66b0bbe81f6a8df server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.671+0000 7fcf67fff640 1 -- 172.21.15.64:0/1971501362 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf60035020 con 0x7fcf7006fe20 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.671+0000 7fcf67fff640 1 -- 172.21.15.64:0/1971501362 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcf60031830 con 0x7fcf7006fe20 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.671+0000 7fcf67fff640 1 -- 172.21.15.64:0/1971501362 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf60031c00 con 0x7fcf7006fe20 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 -- 172.21.15.64:0/1971501362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf7006fe20 msgr2=0x7fcf700ce850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 --2- 172.21.15.64:0/1971501362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf7006fe20 0x7fcf700ce850 secure :-1 s=READY pgs=1745 cs=0 l=1 rev1=1 crypto rx=0x7fcf700722f0 tx=0x7fcf6002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 -- 172.21.15.64:0/1971501362 shutdown_connections 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 --2- 172.21.15.64:0/1971501362 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf7006fe20 0x7fcf700ce850 unknown :-1 s=CLOSED pgs=1745 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 --2- 172.21.15.64:0/1971501362 >> v2:172.21.15.66:3300/0 conn(0x7fcf7015e060 0x7fcf700ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 -- 172.21.15.64:0/1971501362 >> 172.21.15.64:0/1971501362 conn(0x7fcf700d1a90 msgr2=0x7fcf700d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 -- 172.21.15.64:0/1971501362 shutdown_connections 2024-03-04T07:36:16.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.672+0000 7fcf77393640 1 -- 172.21.15.64:0/1971501362 wait complete. 2024-03-04T07:36:16.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.673+0000 7fcf77393640 1 Processor -- start 2024-03-04T07:36:16.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.673+0000 7fcf77393640 1 -- start start 2024-03-04T07:36:16.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf77393640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcf7006fe20 0x7fcf700c7df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:16.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf77393640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf700d0880 0x7fcf70149740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:16.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf77393640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcf7015e060 0x7fcf700ce7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf77393640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf700cdbd0 con 0x7fcf700d0880 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf77393640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf7015e440 con 0x7fcf7006fe20 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf77393640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcf700cdd50 con 0x7fcf7015e060 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf700d0880 0x7fcf70149740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf700d0880 0x7fcf70149740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34854/0 (socket says 172.21.15.64:34854) 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 -- 172.21.15.64:0/2647433352 learned_addr learned my addr 172.21.15.64:0/2647433352 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7512f640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcf7006fe20 0x7fcf700c7df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf75930640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcf7015e060 0x7fcf700ce7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 -- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcf7015e060 msgr2=0x7fcf700ce7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcf7015e060 0x7fcf700ce7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 -- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcf7006fe20 msgr2=0x7fcf700c7df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcf7006fe20 0x7fcf700c7df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 -- 172.21.15.64:0/2647433352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcf700c8330 con 0x7fcf700d0880 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7512f640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcf7006fe20 0x7fcf700c7df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.674+0000 7fcf7492e640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf700d0880 0x7fcf70149740 secure :-1 s=READY pgs=1746 cs=0 l=1 rev1=1 crypto rx=0x7fcf60000c00 tx=0x7fcf600046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.675+0000 7fcf65ffb640 1 -- 172.21.15.64:0/2647433352 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf60040070 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.675+0000 7fcf65ffb640 1 -- 172.21.15.64:0/2647433352 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcf6003a910 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.675+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcf70149c80 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.675+0000 7fcf65ffb640 1 -- 172.21.15.64:0/2647433352 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf60035040 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.675+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcf700b6ea0 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.676+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcf70067ff0 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.676+0000 7fcf65ffb640 1 -- 172.21.15.64:0/2647433352 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcf6004b060 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.677+0000 7fcf65ffb640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf5406b4d0 0x7fcf5406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.677+0000 7fcf65ffb640 1 -- 172.21.15.64:0/2647433352 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fcf600ab8f0 con 0x7fcf700d0880 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.677+0000 7fcf7512f640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf5406b4d0 0x7fcf5406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:16.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.677+0000 7fcf7512f640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf5406b4d0 0x7fcf5406d990 secure :-1 s=READY pgs=792 cs=0 l=1 rev1=1 crypto rx=0x7fcf701f6b70 tx=0x7fcf6c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:16.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.681+0000 7fcf65ffb640 1 -- 172.21.15.64:0/2647433352 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcf70149c80 con 0x7fcf700d0880 2024-03-04T07:36:16.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.901+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fcf700d1f10 con 0x7fcf700d0880 2024-03-04T07:36:16.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.901+0000 7fcf65ffb640 1 -- 172.21.15.64:0/2647433352 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fcf60031800 con 0x7fcf700d0880 2024-03-04T07:36:16.902 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:16.903 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:16.903 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf5406b4d0 msgr2=0x7fcf5406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf5406b4d0 0x7fcf5406d990 secure :-1 s=READY pgs=792 cs=0 l=1 rev1=1 crypto rx=0x7fcf701f6b70 tx=0x7fcf6c009190 comp rx=0 tx=0).stop 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf700d0880 msgr2=0x7fcf70149740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf700d0880 0x7fcf70149740 secure :-1 s=READY pgs=1746 cs=0 l=1 rev1=1 crypto rx=0x7fcf60000c00 tx=0x7fcf600046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 shutdown_connections 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcf5406b4d0 0x7fcf5406d990 unknown :-1 s=CLOSED pgs=792 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcf7015e060 0x7fcf700ce7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcf700d0880 0x7fcf70149740 unknown :-1 s=CLOSED pgs=1746 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 --2- 172.21.15.64:0/2647433352 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcf7006fe20 0x7fcf700c7df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 >> 172.21.15.64:0/2647433352 conn(0x7fcf700d1a90 msgr2=0x7fcf700b3560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 shutdown_connections 2024-03-04T07:36:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:16.903+0000 7fcf77393640 1 -- 172.21.15.64:0/2647433352 wait complete. 2024-03-04T07:36:17.117 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:17.117 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:18.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:18.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.232+0000 7fac1cdee640 1 Processor -- start 2024-03-04T07:36:18.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.232+0000 7fac1cdee640 1 -- start start 2024-03-04T07:36:18.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac1cdee640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fac18072660 0x7fac1815ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:18.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac1cdee640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac180b55d0 0x7fac18071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:18.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac1cdee640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fac180b4160 con 0x7fac1806e700 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac1cdee640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fac180b5b40 con 0x7fac18072660 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac1cdee640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fac180b3ba0 con 0x7fac180b55d0 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac16d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac180b55d0 0x7fac18071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac0ffff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fac18072660 0x7fac1815ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac16d9d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac180b55d0 0x7fac18071980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:47702/0 (socket says 172.21.15.64:47702) 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac1659c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fac1806e700 0x7fac1806d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:54280/0 (socket says 172.21.15.64:54280) 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac16d9d640 1 -- 172.21.15.64:0/1888882769 learned_addr learned my addr 172.21.15.64:0/1888882769 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac16d9d640 1 -- 172.21.15.64:0/1888882769 >> v2:172.21.15.66:3300/0 conn(0x7fac18072660 msgr2=0x7fac1815ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac16d9d640 1 --2- 172.21.15.64:0/1888882769 >> v2:172.21.15.66:3300/0 conn(0x7fac18072660 0x7fac1815ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac16d9d640 1 -- 172.21.15.64:0/1888882769 >> v1:172.21.15.66:6789/0 conn(0x7fac1806e700 legacy=0x7fac1806d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac16d9d640 1 -- 172.21.15.64:0/1888882769 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac180b3e80 con 0x7fac180b55d0 2024-03-04T07:36:18.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.233+0000 7fac0ffff640 1 --2- 172.21.15.64:0/1888882769 >> v2:172.21.15.66:3300/0 conn(0x7fac18072660 0x7fac1815ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:18.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.234+0000 7fac16d9d640 1 --2- 172.21.15.64:0/1888882769 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac180b55d0 0x7fac18071980 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fac10009e50 tx=0x7fac1002bf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5cd09f3715c8d259 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:18.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.234+0000 7fac15d9b640 1 -- 172.21.15.64:0/1888882769 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac1002cb00 con 0x7fac180b55d0 2024-03-04T07:36:18.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.234+0000 7fac15d9b640 1 -- 172.21.15.64:0/1888882769 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fac1002cca0 con 0x7fac180b55d0 2024-03-04T07:36:18.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.234+0000 7fac15d9b640 1 -- 172.21.15.64:0/1888882769 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac10034610 con 0x7fac180b55d0 2024-03-04T07:36:18.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 -- 172.21.15.64:0/1888882769 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac180b55d0 msgr2=0x7fac18071980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1888882769 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac180b55d0 0x7fac18071980 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fac10009e50 tx=0x7fac1002bf60 comp rx=0 tx=0).stop 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 -- 172.21.15.64:0/1888882769 shutdown_connections 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1888882769 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac180b55d0 0x7fac18071980 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1888882769 >> v2:172.21.15.66:3300/0 conn(0x7fac18072660 0x7fac1815ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 -- 172.21.15.64:0/1888882769 >> 172.21.15.64:0/1888882769 conn(0x7fac180b2720 msgr2=0x7fac180b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 -- 172.21.15.64:0/1888882769 shutdown_connections 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.235+0000 7fac1cdee640 1 -- 172.21.15.64:0/1888882769 wait complete. 2024-03-04T07:36:18.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.236+0000 7fac1cdee640 1 Processor -- start 2024-03-04T07:36:18.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.236+0000 7fac1cdee640 1 -- start start 2024-03-04T07:36:18.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1cdee640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac1806e700 0x7fac180cf930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:18.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1cdee640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac18072660 0x7fac1815ff80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1cdee640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac180b55d0 0x7fac181f7230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1cdee640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac18072040 con 0x7fac18072660 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1cdee640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac18071ec0 con 0x7fac180b55d0 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1cdee640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fac180721c0 con 0x7fac1806e700 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac0ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac18072660 0x7fac1815ff80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1659c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac1806e700 0x7fac180cf930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac0ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac18072660 0x7fac1815ff80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34860/0 (socket says 172.21.15.64:34860) 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac16d9d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac180b55d0 0x7fac181f7230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac0ffff640 1 -- 172.21.15.64:0/1224160434 learned_addr learned my addr 172.21.15.64:0/1224160434 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1659c640 1 -- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac180b55d0 msgr2=0x7fac181f7230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1659c640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac180b55d0 0x7fac181f7230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1659c640 1 -- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac18072660 msgr2=0x7fac1815ff80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1659c640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac18072660 0x7fac1815ff80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac1659c640 1 -- 172.21.15.64:0/1224160434 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac180ce890 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.237+0000 7fac16d9d640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac180b55d0 0x7fac181f7230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.238+0000 7fac1659c640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac1806e700 0x7fac180cf930 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fac08000f60 tx=0x7fac0800c800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.238+0000 7fac0effd640 1 -- 172.21.15.64:0/1224160434 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac080074e0 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.238+0000 7fac0effd640 1 -- 172.21.15.64:0/1224160434 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fac080047e0 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.238+0000 7fac0effd640 1 -- 172.21.15.64:0/1224160434 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac08004bd0 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.238+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fac180ceb70 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.238+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fac180cf130 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.239+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fac18067ff0 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.239+0000 7fac0effd640 1 -- 172.21.15.64:0/1224160434 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fac0801a020 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.240+0000 7fac0effd640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fabf806b4d0 0x7fabf806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.240+0000 7fac0effd640 1 -- 172.21.15.64:0/1224160434 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fac08089850 con 0x7fac1806e700 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.240+0000 7fac0ffff640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fabf806b4d0 0x7fabf806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.240+0000 7fac0ffff640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fabf806b4d0 0x7fabf806d990 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7fac00009560 tx=0x7fac00009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:18.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.244+0000 7fac0effd640 1 -- 172.21.15.64:0/1224160434 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fac08058040 con 0x7fac1806e700 2024-03-04T07:36:18.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:36:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.465+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fac18072c30 con 0x7fac1806e700 2024-03-04T07:36:18.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.466+0000 7fac0effd640 1 -- 172.21.15.64:0/1224160434 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fac0805c0e0 con 0x7fac1806e700 2024-03-04T07:36:18.467 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:18.467 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:18.467 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:18.469 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:36:18.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fabf806b4d0 msgr2=0x7fabf806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fabf806b4d0 0x7fabf806d990 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7fac00009560 tx=0x7fac00009190 comp rx=0 tx=0).stop 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac1806e700 msgr2=0x7fac180cf930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac1806e700 0x7fac180cf930 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fac08000f60 tx=0x7fac0800c800 comp rx=0 tx=0).stop 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 shutdown_connections 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fabf806b4d0 0x7fabf806d990 unknown :-1 s=CLOSED pgs=793 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac180b55d0 0x7fac181f7230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac18072660 0x7fac1815ff80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 --2- 172.21.15.64:0/1224160434 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac1806e700 0x7fac180cf930 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 >> 172.21.15.64:0/1224160434 conn(0x7fac180b2720 msgr2=0x7fac1815c090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 shutdown_connections 2024-03-04T07:36:18.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:18.468+0000 7fac1cdee640 1 -- 172.21.15.64:0/1224160434 wait complete. 2024-03-04T07:36:18.649 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:18.681 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:18.681 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:19.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:19.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.794+0000 7fbeaa5e9640 1 Processor -- start 2024-03-04T07:36:19.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.794+0000 7fbeaa5e9640 1 -- start start 2024-03-04T07:36:19.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbeaa5e9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbea406af20 0x7fbea415d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:19.796 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbeaa5e9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbea40cc0e0 con 0x7fbea40b1020 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbeaa5e9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbea40cc3c0 con 0x7fbea406ed10 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbeaa5e9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbea40d28b0 con 0x7fbea406af20 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbea8b86640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbea406af20 0x7fbea415d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbea8b86640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbea406af20 0x7fbea415d700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34874/0 (socket says 172.21.15.64:34874) 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbea3fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbea40b1020 0x7fbea40cd180 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35240/0 (socket says 172.21.15.64:35240) 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbea8b86640 1 -- 172.21.15.64:0/3745945633 learned_addr learned my addr 172.21.15.64:0/3745945633 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbea8b86640 1 -- 172.21.15.64:0/3745945633 >> v1:172.21.15.66:6789/0 conn(0x7fbea406ed10 legacy=0x7fbea40cd890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbea8b86640 1 -- 172.21.15.64:0/3745945633 >> v1:172.21.15.64:6789/0 conn(0x7fbea40b1020 legacy=0x7fbea40cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.795+0000 7fbea2ffd640 1 -- 172.21.15.64:0/3745945633 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 59026158 0 0) 0x7fbea40cc3c0 con 0x7fbea406ed10 2024-03-04T07:36:19.797 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.796+0000 7fbea8b86640 1 -- 172.21.15.64:0/3745945633 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbea40cc6a0 con 0x7fbea406af20 2024-03-04T07:36:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.796+0000 7fbea8b86640 1 --2- 172.21.15.64:0/3745945633 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea415d700 secure :-1 s=READY pgs=1748 cs=0 l=1 rev1=1 crypto rx=0x7fbe9c009e50 tx=0x7fbe9c02be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=117ebe626a0fc392 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.796+0000 7fbea2ffd640 1 -- 172.21.15.64:0/3745945633 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe9c02c960 con 0x7fbea406af20 2024-03-04T07:36:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.796+0000 7fbea2ffd640 1 -- 172.21.15.64:0/3745945633 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe9c02cb00 con 0x7fbea406af20 2024-03-04T07:36:19.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.796+0000 7fbea2ffd640 1 -- 172.21.15.64:0/3745945633 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe9c0343f0 con 0x7fbea406af20 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.797+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/3745945633 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 msgr2=0x7fbea415d700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.797+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/3745945633 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea415d700 secure :-1 s=READY pgs=1748 cs=0 l=1 rev1=1 crypto rx=0x7fbe9c009e50 tx=0x7fbe9c02be40 comp rx=0 tx=0).stop 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.797+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/3745945633 shutdown_connections 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.797+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/3745945633 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea415d700 unknown :-1 s=CLOSED pgs=1748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.797+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/3745945633 >> 172.21.15.64:0/3745945633 conn(0x7fbea40d1a90 msgr2=0x7fbea40d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.797+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/3745945633 shutdown_connections 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.797+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/3745945633 wait complete. 2024-03-04T07:36:19.799 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.798+0000 7fbeaa5e9640 1 Processor -- start 2024-03-04T07:36:19.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbeaa5e9640 1 -- start start 2024-03-04T07:36:19.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbeaa5e9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea4157460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:19.801 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbeaa5e9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbea406ed10 0x7fbea40b33c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbeaa5e9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbea40b1020 0x7fbea40cd920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbeaa5e9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbea40ccc60 con 0x7fbea406af20 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbeaa5e9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbea40cf320 con 0x7fbea406ed10 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbeaa5e9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbea415dc40 con 0x7fbea40b1020 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbea3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea4157460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbea3fff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea4157460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34888/0 (socket says 172.21.15.64:34888) 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbea3fff640 1 -- 172.21.15.64:0/1318475210 learned_addr learned my addr 172.21.15.64:0/1318475210 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbea37fe640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbea406ed10 0x7fbea40b33c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.799+0000 7fbea8b86640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbea40b1020 0x7fbea40cd920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea3fff640 1 -- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbea40b1020 msgr2=0x7fbea40cd920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea3fff640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbea40b1020 0x7fbea40cd920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea3fff640 1 -- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbea406ed10 msgr2=0x7fbea40b33c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:19.802 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea3fff640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbea406ed10 0x7fbea40b33c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea3fff640 1 -- 172.21.15.64:0/1318475210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbea40b5e70 con 0x7fbea406af20 2024-03-04T07:36:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea3fff640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea4157460 secure :-1 s=READY pgs=1749 cs=0 l=1 rev1=1 crypto rx=0x7fbe9800c780 tx=0x7fbe9800cc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea0ff9640 1 -- 172.21.15.64:0/1318475210 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe98018070 con 0x7fbea406af20 2024-03-04T07:36:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea0ff9640 1 -- 172.21.15.64:0/1318475210 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbe980072d0 con 0x7fbea406af20 2024-03-04T07:36:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbea40b6150 con 0x7fbea406af20 2024-03-04T07:36:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbea0ff9640 1 -- 172.21.15.64:0/1318475210 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe980047e0 con 0x7fbea406af20 2024-03-04T07:36:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.800+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbea40b6710 con 0x7fbea406af20 2024-03-04T07:36:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.801+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbea4067ff0 con 0x7fbea406af20 2024-03-04T07:36:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.802+0000 7fbea0ff9640 1 -- 172.21.15.64:0/1318475210 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbe9801b070 con 0x7fbea406af20 2024-03-04T07:36:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.802+0000 7fbea0ff9640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe9006b4d0 0x7fbe9006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.802+0000 7fbea37fe640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe9006b4d0 0x7fbe9006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.802+0000 7fbea0ff9640 1 -- 172.21.15.64:0/1318475210 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbe980895c0 con 0x7fbea406af20 2024-03-04T07:36:19.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.803+0000 7fbea37fe640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe9006b4d0 0x7fbe9006d990 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7fbea40d0e70 tx=0x7fbe94009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:19.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:19.807+0000 7fbea0ff9640 1 -- 172.21.15.64:0/1318475210 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbe98057e70 con 0x7fbea406af20 2024-03-04T07:36:20.031 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.030+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbea40b3b30 con 0x7fbea406af20 2024-03-04T07:36:20.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.031+0000 7fbea0ff9640 1 -- 172.21.15.64:0/1318475210 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fbe98006e20 con 0x7fbea406af20 2024-03-04T07:36:20.032 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:20.032 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:20.032 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe9006b4d0 msgr2=0x7fbe9006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe9006b4d0 0x7fbe9006d990 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7fbea40d0e70 tx=0x7fbe94009040 comp rx=0 tx=0).stop 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 msgr2=0x7fbea4157460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea4157460 secure :-1 s=READY pgs=1749 cs=0 l=1 rev1=1 crypto rx=0x7fbe9800c780 tx=0x7fbe9800cc40 comp rx=0 tx=0).stop 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 shutdown_connections 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbe9006b4d0 0x7fbe9006d990 unknown :-1 s=CLOSED pgs=794 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbea40b1020 0x7fbea40cd920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbea406ed10 0x7fbea40b33c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 --2- 172.21.15.64:0/1318475210 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbea406af20 0x7fbea4157460 unknown :-1 s=CLOSED pgs=1749 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:20.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 >> 172.21.15.64:0/1318475210 conn(0x7fbea40d1a90 msgr2=0x7fbea40d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:20.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 shutdown_connections 2024-03-04T07:36:20.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:20.033+0000 7fbeaa5e9640 1 -- 172.21.15.64:0/1318475210 wait complete. 2024-03-04T07:36:20.246 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:20.246 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:20.652 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:21.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:21.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.359+0000 7f68d8125640 1 Processor -- start 2024-03-04T07:36:21.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.360+0000 7f68d8125640 1 -- start start 2024-03-04T07:36:21.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.360+0000 7f68d8125640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f68d015df00 0x7f68d00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:21.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.360+0000 7f68d8125640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00b9a30 0x7f68d00d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:21.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.360+0000 7f68d8125640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f68d00cea30 con 0x7f68d00b95a0 2024-03-04T07:36:21.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.360+0000 7f68d8125640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f68d00d2af0 con 0x7f68d015df00 2024-03-04T07:36:21.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.360+0000 7f68d8125640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f68d00cf3c0 con 0x7f68d00b9a30 2024-03-04T07:36:21.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.360+0000 7f68d5ec1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f68d015df00 0x7f68d00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:21.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.361+0000 7f68d56c0640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f68d00b95a0 0x7f68d00b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35242/0 (socket says 172.21.15.64:35242) 2024-03-04T07:36:21.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.361+0000 7f68d56c0640 1 -- 172.21.15.64:0/52680480 learned_addr learned my addr 172.21.15.64:0/52680480 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:21.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.361+0000 7f68d66c2640 1 --2- 172.21.15.64:0/52680480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00b9a30 0x7f68d00d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:21.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.361+0000 7f68d5ec1640 1 -- 172.21.15.64:0/52680480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00b9a30 msgr2=0x7f68d00d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:21.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.361+0000 7f68d5ec1640 1 --2- 172.21.15.64:0/52680480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00b9a30 0x7f68d00d3220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.361+0000 7f68d5ec1640 1 -- 172.21.15.64:0/52680480 >> v1:172.21.15.64:6789/0 conn(0x7f68d00b95a0 legacy=0x7f68d00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:21.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.361+0000 7f68d5ec1640 1 -- 172.21.15.64:0/52680480 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68d00cf0e0 con 0x7f68d015df00 2024-03-04T07:36:21.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d5ec1640 1 --2- 172.21.15.64:0/52680480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d015df00 0x7f68d00d25b0 secure :-1 s=READY pgs=1751 cs=0 l=1 rev1=1 crypto rx=0x7f68c80097a0 tx=0x7f68c802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4eaa74df77157e65 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:21.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d4ebf640 1 -- 172.21.15.64:0/52680480 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68c8035020 con 0x7f68d015df00 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d4ebf640 1 -- 172.21.15.64:0/52680480 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f68c8031830 con 0x7f68d015df00 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d4ebf640 1 -- 172.21.15.64:0/52680480 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68c8031c00 con 0x7f68d015df00 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 -- 172.21.15.64:0/52680480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d015df00 msgr2=0x7f68d00d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 --2- 172.21.15.64:0/52680480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d015df00 0x7f68d00d25b0 secure :-1 s=READY pgs=1751 cs=0 l=1 rev1=1 crypto rx=0x7f68c80097a0 tx=0x7f68c802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 -- 172.21.15.64:0/52680480 shutdown_connections 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 --2- 172.21.15.64:0/52680480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00b9a30 0x7f68d00d3220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 --2- 172.21.15.64:0/52680480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d015df00 0x7f68d00d25b0 unknown :-1 s=CLOSED pgs=1751 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 -- 172.21.15.64:0/52680480 >> 172.21.15.64:0/52680480 conn(0x7f68d0071960 msgr2=0x7f68d0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 -- 172.21.15.64:0/52680480 shutdown_connections 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.362+0000 7f68d8125640 1 -- 172.21.15.64:0/52680480 wait complete. 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 Processor -- start 2024-03-04T07:36:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 -- start start 2024-03-04T07:36:21.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d00b95a0 0x7f68d00c8870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68d015df00 0x7f68d0150fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00ca4a0 0x7f68d015e9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68d00cec70 con 0x7f68d00b95a0 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68d00ce750 con 0x7f68d015df00 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d8125640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f68d00cedf0 con 0x7f68d00ca4a0 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.364+0000 7f68d5ec1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d00b95a0 0x7f68d00c8870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d00b95a0 0x7f68d00c8870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34908/0 (socket says 172.21.15.64:34908) 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 -- 172.21.15.64:0/517371365 learned_addr learned my addr 172.21.15.64:0/517371365 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d66c2640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00ca4a0 0x7f68d015e9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d56c0640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68d015df00 0x7f68d0150fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 -- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00ca4a0 msgr2=0x7f68d015e9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00ca4a0 0x7f68d015e9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 -- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68d015df00 msgr2=0x7f68d0150fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68d015df00 0x7f68d0150fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 -- 172.21.15.64:0/517371365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68d00c9400 con 0x7f68d00b95a0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d5ec1640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d00b95a0 0x7f68d00c8870 secure :-1 s=READY pgs=1752 cs=0 l=1 rev1=1 crypto rx=0x7f68c80098c0 tx=0x7f68c80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68c67fc640 1 -- 172.21.15.64:0/517371365 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68c8040070 con 0x7f68d00b95a0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68c67fc640 1 -- 172.21.15.64:0/517371365 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f68c803e070 con 0x7f68d00b95a0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f68d00c96e0 con 0x7f68d00b95a0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68c67fc640 1 -- 172.21.15.64:0/517371365 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68c8035040 con 0x7f68d00b95a0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.365+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f68d00c9ca0 con 0x7f68d00b95a0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.366+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f68d00d4680 con 0x7f68d00b95a0 2024-03-04T07:36:21.369 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.367+0000 7f68c67fc640 1 -- 172.21.15.64:0/517371365 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f68c8038020 con 0x7f68d00b95a0 2024-03-04T07:36:21.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.367+0000 7f68c67fc640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f68b406b4d0 0x7f68b406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:21.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.367+0000 7f68c67fc640 1 -- 172.21.15.64:0/517371365 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f68c80ab550 con 0x7f68d00b95a0 2024-03-04T07:36:21.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.367+0000 7f68d56c0640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f68b406b4d0 0x7f68b406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:21.370 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.368+0000 7f68d56c0640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f68b406b4d0 0x7f68b406d990 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7f68d01f7320 tx=0x7f68c0007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:21.372 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.372+0000 7f68c67fc640 1 -- 172.21.15.64:0/517371365 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f68c8031800 con 0x7f68d00b95a0 2024-03-04T07:36:21.592 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.591+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f68d0067ff0 con 0x7f68d00b95a0 2024-03-04T07:36:21.595 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.592+0000 7f68c67fc640 1 -- 172.21.15.64:0/517371365 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f68c8079d40 con 0x7f68d00b95a0 2024-03-04T07:36:21.596 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:21.596 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f68b406b4d0 msgr2=0x7f68b406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f68b406b4d0 0x7f68b406d990 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7f68d01f7320 tx=0x7f68c0007e80 comp rx=0 tx=0).stop 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d00b95a0 msgr2=0x7f68d00c8870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d00b95a0 0x7f68d00c8870 secure :-1 s=READY pgs=1752 cs=0 l=1 rev1=1 crypto rx=0x7f68c80098c0 tx=0x7f68c80046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 shutdown_connections 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f68b406b4d0 0x7f68b406d990 unknown :-1 s=CLOSED pgs=795 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f68d00ca4a0 0x7f68d015e9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f68d015df00 0x7f68d0150fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 --2- 172.21.15.64:0/517371365 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f68d00b95a0 0x7f68d00c8870 unknown :-1 s=CLOSED pgs=1752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:21.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 >> 172.21.15.64:0/517371365 conn(0x7f68d0071960 msgr2=0x7f68d00b7ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:21.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 shutdown_connections 2024-03-04T07:36:21.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:21.594+0000 7f68d8125640 1 -- 172.21.15.64:0/517371365 wait complete. 2024-03-04T07:36:21.807 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:21.807 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:22.654 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:22.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:22.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.921+0000 7f29b01f4640 1 Processor -- start 2024-03-04T07:36:22.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.921+0000 7f29b01f4640 1 -- start start 2024-03-04T07:36:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29b01f4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f29a815bac0 0x7f29a815cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29b01f4640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29a80729b0 con 0x7f29a80aee10 2024-03-04T07:36:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29b01f4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f29a806c920 con 0x7f29a806d400 2024-03-04T07:36:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29b01f4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f29a806cf70 con 0x7f29a815bac0 2024-03-04T07:36:22.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29adf90640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f29a815bac0 0x7f29a815cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29ae791640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f29a80aee10 0x7f29a8070870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:35258/0 (socket says 172.21.15.64:35258) 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29ae791640 1 -- 172.21.15.64:0/2452317498 learned_addr learned my addr 172.21.15.64:0/2452317498 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29acf8e640 1 -- 172.21.15.64:0/2452317498 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2084255056 0 0) 0x7f29a80729b0 con 0x7f29a80aee10 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29acf8e640 1 -- 172.21.15.64:0/2452317498 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f2994003520 con 0x7f29a80aee10 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29acf8e640 1 -- 172.21.15.64:0/2452317498 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1166714696 0 0) 0x7f29a806c920 con 0x7f29a806d400 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29acf8e640 1 -- 172.21.15.64:0/2452317498 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f29a80729b0 con 0x7f29a806d400 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29adf90640 1 -- 172.21.15.64:0/2452317498 >> v1:172.21.15.66:6789/0 conn(0x7f29a806d400 legacy=0x7f29a80ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29adf90640 1 -- 172.21.15.64:0/2452317498 >> v1:172.21.15.64:6789/0 conn(0x7f29a80aee10 legacy=0x7f29a8070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.922+0000 7f29adf90640 1 -- 172.21.15.64:0/2452317498 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29a806cc00 con 0x7f29a815bac0 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.923+0000 7f29adf90640 1 --2- 172.21.15.64:0/2452317498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a815cbe0 secure :-1 s=READY pgs=1754 cs=0 l=1 rev1=1 crypto rx=0x7f29a40097a0 tx=0x7f29a402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a1e8e6cafb95c24 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.923+0000 7f29acf8e640 1 -- 172.21.15.64:0/2452317498 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29a4034070 con 0x7f29a815bac0 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.923+0000 7f29acf8e640 1 -- 172.21.15.64:0/2452317498 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29a4030610 con 0x7f29a815bac0 2024-03-04T07:36:22.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.923+0000 7f29acf8e640 1 -- 172.21.15.64:0/2452317498 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29a40309e0 con 0x7f29a815bac0 2024-03-04T07:36:22.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.924+0000 7f29b01f4640 1 -- 172.21.15.64:0/2452317498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 msgr2=0x7f29a815cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:22.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.924+0000 7f29b01f4640 1 --2- 172.21.15.64:0/2452317498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a815cbe0 secure :-1 s=READY pgs=1754 cs=0 l=1 rev1=1 crypto rx=0x7f29a40097a0 tx=0x7f29a402b960 comp rx=0 tx=0).stop 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.924+0000 7f29b01f4640 1 -- 172.21.15.64:0/2452317498 shutdown_connections 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.924+0000 7f29b01f4640 1 --2- 172.21.15.64:0/2452317498 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a815cbe0 unknown :-1 s=CLOSED pgs=1754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.924+0000 7f29b01f4640 1 -- 172.21.15.64:0/2452317498 >> 172.21.15.64:0/2452317498 conn(0x7f29a8066d00 msgr2=0x7f29a80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.924+0000 7f29b01f4640 1 -- 172.21.15.64:0/2452317498 shutdown_connections 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.924+0000 7f29b01f4640 1 -- 172.21.15.64:0/2452317498 wait complete. 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.925+0000 7f29b01f4640 1 Processor -- start 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.925+0000 7f29b01f4640 1 -- start start 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29b01f4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29a806d400 0x7f29a81520c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29b01f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29a80aee10 0x7f29a81588e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29b01f4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a81553a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29b01f4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29a815c020 con 0x7f29a815bac0 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29b01f4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29a815bea0 con 0x7f29a806d400 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29b01f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f29a8072580 con 0x7f29a80aee10 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a81553a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a81553a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:34924/0 (socket says 172.21.15.64:34924) 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 -- 172.21.15.64:0/1336450471 learned_addr learned my addr 172.21.15.64:0/1336450471 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29adf90640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29a806d400 0x7f29a81520c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ad78f640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29a80aee10 0x7f29a81588e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 -- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29a80aee10 msgr2=0x7f29a81588e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29a80aee10 0x7f29a81588e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 -- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29a806d400 msgr2=0x7f29a81520c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29a806d400 0x7f29a81520c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.926+0000 7f29ae791640 1 -- 172.21.15.64:0/1336450471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29a8152600 con 0x7f29a815bac0 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.927+0000 7f29ae791640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a81553a0 secure :-1 s=READY pgs=1755 cs=0 l=1 rev1=1 crypto rx=0x7f29a000cba0 tx=0x7f29a000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:22.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.927+0000 7f299e7fc640 1 -- 172.21.15.64:0/1336450471 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29a0017070 con 0x7f29a815bac0 2024-03-04T07:36:22.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.927+0000 7f299e7fc640 1 -- 172.21.15.64:0/1336450471 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f29a000ace0 con 0x7f29a815bac0 2024-03-04T07:36:22.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.927+0000 7f299e7fc640 1 -- 172.21.15.64:0/1336450471 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29a0012620 con 0x7f29a815bac0 2024-03-04T07:36:22.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.927+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29a8140570 con 0x7f29a815bac0 2024-03-04T07:36:22.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.927+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f29a8140b30 con 0x7f29a815bac0 2024-03-04T07:36:22.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.928+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f29a8067ff0 con 0x7f29a815bac0 2024-03-04T07:36:22.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.928+0000 7f299e7fc640 1 -- 172.21.15.64:0/1336450471 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f29a001a070 con 0x7f29a815bac0 2024-03-04T07:36:22.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.929+0000 7f299e7fc640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f298c06b4d0 0x7f298c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:22.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.929+0000 7f299e7fc640 1 -- 172.21.15.64:0/1336450471 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f29a0089f40 con 0x7f29a815bac0 2024-03-04T07:36:22.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.929+0000 7f29adf90640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f298c06b4d0 0x7f298c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:22.932 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.929+0000 7f29adf90640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f298c06b4d0 0x7f298c06d990 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7f29a4036000 tx=0x7f29a403b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:22.934 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:22.933+0000 7f299e7fc640 1 -- 172.21.15.64:0/1336450471 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f29a00587c0 con 0x7f29a815bac0 2024-03-04T07:36:23.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.152+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f29a806d8e0 con 0x7f29a815bac0 2024-03-04T07:36:23.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.153+0000 7f299e7fc640 1 -- 172.21.15.64:0/1336450471 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f29a005c860 con 0x7f29a815bac0 2024-03-04T07:36:23.154 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:23.154 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:23.154 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:23.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f298c06b4d0 msgr2=0x7f298c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:23.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f298c06b4d0 0x7f298c06d990 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7f29a4036000 tx=0x7f29a403b040 comp rx=0 tx=0).stop 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 msgr2=0x7f29a81553a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a81553a0 secure :-1 s=READY pgs=1755 cs=0 l=1 rev1=1 crypto rx=0x7f29a000cba0 tx=0x7f29a000a430 comp rx=0 tx=0).stop 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 shutdown_connections 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f298c06b4d0 0x7f298c06d990 unknown :-1 s=CLOSED pgs=796 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f29a815bac0 0x7f29a81553a0 unknown :-1 s=CLOSED pgs=1755 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f29a80aee10 0x7f29a81588e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 --2- 172.21.15.64:0/1336450471 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f29a806d400 0x7f29a81520c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 >> 172.21.15.64:0/1336450471 conn(0x7f29a8066d00 msgr2=0x7f29a80b1c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 shutdown_connections 2024-03-04T07:36:23.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:23.154+0000 7f29b01f4640 1 -- 172.21.15.64:0/1336450471 wait complete. 2024-03-04T07:36:23.368 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:23.388 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:24.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:24.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.502+0000 7fb02a215640 1 Processor -- start 2024-03-04T07:36:24.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.502+0000 7fb02a215640 1 -- start start 2024-03-04T07:36:24.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb02a215640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb02415cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:24.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb02a215640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb0240729b0 con 0x7fb0240aee10 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb02a215640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb02406c920 con 0x7fb02406d400 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb02a215640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb02406cf70 con 0x7fb02415bac0 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb023fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fb0240aee10 0x7fb024070870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56020/0 (socket says 172.21.15.64:56020) 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb023fff640 1 -- 172.21.15.64:0/1620685066 learned_addr learned my addr 172.21.15.64:0/1620685066 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb0237fe640 1 --2- 172.21.15.64:0/1620685066 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb02415cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb0227fc640 1 -- 172.21.15.64:0/1620685066 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2375900999 0 0) 0x7fb02406c920 con 0x7fb02406d400 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb0227fc640 1 -- 172.21.15.64:0/1620685066 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb00c003520 con 0x7fb02406d400 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb0227fc640 1 -- 172.21.15.64:0/1620685066 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 282823917 0 0) 0x7fb0240729b0 con 0x7fb0240aee10 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb0227fc640 1 -- 172.21.15.64:0/1620685066 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fb02406c920 con 0x7fb0240aee10 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.503+0000 7fb0237fe640 1 -- 172.21.15.64:0/1620685066 >> v1:172.21.15.66:6789/0 conn(0x7fb02406d400 legacy=0x7fb0240ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.504+0000 7fb0237fe640 1 -- 172.21.15.64:0/1620685066 >> v1:172.21.15.64:6789/0 conn(0x7fb0240aee10 legacy=0x7fb024070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.504+0000 7fb0237fe640 1 -- 172.21.15.64:0/1620685066 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb02406cc00 con 0x7fb02415bac0 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.504+0000 7fb0237fe640 1 --2- 172.21.15.64:0/1620685066 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb02415cbe0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb0140097d0 tx=0x7fb01402b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c097db5c6128d2a2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.504+0000 7fb0227fc640 1 -- 172.21.15.64:0/1620685066 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb014035020 con 0x7fb02415bac0 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.504+0000 7fb0227fc640 1 -- 172.21.15.64:0/1620685066 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb014031890 con 0x7fb02415bac0 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb0227fc640 1 -- 172.21.15.64:0/1620685066 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb014031c40 con 0x7fb02415bac0 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb02a215640 1 -- 172.21.15.64:0/1620685066 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 msgr2=0x7fb02415cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb02a215640 1 --2- 172.21.15.64:0/1620685066 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb02415cbe0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb0140097d0 tx=0x7fb01402b930 comp rx=0 tx=0).stop 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb02a215640 1 -- 172.21.15.64:0/1620685066 shutdown_connections 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb02a215640 1 --2- 172.21.15.64:0/1620685066 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb02415cbe0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb02a215640 1 -- 172.21.15.64:0/1620685066 >> 172.21.15.64:0/1620685066 conn(0x7fb024066d00 msgr2=0x7fb0240670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb02a215640 1 -- 172.21.15.64:0/1620685066 shutdown_connections 2024-03-04T07:36:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.505+0000 7fb02a215640 1 -- 172.21.15.64:0/1620685066 wait complete. 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.506+0000 7fb02a215640 1 Processor -- start 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb02a215640 1 -- start start 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb02a215640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb02406d400 0x7fb024144a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb02a215640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0240aee10 0x7fb024155970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb02a215640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb024145470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb02a215640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb02415c020 con 0x7fb02406d400 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb02a215640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb02415bea0 con 0x7fb0240aee10 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb02a215640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb024072580 con 0x7fb02415bac0 2024-03-04T07:36:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb0237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb02406d400 0x7fb024144a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb0237fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb02406d400 0x7fb024144a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38856/0 (socket says 172.21.15.64:38856) 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb0237fe640 1 -- 172.21.15.64:0/439789036 learned_addr learned my addr 172.21.15.64:0/439789036 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb022ffd640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0240aee10 0x7fb024155970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.507+0000 7fb023fff640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb024145470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb0237fe640 1 -- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 msgr2=0x7fb024145470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb0237fe640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb024145470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb0237fe640 1 -- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0240aee10 msgr2=0x7fb024155970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb0237fe640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0240aee10 0x7fb024155970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb0237fe640 1 -- 172.21.15.64:0/439789036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0241482e0 con 0x7fb02406d400 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb0237fe640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb02406d400 0x7fb024144a20 secure :-1 s=READY pgs=1757 cs=0 l=1 rev1=1 crypto rx=0x7fb014000c00 tx=0x7fb0140046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb00bfff640 1 -- 172.21.15.64:0/439789036 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb014035040 con 0x7fb02406d400 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb00bfff640 1 -- 172.21.15.64:0/439789036 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb014030d20 con 0x7fb02406d400 2024-03-04T07:36:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb00bfff640 1 -- 172.21.15.64:0/439789036 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb014006e10 con 0x7fb02406d400 2024-03-04T07:36:24.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb024155170 con 0x7fb02406d400 2024-03-04T07:36:24.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.508+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb024154b40 con 0x7fb02406d400 2024-03-04T07:36:24.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.509+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb024067ff0 con 0x7fb02406d400 2024-03-04T07:36:24.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.510+0000 7fb00bfff640 1 -- 172.21.15.64:0/439789036 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb014038020 con 0x7fb02406d400 2024-03-04T07:36:24.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.510+0000 7fb00bfff640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb01006b4d0 0x7fb01006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:24.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.510+0000 7fb022ffd640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb01006b4d0 0x7fb01006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:24.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.510+0000 7fb00bfff640 1 -- 172.21.15.64:0/439789036 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb0140ac710 con 0x7fb02406d400 2024-03-04T07:36:24.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.511+0000 7fb022ffd640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb01006b4d0 0x7fb01006d990 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7fb018007fd0 tx=0x7fb018009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:24.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.515+0000 7fb00bfff640 1 -- 172.21.15.64:0/439789036 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb01402d040 con 0x7fb02406d400 2024-03-04T07:36:24.655 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:24.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.734+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb02406de50 con 0x7fb02406d400 2024-03-04T07:36:24.735 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.734+0000 7fb00bfff640 1 -- 172.21.15.64:0/439789036 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fb01407af90 con 0x7fb02406d400 2024-03-04T07:36:24.736 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:24.736 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:24.736 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb01006b4d0 msgr2=0x7fb01006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb01006b4d0 0x7fb01006d990 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7fb018007fd0 tx=0x7fb018009040 comp rx=0 tx=0).stop 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb02406d400 msgr2=0x7fb024144a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb02406d400 0x7fb024144a20 secure :-1 s=READY pgs=1757 cs=0 l=1 rev1=1 crypto rx=0x7fb014000c00 tx=0x7fb0140046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 shutdown_connections 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb01006b4d0 0x7fb01006d990 unknown :-1 s=CLOSED pgs=797 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb02415bac0 0x7fb024145470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb0240aee10 0x7fb024155970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 --2- 172.21.15.64:0/439789036 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb02406d400 0x7fb024144a20 unknown :-1 s=CLOSED pgs=1757 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 >> 172.21.15.64:0/439789036 conn(0x7fb024066d00 msgr2=0x7fb02415e1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 shutdown_connections 2024-03-04T07:36:24.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:24.736+0000 7fb02a215640 1 -- 172.21.15.64:0/439789036 wait complete. 2024-03-04T07:36:24.949 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:24.950 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:25.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:26.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.062+0000 7fd4e0cdf640 1 Processor -- start 2024-03-04T07:36:26.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.062+0000 7fd4e0cdf640 1 -- start start 2024-03-04T07:36:26.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4e0cdf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd4dc06df30 0x7fd4dc0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:26.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4e0cdf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd4dc071240 con 0x7fd4dc158520 2024-03-04T07:36:26.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4e0cdf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd4dc071520 con 0x7fd4dc0aec40 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4e0cdf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd4dc0af0f0 con 0x7fd4dc06df30 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4da59c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd4dc158520 0x7fd4dc06d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56036/0 (socket says 172.21.15.64:56036) 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4da59c640 1 -- 172.21.15.64:0/3246606761 learned_addr learned my addr 172.21.15.64:0/3246606761 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4d9d9b640 1 --2- 172.21.15.64:0/3246606761 >> v2:172.21.15.66:3300/0 conn(0x7fd4dc06df30 0x7fd4dc0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4d959a640 1 -- 172.21.15.64:0/3246606761 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3307305000 0 0) 0x7fd4dc071240 con 0x7fd4dc158520 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4d959a640 1 -- 172.21.15.64:0/3246606761 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd4c8003520 con 0x7fd4dc158520 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.063+0000 7fd4d9d9b640 1 -- 172.21.15.64:0/3246606761 >> v1:172.21.15.66:6789/0 conn(0x7fd4dc0aec40 legacy=0x7fd4dc0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.064+0000 7fd4d9d9b640 1 -- 172.21.15.64:0/3246606761 >> v1:172.21.15.64:6789/0 conn(0x7fd4dc158520 legacy=0x7fd4dc06d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.064+0000 7fd4d9d9b640 1 -- 172.21.15.64:0/3246606761 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4dc071800 con 0x7fd4dc06df30 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.064+0000 7fd4d9d9b640 1 --2- 172.21.15.64:0/3246606761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc06df30 0x7fd4dc0ae700 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fd4c40097d0 tx=0x7fd4c402b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6edec4d10b1d9162 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.064+0000 7fd4d959a640 1 -- 172.21.15.64:0/3246606761 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4c4034070 con 0x7fd4dc06df30 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4d959a640 1 -- 172.21.15.64:0/3246606761 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd4c4030610 con 0x7fd4dc06df30 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4d959a640 1 -- 172.21.15.64:0/3246606761 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4c40309c0 con 0x7fd4dc06df30 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3246606761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc06df30 msgr2=0x7fd4dc0ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3246606761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc06df30 0x7fd4dc0ae700 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fd4c40097d0 tx=0x7fd4c402b960 comp rx=0 tx=0).stop 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3246606761 shutdown_connections 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3246606761 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc06df30 0x7fd4dc0ae700 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3246606761 >> 172.21.15.64:0/3246606761 conn(0x7fd4dc066d00 msgr2=0x7fd4dc0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3246606761 shutdown_connections 2024-03-04T07:36:26.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.065+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3246606761 wait complete. 2024-03-04T07:36:26.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 Processor -- start 2024-03-04T07:36:26.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 -- start start 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd4dc06df30 0x7fd4dc1f2710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4dc0aec40 0x7fd4dc1f5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc158520 0x7fd4dc143410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4dc158900 con 0x7fd4dc06df30 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4dc071b70 con 0x7fd4dc158520 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4e0cdf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd4dc158a80 con 0x7fd4dc0aec40 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4da59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd4dc06df30 0x7fd4dc1f2710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.067+0000 7fd4da59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd4dc06df30 0x7fd4dc1f2710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38868/0 (socket says 172.21.15.64:38868) 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4da59c640 1 -- 172.21.15.64:0/3572461344 learned_addr learned my addr 172.21.15.64:0/3572461344 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4d9d9b640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4dc0aec40 0x7fd4dc1f5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4dad9d640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc158520 0x7fd4dc143410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4da59c640 1 -- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4dc0aec40 msgr2=0x7fd4dc1f5160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4da59c640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4dc0aec40 0x7fd4dc1f5160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4da59c640 1 -- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc158520 msgr2=0x7fd4dc143410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4da59c640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc158520 0x7fd4dc143410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4da59c640 1 -- 172.21.15.64:0/3572461344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4dc1f6030 con 0x7fd4dc06df30 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4da59c640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd4dc06df30 0x7fd4dc1f2710 secure :-1 s=READY pgs=1759 cs=0 l=1 rev1=1 crypto rx=0x7fd4cc00cbc0 tx=0x7fd4cc00a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4d2ffd640 1 -- 172.21.15.64:0/3572461344 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4cc017070 con 0x7fd4dc06df30 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4d2ffd640 1 -- 172.21.15.64:0/3572461344 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd4cc00ad40 con 0x7fd4dc06df30 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4dc1f5d00 con 0x7fd4dc06df30 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4d2ffd640 1 -- 172.21.15.64:0/3572461344 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4cc012600 con 0x7fd4dc06df30 2024-03-04T07:36:26.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.068+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd4dc142ef0 con 0x7fd4dc06df30 2024-03-04T07:36:26.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.069+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd4dc067ff0 con 0x7fd4dc06df30 2024-03-04T07:36:26.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.070+0000 7fd4d2ffd640 1 -- 172.21.15.64:0/3572461344 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd4cc01a070 con 0x7fd4dc06df30 2024-03-04T07:36:26.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.070+0000 7fd4d2ffd640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd4bc06b4d0 0x7fd4bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:26.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.070+0000 7fd4d2ffd640 1 -- 172.21.15.64:0/3572461344 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd4cc089fc0 con 0x7fd4dc06df30 2024-03-04T07:36:26.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.070+0000 7fd4d9d9b640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd4bc06b4d0 0x7fd4bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:26.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.071+0000 7fd4d9d9b640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd4bc06b4d0 0x7fd4bc06d990 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7fd4c4036000 tx=0x7fd4c4008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:26.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.074+0000 7fd4d2ffd640 1 -- 172.21.15.64:0/3572461344 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd4cc0587b0 con 0x7fd4dc06df30 2024-03-04T07:36:26.293 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.292+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd4dc0b31f0 con 0x7fd4dc06df30 2024-03-04T07:36:26.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.293+0000 7fd4d2ffd640 1 -- 172.21.15.64:0/3572461344 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fd4cc05c850 con 0x7fd4dc06df30 2024-03-04T07:36:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:26.294 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:26.294 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.294+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd4bc06b4d0 msgr2=0x7fd4bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.294+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd4bc06b4d0 0x7fd4bc06d990 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7fd4c4036000 tx=0x7fd4c4008030 comp rx=0 tx=0).stop 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd4dc06df30 msgr2=0x7fd4dc1f2710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd4dc06df30 0x7fd4dc1f2710 secure :-1 s=READY pgs=1759 cs=0 l=1 rev1=1 crypto rx=0x7fd4cc00cbc0 tx=0x7fd4cc00a430 comp rx=0 tx=0).stop 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 shutdown_connections 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd4bc06b4d0 0x7fd4bc06d990 unknown :-1 s=CLOSED pgs=798 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd4dc158520 0x7fd4dc143410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd4dc0aec40 0x7fd4dc1f5160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 --2- 172.21.15.64:0/3572461344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd4dc06df30 0x7fd4dc1f2710 unknown :-1 s=CLOSED pgs=1759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 >> 172.21.15.64:0/3572461344 conn(0x7fd4dc066d00 msgr2=0x7fd4dc1592d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 shutdown_connections 2024-03-04T07:36:26.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:26.295+0000 7fd4e0cdf640 1 -- 172.21.15.64:0/3572461344 wait complete. 2024-03-04T07:36:26.511 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:26.512 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:26.658 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:27.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:27.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.625+0000 7f3290ed6640 1 Processor -- start 2024-03-04T07:36:27.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.625+0000 7f3290ed6640 1 -- start start 2024-03-04T07:36:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.626+0000 7f3290ed6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0d02e0 0x7f328c0cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.626+0000 7f3290ed6640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f328c0b4260 0x7f328c0b1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.626+0000 7f3290ed6640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f328c0d06c0 0x7f328c0b1a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.626+0000 7f3290ed6640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f328c0b0b80 con 0x7f328c0b4260 2024-03-04T07:36:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.626+0000 7f3290ed6640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f328c0aeec0 con 0x7f328c0d06c0 2024-03-04T07:36:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.626+0000 7f3290ed6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f328c0b0dd0 con 0x7f328c0d02e0 2024-03-04T07:36:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f328c0b4260 0x7f328c0b1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f328ad9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f328c0d06c0 0x7f328c0b1a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f328a59c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0d02e0 0x7f328c0cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f328c0b4260 0x7f328c0b1500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38876/0 (socket says 172.21.15.64:38876) 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 -- 172.21.15.64:0/1161494434 learned_addr learned my addr 172.21.15.64:0/1161494434 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 -- 172.21.15.64:0/1161494434 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0d02e0 msgr2=0x7f328c0cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 --2- 172.21.15.64:0/1161494434 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0d02e0 0x7f328c0cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 -- 172.21.15.64:0/1161494434 >> v2:172.21.15.66:3300/0 conn(0x7f328c0d06c0 msgr2=0x7f328c0b1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 --2- 172.21.15.64:0/1161494434 >> v2:172.21.15.66:3300/0 conn(0x7f328c0d06c0 0x7f328c0b1a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f3289d9b640 1 -- 172.21.15.64:0/1161494434 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f328c0b10b0 con 0x7f328c0b4260 2024-03-04T07:36:27.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.627+0000 7f328ad9d640 1 --2- 172.21.15.64:0/1161494434 >> v2:172.21.15.66:3300/0 conn(0x7f328c0d06c0 0x7f328c0b1a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:27.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.628+0000 7f3289d9b640 1 --2- 172.21.15.64:0/1161494434 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0b4260 0x7f328c0b1500 secure :-1 s=READY pgs=1760 cs=0 l=1 rev1=1 crypto rx=0x7f328c071d50 tx=0x7f327402b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e9ff1e073020052e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:27.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.628+0000 7f328959a640 1 -- 172.21.15.64:0/1161494434 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3274034070 con 0x7f328c0b4260 2024-03-04T07:36:27.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.628+0000 7f328959a640 1 -- 172.21.15.64:0/1161494434 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3274030860 con 0x7f328c0b4260 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.628+0000 7f328959a640 1 -- 172.21.15.64:0/1161494434 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3274030c30 con 0x7f328c0b4260 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 -- 172.21.15.64:0/1161494434 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0b4260 msgr2=0x7f328c0b1500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 --2- 172.21.15.64:0/1161494434 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0b4260 0x7f328c0b1500 secure :-1 s=READY pgs=1760 cs=0 l=1 rev1=1 crypto rx=0x7f328c071d50 tx=0x7f327402b9a0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 -- 172.21.15.64:0/1161494434 shutdown_connections 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 --2- 172.21.15.64:0/1161494434 >> v2:172.21.15.66:3300/0 conn(0x7f328c0d06c0 0x7f328c0b1a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 --2- 172.21.15.64:0/1161494434 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0b4260 0x7f328c0b1500 unknown :-1 s=CLOSED pgs=1760 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 --2- 172.21.15.64:0/1161494434 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0d02e0 0x7f328c0cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 -- 172.21.15.64:0/1161494434 >> 172.21.15.64:0/1161494434 conn(0x7f328c0b36d0 msgr2=0x7f328c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 -- 172.21.15.64:0/1161494434 shutdown_connections 2024-03-04T07:36:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.629+0000 7f3290ed6640 1 -- 172.21.15.64:0/1161494434 wait complete. 2024-03-04T07:36:27.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.630+0000 7f3290ed6640 1 Processor -- start 2024-03-04T07:36:27.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.630+0000 7f3290ed6640 1 -- start start 2024-03-04T07:36:27.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f3290ed6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0b4260 0x7f328c141ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f3290ed6640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f328c0d02e0 0x7f328c0c5b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f3290ed6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0d06c0 0x7f328c154f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f3290ed6640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f328c0b2100 con 0x7f328c0d06c0 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f3290ed6640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f328c0b1f80 con 0x7f328c0d02e0 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f3290ed6640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f328c0b2280 con 0x7f328c0b4260 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0d06c0 0x7f328c154f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0d06c0 0x7f328c154f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38884/0 (socket says 172.21.15.64:38884) 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 -- 172.21.15.64:0/970304506 learned_addr learned my addr 172.21.15.64:0/970304506 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f3289d9b640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f328c0d02e0 0x7f328c0c5b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328a59c640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0b4260 0x7f328c141ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 -- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0b4260 msgr2=0x7f328c141ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:27.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0b4260 0x7f328c141ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 -- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f328c0d02e0 msgr2=0x7f328c0c5b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f328c0d02e0 0x7f328c0c5b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.631+0000 7f328ad9d640 1 -- 172.21.15.64:0/970304506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f328c0c9c00 con 0x7f328c0d06c0 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.632+0000 7f328ad9d640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0d06c0 0x7f328c154f20 secure :-1 s=READY pgs=1761 cs=0 l=1 rev1=1 crypto rx=0x7f328400ef90 tx=0x7f3284009f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.632+0000 7f3282ffd640 1 -- 172.21.15.64:0/970304506 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3284018020 con 0x7f328c0d06c0 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.632+0000 7f3282ffd640 1 -- 172.21.15.64:0/970304506 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3284014480 con 0x7f328c0d06c0 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.632+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f328c153450 con 0x7f328c0d06c0 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.632+0000 7f3282ffd640 1 -- 172.21.15.64:0/970304506 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3284014870 con 0x7f328c0d06c0 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.632+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f328c0c6600 con 0x7f328c0d06c0 2024-03-04T07:36:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.633+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f328c068700 con 0x7f328c0d06c0 2024-03-04T07:36:27.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.633+0000 7f3282ffd640 1 -- 172.21.15.64:0/970304506 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3284005d10 con 0x7f328c0d06c0 2024-03-04T07:36:27.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.634+0000 7f3282ffd640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f326c06b4d0 0x7f326c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:27.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.634+0000 7f3282ffd640 1 -- 172.21.15.64:0/970304506 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f328408ae20 con 0x7f328c0d06c0 2024-03-04T07:36:27.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.634+0000 7f328a59c640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f326c06b4d0 0x7f326c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:27.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.634+0000 7f328a59c640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f326c06b4d0 0x7f326c06d990 secure :-1 s=READY pgs=799 cs=0 l=1 rev1=1 crypto rx=0x7f327c009ac0 tx=0x7f327c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:27.639 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.638+0000 7f3282ffd640 1 -- 172.21.15.64:0/970304506 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3284022030 con 0x7f328c0d06c0 2024-03-04T07:36:27.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.858+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f328c0cbd90 con 0x7f328c0d06c0 2024-03-04T07:36:27.859 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.858+0000 7f3282ffd640 1 -- 172.21.15.64:0/970304506 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f3284010040 con 0x7f328c0d06c0 2024-03-04T07:36:27.859 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:27.859 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:27.860 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:27.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f326c06b4d0 msgr2=0x7f326c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:27.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f326c06b4d0 0x7f326c06d990 secure :-1 s=READY pgs=799 cs=0 l=1 rev1=1 crypto rx=0x7f327c009ac0 tx=0x7f327c009190 comp rx=0 tx=0).stop 2024-03-04T07:36:27.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0d06c0 msgr2=0x7f328c154f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:27.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0d06c0 0x7f328c154f20 secure :-1 s=READY pgs=1761 cs=0 l=1 rev1=1 crypto rx=0x7f328400ef90 tx=0x7f3284009f90 comp rx=0 tx=0).stop 2024-03-04T07:36:27.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 shutdown_connections 2024-03-04T07:36:27.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f326c06b4d0 0x7f326c06d990 unknown :-1 s=CLOSED pgs=799 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f328c0d06c0 0x7f328c154f20 unknown :-1 s=CLOSED pgs=1761 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f328c0d02e0 0x7f328c0c5b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 --2- 172.21.15.64:0/970304506 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f328c0b4260 0x7f328c141ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:27.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 >> 172.21.15.64:0/970304506 conn(0x7f328c0b36d0 msgr2=0x7f328c0ce3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:27.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 shutdown_connections 2024-03-04T07:36:27.862 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:27.860+0000 7f3290ed6640 1 -- 172.21.15.64:0/970304506 wait complete. 2024-03-04T07:36:28.073 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:28.073 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:29.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:29.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.187+0000 7fbd394ed640 1 Processor -- start 2024-03-04T07:36:29.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.187+0000 7fbd394ed640 1 -- start start 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.187+0000 7fbd394ed640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbd340d18c0 0x7fbd34071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd394ed640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbd340b6f20 con 0x7fbd3415df70 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd394ed640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbd340d3030 con 0x7fbd34071a80 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd394ed640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbd34071eb0 con 0x7fbd340d18c0 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd327fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbd340d18c0 0x7fbd34071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd32ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbd3415df70 0x7fbd340d27b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56038/0 (socket says 172.21.15.64:56038) 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd32ffd640 1 -- 172.21.15.64:0/3934107044 learned_addr learned my addr 172.21.15.64:0/3934107044 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd327fc640 1 -- 172.21.15.64:0/3934107044 >> v1:172.21.15.66:6789/0 conn(0x7fbd34071a80 legacy=0x7fbd340b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:29.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd327fc640 1 -- 172.21.15.64:0/3934107044 >> v1:172.21.15.64:6789/0 conn(0x7fbd3415df70 legacy=0x7fbd340d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:29.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd327fc640 1 -- 172.21.15.64:0/3934107044 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd340d3310 con 0x7fbd340d18c0 2024-03-04T07:36:29.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.188+0000 7fbd31ffb640 1 -- 172.21.15.64:0/3934107044 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 880945146 0 0) 0x7fbd340b6f20 con 0x7fbd3415df70 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.189+0000 7fbd327fc640 1 --2- 172.21.15.64:0/3934107044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd340d18c0 0x7fbd34071540 secure :-1 s=READY pgs=1763 cs=0 l=1 rev1=1 crypto rx=0x7fbd280097a0 tx=0x7fbd2802b9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1a5c298047e11d56 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.189+0000 7fbd31ffb640 1 -- 172.21.15.64:0/3934107044 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd2802f810 con 0x7fbd340d18c0 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.189+0000 7fbd31ffb640 1 -- 172.21.15.64:0/3934107044 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbd2802f9b0 con 0x7fbd340d18c0 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.189+0000 7fbd31ffb640 1 -- 172.21.15.64:0/3934107044 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd2802fd80 con 0x7fbd340d18c0 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.189+0000 7fbd394ed640 1 -- 172.21.15.64:0/3934107044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd340d18c0 msgr2=0x7fbd34071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.189+0000 7fbd394ed640 1 --2- 172.21.15.64:0/3934107044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd340d18c0 0x7fbd34071540 secure :-1 s=READY pgs=1763 cs=0 l=1 rev1=1 crypto rx=0x7fbd280097a0 tx=0x7fbd2802b9f0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.190+0000 7fbd394ed640 1 -- 172.21.15.64:0/3934107044 shutdown_connections 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.190+0000 7fbd394ed640 1 --2- 172.21.15.64:0/3934107044 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd340d18c0 0x7fbd34071540 unknown :-1 s=CLOSED pgs=1763 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.190+0000 7fbd394ed640 1 -- 172.21.15.64:0/3934107044 >> 172.21.15.64:0/3934107044 conn(0x7fbd340d3cf0 msgr2=0x7fbd340d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.190+0000 7fbd394ed640 1 -- 172.21.15.64:0/3934107044 shutdown_connections 2024-03-04T07:36:29.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.190+0000 7fbd394ed640 1 -- 172.21.15.64:0/3934107044 wait complete. 2024-03-04T07:36:29.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.191+0000 7fbd394ed640 1 Processor -- start 2024-03-04T07:36:29.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.191+0000 7fbd394ed640 1 -- start start 2024-03-04T07:36:29.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd394ed640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd34071a80 0x7fbd340c78e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:29.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd394ed640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd340d18c0 0x7fbd340c7e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:29.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd394ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd3415df70 0x7fbd341f3ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:29.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd394ed640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd3415e350 con 0x7fbd34071a80 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd394ed640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd340d38d0 con 0x7fbd340d18c0 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd394ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbd3415e4d0 con 0x7fbd3415df70 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd34071a80 0x7fbd340c78e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd34071a80 0x7fbd340c78e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38894/0 (socket says 172.21.15.64:38894) 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 -- 172.21.15.64:0/2190775031 learned_addr learned my addr 172.21.15.64:0/2190775031 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd327fc640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd340d18c0 0x7fbd340c7e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd337fe640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd3415df70 0x7fbd341f3ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 -- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd3415df70 msgr2=0x7fbd341f3ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd3415df70 0x7fbd341f3ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 -- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd340d18c0 msgr2=0x7fbd340c7e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd340d18c0 0x7fbd340c7e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 -- 172.21.15.64:0/2190775031 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd340c8660 con 0x7fbd34071a80 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd327fc640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd340d18c0 0x7fbd340c7e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:29.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.192+0000 7fbd32ffd640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd34071a80 0x7fbd340c78e0 secure :-1 s=READY pgs=1764 cs=0 l=1 rev1=1 crypto rx=0x7fbd2400c3f0 tx=0x7fbd2400c8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:29.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.193+0000 7fbd1b7fe640 1 -- 172.21.15.64:0/2190775031 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd240074e0 con 0x7fbd34071a80 2024-03-04T07:36:29.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.193+0000 7fbd1b7fe640 1 -- 172.21.15.64:0/2190775031 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbd240047e0 con 0x7fbd34071a80 2024-03-04T07:36:29.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.193+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbd340c8940 con 0x7fbd34071a80 2024-03-04T07:36:29.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.193+0000 7fbd1b7fe640 1 -- 172.21.15.64:0/2190775031 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd24004c50 con 0x7fbd34071a80 2024-03-04T07:36:29.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.193+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd340c8f00 con 0x7fbd34071a80 2024-03-04T07:36:29.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.194+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbd34067ff0 con 0x7fbd34071a80 2024-03-04T07:36:29.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.194+0000 7fbd1b7fe640 1 -- 172.21.15.64:0/2190775031 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbd24011070 con 0x7fbd34071a80 2024-03-04T07:36:29.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.194+0000 7fbd1b7fe640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd2006b4d0 0x7fbd2006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:29.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.195+0000 7fbd1b7fe640 1 -- 172.21.15.64:0/2190775031 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbd24089be0 con 0x7fbd34071a80 2024-03-04T07:36:29.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.195+0000 7fbd327fc640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd2006b4d0 0x7fbd2006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:29.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.195+0000 7fbd327fc640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd2006b4d0 0x7fbd2006d990 secure :-1 s=READY pgs=800 cs=0 l=1 rev1=1 crypto rx=0x7fbd28036000 tx=0x7fbd28008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:29.200 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.199+0000 7fbd1b7fe640 1 -- 172.21.15.64:0/2190775031 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbd24058460 con 0x7fbd34071a80 2024-03-04T07:36:29.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.420+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbd340c7210 con 0x7fbd34071a80 2024-03-04T07:36:29.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.420+0000 7fbd1b7fe640 1 -- 172.21.15.64:0/2190775031 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fbd24006e20 con 0x7fbd34071a80 2024-03-04T07:36:29.421 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:29.421 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:29.422 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:29.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd2006b4d0 msgr2=0x7fbd2006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:29.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd2006b4d0 0x7fbd2006d990 secure :-1 s=READY pgs=800 cs=0 l=1 rev1=1 crypto rx=0x7fbd28036000 tx=0x7fbd28008030 comp rx=0 tx=0).stop 2024-03-04T07:36:29.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd34071a80 msgr2=0x7fbd340c78e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:29.423 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd34071a80 0x7fbd340c78e0 secure :-1 s=READY pgs=1764 cs=0 l=1 rev1=1 crypto rx=0x7fbd2400c3f0 tx=0x7fbd2400c8b0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 shutdown_connections 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbd2006b4d0 0x7fbd2006d990 unknown :-1 s=CLOSED pgs=800 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbd3415df70 0x7fbd341f3ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbd340d18c0 0x7fbd340c7e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 --2- 172.21.15.64:0/2190775031 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbd34071a80 0x7fbd340c78e0 unknown :-1 s=CLOSED pgs=1764 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 >> 172.21.15.64:0/2190775031 conn(0x7fbd340d3cf0 msgr2=0x7fbd340b3750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 shutdown_connections 2024-03-04T07:36:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:29.422+0000 7fbd394ed640 1 -- 172.21.15.64:0/2190775031 wait complete. 2024-03-04T07:36:29.635 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:29.635 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:30.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:30.661 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:30.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.750+0000 7f366badf640 1 Processor -- start 2024-03-04T07:36:30.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.750+0000 7f366badf640 1 -- start start 2024-03-04T07:36:30.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.751+0000 7f366badf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f3664168c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.751+0000 7f366badf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36640cf130 con 0x7f366415e060 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.751+0000 7f366badf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36640d0dd0 con 0x7f366406fe20 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.751+0000 7f366badf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36640d10b0 con 0x7f36640d0880 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.751+0000 7f366987b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f366415e060 0x7f36640ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56052/0 (socket says 172.21.15.64:56052) 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.751+0000 7f366987b640 1 -- 172.21.15.64:0/448261662 learned_addr learned my addr 172.21.15.64:0/448261662 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.751+0000 7f366a07c640 1 --2- 172.21.15.64:0/448261662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f3664168c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.752+0000 7f3668879640 1 -- 172.21.15.64:0/448261662 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4069934595 0 0) 0x7f36640d0dd0 con 0x7f366406fe20 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.752+0000 7f3668879640 1 -- 172.21.15.64:0/448261662 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f364c003520 con 0x7f366406fe20 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.752+0000 7f3668879640 1 -- 172.21.15.64:0/448261662 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 215512789 0 0) 0x7f36640cf130 con 0x7f366415e060 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.752+0000 7f366a07c640 1 -- 172.21.15.64:0/448261662 >> v1:172.21.15.66:6789/0 conn(0x7f366406fe20 legacy=0x7f36640cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.752+0000 7f366a07c640 1 -- 172.21.15.64:0/448261662 >> v1:172.21.15.64:6789/0 conn(0x7f366415e060 legacy=0x7f36640ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:30.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.752+0000 7f366a07c640 1 -- 172.21.15.64:0/448261662 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36640cc0e0 con 0x7f36640d0880 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.752+0000 7f366a07c640 1 --2- 172.21.15.64:0/448261662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f3664168c20 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f36540097d0 tx=0x7f365402b930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2126f44a044520b2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f3668879640 1 -- 172.21.15.64:0/448261662 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3654035020 con 0x7f36640d0880 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f3668879640 1 -- 172.21.15.64:0/448261662 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3654031890 con 0x7f36640d0880 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f3668879640 1 -- 172.21.15.64:0/448261662 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3654031c60 con 0x7f36640d0880 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f366badf640 1 -- 172.21.15.64:0/448261662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 msgr2=0x7f3664168c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f366badf640 1 --2- 172.21.15.64:0/448261662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f3664168c20 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f36540097d0 tx=0x7f365402b930 comp rx=0 tx=0).stop 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f366badf640 1 -- 172.21.15.64:0/448261662 shutdown_connections 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f366badf640 1 --2- 172.21.15.64:0/448261662 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f3664168c20 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f366badf640 1 -- 172.21.15.64:0/448261662 >> 172.21.15.64:0/448261662 conn(0x7f36640d1a90 msgr2=0x7f36640d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f366badf640 1 -- 172.21.15.64:0/448261662 shutdown_connections 2024-03-04T07:36:30.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.753+0000 7f366badf640 1 -- 172.21.15.64:0/448261662 wait complete. 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.754+0000 7f366badf640 1 Processor -- start 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.754+0000 7f366badf640 1 -- start start 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366badf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f366406fe20 0x7f366415b230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366badf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f36640ce310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366badf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f366415e060 0x7f36640ce850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366badf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f366415e440 con 0x7f366415e060 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366badf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36640cc6a0 con 0x7f366406fe20 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366badf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36640cdbd0 con 0x7f36640d0880 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366a07c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f366415e060 0x7f36640ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366a07c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f366415e060 0x7f36640ce850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38906/0 (socket says 172.21.15.64:38906) 2024-03-04T07:36:30.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366987b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f366406fe20 0x7f366415b230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366a07c640 1 -- 172.21.15.64:0/1809869653 learned_addr learned my addr 172.21.15.64:0/1809869653 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.755+0000 7f366907a640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f36640ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366a07c640 1 -- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 msgr2=0x7f36640ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366a07c640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f36640ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366a07c640 1 -- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f366406fe20 msgr2=0x7f366415b230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366a07c640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f366406fe20 0x7f366415b230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366a07c640 1 -- 172.21.15.64:0/1809869653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36640c8d00 con 0x7f366415e060 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366987b640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f366406fe20 0x7f366415b230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366a07c640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f366415e060 0x7f36640ce850 secure :-1 s=READY pgs=1766 cs=0 l=1 rev1=1 crypto rx=0x7f3654009720 tx=0x7f36540046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f36627fc640 1 -- 172.21.15.64:0/1809869653 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3654035040 con 0x7f366415e060 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f36627fc640 1 -- 172.21.15.64:0/1809869653 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3654030ca0 con 0x7f366415e060 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36640b7430 con 0x7f366415e060 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f36627fc640 1 -- 172.21.15.64:0/1809869653 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3654004270 con 0x7f366415e060 2024-03-04T07:36:30.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.756+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f366415b770 con 0x7f366415e060 2024-03-04T07:36:30.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.757+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3664067ff0 con 0x7f366415e060 2024-03-04T07:36:30.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.758+0000 7f36627fc640 1 -- 172.21.15.64:0/1809869653 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3654038020 con 0x7f366415e060 2024-03-04T07:36:30.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.758+0000 7f36627fc640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f365006b4d0 0x7f365006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:30.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.758+0000 7f36627fc640 1 -- 172.21.15.64:0/1809869653 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f36540ac410 con 0x7f366415e060 2024-03-04T07:36:30.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.758+0000 7f366987b640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f365006b4d0 0x7f365006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:30.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.758+0000 7f366987b640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f365006b4d0 0x7f365006d990 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7f365c007f30 tx=0x7f365c007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:30.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.762+0000 7f36627fc640 1 -- 172.21.15.64:0/1809869653 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f365402d040 con 0x7f366415e060 2024-03-04T07:36:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.987+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f36640b7950 con 0x7f366415e060 2024-03-04T07:36:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.988+0000 7f36627fc640 1 -- 172.21.15.64:0/1809869653 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f365407ac00 con 0x7f366415e060 2024-03-04T07:36:30.988 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:30.988 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:30.988 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f365006b4d0 msgr2=0x7f365006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f365006b4d0 0x7f365006d990 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7f365c007f30 tx=0x7f365c007e80 comp rx=0 tx=0).stop 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f366415e060 msgr2=0x7f36640ce850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f366415e060 0x7f36640ce850 secure :-1 s=READY pgs=1766 cs=0 l=1 rev1=1 crypto rx=0x7f3654009720 tx=0x7f36540046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 shutdown_connections 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f365006b4d0 0x7f365006d990 unknown :-1 s=CLOSED pgs=801 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f366415e060 0x7f36640ce850 unknown :-1 s=CLOSED pgs=1766 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36640d0880 0x7f36640ce310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 --2- 172.21.15.64:0/1809869653 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f366406fe20 0x7f366415b230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.989+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 >> 172.21.15.64:0/1809869653 conn(0x7f36640d1a90 msgr2=0x7f36640d12f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.990+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 shutdown_connections 2024-03-04T07:36:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:30.990+0000 7f366badf640 1 -- 172.21.15.64:0/1809869653 wait complete. 2024-03-04T07:36:31.202 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:31.202 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:32.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:32.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.317+0000 7f1fadf7a640 1 Processor -- start 2024-03-04T07:36:32.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.317+0000 7f1fadf7a640 1 -- start start 2024-03-04T07:36:32.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fadf7a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1fa806df30 0x7f1fa80ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:32.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fadf7a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1fa80aec40 0x7f1fa80b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:32.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fadf7a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1fa8071240 con 0x7f1fa8158520 2024-03-04T07:36:32.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fadf7a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1fa80af0f0 con 0x7f1fa80aec40 2024-03-04T07:36:32.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fadf7a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1fa8071b70 con 0x7f1fa806df30 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa7fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1fa80aec40 0x7f1fa80b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1fa8158520 0x7f1fa806d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:56068/0 (socket says 172.21.15.64:56068) 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa77fe640 1 -- 172.21.15.64:0/681928456 learned_addr learned my addr 172.21.15.64:0/681928456 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa6ffd640 1 --2- 172.21.15.64:0/681928456 >> v2:172.21.15.66:3300/0 conn(0x7f1fa806df30 0x7f1fa80ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa7fff640 1 -- 172.21.15.64:0/681928456 >> v2:172.21.15.66:3300/0 conn(0x7f1fa806df30 msgr2=0x7f1fa80ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa7fff640 1 --2- 172.21.15.64:0/681928456 >> v2:172.21.15.66:3300/0 conn(0x7f1fa806df30 0x7f1fa80ae700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa7fff640 1 -- 172.21.15.64:0/681928456 >> v1:172.21.15.64:6789/0 conn(0x7f1fa8158520 legacy=0x7f1fa806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:32.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.318+0000 7f1fa7fff640 1 -- 172.21.15.64:0/681928456 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1fa8071520 con 0x7f1fa80aec40 2024-03-04T07:36:32.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.319+0000 7f1fa7fff640 1 --2- 172.21.15.64:0/681928456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa80aec40 0x7f1fa80b3640 secure :-1 s=READY pgs=1768 cs=0 l=1 rev1=1 crypto rx=0x7f1fa0009d90 tx=0x7f1fa002beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=feeaff6270a11197 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:32.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.319+0000 7f1fa67fc640 1 -- 172.21.15.64:0/681928456 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fa0039070 con 0x7f1fa80aec40 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.319+0000 7f1fa67fc640 1 -- 172.21.15.64:0/681928456 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1fa002cc60 con 0x7f1fa80aec40 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.319+0000 7f1fa67fc640 1 -- 172.21.15.64:0/681928456 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1fa0034660 con 0x7f1fa80aec40 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 -- 172.21.15.64:0/681928456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa80aec40 msgr2=0x7f1fa80b3640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/681928456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa80aec40 0x7f1fa80b3640 secure :-1 s=READY pgs=1768 cs=0 l=1 rev1=1 crypto rx=0x7f1fa0009d90 tx=0x7f1fa002beb0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 -- 172.21.15.64:0/681928456 shutdown_connections 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/681928456 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa80aec40 0x7f1fa80b3640 unknown :-1 s=CLOSED pgs=1768 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/681928456 >> v2:172.21.15.66:3300/0 conn(0x7f1fa806df30 0x7f1fa80ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 -- 172.21.15.64:0/681928456 >> 172.21.15.64:0/681928456 conn(0x7f1fa8066d00 msgr2=0x7f1fa80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 -- 172.21.15.64:0/681928456 shutdown_connections 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.320+0000 7f1fadf7a640 1 -- 172.21.15.64:0/681928456 wait complete. 2024-03-04T07:36:32.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.321+0000 7f1fadf7a640 1 Processor -- start 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.321+0000 7f1fadf7a640 1 -- start start 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fadf7a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa806df30 0x7f1fa8155dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fadf7a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fa80aec40 0x7f1fa813c9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fadf7a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fa8158520 0x7f1fa8151f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fadf7a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1fa8158900 con 0x7f1fa806df30 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fadf7a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1fa8072000 con 0x7f1fa8158520 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fadf7a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1fa8158a80 con 0x7f1fa80aec40 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa806df30 0x7f1fa8155dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa806df30 0x7f1fa8155dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38922/0 (socket says 172.21.15.64:38922) 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 -- 172.21.15.64:0/2574040257 learned_addr learned my addr 172.21.15.64:0/2574040257 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa7fff640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fa8158520 0x7f1fa8151f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa6ffd640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fa80aec40 0x7f1fa813c9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 -- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fa80aec40 msgr2=0x7f1fa813c9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fa80aec40 0x7f1fa813c9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 -- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fa8158520 msgr2=0x7f1fa8151f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fa8158520 0x7f1fa8151f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.322+0000 7f1fa77fe640 1 -- 172.21.15.64:0/2574040257 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1fa81563a0 con 0x7f1fa806df30 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.323+0000 7f1fa77fe640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa806df30 0x7f1fa8155dc0 secure :-1 s=READY pgs=1769 cs=0 l=1 rev1=1 crypto rx=0x7f1f9c00c3c0 tx=0x7f1f9c00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.323+0000 7f1f8ffff640 1 -- 172.21.15.64:0/2574040257 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f9c017070 con 0x7f1fa806df30 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.323+0000 7f1f8ffff640 1 -- 172.21.15.64:0/2574040257 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1f9c005ce0 con 0x7f1fa806df30 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.323+0000 7f1f8ffff640 1 -- 172.21.15.64:0/2574040257 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f9c0072f0 con 0x7f1fa806df30 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.323+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1fa8156680 con 0x7f1fa806df30 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.323+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1fa8156c40 con 0x7f1fa806df30 2024-03-04T07:36:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.324+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1fa8157080 con 0x7f1fa806df30 2024-03-04T07:36:32.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.324+0000 7f1f8ffff640 1 -- 172.21.15.64:0/2574040257 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1f9c01a070 con 0x7f1fa806df30 2024-03-04T07:36:32.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.325+0000 7f1f8ffff640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f9406b4d0 0x7f1f9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:32.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.325+0000 7f1f8ffff640 1 -- 172.21.15.64:0/2574040257 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f1f9c0894e0 con 0x7f1fa806df30 2024-03-04T07:36:32.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.325+0000 7f1fa6ffd640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f9406b4d0 0x7f1f9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:32.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.325+0000 7f1fa6ffd640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f9406b4d0 0x7f1f9406d990 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7f1fa8152b20 tx=0x7f1f98009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:32.329 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.329+0000 7f1f8ffff640 1 -- 172.21.15.64:0/2574040257 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f1f9c057d60 con 0x7f1fa806df30 2024-03-04T07:36:32.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.547+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1fa8067ff0 con 0x7f1fa806df30 2024-03-04T07:36:32.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.548+0000 7f1f8ffff640 1 -- 172.21.15.64:0/2574040257 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f1f9c05be00 con 0x7f1fa806df30 2024-03-04T07:36:32.548 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:32.548 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:32.548 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f9406b4d0 msgr2=0x7f1f9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f9406b4d0 0x7f1f9406d990 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7f1fa8152b20 tx=0x7f1f98009190 comp rx=0 tx=0).stop 2024-03-04T07:36:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa806df30 msgr2=0x7f1fa8155dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa806df30 0x7f1fa8155dc0 secure :-1 s=READY pgs=1769 cs=0 l=1 rev1=1 crypto rx=0x7f1f9c00c3c0 tx=0x7f1f9c00c880 comp rx=0 tx=0).stop 2024-03-04T07:36:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 shutdown_connections 2024-03-04T07:36:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1f9406b4d0 0x7f1f9406d990 unknown :-1 s=CLOSED pgs=802 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1fa8158520 0x7f1fa8151f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1fa80aec40 0x7f1fa813c9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 --2- 172.21.15.64:0/2574040257 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1fa806df30 0x7f1fa8155dc0 unknown :-1 s=CLOSED pgs=1769 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 >> 172.21.15.64:0/2574040257 conn(0x7f1fa8066d00 msgr2=0x7f1fa8159590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 shutdown_connections 2024-03-04T07:36:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:32.550+0000 7f1fadf7a640 1 -- 172.21.15.64:0/2574040257 wait complete. 2024-03-04T07:36:32.663 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:32.762 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:32.762 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:33.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:34.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.063+0000 7f748975e640 1 Processor -- start 2024-03-04T07:36:34.063 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.063+0000 7f748975e640 1 -- start start 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f748975e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f748406ed10 0x7f748406af20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f748975e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f74840cd890 0x7f748415d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f748975e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f74840cc0e0 con 0x7f74840b1020 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f748975e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f74840d28b0 con 0x7f74840cd890 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f748975e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f74840cf320 con 0x7f748406ed10 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74837fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f74840cd890 0x7f748415d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74837fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f74840cd890 0x7f748415d700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37694/0 (socket says 172.21.15.64:37694) 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74837fe640 1 -- 172.21.15.64:0/4020941273 learned_addr learned my addr 172.21.15.64:0/4020941273 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74827fc640 1 --2- 172.21.15.64:0/4020941273 >> v2:172.21.15.66:3300/0 conn(0x7f748406ed10 0x7f748406af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74837fe640 1 -- 172.21.15.64:0/4020941273 >> v2:172.21.15.66:3300/0 conn(0x7f748406ed10 msgr2=0x7f748406af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74837fe640 1 --2- 172.21.15.64:0/4020941273 >> v2:172.21.15.66:3300/0 conn(0x7f748406ed10 0x7f748406af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74837fe640 1 -- 172.21.15.64:0/4020941273 >> v1:172.21.15.66:6789/0 conn(0x7f74840b1020 legacy=0x7f74840cd180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.064+0000 7f74837fe640 1 -- 172.21.15.64:0/4020941273 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74840cc3c0 con 0x7f74840cd890 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.065+0000 7f74837fe640 1 --2- 172.21.15.64:0/4020941273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f748415d700 secure :-1 s=READY pgs=1770 cs=0 l=1 rev1=1 crypto rx=0x7f74840723a0 tx=0x7f747c02bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=156703d6e6be78c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.065+0000 7f7481ffb640 1 -- 172.21.15.64:0/4020941273 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f747c02cb20 con 0x7f74840cd890 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.065+0000 7f7481ffb640 1 -- 172.21.15.64:0/4020941273 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f747c02ccc0 con 0x7f74840cd890 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.065+0000 7f7481ffb640 1 -- 172.21.15.64:0/4020941273 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f747c034610 con 0x7f74840cd890 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 -- 172.21.15.64:0/4020941273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 msgr2=0x7f748415d700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 --2- 172.21.15.64:0/4020941273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f748415d700 secure :-1 s=READY pgs=1770 cs=0 l=1 rev1=1 crypto rx=0x7f74840723a0 tx=0x7f747c02bf80 comp rx=0 tx=0).stop 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 -- 172.21.15.64:0/4020941273 shutdown_connections 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 --2- 172.21.15.64:0/4020941273 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f748415d700 unknown :-1 s=CLOSED pgs=1770 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 --2- 172.21.15.64:0/4020941273 >> v2:172.21.15.66:3300/0 conn(0x7f748406ed10 0x7f748406af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 -- 172.21.15.64:0/4020941273 >> 172.21.15.64:0/4020941273 conn(0x7f74840d1a90 msgr2=0x7f74840d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 -- 172.21.15.64:0/4020941273 shutdown_connections 2024-03-04T07:36:34.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.066+0000 7f748975e640 1 -- 172.21.15.64:0/4020941273 wait complete. 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.067+0000 7f748975e640 1 Processor -- start 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.067+0000 7f748975e640 1 -- start start 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f748975e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f748406ed10 0x7f74840b3ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f748975e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74840b1020 0x7f748415db80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f748975e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f74840b6940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f748975e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74840ccc60 con 0x7f74840cd890 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f748975e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74840cc980 con 0x7f74840b1020 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f748975e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f74840cdca0 con 0x7f748406ed10 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f74837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f74840b6940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f74837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f74840b6940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37710/0 (socket says 172.21.15.64:37710) 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f74837fe640 1 -- 172.21.15.64:0/97083856 learned_addr learned my addr 172.21.15.64:0/97083856 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f7482ffd640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f748406ed10 0x7f74840b3ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f74827fc640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74840b1020 0x7f748415db80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:34.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.068+0000 7f74837fe640 1 -- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f748406ed10 msgr2=0x7f74840b3ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f74837fe640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f748406ed10 0x7f74840b3ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f74837fe640 1 -- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74840b1020 msgr2=0x7f748415db80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f74837fe640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74840b1020 0x7f748415db80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f74837fe640 1 -- 172.21.15.64:0/97083856 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74841f4a90 con 0x7f74840cd890 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f74827fc640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74840b1020 0x7f748415db80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f74837fe640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f74840b6940 secure :-1 s=READY pgs=1771 cs=0 l=1 rev1=1 crypto rx=0x7f747c036040 tx=0x7f747c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f746b7fe640 1 -- 172.21.15.64:0/97083856 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f747c043020 con 0x7f74840cd890 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f746b7fe640 1 -- 172.21.15.64:0/97083856 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f747c03e450 con 0x7f74840cd890 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f74840b3130 con 0x7f74840cd890 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f746b7fe640 1 -- 172.21.15.64:0/97083856 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f747c009510 con 0x7f74840cd890 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.069+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f74840b36f0 con 0x7f74840cd890 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.070+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7484067ff0 con 0x7f74840cd890 2024-03-04T07:36:34.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.071+0000 7f746b7fe640 1 -- 172.21.15.64:0/97083856 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f747c046020 con 0x7f74840cd890 2024-03-04T07:36:34.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.071+0000 7f746b7fe640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f747006b4d0 0x7f747006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:34.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.071+0000 7f746b7fe640 1 -- 172.21.15.64:0/97083856 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f747c0ac8d0 con 0x7f74840cd890 2024-03-04T07:36:34.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.071+0000 7f7482ffd640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f747006b4d0 0x7f747006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:34.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.071+0000 7f7482ffd640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f747006b4d0 0x7f747006d990 secure :-1 s=READY pgs=803 cs=0 l=1 rev1=1 crypto rx=0x7f7484070ac0 tx=0x7f7474009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:34.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.075+0000 7f746b7fe640 1 -- 172.21.15.64:0/97083856 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f747c04d030 con 0x7f74840cd890 2024-03-04T07:36:34.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.296+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f748406f2f0 con 0x7f74840cd890 2024-03-04T07:36:34.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.297+0000 7f746b7fe640 1 -- 172.21.15.64:0/97083856 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f747c07b160 con 0x7f74840cd890 2024-03-04T07:36:34.297 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:34.297 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:34.297 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.298+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f747006b4d0 msgr2=0x7f747006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.298+0000 7f748975e640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f747006b4d0 0x7f747006d990 secure :-1 s=READY pgs=803 cs=0 l=1 rev1=1 crypto rx=0x7f7484070ac0 tx=0x7f7474009040 comp rx=0 tx=0).stop 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.298+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 msgr2=0x7f74840b6940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.298+0000 7f748975e640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f74840b6940 secure :-1 s=READY pgs=1771 cs=0 l=1 rev1=1 crypto rx=0x7f747c036040 tx=0x7f747c004a40 comp rx=0 tx=0).stop 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 shutdown_connections 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f747006b4d0 0x7f747006d990 unknown :-1 s=CLOSED pgs=803 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f74840cd890 0x7f74840b6940 unknown :-1 s=CLOSED pgs=1771 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f74840b1020 0x7f748415db80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 --2- 172.21.15.64:0/97083856 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f748406ed10 0x7f74840b3ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:34.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 >> 172.21.15.64:0/97083856 conn(0x7f74840d1a90 msgr2=0x7f748406f510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:34.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 shutdown_connections 2024-03-04T07:36:34.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:34.299+0000 7f748975e640 1 -- 172.21.15.64:0/97083856 wait complete. 2024-03-04T07:36:34.510 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:34.511 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:34.666 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:35.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:35.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.625+0000 7ff3c6b93640 1 Processor -- start 2024-03-04T07:36:35.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.625+0000 7ff3c6b93640 1 -- start start 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c6b93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c6b93640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff3c00b95a0 0x7ff3c00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c6b93640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff3c00b9a30 0x7ff3c00b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c6b93640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff3c00cea30 con 0x7ff3c00b95a0 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c6b93640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff3c00d2af0 con 0x7ff3c00b9a30 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c6b93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff3c00cf3c0 con 0x7ff3c015df00 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3bffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff3c00b95a0 0x7ff3c00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c5130640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff3c00b9a30 0x7ff3c00b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:35.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.626+0000 7ff3c492f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3bffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff3c00b95a0 0x7ff3c00b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37714/0 (socket says 172.21.15.64:37714) 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3bffff640 1 -- 172.21.15.64:0/1252931485 learned_addr learned my addr 172.21.15.64:0/1252931485 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3bffff640 1 -- 172.21.15.64:0/1252931485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 msgr2=0x7ff3c00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3bffff640 1 --2- 172.21.15.64:0/1252931485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3bffff640 1 -- 172.21.15.64:0/1252931485 >> v2:172.21.15.66:3300/0 conn(0x7ff3c00b9a30 msgr2=0x7ff3c00b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3bffff640 1 --2- 172.21.15.64:0/1252931485 >> v2:172.21.15.66:3300/0 conn(0x7ff3c00b9a30 0x7ff3c00b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3bffff640 1 -- 172.21.15.64:0/1252931485 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff3c00cf0e0 con 0x7ff3c00b95a0 2024-03-04T07:36:35.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.627+0000 7ff3c492f640 1 --2- 172.21.15.64:0/1252931485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3bffff640 1 --2- 172.21.15.64:0/1252931485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c00b95a0 0x7ff3c00b5700 secure :-1 s=READY pgs=1772 cs=0 l=1 rev1=1 crypto rx=0x7ff3c00d1160 tx=0x7ff3b002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4c4c4902176a7570 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3bf7fe640 1 -- 172.21.15.64:0/1252931485 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3b0035020 con 0x7ff3c00b95a0 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3bf7fe640 1 -- 172.21.15.64:0/1252931485 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff3b0031870 con 0x7ff3c00b95a0 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3bf7fe640 1 -- 172.21.15.64:0/1252931485 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3b0031c40 con 0x7ff3c00b95a0 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 -- 172.21.15.64:0/1252931485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c00b95a0 msgr2=0x7ff3c00b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/1252931485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c00b95a0 0x7ff3c00b5700 secure :-1 s=READY pgs=1772 cs=0 l=1 rev1=1 crypto rx=0x7ff3c00d1160 tx=0x7ff3b002b910 comp rx=0 tx=0).stop 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 -- 172.21.15.64:0/1252931485 shutdown_connections 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/1252931485 >> v2:172.21.15.66:3300/0 conn(0x7ff3c00b9a30 0x7ff3c00b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/1252931485 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c00b95a0 0x7ff3c00b5700 unknown :-1 s=CLOSED pgs=1772 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/1252931485 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 -- 172.21.15.64:0/1252931485 >> 172.21.15.64:0/1252931485 conn(0x7ff3c0071960 msgr2=0x7ff3c0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 -- 172.21.15.64:0/1252931485 shutdown_connections 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.628+0000 7ff3c6b93640 1 -- 172.21.15.64:0/1252931485 wait complete. 2024-03-04T07:36:35.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.630+0000 7ff3c6b93640 1 Processor -- start 2024-03-04T07:36:35.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.630+0000 7ff3c6b93640 1 -- start start 2024-03-04T07:36:35.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.630+0000 7ff3c6b93640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff3c00b95a0 0x7ff3c01564d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:35.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.630+0000 7ff3c6b93640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00c8430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:35.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.630+0000 7ff3c6b93640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c01f4210 0x7ff3c01582a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:35.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c6b93640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff3c00cec70 con 0x7ff3c01f4210 2024-03-04T07:36:35.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c6b93640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff3c00ce750 con 0x7ff3c00b95a0 2024-03-04T07:36:35.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c6b93640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff3c00cedf0 con 0x7ff3c015df00 2024-03-04T07:36:35.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c5130640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c01f4210 0x7ff3c01582a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c492f640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff3c00b95a0 0x7ff3c01564d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c5130640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c01f4210 0x7ff3c01582a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37728/0 (socket says 172.21.15.64:37728) 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bffff640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00c8430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c5130640 1 -- 172.21.15.64:0/3460194321 learned_addr learned my addr 172.21.15.64:0/3460194321 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bffff640 1 -- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff3c00b95a0 msgr2=0x7ff3c01564d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bffff640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff3c00b95a0 0x7ff3c01564d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bffff640 1 -- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c01f4210 msgr2=0x7ff3c01582a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bffff640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c01f4210 0x7ff3c01582a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bffff640 1 -- 172.21.15.64:0/3460194321 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff3c01f4780 con 0x7ff3c015df00 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c492f640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff3c00b95a0 0x7ff3c01564d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bffff640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00c8430 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7ff3b0033000 tx=0x7ff3b00046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bd7fa640 1 -- 172.21.15.64:0/3460194321 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3b0035040 con 0x7ff3c015df00 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bd7fa640 1 -- 172.21.15.64:0/3460194321 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff3b0031e50 con 0x7ff3c015df00 2024-03-04T07:36:35.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff3c01f4a60 con 0x7ff3c015df00 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.631+0000 7ff3bd7fa640 1 -- 172.21.15.64:0/3460194321 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3b0045480 con 0x7ff3c015df00 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.632+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff3c01f5020 con 0x7ff3c015df00 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.633+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff3c00d4680 con 0x7ff3c015df00 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.633+0000 7ff3bd7fa640 1 -- 172.21.15.64:0/3460194321 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff3b0038020 con 0x7ff3c015df00 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.633+0000 7ff3bd7fa640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff3a006b4d0 0x7ff3a006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.633+0000 7ff3bd7fa640 1 -- 172.21.15.64:0/3460194321 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff3b00ac1c0 con 0x7ff3c015df00 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.633+0000 7ff3c492f640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff3a006b4d0 0x7ff3a006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:35.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.634+0000 7ff3c492f640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff3a006b4d0 0x7ff3a006d990 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7ff3b8009fd0 tx=0x7ff3b800b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:35.637 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.638+0000 7ff3bd7fa640 1 -- 172.21.15.64:0/3460194321 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff3b007a930 con 0x7ff3c015df00 2024-03-04T07:36:35.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.855+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff3c0067ff0 con 0x7ff3c015df00 2024-03-04T07:36:35.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.855+0000 7ff3bd7fa640 1 -- 172.21.15.64:0/3460194321 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7ff3b007e9d0 con 0x7ff3c015df00 2024-03-04T07:36:35.855 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:35.856 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:35.856 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff3a006b4d0 msgr2=0x7ff3a006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff3a006b4d0 0x7ff3a006d990 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7ff3b8009fd0 tx=0x7ff3b800b040 comp rx=0 tx=0).stop 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 msgr2=0x7ff3c00c8430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00c8430 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7ff3b0033000 tx=0x7ff3b00046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 shutdown_connections 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff3a006b4d0 0x7ff3a006d990 unknown :-1 s=CLOSED pgs=804 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff3c01f4210 0x7ff3c01582a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff3c015df00 0x7ff3c00c8430 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 --2- 172.21.15.64:0/3460194321 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff3c00b95a0 0x7ff3c01564d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 >> 172.21.15.64:0/3460194321 conn(0x7ff3c0071960 msgr2=0x7ff3c015be00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 shutdown_connections 2024-03-04T07:36:35.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:35.857+0000 7ff3c6b93640 1 -- 172.21.15.64:0/3460194321 wait complete. 2024-03-04T07:36:36.070 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:36.070 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:36.668 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:37.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:37.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.185+0000 7f6f17084640 1 Processor -- start 2024-03-04T07:36:37.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.185+0000 7f6f17084640 1 -- start start 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.185+0000 7f6f17084640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b95a0 0x7f6f100b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.185+0000 7f6f17084640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6f100b9a10 0x7f6f100b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.185+0000 7f6f17084640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6f100cea30 con 0x7f6f1015df00 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.185+0000 7f6f17084640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6f100cf3c0 con 0x7f6f100b9a10 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.185+0000 7f6f17084640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6f100cf0e0 con 0x7f6f100b95a0 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f0ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b95a0 0x7f6f100b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f15621640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6f100b9a10 0x7f6f100b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:37.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f14e20640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f6f1015df00 0x7f6f100d25b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:43146/0 (socket says 172.21.15.64:43146) 2024-03-04T07:36:37.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f14e20640 1 -- 172.21.15.64:0/3078629622 learned_addr learned my addr 172.21.15.64:0/3078629622 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:37.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f0ffff640 1 -- 172.21.15.64:0/3078629622 >> v2:172.21.15.66:3300/0 conn(0x7f6f100b9a10 msgr2=0x7f6f100b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:37.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f0ffff640 1 --2- 172.21.15.64:0/3078629622 >> v2:172.21.15.66:3300/0 conn(0x7f6f100b9a10 0x7f6f100b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f0ffff640 1 -- 172.21.15.64:0/3078629622 >> v1:172.21.15.66:6789/0 conn(0x7f6f1015df00 legacy=0x7f6f100d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:37.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.186+0000 7f6f0ffff640 1 -- 172.21.15.64:0/3078629622 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f100ce470 con 0x7f6f100b95a0 2024-03-04T07:36:37.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.187+0000 7f6f0ffff640 1 --2- 172.21.15.64:0/3078629622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b95a0 0x7f6f100b5700 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f6efc0097a0 tx=0x7f6efc02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=87d39c0d8c085b45 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.187+0000 7f6f0f7fe640 1 -- 172.21.15.64:0/3078629622 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6efc035020 con 0x7f6f100b95a0 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.187+0000 7f6f0f7fe640 1 -- 172.21.15.64:0/3078629622 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6efc031830 con 0x7f6f100b95a0 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.187+0000 7f6f0f7fe640 1 -- 172.21.15.64:0/3078629622 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6efc031c00 con 0x7f6f100b95a0 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 -- 172.21.15.64:0/3078629622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b95a0 msgr2=0x7f6f100b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 --2- 172.21.15.64:0/3078629622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b95a0 0x7f6f100b5700 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f6efc0097a0 tx=0x7f6efc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 -- 172.21.15.64:0/3078629622 shutdown_connections 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 --2- 172.21.15.64:0/3078629622 >> v2:172.21.15.66:3300/0 conn(0x7f6f100b9a10 0x7f6f100b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 --2- 172.21.15.64:0/3078629622 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b95a0 0x7f6f100b5700 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 -- 172.21.15.64:0/3078629622 >> 172.21.15.64:0/3078629622 conn(0x7f6f10071960 msgr2=0x7f6f10071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 -- 172.21.15.64:0/3078629622 shutdown_connections 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.188+0000 7f6f17084640 1 -- 172.21.15.64:0/3078629622 wait complete. 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.189+0000 7f6f17084640 1 Processor -- start 2024-03-04T07:36:37.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.189+0000 7f6f17084640 1 -- start start 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f17084640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f100b95a0 0x7f6f10155660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f17084640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b9a10 0x7f6f100cca10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f17084640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f1015df00 0x7f6f100d35f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f17084640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f100cedf0 con 0x7f6f1015df00 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f17084640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f100cec70 con 0x7f6f100b95a0 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f17084640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6f100d3220 con 0x7f6f100b9a10 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f15621640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f1015df00 0x7f6f100d35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f100b95a0 0x7f6f10155660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f0ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b9a10 0x7f6f100cca10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f100b95a0 0x7f6f10155660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:44758/0 (socket says 172.21.15.64:44758) 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f15621640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f1015df00 0x7f6f100d35f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37738/0 (socket says 172.21.15.64:37738) 2024-03-04T07:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 -- 172.21.15.64:0/1920774089 learned_addr learned my addr 172.21.15.64:0/1920774089 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f0ffff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b9a10 0x7f6f100cca10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:47940/0 (socket says 172.21.15.64:47940) 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 -- 172.21.15.64:0/1920774089 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b9a10 msgr2=0x7f6f100cca10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b9a10 0x7f6f100cca10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 -- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f1015df00 msgr2=0x7f6f100d35f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f1015df00 0x7f6f100d35f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 -- 172.21.15.64:0/1920774089 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f1015e480 con 0x7f6f100b95a0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f15621640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f1015df00 0x7f6f100d35f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.190+0000 7f6f14e20640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f100b95a0 0x7f6f10155660 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7f6f0400c420 tx=0x7f6f0400c8e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.191+0000 7f6f0d7fa640 1 -- 172.21.15.64:0/1920774089 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f04017070 con 0x7f6f100b95a0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.191+0000 7f6f0d7fa640 1 -- 172.21.15.64:0/1920774089 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6f04005ce0 con 0x7f6f100b95a0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.191+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6f1015e760 con 0x7f6f100b95a0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.191+0000 7f6f0d7fa640 1 -- 172.21.15.64:0/1920774089 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f040072f0 con 0x7f6f100b95a0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.191+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6f1015ed20 con 0x7f6f100b95a0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.192+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6f10067ff0 con 0x7f6f100b95a0 2024-03-04T07:36:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.192+0000 7f6f0d7fa640 1 -- 172.21.15.64:0/1920774089 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6f0401a070 con 0x7f6f100b95a0 2024-03-04T07:36:37.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.192+0000 7f6f0d7fa640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ef406b4d0 0x7f6ef406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:37.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.193+0000 7f6f0d7fa640 1 -- 172.21.15.64:0/1920774089 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6f040894d0 con 0x7f6f100b95a0 2024-03-04T07:36:37.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.193+0000 7f6f0ffff640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ef406b4d0 0x7f6ef406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:37.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.193+0000 7f6f0ffff640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ef406b4d0 0x7f6ef406d990 secure :-1 s=READY pgs=805 cs=0 l=1 rev1=1 crypto rx=0x7f6efc036000 tx=0x7f6efc008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:37.197 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.197+0000 7f6f0d7fa640 1 -- 172.21.15.64:0/1920774089 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6f04057cb0 con 0x7f6f100b95a0 2024-03-04T07:36:37.416 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.416+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6f10071e50 con 0x7f6f100b95a0 2024-03-04T07:36:37.417 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.417+0000 7f6f0d7fa640 1 -- 172.21.15.64:0/1920774089 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f6f0405bd50 con 0x7f6f100b95a0 2024-03-04T07:36:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:37.417 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:37.417 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:37.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ef406b4d0 msgr2=0x7f6ef406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:37.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ef406b4d0 0x7f6ef406d990 secure :-1 s=READY pgs=805 cs=0 l=1 rev1=1 crypto rx=0x7f6efc036000 tx=0x7f6efc008030 comp rx=0 tx=0).stop 2024-03-04T07:36:37.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f100b95a0 msgr2=0x7f6f10155660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:37.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f100b95a0 0x7f6f10155660 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7f6f0400c420 tx=0x7f6f0400c8e0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 shutdown_connections 2024-03-04T07:36:37.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ef406b4d0 0x7f6ef406d990 unknown :-1 s=CLOSED pgs=805 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.419 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6f1015df00 0x7f6f100d35f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6f100b9a10 0x7f6f100cca10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 --2- 172.21.15.64:0/1920774089 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6f100b95a0 0x7f6f10155660 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:37.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 >> 172.21.15.64:0/1920774089 conn(0x7f6f10071960 msgr2=0x7f6f100b6d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:37.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 shutdown_connections 2024-03-04T07:36:37.420 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:37.419+0000 7f6f17084640 1 -- 172.21.15.64:0/1920774089 wait complete. 2024-03-04T07:36:37.631 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:37.632 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:38.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:38.669 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:38.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.746+0000 7fef5c284640 1 Processor -- start 2024-03-04T07:36:38.746 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.746+0000 7fef5c284640 1 -- start start 2024-03-04T07:36:38.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5c284640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef5406e700 0x7fef5415ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:38.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5c284640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fef540b4160 con 0x7fef54072660 2024-03-04T07:36:38.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5c284640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fef540b5b40 con 0x7fef540b55d0 2024-03-04T07:36:38.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5c284640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fef540b3ba0 con 0x7fef5406e700 2024-03-04T07:36:38.749 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5981f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fef54072660 0x7fef5406d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51292/0 (socket says 172.21.15.64:51292) 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5981f640 1 -- 172.21.15.64:0/3270909387 learned_addr learned my addr 172.21.15.64:0/3270909387 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5a020640 1 --2- 172.21.15.64:0/3270909387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef5406e700 0x7fef5415ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5901e640 1 -- 172.21.15.64:0/3270909387 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 620212333 0 0) 0x7fef540b4160 con 0x7fef54072660 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5901e640 1 -- 172.21.15.64:0/3270909387 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fef3c003520 con 0x7fef54072660 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.747+0000 7fef5901e640 1 -- 172.21.15.64:0/3270909387 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 287282281 0 0) 0x7fef540b5b40 con 0x7fef540b55d0 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.748+0000 7fef5901e640 1 -- 172.21.15.64:0/3270909387 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fef540b4160 con 0x7fef540b55d0 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.748+0000 7fef5a020640 1 -- 172.21.15.64:0/3270909387 >> v1:172.21.15.66:6789/0 conn(0x7fef540b55d0 legacy=0x7fef54071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.748+0000 7fef5a020640 1 -- 172.21.15.64:0/3270909387 >> v1:172.21.15.64:6789/0 conn(0x7fef54072660 legacy=0x7fef5406d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.748+0000 7fef5a020640 1 -- 172.21.15.64:0/3270909387 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef540b3e80 con 0x7fef5406e700 2024-03-04T07:36:38.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.748+0000 7fef5a020640 1 --2- 172.21.15.64:0/3270909387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef5406e700 0x7fef5415ef80 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fef4c009dc0 tx=0x7fef4c02bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5ac52f61243ffa64 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5901e640 1 -- 172.21.15.64:0/3270909387 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef4c039070 con 0x7fef5406e700 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5901e640 1 -- 172.21.15.64:0/3270909387 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fef4c02ccd0 con 0x7fef5406e700 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5901e640 1 -- 172.21.15.64:0/3270909387 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef4c034640 con 0x7fef5406e700 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5c284640 1 -- 172.21.15.64:0/3270909387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef5406e700 msgr2=0x7fef5415ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5c284640 1 --2- 172.21.15.64:0/3270909387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef5406e700 0x7fef5415ef80 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fef4c009dc0 tx=0x7fef4c02bf20 comp rx=0 tx=0).stop 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5c284640 1 -- 172.21.15.64:0/3270909387 shutdown_connections 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5c284640 1 --2- 172.21.15.64:0/3270909387 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef5406e700 0x7fef5415ef80 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5c284640 1 -- 172.21.15.64:0/3270909387 >> 172.21.15.64:0/3270909387 conn(0x7fef540b2720 msgr2=0x7fef540b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5c284640 1 -- 172.21.15.64:0/3270909387 shutdown_connections 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.749+0000 7fef5c284640 1 -- 172.21.15.64:0/3270909387 wait complete. 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.750+0000 7fef5c284640 1 Processor -- start 2024-03-04T07:36:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.750+0000 7fef5c284640 1 -- start start 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5c284640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef5406e700 0x7fef541470e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5c284640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef54072660 0x7fef540b0320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5c284640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef540b55d0 0x7fef540ae040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5c284640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef54072210 con 0x7fef5406e700 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5c284640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef54072090 con 0x7fef540b55d0 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5c284640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fef540670e0 con 0x7fef54072660 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5a020640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef5406e700 0x7fef541470e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5a020640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef5406e700 0x7fef541470e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37754/0 (socket says 172.21.15.64:37754) 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5a020640 1 -- 172.21.15.64:0/1591110303 learned_addr learned my addr 172.21.15.64:0/1591110303 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5981f640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef54072660 0x7fef540b0320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5a821640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef540b55d0 0x7fef540ae040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5a020640 1 -- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef54072660 msgr2=0x7fef540b0320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.751+0000 7fef5a020640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef54072660 0x7fef540b0320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef5a020640 1 -- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef540b55d0 msgr2=0x7fef540ae040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:38.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef5a020640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef540b55d0 0x7fef540ae040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef5a020640 1 -- 172.21.15.64:0/1591110303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef540af5d0 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef5a020640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef5406e700 0x7fef541470e0 secure :-1 s=READY pgs=1774 cs=0 l=1 rev1=1 crypto rx=0x7fef4c009ee0 tx=0x7fef4c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef52ffd640 1 -- 172.21.15.64:0/1591110303 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef4c039050 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef52ffd640 1 -- 172.21.15.64:0/1591110303 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fef4c045070 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fef540af840 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef52ffd640 1 -- 172.21.15.64:0/1591110303 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef4c009450 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.752+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef540afe00 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.753+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fef5406ef00 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.753+0000 7fef52ffd640 1 -- 172.21.15.64:0/1591110303 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fef4c03e830 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.754+0000 7fef52ffd640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef4006b4d0 0x7fef4006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.754+0000 7fef52ffd640 1 -- 172.21.15.64:0/1591110303 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fef4c0ace40 con 0x7fef5406e700 2024-03-04T07:36:38.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.754+0000 7fef5981f640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef4006b4d0 0x7fef4006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:38.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.754+0000 7fef5981f640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef4006b4d0 0x7fef4006d990 secure :-1 s=READY pgs=806 cs=0 l=1 rev1=1 crypto rx=0x7fef5415bcb0 tx=0x7fef48007d70 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:38.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.758+0000 7fef52ffd640 1 -- 172.21.15.64:0/1591110303 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fef5406ef00 con 0x7fef5406e700 2024-03-04T07:36:38.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.979+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fef54067ff0 con 0x7fef5406e700 2024-03-04T07:36:38.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.979+0000 7fef52ffd640 1 -- 172.21.15.64:0/1591110303 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fef4c07b8a0 con 0x7fef5406e700 2024-03-04T07:36:38.979 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:38.980 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:38.980 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef4006b4d0 msgr2=0x7fef4006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef4006b4d0 0x7fef4006d990 secure :-1 s=READY pgs=806 cs=0 l=1 rev1=1 crypto rx=0x7fef5415bcb0 tx=0x7fef48007d70 comp rx=0 tx=0).stop 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef5406e700 msgr2=0x7fef541470e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef5406e700 0x7fef541470e0 secure :-1 s=READY pgs=1774 cs=0 l=1 rev1=1 crypto rx=0x7fef4c009ee0 tx=0x7fef4c004a40 comp rx=0 tx=0).stop 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 shutdown_connections 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fef4006b4d0 0x7fef4006d990 unknown :-1 s=CLOSED pgs=806 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fef540b55d0 0x7fef540ae040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fef54072660 0x7fef540b0320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 --2- 172.21.15.64:0/1591110303 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fef5406e700 0x7fef541470e0 unknown :-1 s=CLOSED pgs=1774 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 >> 172.21.15.64:0/1591110303 conn(0x7fef540b2720 msgr2=0x7fef54160c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 shutdown_connections 2024-03-04T07:36:38.982 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:38.981+0000 7fef5c284640 1 -- 172.21.15.64:0/1591110303 wait complete. 2024-03-04T07:36:39.197 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:39.197 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:40.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:40.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.312+0000 7f507b0e1640 1 Processor -- start 2024-03-04T07:36:40.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.312+0000 7f507b0e1640 1 -- start start 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507b0e1640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f5074072090 0x7f507415ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507b0e1640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f50740b4160 con 0x7f507406e700 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507b0e1640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f50740b5b40 con 0x7f5074072660 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507b0e1640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f50740b3ba0 con 0x7f5074072090 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f5078e7d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f507406e700 0x7f507406d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:51304/0 (socket says 172.21.15.64:51304) 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f5078e7d640 1 -- 172.21.15.64:0/1992707753 learned_addr learned my addr 172.21.15.64:0/1992707753 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507967e640 1 --2- 172.21.15.64:0/1992707753 >> v2:172.21.15.66:3300/0 conn(0x7f5074072090 0x7f507415ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:40.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507967e640 1 -- 172.21.15.64:0/1992707753 >> v1:172.21.15.66:6789/0 conn(0x7f5074072660 legacy=0x7f5074071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:40.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507967e640 1 -- 172.21.15.64:0/1992707753 >> v1:172.21.15.64:6789/0 conn(0x7f507406e700 legacy=0x7f507406d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:40.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.313+0000 7f507967e640 1 -- 172.21.15.64:0/1992707753 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50740b3e80 con 0x7f5074072090 2024-03-04T07:36:40.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.314+0000 7f507967e640 1 --2- 172.21.15.64:0/1992707753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f507415ef80 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7f506c009d90 tx=0x7f506c02bf50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=68a961dcee0c563 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:40.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.314+0000 7f50737fe640 1 -- 172.21.15.64:0/1992707753 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f506c039070 con 0x7f5074072090 2024-03-04T07:36:40.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.314+0000 7f50737fe640 1 -- 172.21.15.64:0/1992707753 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f506c02cd00 con 0x7f5074072090 2024-03-04T07:36:40.314 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.314+0000 7f50737fe640 1 -- 172.21.15.64:0/1992707753 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f506c034660 con 0x7f5074072090 2024-03-04T07:36:40.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.315+0000 7f507b0e1640 1 -- 172.21.15.64:0/1992707753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 msgr2=0x7f507415ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:40.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.315+0000 7f507b0e1640 1 --2- 172.21.15.64:0/1992707753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f507415ef80 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7f506c009d90 tx=0x7f506c02bf50 comp rx=0 tx=0).stop 2024-03-04T07:36:40.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.315+0000 7f507b0e1640 1 -- 172.21.15.64:0/1992707753 shutdown_connections 2024-03-04T07:36:40.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.315+0000 7f507b0e1640 1 --2- 172.21.15.64:0/1992707753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f507415ef80 unknown :-1 s=CLOSED pgs=653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:40.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.315+0000 7f507b0e1640 1 -- 172.21.15.64:0/1992707753 >> 172.21.15.64:0/1992707753 conn(0x7f50740b2720 msgr2=0x7f50740b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:40.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.315+0000 7f507b0e1640 1 -- 172.21.15.64:0/1992707753 shutdown_connections 2024-03-04T07:36:40.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.315+0000 7f507b0e1640 1 -- 172.21.15.64:0/1992707753 wait complete. 2024-03-04T07:36:40.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.316+0000 7f507b0e1640 1 Processor -- start 2024-03-04T07:36:40.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.316+0000 7f507b0e1640 1 -- start start 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507b0e1640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f507406e700 0x7f5074151540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507b0e1640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f50740ae280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507b0e1640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5074072660 0x7f50740ae7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507b0e1640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50740b5750 con 0x7f5074072660 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507b0e1640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50740b55d0 con 0x7f5074072090 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507b0e1640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f50740b58d0 con 0x7f507406e700 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507967e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5074072660 0x7f50740ae7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507967e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5074072660 0x7f50740ae7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37770/0 (socket says 172.21.15.64:37770) 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f507967e640 1 -- 172.21.15.64:0/2651916923 learned_addr learned my addr 172.21.15.64:0/2651916923 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:40.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f5073fff640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f50740ae280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.317+0000 7f5078e7d640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f507406e700 0x7f5074151540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507967e640 1 -- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f507406e700 msgr2=0x7f5074151540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507967e640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f507406e700 0x7f5074151540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507967e640 1 -- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 msgr2=0x7f50740ae280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507967e640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f50740ae280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507967e640 1 -- 172.21.15.64:0/2651916923 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50741f4960 con 0x7f5074072660 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f5073fff640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f50740ae280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507967e640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5074072660 0x7f50740ae7c0 secure :-1 s=READY pgs=1776 cs=0 l=1 rev1=1 crypto rx=0x7f506c009d90 tx=0x7f506c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f50717fa640 1 -- 172.21.15.64:0/2651916923 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f506c039050 con 0x7f5074072660 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f50717fa640 1 -- 172.21.15.64:0/2651916923 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f506c045070 con 0x7f5074072660 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5074151ed0 con 0x7f5074072660 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f50717fa640 1 -- 172.21.15.64:0/2651916923 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f506c03e830 con 0x7f5074072660 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.318+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5074152490 con 0x7f5074072660 2024-03-04T07:36:40.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.319+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5074067ff0 con 0x7f5074072660 2024-03-04T07:36:40.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.320+0000 7f50717fa640 1 -- 172.21.15.64:0/2651916923 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f506c03e9d0 con 0x7f5074072660 2024-03-04T07:36:40.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.320+0000 7f50717fa640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f505806b4d0 0x7f505806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:40.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.320+0000 7f5078e7d640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f505806b4d0 0x7f505806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:40.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.320+0000 7f50717fa640 1 -- 172.21.15.64:0/2651916923 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f506c0acf10 con 0x7f5074072660 2024-03-04T07:36:40.321 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.321+0000 7f5078e7d640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f505806b4d0 0x7f505806d990 secure :-1 s=READY pgs=807 cs=0 l=1 rev1=1 crypto rx=0x7f5068005880 tx=0x7f5068009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:40.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.325+0000 7f50717fa640 1 -- 172.21.15.64:0/2651916923 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f50741f4960 con 0x7f5074072660 2024-03-04T07:36:40.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.542+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5074072c30 con 0x7f5074072660 2024-03-04T07:36:40.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.542+0000 7f50717fa640 1 -- 172.21.15.64:0/2651916923 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f506c07b820 con 0x7f5074072660 2024-03-04T07:36:40.542 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:40.543 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:40.543 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f505806b4d0 msgr2=0x7f505806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f505806b4d0 0x7f505806d990 secure :-1 s=READY pgs=807 cs=0 l=1 rev1=1 crypto rx=0x7f5068005880 tx=0x7f5068009040 comp rx=0 tx=0).stop 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5074072660 msgr2=0x7f50740ae7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5074072660 0x7f50740ae7c0 secure :-1 s=READY pgs=1776 cs=0 l=1 rev1=1 crypto rx=0x7f506c009d90 tx=0x7f506c004a40 comp rx=0 tx=0).stop 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 shutdown_connections 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f505806b4d0 0x7f505806d990 unknown :-1 s=CLOSED pgs=807 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f5074072660 0x7f50740ae7c0 unknown :-1 s=CLOSED pgs=1776 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f5074072090 0x7f50740ae280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 --2- 172.21.15.64:0/2651916923 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f507406e700 0x7f5074151540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:40.545 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 >> 172.21.15.64:0/2651916923 conn(0x7f50740b2720 msgr2=0x7f5074066fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:40.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 shutdown_connections 2024-03-04T07:36:40.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:40.545+0000 7f507b0e1640 1 -- 172.21.15.64:0/2651916923 wait complete. 2024-03-04T07:36:40.672 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:40.756 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:40.756 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:41.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:41.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.871+0000 7ff9daf81640 1 Processor -- start 2024-03-04T07:36:41.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.871+0000 7ff9daf81640 1 -- start start 2024-03-04T07:36:41.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9daf81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d415b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:41.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9daf81640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff9d4157f60 0x7ff9d415b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:41.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9daf81640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff9d40cbab0 0x7ff9d4070b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9daf81640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff9d406c7b0 con 0x7ff9d4157f60 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9daf81640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff9d406ca90 con 0x7ff9d40cbab0 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9daf81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9d406cd70 con 0x7ff9d4157b80 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff9d4157f60 0x7ff9d415b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7ff9d4157f60 0x7ff9d415b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37772/0 (socket says 172.21.15.64:37772) 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d8d1d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d415b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 -- 172.21.15.64:0/3875048256 learned_addr learned my addr 172.21.15.64:0/3875048256 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d951e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff9d40cbab0 0x7ff9d4070b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 -- 172.21.15.64:0/3875048256 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 msgr2=0x7ff9d415b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 --2- 172.21.15.64:0/3875048256 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d415b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 -- 172.21.15.64:0/3875048256 >> v2:172.21.15.66:3300/0 conn(0x7ff9d40cbab0 msgr2=0x7ff9d4070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 --2- 172.21.15.64:0/3875048256 >> v2:172.21.15.66:3300/0 conn(0x7ff9d40cbab0 0x7ff9d4070b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.872+0000 7ff9d3fff640 1 -- 172.21.15.64:0/3875048256 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9d406d050 con 0x7ff9d4157f60 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.873+0000 7ff9d3fff640 1 --2- 172.21.15.64:0/3875048256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d415b9a0 secure :-1 s=READY pgs=1777 cs=0 l=1 rev1=1 crypto rx=0x7ff9d40aeb70 tx=0x7ff9c002dea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ee632bafa80935e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.873+0000 7ff9d37fe640 1 -- 172.21.15.64:0/3875048256 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9c0034070 con 0x7ff9d4157f60 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.873+0000 7ff9d37fe640 1 -- 172.21.15.64:0/3875048256 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff9c002ec50 con 0x7ff9d4157f60 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.873+0000 7ff9d37fe640 1 -- 172.21.15.64:0/3875048256 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9c0038400 con 0x7ff9d4157f60 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 -- 172.21.15.64:0/3875048256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 msgr2=0x7ff9d415b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 --2- 172.21.15.64:0/3875048256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d415b9a0 secure :-1 s=READY pgs=1777 cs=0 l=1 rev1=1 crypto rx=0x7ff9d40aeb70 tx=0x7ff9c002dea0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 -- 172.21.15.64:0/3875048256 shutdown_connections 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 --2- 172.21.15.64:0/3875048256 >> v2:172.21.15.66:3300/0 conn(0x7ff9d40cbab0 0x7ff9d4070b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 --2- 172.21.15.64:0/3875048256 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d415b9a0 unknown :-1 s=CLOSED pgs=1777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 --2- 172.21.15.64:0/3875048256 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d415b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 -- 172.21.15.64:0/3875048256 >> 172.21.15.64:0/3875048256 conn(0x7ff9d4066d00 msgr2=0x7ff9d40670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:41.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 -- 172.21.15.64:0/3875048256 shutdown_connections 2024-03-04T07:36:41.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.874+0000 7ff9daf81640 1 -- 172.21.15.64:0/3875048256 wait complete. 2024-03-04T07:36:41.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.875+0000 7ff9daf81640 1 Processor -- start 2024-03-04T07:36:41.876 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.875+0000 7ff9daf81640 1 -- start start 2024-03-04T07:36:41.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9daf81640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9d40cbab0 0x7ff9d4147730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:41.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9daf81640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d40b4040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:41.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9daf81640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d40b02c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:41.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9daf81640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9d40cc1d0 con 0x7ff9d4157f60 2024-03-04T07:36:41.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9daf81640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9d40cc050 con 0x7ff9d40cbab0 2024-03-04T07:36:41.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9daf81640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff9d4071080 con 0x7ff9d4157b80 2024-03-04T07:36:41.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d40b02c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d40b02c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:37778/0 (socket says 172.21.15.64:37778) 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d3fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d40b4040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 -- 172.21.15.64:0/1566300778 learned_addr learned my addr 172.21.15.64:0/1566300778 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d8d1d640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9d40cbab0 0x7ff9d4147730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 -- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 msgr2=0x7ff9d40b4040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d40b4040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 -- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9d40cbab0 msgr2=0x7ff9d4147730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9d40cbab0 0x7ff9d4147730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.876+0000 7ff9d951e640 1 -- 172.21.15.64:0/1566300778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9d40cdc90 con 0x7ff9d4157f60 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.877+0000 7ff9d3fff640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d40b4040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.877+0000 7ff9d951e640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d40b02c0 secure :-1 s=READY pgs=1778 cs=0 l=1 rev1=1 crypto rx=0x7ff9cc00eed0 tx=0x7ff9cc00b480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.877+0000 7ff9d17fa640 1 -- 172.21.15.64:0/1566300778 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9cc018020 con 0x7ff9d4157f60 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.877+0000 7ff9d17fa640 1 -- 172.21.15.64:0/1566300778 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff9cc014400 con 0x7ff9d4157f60 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.877+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff9d4147c70 con 0x7ff9d4157f60 2024-03-04T07:36:41.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.877+0000 7ff9d17fa640 1 -- 172.21.15.64:0/1566300778 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9cc0147f0 con 0x7ff9d4157f60 2024-03-04T07:36:41.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.877+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff9d40b2d80 con 0x7ff9d4157f60 2024-03-04T07:36:41.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.878+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff9d40cde60 con 0x7ff9d4157f60 2024-03-04T07:36:41.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.878+0000 7ff9d17fa640 1 -- 172.21.15.64:0/1566300778 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff9cc005d10 con 0x7ff9d4157f60 2024-03-04T07:36:41.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.879+0000 7ff9d17fa640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9b806b4d0 0x7ff9b806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:41.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.879+0000 7ff9d8d1d640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9b806b4d0 0x7ff9b806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:41.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.879+0000 7ff9d17fa640 1 -- 172.21.15.64:0/1566300778 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7ff9cc08aa70 con 0x7ff9d4157f60 2024-03-04T07:36:41.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.879+0000 7ff9d8d1d640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9b806b4d0 0x7ff9b806d990 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7ff9c8009b40 tx=0x7ff9c8009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:41.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:41.883+0000 7ff9d17fa640 1 -- 172.21.15.64:0/1566300778 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff9cc022030 con 0x7ff9d4157f60 2024-03-04T07:36:42.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.106+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff9d40d07a0 con 0x7ff9d4157f60 2024-03-04T07:36:42.106 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.106+0000 7ff9d17fa640 1 -- 172.21.15.64:0/1566300778 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7ff9cc059260 con 0x7ff9d4157f60 2024-03-04T07:36:42.106 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:42.106 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:42.107 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:42.108 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9b806b4d0 msgr2=0x7ff9b806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9b806b4d0 0x7ff9b806d990 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7ff9c8009b40 tx=0x7ff9c8009190 comp rx=0 tx=0).stop 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 msgr2=0x7ff9d40b02c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d40b02c0 secure :-1 s=READY pgs=1778 cs=0 l=1 rev1=1 crypto rx=0x7ff9cc00eed0 tx=0x7ff9cc00b480 comp rx=0 tx=0).stop 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 shutdown_connections 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff9b806b4d0 0x7ff9b806d990 unknown :-1 s=CLOSED pgs=808 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff9d4157f60 0x7ff9d40b02c0 unknown :-1 s=CLOSED pgs=1778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff9d4157b80 0x7ff9d40b4040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 --2- 172.21.15.64:0/1566300778 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff9d40cbab0 0x7ff9d4147730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 >> 172.21.15.64:0/1566300778 conn(0x7ff9d4066d00 msgr2=0x7ff9d415c8e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 shutdown_connections 2024-03-04T07:36:42.109 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:42.108+0000 7ff9daf81640 1 -- 172.21.15.64:0/1566300778 wait complete. 2024-03-04T07:36:42.320 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:42.321 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:42.673 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:43.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:43.434 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.434+0000 7fe128fac640 1 Processor -- start 2024-03-04T07:36:43.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.435+0000 7fe128fac640 1 -- start start 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.435+0000 7fe128fac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe12415df00 0x7fe1240d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.435+0000 7fe128fac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.435+0000 7fe128fac640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fe1240cea30 con 0x7fe1240b9a30 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.435+0000 7fe128fac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe1240d2af0 con 0x7fe12415df00 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.435+0000 7fe128fac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe1240cf3c0 con 0x7fe1240b95a0 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe121d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe12259c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe12415df00 0x7fe1240d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:43.436 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe122d9d640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fe1240b9a30 0x7fe1240d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:60176/0 (socket says 172.21.15.64:60176) 2024-03-04T07:36:43.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe121d9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:52172/0 (socket says 172.21.15.64:52172) 2024-03-04T07:36:43.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe122d9d640 1 -- 172.21.15.64:0/2226834346 learned_addr learned my addr 172.21.15.64:0/2226834346 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:43.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe121d9b640 1 -- 172.21.15.64:0/2226834346 >> v2:172.21.15.66:3300/0 conn(0x7fe12415df00 msgr2=0x7fe1240d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:43.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe121d9b640 1 --2- 172.21.15.64:0/2226834346 >> v2:172.21.15.66:3300/0 conn(0x7fe12415df00 0x7fe1240d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe121d9b640 1 -- 172.21.15.64:0/2226834346 >> v1:172.21.15.66:6789/0 conn(0x7fe1240b9a30 legacy=0x7fe1240d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:43.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe121d9b640 1 -- 172.21.15.64:0/2226834346 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1240cf0e0 con 0x7fe1240b95a0 2024-03-04T07:36:43.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.436+0000 7fe12259c640 1 --2- 172.21.15.64:0/2226834346 >> v2:172.21.15.66:3300/0 conn(0x7fe12415df00 0x7fe1240d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:43.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.437+0000 7fe121d9b640 1 --2- 172.21.15.64:0/2226834346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b5700 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fe10c0097a0 tx=0x7fe10c02b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b4787d31d2e78cc3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:43.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.437+0000 7fe12159a640 1 -- 172.21.15.64:0/2226834346 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe10c035020 con 0x7fe1240b95a0 2024-03-04T07:36:43.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.437+0000 7fe12159a640 1 -- 172.21.15.64:0/2226834346 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe10c031830 con 0x7fe1240b95a0 2024-03-04T07:36:43.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.437+0000 7fe12159a640 1 -- 172.21.15.64:0/2226834346 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe10c031c00 con 0x7fe1240b95a0 2024-03-04T07:36:43.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 -- 172.21.15.64:0/2226834346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 msgr2=0x7fe1240b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:43.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 --2- 172.21.15.64:0/2226834346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b5700 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fe10c0097a0 tx=0x7fe10c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 -- 172.21.15.64:0/2226834346 shutdown_connections 2024-03-04T07:36:43.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 --2- 172.21.15.64:0/2226834346 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b5700 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 --2- 172.21.15.64:0/2226834346 >> v2:172.21.15.66:3300/0 conn(0x7fe12415df00 0x7fe1240d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 -- 172.21.15.64:0/2226834346 >> 172.21.15.64:0/2226834346 conn(0x7fe124071960 msgr2=0x7fe124071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:43.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 -- 172.21.15.64:0/2226834346 shutdown_connections 2024-03-04T07:36:43.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.438+0000 7fe128fac640 1 -- 172.21.15.64:0/2226834346 wait complete. 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.439+0000 7fe128fac640 1 Processor -- start 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.439+0000 7fe128fac640 1 -- start start 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe128fac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b2f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe128fac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe12415df00 0x7fe12414efa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe128fac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe1240b4790 0x7fe1240b5320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe128fac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1240cec70 con 0x7fe12415df00 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe128fac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1240ce750 con 0x7fe1240b4790 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe128fac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe1240cedf0 con 0x7fe1240b95a0 2024-03-04T07:36:43.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe12415df00 0x7fe12414efa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe12415df00 0x7fe12414efa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60856/0 (socket says 172.21.15.64:60856) 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe12259c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b2f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 -- 172.21.15.64:0/2041535760 learned_addr learned my addr 172.21.15.64:0/2041535760 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe122d9d640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe1240b4790 0x7fe1240b5320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 -- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 msgr2=0x7fe1240b2f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b2f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 -- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe1240b4790 msgr2=0x7fe1240b5320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe1240b4790 0x7fe1240b5320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 -- 172.21.15.64:0/2041535760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe12414f4e0 con 0x7fe12415df00 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe12259c640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b2f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.440+0000 7fe121d9b640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe12415df00 0x7fe12414efa0 secure :-1 s=READY pgs=1779 cs=0 l=1 rev1=1 crypto rx=0x7fe10c02bee0 tx=0x7fe10c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.441+0000 7fe11affd640 1 -- 172.21.15.64:0/2041535760 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe10c035040 con 0x7fe12415df00 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.441+0000 7fe11affd640 1 -- 172.21.15.64:0/2041535760 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe10c03b920 con 0x7fe12415df00 2024-03-04T07:36:43.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.441+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe1240b36f0 con 0x7fe12415df00 2024-03-04T07:36:43.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.441+0000 7fe11affd640 1 -- 172.21.15.64:0/2041535760 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe10c03bd10 con 0x7fe12415df00 2024-03-04T07:36:43.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.441+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe1240b3cb0 con 0x7fe12415df00 2024-03-04T07:36:43.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.442+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe124067ff0 con 0x7fe12415df00 2024-03-04T07:36:43.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.442+0000 7fe11affd640 1 -- 172.21.15.64:0/2041535760 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe10c038020 con 0x7fe12415df00 2024-03-04T07:36:43.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.443+0000 7fe11affd640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe10406b4d0 0x7fe10406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:43.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.443+0000 7fe11affd640 1 -- 172.21.15.64:0/2041535760 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fe10c0ac200 con 0x7fe12415df00 2024-03-04T07:36:43.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.443+0000 7fe12259c640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe10406b4d0 0x7fe10406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:43.444 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.443+0000 7fe12259c640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe10406b4d0 0x7fe10406d990 secure :-1 s=READY pgs=809 cs=0 l=1 rev1=1 crypto rx=0x7fe114009fd0 tx=0x7fe11400b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:43.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.447+0000 7fe11affd640 1 -- 172.21.15.64:0/2041535760 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe10c031800 con 0x7fe12415df00 2024-03-04T07:36:43.667 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.668+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe1240caeb0 con 0x7fe12415df00 2024-03-04T07:36:43.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.668+0000 7fe11affd640 1 -- 172.21.15.64:0/2041535760 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fe10c07aa00 con 0x7fe12415df00 2024-03-04T07:36:43.668 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:43.668 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:43.669 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:43.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe10406b4d0 msgr2=0x7fe10406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:43.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe10406b4d0 0x7fe10406d990 secure :-1 s=READY pgs=809 cs=0 l=1 rev1=1 crypto rx=0x7fe114009fd0 tx=0x7fe11400b040 comp rx=0 tx=0).stop 2024-03-04T07:36:43.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe12415df00 msgr2=0x7fe12414efa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe12415df00 0x7fe12414efa0 secure :-1 s=READY pgs=1779 cs=0 l=1 rev1=1 crypto rx=0x7fe10c02bee0 tx=0x7fe10c0046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 shutdown_connections 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe10406b4d0 0x7fe10406d990 unknown :-1 s=CLOSED pgs=809 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe1240b4790 0x7fe1240b5320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe12415df00 0x7fe12414efa0 unknown :-1 s=CLOSED pgs=1779 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 --2- 172.21.15.64:0/2041535760 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe1240b95a0 0x7fe1240b2f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 >> 172.21.15.64:0/2041535760 conn(0x7fe124071960 msgr2=0x7fe1240b8b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 shutdown_connections 2024-03-04T07:36:43.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:43.670+0000 7fe128fac640 1 -- 172.21.15.64:0/2041535760 wait complete. 2024-03-04T07:36:43.883 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:43.883 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:44.675 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:44.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:44.998 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:44.998+0000 7fb29769a640 1 Processor -- start 2024-03-04T07:36:44.999 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:44.999+0000 7fb29769a640 1 -- start start 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:44.999+0000 7fb29769a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb2900b95a0 0x7fb2900b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:44.999+0000 7fb29769a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb2900b9a10 0x7fb2900b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:44.999+0000 7fb29769a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fb2900cea30 con 0x7fb29015df00 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:44.999+0000 7fb29769a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb2900cf3c0 con 0x7fb2900b9a10 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:44.999+0000 7fb29769a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb2900cf0e0 con 0x7fb2900b95a0 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb295c37640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb2900b9a10 0x7fb2900b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb295c37640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb2900b9a10 0x7fb2900b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60868/0 (socket says 172.21.15.64:60868) 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb295c37640 1 -- 172.21.15.64:0/3128753655 learned_addr learned my addr 172.21.15.64:0/3128753655 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb294c35640 1 --2- 172.21.15.64:0/3128753655 >> v2:172.21.15.66:3300/0 conn(0x7fb2900b95a0 0x7fb2900b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:45.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb295c37640 1 -- 172.21.15.64:0/3128753655 >> v2:172.21.15.66:3300/0 conn(0x7fb2900b95a0 msgr2=0x7fb2900b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:45.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb295c37640 1 --2- 172.21.15.64:0/3128753655 >> v2:172.21.15.66:3300/0 conn(0x7fb2900b95a0 0x7fb2900b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb295c37640 1 -- 172.21.15.64:0/3128753655 >> v1:172.21.15.66:6789/0 conn(0x7fb29015df00 legacy=0x7fb2900d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:45.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.000+0000 7fb295c37640 1 -- 172.21.15.64:0/3128753655 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2900ce470 con 0x7fb2900b9a10 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb295c37640 1 --2- 172.21.15.64:0/3128753655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb2900b5c40 secure :-1 s=READY pgs=1780 cs=0 l=1 rev1=1 crypto rx=0x7fb2900d1760 tx=0x7fb28802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f7e5706410d97952 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb28ffff640 1 -- 172.21.15.64:0/3128753655 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb288034070 con 0x7fb2900b9a10 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb28ffff640 1 -- 172.21.15.64:0/3128753655 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb288030860 con 0x7fb2900b9a10 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb28ffff640 1 -- 172.21.15.64:0/3128753655 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb288030c30 con 0x7fb2900b9a10 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb29769a640 1 -- 172.21.15.64:0/3128753655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 msgr2=0x7fb2900b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb29769a640 1 --2- 172.21.15.64:0/3128753655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb2900b5c40 secure :-1 s=READY pgs=1780 cs=0 l=1 rev1=1 crypto rx=0x7fb2900d1760 tx=0x7fb28802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb29769a640 1 -- 172.21.15.64:0/3128753655 shutdown_connections 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb29769a640 1 --2- 172.21.15.64:0/3128753655 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb2900b5c40 unknown :-1 s=CLOSED pgs=1780 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb29769a640 1 --2- 172.21.15.64:0/3128753655 >> v2:172.21.15.66:3300/0 conn(0x7fb2900b95a0 0x7fb2900b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.001+0000 7fb29769a640 1 -- 172.21.15.64:0/3128753655 >> 172.21.15.64:0/3128753655 conn(0x7fb290071960 msgr2=0x7fb290071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:45.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.002+0000 7fb29769a640 1 -- 172.21.15.64:0/3128753655 shutdown_connections 2024-03-04T07:36:45.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.002+0000 7fb29769a640 1 -- 172.21.15.64:0/3128753655 wait complete. 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 Processor -- start 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 -- start start 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2900b95a0 0x7fb2900cd5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb29015f180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb29015df00 0x7fb2901e8aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2900cedf0 con 0x7fb2900b9a10 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2900cec70 con 0x7fb29015df00 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.003+0000 7fb29769a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb2900d3220 con 0x7fb2900b95a0 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb29015f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb29015f180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60884/0 (socket says 172.21.15.64:60884) 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 -- 172.21.15.64:0/522921487 learned_addr learned my addr 172.21.15.64:0/522921487 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb295436640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2900b95a0 0x7fb2900cd5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb295c37640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb29015df00 0x7fb2901e8aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 -- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2900b95a0 msgr2=0x7fb2900cd5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2900b95a0 0x7fb2900cd5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 -- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb29015df00 msgr2=0x7fb2901e8aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb29015df00 0x7fb2901e8aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 -- 172.21.15.64:0/522921487 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb290160c70 con 0x7fb2900b9a10 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb294c35640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb29015f180 secure :-1 s=READY pgs=1781 cs=0 l=1 rev1=1 crypto rx=0x7fb28400e9b0 tx=0x7fb28400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb28dffb640 1 -- 172.21.15.64:0/522921487 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb284017070 con 0x7fb2900b9a10 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb28dffb640 1 -- 172.21.15.64:0/522921487 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb28400c9a0 con 0x7fb2900b9a10 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb28dffb640 1 -- 172.21.15.64:0/522921487 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb28400cd90 con 0x7fb2900b9a10 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb29015fbd0 con 0x7fb2900b9a10 2024-03-04T07:36:45.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.004+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb290160190 con 0x7fb2900b9a10 2024-03-04T07:36:45.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.005+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb2900b6d70 con 0x7fb2900b9a10 2024-03-04T07:36:45.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.006+0000 7fb28dffb640 1 -- 172.21.15.64:0/522921487 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb28401a070 con 0x7fb2900b9a10 2024-03-04T07:36:45.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.006+0000 7fb28dffb640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb27c06b500 0x7fb27c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:45.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.007+0000 7fb295436640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb27c06b500 0x7fb27c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:45.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.007+0000 7fb28dffb640 1 -- 172.21.15.64:0/522921487 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb28408a180 con 0x7fb2900b9a10 2024-03-04T07:36:45.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.007+0000 7fb295436640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb27c06b500 0x7fb27c06d9c0 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7fb280001e70 tx=0x7fb280009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:45.011 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.011+0000 7fb28dffb640 1 -- 172.21.15.64:0/522921487 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb2900b6d70 con 0x7fb2900b9a10 2024-03-04T07:36:45.231 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.231+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb290067ff0 con 0x7fb2900b9a10 2024-03-04T07:36:45.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.232+0000 7fb28dffb640 1 -- 172.21.15.64:0/522921487 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fb284058970 con 0x7fb2900b9a10 2024-03-04T07:36:45.232 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:45.232 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:45.232 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:45.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb27c06b500 msgr2=0x7fb27c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:45.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb27c06b500 0x7fb27c06d9c0 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7fb280001e70 tx=0x7fb280009040 comp rx=0 tx=0).stop 2024-03-04T07:36:45.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 msgr2=0x7fb29015f180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:45.234 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb29015f180 secure :-1 s=READY pgs=1781 cs=0 l=1 rev1=1 crypto rx=0x7fb28400e9b0 tx=0x7fb28400ee70 comp rx=0 tx=0).stop 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 shutdown_connections 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb27c06b500 0x7fb27c06d9c0 unknown :-1 s=CLOSED pgs=810 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb29015df00 0x7fb2901e8aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb2900b9a10 0x7fb29015f180 unknown :-1 s=CLOSED pgs=1781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 --2- 172.21.15.64:0/522921487 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb2900b95a0 0x7fb2900cd5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 >> 172.21.15.64:0/522921487 conn(0x7fb290071960 msgr2=0x7fb29015d1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 shutdown_connections 2024-03-04T07:36:45.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:45.234+0000 7fb29769a640 1 -- 172.21.15.64:0/522921487 wait complete. 2024-03-04T07:36:45.447 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:45.447 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:46.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:46.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.562+0000 7f4ce02f4640 1 Processor -- start 2024-03-04T07:36:46.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.562+0000 7f4ce02f4640 1 -- start start 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4ce02f4640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4cd80b95a0 0x7f4cd80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4ce02f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd80b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4ce02f4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4cd80cea30 con 0x7f4cd815df00 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4ce02f4640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4cd80cf3c0 con 0x7f4cd80b95a0 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4ce02f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4cd80cf0e0 con 0x7f4cd80b9a10 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd88f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4cd80b95a0 0x7f4cd80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd88f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4cd80b95a0 0x7f4cd80b5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60892/0 (socket says 172.21.15.64:60892) 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd88f640 1 -- 172.21.15.64:0/2616070841 learned_addr learned my addr 172.21.15.64:0/2616070841 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cde891640 1 --2- 172.21.15.64:0/2616070841 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd80b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd88f640 1 -- 172.21.15.64:0/2616070841 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 msgr2=0x7f4cd80b5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:46.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd88f640 1 --2- 172.21.15.64:0/2616070841 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd80b5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd88f640 1 -- 172.21.15.64:0/2616070841 >> v1:172.21.15.66:6789/0 conn(0x7f4cd815df00 legacy=0x7f4cd80d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd88f640 1 -- 172.21.15.64:0/2616070841 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cd80ce470 con 0x7f4cd80b95a0 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.563+0000 7f4cdd08e640 1 -- 172.21.15.64:0/2616070841 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 738790322 0 0) 0x7f4cd80cea30 con 0x7f4cd815df00 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.564+0000 7f4cdd88f640 1 --2- 172.21.15.64:0/2616070841 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd80b95a0 0x7f4cd80b5700 secure :-1 s=READY pgs=1782 cs=0 l=1 rev1=1 crypto rx=0x7f4cc8009870 tx=0x7f4cc802b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf98a5ff9f151fc9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.564+0000 7f4cdd08e640 1 -- 172.21.15.64:0/2616070841 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cc8034070 con 0x7f4cd80b95a0 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.564+0000 7f4cdd08e640 1 -- 172.21.15.64:0/2616070841 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cc8030840 con 0x7f4cd80b95a0 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.564+0000 7f4cdd08e640 1 -- 172.21.15.64:0/2616070841 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cc8030c10 con 0x7f4cd80b95a0 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2616070841 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd80b95a0 msgr2=0x7f4cd80b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2616070841 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd80b95a0 0x7f4cd80b5700 secure :-1 s=READY pgs=1782 cs=0 l=1 rev1=1 crypto rx=0x7f4cc8009870 tx=0x7f4cc802b980 comp rx=0 tx=0).stop 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2616070841 shutdown_connections 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2616070841 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd80b5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2616070841 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd80b95a0 0x7f4cd80b5700 unknown :-1 s=CLOSED pgs=1782 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2616070841 >> 172.21.15.64:0/2616070841 conn(0x7f4cd8071960 msgr2=0x7f4cd8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2616070841 shutdown_connections 2024-03-04T07:36:46.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.565+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2616070841 wait complete. 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.566+0000 7f4ce02f4640 1 Processor -- start 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.566+0000 7f4ce02f4640 1 -- start start 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4ce02f4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4cd80b95a0 0x7f4cd80c77b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4ce02f4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd814fa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4ce02f4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd815df00 0x7f4cd80c8fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4ce02f4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cd80cedf0 con 0x7f4cd815df00 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4ce02f4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cd80cec70 con 0x7f4cd80b95a0 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4ce02f4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4cd80d3220 con 0x7f4cd80b9a10 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cde891640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd815df00 0x7f4cd80c8fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cdd88f640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd814fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cde090640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4cd80b95a0 0x7f4cd80c77b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:46.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cde891640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd815df00 0x7f4cd80c8fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60908/0 (socket says 172.21.15.64:60908) 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cde891640 1 -- 172.21.15.64:0/2930792727 learned_addr learned my addr 172.21.15.64:0/2930792727 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cdd88f640 1 -- 172.21.15.64:0/2930792727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4cd80b95a0 msgr2=0x7f4cd80c77b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cdd88f640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4cd80b95a0 0x7f4cd80c77b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cdd88f640 1 -- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd815df00 msgr2=0x7f4cd80c8fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cdd88f640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd815df00 0x7f4cd80c8fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cdd88f640 1 -- 172.21.15.64:0/2930792727 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cd814ffa0 con 0x7f4cd80b9a10 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.567+0000 7f4cde891640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd815df00 0x7f4cd80c8fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.568+0000 7f4cdd88f640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd814fa60 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f4cc8033000 tx=0x7f4cc80046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.568+0000 7f4cceffd640 1 -- 172.21.15.64:0/2930792727 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cc8034040 con 0x7f4cd80b9a10 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.568+0000 7f4cceffd640 1 -- 172.21.15.64:0/2930792727 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4cc8030640 con 0x7f4cd80b9a10 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.568+0000 7f4cceffd640 1 -- 172.21.15.64:0/2930792727 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cc8039cc0 con 0x7f4cd80b9a10 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.568+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4cd80c7f20 con 0x7f4cd80b9a10 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.568+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4cd80c84e0 con 0x7f4cd80b9a10 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.569+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4cd8067ff0 con 0x7f4cd80b9a10 2024-03-04T07:36:46.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.569+0000 7f4cceffd640 1 -- 172.21.15.64:0/2930792727 <== mon.2 v2:172.21.15.66:3301/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4cc8037070 con 0x7f4cd80b9a10 2024-03-04T07:36:46.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.569+0000 7f4cceffd640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cbc06b4d0 0x7f4cbc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:46.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.570+0000 7f4cceffd640 1 -- 172.21.15.64:0/2930792727 <== mon.2 v2:172.21.15.66:3301/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4cc80abe70 con 0x7f4cd80b9a10 2024-03-04T07:36:46.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.570+0000 7f4cde090640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cbc06b4d0 0x7f4cbc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:46.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.570+0000 7f4cde090640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cbc06b4d0 0x7f4cbc06d990 secure :-1 s=READY pgs=811 cs=0 l=1 rev1=1 crypto rx=0x7f4cd4000f60 tx=0x7f4cd4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:46.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.574+0000 7f4cceffd640 1 -- 172.21.15.64:0/2930792727 <== mon.2 v2:172.21.15.66:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4cc807a5c0 con 0x7f4cd80b9a10 2024-03-04T07:36:46.676 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:46.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.795+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4cd815d470 con 0x7f4cd80b9a10 2024-03-04T07:36:46.795 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.795+0000 7f4cceffd640 1 -- 172.21.15.64:0/2930792727 <== mon.2 v2:172.21.15.66:3301/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f4cc807e660 con 0x7f4cd80b9a10 2024-03-04T07:36:46.795 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:46.796 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:46.796 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cbc06b4d0 msgr2=0x7f4cbc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cbc06b4d0 0x7f4cbc06d990 secure :-1 s=READY pgs=811 cs=0 l=1 rev1=1 crypto rx=0x7f4cd4000f60 tx=0x7f4cd4009040 comp rx=0 tx=0).stop 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 msgr2=0x7f4cd814fa60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd814fa60 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f4cc8033000 tx=0x7f4cc80046c0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 shutdown_connections 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4cbc06b4d0 0x7f4cbc06d990 unknown :-1 s=CLOSED pgs=811 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4cd815df00 0x7f4cd80c8fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4cd80b9a10 0x7f4cd814fa60 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 --2- 172.21.15.64:0/2930792727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4cd80b95a0 0x7f4cd80c77b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 >> 172.21.15.64:0/2930792727 conn(0x7f4cd8071960 msgr2=0x7f4cd80b3630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.797+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 shutdown_connections 2024-03-04T07:36:46.798 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:46.798+0000 7f4ce02f4640 1 -- 172.21.15.64:0/2930792727 wait complete. 2024-03-04T07:36:47.009 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:47.009 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:48.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:48.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.124+0000 7f43bf20e640 1 Processor -- start 2024-03-04T07:36:48.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.124+0000 7f43bf20e640 1 -- start start 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.124+0000 7f43bf20e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80cf660 0x7f43b80cfb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.124+0000 7f43bf20e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f43b80d2580 0x7f43b80d0050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.124+0000 7f43bf20e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f43b80cf270 con 0x7f43b815bea0 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.124+0000 7f43bf20e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f43b815c280 con 0x7f43b80d2580 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.124+0000 7f43bf20e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43b80cc850 con 0x7f43b80cf660 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43bd7ab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f43b80d2580 0x7f43b80d0050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43bd7ab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f43b80d2580 0x7f43b80d0050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60918/0 (socket says 172.21.15.64:60918) 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43bd7ab640 1 -- 172.21.15.64:0/3784776064 learned_addr learned my addr 172.21.15.64:0/3784776064 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43b7fff640 1 --2- 172.21.15.64:0/3784776064 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80cf660 0x7f43b80cfb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:48.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43bd7ab640 1 -- 172.21.15.64:0/3784776064 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80cf660 msgr2=0x7f43b80cfb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:48.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43bd7ab640 1 --2- 172.21.15.64:0/3784776064 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80cf660 0x7f43b80cfb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43bd7ab640 1 -- 172.21.15.64:0/3784776064 >> v1:172.21.15.66:6789/0 conn(0x7f43b815bea0 legacy=0x7f43b80cd330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:48.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.125+0000 7f43bd7ab640 1 -- 172.21.15.64:0/3784776064 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43b80ccb30 con 0x7f43b80d2580 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43bd7ab640 1 --2- 172.21.15.64:0/3784776064 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b80d2580 0x7f43b80d0050 secure :-1 s=READY pgs=1783 cs=0 l=1 rev1=1 crypto rx=0x7f43b0009e80 tx=0x7f43b002be20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d1a2093c83a8ce1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43b77fe640 1 -- 172.21.15.64:0/3784776064 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43b002c9c0 con 0x7f43b80d2580 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43b77fe640 1 -- 172.21.15.64:0/3784776064 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43b002cb60 con 0x7f43b80d2580 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43b77fe640 1 -- 172.21.15.64:0/3784776064 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43b00343f0 con 0x7f43b80d2580 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43bf20e640 1 -- 172.21.15.64:0/3784776064 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b80d2580 msgr2=0x7f43b80d0050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43bf20e640 1 --2- 172.21.15.64:0/3784776064 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b80d2580 0x7f43b80d0050 secure :-1 s=READY pgs=1783 cs=0 l=1 rev1=1 crypto rx=0x7f43b0009e80 tx=0x7f43b002be20 comp rx=0 tx=0).stop 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43bf20e640 1 -- 172.21.15.64:0/3784776064 shutdown_connections 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43bf20e640 1 --2- 172.21.15.64:0/3784776064 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b80d2580 0x7f43b80d0050 unknown :-1 s=CLOSED pgs=1783 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43bf20e640 1 --2- 172.21.15.64:0/3784776064 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80cf660 0x7f43b80cfb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.126+0000 7f43bf20e640 1 -- 172.21.15.64:0/3784776064 >> 172.21.15.64:0/3784776064 conn(0x7f43b80d1a90 msgr2=0x7f43b80d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:48.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.127+0000 7f43bf20e640 1 -- 172.21.15.64:0/3784776064 shutdown_connections 2024-03-04T07:36:48.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.127+0000 7f43bf20e640 1 -- 172.21.15.64:0/3784776064 wait complete. 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.128+0000 7f43bf20e640 1 Processor -- start 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.128+0000 7f43bf20e640 1 -- start start 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bf20e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43b80cf660 0x7f43b8155ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bf20e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80d2580 0x7f43b80b6e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bf20e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b815bea0 0x7f43b80b7390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bf20e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43b80ceeb0 con 0x7f43b815bea0 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bf20e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43b80cc500 con 0x7f43b80cf660 2024-03-04T07:36:48.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bf20e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f43b80cf030 con 0x7f43b80d2580 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b815bea0 0x7f43b80b7390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b815bea0 0x7f43b80b7390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60932/0 (socket says 172.21.15.64:60932) 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 -- 172.21.15.64:0/4215749115 learned_addr learned my addr 172.21.15.64:0/4215749115 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43b7fff640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80d2580 0x7f43b80b6e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bcfaa640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43b80cf660 0x7f43b8155ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 -- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80d2580 msgr2=0x7f43b80b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80d2580 0x7f43b80b6e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 -- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43b80cf660 msgr2=0x7f43b8155ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43b80cf660 0x7f43b8155ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 -- 172.21.15.64:0/4215749115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f43b80c97a0 con 0x7f43b815bea0 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43b7fff640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80d2580 0x7f43b80b6e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43bd7ab640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b815bea0 0x7f43b80b7390 secure :-1 s=READY pgs=1784 cs=0 l=1 rev1=1 crypto rx=0x7f43b0036040 tx=0x7f43b0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:48.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.129+0000 7f43b57fa640 1 -- 172.21.15.64:0/4215749115 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43b0043020 con 0x7f43b815bea0 2024-03-04T07:36:48.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.130+0000 7f43b57fa640 1 -- 172.21.15.64:0/4215749115 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f43b003e450 con 0x7f43b815bea0 2024-03-04T07:36:48.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.130+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f43b80c9a80 con 0x7f43b815bea0 2024-03-04T07:36:48.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.130+0000 7f43b57fa640 1 -- 172.21.15.64:0/4215749115 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f43b0009510 con 0x7f43b815bea0 2024-03-04T07:36:48.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.130+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f43b80ca040 con 0x7f43b815bea0 2024-03-04T07:36:48.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.131+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f43b8067ff0 con 0x7f43b815bea0 2024-03-04T07:36:48.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.131+0000 7f43b57fa640 1 -- 172.21.15.64:0/4215749115 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f43b0046020 con 0x7f43b815bea0 2024-03-04T07:36:48.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.131+0000 7f43b57fa640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f439c06b500 0x7f439c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:48.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.132+0000 7f43b57fa640 1 -- 172.21.15.64:0/4215749115 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f43b00ac800 con 0x7f43b815bea0 2024-03-04T07:36:48.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.132+0000 7f43bcfaa640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f439c06b500 0x7f439c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:48.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.132+0000 7f43bcfaa640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f439c06b500 0x7f439c06d9c0 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7f43ac0016e0 tx=0x7f43ac009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:48.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.136+0000 7f43b57fa640 1 -- 172.21.15.64:0/4215749115 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f43b004d030 con 0x7f43b815bea0 2024-03-04T07:36:48.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.359+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f43b8070b20 con 0x7f43b815bea0 2024-03-04T07:36:48.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.359+0000 7f43b57fa640 1 -- 172.21.15.64:0/4215749115 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f43b007b000 con 0x7f43b815bea0 2024-03-04T07:36:48.360 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:48.360 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:48.360 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.361+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f439c06b500 msgr2=0x7f439c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.361+0000 7f43bf20e640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f439c06b500 0x7f439c06d9c0 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7f43ac0016e0 tx=0x7f43ac009040 comp rx=0 tx=0).stop 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b815bea0 msgr2=0x7f43b80b7390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b815bea0 0x7f43b80b7390 secure :-1 s=READY pgs=1784 cs=0 l=1 rev1=1 crypto rx=0x7f43b0036040 tx=0x7f43b0004a40 comp rx=0 tx=0).stop 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 shutdown_connections 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f439c06b500 0x7f439c06d9c0 unknown :-1 s=CLOSED pgs=812 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f43b815bea0 0x7f43b80b7390 unknown :-1 s=CLOSED pgs=1784 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f43b80d2580 0x7f43b80b6e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 --2- 172.21.15.64:0/4215749115 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f43b80cf660 0x7f43b8155ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:48.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 >> 172.21.15.64:0/4215749115 conn(0x7f43b80d1a90 msgr2=0x7f43b80cd330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:48.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 shutdown_connections 2024-03-04T07:36:48.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:48.362+0000 7f43bf20e640 1 -- 172.21.15.64:0/4215749115 wait complete. 2024-03-04T07:36:48.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:36:48.557 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:36:48.574 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:48.574 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:48.679 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:49.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:49.688 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.689+0000 7f79a613e640 1 Processor -- start 2024-03-04T07:36:49.689 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.689+0000 7f79a613e640 1 -- start start 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.689+0000 7f79a613e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f79a015bda0 0x7f79a00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.689+0000 7f79a613e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00cea70 0x7f79a006f290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.689+0000 7f79a613e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f79a00cd350 con 0x7f79a00b0ed0 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.689+0000 7f79a613e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f79a006eb20 con 0x7f79a015bda0 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.689+0000 7f79a613e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79a00b1330 con 0x7f79a00cea70 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f79a00b0ed0 0x7f79a00ce360 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42920/0 (socket says 172.21.15.64:42920) 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799effd640 1 -- 172.21.15.64:0/3394016158 learned_addr learned my addr 172.21.15.64:0/3394016158 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799f7fe640 1 --2- 172.21.15.64:0/3394016158 >> v2:172.21.15.66:3300/0 conn(0x7f79a015bda0 0x7f79a00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:49.690 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799ffff640 1 --2- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00cea70 0x7f79a006f290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:49.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799e7fc640 1 -- 172.21.15.64:0/3394016158 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2746420721 0 0) 0x7f79a00cd350 con 0x7f79a00b0ed0 2024-03-04T07:36:49.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799e7fc640 1 -- 172.21.15.64:0/3394016158 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7988003520 con 0x7f79a00b0ed0 2024-03-04T07:36:49.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799f7fe640 1 -- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00cea70 msgr2=0x7f79a006f290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:49.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799f7fe640 1 --2- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00cea70 0x7f79a006f290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799f7fe640 1 -- 172.21.15.64:0/3394016158 >> v1:172.21.15.64:6789/0 conn(0x7f79a00b0ed0 legacy=0x7f79a00ce360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:49.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799f7fe640 1 -- 172.21.15.64:0/3394016158 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79a00cd630 con 0x7f79a015bda0 2024-03-04T07:36:49.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.690+0000 7f799ffff640 1 --2- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00cea70 0x7f79a006f290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:49.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.691+0000 7f799f7fe640 1 --2- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 0x7f79a00d25b0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f79900097a0 tx=0x7f799002b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ae84aa00d55a677e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:49.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.691+0000 7f799e7fc640 1 -- 172.21.15.64:0/3394016158 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7990035020 con 0x7f79a015bda0 2024-03-04T07:36:49.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.691+0000 7f799e7fc640 1 -- 172.21.15.64:0/3394016158 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7990031830 con 0x7f79a015bda0 2024-03-04T07:36:49.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.691+0000 7f799e7fc640 1 -- 172.21.15.64:0/3394016158 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7990031c00 con 0x7f79a015bda0 2024-03-04T07:36:49.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 -- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 msgr2=0x7f79a00d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:49.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 --2- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 0x7f79a00d25b0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f79900097a0 tx=0x7f799002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 -- 172.21.15.64:0/3394016158 shutdown_connections 2024-03-04T07:36:49.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 --2- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00cea70 0x7f79a006f290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 --2- 172.21.15.64:0/3394016158 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 0x7f79a00d25b0 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 -- 172.21.15.64:0/3394016158 >> 172.21.15.64:0/3394016158 conn(0x7f79a00d1a90 msgr2=0x7f79a00d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:49.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 -- 172.21.15.64:0/3394016158 shutdown_connections 2024-03-04T07:36:49.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.692+0000 7f79a613e640 1 -- 172.21.15.64:0/3394016158 wait complete. 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.693+0000 7f79a613e640 1 Processor -- start 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.693+0000 7f79a613e640 1 -- start start 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f79a613e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00b0ed0 0x7f79a014f4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f79a613e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79a00cea70 0x7f79a00b48e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f79a613e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 0x7f79a00b4e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f79a613e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79a015c180 con 0x7f79a00cea70 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f79a613e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79a006ed70 con 0x7f79a015bda0 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f79a613e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f79a015c300 con 0x7f79a00b0ed0 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79a00cea70 0x7f79a00b48e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:49.695 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79a00cea70 0x7f79a00b48e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60946/0 (socket says 172.21.15.64:60946) 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 -- 172.21.15.64:0/2720129425 learned_addr learned my addr 172.21.15.64:0/2720129425 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799f7fe640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00b0ed0 0x7f79a014f4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799ffff640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 0x7f79a00b4e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 -- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00b0ed0 msgr2=0x7f79a014f4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00b0ed0 0x7f79a014f4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 -- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 msgr2=0x7f79a00b4e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 0x7f79a00b4e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 -- 172.21.15.64:0/2720129425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79a00b5450 con 0x7f79a00cea70 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.694+0000 7f799effd640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79a00cea70 0x7f79a00b48e0 secure :-1 s=READY pgs=1786 cs=0 l=1 rev1=1 crypto rx=0x7f799400c390 tx=0x7f799400c790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.695+0000 7f7987fff640 1 -- 172.21.15.64:0/2720129425 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7994017070 con 0x7f79a00cea70 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.695+0000 7f7987fff640 1 -- 172.21.15.64:0/2720129425 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7994005ce0 con 0x7f79a00cea70 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.695+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f79a00b5730 con 0x7f79a00cea70 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.695+0000 7f7987fff640 1 -- 172.21.15.64:0/2720129425 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79940072f0 con 0x7f79a00cea70 2024-03-04T07:36:49.696 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.695+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f79a00b5cf0 con 0x7f79a00cea70 2024-03-04T07:36:49.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.696+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f79a0067ff0 con 0x7f79a00cea70 2024-03-04T07:36:49.697 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.696+0000 7f7987fff640 1 -- 172.21.15.64:0/2720129425 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f799401a070 con 0x7f79a00cea70 2024-03-04T07:36:49.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.696+0000 7f7987fff640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f798c06b500 0x7f798c06d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:49.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.697+0000 7f7987fff640 1 -- 172.21.15.64:0/2720129425 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f79940893b0 con 0x7f79a00cea70 2024-03-04T07:36:49.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.697+0000 7f799f7fe640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f798c06b500 0x7f798c06d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:49.698 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.697+0000 7f799f7fe640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f798c06b500 0x7f798c06d9c0 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7f7990036000 tx=0x7f7990008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:49.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.701+0000 7f7987fff640 1 -- 172.21.15.64:0/2720129425 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7994057c30 con 0x7f79a00cea70 2024-03-04T07:36:49.921 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.921+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f79a00b4680 con 0x7f79a00cea70 2024-03-04T07:36:49.922 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.922+0000 7f7987fff640 1 -- 172.21.15.64:0/2720129425 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f799405bcd0 con 0x7f79a00cea70 2024-03-04T07:36:49.922 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:49.922 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:49.922 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:49.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f798c06b500 msgr2=0x7f798c06d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:49.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f798c06b500 0x7f798c06d9c0 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7f7990036000 tx=0x7f7990008030 comp rx=0 tx=0).stop 2024-03-04T07:36:49.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79a00cea70 msgr2=0x7f79a00b48e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:49.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79a00cea70 0x7f79a00b48e0 secure :-1 s=READY pgs=1786 cs=0 l=1 rev1=1 crypto rx=0x7f799400c390 tx=0x7f799400c790 comp rx=0 tx=0).stop 2024-03-04T07:36:49.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 shutdown_connections 2024-03-04T07:36:49.924 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f798c06b500 0x7f798c06d9c0 unknown :-1 s=CLOSED pgs=813 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79a015bda0 0x7f79a00b4e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f79a00cea70 0x7f79a00b48e0 unknown :-1 s=CLOSED pgs=1786 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 --2- 172.21.15.64:0/2720129425 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f79a00b0ed0 0x7f79a014f4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:49.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 >> 172.21.15.64:0/2720129425 conn(0x7f79a00d1a90 msgr2=0x7f79a00b3bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:49.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 shutdown_connections 2024-03-04T07:36:49.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:49.924+0000 7f79a613e640 1 -- 172.21.15.64:0/2720129425 wait complete. 2024-03-04T07:36:50.136 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:50.136 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:51.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:51.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.251+0000 7f3b04f8a640 1 Processor -- start 2024-03-04T07:36:51.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.251+0000 7f3b04f8a640 1 -- start start 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3b04f8a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3b0015e030 0x7f3b0006ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3b04f8a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3b0006ef70 0x7f3b000b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3b04f8a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3b0015e410 con 0x7f3b000b2290 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3b04f8a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3b00162700 con 0x7f3b0015e030 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3b04f8a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3b00161af0 con 0x7f3b0006ef70 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3afe59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3b0015e030 0x7f3b0006ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3afe59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3b0015e030 0x7f3b0006ea30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60962/0 (socket says 172.21.15.64:60962) 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3afe59c640 1 -- 172.21.15.64:0/2514900062 learned_addr learned my addr 172.21.15.64:0/2514900062 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.252+0000 7f3afdd9b640 1 --2- 172.21.15.64:0/2514900062 >> v2:172.21.15.66:3300/0 conn(0x7f3b0006ef70 0x7f3b000b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.253+0000 7f3afe59c640 1 -- 172.21.15.64:0/2514900062 >> v2:172.21.15.66:3300/0 conn(0x7f3b0006ef70 msgr2=0x7f3b000b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.253+0000 7f3afe59c640 1 --2- 172.21.15.64:0/2514900062 >> v2:172.21.15.66:3300/0 conn(0x7f3b0006ef70 0x7f3b000b1d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.253+0000 7f3afe59c640 1 -- 172.21.15.64:0/2514900062 >> v1:172.21.15.66:6789/0 conn(0x7f3b000b2290 legacy=0x7f3b000d0040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:51.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.253+0000 7f3afe59c640 1 -- 172.21.15.64:0/2514900062 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b00161dd0 con 0x7f3b0015e030 2024-03-04T07:36:51.254 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.253+0000 7f3afd59a640 1 -- 172.21.15.64:0/2514900062 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1295617014 0 0) 0x7f3af8001110 con 0x7f3b000b2290 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.253+0000 7f3afe59c640 1 --2- 172.21.15.64:0/2514900062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b0015e030 0x7f3b0006ea30 secure :-1 s=READY pgs=1787 cs=0 l=1 rev1=1 crypto rx=0x7f3b000720a0 tx=0x7f3af002bf80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ddf1e471efd5d31a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3afd59a640 1 -- 172.21.15.64:0/2514900062 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3af002cb20 con 0x7f3b0015e030 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3afd59a640 1 -- 172.21.15.64:0/2514900062 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3af002ccc0 con 0x7f3b0015e030 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3afd59a640 1 -- 172.21.15.64:0/2514900062 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3af0034610 con 0x7f3b0015e030 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 -- 172.21.15.64:0/2514900062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b0015e030 msgr2=0x7f3b0006ea30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/2514900062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b0015e030 0x7f3b0006ea30 secure :-1 s=READY pgs=1787 cs=0 l=1 rev1=1 crypto rx=0x7f3b000720a0 tx=0x7f3af002bf80 comp rx=0 tx=0).stop 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 -- 172.21.15.64:0/2514900062 shutdown_connections 2024-03-04T07:36:51.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/2514900062 >> v2:172.21.15.66:3300/0 conn(0x7f3b0006ef70 0x7f3b000b1d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/2514900062 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b0015e030 0x7f3b0006ea30 unknown :-1 s=CLOSED pgs=1787 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 -- 172.21.15.64:0/2514900062 >> 172.21.15.64:0/2514900062 conn(0x7f3b000b55e0 msgr2=0x7f3b000b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:51.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 -- 172.21.15.64:0/2514900062 shutdown_connections 2024-03-04T07:36:51.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.254+0000 7f3b04f8a640 1 -- 172.21.15.64:0/2514900062 wait complete. 2024-03-04T07:36:51.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 Processor -- start 2024-03-04T07:36:51.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 -- start start 2024-03-04T07:36:51.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b0006ef70 0x7f3b0015c9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:51.258 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b000b2290 0x7f3b0009cb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b0015e030 0x7f3b0009d070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b0006e220 con 0x7f3b000b2290 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b00162390 con 0x7f3b0006ef70 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.256+0000 7f3b04f8a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b0006e3a0 con 0x7f3b0015e030 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b000b2290 0x7f3b0009cb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b000b2290 0x7f3b0009cb30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60974/0 (socket says 172.21.15.64:60974) 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 -- 172.21.15.64:0/717041830 learned_addr learned my addr 172.21.15.64:0/717041830 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afe59c640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b0006ef70 0x7f3b0015c9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afed9d640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b0015e030 0x7f3b0009d070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 -- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b0015e030 msgr2=0x7f3b0009d070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b0015e030 0x7f3b0009d070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 -- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b0006ef70 msgr2=0x7f3b0015c9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b0006ef70 0x7f3b0015c9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 -- 172.21.15.64:0/717041830 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b0015cf20 con 0x7f3b000b2290 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3afdd9b640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b000b2290 0x7f3b0009cb30 secure :-1 s=READY pgs=1788 cs=0 l=1 rev1=1 crypto rx=0x7f3ae800e9e0 tx=0x7f3ae800eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3af6ffd640 1 -- 172.21.15.64:0/717041830 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ae8017070 con 0x7f3b000b2290 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3af6ffd640 1 -- 172.21.15.64:0/717041830 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3ae800c9d0 con 0x7f3b000b2290 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b0009d8d0 con 0x7f3b000b2290 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3af6ffd640 1 -- 172.21.15.64:0/717041830 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ae800cdc0 con 0x7f3b000b2290 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.257+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b0015a540 con 0x7f3b000b2290 2024-03-04T07:36:51.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.258+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3b0009ddf0 con 0x7f3b000b2290 2024-03-04T07:36:51.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.259+0000 7f3af6ffd640 1 -- 172.21.15.64:0/717041830 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3ae801a070 con 0x7f3b000b2290 2024-03-04T07:36:51.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.259+0000 7f3af6ffd640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ae006b4d0 0x7f3ae006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:51.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.259+0000 7f3af6ffd640 1 -- 172.21.15.64:0/717041830 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3ae808a860 con 0x7f3b000b2290 2024-03-04T07:36:51.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.260+0000 7f3afe59c640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ae006b4d0 0x7f3ae006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:51.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.260+0000 7f3afe59c640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ae006b4d0 0x7f3ae006d990 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f3af0035000 tx=0x7f3af0030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:51.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.263+0000 7f3af6ffd640 1 -- 172.21.15.64:0/717041830 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3b0009ddf0 con 0x7f3b000b2290 2024-03-04T07:36:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.483+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3b00161490 con 0x7f3b000b2290 2024-03-04T07:36:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.484+0000 7f3af6ffd640 1 -- 172.21.15.64:0/717041830 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f3ae80591c0 con 0x7f3b000b2290 2024-03-04T07:36:51.484 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:51.484 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:51.484 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ae006b4d0 msgr2=0x7f3ae006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ae006b4d0 0x7f3ae006d990 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f3af0035000 tx=0x7f3af0030000 comp rx=0 tx=0).stop 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b000b2290 msgr2=0x7f3b0009cb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b000b2290 0x7f3b0009cb30 secure :-1 s=READY pgs=1788 cs=0 l=1 rev1=1 crypto rx=0x7f3ae800e9e0 tx=0x7f3ae800eea0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 shutdown_connections 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ae006b4d0 0x7f3ae006d990 unknown :-1 s=CLOSED pgs=814 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b0015e030 0x7f3b0009d070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.486 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b000b2290 0x7f3b0009cb30 unknown :-1 s=CLOSED pgs=1788 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 --2- 172.21.15.64:0/717041830 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b0006ef70 0x7f3b0015c9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:51.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 >> 172.21.15.64:0/717041830 conn(0x7f3b000b55e0 msgr2=0x7f3b001589e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:51.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 shutdown_connections 2024-03-04T07:36:51.487 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:51.486+0000 7f3b04f8a640 1 -- 172.21.15.64:0/717041830 wait complete. 2024-03-04T07:36:51.698 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:51.698 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:52.683 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:52.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.814+0000 7f619eff4640 1 Processor -- start 2024-03-04T07:36:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.814+0000 7f619eff4640 1 -- start start 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619eff4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f619806b5c0 0x7f61980b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619eff4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f61980ae9b0 0x7f61980d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619eff4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f61980b3340 con 0x7f61980b0e90 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619eff4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f61980cfa00 con 0x7f61980ae9b0 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619eff4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f61980cee90 con 0x7f619806b5c0 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619d591640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f61980ae9b0 0x7f61980d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619cd90640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f619806b5c0 0x7f61980b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f6197fff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f61980b0e90 0x7f619806b9a0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:60216/0 (socket says 172.21.15.64:60216) 2024-03-04T07:36:52.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f6197fff640 1 -- 172.21.15.64:0/1159872736 learned_addr learned my addr 172.21.15.64:0/1159872736 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:52.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619d591640 1 -- 172.21.15.64:0/1159872736 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f619806b5c0 msgr2=0x7f61980b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:52.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619d591640 1 --2- 172.21.15.64:0/1159872736 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f619806b5c0 0x7f61980b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:52.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619d591640 1 -- 172.21.15.64:0/1159872736 >> v1:172.21.15.66:6789/0 conn(0x7f61980b0e90 legacy=0x7f619806b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:52.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f61977fe640 1 -- 172.21.15.64:0/1159872736 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4125831246 0 0) 0x7f61980b3340 con 0x7f61980b0e90 2024-03-04T07:36:52.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.815+0000 7f619d591640 1 -- 172.21.15.64:0/1159872736 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61980cfc50 con 0x7f61980ae9b0 2024-03-04T07:36:52.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.816+0000 7f619d591640 1 --2- 172.21.15.64:0/1159872736 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 0x7f61980d03a0 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f61800097a0 tx=0x7f618002b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8d42f63e195f445d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:52.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.816+0000 7f61977fe640 1 -- 172.21.15.64:0/1159872736 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61800315a0 con 0x7f61980ae9b0 2024-03-04T07:36:52.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.816+0000 7f61977fe640 1 -- 172.21.15.64:0/1159872736 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6180037070 con 0x7f61980ae9b0 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.816+0000 7f61977fe640 1 -- 172.21.15.64:0/1159872736 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6180031960 con 0x7f61980ae9b0 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 -- 172.21.15.64:0/1159872736 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 msgr2=0x7f61980d03a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 --2- 172.21.15.64:0/1159872736 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 0x7f61980d03a0 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f61800097a0 tx=0x7f618002b840 comp rx=0 tx=0).stop 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 -- 172.21.15.64:0/1159872736 shutdown_connections 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 --2- 172.21.15.64:0/1159872736 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 0x7f61980d03a0 unknown :-1 s=CLOSED pgs=661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 --2- 172.21.15.64:0/1159872736 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f619806b5c0 0x7f61980b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 -- 172.21.15.64:0/1159872736 >> 172.21.15.64:0/1159872736 conn(0x7f61980b36d0 msgr2=0x7f61980b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 -- 172.21.15.64:0/1159872736 shutdown_connections 2024-03-04T07:36:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.817+0000 7f619eff4640 1 -- 172.21.15.64:0/1159872736 wait complete. 2024-03-04T07:36:52.820 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.818+0000 7f619eff4640 1 Processor -- start 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.818+0000 7f619eff4640 1 -- start start 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619eff4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f619806b5c0 0x7f61981e8890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619eff4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 0x7f61981f3780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619eff4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61980b0e90 0x7f61981e3b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619eff4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f619806c910 con 0x7f619806b5c0 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619eff4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f619806c790 con 0x7f61980ae9b0 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619eff4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f61980aee00 con 0x7f61980b0e90 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619cd90640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f619806b5c0 0x7f61981e8890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619cd90640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f619806b5c0 0x7f61981e8890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60986/0 (socket says 172.21.15.64:60986) 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619cd90640 1 -- 172.21.15.64:0/3250236124 learned_addr learned my addr 172.21.15.64:0/3250236124 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f6197fff640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 0x7f61981f3780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619d591640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61980b0e90 0x7f61981e3b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.819+0000 7f619cd90640 1 -- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61980b0e90 msgr2=0x7f61981e3b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f619cd90640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61980b0e90 0x7f61981e3b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f619cd90640 1 -- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 msgr2=0x7f61981f3780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f619cd90640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 0x7f61981f3780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f619cd90640 1 -- 172.21.15.64:0/3250236124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61981e77f0 con 0x7f619806b5c0 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f619cd90640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f619806b5c0 0x7f61981e8890 secure :-1 s=READY pgs=1789 cs=0 l=1 rev1=1 crypto rx=0x7f619000efc0 tx=0x7f619000c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f61957fa640 1 -- 172.21.15.64:0/3250236124 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6190018020 con 0x7f619806b5c0 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f61957fa640 1 -- 172.21.15.64:0/3250236124 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6190014480 con 0x7f619806b5c0 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61981e7ad0 con 0x7f619806b5c0 2024-03-04T07:36:52.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f61957fa640 1 -- 172.21.15.64:0/3250236124 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6190014850 con 0x7f619806b5c0 2024-03-04T07:36:52.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.820+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61981e8090 con 0x7f619806b5c0 2024-03-04T07:36:52.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.821+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6198068700 con 0x7f619806b5c0 2024-03-04T07:36:52.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.822+0000 7f61957fa640 1 -- 172.21.15.64:0/3250236124 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6190005d10 con 0x7f619806b5c0 2024-03-04T07:36:52.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.822+0000 7f61957fa640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f617806b4d0 0x7f617806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:52.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.822+0000 7f61957fa640 1 -- 172.21.15.64:0/3250236124 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f619008af50 con 0x7f619806b5c0 2024-03-04T07:36:52.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.822+0000 7f6197fff640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f617806b4d0 0x7f617806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:52.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.823+0000 7f6197fff640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f617806b4d0 0x7f617806d990 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f61980d11a0 tx=0x7f6188007d50 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:52.826 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:52.826+0000 7f61957fa640 1 -- 172.21.15.64:0/3250236124 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6190022030 con 0x7f619806b5c0 2024-03-04T07:36:53.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.048+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f61981e6290 con 0x7f619806b5c0 2024-03-04T07:36:53.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.049+0000 7f61957fa640 1 -- 172.21.15.64:0/3250236124 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f6190010040 con 0x7f619806b5c0 2024-03-04T07:36:53.049 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:53.049 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:53.049 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:53.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f617806b4d0 msgr2=0x7f617806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:53.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f617806b4d0 0x7f617806d990 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f61980d11a0 tx=0x7f6188007d50 comp rx=0 tx=0).stop 2024-03-04T07:36:53.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f619806b5c0 msgr2=0x7f61981e8890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:53.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f619806b5c0 0x7f61981e8890 secure :-1 s=READY pgs=1789 cs=0 l=1 rev1=1 crypto rx=0x7f619000efc0 tx=0x7f619000c490 comp rx=0 tx=0).stop 2024-03-04T07:36:53.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 shutdown_connections 2024-03-04T07:36:53.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f617806b4d0 0x7f617806d990 unknown :-1 s=CLOSED pgs=815 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:53.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f61980b0e90 0x7f61981e3b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:53.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f61980ae9b0 0x7f61981f3780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:53.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 --2- 172.21.15.64:0/3250236124 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f619806b5c0 0x7f61981e8890 unknown :-1 s=CLOSED pgs=1789 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:53.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 >> 172.21.15.64:0/3250236124 conn(0x7f61980b36d0 msgr2=0x7f61981e5060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:53.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 shutdown_connections 2024-03-04T07:36:53.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:53.051+0000 7f619eff4640 1 -- 172.21.15.64:0/3250236124 wait complete. 2024-03-04T07:36:53.263 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:53.264 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:54.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:54.381 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.381+0000 7f9dfd9a0640 1 Processor -- start 2024-03-04T07:36:54.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.381+0000 7f9dfd9a0640 1 -- start start 2024-03-04T07:36:54.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd9a0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f9df8067410 0x7f9df8161780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd9a0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9df806ef80 0x7f9df8157a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd9a0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9df80b5330 con 0x7f9df8162260 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd9a0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9df80b3eb0 con 0x7f9df806ef80 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd9a0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f9df80b5c00 con 0x7f9df8067410 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd19f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9df806ef80 0x7f9df8157a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd19f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f9df806ef80 0x7f9df8157a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60356/0 (socket says 172.21.15.64:60356) 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9dfd19f640 1 -- 172.21.15.64:0/4222700873 learned_addr learned my addr 172.21.15.64:0/4222700873 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.382+0000 7f9df7fff640 1 --2- 172.21.15.64:0/4222700873 >> v2:172.21.15.66:3300/0 conn(0x7f9df8067410 0x7f9df8161780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:54.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.383+0000 7f9dfd19f640 1 -- 172.21.15.64:0/4222700873 >> v2:172.21.15.66:3300/0 conn(0x7f9df8067410 msgr2=0x7f9df8161780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.383+0000 7f9dfd19f640 1 --2- 172.21.15.64:0/4222700873 >> v2:172.21.15.66:3300/0 conn(0x7f9df8067410 0x7f9df8161780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.383+0000 7f9dfd19f640 1 -- 172.21.15.64:0/4222700873 >> v1:172.21.15.66:6789/0 conn(0x7f9df8162260 legacy=0x7f9df8066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.383+0000 7f9dfd19f640 1 -- 172.21.15.64:0/4222700873 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9df80b3bd0 con 0x7f9df806ef80 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.383+0000 7f9df77fe640 1 -- 172.21.15.64:0/4222700873 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2178193885 0 0) 0x7f9df0001490 con 0x7f9df8162260 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.383+0000 7f9dfd19f640 1 --2- 172.21.15.64:0/4222700873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df806ef80 0x7f9df8157a60 secure :-1 s=READY pgs=1790 cs=0 l=1 rev1=1 crypto rx=0x7f9df806e970 tx=0x7f9de802b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71b0cad554e79f14 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9df77fe640 1 -- 172.21.15.64:0/4222700873 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9de8034070 con 0x7f9df806ef80 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9df77fe640 1 -- 172.21.15.64:0/4222700873 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9de8030860 con 0x7f9df806ef80 2024-03-04T07:36:54.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9df77fe640 1 -- 172.21.15.64:0/4222700873 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9de8030c30 con 0x7f9df806ef80 2024-03-04T07:36:54.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/4222700873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df806ef80 msgr2=0x7f9df8157a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:54.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/4222700873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df806ef80 0x7f9df8157a60 secure :-1 s=READY pgs=1790 cs=0 l=1 rev1=1 crypto rx=0x7f9df806e970 tx=0x7f9de802b9a0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/4222700873 shutdown_connections 2024-03-04T07:36:54.388 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/4222700873 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df806ef80 0x7f9df8157a60 unknown :-1 s=CLOSED pgs=1790 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/4222700873 >> v2:172.21.15.66:3300/0 conn(0x7f9df8067410 0x7f9df8161780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/4222700873 >> 172.21.15.64:0/4222700873 conn(0x7f9df80b2460 msgr2=0x7f9df80b2840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/4222700873 shutdown_connections 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.384+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/4222700873 wait complete. 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.385+0000 7f9dfd9a0640 1 Processor -- start 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.385+0000 7f9dfd9a0640 1 -- start start 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.386+0000 7f9dfd9a0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9df8067410 0x7f9df80b16d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.386+0000 7f9dfd9a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9df806ef80 0x7f9df8148e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.386+0000 7f9dfd9a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df8162260 0x7f9df81493b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.386+0000 7f9dfd9a0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9df8162640 con 0x7f9df8162260 2024-03-04T07:36:54.389 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.386+0000 7f9dfd9a0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9df80b5890 con 0x7f9df8067410 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.386+0000 7f9dfd9a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9df80affe0 con 0x7f9df806ef80 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df8162260 0x7f9df81493b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df8162260 0x7f9df81493b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60372/0 (socket says 172.21.15.64:60372) 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9df7fff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9df806ef80 0x7f9df8148e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 -- 172.21.15.64:0/2882638721 learned_addr learned my addr 172.21.15.64:0/2882638721 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfc99e640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9df8067410 0x7f9df80b16d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 -- 172.21.15.64:0/2882638721 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9df806ef80 msgr2=0x7f9df8148e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9df806ef80 0x7f9df8148e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 -- 172.21.15.64:0/2882638721 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9df8067410 msgr2=0x7f9df80b16d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9df8067410 0x7f9df80b16d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 -- 172.21.15.64:0/2882638721 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9df81498f0 con 0x7f9df8162260 2024-03-04T07:36:54.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd19f640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df8162260 0x7f9df81493b0 secure :-1 s=READY pgs=1791 cs=0 l=1 rev1=1 crypto rx=0x7f9de802b9e0 tx=0x7f9de8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:54.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9df57fa640 1 -- 172.21.15.64:0/2882638721 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9de8048070 con 0x7f9df8162260 2024-03-04T07:36:54.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9df57fa640 1 -- 172.21.15.64:0/2882638721 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9de8030660 con 0x7f9df8162260 2024-03-04T07:36:54.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9df8149e60 con 0x7f9df8162260 2024-03-04T07:36:54.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9df57fa640 1 -- 172.21.15.64:0/2882638721 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9de8034040 con 0x7f9df8162260 2024-03-04T07:36:54.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.387+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9df814a700 con 0x7f9df8162260 2024-03-04T07:36:54.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.388+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9df814a250 con 0x7f9df8162260 2024-03-04T07:36:54.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.389+0000 7f9df57fa640 1 -- 172.21.15.64:0/2882638721 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9de804b070 con 0x7f9df8162260 2024-03-04T07:36:54.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.389+0000 7f9df57fa640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9de406b4d0 0x7f9de406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:54.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.389+0000 7f9df57fa640 1 -- 172.21.15.64:0/2882638721 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f9de80ac240 con 0x7f9df8162260 2024-03-04T07:36:54.392 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.389+0000 7f9dfc99e640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9de406b4d0 0x7f9de406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:54.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.390+0000 7f9dfc99e640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9de406b4d0 0x7f9de406d990 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f9df00012e0 tx=0x7f9df0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:54.393 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.394+0000 7f9df57fa640 1 -- 172.21.15.64:0/2882638721 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9de8030830 con 0x7f9df8162260 2024-03-04T07:36:54.611 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.612+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9df804f330 con 0x7f9df8162260 2024-03-04T07:36:54.612 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.612+0000 7f9df57fa640 1 -- 172.21.15.64:0/2882638721 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f9de807ab50 con 0x7f9df8162260 2024-03-04T07:36:54.612 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:54.612 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:54.613 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:54.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9de406b4d0 msgr2=0x7f9de406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:54.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9de406b4d0 0x7f9de406d990 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f9df00012e0 tx=0x7f9df0009040 comp rx=0 tx=0).stop 2024-03-04T07:36:54.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df8162260 msgr2=0x7f9df81493b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:54.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df8162260 0x7f9df81493b0 secure :-1 s=READY pgs=1791 cs=0 l=1 rev1=1 crypto rx=0x7f9de802b9e0 tx=0x7f9de8004730 comp rx=0 tx=0).stop 2024-03-04T07:36:54.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 shutdown_connections 2024-03-04T07:36:54.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f9de406b4d0 0x7f9de406d990 unknown :-1 s=CLOSED pgs=816 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.614 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9df8162260 0x7f9df81493b0 unknown :-1 s=CLOSED pgs=1791 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9df806ef80 0x7f9df8148e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 --2- 172.21.15.64:0/2882638721 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9df8067410 0x7f9df80b16d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:54.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 >> 172.21.15.64:0/2882638721 conn(0x7f9df80b2460 msgr2=0x7f9df815d2a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:54.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 shutdown_connections 2024-03-04T07:36:54.615 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:54.614+0000 7f9dfd9a0640 1 -- 172.21.15.64:0/2882638721 wait complete. 2024-03-04T07:36:54.685 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:54.827 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:54.827 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:55.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:55.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.942+0000 7f08ad7dd640 1 Processor -- start 2024-03-04T07:36:55.942 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.942+0000 7f08ad7dd640 1 -- start start 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.942+0000 7f08ad7dd640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f08a806df30 0x7f08a80ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.942+0000 7f08ad7dd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a80b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.942+0000 7f08ad7dd640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f08a8071240 con 0x7f08a8158520 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.942+0000 7f08ad7dd640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f08a80af0f0 con 0x7f08a806df30 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.942+0000 7f08ad7dd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f08a8071b70 con 0x7f08a80aec40 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a6ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f08a8158520 0x7f08a806d820 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:53638/0 (socket says 172.21.15.64:53638) 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a6ffd640 1 -- 172.21.15.64:0/3405033239 learned_addr learned my addr 172.21.15.64:0/3405033239 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a77fe640 1 --2- 172.21.15.64:0/3405033239 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a80b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a67fc640 1 --2- 172.21.15.64:0/3405033239 >> v2:172.21.15.66:3300/0 conn(0x7f08a806df30 0x7f08a80ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a5ffb640 1 -- 172.21.15.64:0/3405033239 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 376478308 0 0) 0x7f08a8071240 con 0x7f08a8158520 2024-03-04T07:36:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a77fe640 1 -- 172.21.15.64:0/3405033239 >> v2:172.21.15.66:3300/0 conn(0x7f08a806df30 msgr2=0x7f08a80ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a77fe640 1 --2- 172.21.15.64:0/3405033239 >> v2:172.21.15.66:3300/0 conn(0x7f08a806df30 0x7f08a80ae700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a77fe640 1 -- 172.21.15.64:0/3405033239 >> v1:172.21.15.64:6789/0 conn(0x7f08a8158520 legacy=0x7f08a806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a77fe640 1 -- 172.21.15.64:0/3405033239 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08a8071520 con 0x7f08a80aec40 2024-03-04T07:36:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.943+0000 7f08a67fc640 1 --2- 172.21.15.64:0/3405033239 >> v2:172.21.15.66:3300/0 conn(0x7f08a806df30 0x7f08a80ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.944+0000 7f08a77fe640 1 --2- 172.21.15.64:0/3405033239 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a80b3640 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f08a0009d90 tx=0x7f08a002bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d9de01b1950dc1c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.944+0000 7f08a5ffb640 1 -- 172.21.15.64:0/3405033239 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08a0039070 con 0x7f08a80aec40 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.944+0000 7f08a5ffb640 1 -- 172.21.15.64:0/3405033239 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08a002cc50 con 0x7f08a80aec40 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.944+0000 7f08a5ffb640 1 -- 172.21.15.64:0/3405033239 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08a0034440 con 0x7f08a80aec40 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.944+0000 7f08ad7dd640 1 -- 172.21.15.64:0/3405033239 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 msgr2=0x7f08a80b3640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.944+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/3405033239 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a80b3640 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f08a0009d90 tx=0x7f08a002bea0 comp rx=0 tx=0).stop 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.945+0000 7f08ad7dd640 1 -- 172.21.15.64:0/3405033239 shutdown_connections 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.945+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/3405033239 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a80b3640 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.945+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/3405033239 >> v2:172.21.15.66:3300/0 conn(0x7f08a806df30 0x7f08a80ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.945+0000 7f08ad7dd640 1 -- 172.21.15.64:0/3405033239 >> 172.21.15.64:0/3405033239 conn(0x7f08a8066d00 msgr2=0x7f08a80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.945+0000 7f08ad7dd640 1 -- 172.21.15.64:0/3405033239 shutdown_connections 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.945+0000 7f08ad7dd640 1 -- 172.21.15.64:0/3405033239 wait complete. 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.946+0000 7f08ad7dd640 1 Processor -- start 2024-03-04T07:36:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.946+0000 7f08ad7dd640 1 -- start start 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08ad7dd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08a806df30 0x7f08a81477e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08ad7dd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a8145d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08ad7dd640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08a8158520 0x7f08a80ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08ad7dd640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08a8158900 con 0x7f08a806df30 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08ad7dd640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08a8072000 con 0x7f08a8158520 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08ad7dd640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f08a8158a80 con 0x7f08a80aec40 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08a806df30 0x7f08a81477e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08a806df30 0x7f08a81477e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60376/0 (socket says 172.21.15.64:60376) 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 -- 172.21.15.64:0/2248698809 learned_addr learned my addr 172.21.15.64:0/2248698809 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a77fe640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08a8158520 0x7f08a80ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a67fc640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a8145d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 -- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 msgr2=0x7f08a8145d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a8145d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 -- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08a8158520 msgr2=0x7f08a80ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08a8158520 0x7f08a80ad8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 -- 172.21.15.64:0/2248698809 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08a8146740 con 0x7f08a806df30 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a77fe640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08a8158520 0x7f08a80ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.947+0000 7f08a6ffd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08a806df30 0x7f08a81477e0 secure :-1 s=READY pgs=1793 cs=0 l=1 rev1=1 crypto rx=0x7f089800c430 tx=0x7f089800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.948+0000 7f089f7fe640 1 -- 172.21.15.64:0/2248698809 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08980074e0 con 0x7f08a806df30 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.948+0000 7f089f7fe640 1 -- 172.21.15.64:0/2248698809 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f08980047e0 con 0x7f08a806df30 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.948+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08a8146a20 con 0x7f08a806df30 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.948+0000 7f089f7fe640 1 -- 172.21.15.64:0/2248698809 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0898004bd0 con 0x7f08a806df30 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.948+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08a8146fe0 con 0x7f08a806df30 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.949+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f08a8067ff0 con 0x7f08a806df30 2024-03-04T07:36:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.949+0000 7f089f7fe640 1 -- 172.21.15.64:0/2248698809 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0898011070 con 0x7f08a806df30 2024-03-04T07:36:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.949+0000 7f089f7fe640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f088806b4d0 0x7f088806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.950+0000 7f089f7fe640 1 -- 172.21.15.64:0/2248698809 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0898089d50 con 0x7f08a806df30 2024-03-04T07:36:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.950+0000 7f08a67fc640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f088806b4d0 0x7f088806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.950+0000 7f08a67fc640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f088806b4d0 0x7f088806d990 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7f0890009560 tx=0x7f0890009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:55.954 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:55.954+0000 7f089f7fe640 1 -- 172.21.15.64:0/2248698809 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f089808d030 con 0x7f08a806df30 2024-03-04T07:36:56.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.175+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f08a80b31f0 con 0x7f08a806df30 2024-03-04T07:36:56.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.176+0000 7f089f7fe640 1 -- 172.21.15.64:0/2248698809 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f0898058540 con 0x7f08a806df30 2024-03-04T07:36:56.176 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:56.176 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:56.176 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:56.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f088806b4d0 msgr2=0x7f088806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:56.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f088806b4d0 0x7f088806d990 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7f0890009560 tx=0x7f0890009190 comp rx=0 tx=0).stop 2024-03-04T07:36:56.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08a806df30 msgr2=0x7f08a81477e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:56.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08a806df30 0x7f08a81477e0 secure :-1 s=READY pgs=1793 cs=0 l=1 rev1=1 crypto rx=0x7f089800c430 tx=0x7f089800c8f0 comp rx=0 tx=0).stop 2024-03-04T07:36:56.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 shutdown_connections 2024-03-04T07:36:56.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f088806b4d0 0x7f088806d990 unknown :-1 s=CLOSED pgs=817 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:56.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f08a8158520 0x7f08a80ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:56.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f08a80aec40 0x7f08a8145d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:56.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 --2- 172.21.15.64:0/2248698809 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f08a806df30 0x7f08a81477e0 unknown :-1 s=CLOSED pgs=1793 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:56.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 >> 172.21.15.64:0/2248698809 conn(0x7f08a8066d00 msgr2=0x7f08a80b3640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:56.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 shutdown_connections 2024-03-04T07:36:56.179 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:56.178+0000 7f08ad7dd640 1 -- 172.21.15.64:0/2248698809 wait complete. 2024-03-04T07:36:56.390 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:56.390 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:56.687 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:57.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:57.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.506+0000 7f6bacac9640 1 Processor -- start 2024-03-04T07:36:57.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.506+0000 7f6bacac9640 1 -- start start 2024-03-04T07:36:57.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6bacac9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba815e030 0x7f6ba806ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:57.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6bacac9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ba806ef70 0x7f6ba80b1d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:57.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6bacac9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6ba80b2290 0x7f6ba80d0040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:57.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6bacac9640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ba8162700 con 0x7f6ba806ef70 2024-03-04T07:36:57.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6bacac9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ba815e410 con 0x7f6ba80b2290 2024-03-04T07:36:57.508 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6bacac9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ba8161af0 con 0x7f6ba815e030 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ba806ef70 0x7f6ba80b1d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ba806ef70 0x7f6ba80b1d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60386/0 (socket says 172.21.15.64:60386) 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 -- 172.21.15.64:0/1137398648 learned_addr learned my addr 172.21.15.64:0/1137398648 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6ba6d9d640 1 --2- 172.21.15.64:0/1137398648 >> v2:172.21.15.66:3300/0 conn(0x7f6ba80b2290 0x7f6ba80d0040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6ba659c640 1 --2- 172.21.15.64:0/1137398648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba815e030 0x7f6ba806ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 -- 172.21.15.64:0/1137398648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba815e030 msgr2=0x7f6ba806ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 --2- 172.21.15.64:0/1137398648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba815e030 0x7f6ba806ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 -- 172.21.15.64:0/1137398648 >> v2:172.21.15.66:3300/0 conn(0x7f6ba80b2290 msgr2=0x7f6ba80d0040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 --2- 172.21.15.64:0/1137398648 >> v2:172.21.15.66:3300/0 conn(0x7f6ba80b2290 0x7f6ba80d0040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6b9ffff640 1 -- 172.21.15.64:0/1137398648 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ba8161dd0 con 0x7f6ba806ef70 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.507+0000 7f6ba6d9d640 1 --2- 172.21.15.64:0/1137398648 >> v2:172.21.15.66:3300/0 conn(0x7f6ba80b2290 0x7f6ba80d0040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:36:57.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.508+0000 7f6b9ffff640 1 --2- 172.21.15.64:0/1137398648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80b1d50 secure :-1 s=READY pgs=1794 cs=0 l=1 rev1=1 crypto rx=0x7f6ba806cee0 tx=0x7f6b9002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=140c199b7c9a4faa server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:57.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.508+0000 7f6ba5d9b640 1 -- 172.21.15.64:0/1137398648 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b90035020 con 0x7f6ba806ef70 2024-03-04T07:36:57.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.508+0000 7f6ba5d9b640 1 -- 172.21.15.64:0/1137398648 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b90031870 con 0x7f6ba806ef70 2024-03-04T07:36:57.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.508+0000 7f6ba5d9b640 1 -- 172.21.15.64:0/1137398648 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b90031c40 con 0x7f6ba806ef70 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 -- 172.21.15.64:0/1137398648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 msgr2=0x7f6ba80b1d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1137398648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80b1d50 secure :-1 s=READY pgs=1794 cs=0 l=1 rev1=1 crypto rx=0x7f6ba806cee0 tx=0x7f6b9002b910 comp rx=0 tx=0).stop 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 -- 172.21.15.64:0/1137398648 shutdown_connections 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1137398648 >> v2:172.21.15.66:3300/0 conn(0x7f6ba80b2290 0x7f6ba80d0040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1137398648 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80b1d50 unknown :-1 s=CLOSED pgs=1794 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1137398648 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba815e030 0x7f6ba806ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 -- 172.21.15.64:0/1137398648 >> 172.21.15.64:0/1137398648 conn(0x7f6ba80b55e0 msgr2=0x7f6ba80b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 -- 172.21.15.64:0/1137398648 shutdown_connections 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.509+0000 7f6bacac9640 1 -- 172.21.15.64:0/1137398648 wait complete. 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.510+0000 7f6bacac9640 1 Processor -- start 2024-03-04T07:36:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.510+0000 7f6bacac9640 1 -- start start 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6bacac9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80adf40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6bacac9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba80b2290 0x7f6ba80b0a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6bacac9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ba815e030 0x7f6ba80b0120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6bacac9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ba80b31e0 con 0x7f6ba806ef70 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6bacac9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ba8162390 con 0x7f6ba815e030 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6bacac9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ba80b3360 con 0x7f6ba80b2290 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80adf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80adf40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60396/0 (socket says 172.21.15.64:60396) 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 -- 172.21.15.64:0/1282043296 learned_addr learned my addr 172.21.15.64:0/1282043296 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba6d9d640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ba815e030 0x7f6ba80b0120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6b9ffff640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba80b2290 0x7f6ba80b0a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 -- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba80b2290 msgr2=0x7f6ba80b0a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba80b2290 0x7f6ba80b0a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 -- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ba815e030 msgr2=0x7f6ba80b0120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ba815e030 0x7f6ba80b0120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.511+0000 7f6ba659c640 1 -- 172.21.15.64:0/1282043296 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ba8145ba0 con 0x7f6ba806ef70 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.512+0000 7f6ba659c640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80adf40 secure :-1 s=READY pgs=1795 cs=0 l=1 rev1=1 crypto rx=0x7f6b9800e9b0 tx=0x7f6b9800edb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.512+0000 7f6b9effd640 1 -- 172.21.15.64:0/1282043296 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b98017070 con 0x7f6ba806ef70 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.512+0000 7f6b9effd640 1 -- 172.21.15.64:0/1282043296 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6b9800c940 con 0x7f6ba806ef70 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.512+0000 7f6b9effd640 1 -- 172.21.15.64:0/1282043296 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6b9800cd30 con 0x7f6ba806ef70 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.512+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ba8145e40 con 0x7f6ba806ef70 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.512+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ba80b0660 con 0x7f6ba806ef70 2024-03-04T07:36:57.515 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.513+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ba8067ff0 con 0x7f6ba806ef70 2024-03-04T07:36:57.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.513+0000 7f6b9effd640 1 -- 172.21.15.64:0/1282043296 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6b9801a070 con 0x7f6ba806ef70 2024-03-04T07:36:57.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.514+0000 7f6b9effd640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:57.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.514+0000 7f6b9effd640 1 -- 172.21.15.64:0/1282043296 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6b9808a380 con 0x7f6ba806ef70 2024-03-04T07:36:57.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.514+0000 7f6b9ffff640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:57.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.514+0000 7f6b9ffff640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7f6b90036000 tx=0x7f6b9003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:57.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.518+0000 7f6b9effd640 1 -- 172.21.15.64:0/1282043296 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6b98058c00 con 0x7f6ba806ef70 2024-03-04T07:36:57.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.736+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6ba80b2910 con 0x7f6ba806ef70 2024-03-04T07:36:57.736 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.736+0000 7f6b9effd640 1 -- 172.21.15.64:0/1282043296 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f6b9805cca0 con 0x7f6ba806ef70 2024-03-04T07:36:57.737 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:57.737 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:57.737 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:57.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 msgr2=0x7f6b8806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:57.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7f6b90036000 tx=0x7f6b9003c040 comp rx=0 tx=0).stop 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 msgr2=0x7f6ba80adf40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80adf40 secure :-1 s=READY pgs=1795 cs=0 l=1 rev1=1 crypto rx=0x7f6b9800e9b0 tx=0x7f6b9800edb0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 shutdown_connections 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6b8806b500 0x7f6b8806d9c0 unknown :-1 s=CLOSED pgs=818 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ba815e030 0x7f6ba80b0120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ba80b2290 0x7f6ba80b0a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 --2- 172.21.15.64:0/1282043296 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ba806ef70 0x7f6ba80adf40 unknown :-1 s=CLOSED pgs=1795 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 >> 172.21.15.64:0/1282043296 conn(0x7f6ba80b55e0 msgr2=0x7f6ba806dfe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 shutdown_connections 2024-03-04T07:36:57.739 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:57.738+0000 7f6bacac9640 1 -- 172.21.15.64:0/1282043296 wait complete. 2024-03-04T07:36:57.950 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:57.951 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:36:58.689 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:36:58.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:36:59.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.065+0000 7fddd89d7640 1 Processor -- start 2024-03-04T07:36:59.065 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.065+0000 7fddd89d7640 1 -- start start 2024-03-04T07:36:59.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.066+0000 7fddd89d7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fddd40d0c00 0x7fddd40d1050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:59.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.066+0000 7fddd89d7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fddd40cece0 con 0x7fddd40ce2d0 2024-03-04T07:36:59.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.066+0000 7fddd89d7640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fddd40ce760 con 0x7fddd4162d70 2024-03-04T07:36:59.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.066+0000 7fddd89d7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fddd40d1590 con 0x7fddd40d0c00 2024-03-04T07:36:59.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd1d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fddd40ce2d0 0x7fddd40d24b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:53642/0 (socket says 172.21.15.64:53642) 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd1d9b640 1 -- 172.21.15.64:0/2097774099 learned_addr learned my addr 172.21.15.64:0/2097774099 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd2d9d640 1 --2- 172.21.15.64:0/2097774099 >> v2:172.21.15.66:3300/0 conn(0x7fddd40d0c00 0x7fddd40d1050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd159a640 1 -- 172.21.15.64:0/2097774099 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1154354487 0 0) 0x7fddbc001110 con 0x7fddd40ce2d0 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd159a640 1 -- 172.21.15.64:0/2097774099 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fddc0003520 con 0x7fddd40ce2d0 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd159a640 1 -- 172.21.15.64:0/2097774099 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3663113646 0 0) 0x7fddd40ce760 con 0x7fddd4162d70 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd159a640 1 -- 172.21.15.64:0/2097774099 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fddbc001110 con 0x7fddd4162d70 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd2d9d640 1 -- 172.21.15.64:0/2097774099 >> v1:172.21.15.66:6789/0 conn(0x7fddd4162d70 legacy=0x7fddd4163150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd2d9d640 1 -- 172.21.15.64:0/2097774099 >> v1:172.21.15.64:6789/0 conn(0x7fddd40ce2d0 legacy=0x7fddd40d24b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:59.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.067+0000 7fddd2d9d640 1 -- 172.21.15.64:0/2097774099 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fddd40cea00 con 0x7fddd40d0c00 2024-03-04T07:36:59.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.068+0000 7fddd2d9d640 1 --2- 172.21.15.64:0/2097774099 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40d0c00 0x7fddd40d1050 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7fddc4009d90 tx=0x7fddc402beb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e8985ef934a964b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:59.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.068+0000 7fddd159a640 1 -- 172.21.15.64:0/2097774099 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fddc4039070 con 0x7fddd40d0c00 2024-03-04T07:36:59.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.068+0000 7fddd159a640 1 -- 172.21.15.64:0/2097774099 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fddc402cc60 con 0x7fddd40d0c00 2024-03-04T07:36:59.069 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.068+0000 7fddd159a640 1 -- 172.21.15.64:0/2097774099 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fddc4034640 con 0x7fddd40d0c00 2024-03-04T07:36:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.068+0000 7fddd89d7640 1 -- 172.21.15.64:0/2097774099 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40d0c00 msgr2=0x7fddd40d1050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.068+0000 7fddd89d7640 1 --2- 172.21.15.64:0/2097774099 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40d0c00 0x7fddd40d1050 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7fddc4009d90 tx=0x7fddc402beb0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.069+0000 7fddd89d7640 1 -- 172.21.15.64:0/2097774099 shutdown_connections 2024-03-04T07:36:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.069+0000 7fddd89d7640 1 --2- 172.21.15.64:0/2097774099 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40d0c00 0x7fddd40d1050 unknown :-1 s=CLOSED pgs=664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.069+0000 7fddd89d7640 1 -- 172.21.15.64:0/2097774099 >> 172.21.15.64:0/2097774099 conn(0x7fddd4071960 msgr2=0x7fddd4071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.069+0000 7fddd89d7640 1 -- 172.21.15.64:0/2097774099 shutdown_connections 2024-03-04T07:36:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.069+0000 7fddd89d7640 1 -- 172.21.15.64:0/2097774099 wait complete. 2024-03-04T07:36:59.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.070+0000 7fddd89d7640 1 Processor -- start 2024-03-04T07:36:59.072 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.070+0000 7fddd89d7640 1 -- start start 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd89d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40ce2d0 0x7fddd415a6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd89d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fddd40d0c00 0x7fddd40b8590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd89d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fddd4162d70 0x7fddd40b8ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd89d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fddd40d3710 con 0x7fddd4162d70 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd89d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fddd40d1cd0 con 0x7fddd40ce2d0 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd89d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fddd40d3890 con 0x7fddd40d0c00 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fddd4162d70 0x7fddd40b8ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fddd4162d70 0x7fddd40b8ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60400/0 (socket says 172.21.15.64:60400) 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 -- 172.21.15.64:0/979424100 learned_addr learned my addr 172.21.15.64:0/979424100 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd259c640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40ce2d0 0x7fddd415a6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd1d9b640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fddd40d0c00 0x7fddd40b8590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 -- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fddd40d0c00 msgr2=0x7fddd40b8590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fddd40d0c00 0x7fddd40b8590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 -- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40ce2d0 msgr2=0x7fddd415a6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40ce2d0 0x7fddd415a6f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 -- 172.21.15.64:0/979424100 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fddd41f7db0 con 0x7fddd4162d70 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.071+0000 7fddd2d9d640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fddd4162d70 0x7fddd40b8ad0 secure :-1 s=READY pgs=1797 cs=0 l=1 rev1=1 crypto rx=0x7fddc4009d90 tx=0x7fddc4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.072+0000 7fddcaffd640 1 -- 172.21.15.64:0/979424100 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fddc4039050 con 0x7fddd4162d70 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.072+0000 7fddcaffd640 1 -- 172.21.15.64:0/979424100 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fddc4045070 con 0x7fddd4162d70 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.072+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fddd415b0d0 con 0x7fddd4162d70 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.072+0000 7fddcaffd640 1 -- 172.21.15.64:0/979424100 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fddc4009450 con 0x7fddd4162d70 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.072+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fddd41f8350 con 0x7fddd4162d70 2024-03-04T07:36:59.074 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.073+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fddd4067ff0 con 0x7fddd4162d70 2024-03-04T07:36:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.073+0000 7fddcaffd640 1 -- 172.21.15.64:0/979424100 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fddc403e830 con 0x7fddd4162d70 2024-03-04T07:36:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.073+0000 7fddcaffd640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fddb406b4d0 0x7fddb406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:36:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.074+0000 7fddcaffd640 1 -- 172.21.15.64:0/979424100 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fddc40acee0 con 0x7fddd4162d70 2024-03-04T07:36:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.074+0000 7fddd259c640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fddb406b4d0 0x7fddb406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:36:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.074+0000 7fddd259c640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fddb406b4d0 0x7fddb406d990 secure :-1 s=READY pgs=819 cs=0 l=1 rev1=1 crypto rx=0x7fddcc007fd0 tx=0x7fddcc009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:36:59.078 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.078+0000 7fddcaffd640 1 -- 172.21.15.64:0/979424100 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fddc407b940 con 0x7fddd4162d70 2024-03-04T07:36:59.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.296+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fddd41f86c0 con 0x7fddd4162d70 2024-03-04T07:36:59.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.296+0000 7fddcaffd640 1 -- 172.21.15.64:0/979424100 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fddc407f9e0 con 0x7fddd4162d70 2024-03-04T07:36:59.296 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:36:59.297 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:36:59.297 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:36:59.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fddb406b4d0 msgr2=0x7fddb406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:59.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fddb406b4d0 0x7fddb406d990 secure :-1 s=READY pgs=819 cs=0 l=1 rev1=1 crypto rx=0x7fddcc007fd0 tx=0x7fddcc009040 comp rx=0 tx=0).stop 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fddd4162d70 msgr2=0x7fddd40b8ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fddd4162d70 0x7fddd40b8ad0 secure :-1 s=READY pgs=1797 cs=0 l=1 rev1=1 crypto rx=0x7fddc4009d90 tx=0x7fddc4004a40 comp rx=0 tx=0).stop 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 shutdown_connections 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fddb406b4d0 0x7fddb406d990 unknown :-1 s=CLOSED pgs=819 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fddd4162d70 0x7fddd40b8ad0 unknown :-1 s=CLOSED pgs=1797 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fddd40d0c00 0x7fddd40b8590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 --2- 172.21.15.64:0/979424100 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fddd40ce2d0 0x7fddd415a6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 >> 172.21.15.64:0/979424100 conn(0x7fddd4071960 msgr2=0x7fddd40b4640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 shutdown_connections 2024-03-04T07:36:59.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:36:59.298+0000 7fddd89d7640 1 -- 172.21.15.64:0/979424100 wait complete. 2024-03-04T07:36:59.510 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:36:59.510 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:00.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:00.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.626+0000 7f4f22b9a640 1 Processor -- start 2024-03-04T07:37:00.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.626+0000 7f4f22b9a640 1 -- start start 2024-03-04T07:37:00.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f22b9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c15e060 0x7f4f1c0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:00.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f22b9a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f1c06fe20 0x7f4f1c0ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:00.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f22b9a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f4f1c0d0dd0 con 0x7f4f1c0d0880 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f22b9a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f4f1c0d10b0 con 0x7f4f1c06fe20 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f22b9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f1c0cc0e0 con 0x7f4f1c15e060 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f1bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f1c06fe20 0x7f4f1c0ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f20936640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c15e060 0x7f4f1c0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f21137640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f4f1c0d0880 0x7f4f1c06ed30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:55706/0 (socket says 172.21.15.64:55706) 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f1bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f4f1c06fe20 0x7f4f1c0ce850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60408/0 (socket says 172.21.15.64:60408) 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f21137640 1 -- 172.21.15.64:0/882658173 learned_addr learned my addr 172.21.15.64:0/882658173 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f1b7fe640 1 -- 172.21.15.64:0/882658173 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1874338123 0 0) 0x7f4f1c0d0dd0 con 0x7f4f1c0d0880 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f1b7fe640 1 -- 172.21.15.64:0/882658173 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f4f0c003520 con 0x7f4f1c0d0880 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f20936640 1 -- 172.21.15.64:0/882658173 >> v2:172.21.15.64:3300/0 conn(0x7f4f1c06fe20 msgr2=0x7f4f1c0ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f20936640 1 --2- 172.21.15.64:0/882658173 >> v2:172.21.15.64:3300/0 conn(0x7f4f1c06fe20 0x7f4f1c0ce850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f20936640 1 -- 172.21.15.64:0/882658173 >> v1:172.21.15.66:6789/0 conn(0x7f4f1c0d0880 legacy=0x7f4f1c06ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:00.628 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f20936640 1 -- 172.21.15.64:0/882658173 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f1c0cc3c0 con 0x7f4f1c15e060 2024-03-04T07:37:00.629 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.627+0000 7f4f1bfff640 1 --2- 172.21.15.64:0/882658173 >> v2:172.21.15.64:3300/0 conn(0x7f4f1c06fe20 0x7f4f1c0ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:00.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.628+0000 7f4f20936640 1 --2- 172.21.15.64:0/882658173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c15e060 0x7f4f1c0ce310 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f4f100097a0 tx=0x7f4f1002b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4eb56c2f96ceeb90 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:00.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.628+0000 7f4f1b7fe640 1 -- 172.21.15.64:0/882658173 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f10035020 con 0x7f4f1c15e060 2024-03-04T07:37:00.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.628+0000 7f4f1b7fe640 1 -- 172.21.15.64:0/882658173 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f10031830 con 0x7f4f1c15e060 2024-03-04T07:37:00.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.628+0000 7f4f1b7fe640 1 -- 172.21.15.64:0/882658173 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f10031be0 con 0x7f4f1c15e060 2024-03-04T07:37:00.630 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 -- 172.21.15.64:0/882658173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c15e060 msgr2=0x7f4f1c0ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/882658173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c15e060 0x7f4f1c0ce310 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f4f100097a0 tx=0x7f4f1002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 -- 172.21.15.64:0/882658173 shutdown_connections 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/882658173 >> v2:172.21.15.64:3300/0 conn(0x7f4f1c06fe20 0x7f4f1c0ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/882658173 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c15e060 0x7f4f1c0ce310 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 -- 172.21.15.64:0/882658173 >> 172.21.15.64:0/882658173 conn(0x7f4f1c0d1a90 msgr2=0x7f4f1c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 -- 172.21.15.64:0/882658173 shutdown_connections 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.629+0000 7f4f22b9a640 1 -- 172.21.15.64:0/882658173 wait complete. 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.630+0000 7f4f22b9a640 1 Processor -- start 2024-03-04T07:37:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.630+0000 7f4f22b9a640 1 -- start start 2024-03-04T07:37:00.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f22b9a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f1c06fe20 0x7f4f1c152f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:00.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f22b9a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c0d0880 0x7f4f1c1f3c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:00.633 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f22b9a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f1c15e060 0x7f4f1c1f41c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f22b9a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f1c0cdbd0 con 0x7f4f1c06fe20 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f22b9a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f1c15e440 con 0x7f4f1c15e060 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f22b9a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4f1c0cdd50 con 0x7f4f1c0d0880 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f20936640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f1c06fe20 0x7f4f1c152f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f20936640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f1c06fe20 0x7f4f1c152f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60424/0 (socket says 172.21.15.64:60424) 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f20936640 1 -- 172.21.15.64:0/3465131073 learned_addr learned my addr 172.21.15.64:0/3465131073 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f1bfff640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c0d0880 0x7f4f1c1f3c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.631+0000 7f4f21137640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f1c15e060 0x7f4f1c1f41c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f20936640 1 -- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c0d0880 msgr2=0x7f4f1c1f3c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f20936640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c0d0880 0x7f4f1c1f3c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f20936640 1 -- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f1c15e060 msgr2=0x7f4f1c1f41c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f20936640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f1c15e060 0x7f4f1c1f41c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f20936640 1 -- 172.21.15.64:0/3465131073 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f1c0b4b80 con 0x7f4f1c06fe20 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f20936640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f1c06fe20 0x7f4f1c152f40 secure :-1 s=READY pgs=1798 cs=0 l=1 rev1=1 crypto rx=0x7f4f1002b910 tx=0x7f4f100046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f197fa640 1 -- 172.21.15.64:0/3465131073 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f10035040 con 0x7f4f1c06fe20 2024-03-04T07:37:00.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f197fa640 1 -- 172.21.15.64:0/3465131073 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f4f100044d0 con 0x7f4f1c06fe20 2024-03-04T07:37:00.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f1c1f47f0 con 0x7f4f1c06fe20 2024-03-04T07:37:00.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f197fa640 1 -- 172.21.15.64:0/3465131073 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f10003850 con 0x7f4f1c06fe20 2024-03-04T07:37:00.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.632+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f1c15f510 con 0x7f4f1c06fe20 2024-03-04T07:37:00.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.633+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f1c067ff0 con 0x7f4f1c06fe20 2024-03-04T07:37:00.635 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.634+0000 7f4f197fa640 1 -- 172.21.15.64:0/3465131073 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f4f10038020 con 0x7f4f1c06fe20 2024-03-04T07:37:00.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.634+0000 7f4f197fa640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f0006b4d0 0x7f4f0006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:00.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.634+0000 7f4f197fa640 1 -- 172.21.15.64:0/3465131073 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f4f100ac2e0 con 0x7f4f1c06fe20 2024-03-04T07:37:00.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.634+0000 7f4f1bfff640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f0006b4d0 0x7f4f0006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:00.636 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.635+0000 7f4f1bfff640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f0006b4d0 0x7f4f0006d990 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7f4f08009560 tx=0x7f4f08009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:00.638 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.638+0000 7f4f197fa640 1 -- 172.21.15.64:0/3465131073 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f4f1007aad0 con 0x7f4f1c06fe20 2024-03-04T07:37:00.691 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:00.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.858+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4f1c0ce330 con 0x7f4f1c06fe20 2024-03-04T07:37:00.858 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.858+0000 7f4f197fa640 1 -- 172.21.15.64:0/3465131073 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f4f1007eb70 con 0x7f4f1c06fe20 2024-03-04T07:37:00.858 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:00.858 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:00.859 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:00.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f0006b4d0 msgr2=0x7f4f0006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:00.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f0006b4d0 0x7f4f0006d990 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7f4f08009560 tx=0x7f4f08009190 comp rx=0 tx=0).stop 2024-03-04T07:37:00.860 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f1c06fe20 msgr2=0x7f4f1c152f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f1c06fe20 0x7f4f1c152f40 secure :-1 s=READY pgs=1798 cs=0 l=1 rev1=1 crypto rx=0x7f4f1002b910 tx=0x7f4f100046c0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 shutdown_connections 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f4f0006b4d0 0x7f4f0006d990 unknown :-1 s=CLOSED pgs=820 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f4f1c15e060 0x7f4f1c1f41c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f4f1c0d0880 0x7f4f1c1f3c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 --2- 172.21.15.64:0/3465131073 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f4f1c06fe20 0x7f4f1c152f40 unknown :-1 s=CLOSED pgs=1798 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 >> 172.21.15.64:0/3465131073 conn(0x7f4f1c0d1a90 msgr2=0x7f4f1c15d070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 shutdown_connections 2024-03-04T07:37:00.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:00.860+0000 7f4f22b9a640 1 -- 172.21.15.64:0/3465131073 wait complete. 2024-03-04T07:37:01.072 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:01.072 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:02.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.187+0000 7fbceacdf640 1 Processor -- start 2024-03-04T07:37:02.187 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.187+0000 7fbceacdf640 1 -- start start 2024-03-04T07:37:02.188 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.188+0000 7fbceacdf640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbce415e060 0x7fbce40ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.188+0000 7fbceacdf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbce40d0dd0 con 0x7fbce406fe20 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.188+0000 7fbceacdf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fbce40d10b0 con 0x7fbce40d0880 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.188+0000 7fbceacdf640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fbce40cc0e0 con 0x7fbce415e060 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.188+0000 7fbce8a7b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fbce415e060 0x7fbce40ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.188+0000 7fbce3fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fbce406fe20 0x7fbce40ce850 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:53652/0 (socket says 172.21.15.64:53652) 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.188+0000 7fbce3fff640 1 -- 172.21.15.64:0/4030183060 learned_addr learned my addr 172.21.15.64:0/4030183060 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.189+0000 7fbce8a7b640 1 -- 172.21.15.64:0/4030183060 >> v1:172.21.15.66:6789/0 conn(0x7fbce40d0880 legacy=0x7fbce40ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.189+0000 7fbce8a7b640 1 -- 172.21.15.64:0/4030183060 >> v1:172.21.15.64:6789/0 conn(0x7fbce406fe20 legacy=0x7fbce40ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.189+0000 7fbce37fe640 1 -- 172.21.15.64:0/4030183060 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3138220341 0 0) 0x7fbce40d10b0 con 0x7fbce40d0880 2024-03-04T07:37:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.189+0000 7fbce8a7b640 1 -- 172.21.15.64:0/4030183060 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbce40cc3c0 con 0x7fbce415e060 2024-03-04T07:37:02.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.189+0000 7fbce8a7b640 1 --2- 172.21.15.64:0/4030183060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce415e060 0x7fbce40ce310 secure :-1 s=READY pgs=1800 cs=0 l=1 rev1=1 crypto rx=0x7fbcd40097a0 tx=0x7fbcd402b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fe53c2a360d3c4c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:02.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.189+0000 7fbce37fe640 1 -- 172.21.15.64:0/4030183060 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcd4034070 con 0x7fbce415e060 2024-03-04T07:37:02.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbce37fe640 1 -- 172.21.15.64:0/4030183060 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbcd4030610 con 0x7fbce415e060 2024-03-04T07:37:02.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbce37fe640 1 -- 172.21.15.64:0/4030183060 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcd40309e0 con 0x7fbce415e060 2024-03-04T07:37:02.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbceacdf640 1 -- 172.21.15.64:0/4030183060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce415e060 msgr2=0x7fbce40ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4030183060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce415e060 0x7fbce40ce310 secure :-1 s=READY pgs=1800 cs=0 l=1 rev1=1 crypto rx=0x7fbcd40097a0 tx=0x7fbcd402b960 comp rx=0 tx=0).stop 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbceacdf640 1 -- 172.21.15.64:0/4030183060 shutdown_connections 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4030183060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce415e060 0x7fbce40ce310 unknown :-1 s=CLOSED pgs=1800 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbceacdf640 1 -- 172.21.15.64:0/4030183060 >> 172.21.15.64:0/4030183060 conn(0x7fbce40d1a90 msgr2=0x7fbce40d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbceacdf640 1 -- 172.21.15.64:0/4030183060 shutdown_connections 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.190+0000 7fbceacdf640 1 -- 172.21.15.64:0/4030183060 wait complete. 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.191+0000 7fbceacdf640 1 Processor -- start 2024-03-04T07:37:02.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.191+0000 7fbceacdf640 1 -- start start 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbceacdf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce406fe20 0x7fbce4160ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbceacdf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbce40d0880 0x7fbce414def0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbceacdf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbce415e060 0x7fbce415ed20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbceacdf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbce40cef60 con 0x7fbce406fe20 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbceacdf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbce415e440 con 0x7fbce415e060 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbceacdf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbce40cf0e0 con 0x7fbce40d0880 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbce8a7b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce406fe20 0x7fbce4160ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbce8a7b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce406fe20 0x7fbce4160ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:60446/0 (socket says 172.21.15.64:60446) 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbce8a7b640 1 -- 172.21.15.64:0/4200297781 learned_addr learned my addr 172.21.15.64:0/4200297781 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbce3fff640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbce40d0880 0x7fbce414def0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.192+0000 7fbce927c640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbce415e060 0x7fbce415ed20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce8a7b640 1 -- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbce40d0880 msgr2=0x7fbce414def0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:02.194 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce8a7b640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbce40d0880 0x7fbce414def0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce8a7b640 1 -- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbce415e060 msgr2=0x7fbce415ed20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce8a7b640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbce415e060 0x7fbce415ed20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce8a7b640 1 -- 172.21.15.64:0/4200297781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbce414e430 con 0x7fbce406fe20 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce8a7b640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce406fe20 0x7fbce4160ee0 secure :-1 s=READY pgs=1801 cs=0 l=1 rev1=1 crypto rx=0x7fbcd402bf70 tx=0x7fbcd40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce17fa640 1 -- 172.21.15.64:0/4200297781 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcd4034040 con 0x7fbce406fe20 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce17fa640 1 -- 172.21.15.64:0/4200297781 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fbcd403acc0 con 0x7fbce406fe20 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbce40b7660 con 0x7fbce406fe20 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbce17fa640 1 -- 172.21.15.64:0/4200297781 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcd4044630 con 0x7fbce406fe20 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.193+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbce40b6b80 con 0x7fbce406fe20 2024-03-04T07:37:02.195 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.194+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbce4067ff0 con 0x7fbce406fe20 2024-03-04T07:37:02.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.195+0000 7fbce17fa640 1 -- 172.21.15.64:0/4200297781 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fbcd4037070 con 0x7fbce406fe20 2024-03-04T07:37:02.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.195+0000 7fbce17fa640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbcd006b4d0 0x7fbcd006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:02.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.195+0000 7fbce17fa640 1 -- 172.21.15.64:0/4200297781 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fbcd40ac160 con 0x7fbce406fe20 2024-03-04T07:37:02.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.195+0000 7fbce3fff640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbcd006b4d0 0x7fbcd006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:02.196 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.195+0000 7fbce3fff640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbcd006b4d0 0x7fbcd006d990 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7fbce40b78c0 tx=0x7fbcd8007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:02.199 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.199+0000 7fbce17fa640 1 -- 172.21.15.64:0/4200297781 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fbce414e430 con 0x7fbce406fe20 2024-03-04T07:37:02.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.421+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbce4161820 con 0x7fbce406fe20 2024-03-04T07:37:02.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.422+0000 7fbce17fa640 1 -- 172.21.15.64:0/4200297781 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fbcd403f070 con 0x7fbce406fe20 2024-03-04T07:37:02.422 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:02.422 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:02.422 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.423+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbcd006b4d0 msgr2=0x7fbcd006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.423+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbcd006b4d0 0x7fbcd006d990 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7fbce40b78c0 tx=0x7fbcd8007e80 comp rx=0 tx=0).stop 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.423+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce406fe20 msgr2=0x7fbce4160ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.423+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce406fe20 0x7fbce4160ee0 secure :-1 s=READY pgs=1801 cs=0 l=1 rev1=1 crypto rx=0x7fbcd402bf70 tx=0x7fbcd40046c0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 shutdown_connections 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fbcd006b4d0 0x7fbcd006d990 unknown :-1 s=CLOSED pgs=821 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fbce415e060 0x7fbce415ed20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fbce40d0880 0x7fbce414def0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 --2- 172.21.15.64:0/4200297781 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fbce406fe20 0x7fbce4160ee0 unknown :-1 s=CLOSED pgs=1801 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:02.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 >> 172.21.15.64:0/4200297781 conn(0x7fbce40d1a90 msgr2=0x7fbce40ccbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:02.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 shutdown_connections 2024-03-04T07:37:02.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:02.424+0000 7fbceacdf640 1 -- 172.21.15.64:0/4200297781 wait complete. 2024-03-04T07:37:02.635 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:02.636 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:02.693 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:03.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:03.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.751+0000 7f3fa035c640 1 Processor -- start 2024-03-04T07:37:03.751 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.751+0000 7f3fa035c640 1 -- start start 2024-03-04T07:37:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.752+0000 7f3fa035c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3f980b9a10 0x7f3f980d3220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.752+0000 7f3fa035c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3f980cea30 con 0x7f3f980b95a0 2024-03-04T07:37:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.752+0000 7f3fa035c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3f980cf3c0 con 0x7f3f9815df00 2024-03-04T07:37:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.752+0000 7f3fa035c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3f980cf0e0 con 0x7f3f980b9a10 2024-03-04T07:37:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.752+0000 7f3f9e8f9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3f980b9a10 0x7f3f980d3220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.752+0000 7f3f9d8f7640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3f980b95a0 0x7f3f980b5700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34568/0 (socket says 172.21.15.64:34568) 2024-03-04T07:37:03.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.752+0000 7f3f9d8f7640 1 -- 172.21.15.64:0/3680025384 learned_addr learned my addr 172.21.15.64:0/3680025384 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.753+0000 7f3f9e8f9640 1 -- 172.21.15.64:0/3680025384 >> v1:172.21.15.66:6789/0 conn(0x7f3f9815df00 legacy=0x7f3f980d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.753+0000 7f3f9e8f9640 1 -- 172.21.15.64:0/3680025384 >> v1:172.21.15.64:6789/0 conn(0x7f3f980b95a0 legacy=0x7f3f980b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:03.753 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.753+0000 7f3f9d0f6640 1 -- 172.21.15.64:0/3680025384 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1271809147 0 0) 0x7f3f980cf3c0 con 0x7f3f9815df00 2024-03-04T07:37:03.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.753+0000 7f3f9e8f9640 1 -- 172.21.15.64:0/3680025384 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f980ce470 con 0x7f3f980b9a10 2024-03-04T07:37:03.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.753+0000 7f3f9e8f9640 1 --2- 172.21.15.64:0/3680025384 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b9a10 0x7f3f980d3220 secure :-1 s=READY pgs=1803 cs=0 l=1 rev1=1 crypto rx=0x7f3f980cf590 tx=0x7f3f9002bd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c13ad2b0f4f6aa8 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:03.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3f9d0f6640 1 -- 172.21.15.64:0/3680025384 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f90039070 con 0x7f3f980b9a10 2024-03-04T07:37:03.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3f9d0f6640 1 -- 172.21.15.64:0/3680025384 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f9002caa0 con 0x7f3f980b9a10 2024-03-04T07:37:03.754 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3f9d0f6640 1 -- 172.21.15.64:0/3680025384 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f90034440 con 0x7f3f980b9a10 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3fa035c640 1 -- 172.21.15.64:0/3680025384 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b9a10 msgr2=0x7f3f980d3220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3fa035c640 1 --2- 172.21.15.64:0/3680025384 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b9a10 0x7f3f980d3220 secure :-1 s=READY pgs=1803 cs=0 l=1 rev1=1 crypto rx=0x7f3f980cf590 tx=0x7f3f9002bd70 comp rx=0 tx=0).stop 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3fa035c640 1 -- 172.21.15.64:0/3680025384 shutdown_connections 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3fa035c640 1 --2- 172.21.15.64:0/3680025384 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b9a10 0x7f3f980d3220 unknown :-1 s=CLOSED pgs=1803 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3fa035c640 1 -- 172.21.15.64:0/3680025384 >> 172.21.15.64:0/3680025384 conn(0x7f3f98071960 msgr2=0x7f3f98071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3fa035c640 1 -- 172.21.15.64:0/3680025384 shutdown_connections 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.754+0000 7f3fa035c640 1 -- 172.21.15.64:0/3680025384 wait complete. 2024-03-04T07:37:03.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.755+0000 7f3fa035c640 1 Processor -- start 2024-03-04T07:37:03.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.755+0000 7f3fa035c640 1 -- start start 2024-03-04T07:37:03.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3fa035c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b95a0 0x7f3f980b1ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:03.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3fa035c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f980b9a10 0x7f3f98160cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:03.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3fa035c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f9815df00 0x7f3f9814d9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3fa035c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f980cedf0 con 0x7f3f980b95a0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3fa035c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f980cec70 con 0x7f3f9815df00 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3fa035c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f980b5e10 con 0x7f3f980b9a10 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3f9e0f8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b95a0 0x7f3f980b1ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3f9e0f8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b95a0 0x7f3f980b1ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51866/0 (socket says 172.21.15.64:51866) 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3f9e0f8640 1 -- 172.21.15.64:0/970684592 learned_addr learned my addr 172.21.15.64:0/970684592 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3f9d8f7640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f980b9a10 0x7f3f98160cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.756+0000 7f3f9e8f9640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f9815df00 0x7f3f9814d9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f9e0f8640 1 -- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f980b9a10 msgr2=0x7f3f98160cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f9e0f8640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f980b9a10 0x7f3f98160cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f9e0f8640 1 -- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f9815df00 msgr2=0x7f3f9814d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f9e0f8640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f9815df00 0x7f3f9814d9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f9e0f8640 1 -- 172.21.15.64:0/970684592 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f980cbff0 con 0x7f3f980b95a0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f9e0f8640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b95a0 0x7f3f980b1ba0 secure :-1 s=READY pgs=1804 cs=0 l=1 rev1=1 crypto rx=0x7f3f9400c440 tx=0x7f3f9400c900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f8effd640 1 -- 172.21.15.64:0/970684592 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f94017070 con 0x7f3f980b95a0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f8effd640 1 -- 172.21.15.64:0/970684592 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f94005480 con 0x7f3f980b95a0 2024-03-04T07:37:03.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f980cc2d0 con 0x7f3f980b95a0 2024-03-04T07:37:03.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3f8effd640 1 -- 172.21.15.64:0/970684592 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f94003830 con 0x7f3f980b95a0 2024-03-04T07:37:03.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.757+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f980b20e0 con 0x7f3f980b95a0 2024-03-04T07:37:03.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.758+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3f98067ff0 con 0x7f3f980b95a0 2024-03-04T07:37:03.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.758+0000 7f3f8effd640 1 -- 172.21.15.64:0/970684592 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3f9401a070 con 0x7f3f980b95a0 2024-03-04T07:37:03.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.759+0000 7f3f8effd640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f7c06b4d0 0x7f3f7c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:03.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.759+0000 7f3f8effd640 1 -- 172.21.15.64:0/970684592 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3f94089470 con 0x7f3f980b95a0 2024-03-04T07:37:03.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.759+0000 7f3f9d8f7640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f7c06b4d0 0x7f3f7c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:03.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.759+0000 7f3f9d8f7640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f7c06b4d0 0x7f3f7c06d990 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7f3f981f6e40 tx=0x7f3f88007e20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:03.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.763+0000 7f3f8effd640 1 -- 172.21.15.64:0/970684592 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3f94057c60 con 0x7f3f980b95a0 2024-03-04T07:37:03.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.985+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3f9815b820 con 0x7f3f980b95a0 2024-03-04T07:37:03.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.985+0000 7f3f8effd640 1 -- 172.21.15.64:0/970684592 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f3f9405bd00 con 0x7f3f980b95a0 2024-03-04T07:37:03.985 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:03.985 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:03.986 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:03.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f7c06b4d0 msgr2=0x7f3f7c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f7c06b4d0 0x7f3f7c06d990 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7f3f981f6e40 tx=0x7f3f88007e20 comp rx=0 tx=0).stop 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b95a0 msgr2=0x7f3f980b1ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b95a0 0x7f3f980b1ba0 secure :-1 s=READY pgs=1804 cs=0 l=1 rev1=1 crypto rx=0x7f3f9400c440 tx=0x7f3f9400c900 comp rx=0 tx=0).stop 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 shutdown_connections 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3f7c06b4d0 0x7f3f7c06d990 unknown :-1 s=CLOSED pgs=822 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f9815df00 0x7f3f9814d9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f980b9a10 0x7f3f98160cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 --2- 172.21.15.64:0/970684592 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f980b95a0 0x7f3f980b1ba0 unknown :-1 s=CLOSED pgs=1804 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 >> 172.21.15.64:0/970684592 conn(0x7f3f98071960 msgr2=0x7f3f980b7bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 shutdown_connections 2024-03-04T07:37:03.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:03.987+0000 7f3fa035c640 1 -- 172.21.15.64:0/970684592 wait complete. 2024-03-04T07:37:04.199 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:04.199 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:04.695 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:05.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:05.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.313+0000 7fba8c7b5640 1 Processor -- start 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.313+0000 7fba8c7b5640 1 -- start start 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.314+0000 7fba8c7b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415df00 0x7fba840d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.314+0000 7fba8c7b5640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fba840b95a0 0x7fba840b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.314+0000 7fba8c7b5640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fba840cea30 con 0x7fba840b9a30 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.314+0000 7fba8c7b5640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fba840d2af0 con 0x7fba840b95a0 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.314+0000 7fba8c7b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fba840cf3c0 con 0x7fba8415df00 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba8a551640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415df00 0x7fba840d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba89d50640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fba840b95a0 0x7fba840b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba8a551640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415df00 0x7fba840d25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:54436/0 (socket says 172.21.15.64:54436) 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba8ad52640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fba840b9a30 0x7fba840d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:57078/0 (socket says 172.21.15.64:57078) 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba8a551640 1 -- 172.21.15.64:0/1462357808 learned_addr learned my addr 172.21.15.64:0/1462357808 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba8954f640 1 -- 172.21.15.64:0/1462357808 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1689172371 0 0) 0x7fba840cea30 con 0x7fba840b9a30 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba8954f640 1 -- 172.21.15.64:0/1462357808 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fba70003520 con 0x7fba840b9a30 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba89d50640 1 -- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415df00 msgr2=0x7fba840d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba89d50640 1 --2- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415df00 0x7fba840d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba89d50640 1 -- 172.21.15.64:0/1462357808 >> v1:172.21.15.66:6789/0 conn(0x7fba840b9a30 legacy=0x7fba840d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba89d50640 1 -- 172.21.15.64:0/1462357808 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba840cf0e0 con 0x7fba840b95a0 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.315+0000 7fba8a551640 1 --2- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415df00 0x7fba840d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.316+0000 7fba89d50640 1 --2- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba840b95a0 0x7fba840b5700 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7fba740097a0 tx=0x7fba7402b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ff190a708d98af78 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.316+0000 7fba8954f640 1 -- 172.21.15.64:0/1462357808 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba74035020 con 0x7fba840b95a0 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.316+0000 7fba8954f640 1 -- 172.21.15.64:0/1462357808 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba74031830 con 0x7fba840b95a0 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.316+0000 7fba8954f640 1 -- 172.21.15.64:0/1462357808 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba74031c00 con 0x7fba840b95a0 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.316+0000 7fba8c7b5640 1 -- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba840b95a0 msgr2=0x7fba840b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.316+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba840b95a0 0x7fba840b5700 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7fba740097a0 tx=0x7fba7402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.317+0000 7fba8c7b5640 1 -- 172.21.15.64:0/1462357808 shutdown_connections 2024-03-04T07:37:05.319 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.317+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba840b95a0 0x7fba840b5700 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.317+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/1462357808 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415df00 0x7fba840d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.317+0000 7fba8c7b5640 1 -- 172.21.15.64:0/1462357808 >> 172.21.15.64:0/1462357808 conn(0x7fba84071960 msgr2=0x7fba84071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:05.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.317+0000 7fba8c7b5640 1 -- 172.21.15.64:0/1462357808 shutdown_connections 2024-03-04T07:37:05.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.317+0000 7fba8c7b5640 1 -- 172.21.15.64:0/1462357808 wait complete. 2024-03-04T07:37:05.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.318+0000 7fba8c7b5640 1 Processor -- start 2024-03-04T07:37:05.320 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.318+0000 7fba8c7b5640 1 -- start start 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.318+0000 7fba8c7b5640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba840b95a0 0x7fba841f4810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.318+0000 7fba8c7b5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba8415df00 0x7fba8415ecf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.318+0000 7fba8c7b5640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415fe50 0x7fba840b8830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.318+0000 7fba8c7b5640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba840cec70 con 0x7fba840b95a0 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8c7b5640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba840ce750 con 0x7fba8415df00 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8c7b5640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fba840cedf0 con 0x7fba8415fe50 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba840b95a0 0x7fba841f4810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba840b95a0 0x7fba841f4810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51878/0 (socket says 172.21.15.64:51878) 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 -- 172.21.15.64:0/2982909087 learned_addr learned my addr 172.21.15.64:0/2982909087 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8ad52640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415fe50 0x7fba840b8830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba89d50640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba8415df00 0x7fba8415ecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:05.323 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 -- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415fe50 msgr2=0x7fba840b8830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415fe50 0x7fba840b8830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 -- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba8415df00 msgr2=0x7fba8415ecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba8415df00 0x7fba8415ecf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 -- 172.21.15.64:0/2982909087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba8415f230 con 0x7fba840b95a0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba89d50640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba8415df00 0x7fba8415ecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8a551640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba840b95a0 0x7fba841f4810 secure :-1 s=READY pgs=1805 cs=0 l=1 rev1=1 crypto rx=0x7fba8000e9e0 tx=0x7fba8000eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba7affd640 1 -- 172.21.15.64:0/2982909087 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba80017070 con 0x7fba840b95a0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba7affd640 1 -- 172.21.15.64:0/2982909087 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fba8000c9d0 con 0x7fba840b95a0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fba840b31d0 con 0x7fba840b95a0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba7affd640 1 -- 172.21.15.64:0/2982909087 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba8000cd80 con 0x7fba840b95a0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.319+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fba840b3790 con 0x7fba840b95a0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.320+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fba84067ff0 con 0x7fba840b95a0 2024-03-04T07:37:05.324 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.321+0000 7fba7affd640 1 -- 172.21.15.64:0/2982909087 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fba8001a070 con 0x7fba840b95a0 2024-03-04T07:37:05.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.321+0000 7fba7affd640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba6806b4d0 0x7fba6806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:05.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.321+0000 7fba89d50640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba6806b4d0 0x7fba6806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:05.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.322+0000 7fba7affd640 1 -- 172.21.15.64:0/2982909087 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fba8008a110 con 0x7fba840b95a0 2024-03-04T07:37:05.325 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.322+0000 7fba89d50640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba6806b4d0 0x7fba6806d990 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7fba74036000 tx=0x7fba74008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:05.326 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.326+0000 7fba7affd640 1 -- 172.21.15.64:0/2982909087 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fba8415f230 con 0x7fba840b95a0 2024-03-04T07:37:05.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.549+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fba8415d270 con 0x7fba840b95a0 2024-03-04T07:37:05.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.549+0000 7fba7affd640 1 -- 172.21.15.64:0/2982909087 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fba80058870 con 0x7fba840b95a0 2024-03-04T07:37:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:05.549 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:05.550 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:05.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba6806b4d0 msgr2=0x7fba6806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:05.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba6806b4d0 0x7fba6806d990 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7fba74036000 tx=0x7fba74008030 comp rx=0 tx=0).stop 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba840b95a0 msgr2=0x7fba841f4810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba840b95a0 0x7fba841f4810 secure :-1 s=READY pgs=1805 cs=0 l=1 rev1=1 crypto rx=0x7fba8000e9e0 tx=0x7fba8000eea0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 shutdown_connections 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fba6806b4d0 0x7fba6806d990 unknown :-1 s=CLOSED pgs=823 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fba8415fe50 0x7fba840b8830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fba8415df00 0x7fba8415ecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 --2- 172.21.15.64:0/2982909087 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fba840b95a0 0x7fba841f4810 unknown :-1 s=CLOSED pgs=1805 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 >> 172.21.15.64:0/2982909087 conn(0x7fba84071960 msgr2=0x7fba841618b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 shutdown_connections 2024-03-04T07:37:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:05.551+0000 7fba8c7b5640 1 -- 172.21.15.64:0/2982909087 wait complete. 2024-03-04T07:37:05.763 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:05.763 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:06.698 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:06.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.879+0000 7f25c798c640 1 Processor -- start 2024-03-04T07:37:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.879+0000 7f25c798c640 1 -- start start 2024-03-04T07:37:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c798c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f25c00d18c0 0x7f25c0071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:06.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c798c640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f25c00b6f20 con 0x7f25c015df70 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c798c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f25c00d3030 con 0x7f25c0071a80 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c798c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f25c0071eb0 con 0x7f25c00d18c0 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c5728640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f25c015df70 0x7f25c00d27b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34584/0 (socket says 172.21.15.64:34584) 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c5728640 1 -- 172.21.15.64:0/2087418309 learned_addr learned my addr 172.21.15.64:0/2087418309 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c4f27640 1 --2- 172.21.15.64:0/2087418309 >> v2:172.21.15.66:3300/0 conn(0x7f25c00d18c0 0x7f25c0071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25b7fff640 1 -- 172.21.15.64:0/2087418309 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 878564971 0 0) 0x7f25c00b6f20 con 0x7f25c015df70 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25b7fff640 1 -- 172.21.15.64:0/2087418309 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f25ac003520 con 0x7f25c015df70 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c4f27640 1 -- 172.21.15.64:0/2087418309 >> v1:172.21.15.66:6789/0 conn(0x7f25c0071a80 legacy=0x7f25c00b2700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c4f27640 1 -- 172.21.15.64:0/2087418309 >> v1:172.21.15.64:6789/0 conn(0x7f25c015df70 legacy=0x7f25c00d27b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:06.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.880+0000 7f25c4f27640 1 -- 172.21.15.64:0/2087418309 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f25c00d3310 con 0x7f25c00d18c0 2024-03-04T07:37:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.881+0000 7f25c4f27640 1 --2- 172.21.15.64:0/2087418309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c0071540 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f25b00097a0 tx=0x7f25b002b9f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6e2129b54defa82d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.881+0000 7f25b7fff640 1 -- 172.21.15.64:0/2087418309 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f25b002f810 con 0x7f25c00d18c0 2024-03-04T07:37:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.881+0000 7f25b7fff640 1 -- 172.21.15.64:0/2087418309 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f25b002f9b0 con 0x7f25c00d18c0 2024-03-04T07:37:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.881+0000 7f25b7fff640 1 -- 172.21.15.64:0/2087418309 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f25b002fd60 con 0x7f25c00d18c0 2024-03-04T07:37:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.882+0000 7f25c798c640 1 -- 172.21.15.64:0/2087418309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 msgr2=0x7f25c0071540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.882+0000 7f25c798c640 1 --2- 172.21.15.64:0/2087418309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c0071540 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f25b00097a0 tx=0x7f25b002b9f0 comp rx=0 tx=0).stop 2024-03-04T07:37:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.882+0000 7f25c798c640 1 -- 172.21.15.64:0/2087418309 shutdown_connections 2024-03-04T07:37:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.882+0000 7f25c798c640 1 --2- 172.21.15.64:0/2087418309 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c0071540 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.882+0000 7f25c798c640 1 -- 172.21.15.64:0/2087418309 >> 172.21.15.64:0/2087418309 conn(0x7f25c00d3cf0 msgr2=0x7f25c00d4100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.882+0000 7f25c798c640 1 -- 172.21.15.64:0/2087418309 shutdown_connections 2024-03-04T07:37:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.882+0000 7f25c798c640 1 -- 172.21.15.64:0/2087418309 wait complete. 2024-03-04T07:37:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.883+0000 7f25c798c640 1 Processor -- start 2024-03-04T07:37:06.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.883+0000 7f25c798c640 1 -- start start 2024-03-04T07:37:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c798c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f25c0071a80 0x7f25c00c8f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c798c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c01e7250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c798c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25c015df70 0x7f25c01e7c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c798c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f25c015e350 con 0x7f25c0071a80 2024-03-04T07:37:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c798c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f25c00d38d0 con 0x7f25c00d18c0 2024-03-04T07:37:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c798c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f25c015e4d0 con 0x7f25c015df70 2024-03-04T07:37:06.885 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f25c0071a80 0x7f25c00c8f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f25c0071a80 0x7f25c00c8f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51886/0 (socket says 172.21.15.64:51886) 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 -- 172.21.15.64:0/2945710216 learned_addr learned my addr 172.21.15.64:0/2945710216 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c4f27640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c01e7250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5f29640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25c015df70 0x7f25c01e7c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 -- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25c015df70 msgr2=0x7f25c01e7c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25c015df70 0x7f25c01e7c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 -- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 msgr2=0x7f25c01e7250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c01e7250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c5728640 1 -- 172.21.15.64:0/2945710216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f25c00cc880 con 0x7f25c0071a80 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.884+0000 7f25c4f27640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c01e7250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.885+0000 7f25c5728640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f25c0071a80 0x7f25c00c8f90 secure :-1 s=READY pgs=1807 cs=0 l=1 rev1=1 crypto rx=0x7f25bc00c410 tx=0x7f25bc00c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.885+0000 7f25b5ffb640 1 -- 172.21.15.64:0/2945710216 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f25bc017070 con 0x7f25c0071a80 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.885+0000 7f25b5ffb640 1 -- 172.21.15.64:0/2945710216 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f25bc005ce0 con 0x7f25c0071a80 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.885+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f25c00ccb60 con 0x7f25c0071a80 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.885+0000 7f25b5ffb640 1 -- 172.21.15.64:0/2945710216 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f25bc0072f0 con 0x7f25c0071a80 2024-03-04T07:37:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.885+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f25c00cd120 con 0x7f25c0071a80 2024-03-04T07:37:06.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.886+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f25c0067ff0 con 0x7f25c0071a80 2024-03-04T07:37:06.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.887+0000 7f25b5ffb640 1 -- 172.21.15.64:0/2945710216 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f25bc01a070 con 0x7f25c0071a80 2024-03-04T07:37:06.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.887+0000 7f25b5ffb640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f25a406b4d0 0x7f25a406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:06.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.887+0000 7f25b5ffb640 1 -- 172.21.15.64:0/2945710216 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f25bc089650 con 0x7f25c0071a80 2024-03-04T07:37:06.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.887+0000 7f25c4f27640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f25a406b4d0 0x7f25a406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:06.888 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.888+0000 7f25c4f27640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f25a406b4d0 0x7f25a406d990 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7f25b0036000 tx=0x7f25b0008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:06.891 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:06.891+0000 7f25b5ffb640 1 -- 172.21.15.64:0/2945710216 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f25bc057f00 con 0x7f25c0071a80 2024-03-04T07:37:07.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.111+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f25c00cf1c0 con 0x7f25c0071a80 2024-03-04T07:37:07.111 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.111+0000 7f25b5ffb640 1 -- 172.21.15.64:0/2945710216 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f25bc05bfa0 con 0x7f25c0071a80 2024-03-04T07:37:07.112 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:07.112 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:07.112 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:07.113 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f25a406b4d0 msgr2=0x7f25a406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f25a406b4d0 0x7f25a406d990 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7f25b0036000 tx=0x7f25b0008030 comp rx=0 tx=0).stop 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f25c0071a80 msgr2=0x7f25c00c8f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f25c0071a80 0x7f25c00c8f90 secure :-1 s=READY pgs=1807 cs=0 l=1 rev1=1 crypto rx=0x7f25bc00c410 tx=0x7f25bc00c8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 shutdown_connections 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f25a406b4d0 0x7f25a406d990 unknown :-1 s=CLOSED pgs=824 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f25c015df70 0x7f25c01e7c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f25c00d18c0 0x7f25c01e7250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 --2- 172.21.15.64:0/2945710216 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f25c0071a80 0x7f25c00c8f90 unknown :-1 s=CLOSED pgs=1807 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 >> 172.21.15.64:0/2945710216 conn(0x7f25c00d3cf0 msgr2=0x7f25c00b42b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 shutdown_connections 2024-03-04T07:37:07.114 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:07.113+0000 7f25c798c640 1 -- 172.21.15.64:0/2945710216 wait complete. 2024-03-04T07:37:07.325 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:07.325 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:08.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:08.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.439+0000 7f3b5f857640 1 Processor -- start 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.440+0000 7f3b5f857640 1 -- start start 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.440+0000 7f3b5f857640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3b58071540 0x7f3b580b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.440+0000 7f3b5f857640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580b2b50 0x7f3b580b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.440+0000 7f3b5f857640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3b580ce610 con 0x7f3b580d3400 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.440+0000 7f3b5f857640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3b580d2f70 con 0x7f3b58071540 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.440+0000 7f3b5f857640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b580d4be0 con 0x7f3b580b2b50 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5ddf4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580b2b50 0x7f3b580b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5d5f3640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3b58071540 0x7f3b580b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5cdf2640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f3b580d3400 0x7f3b58071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:57098/0 (socket says 172.21.15.64:57098) 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5cdf2640 1 -- 172.21.15.64:0/1787065299 learned_addr learned my addr 172.21.15.64:0/1787065299 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b4ffff640 1 -- 172.21.15.64:0/1787065299 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2788378479 0 0) 0x7f3b580ce610 con 0x7f3b580d3400 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b4ffff640 1 -- 172.21.15.64:0/1787065299 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3b44003520 con 0x7f3b580d3400 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5d5f3640 1 -- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580b2b50 msgr2=0x7f3b580b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:08.447 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5d5f3640 1 --2- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580b2b50 0x7f3b580b6e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5d5f3640 1 -- 172.21.15.64:0/1787065299 >> v1:172.21.15.66:6789/0 conn(0x7f3b580d3400 legacy=0x7f3b58071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5d5f3640 1 -- 172.21.15.64:0/1787065299 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b580d2920 con 0x7f3b58071540 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.441+0000 7f3b5ddf4640 1 --2- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580b2b50 0x7f3b580b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.442+0000 7f3b5d5f3640 1 --2- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b58071540 0x7f3b580b30a0 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f3b500097a0 tx=0x7f3b5002b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=51cba375d983fc9c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.442+0000 7f3b4ffff640 1 -- 172.21.15.64:0/1787065299 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b50035020 con 0x7f3b58071540 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.442+0000 7f3b4ffff640 1 -- 172.21.15.64:0/1787065299 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b50031830 con 0x7f3b58071540 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.442+0000 7f3b4ffff640 1 -- 172.21.15.64:0/1787065299 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b50031be0 con 0x7f3b58071540 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.442+0000 7f3b5f857640 1 -- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b58071540 msgr2=0x7f3b580b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.442+0000 7f3b5f857640 1 --2- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b58071540 0x7f3b580b30a0 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f3b500097a0 tx=0x7f3b5002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.443+0000 7f3b5f857640 1 -- 172.21.15.64:0/1787065299 shutdown_connections 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.443+0000 7f3b5f857640 1 --2- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580b2b50 0x7f3b580b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.443+0000 7f3b5f857640 1 --2- 172.21.15.64:0/1787065299 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b58071540 0x7f3b580b30a0 unknown :-1 s=CLOSED pgs=673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.443+0000 7f3b5f857640 1 -- 172.21.15.64:0/1787065299 >> 172.21.15.64:0/1787065299 conn(0x7f3b580d3ca0 msgr2=0x7f3b580d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:08.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.443+0000 7f3b5f857640 1 -- 172.21.15.64:0/1787065299 shutdown_connections 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.443+0000 7f3b5f857640 1 -- 172.21.15.64:0/1787065299 wait complete. 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.444+0000 7f3b5f857640 1 Processor -- start 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.444+0000 7f3b5f857640 1 -- start start 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5f857640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b58071540 0x7f3b580c9940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5f857640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b580b2b50 0x7f3b5815f640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5f857640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580d3400 0x7f3b5815fb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5f857640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b5815e370 con 0x7f3b58071540 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5f857640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b580ce8f0 con 0x7f3b580b2b50 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5f857640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b58158880 con 0x7f3b580d3400 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b58071540 0x7f3b580c9940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b58071540 0x7f3b580c9940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51894/0 (socket says 172.21.15.64:51894) 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 -- 172.21.15.64:0/3651535153 learned_addr learned my addr 172.21.15.64:0/3651535153 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:08.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5cdf2640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b580b2b50 0x7f3b5815f640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5ddf4640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580d3400 0x7f3b5815fb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 -- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580d3400 msgr2=0x7f3b5815fb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580d3400 0x7f3b5815fb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 -- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b580b2b50 msgr2=0x7f3b5815f640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b580b2b50 0x7f3b5815f640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 -- 172.21.15.64:0/3651535153 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b581f4410 con 0x7f3b58071540 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5ddf4640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580d3400 0x7f3b5815fb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b5d5f3640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b58071540 0x7f3b580c9940 secure :-1 s=READY pgs=1808 cs=0 l=1 rev1=1 crypto rx=0x7f3b500098c0 tx=0x7f3b500046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b4dffb640 1 -- 172.21.15.64:0/3651535153 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b50035040 con 0x7f3b58071540 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.445+0000 7f3b4dffb640 1 -- 172.21.15.64:0/3651535153 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3b5003b840 con 0x7f3b58071540 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.446+0000 7f3b4dffb640 1 -- 172.21.15.64:0/3651535153 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b5003bc30 con 0x7f3b58071540 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.446+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b58158ad0 con 0x7f3b58071540 2024-03-04T07:37:08.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.446+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b581f47f0 con 0x7f3b58071540 2024-03-04T07:37:08.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.447+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3b58068700 con 0x7f3b58071540 2024-03-04T07:37:08.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.447+0000 7f3b4dffb640 1 -- 172.21.15.64:0/3651535153 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3b50038020 con 0x7f3b58071540 2024-03-04T07:37:08.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.448+0000 7f3b4dffb640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3b3c06b4d0 0x7f3b3c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:08.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.448+0000 7f3b5cdf2640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3b3c06b4d0 0x7f3b3c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:08.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.448+0000 7f3b4dffb640 1 -- 172.21.15.64:0/3651535153 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3b500ac080 con 0x7f3b58071540 2024-03-04T07:37:08.451 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.448+0000 7f3b5cdf2640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3b3c06b4d0 0x7f3b3c06d990 secure :-1 s=READY pgs=825 cs=0 l=1 rev1=1 crypto rx=0x7f3b580b8920 tx=0x7f3b48009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:08.453 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.452+0000 7f3b4dffb640 1 -- 172.21.15.64:0/3651535153 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3b5007af50 con 0x7f3b58071540 2024-03-04T07:37:08.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.670+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3b5815ef10 con 0x7f3b58071540 2024-03-04T07:37:08.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.671+0000 7f3b4dffb640 1 -- 172.21.15.64:0/3651535153 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f3b5007a850 con 0x7f3b58071540 2024-03-04T07:37:08.672 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:08.672 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:08.672 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3b3c06b4d0 msgr2=0x7f3b3c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3b3c06b4d0 0x7f3b3c06d990 secure :-1 s=READY pgs=825 cs=0 l=1 rev1=1 crypto rx=0x7f3b580b8920 tx=0x7f3b48009040 comp rx=0 tx=0).stop 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b58071540 msgr2=0x7f3b580c9940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b58071540 0x7f3b580c9940 secure :-1 s=READY pgs=1808 cs=0 l=1 rev1=1 crypto rx=0x7f3b500098c0 tx=0x7f3b500046c0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 shutdown_connections 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3b3c06b4d0 0x7f3b3c06d990 unknown :-1 s=CLOSED pgs=825 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3b580d3400 0x7f3b5815fb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3b580b2b50 0x7f3b5815f640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 --2- 172.21.15.64:0/3651535153 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3b58071540 0x7f3b580c9940 unknown :-1 s=CLOSED pgs=1808 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:08.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 >> 172.21.15.64:0/3651535153 conn(0x7f3b580d3ca0 msgr2=0x7f3b58071920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:08.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 shutdown_connections 2024-03-04T07:37:08.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:08.673+0000 7f3b5f857640 1 -- 172.21.15.64:0/3651535153 wait complete. 2024-03-04T07:37:08.699 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:08.886 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:08.886 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:09.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:10.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.000+0000 7fd065476640 1 Processor -- start 2024-03-04T07:37:10.001 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.000+0000 7fd065476640 1 -- start start 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.000+0000 7fd065476640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600b0e90 0x7fd06006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.000+0000 7fd065476640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fd0600ae9b0 0x7fd0600d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.000+0000 7fd065476640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd0600cee90 con 0x7fd06006b5c0 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.000+0000 7fd065476640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd0600cfa00 con 0x7fd0600ae9b0 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.000+0000 7fd065476640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0600cfc50 con 0x7fd0600b0e90 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd05effd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fd06006b5c0 0x7fd0600b2e00 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34588/0 (socket says 172.21.15.64:34588) 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd05effd640 1 -- 172.21.15.64:0/1572057800 learned_addr learned my addr 172.21.15.64:0/1572057800 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd057fff640 1 --2- 172.21.15.64:0/1572057800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600b0e90 0x7fd06006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:10.002 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd05f7fe640 1 --2- 172.21.15.64:0/1572057800 >> v2:172.21.15.66:3300/0 conn(0x7fd0600ae9b0 0x7fd0600d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:10.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd05e7fc640 1 -- 172.21.15.64:0/1572057800 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2135074308 0 0) 0x7fd0600cee90 con 0x7fd06006b5c0 2024-03-04T07:37:10.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd05e7fc640 1 -- 172.21.15.64:0/1572057800 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fd04c003520 con 0x7fd06006b5c0 2024-03-04T07:37:10.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd057fff640 1 -- 172.21.15.64:0/1572057800 >> v2:172.21.15.66:3300/0 conn(0x7fd0600ae9b0 msgr2=0x7fd0600d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:10.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd057fff640 1 --2- 172.21.15.64:0/1572057800 >> v2:172.21.15.66:3300/0 conn(0x7fd0600ae9b0 0x7fd0600d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd057fff640 1 -- 172.21.15.64:0/1572057800 >> v1:172.21.15.64:6789/0 conn(0x7fd06006b5c0 legacy=0x7fd0600b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:10.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.001+0000 7fd057fff640 1 -- 172.21.15.64:0/1572057800 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0600cff30 con 0x7fd0600b0e90 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.002+0000 7fd057fff640 1 --2- 172.21.15.64:0/1572057800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600b0e90 0x7fd06006b9a0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fd0480097a0 tx=0x7fd04802b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2daa08d835adc893 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.002+0000 7fd05e7fc640 1 -- 172.21.15.64:0/1572057800 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd0480315a0 con 0x7fd0600b0e90 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.002+0000 7fd05e7fc640 1 -- 172.21.15.64:0/1572057800 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd048037070 con 0x7fd0600b0e90 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.002+0000 7fd05e7fc640 1 -- 172.21.15.64:0/1572057800 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd048031940 con 0x7fd0600b0e90 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 -- 172.21.15.64:0/1572057800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600b0e90 msgr2=0x7fd06006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 --2- 172.21.15.64:0/1572057800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600b0e90 0x7fd06006b9a0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fd0480097a0 tx=0x7fd04802b840 comp rx=0 tx=0).stop 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 -- 172.21.15.64:0/1572057800 shutdown_connections 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 --2- 172.21.15.64:0/1572057800 >> v2:172.21.15.66:3300/0 conn(0x7fd0600ae9b0 0x7fd0600d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 --2- 172.21.15.64:0/1572057800 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600b0e90 0x7fd06006b9a0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 -- 172.21.15.64:0/1572057800 >> 172.21.15.64:0/1572057800 conn(0x7fd0600b36d0 msgr2=0x7fd0600b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:10.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 -- 172.21.15.64:0/1572057800 shutdown_connections 2024-03-04T07:37:10.005 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.003+0000 7fd065476640 1 -- 172.21.15.64:0/1572057800 wait complete. 2024-03-04T07:37:10.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.004+0000 7fd065476640 1 Processor -- start 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.004+0000 7fd065476640 1 -- start start 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd06006b5c0 0x7fd0600cb450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600ae9b0 0x7fd0600c6cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd0600b0e90 0x7fd0601f3e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd06006c910 con 0x7fd0600b0e90 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd06006c790 con 0x7fd06006b5c0 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd0600aee20 con 0x7fd0600ae9b0 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd0600b0e90 0x7fd0601f3e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd0600b0e90 0x7fd0601f3e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51904/0 (socket says 172.21.15.64:51904) 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 -- 172.21.15.64:0/1503643345 learned_addr learned my addr 172.21.15.64:0/1503643345 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd057fff640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600ae9b0 0x7fd0600c6cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05effd640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd06006b5c0 0x7fd0600cb450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:10.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 -- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600ae9b0 msgr2=0x7fd0600c6cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600ae9b0 0x7fd0600c6cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 -- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd06006b5c0 msgr2=0x7fd0600cb450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd06006b5c0 0x7fd0600cb450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 -- 172.21.15.64:0/1503643345 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0601f46d0 con 0x7fd0600b0e90 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd05f7fe640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd0600b0e90 0x7fd0601f3e30 secure :-1 s=READY pgs=1810 cs=0 l=1 rev1=1 crypto rx=0x7fd05800efc0 tx=0x7fd05800c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd0577fe640 1 -- 172.21.15.64:0/1503643345 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd058018020 con 0x7fd0600b0e90 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd0577fe640 1 -- 172.21.15.64:0/1503643345 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd058014480 con 0x7fd0600b0e90 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd0601f49b0 con 0x7fd0600b0e90 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd0577fe640 1 -- 172.21.15.64:0/1503643345 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd058014870 con 0x7fd0600b0e90 2024-03-04T07:37:10.008 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.005+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0600c5630 con 0x7fd0600b0e90 2024-03-04T07:37:10.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.007+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd060068700 con 0x7fd0600b0e90 2024-03-04T07:37:10.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.007+0000 7fd0577fe640 1 -- 172.21.15.64:0/1503643345 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd058005d10 con 0x7fd0600b0e90 2024-03-04T07:37:10.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.007+0000 7fd0577fe640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd04006b4d0 0x7fd04006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:10.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.007+0000 7fd0577fe640 1 -- 172.21.15.64:0/1503643345 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd05808abb0 con 0x7fd0600b0e90 2024-03-04T07:37:10.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.007+0000 7fd05effd640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd04006b4d0 0x7fd04006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:10.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.008+0000 7fd05effd640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd04006b4d0 0x7fd04006d990 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fd050005880 tx=0x7fd050009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:10.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.012+0000 7fd0577fe640 1 -- 172.21.15.64:0/1503643345 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd058022070 con 0x7fd0600b0e90 2024-03-04T07:37:10.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.236+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd0600cf180 con 0x7fd0600b0e90 2024-03-04T07:37:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.237+0000 7fd0577fe640 1 -- 172.21.15.64:0/1503643345 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fd058010040 con 0x7fd0600b0e90 2024-03-04T07:37:10.238 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:10.238 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:10.238 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:10.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd04006b4d0 msgr2=0x7fd04006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:10.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd04006b4d0 0x7fd04006d990 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fd050005880 tx=0x7fd050009040 comp rx=0 tx=0).stop 2024-03-04T07:37:10.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd0600b0e90 msgr2=0x7fd0601f3e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:10.240 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd0600b0e90 0x7fd0601f3e30 secure :-1 s=READY pgs=1810 cs=0 l=1 rev1=1 crypto rx=0x7fd05800efc0 tx=0x7fd05800c490 comp rx=0 tx=0).stop 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 shutdown_connections 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd04006b4d0 0x7fd04006d990 unknown :-1 s=CLOSED pgs=826 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd0600b0e90 0x7fd0601f3e30 unknown :-1 s=CLOSED pgs=1810 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd0600ae9b0 0x7fd0600c6cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 --2- 172.21.15.64:0/1503643345 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd06006b5c0 0x7fd0600cb450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 >> 172.21.15.64:0/1503643345 conn(0x7fd0600b36d0 msgr2=0x7fd0601e51d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 shutdown_connections 2024-03-04T07:37:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:10.239+0000 7fd065476640 1 -- 172.21.15.64:0/1503643345 wait complete. 2024-03-04T07:37:10.452 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:10.452 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:10.702 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:11.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:11.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.567+0000 7f176094b640 1 Processor -- start 2024-03-04T07:37:11.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.567+0000 7f176094b640 1 -- start start 2024-03-04T07:37:11.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f176094b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f175c158520 0x7f175c06d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f176094b640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f175c0aee10 0x7f175c0b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f176094b640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f175c071240 con 0x7f175c06dd60 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f176094b640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f175c071b70 con 0x7f175c158520 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f176094b640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f175c071520 con 0x7f175c0aee10 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f175c158520 0x7f175c06d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f175c158520 0x7f175c06d820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51910/0 (socket says 172.21.15.64:51910) 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f1753fff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f175c06dd60 0x7f175c0ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34594/0 (socket says 172.21.15.64:34594) 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175a59c640 1 -- 172.21.15.64:0/3543609344 learned_addr learned my addr 172.21.15.64:0/3543609344 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175ad9d640 1 --2- 172.21.15.64:0/3543609344 >> v2:172.21.15.66:3300/0 conn(0x7f175c0aee10 0x7f175c0b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175a59c640 1 -- 172.21.15.64:0/3543609344 >> v2:172.21.15.66:3300/0 conn(0x7f175c0aee10 msgr2=0x7f175c0b3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175a59c640 1 --2- 172.21.15.64:0/3543609344 >> v2:172.21.15.66:3300/0 conn(0x7f175c0aee10 0x7f175c0b3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175a59c640 1 -- 172.21.15.64:0/3543609344 >> v1:172.21.15.64:6789/0 conn(0x7f175c06dd60 legacy=0x7f175c0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:11.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.568+0000 7f175a59c640 1 -- 172.21.15.64:0/3543609344 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f175c071800 con 0x7f175c158520 2024-03-04T07:37:11.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.569+0000 7f175a59c640 1 --2- 172.21.15.64:0/3543609344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c06d820 secure :-1 s=READY pgs=1812 cs=0 l=1 rev1=1 crypto rx=0x7f175c0b2d50 tx=0x7f174c02b7e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9701c0ed8c3db397 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:11.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.569+0000 7f1759d9b640 1 -- 172.21.15.64:0/3543609344 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f174c037070 con 0x7f175c158520 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.569+0000 7f1759d9b640 1 -- 172.21.15.64:0/3543609344 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f174c004230 con 0x7f175c158520 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.569+0000 7f1759d9b640 1 -- 172.21.15.64:0/3543609344 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f174c003610 con 0x7f175c158520 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 -- 172.21.15.64:0/3543609344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 msgr2=0x7f175c06d820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 --2- 172.21.15.64:0/3543609344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c06d820 secure :-1 s=READY pgs=1812 cs=0 l=1 rev1=1 crypto rx=0x7f175c0b2d50 tx=0x7f174c02b7e0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 -- 172.21.15.64:0/3543609344 shutdown_connections 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 --2- 172.21.15.64:0/3543609344 >> v2:172.21.15.66:3300/0 conn(0x7f175c0aee10 0x7f175c0b3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 --2- 172.21.15.64:0/3543609344 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c06d820 unknown :-1 s=CLOSED pgs=1812 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 -- 172.21.15.64:0/3543609344 >> 172.21.15.64:0/3543609344 conn(0x7f175c066d00 msgr2=0x7f175c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 -- 172.21.15.64:0/3543609344 shutdown_connections 2024-03-04T07:37:11.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.570+0000 7f176094b640 1 -- 172.21.15.64:0/3543609344 wait complete. 2024-03-04T07:37:11.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.571+0000 7f176094b640 1 Processor -- start 2024-03-04T07:37:11.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.571+0000 7f176094b640 1 -- start start 2024-03-04T07:37:11.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f176094b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f175c06dd60 0x7f175c0ce800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:11.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f176094b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f175c0aee10 0x7f175c0ced40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:11.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f176094b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c145f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:11.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f176094b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f175c158a80 con 0x7f175c158520 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f176094b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f175c158900 con 0x7f175c0aee10 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f176094b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f175c0b3b80 con 0x7f175c06dd60 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c145f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c145f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51912/0 (socket says 172.21.15.64:51912) 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175a59c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f175c06dd60 0x7f175c0ce800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 -- 172.21.15.64:0/3806136679 learned_addr learned my addr 172.21.15.64:0/3806136679 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f1753fff640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f175c0aee10 0x7f175c0ced40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 -- 172.21.15.64:0/3806136679 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f175c06dd60 msgr2=0x7f175c0ce800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f175c06dd60 0x7f175c0ce800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 -- 172.21.15.64:0/3806136679 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f175c0aee10 msgr2=0x7f175c0ced40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f175c0aee10 0x7f175c0ced40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.572+0000 7f175ad9d640 1 -- 172.21.15.64:0/3806136679 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f175c0a9760 con 0x7f175c158520 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.573+0000 7f175ad9d640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c145f80 secure :-1 s=READY pgs=1813 cs=0 l=1 rev1=1 crypto rx=0x7f1754009e30 tx=0x7f175400c490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.573+0000 7f1752ffd640 1 -- 172.21.15.64:0/3806136679 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1754018020 con 0x7f175c158520 2024-03-04T07:37:11.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.573+0000 7f1752ffd640 1 -- 172.21.15.64:0/3806136679 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1754014480 con 0x7f175c158520 2024-03-04T07:37:11.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.573+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f175c0cf370 con 0x7f175c158520 2024-03-04T07:37:11.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.573+0000 7f1752ffd640 1 -- 172.21.15.64:0/3806136679 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1754014870 con 0x7f175c158520 2024-03-04T07:37:11.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.573+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f175c144090 con 0x7f175c158520 2024-03-04T07:37:11.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.574+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f175c0cf890 con 0x7f175c158520 2024-03-04T07:37:11.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.574+0000 7f1752ffd640 1 -- 172.21.15.64:0/3806136679 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1754005d10 con 0x7f175c158520 2024-03-04T07:37:11.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.575+0000 7f1752ffd640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f173c06b4d0 0x7f173c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:11.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.575+0000 7f1752ffd640 1 -- 172.21.15.64:0/3806136679 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f175408aed0 con 0x7f175c158520 2024-03-04T07:37:11.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.575+0000 7f175a59c640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f173c06b4d0 0x7f173c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:11.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.575+0000 7f175a59c640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f173c06b4d0 0x7f173c06d990 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7f174c034000 tx=0x7f174c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:11.580 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.579+0000 7f1752ffd640 1 -- 172.21.15.64:0/3806136679 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f175c0cf890 con 0x7f175c158520 2024-03-04T07:37:11.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.799+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f175c067ff0 con 0x7f175c158520 2024-03-04T07:37:11.800 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.799+0000 7f1752ffd640 1 -- 172.21.15.64:0/3806136679 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f1754022030 con 0x7f175c158520 2024-03-04T07:37:11.801 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:11.801 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:11.801 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.801+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f173c06b4d0 msgr2=0x7f173c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.801+0000 7f176094b640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f173c06b4d0 0x7f173c06d990 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7f174c034000 tx=0x7f174c008030 comp rx=0 tx=0).stop 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.801+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 msgr2=0x7f175c145f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.801+0000 7f176094b640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c145f80 secure :-1 s=READY pgs=1813 cs=0 l=1 rev1=1 crypto rx=0x7f1754009e30 tx=0x7f175400c490 comp rx=0 tx=0).stop 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 shutdown_connections 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f173c06b4d0 0x7f173c06d990 unknown :-1 s=CLOSED pgs=827 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f175c158520 0x7f175c145f80 unknown :-1 s=CLOSED pgs=1813 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f175c0aee10 0x7f175c0ced40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 --2- 172.21.15.64:0/3806136679 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f175c06dd60 0x7f175c0ce800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 >> 172.21.15.64:0/3806136679 conn(0x7f175c066d00 msgr2=0x7f175c1537c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 shutdown_connections 2024-03-04T07:37:11.803 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:11.802+0000 7f176094b640 1 -- 172.21.15.64:0/3806136679 wait complete. 2024-03-04T07:37:12.015 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:12.015 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:12.704 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:13.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:13.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.129+0000 7f7092302640 1 Processor -- start 2024-03-04T07:37:13.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.129+0000 7f7092302640 1 -- start start 2024-03-04T07:37:13.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f7092302640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f708c15e060 0x7f708c0ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:13.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f7092302640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f708c0d0dd0 con 0x7f708c0d0880 2024-03-04T07:37:13.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f7092302640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f708c0d10b0 con 0x7f708c06fe20 2024-03-04T07:37:13.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f7092302640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f708c0cc0e0 con 0x7f708c15e060 2024-03-04T07:37:13.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f709089f640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f708c0d0880 0x7f708c0ccbc0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:34604/0 (socket says 172.21.15.64:34604) 2024-03-04T07:37:13.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f708bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f708c15e060 0x7f708c0ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:13.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f709089f640 1 -- 172.21.15.64:0/2118227278 learned_addr learned my addr 172.21.15.64:0/2118227278 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:13.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f708bfff640 1 -- 172.21.15.64:0/2118227278 >> v1:172.21.15.66:6789/0 conn(0x7f708c06fe20 legacy=0x7f708c0ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:13.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f708affd640 1 -- 172.21.15.64:0/2118227278 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4044408163 0 0) 0x7f708c0d10b0 con 0x7f708c06fe20 2024-03-04T07:37:13.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f708bfff640 1 -- 172.21.15.64:0/2118227278 >> v1:172.21.15.64:6789/0 conn(0x7f708c0d0880 legacy=0x7f708c0ccbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:13.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.130+0000 7f708bfff640 1 -- 172.21.15.64:0/2118227278 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f708c0cc3c0 con 0x7f708c15e060 2024-03-04T07:37:13.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.131+0000 7f708bfff640 1 --2- 172.21.15.64:0/2118227278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0ce310 secure :-1 s=READY pgs=1815 cs=0 l=1 rev1=1 crypto rx=0x7f70800097a0 tx=0x7f708002b790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=28578ba675c53e4f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:13.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.131+0000 7f708affd640 1 -- 172.21.15.64:0/2118227278 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7080037070 con 0x7f708c15e060 2024-03-04T07:37:13.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.131+0000 7f708affd640 1 -- 172.21.15.64:0/2118227278 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7080030710 con 0x7f708c15e060 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.131+0000 7f708affd640 1 -- 172.21.15.64:0/2118227278 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7080030ae0 con 0x7f708c15e060 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.132+0000 7f7092302640 1 -- 172.21.15.64:0/2118227278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 msgr2=0x7f708c0ce310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.132+0000 7f7092302640 1 --2- 172.21.15.64:0/2118227278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0ce310 secure :-1 s=READY pgs=1815 cs=0 l=1 rev1=1 crypto rx=0x7f70800097a0 tx=0x7f708002b790 comp rx=0 tx=0).stop 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.132+0000 7f7092302640 1 -- 172.21.15.64:0/2118227278 shutdown_connections 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.132+0000 7f7092302640 1 --2- 172.21.15.64:0/2118227278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0ce310 unknown :-1 s=CLOSED pgs=1815 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.132+0000 7f7092302640 1 -- 172.21.15.64:0/2118227278 >> 172.21.15.64:0/2118227278 conn(0x7f708c0d1a90 msgr2=0x7f708c0d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.132+0000 7f7092302640 1 -- 172.21.15.64:0/2118227278 shutdown_connections 2024-03-04T07:37:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.132+0000 7f7092302640 1 -- 172.21.15.64:0/2118227278 wait complete. 2024-03-04T07:37:13.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.133+0000 7f7092302640 1 Processor -- start 2024-03-04T07:37:13.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.133+0000 7f7092302640 1 -- start start 2024-03-04T07:37:13.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f7092302640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f708c06fe20 0x7f708c159b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:13.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f7092302640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f708c0d0880 0x7f708c0b2d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:13.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f7092302640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0b2130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f7092302640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f708c0cef60 con 0x7f708c15e060 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f7092302640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f708c15e440 con 0x7f708c0d0880 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f7092302640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f708c0cf0e0 con 0x7f708c06fe20 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f709089f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0b2130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f709089f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0b2130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:51920/0 (socket says 172.21.15.64:51920) 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f709089f640 1 -- 172.21.15.64:0/2785337215 learned_addr learned my addr 172.21.15.64:0/2785337215 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f708bfff640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f708c06fe20 0x7f708c159b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.134+0000 7f708b7fe640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f708c0d0880 0x7f708c0b2d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f709089f640 1 -- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f708c06fe20 msgr2=0x7f708c159b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f709089f640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f708c06fe20 0x7f708c159b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f709089f640 1 -- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f708c0d0880 msgr2=0x7f708c0b2d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f709089f640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f708c0d0880 0x7f708c0b2d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f709089f640 1 -- 172.21.15.64:0/2785337215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f708c0caf20 con 0x7f708c15e060 2024-03-04T07:37:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f709089f640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0b2130 secure :-1 s=READY pgs=1816 cs=0 l=1 rev1=1 crypto rx=0x7f708400caf0 tx=0x7f70840020a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f7088ff9640 1 -- 172.21.15.64:0/2785337215 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7084018070 con 0x7f708c15e060 2024-03-04T07:37:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f7088ff9640 1 -- 172.21.15.64:0/2785337215 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f708400a700 con 0x7f708c15e060 2024-03-04T07:37:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f708c0cb200 con 0x7f708c15e060 2024-03-04T07:37:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f7088ff9640 1 -- 172.21.15.64:0/2785337215 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f708400ab30 con 0x7f708c15e060 2024-03-04T07:37:13.138 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.135+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f708c0cb7c0 con 0x7f708c15e060 2024-03-04T07:37:13.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.136+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f708c067ff0 con 0x7f708c15e060 2024-03-04T07:37:13.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.136+0000 7f7088ff9640 1 -- 172.21.15.64:0/2785337215 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f708401b070 con 0x7f708c15e060 2024-03-04T07:37:13.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.137+0000 7f7088ff9640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f707806b4d0 0x7f707806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:13.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.137+0000 7f7088ff9640 1 -- 172.21.15.64:0/2785337215 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f70840898b0 con 0x7f708c15e060 2024-03-04T07:37:13.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.137+0000 7f708bfff640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f707806b4d0 0x7f707806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:13.139 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.137+0000 7f708bfff640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f707806b4d0 0x7f707806d990 secure :-1 s=READY pgs=828 cs=0 l=1 rev1=1 crypto rx=0x7f7080034000 tx=0x7f708002f000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:13.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.141+0000 7f7088ff9640 1 -- 172.21.15.64:0/2785337215 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7084058130 con 0x7f708c15e060 2024-03-04T07:37:13.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.364+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f708c0b3250 con 0x7f708c15e060 2024-03-04T07:37:13.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.364+0000 7f7088ff9640 1 -- 172.21.15.64:0/2785337215 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f708405c1d0 con 0x7f708c15e060 2024-03-04T07:37:13.365 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:13.366 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:13.366 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:13.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f707806b4d0 msgr2=0x7f707806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:13.367 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f707806b4d0 0x7f707806d990 secure :-1 s=READY pgs=828 cs=0 l=1 rev1=1 crypto rx=0x7f7080034000 tx=0x7f708002f000 comp rx=0 tx=0).stop 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 msgr2=0x7f708c0b2130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0b2130 secure :-1 s=READY pgs=1816 cs=0 l=1 rev1=1 crypto rx=0x7f708400caf0 tx=0x7f70840020a0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 shutdown_connections 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f707806b4d0 0x7f707806d990 unknown :-1 s=CLOSED pgs=828 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f708c15e060 0x7f708c0b2130 unknown :-1 s=CLOSED pgs=1816 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f708c0d0880 0x7f708c0b2d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 --2- 172.21.15.64:0/2785337215 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f708c06fe20 0x7f708c159b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 >> 172.21.15.64:0/2785337215 conn(0x7f708c0d1a90 msgr2=0x7f708c15d2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 shutdown_connections 2024-03-04T07:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:13.366+0000 7f7092302640 1 -- 172.21.15.64:0/2785337215 wait complete. 2024-03-04T07:37:13.579 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 91, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-03-04T07:33:37.247681+0000', 'modified': '2024-03-04T07:36:06.905493+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}, 'max_mds': 1, 'in': [0, 1], 'up': {'mds_0': 8577, 'mds_1': 6728}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_6728': {'gid': 6728, 'name': 'b', 'rank': 1, 'incarnation': 83, 'state': 'up:stopping', 'state_seq': 5, 'addr': '172.21.15.66:6835/2326824208', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6833', 'nonce': 2326824208}, {'type': 'v1', 'addr': '172.21.15.66:6835', 'nonce': 2326824208}]}, 'join_fscid': -1, 'laggy_since': '2024-03-04T07:34:33.556697+0000', 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}, 'gid_8577': {'gid': 8577, 'name': 'd', 'rank': 0, 'incarnation': 78, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.66:6834/546842609', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.66:6832', 'nonce': 546842609}, {'type': 'v1', 'addr': '172.21.15.66:6834', 'nonce': 546842609}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments'}}}}, 'data_pools': [19], 'metadata_pool': 18, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 8577, 'qdb_cluster': [8577]} 2024-03-04T07:37:13.579 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_6728:up:stopping 2024-03-04T07:37:14.581 DEBUG:tasks.cephfs.filesystem:status = { "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "default_fscid": 9, "epoch": 91, "feature_flags": { "enable_multiple": true, "ever_enabled_multiple": true }, "filesystems": [ { "id": 9, "mdsmap": { "bal_rank_mask": "-1", "balancer": "", "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_7": "mds uses inline data", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "created": "2024-03-04T07:33:37.247681+0000", "damaged": [], "data_pools": [ 19 ], "enabled": true, "epoch": 91, "ever_allowed_features": 0, "explicitly_allowed_features": 0, "failed": [], "flags": 18, "flags_state": { "allow_multimds_snaps": true, "allow_snaps": true, "allow_standby_replay": false, "balance_automate": false, "joinable": true, "refuse_client_session": false, "refuse_standby_for_another_fs": false }, "fs_name": "cephfs", "in": [ 0, 1 ], "info": { "gid_6728": { "addr": "172.21.15.66:6835/2326824208", "addrs": { "addrvec": [ { "addr": "172.21.15.66:6833", "nonce": 2326824208, "type": "v2" }, { "addr": "172.21.15.66:6835", "nonce": 2326824208, "type": "v1" } ] }, "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_7": "mds uses inline data", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "export_targets": [], "features": 4540701547738038271, "flags": 0, "gid": 6728, "incarnation": 83, "join_fscid": -1, "laggy_since": "2024-03-04T07:34:33.556697+0000", "name": "b", "rank": 1, "state": "up:stopping", "state_seq": 5 }, "gid_8577": { "addr": "172.21.15.66:6834/546842609", "addrs": { "addrvec": [ { "addr": "172.21.15.66:6832", "nonce": 546842609, "type": "v2" }, { "addr": "172.21.15.66:6834", "nonce": 546842609, "type": "v1" } ] }, "compat": { "compat": {}, "incompat": { "feature_1": "base v0.20", "feature_10": "snaprealm v2", "feature_11": "minor log segments", "feature_2": "client writeable ranges", "feature_3": "default file layouts on dirs", "feature_4": "dir inode in separate object", "feature_5": "mds uses versioned encoding", "feature_6": "dirfrag is stored in omap", "feature_7": "mds uses inline data", "feature_8": "no anchor table", "feature_9": "file layout v2" }, "ro_compat": {} }, "export_targets": [], "features": 4540701547738038271, "flags": 0, "gid": 8577, "incarnation": 78, "join_fscid": -1, "name": "d", "rank": 0, "state": "up:active", "state_seq": 3 } }, "last_failure": 0, "last_failure_osd_epoch": 0, "max_file_size": 1099511627776, "max_mds": 1, "max_xattr_size": 65536, "metadata_pool": 18, "modified": "2024-03-04T07:36:06.905493+0000", "qdb_cluster": [ 8577 ], "qdb_leader": 8577, "required_client_features": {}, "root": 0, "session_autoclose": 300, "session_timeout": 60, "standby_count_wanted": 0, "stopped": [], "tableserver": 0, "up": { "mds_0": 8577, "mds_1": 6728 } } } ], "standbys": [] } 2024-03-04T07:37:14.581 INFO:teuthology.nuke.actions:Clearing teuthology firewall rules... 2024-03-04T07:37:14.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:37:14.630 DEBUG:teuthology.orchestra.run.smithi066:> sudo sh -c 'iptables-save | grep -v teuthology | iptables-restore' 2024-03-04T07:37:14.686 INFO:teuthology.nuke.actions:Cleared teuthology firewall rules. 2024-03-04T07:37:14.686 INFO:tasks.cephfs.mount:Terminating background process 2024-03-04T07:37:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14 stdin-killer INFO: expiration expected; waiting 5 seconds for command to complete 2024-03-04T07:37:14.706 INFO:teuthology.orchestra.run.smithi064.stderr:open_dir_background: keeping dir open 2024-03-04T07:37:14.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-04T07:37:14.718 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:37:14.718 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:14.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:37:14.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.928+0000 7fc358629640 1 Processor -- start 2024-03-04T07:37:14.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.928+0000 7fc358629640 1 -- start start 2024-03-04T07:37:14.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.928+0000 7fc358629640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3500d0880 0x7fc350168c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:14.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc358629640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc3500cf130 con 0x7fc35015e060 2024-03-04T07:37:14.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc358629640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc3500d0dd0 con 0x7fc35006fe20 2024-03-04T07:37:14.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc358629640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc3500d10b0 con 0x7fc3500d0880 2024-03-04T07:37:14.930 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc3563c5640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc35015e060 0x7fc3500ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42552/0 (socket says 172.21.15.64:42552) 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc3563c5640 1 -- 172.21.15.64:0/50231292 learned_addr learned my addr 172.21.15.64:0/50231292 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc356bc6640 1 --2- 172.21.15.64:0/50231292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3500d0880 0x7fc350168c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc3553c3640 1 -- 172.21.15.64:0/50231292 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2537879014 0 0) 0x7fc3500d0dd0 con 0x7fc35006fe20 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc3553c3640 1 -- 172.21.15.64:0/50231292 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc33c003520 con 0x7fc35006fe20 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc3553c3640 1 -- 172.21.15.64:0/50231292 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2717095980 0 0) 0x7fc3500cf130 con 0x7fc35015e060 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc3553c3640 1 -- 172.21.15.64:0/50231292 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc3500d0dd0 con 0x7fc35015e060 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc356bc6640 1 -- 172.21.15.64:0/50231292 >> v1:172.21.15.66:6789/0 conn(0x7fc35006fe20 legacy=0x7fc3500cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc356bc6640 1 -- 172.21.15.64:0/50231292 >> v1:172.21.15.64:6789/0 conn(0x7fc35015e060 legacy=0x7fc3500ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:14.931 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.929+0000 7fc356bc6640 1 -- 172.21.15.64:0/50231292 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3500cc0e0 con 0x7fc3500d0880 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.930+0000 7fc356bc6640 1 --2- 172.21.15.64:0/50231292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3500d0880 0x7fc350168c20 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fc348009dc0 tx=0x7fc34802bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=36be7595493a392b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.930+0000 7fc3553c3640 1 -- 172.21.15.64:0/50231292 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc348039070 con 0x7fc3500d0880 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.930+0000 7fc3553c3640 1 -- 172.21.15.64:0/50231292 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc34802ccd0 con 0x7fc3500d0880 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.930+0000 7fc3553c3640 1 -- 172.21.15.64:0/50231292 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc348034660 con 0x7fc3500d0880 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.931+0000 7fc358629640 1 -- 172.21.15.64:0/50231292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3500d0880 msgr2=0x7fc350168c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.931+0000 7fc358629640 1 --2- 172.21.15.64:0/50231292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3500d0880 0x7fc350168c20 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fc348009dc0 tx=0x7fc34802bf20 comp rx=0 tx=0).stop 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.931+0000 7fc358629640 1 -- 172.21.15.64:0/50231292 shutdown_connections 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.931+0000 7fc358629640 1 --2- 172.21.15.64:0/50231292 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc3500d0880 0x7fc350168c20 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.931+0000 7fc358629640 1 -- 172.21.15.64:0/50231292 >> 172.21.15.64:0/50231292 conn(0x7fc3500d1a90 msgr2=0x7fc3500d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.931+0000 7fc358629640 1 -- 172.21.15.64:0/50231292 shutdown_connections 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.931+0000 7fc358629640 1 -- 172.21.15.64:0/50231292 wait complete. 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.932+0000 7fc358629640 1 Processor -- start 2024-03-04T07:37:14.933 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.932+0000 7fc358629640 1 -- start start 2024-03-04T07:37:14.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc358629640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc35006fe20 0x7fc3500b2cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:14.935 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc358629640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3500d0880 0x7fc3501f4c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc358629640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc35015e060 0x7fc3501f51a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc358629640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc35015e440 con 0x7fc3500d0880 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc358629640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3500cc6a0 con 0x7fc35006fe20 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc358629640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc3500cdbd0 con 0x7fc35015e060 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc355bc4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3500d0880 0x7fc3501f4c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc3563c5640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc35006fe20 0x7fc3500b2cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc356bc6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc35015e060 0x7fc3501f51a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc355bc4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3500d0880 0x7fc3501f4c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42198/0 (socket says 172.21.15.64:42198) 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc356bc6640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc35015e060 0x7fc3501f51a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3301/0 says I am v2:172.21.15.64:57928/0 (socket says 172.21.15.64:57928) 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc355bc4640 1 -- 172.21.15.64:0/1665341145 learned_addr learned my addr 172.21.15.64:0/1665341145 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc3563c5640 1 -- 172.21.15.64:0/1665341145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc35015e060 msgr2=0x7fc3501f51a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc3563c5640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc35015e060 0x7fc3501f51a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:14.936 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc3563c5640 1 -- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3500d0880 msgr2=0x7fc3501f4c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc3563c5640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3500d0880 0x7fc3501f4c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc3563c5640 1 -- 172.21.15.64:0/1665341145 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3500c5a60 con 0x7fc35006fe20 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.933+0000 7fc3563c5640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc35006fe20 0x7fc3500b2cc0 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fc34c00c3c0 tx=0x7fc34c001e70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.934+0000 7fc346ffd640 1 -- 172.21.15.64:0/1665341145 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc34c017070 con 0x7fc35006fe20 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.934+0000 7fc346ffd640 1 -- 172.21.15.64:0/1665341145 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc34c005ce0 con 0x7fc35006fe20 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.934+0000 7fc346ffd640 1 -- 172.21.15.64:0/1665341145 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc34c0072d0 con 0x7fc35006fe20 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.934+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3500c5d40 con 0x7fc35006fe20 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.934+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3500c6300 con 0x7fc35006fe20 2024-03-04T07:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.935+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc350067ff0 con 0x7fc35006fe20 2024-03-04T07:37:14.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.935+0000 7fc346ffd640 1 -- 172.21.15.64:0/1665341145 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc34c01a070 con 0x7fc35006fe20 2024-03-04T07:37:14.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.935+0000 7fc346ffd640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc33406b500 0x7fc33406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:14.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.936+0000 7fc346ffd640 1 -- 172.21.15.64:0/1665341145 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc34c089460 con 0x7fc35006fe20 2024-03-04T07:37:14.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.936+0000 7fc355bc4640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc33406b500 0x7fc33406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:14.938 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.936+0000 7fc355bc4640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc33406b500 0x7fc33406d9c0 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7fc35006cea0 tx=0x7fc340009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:14.941 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:14.940+0000 7fc346ffd640 1 -- 172.21.15.64:0/1665341145 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc34c057c50 con 0x7fc35006fe20 2024-03-04T07:37:15.124 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.123+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc3500b04f0 con 0x7fc35006fe20 2024-03-04T07:37:15.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.126+0000 7fc346ffd640 1 -- 172.21.15.64:0/1665341145 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v127) v1 ==== 74+0+16673 (secure 0 0 0) 0x7fc34c05bcf0 con 0x7fc35006fe20 2024-03-04T07:37:15.126 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:15.127 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":127,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:34:36.563800+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:33:34.914683+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-04T07:33:36.218855+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"123","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.25,"score_stable":1.25,"optimal_score":1,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"19.25","mappings":[{"from":2,"to":0}]},{"pgid":"19.3f","mappings":[{"from":2,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.127+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc33406b500 msgr2=0x7fc33406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.127+0000 7fc358629640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc33406b500 0x7fc33406d9c0 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7fc35006cea0 tx=0x7fc340009040 comp rx=0 tx=0).stop 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.127+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc35006fe20 msgr2=0x7fc3500b2cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.127+0000 7fc358629640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc35006fe20 0x7fc3500b2cc0 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7fc34c00c3c0 tx=0x7fc34c001e70 comp rx=0 tx=0).stop 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 shutdown_connections 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc33406b500 0x7fc33406d9c0 unknown :-1 s=CLOSED pgs=829 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc35015e060 0x7fc3501f51a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc3500d0880 0x7fc3501f4c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 --2- 172.21.15.64:0/1665341145 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc35006fe20 0x7fc3500b2cc0 unknown :-1 s=CLOSED pgs=676 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 >> 172.21.15.64:0/1665341145 conn(0x7fc3500d1a90 msgr2=0x7fc3500d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 shutdown_connections 2024-03-04T07:37:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.128+0000 7fc358629640 1 -- 172.21.15.64:0/1665341145 wait complete. 2024-03-04T07:37:15.140 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 18, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:33:34.914683+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '122', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 19, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:33:36.218855+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '123', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.25, 'score_stable': 1.25, 'optimal_score': 1, 'raw_score_acting': 1.25, 'raw_score_stable': 1.25, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:37:15.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:37:15.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.243+0000 7f1049535640 1 Processor -- start 2024-03-04T07:37:15.244 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.243+0000 7f1049535640 1 -- start start 2024-03-04T07:37:15.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f1049535640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f104415dd50 0x7f10440ce9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f1049535640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1044072680 0x7f10440d2760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f1049535640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1044072cf0 con 0x7f10440cef20 2024-03-04T07:37:15.245 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f1049535640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1044072ad0 con 0x7f1044072680 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f1049535640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10440cf380 con 0x7f104415dd50 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f10437fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f1044072680 0x7f10440d2760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f10427fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f10440cef20 0x7f1044071f70 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42560/0 (socket says 172.21.15.64:42560) 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f10427fc640 1 -- 172.21.15.64:0/3347219618 learned_addr learned my addr 172.21.15.64:0/3347219618 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f1042ffd640 1 --2- 172.21.15.64:0/3347219618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f104415dd50 0x7f10440ce9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f10437fe640 1 -- 172.21.15.64:0/3347219618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f104415dd50 msgr2=0x7f10440ce9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f10437fe640 1 --2- 172.21.15.64:0/3347219618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f104415dd50 0x7f10440ce9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f10437fe640 1 -- 172.21.15.64:0/3347219618 >> v1:172.21.15.64:6789/0 conn(0x7f10440cef20 legacy=0x7f1044071f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.246 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.244+0000 7f10437fe640 1 -- 172.21.15.64:0/3347219618 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1044072fd0 con 0x7f1044072680 2024-03-04T07:37:15.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.245+0000 7f10437fe640 1 --2- 172.21.15.64:0/3347219618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10440d2760 secure :-1 s=READY pgs=1819 cs=0 l=1 rev1=1 crypto rx=0x7f1034009d90 tx=0x7f103402be10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9db431eccbcbdbc server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.245+0000 7f1041ffb640 1 -- 172.21.15.64:0/3347219618 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1034039070 con 0x7f1044072680 2024-03-04T07:37:15.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.245+0000 7f1041ffb640 1 -- 172.21.15.64:0/3347219618 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f103402cbc0 con 0x7f1044072680 2024-03-04T07:37:15.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.245+0000 7f1041ffb640 1 -- 172.21.15.64:0/3347219618 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1034034440 con 0x7f1044072680 2024-03-04T07:37:15.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 -- 172.21.15.64:0/3347219618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 msgr2=0x7f10440d2760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 --2- 172.21.15.64:0/3347219618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10440d2760 secure :-1 s=READY pgs=1819 cs=0 l=1 rev1=1 crypto rx=0x7f1034009d90 tx=0x7f103402be10 comp rx=0 tx=0).stop 2024-03-04T07:37:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 -- 172.21.15.64:0/3347219618 shutdown_connections 2024-03-04T07:37:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 --2- 172.21.15.64:0/3347219618 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10440d2760 unknown :-1 s=CLOSED pgs=1819 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 --2- 172.21.15.64:0/3347219618 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f104415dd50 0x7f10440ce9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 -- 172.21.15.64:0/3347219618 >> 172.21.15.64:0/3347219618 conn(0x7f10440d3ef0 msgr2=0x7f10440d4300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 -- 172.21.15.64:0/3347219618 shutdown_connections 2024-03-04T07:37:15.248 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.246+0000 7f1049535640 1 -- 172.21.15.64:0/3347219618 wait complete. 2024-03-04T07:37:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.247+0000 7f1049535640 1 Processor -- start 2024-03-04T07:37:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.247+0000 7f1049535640 1 -- start start 2024-03-04T07:37:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1049535640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10441f4ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1049535640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10440cef20 0x7f10440c9ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1049535640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f104415dd50 0x7f10440cd3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1049535640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f104415e130 con 0x7f1044072680 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1049535640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10440d2240 con 0x7f104415dd50 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1049535640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f104415e2b0 con 0x7f10440cef20 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10441f4ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10441f4ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42222/0 (socket says 172.21.15.64:42222) 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 -- 172.21.15.64:0/1250578963 learned_addr learned my addr 172.21.15.64:0/1250578963 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f10437fe640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f104415dd50 0x7f10440cd3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f10427fc640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10440cef20 0x7f10440c9ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 -- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10440cef20 msgr2=0x7f10440c9ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10440cef20 0x7f10440c9ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 -- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f104415dd50 msgr2=0x7f10440cd3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f104415dd50 0x7f10440cd3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 -- 172.21.15.64:0/1250578963 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10440cd910 con 0x7f1044072680 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f10437fe640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f104415dd50 0x7f10440cd3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:15.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.248+0000 7f1042ffd640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10441f4ce0 secure :-1 s=READY pgs=1820 cs=0 l=1 rev1=1 crypto rx=0x7f103c00e8a0 tx=0x7f103c00ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.249+0000 7f102b7fe640 1 -- 172.21.15.64:0/1250578963 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f103c017070 con 0x7f1044072680 2024-03-04T07:37:15.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.249+0000 7f102b7fe640 1 -- 172.21.15.64:0/1250578963 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f103c00c940 con 0x7f1044072680 2024-03-04T07:37:15.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.249+0000 7f102b7fe640 1 -- 172.21.15.64:0/1250578963 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f103c00cd30 con 0x7f1044072680 2024-03-04T07:37:15.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.249+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10441e7720 con 0x7f1044072680 2024-03-04T07:37:15.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.249+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10441e6c40 con 0x7f1044072680 2024-03-04T07:37:15.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.250+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f104406cdd0 con 0x7f1044072680 2024-03-04T07:37:15.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.250+0000 7f102b7fe640 1 -- 172.21.15.64:0/1250578963 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f103c01a070 con 0x7f1044072680 2024-03-04T07:37:15.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.251+0000 7f102b7fe640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f103006b4d0 0x7f103006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.251+0000 7f10427fc640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f103006b4d0 0x7f103006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.251+0000 7f102b7fe640 1 -- 172.21.15.64:0/1250578963 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f103c089ff0 con 0x7f1044072680 2024-03-04T07:37:15.253 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.251+0000 7f10427fc640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f103006b4d0 0x7f103006d990 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f10440724a0 tx=0x7f1038007e80 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.255+0000 7f102b7fe640 1 -- 172.21.15.64:0/1250578963 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f103c058870 con 0x7f1044072680 2024-03-04T07:37:15.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.437+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f104404f410 con 0x7f1044072680 2024-03-04T07:37:15.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.437+0000 7f102b7fe640 1 -- 172.21.15.64:0/1250578963 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v127) v1 ==== 93+0+10 (secure 0 0 0) 0x7f103c05c910 con 0x7f1044072680 2024-03-04T07:37:15.438 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:37:15.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f103006b4d0 msgr2=0x7f103006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f103006b4d0 0x7f103006d990 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f10440724a0 tx=0x7f1038007e80 comp rx=0 tx=0).stop 2024-03-04T07:37:15.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 msgr2=0x7f10441f4ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10441f4ce0 secure :-1 s=READY pgs=1820 cs=0 l=1 rev1=1 crypto rx=0x7f103c00e8a0 tx=0x7f103c00ed60 comp rx=0 tx=0).stop 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 shutdown_connections 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f103006b4d0 0x7f103006d990 unknown :-1 s=CLOSED pgs=830 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f104415dd50 0x7f10440cd3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f10440cef20 0x7f10440c9ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 --2- 172.21.15.64:0/1250578963 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1044072680 0x7f10441f4ce0 unknown :-1 s=CLOSED pgs=1820 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 >> 172.21.15.64:0/1250578963 conn(0x7f10440d3ef0 msgr2=0x7f104415c020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 shutdown_connections 2024-03-04T07:37:15.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.439+0000 7f1049535640 1 -- 172.21.15.64:0/1250578963 wait complete. 2024-03-04T07:37:15.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:37:15.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.554+0000 7fd1b695c640 1 Processor -- start 2024-03-04T07:37:15.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.554+0000 7fd1b695c640 1 -- start start 2024-03-04T07:37:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1b695c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd1b00d07c0 0x7fd1b01624c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1b695c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00cf9f0 0x7fd1b015d660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1b695c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fd1b006d4d0 con 0x7fd1b0072a30 2024-03-04T07:37:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1b695c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fd1b006dba0 con 0x7fd1b00d07c0 2024-03-04T07:37:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1b695c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1b00b4950 con 0x7fd1b00cf9f0 2024-03-04T07:37:15.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1af7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd1b00d07c0 0x7fd1b01624c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1af7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fd1b00d07c0 0x7fd1b01624c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42238/0 (socket says 172.21.15.64:42238) 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1af7fe640 1 -- 172.21.15.64:0/947256058 learned_addr learned my addr 172.21.15.64:0/947256058 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1b4ef9640 1 --2- 172.21.15.64:0/947256058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00cf9f0 0x7fd1b015d660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1aeffd640 1 -- 172.21.15.64:0/947256058 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3534794273 0 0) 0x7fd1b006d4d0 con 0x7fd1b0072a30 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1af7fe640 1 -- 172.21.15.64:0/947256058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00cf9f0 msgr2=0x7fd1b015d660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1af7fe640 1 --2- 172.21.15.64:0/947256058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00cf9f0 0x7fd1b015d660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1af7fe640 1 -- 172.21.15.64:0/947256058 >> v1:172.21.15.66:6789/0 conn(0x7fd1b0072a30 legacy=0x7fd1b00d00b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.555+0000 7fd1af7fe640 1 -- 172.21.15.64:0/947256058 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1b006d7b0 con 0x7fd1b00d07c0 2024-03-04T07:37:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.556+0000 7fd1af7fe640 1 --2- 172.21.15.64:0/947256058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b00d07c0 0x7fd1b01624c0 secure :-1 s=READY pgs=1821 cs=0 l=1 rev1=1 crypto rx=0x7fd1a0009870 tx=0x7fd1a002b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=df3e3eef62363536 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.556+0000 7fd1aeffd640 1 -- 172.21.15.64:0/947256058 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1a00317c0 con 0x7fd1b00d07c0 2024-03-04T07:37:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.556+0000 7fd1aeffd640 1 -- 172.21.15.64:0/947256058 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd1a0037070 con 0x7fd1b00d07c0 2024-03-04T07:37:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.556+0000 7fd1aeffd640 1 -- 172.21.15.64:0/947256058 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1a0031b80 con 0x7fd1b00d07c0 2024-03-04T07:37:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 -- 172.21.15.64:0/947256058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b00d07c0 msgr2=0x7fd1b01624c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 --2- 172.21.15.64:0/947256058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b00d07c0 0x7fd1b01624c0 secure :-1 s=READY pgs=1821 cs=0 l=1 rev1=1 crypto rx=0x7fd1a0009870 tx=0x7fd1a002b860 comp rx=0 tx=0).stop 2024-03-04T07:37:15.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 -- 172.21.15.64:0/947256058 shutdown_connections 2024-03-04T07:37:15.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 --2- 172.21.15.64:0/947256058 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00cf9f0 0x7fd1b015d660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 --2- 172.21.15.64:0/947256058 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b00d07c0 0x7fd1b01624c0 unknown :-1 s=CLOSED pgs=1821 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 -- 172.21.15.64:0/947256058 >> 172.21.15.64:0/947256058 conn(0x7fd1b006af20 msgr2=0x7fd1b006b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:15.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 -- 172.21.15.64:0/947256058 shutdown_connections 2024-03-04T07:37:15.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.557+0000 7fd1b695c640 1 -- 172.21.15.64:0/947256058 wait complete. 2024-03-04T07:37:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.558+0000 7fd1b695c640 1 Processor -- start 2024-03-04T07:37:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.558+0000 7fd1b695c640 1 -- start start 2024-03-04T07:37:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1b695c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b0072a30 0x7fd1b01f5200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1b695c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1b00cf9f0 0x7fd1b0158650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1b695c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00d07c0 0x7fd1b0159a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1b695c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1b0072e10 con 0x7fd1b0072a30 2024-03-04T07:37:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1b695c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1b0071130 con 0x7fd1b00cf9f0 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1b695c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1b0072f90 con 0x7fd1b00d07c0 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b0072a30 0x7fd1b01f5200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b0072a30 0x7fd1b01f5200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42242/0 (socket says 172.21.15.64:42242) 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 -- 172.21.15.64:0/3616062278 learned_addr learned my addr 172.21.15.64:0/3616062278 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1af7fe640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1b00cf9f0 0x7fd1b0158650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1b4ef9640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00d07c0 0x7fd1b0159a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 -- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00d07c0 msgr2=0x7fd1b0159a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00d07c0 0x7fd1b0159a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 -- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1b00cf9f0 msgr2=0x7fd1b0158650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1b00cf9f0 0x7fd1b0158650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 -- 172.21.15.64:0/3616062278 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1b0159060 con 0x7fd1b0072a30 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1affff640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b0072a30 0x7fd1b01f5200 secure :-1 s=READY pgs=1822 cs=0 l=1 rev1=1 crypto rx=0x7fd1a400c410 tx=0x7fd1a400c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.559+0000 7fd1acff9640 1 -- 172.21.15.64:0/3616062278 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1a4017070 con 0x7fd1b0072a30 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.560+0000 7fd1acff9640 1 -- 172.21.15.64:0/3616062278 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fd1a40035f0 con 0x7fd1b0072a30 2024-03-04T07:37:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.560+0000 7fd1acff9640 1 -- 172.21.15.64:0/3616062278 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1a40039e0 con 0x7fd1b0072a30 2024-03-04T07:37:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.560+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd1b015c3e0 con 0x7fd1b0072a30 2024-03-04T07:37:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.560+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd1b0071ec0 con 0x7fd1b0072a30 2024-03-04T07:37:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.561+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd1b009e320 con 0x7fd1b0072a30 2024-03-04T07:37:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.561+0000 7fd1acff9640 1 -- 172.21.15.64:0/3616062278 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fd1a401a070 con 0x7fd1b0072a30 2024-03-04T07:37:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.561+0000 7fd1acff9640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd19c06b4d0 0x7fd19c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.561+0000 7fd1acff9640 1 -- 172.21.15.64:0/3616062278 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fd1a4089600 con 0x7fd1b0072a30 2024-03-04T07:37:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.561+0000 7fd1af7fe640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd19c06b4d0 0x7fd19c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.562+0000 7fd1af7fe640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd19c06b4d0 0x7fd19c06d990 secure :-1 s=READY pgs=831 cs=0 l=1 rev1=1 crypto rx=0x7fd1a0036000 tx=0x7fd1a0008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.566+0000 7fd1acff9640 1 -- 172.21.15.64:0/3616062278 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fd1a4057df0 con 0x7fd1b0072a30 2024-03-04T07:37:15.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.746+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fd1b00add80 con 0x7fd1b0072a30 2024-03-04T07:37:15.747 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.746+0000 7fd1acff9640 1 -- 172.21.15.64:0/3616062278 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v127) v1 ==== 104+0+11 (secure 0 0 0) 0x7fd1a405be90 con 0x7fd1b0072a30 2024-03-04T07:37:15.747 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd19c06b4d0 msgr2=0x7fd19c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd19c06b4d0 0x7fd19c06d990 secure :-1 s=READY pgs=831 cs=0 l=1 rev1=1 crypto rx=0x7fd1a0036000 tx=0x7fd1a0008030 comp rx=0 tx=0).stop 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b0072a30 msgr2=0x7fd1b01f5200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b0072a30 0x7fd1b01f5200 secure :-1 s=READY pgs=1822 cs=0 l=1 rev1=1 crypto rx=0x7fd1a400c410 tx=0x7fd1a400c8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 shutdown_connections 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fd19c06b4d0 0x7fd19c06d990 unknown :-1 s=CLOSED pgs=831 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fd1b00d07c0 0x7fd1b0159a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fd1b00cf9f0 0x7fd1b0158650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 --2- 172.21.15.64:0/3616062278 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fd1b0072a30 0x7fd1b01f5200 unknown :-1 s=CLOSED pgs=1822 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 >> 172.21.15.64:0/3616062278 conn(0x7fd1b006af20 msgr2=0x7fd1b00ae970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 shutdown_connections 2024-03-04T07:37:15.750 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.749+0000 7fd1b695c640 1 -- 172.21.15.64:0/3616062278 wait complete. 2024-03-04T07:37:15.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:37:15.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.864+0000 7f7212c64640 1 Processor -- start 2024-03-04T07:37:15.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.864+0000 7f7212c64640 1 -- start start 2024-03-04T07:37:15.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7212c64640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c0b41f0 0x7f720c06bbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7212c64640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f720c06c130 0x7f720c0cdac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7212c64640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f720c15e6d0 con 0x7f720c15e2f0 2024-03-04T07:37:15.866 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7212c64640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f720c0d0a10 con 0x7f720c06c130 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7212c64640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f720c0ccd00 con 0x7f720c0b41f0 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f720bfff640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c0b41f0 0x7f720c06bbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7210a00640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f720c15e2f0 0x7f720c06b4e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49566/0 (socket says 172.21.15.64:49566) 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7210a00640 1 -- 172.21.15.64:0/385831911 learned_addr learned my addr 172.21.15.64:0/385831911 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f7211201640 1 --2- 172.21.15.64:0/385831911 >> v2:172.21.15.64:3300/0 conn(0x7f720c06c130 0x7f720c0cdac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f720bfff640 1 -- 172.21.15.64:0/385831911 >> v2:172.21.15.64:3300/0 conn(0x7f720c06c130 msgr2=0x7f720c0cdac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f720bfff640 1 --2- 172.21.15.64:0/385831911 >> v2:172.21.15.64:3300/0 conn(0x7f720c06c130 0x7f720c0cdac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f720bfff640 1 -- 172.21.15.64:0/385831911 >> v1:172.21.15.66:6789/0 conn(0x7f720c15e2f0 legacy=0x7f720c06b4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f720b7fe640 1 -- 172.21.15.64:0/385831911 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3367729816 0 0) 0x7f720c15e6d0 con 0x7f720c15e2f0 2024-03-04T07:37:15.867 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.865+0000 7f720bfff640 1 -- 172.21.15.64:0/385831911 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f720c0ccfe0 con 0x7f720c0b41f0 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.866+0000 7f720bfff640 1 --2- 172.21.15.64:0/385831911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c0b41f0 0x7f720c06bbf0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f72000097a0 tx=0x7f720002b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f59a2102dd75e1ee server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.866+0000 7f720b7fe640 1 -- 172.21.15.64:0/385831911 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72000315a0 con 0x7f720c0b41f0 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.866+0000 7f720b7fe640 1 -- 172.21.15.64:0/385831911 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7200037070 con 0x7f720c0b41f0 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.866+0000 7f720b7fe640 1 -- 172.21.15.64:0/385831911 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7200031960 con 0x7f720c0b41f0 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 -- 172.21.15.64:0/385831911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c0b41f0 msgr2=0x7f720c06bbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 --2- 172.21.15.64:0/385831911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c0b41f0 0x7f720c06bbf0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f72000097a0 tx=0x7f720002b840 comp rx=0 tx=0).stop 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 -- 172.21.15.64:0/385831911 shutdown_connections 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 --2- 172.21.15.64:0/385831911 >> v2:172.21.15.64:3300/0 conn(0x7f720c06c130 0x7f720c0cdac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 --2- 172.21.15.64:0/385831911 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c0b41f0 0x7f720c06bbf0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.869 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 -- 172.21.15.64:0/385831911 >> 172.21.15.64:0/385831911 conn(0x7f720c0b36d0 msgr2=0x7f720c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:15.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 -- 172.21.15.64:0/385831911 shutdown_connections 2024-03-04T07:37:15.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.867+0000 7f7212c64640 1 -- 172.21.15.64:0/385831911 wait complete. 2024-03-04T07:37:15.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.868+0000 7f7212c64640 1 Processor -- start 2024-03-04T07:37:15.870 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.868+0000 7f7212c64640 1 -- start start 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7212c64640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f720c06c130 0x7f720c1f6bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7212c64640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f720c0b41f0 0x7f720c06eb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7212c64640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c15e2f0 0x7f720c06f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7212c64640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f720c0ce000 con 0x7f720c06c130 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7212c64640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f720c0cd5a0 con 0x7f720c0b41f0 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7212c64640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f720c0ce180 con 0x7f720c15e2f0 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f720c06c130 0x7f720c1f6bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f720c06c130 0x7f720c1f6bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42268/0 (socket says 172.21.15.64:42268) 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 -- 172.21.15.64:0/1569007897 learned_addr learned my addr 172.21.15.64:0/1569007897 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7211201640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c15e2f0 0x7f720c06f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f720bfff640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f720c0b41f0 0x7f720c06eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 -- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c15e2f0 msgr2=0x7f720c06f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c15e2f0 0x7f720c06f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 -- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f720c0b41f0 msgr2=0x7f720c06eb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f720c0b41f0 0x7f720c06eb90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.869+0000 7f7210a00640 1 -- 172.21.15.64:0/1569007897 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f720c06f770 con 0x7f720c06c130 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.870+0000 7f7210a00640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f720c06c130 0x7f720c1f6bc0 secure :-1 s=READY pgs=1823 cs=0 l=1 rev1=1 crypto rx=0x7f71fc00c7f0 tx=0x7f71fc00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.870+0000 7f72097fa640 1 -- 172.21.15.64:0/1569007897 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71fc018070 con 0x7f720c06c130 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.870+0000 7f72097fa640 1 -- 172.21.15.64:0/1569007897 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f71fc00a640 con 0x7f720c06c130 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.870+0000 7f72097fa640 1 -- 172.21.15.64:0/1569007897 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71fc00aa30 con 0x7f720c06c130 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.870+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f720c06fa50 con 0x7f720c06c130 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.870+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f720c070010 con 0x7f720c06c130 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.871+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f720c09fd30 con 0x7f720c06c130 2024-03-04T07:37:15.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.871+0000 7f72097fa640 1 -- 172.21.15.64:0/1569007897 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f71fc00abd0 con 0x7f720c06c130 2024-03-04T07:37:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.872+0000 7f72097fa640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f71f806b4d0 0x7f71f806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.872+0000 7f720bfff640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f71f806b4d0 0x7f71f806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.872+0000 7f72097fa640 1 -- 172.21.15.64:0/1569007897 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f71fc0897e0 con 0x7f720c06c130 2024-03-04T07:37:15.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.872+0000 7f720bfff640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f71f806b4d0 0x7f71f806d990 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f7200036000 tx=0x7f7200008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:15.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:15.876+0000 7f72097fa640 1 -- 172.21.15.64:0/1569007897 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f71fc057fd0 con 0x7f720c06c130 2024-03-04T07:37:16.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.058+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f720c04f410 con 0x7f720c06c130 2024-03-04T07:37:16.059 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.058+0000 7f72097fa640 1 -- 172.21.15.64:0/1569007897 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v127) v1 ==== 100+0+11 (secure 0 0 0) 0x7f71fc05c070 con 0x7f720c06c130 2024-03-04T07:37:16.059 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:37:16.061 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f71f806b4d0 msgr2=0x7f71f806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f71f806b4d0 0x7f71f806d990 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f7200036000 tx=0x7f7200008030 comp rx=0 tx=0).stop 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f720c06c130 msgr2=0x7f720c1f6bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f720c06c130 0x7f720c1f6bc0 secure :-1 s=READY pgs=1823 cs=0 l=1 rev1=1 crypto rx=0x7f71fc00c7f0 tx=0x7f71fc00ccb0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 shutdown_connections 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f71f806b4d0 0x7f71f806d990 unknown :-1 s=CLOSED pgs=832 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f720c15e2f0 0x7f720c06f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f720c0b41f0 0x7f720c06eb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 --2- 172.21.15.64:0/1569007897 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f720c06c130 0x7f720c1f6bc0 unknown :-1 s=CLOSED pgs=1823 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 >> 172.21.15.64:0/1569007897 conn(0x7f720c0b36d0 msgr2=0x7f720c0b3ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 shutdown_connections 2024-03-04T07:37:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.060+0000 7f7212c64640 1 -- 172.21.15.64:0/1569007897 wait complete. 2024-03-04T07:37:16.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-04T07:37:16.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.174+0000 7f54b25d7640 1 Processor -- start 2024-03-04T07:37:16.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.174+0000 7f54b25d7640 1 -- start start 2024-03-04T07:37:16.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b25d7640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f54ac072660 0x7f54ac15ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b25d7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f54ac0b55d0 0x7f54ac071980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b25d7640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f54ac0b4160 con 0x7f54ac06e700 2024-03-04T07:37:16.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b25d7640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f54ac0b5b40 con 0x7f54ac0b55d0 2024-03-04T07:37:16.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b25d7640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f54ac0b3ba0 con 0x7f54ac072660 2024-03-04T07:37:16.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b0b74640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f54ac0b55d0 0x7f54ac071980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.177 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54abfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f54ac06e700 0x7f54ac06d9e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42576/0 (socket says 172.21.15.64:42576) 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54abfff640 1 -- 172.21.15.64:0/2975442947 learned_addr learned my addr 172.21.15.64:0/2975442947 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b0b74640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f54ac0b55d0 0x7f54ac071980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42280/0 (socket says 172.21.15.64:42280) 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54ab7fe640 1 --2- 172.21.15.64:0/2975442947 >> v2:172.21.15.66:3300/0 conn(0x7f54ac072660 0x7f54ac15ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54aaffd640 1 -- 172.21.15.64:0/2975442947 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 513042170 0 0) 0x7f54ac0b4160 con 0x7f54ac06e700 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54aaffd640 1 -- 172.21.15.64:0/2975442947 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f5498003520 con 0x7f54ac06e700 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b0b74640 1 -- 172.21.15.64:0/2975442947 >> v2:172.21.15.66:3300/0 conn(0x7f54ac072660 msgr2=0x7f54ac15ef80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b0b74640 1 --2- 172.21.15.64:0/2975442947 >> v2:172.21.15.66:3300/0 conn(0x7f54ac072660 0x7f54ac15ef80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b0b74640 1 -- 172.21.15.64:0/2975442947 >> v1:172.21.15.64:6789/0 conn(0x7f54ac06e700 legacy=0x7f54ac06d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54b0b74640 1 -- 172.21.15.64:0/2975442947 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54ac0b3e80 con 0x7f54ac0b55d0 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.175+0000 7f54ab7fe640 1 --2- 172.21.15.64:0/2975442947 >> v2:172.21.15.66:3300/0 conn(0x7f54ac072660 0x7f54ac15ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.176+0000 7f54b0b74640 1 --2- 172.21.15.64:0/2975442947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac0b55d0 0x7f54ac071980 secure :-1 s=READY pgs=1825 cs=0 l=1 rev1=1 crypto rx=0x7f54ac147ce0 tx=0x7f54a402bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=28da1a626a8cb52d server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.176+0000 7f54aaffd640 1 -- 172.21.15.64:0/2975442947 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54a4039070 con 0x7f54ac0b55d0 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.176+0000 7f54aaffd640 1 -- 172.21.15.64:0/2975442947 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f54a402cc80 con 0x7f54ac0b55d0 2024-03-04T07:37:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.176+0000 7f54aaffd640 1 -- 172.21.15.64:0/2975442947 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54a4034660 con 0x7f54ac0b55d0 2024-03-04T07:37:16.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 -- 172.21.15.64:0/2975442947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac0b55d0 msgr2=0x7f54ac071980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.181 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2975442947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac0b55d0 0x7f54ac071980 secure :-1 s=READY pgs=1825 cs=0 l=1 rev1=1 crypto rx=0x7f54ac147ce0 tx=0x7f54a402bed0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 -- 172.21.15.64:0/2975442947 shutdown_connections 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2975442947 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac0b55d0 0x7f54ac071980 unknown :-1 s=CLOSED pgs=1825 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2975442947 >> v2:172.21.15.66:3300/0 conn(0x7f54ac072660 0x7f54ac15ef80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 -- 172.21.15.64:0/2975442947 >> 172.21.15.64:0/2975442947 conn(0x7f54ac0b2720 msgr2=0x7f54ac0b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 -- 172.21.15.64:0/2975442947 shutdown_connections 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.177+0000 7f54b25d7640 1 -- 172.21.15.64:0/2975442947 wait complete. 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.178+0000 7f54b25d7640 1 Processor -- start 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.178+0000 7f54b25d7640 1 -- start start 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54b25d7640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac06e700 0x7f54ac0cfee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54b25d7640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54ac072660 0x7f54ac1517d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54b25d7640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54ac0b55d0 0x7f54ac1602e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54b25d7640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54ac072040 con 0x7f54ac06e700 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54b25d7640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54ac071ec0 con 0x7f54ac0b55d0 2024-03-04T07:37:16.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54b25d7640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f54ac0721c0 con 0x7f54ac072660 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac06e700 0x7f54ac0cfee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac06e700 0x7f54ac0cfee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42290/0 (socket says 172.21.15.64:42290) 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 -- 172.21.15.64:0/2806319480 learned_addr learned my addr 172.21.15.64:0/2806319480 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54ab7fe640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54ac072660 0x7f54ac1517d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54b0b74640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54ac0b55d0 0x7f54ac1602e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 -- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54ac072660 msgr2=0x7f54ac1517d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54ac072660 0x7f54ac1517d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 -- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54ac0b55d0 msgr2=0x7f54ac1602e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54ac0b55d0 0x7f54ac1602e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 -- 172.21.15.64:0/2806319480 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54ac0ce470 con 0x7f54ac06e700 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.179+0000 7f54abfff640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac06e700 0x7f54ac0cfee0 secure :-1 s=READY pgs=1826 cs=0 l=1 rev1=1 crypto rx=0x7f549400c430 tx=0x7f549400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.180+0000 7f54a8ff9640 1 -- 172.21.15.64:0/2806319480 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5494017070 con 0x7f54ac06e700 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.180+0000 7f54a8ff9640 1 -- 172.21.15.64:0/2806319480 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f5494005ce0 con 0x7f54ac06e700 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.180+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f54ac0ce7b0 con 0x7f54ac06e700 2024-03-04T07:37:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.180+0000 7f54a8ff9640 1 -- 172.21.15.64:0/2806319480 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54940072f0 con 0x7f54ac06e700 2024-03-04T07:37:16.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.180+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54ac0ceff0 con 0x7f54ac06e700 2024-03-04T07:37:16.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.181+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f54ac0b33b0 con 0x7f54ac06e700 2024-03-04T07:37:16.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.181+0000 7f54a8ff9640 1 -- 172.21.15.64:0/2806319480 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f549401a070 con 0x7f54ac06e700 2024-03-04T07:37:16.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.181+0000 7f54a8ff9640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f548c06b4d0 0x7f548c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.181+0000 7f54a8ff9640 1 -- 172.21.15.64:0/2806319480 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f54940895e0 con 0x7f54ac06e700 2024-03-04T07:37:16.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.182+0000 7f54ab7fe640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f548c06b4d0 0x7f548c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.182+0000 7f54ab7fe640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f548c06b4d0 0x7f548c06d990 secure :-1 s=READY pgs=833 cs=0 l=1 rev1=1 crypto rx=0x7f549c009560 tx=0x7f549c009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:16.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.186+0000 7f54a8ff9640 1 -- 172.21.15.64:0/2806319480 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f54ac0b33b0 con 0x7f54ac06e700 2024-03-04T07:37:16.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.382+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f54ac0cf1f0 con 0x7f54ac06e700 2024-03-04T07:37:16.384 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.383+0000 7f54a8ff9640 1 -- 172.21.15.64:0/2806319480 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 0 entries v127) v1 ==== 80+0+0 (secure 0 0 0) 0x7f54ac0cf1f0 con 0x7f54ac06e700 2024-03-04T07:37:16.384 INFO:teuthology.orchestra.run.smithi064.stderr:listed 0 entries 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f548c06b4d0 msgr2=0x7f548c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f548c06b4d0 0x7f548c06d990 secure :-1 s=READY pgs=833 cs=0 l=1 rev1=1 crypto rx=0x7f549c009560 tx=0x7f549c009190 comp rx=0 tx=0).stop 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac06e700 msgr2=0x7f54ac0cfee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac06e700 0x7f54ac0cfee0 secure :-1 s=READY pgs=1826 cs=0 l=1 rev1=1 crypto rx=0x7f549400c430 tx=0x7f549400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 shutdown_connections 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f548c06b4d0 0x7f548c06d990 unknown :-1 s=CLOSED pgs=833 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f54ac0b55d0 0x7f54ac1602e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f54ac072660 0x7f54ac1517d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 --2- 172.21.15.64:0/2806319480 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f54ac06e700 0x7f54ac0cfee0 unknown :-1 s=CLOSED pgs=1826 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 >> 172.21.15.64:0/2806319480 conn(0x7f54ac0b2720 msgr2=0x7f54ac15c070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:16.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 shutdown_connections 2024-03-04T07:37:16.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.385+0000 7f54b25d7640 1 -- 172.21.15.64:0/2806319480 wait complete. 2024-03-04T07:37:16.397 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi064.front.sepia.ceph.com... 2024-03-04T07:37:16.397 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:16.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo fusermount -u /home/ubuntu/cephtest/mnt.0 2024-03-04T07:37:16.422 INFO:tasks.cephfs.fuse_mount.ceph-fuse.0.smithi064.stderr:ceph-fuse[78502]: fuse finished with error 0 and tester_r 0 2024-03-04T07:37:16.424 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:37:16.425 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:16.425 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-04T07:37:16.590 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:37:16.590 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:16.605 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:37:16.605 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:37:16.605 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:16.606 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.1) 2024-03-04T07:37:16.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:37:16.660 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.1': No such file or directory 2024-03-04T07:37:16.660 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:37:16.660 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:16.676 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:37:16.676 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:37:16.676 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:16.676 DEBUG:teuthology.orchestra.run.smithi064:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.2) 2024-03-04T07:37:16.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:37:16.730 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.2': No such file or directory 2024-03-04T07:37:16.730 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:37:16.730 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:16.745 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:37:16.746 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:37:16.746 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:16.746 DEBUG:teuthology.orchestra.run.smithi066:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.3) 2024-03-04T07:37:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:37:16.800 INFO:teuthology.orchestra.run.smithi066.stderr:rmdir: failed to remove '/home/ubuntu/cephtest/mnt.3': No such file or directory 2024-03-04T07:37:16.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:16.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.900+0000 7f36df793640 1 Processor -- start 2024-03-04T07:37:16.902 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36df793640 1 -- start start 2024-03-04T07:37:16.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36df793640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f36d815df00 0x7f36d80d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36df793640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d80b95a0 0x7f36d80b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36df793640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f36d80cea30 con 0x7f36d80b9a30 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36df793640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f36d80d2af0 con 0x7f36d815df00 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36df793640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36d80cf3c0 con 0x7f36d80b95a0 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36dd52f640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f36d815df00 0x7f36d80d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36ddd30640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f36d80b9a30 0x7f36d80d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42584/0 (socket says 172.21.15.64:42584) 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.901+0000 7f36ddd30640 1 -- 172.21.15.64:0/429486795 learned_addr learned my addr 172.21.15.64:0/429486795 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.902+0000 7f36dcd2e640 1 --2- 172.21.15.64:0/429486795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d80b95a0 0x7f36d80b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.902+0000 7f36dd52f640 1 -- 172.21.15.64:0/429486795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d80b95a0 msgr2=0x7f36d80b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.902+0000 7f36dd52f640 1 --2- 172.21.15.64:0/429486795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d80b95a0 0x7f36d80b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.904 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.902+0000 7f36dd52f640 1 -- 172.21.15.64:0/429486795 >> v1:172.21.15.64:6789/0 conn(0x7f36d80b9a30 legacy=0x7f36d80d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.902+0000 7f36cffff640 1 -- 172.21.15.64:0/429486795 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1217171480 0 0) 0x7f36d80cea30 con 0x7f36d80b9a30 2024-03-04T07:37:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.902+0000 7f36dd52f640 1 -- 172.21.15.64:0/429486795 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36d80cf0e0 con 0x7f36d815df00 2024-03-04T07:37:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36dd52f640 1 --2- 172.21.15.64:0/429486795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d815df00 0x7f36d80d25b0 secure :-1 s=READY pgs=1828 cs=0 l=1 rev1=1 crypto rx=0x7f36d40097a0 tx=0x7f36d402b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=acddec2187fe24b1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36cffff640 1 -- 172.21.15.64:0/429486795 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36d4035020 con 0x7f36d815df00 2024-03-04T07:37:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36cffff640 1 -- 172.21.15.64:0/429486795 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36d4031830 con 0x7f36d815df00 2024-03-04T07:37:16.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36cffff640 1 -- 172.21.15.64:0/429486795 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36d4031c00 con 0x7f36d815df00 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 -- 172.21.15.64:0/429486795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d815df00 msgr2=0x7f36d80d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 --2- 172.21.15.64:0/429486795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d815df00 0x7f36d80d25b0 secure :-1 s=READY pgs=1828 cs=0 l=1 rev1=1 crypto rx=0x7f36d40097a0 tx=0x7f36d402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 -- 172.21.15.64:0/429486795 shutdown_connections 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 --2- 172.21.15.64:0/429486795 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d80b95a0 0x7f36d80b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 --2- 172.21.15.64:0/429486795 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d815df00 0x7f36d80d25b0 unknown :-1 s=CLOSED pgs=1828 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 -- 172.21.15.64:0/429486795 >> 172.21.15.64:0/429486795 conn(0x7f36d8071960 msgr2=0x7f36d8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 -- 172.21.15.64:0/429486795 shutdown_connections 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.903+0000 7f36df793640 1 -- 172.21.15.64:0/429486795 wait complete. 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.904+0000 7f36df793640 1 Processor -- start 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36df793640 1 -- start start 2024-03-04T07:37:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36df793640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d80b95a0 0x7f36d80b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36df793640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d815df00 0x7f36d80cc450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36df793640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36d80cd8e0 0x7f36d8157ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36df793640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36d80cec70 con 0x7f36d80b95a0 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36df793640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36d80ce750 con 0x7f36d80cd8e0 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36df793640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f36d80cedf0 con 0x7f36d815df00 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36dd52f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d80b95a0 0x7f36d80b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36dd52f640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d80b95a0 0x7f36d80b4eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42310/0 (socket says 172.21.15.64:42310) 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36dd52f640 1 -- 172.21.15.64:0/2918176587 learned_addr learned my addr 172.21.15.64:0/2918176587 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36dcd2e640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d815df00 0x7f36d80cc450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.905+0000 7f36ddd30640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36d80cd8e0 0x7f36d8157ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36dd52f640 1 -- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d815df00 msgr2=0x7f36d80cc450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36dd52f640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d815df00 0x7f36d80cc450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36dd52f640 1 -- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36d80cd8e0 msgr2=0x7f36d8157ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36dd52f640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36d80cd8e0 0x7f36d8157ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36dd52f640 1 -- 172.21.15.64:0/2918176587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36d815e2e0 con 0x7f36d80b95a0 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36dd52f640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d80b95a0 0x7f36d80b4eb0 secure :-1 s=READY pgs=1829 cs=0 l=1 rev1=1 crypto rx=0x7f36d40098c0 tx=0x7f36d40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36cdffb640 1 -- 172.21.15.64:0/2918176587 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36d4040070 con 0x7f36d80b95a0 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36cdffb640 1 -- 172.21.15.64:0/2918176587 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f36d403e070 con 0x7f36d80b95a0 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36d815e5c0 con 0x7f36d80b95a0 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36cdffb640 1 -- 172.21.15.64:0/2918176587 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f36d4035040 con 0x7f36d80b95a0 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.906+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f36d815eb80 con 0x7f36d80b95a0 2024-03-04T07:37:16.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.907+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f36d8067ff0 con 0x7f36d80b95a0 2024-03-04T07:37:16.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.908+0000 7f36cdffb640 1 -- 172.21.15.64:0/2918176587 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f36d4038020 con 0x7f36d80b95a0 2024-03-04T07:37:16.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.908+0000 7f36cdffb640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36bc06b4d0 0x7f36bc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:16.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.908+0000 7f36cdffb640 1 -- 172.21.15.64:0/2918176587 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f36d40ab5c0 con 0x7f36d80b95a0 2024-03-04T07:37:16.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.908+0000 7f36dcd2e640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36bc06b4d0 0x7f36bc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:16.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.908+0000 7f36dcd2e640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36bc06b4d0 0x7f36bc06d990 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f36d80cc330 tx=0x7f36c8009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:16.913 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:16.913+0000 7f36cdffb640 1 -- 172.21.15.64:0/2918176587 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f36d4031800 con 0x7f36d80b95a0 2024-03-04T07:37:17.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.131+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f36d815b820 con 0x7f36d80b95a0 2024-03-04T07:37:17.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.131+0000 7f36cdffb640 1 -- 172.21.15.64:0/2918176587 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f36d4079db0 con 0x7f36d80b95a0 2024-03-04T07:37:17.132 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:17.133 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:17.133 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36bc06b4d0 msgr2=0x7f36bc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36bc06b4d0 0x7f36bc06d990 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f36d80cc330 tx=0x7f36c8009190 comp rx=0 tx=0).stop 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d80b95a0 msgr2=0x7f36d80b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d80b95a0 0x7f36d80b4eb0 secure :-1 s=READY pgs=1829 cs=0 l=1 rev1=1 crypto rx=0x7f36d40098c0 tx=0x7f36d40046c0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 shutdown_connections 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f36bc06b4d0 0x7f36bc06d990 unknown :-1 s=CLOSED pgs=834 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f36d80cd8e0 0x7f36d8157ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f36d815df00 0x7f36d80cc450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 --2- 172.21.15.64:0/2918176587 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f36d80b95a0 0x7f36d80b4eb0 unknown :-1 s=CLOSED pgs=1829 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.133+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 >> 172.21.15.64:0/2918176587 conn(0x7f36d8071960 msgr2=0x7f36d815d130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:17.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.134+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 shutdown_connections 2024-03-04T07:37:17.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.134+0000 7f36df793640 1 -- 172.21.15.64:0/2918176587 wait complete. 2024-03-04T07:37:17.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:37:17.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.246+0000 7f6d90b8c640 1 Processor -- start 2024-03-04T07:37:17.247 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.246+0000 7f6d90b8c640 1 -- start start 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d90b8c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6d8c06b5c0 0x7f6d8c0b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d90b8c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6d8c0b0e90 0x7f6d8c06b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d90b8c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0d03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d90b8c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6d8c0cfa00 con 0x7f6d8c0b0e90 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d90b8c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6d8c0b3340 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d90b8c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6d8c0cee90 con 0x7f6d8c0ae9b0 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d89d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6d8c0b0e90 0x7f6d8c06b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d89d9b640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6d8c0b0e90 0x7f6d8c06b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42326/0 (socket says 172.21.15.64:42326) 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d89d9b640 1 -- 172.21.15.64:0/1626553936 learned_addr learned my addr 172.21.15.64:0/1626553936 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d8ad9d640 1 --2- 172.21.15.64:0/1626553936 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0d03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.247+0000 7f6d8a59c640 1 --2- 172.21.15.64:0/1626553936 >> v2:172.21.15.66:3300/0 conn(0x7f6d8c06b5c0 0x7f6d8c0b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d89d9b640 1 -- 172.21.15.64:0/1626553936 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0ae9b0 msgr2=0x7f6d8c0d03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d89d9b640 1 --2- 172.21.15.64:0/1626553936 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0d03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d89d9b640 1 -- 172.21.15.64:0/1626553936 >> v2:172.21.15.66:3300/0 conn(0x7f6d8c06b5c0 msgr2=0x7f6d8c0b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d89d9b640 1 --2- 172.21.15.64:0/1626553936 >> v2:172.21.15.66:3300/0 conn(0x7f6d8c06b5c0 0x7f6d8c0b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d89d9b640 1 -- 172.21.15.64:0/1626553936 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d8c0cfc50 con 0x7f6d8c0b0e90 2024-03-04T07:37:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d89d9b640 1 --2- 172.21.15.64:0/1626553936 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c0b0e90 0x7f6d8c06b9a0 secure :-1 s=READY pgs=1830 cs=0 l=1 rev1=1 crypto rx=0x7f6d8c071b00 tx=0x7f6d7402b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb460bab5a3be58b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d8959a640 1 -- 172.21.15.64:0/1626553936 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d7402e070 con 0x7f6d8c0b0e90 2024-03-04T07:37:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d8959a640 1 -- 172.21.15.64:0/1626553936 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d74037070 con 0x7f6d8c0b0e90 2024-03-04T07:37:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.248+0000 7f6d8959a640 1 -- 172.21.15.64:0/1626553936 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d74031a00 con 0x7f6d8c0b0e90 2024-03-04T07:37:17.251 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1626553936 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c0b0e90 msgr2=0x7f6d8c06b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1626553936 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c0b0e90 0x7f6d8c06b9a0 secure :-1 s=READY pgs=1830 cs=0 l=1 rev1=1 crypto rx=0x7f6d8c071b00 tx=0x7f6d7402b880 comp rx=0 tx=0).stop 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1626553936 shutdown_connections 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1626553936 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0d03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1626553936 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c0b0e90 0x7f6d8c06b9a0 unknown :-1 s=CLOSED pgs=1830 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1626553936 >> v2:172.21.15.66:3300/0 conn(0x7f6d8c06b5c0 0x7f6d8c0b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1626553936 >> 172.21.15.64:0/1626553936 conn(0x7f6d8c0b36d0 msgr2=0x7f6d8c0b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1626553936 shutdown_connections 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.249+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1626553936 wait complete. 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.250+0000 7f6d90b8c640 1 Processor -- start 2024-03-04T07:37:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.250+0000 7f6d90b8c640 1 -- start start 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d90b8c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c06b5c0 0x7f6d8c1550d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d90b8c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0aee00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d90b8c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0b0e90 0x7f6d8c0af340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d90b8c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d8c06c790 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d90b8c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d8c06bee0 con 0x7f6d8c0ae9b0 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d90b8c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6d8c06c910 con 0x7f6d8c0b0e90 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c06b5c0 0x7f6d8c1550d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8a59c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c06b5c0 0x7f6d8c1550d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42334/0 (socket says 172.21.15.64:42334) 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8a59c640 1 -- 172.21.15.64:0/1410763468 learned_addr learned my addr 172.21.15.64:0/1410763468 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d89d9b640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0aee00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8ad9d640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0b0e90 0x7f6d8c0af340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8a59c640 1 -- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0b0e90 msgr2=0x7f6d8c0af340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8a59c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0b0e90 0x7f6d8c0af340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8a59c640 1 -- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6d8c0ae9b0 msgr2=0x7f6d8c0aee00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.251+0000 7f6d8a59c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0aee00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.252+0000 7f6d8a59c640 1 -- 172.21.15.64:0/1410763468 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d8c0c8c70 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.252+0000 7f6d8a59c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c06b5c0 0x7f6d8c1550d0 secure :-1 s=READY pgs=1831 cs=0 l=1 rev1=1 crypto rx=0x7f6d8400e870 tx=0x7f6d8400ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.252+0000 7f6d82ffd640 1 -- 172.21.15.64:0/1410763468 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d84017070 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.252+0000 7f6d82ffd640 1 -- 172.21.15.64:0/1410763468 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6d8400ee60 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.252+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6d8c0c8f50 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.252+0000 7f6d82ffd640 1 -- 172.21.15.64:0/1410763468 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d8400cb80 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.252+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6d8c0c9510 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.253+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6d8c068700 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.253+0000 7f6d82ffd640 1 -- 172.21.15.64:0/1410763468 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6d8401a070 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.254+0000 7f6d82ffd640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d6c06b4d0 0x7f6d6c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.254+0000 7f6d82ffd640 1 -- 172.21.15.64:0/1410763468 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f6d8408a310 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.254+0000 7f6d89d9b640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d6c06b4d0 0x7f6d6c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.254+0000 7f6d89d9b640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d6c06b4d0 0x7f6d6c06d990 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f6d74036000 tx=0x7f6d7403d040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.259 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.258+0000 7f6d82ffd640 1 -- 172.21.15.64:0/1410763468 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6d84058b90 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.441+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6d8c0d0b70 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.442+0000 7f6d82ffd640 1 -- 172.21.15.64:0/1410763468 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v127) v1 ==== 74+0+16673 (secure 0 0 0) 0x7f6d8408a130 con 0x7f6d8c06b5c0 2024-03-04T07:37:17.443 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:17.443 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":127,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:34:36.563800+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:33:34.914683+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-04T07:33:36.218855+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"123","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.25,"score_stable":1.25,"optimal_score":1,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"19.25","mappings":[{"from":2,"to":0}]},{"pgid":"19.3f","mappings":[{"from":2,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:37:17.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d6c06b4d0 msgr2=0x7f6d6c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d6c06b4d0 0x7f6d6c06d990 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f6d74036000 tx=0x7f6d7403d040 comp rx=0 tx=0).stop 2024-03-04T07:37:17.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c06b5c0 msgr2=0x7f6d8c1550d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.445 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c06b5c0 0x7f6d8c1550d0 secure :-1 s=READY pgs=1831 cs=0 l=1 rev1=1 crypto rx=0x7f6d8400e870 tx=0x7f6d8400ed30 comp rx=0 tx=0).stop 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 shutdown_connections 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6d6c06b4d0 0x7f6d6c06d990 unknown :-1 s=CLOSED pgs=835 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6d8c0b0e90 0x7f6d8c0af340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6d8c0ae9b0 0x7f6d8c0aee00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 --2- 172.21.15.64:0/1410763468 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6d8c06b5c0 0x7f6d8c1550d0 unknown :-1 s=CLOSED pgs=1831 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 >> 172.21.15.64:0/1410763468 conn(0x7f6d8c0b36d0 msgr2=0x7f6d8c0cdf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 shutdown_connections 2024-03-04T07:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.444+0000 7f6d90b8c640 1 -- 172.21.15.64:0/1410763468 wait complete. 2024-03-04T07:37:17.456 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-04T07:26:41.184112+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '16', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 8, 'score_stable': 8, 'optimal_score': 0.25, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 18, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-04T07:33:34.914683+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '122', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 19, 'pool_name': 'cephfs_data', 'create_time': '2024-03-04T07:33:36.218855+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '123', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.25, 'score_stable': 1.25, 'optimal_score': 1, 'raw_score_acting': 1.25, 'raw_score_stable': 1.25, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-04T07:37:17.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-04T07:37:17.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.560+0000 7f3a56d1e640 1 Processor -- start 2024-03-04T07:37:17.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.560+0000 7f3a56d1e640 1 -- start start 2024-03-04T07:37:17.562 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a56d1e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3a5015e2f0 0x7f3a500b41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a56d1e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3a5006b4e0 0x7f3a5006b940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a56d1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5006be80 0x7f3a500cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a56d1e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a5015e6d0 con 0x7f3a5006b4e0 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a56d1e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3a500ccd00 con 0x7f3a5015e2f0 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a56d1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3a500ccfe0 con 0x7f3a5006be80 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3a5006b4e0 0x7f3a5006b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f3a5006b4e0 0x7f3a5006b940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42342/0 (socket says 172.21.15.64:42342) 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 -- 172.21.15.64:0/2026780767 learned_addr learned my addr 172.21.15.64:0/2026780767 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a552bb640 1 --2- 172.21.15.64:0/2026780767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5006be80 0x7f3a500cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a54aba640 1 --2- 172.21.15.64:0/2026780767 >> v2:172.21.15.66:3300/0 conn(0x7f3a5015e2f0 0x7f3a500b41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 -- 172.21.15.64:0/2026780767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5006be80 msgr2=0x7f3a500cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:17.563 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 --2- 172.21.15.64:0/2026780767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5006be80 0x7f3a500cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 -- 172.21.15.64:0/2026780767 >> v2:172.21.15.66:3300/0 conn(0x7f3a5015e2f0 msgr2=0x7f3a500b41f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:17.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 --2- 172.21.15.64:0/2026780767 >> v2:172.21.15.66:3300/0 conn(0x7f3a5015e2f0 0x7f3a500b41f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.561+0000 7f3a4ffff640 1 -- 172.21.15.64:0/2026780767 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a500cd2c0 con 0x7f3a5006b4e0 2024-03-04T07:37:17.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.562+0000 7f3a4ffff640 1 --2- 172.21.15.64:0/2026780767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006b4e0 0x7f3a5006b940 secure :-1 s=READY pgs=1832 cs=0 l=1 rev1=1 crypto rx=0x7f3a50071dc0 tx=0x7f3a4002bdb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fd355aa0fed06b67 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.562+0000 7f3a4f7fe640 1 -- 172.21.15.64:0/2026780767 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a4002c950 con 0x7f3a5006b4e0 2024-03-04T07:37:17.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.562+0000 7f3a4f7fe640 1 -- 172.21.15.64:0/2026780767 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a4002caf0 con 0x7f3a5006b4e0 2024-03-04T07:37:17.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.562+0000 7f3a4f7fe640 1 -- 172.21.15.64:0/2026780767 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a400343f0 con 0x7f3a5006b4e0 2024-03-04T07:37:17.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 -- 172.21.15.64:0/2026780767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006b4e0 msgr2=0x7f3a5006b940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/2026780767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006b4e0 0x7f3a5006b940 secure :-1 s=READY pgs=1832 cs=0 l=1 rev1=1 crypto rx=0x7f3a50071dc0 tx=0x7f3a4002bdb0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 -- 172.21.15.64:0/2026780767 shutdown_connections 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/2026780767 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5006be80 0x7f3a500cfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/2026780767 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006b4e0 0x7f3a5006b940 unknown :-1 s=CLOSED pgs=1832 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/2026780767 >> v2:172.21.15.66:3300/0 conn(0x7f3a5015e2f0 0x7f3a500b41f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 -- 172.21.15.64:0/2026780767 >> 172.21.15.64:0/2026780767 conn(0x7f3a500b36d0 msgr2=0x7f3a500b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 -- 172.21.15.64:0/2026780767 shutdown_connections 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.563+0000 7f3a56d1e640 1 -- 172.21.15.64:0/2026780767 wait complete. 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.564+0000 7f3a56d1e640 1 Processor -- start 2024-03-04T07:37:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.564+0000 7f3a56d1e640 1 -- start start 2024-03-04T07:37:17.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a56d1e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a5006b4e0 0x7f3a50159f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a56d1e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006be80 0x7f3a500ca720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a56d1e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5015e2f0 0x7f3a500b6990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a56d1e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a500cff40 con 0x7f3a5006be80 2024-03-04T07:37:17.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a56d1e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a5006c310 con 0x7f3a5006b4e0 2024-03-04T07:37:17.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a56d1e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3a500d00c0 con 0x7f3a5015e2f0 2024-03-04T07:37:17.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006be80 0x7f3a500ca720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006be80 0x7f3a500ca720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42344/0 (socket says 172.21.15.64:42344) 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 -- 172.21.15.64:0/111989412 learned_addr learned my addr 172.21.15.64:0/111989412 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a552bb640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5015e2f0 0x7f3a500b6990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a54aba640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a5006b4e0 0x7f3a50159f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 -- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5015e2f0 msgr2=0x7f3a500b6990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5015e2f0 0x7f3a500b6990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 -- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a5006b4e0 msgr2=0x7f3a50159f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a5006b4e0 0x7f3a50159f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.565+0000 7f3a4ffff640 1 -- 172.21.15.64:0/111989412 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a500c8e20 con 0x7f3a5006be80 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.566+0000 7f3a4ffff640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006be80 0x7f3a500ca720 secure :-1 s=READY pgs=1833 cs=0 l=1 rev1=1 crypto rx=0x7f3a40000f80 tx=0x7f3a40004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.566+0000 7f3a4d7fa640 1 -- 172.21.15.64:0/111989412 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a40043020 con 0x7f3a5006be80 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.566+0000 7f3a4d7fa640 1 -- 172.21.15.64:0/111989412 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3a4003e450 con 0x7f3a5006be80 2024-03-04T07:37:17.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.566+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a500c9100 con 0x7f3a5006be80 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.566+0000 7f3a4d7fa640 1 -- 172.21.15.64:0/111989412 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a4003e840 con 0x7f3a5006be80 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.566+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a500c96c0 con 0x7f3a5006be80 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.567+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a500c8bb0 con 0x7f3a5006be80 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.567+0000 7f3a4d7fa640 1 -- 172.21.15.64:0/111989412 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3a40046020 con 0x7f3a5006be80 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.568+0000 7f3a4d7fa640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a3006b4d0 0x7f3a3006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.568+0000 7f3a4d7fa640 1 -- 172.21.15.64:0/111989412 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f3a400aca60 con 0x7f3a5006be80 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.568+0000 7f3a54aba640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a3006b4d0 0x7f3a3006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.568+0000 7f3a54aba640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a3006b4d0 0x7f3a3006d990 secure :-1 s=READY pgs=836 cs=0 l=1 rev1=1 crypto rx=0x7f3a5015dca0 tx=0x7f3a48009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.572+0000 7f3a4d7fa640 1 -- 172.21.15.64:0/111989412 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3a4004d030 con 0x7f3a5006be80 2024-03-04T07:37:17.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.751+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f3a5009fd30 con 0x7f3a5006be80 2024-03-04T07:37:17.752 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.752+0000 7f3a4d7fa640 1 -- 172.21.15.64:0/111989412 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v127) v1 ==== 93+0+10 (secure 0 0 0) 0x7f3a4007b2e0 con 0x7f3a5006be80 2024-03-04T07:37:17.753 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-03-04T07:37:17.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a3006b4d0 msgr2=0x7f3a3006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a3006b4d0 0x7f3a3006d990 secure :-1 s=READY pgs=836 cs=0 l=1 rev1=1 crypto rx=0x7f3a5015dca0 tx=0x7f3a48009190 comp rx=0 tx=0).stop 2024-03-04T07:37:17.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006be80 msgr2=0x7f3a500ca720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.755 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006be80 0x7f3a500ca720 secure :-1 s=READY pgs=1833 cs=0 l=1 rev1=1 crypto rx=0x7f3a40000f80 tx=0x7f3a40004a40 comp rx=0 tx=0).stop 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 shutdown_connections 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3a3006b4d0 0x7f3a3006d990 unknown :-1 s=CLOSED pgs=836 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3a5015e2f0 0x7f3a500b6990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3a5006be80 0x7f3a500ca720 unknown :-1 s=CLOSED pgs=1833 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 --2- 172.21.15.64:0/111989412 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3a5006b4e0 0x7f3a50159f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 >> 172.21.15.64:0/111989412 conn(0x7f3a500b36d0 msgr2=0x7f3a5015c8f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 shutdown_connections 2024-03-04T07:37:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.754+0000 7f3a56d1e640 1 -- 172.21.15.64:0/111989412 wait complete. 2024-03-04T07:37:17.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-04T07:37:17.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.870+0000 7fa164925640 1 Processor -- start 2024-03-04T07:37:17.871 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.870+0000 7fa164925640 1 -- start start 2024-03-04T07:37:17.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa164925640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fa1600b3060 0x7fa160071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.872 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa164925640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa16015e4b0 con 0x7fa1600b2b10 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa164925640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa1600b97e0 con 0x7fa1600717b0 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa164925640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fa1600b7c70 con 0x7fa1600b3060 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15ed9d640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa1600b2b10 0x7fa1600b7560 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42596/0 (socket says 172.21.15.64:42596) 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15ed9d640 1 -- 172.21.15.64:0/4012805190 learned_addr learned my addr 172.21.15.64:0/4012805190 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15dd9b640 1 --2- 172.21.15.64:0/4012805190 >> v2:172.21.15.66:3300/0 conn(0x7fa1600b3060 0x7fa160071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15d59a640 1 -- 172.21.15.64:0/4012805190 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 314010330 0 0) 0x7fa150001490 con 0x7fa1600b2b10 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15d59a640 1 -- 172.21.15.64:0/4012805190 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa148003520 con 0x7fa1600b2b10 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15dd9b640 1 -- 172.21.15.64:0/4012805190 >> v1:172.21.15.66:6789/0 conn(0x7fa1600717b0 legacy=0x7fa1600b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15d59a640 1 -- 172.21.15.64:0/4012805190 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1978376434 0 0) 0x7fa1600b97e0 con 0x7fa1600717b0 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15dd9b640 1 -- 172.21.15.64:0/4012805190 >> v1:172.21.15.64:6789/0 conn(0x7fa1600b2b10 legacy=0x7fa1600b7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.873 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.871+0000 7fa15dd9b640 1 -- 172.21.15.64:0/4012805190 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1600b9a80 con 0x7fa1600b3060 2024-03-04T07:37:17.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.872+0000 7fa15dd9b640 1 --2- 172.21.15.64:0/4012805190 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600b3060 0x7fa160071b90 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7fa1540097a0 tx=0x7fa15402b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6a30e0ca352a0d8f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.874 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.872+0000 7fa15d59a640 1 -- 172.21.15.64:0/4012805190 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa154034070 con 0x7fa1600b3060 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.872+0000 7fa15d59a640 1 -- 172.21.15.64:0/4012805190 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa154030610 con 0x7fa1600b3060 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.872+0000 7fa15d59a640 1 -- 172.21.15.64:0/4012805190 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1540309e0 con 0x7fa1600b3060 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.873+0000 7fa164925640 1 -- 172.21.15.64:0/4012805190 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600b3060 msgr2=0x7fa160071b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.873+0000 7fa164925640 1 --2- 172.21.15.64:0/4012805190 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600b3060 0x7fa160071b90 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7fa1540097a0 tx=0x7fa15402b960 comp rx=0 tx=0).stop 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.873+0000 7fa164925640 1 -- 172.21.15.64:0/4012805190 shutdown_connections 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.873+0000 7fa164925640 1 --2- 172.21.15.64:0/4012805190 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600b3060 0x7fa160071b90 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.873+0000 7fa164925640 1 -- 172.21.15.64:0/4012805190 >> 172.21.15.64:0/4012805190 conn(0x7fa1600d3d20 msgr2=0x7fa1600d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.873+0000 7fa164925640 1 -- 172.21.15.64:0/4012805190 shutdown_connections 2024-03-04T07:37:17.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.873+0000 7fa164925640 1 -- 172.21.15.64:0/4012805190 wait complete. 2024-03-04T07:37:17.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.874+0000 7fa164925640 1 Processor -- start 2024-03-04T07:37:17.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.874+0000 7fa164925640 1 -- start start 2024-03-04T07:37:17.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa164925640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600717b0 0x7fa1600c96a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa164925640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1600b2b10 0x7fa1601e7b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa164925640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1600b3060 0x7fa1600ccfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa164925640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1600d4bf0 con 0x7fa1600b3060 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa164925640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1600d2e10 con 0x7fa1600717b0 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa164925640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa1600d32a0 con 0x7fa1600b2b10 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1600b3060 0x7fa1600ccfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1600b3060 0x7fa1600ccfc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42348/0 (socket says 172.21.15.64:42348) 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 -- 172.21.15.64:0/2494885715 learned_addr learned my addr 172.21.15.64:0/2494885715 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15dd9b640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1600b2b10 0x7fa1601e7b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15e59c640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600717b0 0x7fa1600c96a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 -- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1600b2b10 msgr2=0x7fa1601e7b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1600b2b10 0x7fa1601e7b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 -- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600717b0 msgr2=0x7fa1600c96a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:17.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600717b0 0x7fa1600c96a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.875+0000 7fa15ed9d640 1 -- 172.21.15.64:0/2494885715 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1600c9be0 con 0x7fa1600b3060 2024-03-04T07:37:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.876+0000 7fa15ed9d640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1600b3060 0x7fa1600ccfc0 secure :-1 s=READY pgs=1835 cs=0 l=1 rev1=1 crypto rx=0x7fa15000cb50 tx=0x7fa15000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.876+0000 7fa14effd640 1 -- 172.21.15.64:0/2494885715 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa150017070 con 0x7fa1600b3060 2024-03-04T07:37:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.876+0000 7fa14effd640 1 -- 172.21.15.64:0/2494885715 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa15000ace0 con 0x7fa1600b3060 2024-03-04T07:37:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.876+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa160148260 con 0x7fa1600b3060 2024-03-04T07:37:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.876+0000 7fa14effd640 1 -- 172.21.15.64:0/2494885715 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa150012620 con 0x7fa1600b3060 2024-03-04T07:37:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.876+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa1601487e0 con 0x7fa1600b3060 2024-03-04T07:37:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.877+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa16006ce40 con 0x7fa1600b3060 2024-03-04T07:37:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.877+0000 7fa14effd640 1 -- 172.21.15.64:0/2494885715 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa15001a070 con 0x7fa1600b3060 2024-03-04T07:37:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.878+0000 7fa14effd640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14006b4d0 0x7fa14006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.878+0000 7fa14effd640 1 -- 172.21.15.64:0/2494885715 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fa150089ff0 con 0x7fa1600b3060 2024-03-04T07:37:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.878+0000 7fa15e59c640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14006b4d0 0x7fa14006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.878+0000 7fa15e59c640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14006b4d0 0x7fa14006d990 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7fa158000f60 tx=0x7fa158009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:17.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:17.882+0000 7fa14effd640 1 -- 172.21.15.64:0/2494885715 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa1600c9be0 con 0x7fa1600b3060 2024-03-04T07:37:18.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.063+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fa1600b91e0 con 0x7fa1600b3060 2024-03-04T07:37:18.064 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.063+0000 7fa14effd640 1 -- 172.21.15.64:0/2494885715 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v127) v1 ==== 104+0+11 (secure 0 0 0) 0x7fa16015e4b0 con 0x7fa1600b3060 2024-03-04T07:37:18.064 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 32 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14006b4d0 msgr2=0x7fa14006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14006b4d0 0x7fa14006d990 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7fa158000f60 tx=0x7fa158009040 comp rx=0 tx=0).stop 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1600b3060 msgr2=0x7fa1600ccfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1600b3060 0x7fa1600ccfc0 secure :-1 s=READY pgs=1835 cs=0 l=1 rev1=1 crypto rx=0x7fa15000cb50 tx=0x7fa15000a430 comp rx=0 tx=0).stop 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 shutdown_connections 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa14006b4d0 0x7fa14006d990 unknown :-1 s=CLOSED pgs=837 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa1600b3060 0x7fa1600ccfc0 unknown :-1 s=CLOSED pgs=1835 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa1600b2b10 0x7fa1601e7b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 --2- 172.21.15.64:0/2494885715 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa1600717b0 0x7fa1600c96a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.067 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.065+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 >> 172.21.15.64:0/2494885715 conn(0x7fa1600d3d20 msgr2=0x7fa1600b6e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:18.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.066+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 shutdown_connections 2024-03-04T07:37:18.068 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.066+0000 7fa164925640 1 -- 172.21.15.64:0/2494885715 wait complete. 2024-03-04T07:37:18.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-04T07:37:18.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.181+0000 7fad966aa640 1 Processor -- start 2024-03-04T07:37:18.182 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.181+0000 7fad966aa640 1 -- start start 2024-03-04T07:37:18.184 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad966aa640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fad901618e0 0x7fad9006b130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad966aa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900ce930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad966aa640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fad900cee70 con 0x7fad90161d40 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad966aa640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fad900cfce0 con 0x7fad901618e0 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad966aa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fad9015f730 con 0x7fad900ce4f0 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fad901618e0 0x7fad9006b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8ffff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fad901618e0 0x7fad9006b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42362/0 (socket says 172.21.15.64:42362) 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8ffff640 1 -- 172.21.15.64:0/1986601105 learned_addr learned my addr 172.21.15.64:0/1986601105 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8f7fe640 1 --2- 172.21.15.64:0/1986601105 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900ce930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8ffff640 1 -- 172.21.15.64:0/1986601105 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 msgr2=0x7fad900ce930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8ffff640 1 --2- 172.21.15.64:0/1986601105 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900ce930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8ffff640 1 -- 172.21.15.64:0/1986601105 >> v1:172.21.15.66:6789/0 conn(0x7fad90161d40 legacy=0x7fad901621d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8ffff640 1 -- 172.21.15.64:0/1986601105 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fad9015ebb0 con 0x7fad901618e0 2024-03-04T07:37:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.182+0000 7fad8effd640 1 -- 172.21.15.64:0/1986601105 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2490773215 0 0) 0x7fad88001490 con 0x7fad90161d40 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.183+0000 7fad8ffff640 1 --2- 172.21.15.64:0/1986601105 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad901618e0 0x7fad9006b130 secure :-1 s=READY pgs=1836 cs=0 l=1 rev1=1 crypto rx=0x7fad80009870 tx=0x7fad8002ba20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1af5f885a946c60b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.183+0000 7fad8effd640 1 -- 172.21.15.64:0/1986601105 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fad8002f840 con 0x7fad901618e0 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.183+0000 7fad8effd640 1 -- 172.21.15.64:0/1986601105 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fad8002f9e0 con 0x7fad901618e0 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.183+0000 7fad8effd640 1 -- 172.21.15.64:0/1986601105 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fad8002fdb0 con 0x7fad901618e0 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.183+0000 7fad966aa640 1 -- 172.21.15.64:0/1986601105 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad901618e0 msgr2=0x7fad9006b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.183+0000 7fad966aa640 1 --2- 172.21.15.64:0/1986601105 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad901618e0 0x7fad9006b130 secure :-1 s=READY pgs=1836 cs=0 l=1 rev1=1 crypto rx=0x7fad80009870 tx=0x7fad8002ba20 comp rx=0 tx=0).stop 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.184+0000 7fad966aa640 1 -- 172.21.15.64:0/1986601105 shutdown_connections 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.184+0000 7fad966aa640 1 --2- 172.21.15.64:0/1986601105 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900ce930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.184+0000 7fad966aa640 1 --2- 172.21.15.64:0/1986601105 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad901618e0 0x7fad9006b130 unknown :-1 s=CLOSED pgs=1836 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.184+0000 7fad966aa640 1 -- 172.21.15.64:0/1986601105 >> 172.21.15.64:0/1986601105 conn(0x7fad900b3660 msgr2=0x7fad900b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.184+0000 7fad966aa640 1 -- 172.21.15.64:0/1986601105 shutdown_connections 2024-03-04T07:37:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.184+0000 7fad966aa640 1 -- 172.21.15.64:0/1986601105 wait complete. 2024-03-04T07:37:18.189 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.185+0000 7fad966aa640 1 Processor -- start 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.185+0000 7fad966aa640 1 -- start start 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.185+0000 7fad966aa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900c7af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad966aa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad90161d40 0x7fad900c9c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad966aa640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fad9015bba0 0x7fad900b4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad966aa640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad9015f450 con 0x7fad90161d40 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad966aa640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad9015f170 con 0x7fad9015bba0 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad966aa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fad9006b670 con 0x7fad900ce4f0 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad90161d40 0x7fad900c9c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad90161d40 0x7fad900c9c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42378/0 (socket says 172.21.15.64:42378) 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 -- 172.21.15.64:0/2034509940 learned_addr learned my addr 172.21.15.64:0/2034509940 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8ffff640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900c7af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad94c47640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fad9015bba0 0x7fad900b4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.190 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 -- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 msgr2=0x7fad900c7af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900c7af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 -- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fad9015bba0 msgr2=0x7fad900b4e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fad9015bba0 0x7fad900b4e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 -- 172.21.15.64:0/2034509940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fad900ca1b0 con 0x7fad90161d40 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8f7fe640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad90161d40 0x7fad900c9c70 secure :-1 s=READY pgs=1837 cs=0 l=1 rev1=1 crypto rx=0x7fad8400e9b0 tx=0x7fad8400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8cff9640 1 -- 172.21.15.64:0/2034509940 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fad84017070 con 0x7fad90161d40 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8cff9640 1 -- 172.21.15.64:0/2034509940 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fad8400c9a0 con 0x7fad90161d40 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad8cff9640 1 -- 172.21.15.64:0/2034509940 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fad8400cd90 con 0x7fad90161d40 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fad900ca450 con 0x7fad90161d40 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.186+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fad900caa10 con 0x7fad90161d40 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.187+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fad9009fc50 con 0x7fad90161d40 2024-03-04T07:37:18.191 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.188+0000 7fad8cff9640 1 -- 172.21.15.64:0/2034509940 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fad8401a070 con 0x7fad90161d40 2024-03-04T07:37:18.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.188+0000 7fad8cff9640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fad7c06b4d0 0x7fad7c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.188+0000 7fad8cff9640 1 -- 172.21.15.64:0/2034509940 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fad8408a1d0 con 0x7fad90161d40 2024-03-04T07:37:18.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.188+0000 7fad8ffff640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fad7c06b4d0 0x7fad7c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.192 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.189+0000 7fad8ffff640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fad7c06b4d0 0x7fad7c06d990 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7fad80036000 tx=0x7fad80008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.193 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.193+0000 7fad8cff9640 1 -- 172.21.15.64:0/2034509940 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fad840589c0 con 0x7fad90161d40 2024-03-04T07:37:18.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.372+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fad90160190 con 0x7fad90161d40 2024-03-04T07:37:18.373 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.372+0000 7fad8cff9640 1 -- 172.21.15.64:0/2034509940 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v127) v1 ==== 100+0+11 (secure 0 0 0) 0x7fad8405ca60 con 0x7fad90161d40 2024-03-04T07:37:18.374 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 64 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fad7c06b4d0 msgr2=0x7fad7c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fad7c06b4d0 0x7fad7c06d990 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7fad80036000 tx=0x7fad80008030 comp rx=0 tx=0).stop 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad90161d40 msgr2=0x7fad900c9c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad90161d40 0x7fad900c9c70 secure :-1 s=READY pgs=1837 cs=0 l=1 rev1=1 crypto rx=0x7fad8400e9b0 tx=0x7fad8400ee70 comp rx=0 tx=0).stop 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 shutdown_connections 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fad7c06b4d0 0x7fad7c06d990 unknown :-1 s=CLOSED pgs=838 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fad9015bba0 0x7fad900b4e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fad90161d40 0x7fad900c9c70 unknown :-1 s=CLOSED pgs=1837 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 --2- 172.21.15.64:0/2034509940 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fad900ce4f0 0x7fad900c7af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.374+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 >> 172.21.15.64:0/2034509940 conn(0x7fad900b3660 msgr2=0x7fad9015e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.375+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 shutdown_connections 2024-03-04T07:37:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.375+0000 7fad966aa640 1 -- 172.21.15.64:0/2034509940 wait complete. 2024-03-04T07:37:18.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:18.488 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.487+0000 7fb075a47640 1 Processor -- start 2024-03-04T07:37:18.489 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb075a47640 1 -- start start 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb075a47640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fb07015e000 0x7fb070071540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb075a47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0700b2c50 0x7fb070071a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb075a47640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb070161cb0 0x7fb070162140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb075a47640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0700cece0 con 0x7fb070161cb0 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb075a47640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fb0700ce970 con 0x7fb07015e000 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb075a47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb0700cef30 con 0x7fb0700b2c50 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.488+0000 7fb06f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb070161cb0 0x7fb070162140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06f7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fb070161cb0 0x7fb070162140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42392/0 (socket says 172.21.15.64:42392) 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06f7fe640 1 -- 172.21.15.64:0/2993096158 learned_addr learned my addr 172.21.15.64:0/2993096158 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:18.490 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06effd640 1 --2- 172.21.15.64:0/2993096158 >> v2:172.21.15.66:3300/0 conn(0x7fb07015e000 0x7fb070071540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06e7fc640 1 --2- 172.21.15.64:0/2993096158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0700b2c50 0x7fb070071a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06f7fe640 1 -- 172.21.15.64:0/2993096158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0700b2c50 msgr2=0x7fb070071a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:18.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06f7fe640 1 --2- 172.21.15.64:0/2993096158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0700b2c50 0x7fb070071a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06f7fe640 1 -- 172.21.15.64:0/2993096158 >> v2:172.21.15.66:3300/0 conn(0x7fb07015e000 msgr2=0x7fb070071540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:18.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06f7fe640 1 --2- 172.21.15.64:0/2993096158 >> v2:172.21.15.66:3300/0 conn(0x7fb07015e000 0x7fb070071540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.491 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.489+0000 7fb06f7fe640 1 -- 172.21.15.64:0/2993096158 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0700cf210 con 0x7fb070161cb0 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb06f7fe640 1 --2- 172.21.15.64:0/2993096158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb070161cb0 0x7fb070162140 secure :-1 s=READY pgs=1838 cs=0 l=1 rev1=1 crypto rx=0x7fb070062c70 tx=0x7fb06802be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8d5086c38e3f5b82 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb06dffb640 1 -- 172.21.15.64:0/2993096158 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb06802c9e0 con 0x7fb070161cb0 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb06dffb640 1 -- 172.21.15.64:0/2993096158 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb06802cb80 con 0x7fb070161cb0 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb06dffb640 1 -- 172.21.15.64:0/2993096158 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0680343f0 con 0x7fb070161cb0 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 -- 172.21.15.64:0/2993096158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb070161cb0 msgr2=0x7fb070162140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 --2- 172.21.15.64:0/2993096158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb070161cb0 0x7fb070162140 secure :-1 s=READY pgs=1838 cs=0 l=1 rev1=1 crypto rx=0x7fb070062c70 tx=0x7fb06802be40 comp rx=0 tx=0).stop 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 -- 172.21.15.64:0/2993096158 shutdown_connections 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 --2- 172.21.15.64:0/2993096158 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb070161cb0 0x7fb070162140 unknown :-1 s=CLOSED pgs=1838 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 --2- 172.21.15.64:0/2993096158 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb0700b2c50 0x7fb070071a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 --2- 172.21.15.64:0/2993096158 >> v2:172.21.15.66:3300/0 conn(0x7fb07015e000 0x7fb070071540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 -- 172.21.15.64:0/2993096158 >> 172.21.15.64:0/2993096158 conn(0x7fb0700d3c80 msgr2=0x7fb0700d4090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:18.493 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 -- 172.21.15.64:0/2993096158 shutdown_connections 2024-03-04T07:37:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.490+0000 7fb075a47640 1 -- 172.21.15.64:0/2993096158 wait complete. 2024-03-04T07:37:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.491+0000 7fb075a47640 1 Processor -- start 2024-03-04T07:37:18.494 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb075a47640 1 -- start start 2024-03-04T07:37:18.496 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb075a47640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0700b2c50 0x7fb0700cd5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb075a47640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb07015e000 0x7fb0701f4590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb075a47640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb070161cb0 0x7fb0700b56e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb075a47640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0700b3320 con 0x7fb0700b2c50 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb075a47640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0700b31a0 con 0x7fb07015e000 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb075a47640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb07015e3e0 con 0x7fb070161cb0 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb06effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0700b2c50 0x7fb0700cd5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb06effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0700b2c50 0x7fb0700cd5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42402/0 (socket says 172.21.15.64:42402) 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.492+0000 7fb06effd640 1 -- 172.21.15.64:0/1105197178 learned_addr learned my addr 172.21.15.64:0/1105197178 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06f7fe640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb070161cb0 0x7fb0700b56e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06e7fc640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb07015e000 0x7fb0701f4590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06effd640 1 -- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb070161cb0 msgr2=0x7fb0700b56e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06effd640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb070161cb0 0x7fb0700b56e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06effd640 1 -- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb07015e000 msgr2=0x7fb0701f4590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.497 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06effd640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb07015e000 0x7fb0701f4590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06effd640 1 -- 172.21.15.64:0/1105197178 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0700b3ab0 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb06effd640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0700b2c50 0x7fb0700cd5a0 secure :-1 s=READY pgs=1839 cs=0 l=1 rev1=1 crypto rx=0x7fb06000e9b0 tx=0x7fb06000ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb057fff640 1 -- 172.21.15.64:0/1105197178 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb060017070 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb057fff640 1 -- 172.21.15.64:0/1105197178 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb06000c9a0 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0700b3d90 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb057fff640 1 -- 172.21.15.64:0/1105197178 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb06000cd90 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.493+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0700b4350 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.494+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb070067ff0 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.495+0000 7fb057fff640 1 -- 172.21.15.64:0/1105197178 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fb06001a070 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.495+0000 7fb057fff640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb05c06b4d0 0x7fb05c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.495+0000 7fb057fff640 1 -- 172.21.15.64:0/1105197178 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fb06008a140 con 0x7fb0700b2c50 2024-03-04T07:37:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.495+0000 7fb06e7fc640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb05c06b4d0 0x7fb05c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.496+0000 7fb06e7fc640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb05c06b4d0 0x7fb05c06d990 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7fb0700b5690 tx=0x7fb064009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.499+0000 7fb057fff640 1 -- 172.21.15.64:0/1105197178 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fb0600589c0 con 0x7fb0700b2c50 2024-03-04T07:37:18.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:18.610 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:18.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.722+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb0700b2700 con 0x7fb0700b2c50 2024-03-04T07:37:18.723 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.722+0000 7fb057fff640 1 -- 172.21.15.64:0/1105197178 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fb06005ca60 con 0x7fb0700b2c50 2024-03-04T07:37:18.723 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:18.723 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:18.724 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb05c06b4d0 msgr2=0x7fb05c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb05c06b4d0 0x7fb05c06d990 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7fb0700b5690 tx=0x7fb064009190 comp rx=0 tx=0).stop 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0700b2c50 msgr2=0x7fb0700cd5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0700b2c50 0x7fb0700cd5a0 secure :-1 s=READY pgs=1839 cs=0 l=1 rev1=1 crypto rx=0x7fb06000e9b0 tx=0x7fb06000ee70 comp rx=0 tx=0).stop 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 shutdown_connections 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fb05c06b4d0 0x7fb05c06d990 unknown :-1 s=CLOSED pgs=839 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fb070161cb0 0x7fb0700b56e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fb07015e000 0x7fb0701f4590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 --2- 172.21.15.64:0/1105197178 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fb0700b2c50 0x7fb0700cd5a0 unknown :-1 s=CLOSED pgs=1839 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 >> 172.21.15.64:0/1105197178 conn(0x7fb0700d3c80 msgr2=0x7fb0700b95e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.724+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 shutdown_connections 2024-03-04T07:37:18.726 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.725+0000 7fb075a47640 1 -- 172.21.15.64:0/1105197178 wait complete. 2024-03-04T07:37:18.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:37:18.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.837+0000 7f7bf204e640 1 Processor -- start 2024-03-04T07:37:18.838 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.837+0000 7f7bf204e640 1 -- start start 2024-03-04T07:37:18.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bf204e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7bec15bac0 0x7f7bec15cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.839 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bf204e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec06d400 0x7f7bec0ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bf204e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7bec0729b0 con 0x7f7bec0aec40 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bf204e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7bec0af0b0 con 0x7f7bec15bac0 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bf204e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7bec06cf70 con 0x7f7bec06d400 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7beb7fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f7bec15bac0 0x7f7bec15cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bebfff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7bec0aec40 0x7f7bec070870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42612/0 (socket says 172.21.15.64:42612) 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bebfff640 1 -- 172.21.15.64:0/2684644623 learned_addr learned my addr 172.21.15.64:0/2684644623 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7beaffd640 1 --2- 172.21.15.64:0/2684644623 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec06d400 0x7f7bec0ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7beb7fe640 1 -- 172.21.15.64:0/2684644623 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec06d400 msgr2=0x7f7bec0ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7beb7fe640 1 --2- 172.21.15.64:0/2684644623 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec06d400 0x7f7bec0ae700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7beb7fe640 1 -- 172.21.15.64:0/2684644623 >> v1:172.21.15.64:6789/0 conn(0x7f7bec0aec40 legacy=0x7f7bec070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7beb7fe640 1 -- 172.21.15.64:0/2684644623 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7bec06c920 con 0x7f7bec15bac0 2024-03-04T07:37:18.840 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.838+0000 7f7bea7fc640 1 -- 172.21.15.64:0/2684644623 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3689964784 0 0) 0x7f7bec0729b0 con 0x7f7bec0aec40 2024-03-04T07:37:18.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.839+0000 7f7beb7fe640 1 --2- 172.21.15.64:0/2684644623 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec15bac0 0x7f7bec15cbe0 secure :-1 s=READY pgs=1841 cs=0 l=1 rev1=1 crypto rx=0x7f7bdc0097a0 tx=0x7f7bdc02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5ff27c4c51b47ca1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.839+0000 7f7bea7fc640 1 -- 172.21.15.64:0/2684644623 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7bdc035020 con 0x7f7bec15bac0 2024-03-04T07:37:18.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.839+0000 7f7bea7fc640 1 -- 172.21.15.64:0/2684644623 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7bdc031830 con 0x7f7bec15bac0 2024-03-04T07:37:18.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.839+0000 7f7bea7fc640 1 -- 172.21.15.64:0/2684644623 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7bdc031c00 con 0x7f7bec15bac0 2024-03-04T07:37:18.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 -- 172.21.15.64:0/2684644623 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec15bac0 msgr2=0x7f7bec15cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2684644623 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec15bac0 0x7f7bec15cbe0 secure :-1 s=READY pgs=1841 cs=0 l=1 rev1=1 crypto rx=0x7f7bdc0097a0 tx=0x7f7bdc02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.842 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 -- 172.21.15.64:0/2684644623 shutdown_connections 2024-03-04T07:37:18.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2684644623 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec06d400 0x7f7bec0ae700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2684644623 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec15bac0 0x7f7bec15cbe0 unknown :-1 s=CLOSED pgs=1841 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 -- 172.21.15.64:0/2684644623 >> 172.21.15.64:0/2684644623 conn(0x7f7bec066d00 msgr2=0x7f7bec0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:18.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 -- 172.21.15.64:0/2684644623 shutdown_connections 2024-03-04T07:37:18.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.840+0000 7f7bf204e640 1 -- 172.21.15.64:0/2684644623 wait complete. 2024-03-04T07:37:18.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.841+0000 7f7bf204e640 1 Processor -- start 2024-03-04T07:37:18.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.841+0000 7f7bf204e640 1 -- start start 2024-03-04T07:37:18.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7bf204e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7bec06d400 0x7f7bec147150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7bf204e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec0aec40 0x7f7bec1e42f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7bf204e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec15bac0 0x7f7bec1e5a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7bf204e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7bec15bea0 con 0x7f7bec0aec40 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7bf204e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7bec0cc490 con 0x7f7bec06d400 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7bf204e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7bec15c020 con 0x7f7bec15bac0 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec0aec40 0x7f7bec1e42f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec0aec40 0x7f7bec1e42f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42430/0 (socket says 172.21.15.64:42430) 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 -- 172.21.15.64:0/2306961249 learned_addr learned my addr 172.21.15.64:0/2306961249 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beb7fe640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7bec06d400 0x7f7bec147150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7bebfff640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec15bac0 0x7f7bec1e5a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 -- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec15bac0 msgr2=0x7f7bec1e5a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec15bac0 0x7f7bec1e5a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 -- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7bec06d400 msgr2=0x7f7bec147150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:18.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7bec06d400 0x7f7bec147150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.842+0000 7f7beaffd640 1 -- 172.21.15.64:0/2306961249 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7bec1e4cd0 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.843+0000 7f7beaffd640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec0aec40 0x7f7bec1e42f0 secure :-1 s=READY pgs=1842 cs=0 l=1 rev1=1 crypto rx=0x7f7be000e9e0 tx=0x7f7be000eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.843+0000 7f7bd3fff640 1 -- 172.21.15.64:0/2306961249 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7be0017070 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.843+0000 7f7bd3fff640 1 -- 172.21.15.64:0/2306961249 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7be000c9d0 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.843+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7bec1e4fb0 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.843+0000 7f7bd3fff640 1 -- 172.21.15.64:0/2306961249 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7be000cdc0 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.843+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7bec157e20 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.844+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7bec067ff0 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.844+0000 7f7bd3fff640 1 -- 172.21.15.64:0/2306961249 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7be001a070 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.845+0000 7f7bd3fff640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7bd806b4d0 0x7f7bd806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.845+0000 7f7bd3fff640 1 -- 172.21.15.64:0/2306961249 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f7be008a4b0 con 0x7f7bec0aec40 2024-03-04T07:37:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.845+0000 7f7beb7fe640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7bd806b4d0 0x7f7bd806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:18.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.845+0000 7f7beb7fe640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7bd806b4d0 0x7f7bd806d990 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7f7bdc036000 tx=0x7f7bdc008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:18.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:18.849+0000 7f7bd3fff640 1 -- 172.21.15.64:0/2306961249 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7be0058d30 con 0x7f7bec0aec40 2024-03-04T07:37:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.032+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f7bec0aea20 con 0x7f7bec0aec40 2024-03-04T07:37:19.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.033+0000 7f7bd3fff640 1 -- 172.21.15.64:0/2306961249 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v127) v1 ==== 74+0+16673 (secure 0 0 0) 0x7f7be005cdd0 con 0x7f7bec0aec40 2024-03-04T07:37:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:19.034 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":127,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:34:36.563800+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:33:34.914683+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-04T07:33:36.218855+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"123","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.25,"score_stable":1.25,"optimal_score":1,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"19.25","mappings":[{"from":2,"to":0}]},{"pgid":"19.3f","mappings":[{"from":2,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:37:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7bd806b4d0 msgr2=0x7f7bd806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7bd806b4d0 0x7f7bd806d990 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7f7bdc036000 tx=0x7f7bdc008030 comp rx=0 tx=0).stop 2024-03-04T07:37:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec0aec40 msgr2=0x7f7bec1e42f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.036 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec0aec40 0x7f7bec1e42f0 secure :-1 s=READY pgs=1842 cs=0 l=1 rev1=1 crypto rx=0x7f7be000e9e0 tx=0x7f7be000eea0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 shutdown_connections 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7bd806b4d0 0x7f7bd806d990 unknown :-1 s=CLOSED pgs=840 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7bec15bac0 0x7f7bec1e5a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7bec0aec40 0x7f7bec1e42f0 unknown :-1 s=CLOSED pgs=1842 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 --2- 172.21.15.64:0/2306961249 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7bec06d400 0x7f7bec147150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 >> 172.21.15.64:0/2306961249 conn(0x7f7bec066d00 msgr2=0x7f7bec0b1420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 shutdown_connections 2024-03-04T07:37:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.035+0000 7f7bf204e640 1 -- 172.21.15.64:0/2306961249 wait complete. 2024-03-04T07:37:19.047 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:37:19.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.148+0000 7f0cc6f0c640 1 Processor -- start 2024-03-04T07:37:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.148+0000 7f0cc6f0c640 1 -- start start 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc6f0c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0cc015bea0 0x7f0cc00cd330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc6f0c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0cc00d0220 0x7f0cc006eb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc6f0c640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f0cc00cf270 con 0x7f0cc00cf660 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc6f0c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0cc015c280 con 0x7f0cc00d0220 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc6f0c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f0cc00cc850 con 0x7f0cc015bea0 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc54a9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0cc00d0220 0x7f0cc006eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc4ca8640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f0cc015bea0 0x7f0cc00cd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc54a9640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f0cc00d0220 0x7f0cc006eb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42436/0 (socket says 172.21.15.64:42436) 2024-03-04T07:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc54a9640 1 -- 172.21.15.64:0/1103262689 learned_addr learned my addr 172.21.15.64:0/1103262689 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cbffff640 1 --1- 172.21.15.64:0/1103262689 >> v1:172.21.15.66:6789/0 conn(0x7f0cc00cf660 0x7f0cc00cfb10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49600/0 (socket says 172.21.15.64:49600) 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc54a9640 1 -- 172.21.15.64:0/1103262689 >> v2:172.21.15.66:3300/0 conn(0x7f0cc015bea0 msgr2=0x7f0cc00cd330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc54a9640 1 --2- 172.21.15.64:0/1103262689 >> v2:172.21.15.66:3300/0 conn(0x7f0cc015bea0 0x7f0cc00cd330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc54a9640 1 -- 172.21.15.64:0/1103262689 >> v1:172.21.15.66:6789/0 conn(0x7f0cc00cf660 legacy=0x7f0cc00cfb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.149+0000 7f0cc54a9640 1 -- 172.21.15.64:0/1103262689 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cc00ccb30 con 0x7f0cc00d0220 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.150+0000 7f0cc54a9640 1 --2- 172.21.15.64:0/1103262689 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc006eb00 secure :-1 s=READY pgs=1843 cs=0 l=1 rev1=1 crypto rx=0x7f0cc0071fa0 tx=0x7f0cb802bee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=20afddfbf16e8206 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.150+0000 7f0cbf7fe640 1 -- 172.21.15.64:0/1103262689 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cb802ca80 con 0x7f0cc00d0220 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.150+0000 7f0cbf7fe640 1 -- 172.21.15.64:0/1103262689 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0cb802cc20 con 0x7f0cc00d0220 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.150+0000 7f0cbf7fe640 1 -- 172.21.15.64:0/1103262689 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cb80343f0 con 0x7f0cc00d0220 2024-03-04T07:37:19.153 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1103262689 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 msgr2=0x7f0cc006eb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1103262689 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc006eb00 secure :-1 s=READY pgs=1843 cs=0 l=1 rev1=1 crypto rx=0x7f0cc0071fa0 tx=0x7f0cb802bee0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1103262689 shutdown_connections 2024-03-04T07:37:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1103262689 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc006eb00 unknown :-1 s=CLOSED pgs=1843 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1103262689 >> v2:172.21.15.66:3300/0 conn(0x7f0cc015bea0 0x7f0cc00cd330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1103262689 >> 172.21.15.64:0/1103262689 conn(0x7f0cc00d1a90 msgr2=0x7f0cc00d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1103262689 shutdown_connections 2024-03-04T07:37:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.151+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1103262689 wait complete. 2024-03-04T07:37:19.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.152+0000 7f0cc6f0c640 1 Processor -- start 2024-03-04T07:37:19.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.152+0000 7f0cc6f0c640 1 -- start start 2024-03-04T07:37:19.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc6f0c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cc00cf660 0x7f0cc0156000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc6f0c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc01f1ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc6f0c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cc015bea0 0x7f0cc00b3020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc6f0c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cc00cc500 con 0x7f0cc00d0220 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc6f0c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cc00cd870 con 0x7f0cc015bea0 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc6f0c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0cc00ceeb0 con 0x7f0cc00cf660 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc01f1ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc01f1ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42438/0 (socket says 172.21.15.64:42438) 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 -- 172.21.15.64:0/1786703004 learned_addr learned my addr 172.21.15.64:0/1786703004 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc4ca8640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cc00cf660 0x7f0cc0156000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc54a9640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cc015bea0 0x7f0cc00b3020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 -- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cc00cf660 msgr2=0x7f0cc0156000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cc00cf660 0x7f0cc0156000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 -- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cc015bea0 msgr2=0x7f0cc00b3020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cc015bea0 0x7f0cc00b3020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 -- 172.21.15.64:0/1786703004 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cc01f2860 con 0x7f0cc00d0220 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbffff640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc01f1ec0 secure :-1 s=READY pgs=1844 cs=0 l=1 rev1=1 crypto rx=0x7f0cac00c440 tx=0x7f0cac00c900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbd7fa640 1 -- 172.21.15.64:0/1786703004 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cac0074e0 con 0x7f0cc00d0220 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbd7fa640 1 -- 172.21.15.64:0/1786703004 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0cac007680 con 0x7f0cc00d0220 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0cc01f2b40 con 0x7f0cc00d0220 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.153+0000 7f0cbd7fa640 1 -- 172.21.15.64:0/1786703004 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0cac004a20 con 0x7f0cc00d0220 2024-03-04T07:37:19.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.154+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0cc01f34f0 con 0x7f0cc00d0220 2024-03-04T07:37:19.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.155+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0cc01f36f0 con 0x7f0cc00d0220 2024-03-04T07:37:19.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.155+0000 7f0cbd7fa640 1 -- 172.21.15.64:0/1786703004 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f0cac01a020 con 0x7f0cc00d0220 2024-03-04T07:37:19.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.155+0000 7f0cbd7fa640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ca406b500 0x7f0ca406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.155+0000 7f0cc4ca8640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ca406b500 0x7f0ca406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.155+0000 7f0cbd7fa640 1 -- 172.21.15.64:0/1786703004 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f0cac0897d0 con 0x7f0cc00d0220 2024-03-04T07:37:19.160 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.156+0000 7f0cc4ca8640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ca406b500 0x7f0ca406d9c0 secure :-1 s=READY pgs=841 cs=0 l=1 rev1=1 crypto rx=0x7f0cb4009aa0 tx=0x7f0cb4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.160+0000 7f0cbd7fa640 1 -- 172.21.15.64:0/1786703004 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f0cc01f36f0 con 0x7f0cc00d0220 2024-03-04T07:37:19.382 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.382+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0cc0067ff0 con 0x7f0cc00d0220 2024-03-04T07:37:19.383 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.382+0000 7f0cbd7fa640 1 -- 172.21.15.64:0/1786703004 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7f0cac057fc0 con 0x7f0cc00d0220 2024-03-04T07:37:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:19.383 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:19.384 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ca406b500 msgr2=0x7f0ca406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ca406b500 0x7f0ca406d9c0 secure :-1 s=READY pgs=841 cs=0 l=1 rev1=1 crypto rx=0x7f0cb4009aa0 tx=0x7f0cb4009190 comp rx=0 tx=0).stop 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 msgr2=0x7f0cc01f1ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc01f1ec0 secure :-1 s=READY pgs=1844 cs=0 l=1 rev1=1 crypto rx=0x7f0cac00c440 tx=0x7f0cac00c900 comp rx=0 tx=0).stop 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 shutdown_connections 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f0ca406b500 0x7f0ca406d9c0 unknown :-1 s=CLOSED pgs=841 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f0cc015bea0 0x7f0cc00b3020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f0cc00d0220 0x7f0cc01f1ec0 unknown :-1 s=CLOSED pgs=1844 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 --2- 172.21.15.64:0/1786703004 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f0cc00cf660 0x7f0cc0156000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 >> 172.21.15.64:0/1786703004 conn(0x7f0cc00d1a90 msgr2=0x7f0cc00cfaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 shutdown_connections 2024-03-04T07:37:19.386 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.384+0000 7f0cc6f0c640 1 -- 172.21.15.64:0/1786703004 wait complete. 2024-03-04T07:37:19.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:19.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.497+0000 7fac4b888640 1 Processor -- start 2024-03-04T07:37:19.498 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.497+0000 7fac4b888640 1 -- start start 2024-03-04T07:37:19.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac4b888640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072090 0x7fac4415ef80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.499 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac4b888640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fac440b4160 con 0x7fac44072660 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac4b888640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fac440b5b40 con 0x7fac4406e700 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac4b888640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fac440b3ba0 con 0x7fac44072090 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac48e23640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fac44072660 0x7fac44071980 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42618/0 (socket says 172.21.15.64:42618) 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac48e23640 1 -- 172.21.15.64:0/582085059 learned_addr learned my addr 172.21.15.64:0/582085059 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac49e25640 1 --2- 172.21.15.64:0/582085059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072090 0x7fac4415ef80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac3bfff640 1 -- 172.21.15.64:0/582085059 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 259602872 0 0) 0x7fac440b5b40 con 0x7fac4406e700 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac3bfff640 1 -- 172.21.15.64:0/582085059 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fac30003520 con 0x7fac4406e700 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac3bfff640 1 -- 172.21.15.64:0/582085059 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2566457232 0 0) 0x7fac440b4160 con 0x7fac44072660 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac3bfff640 1 -- 172.21.15.64:0/582085059 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fac440b5b40 con 0x7fac44072660 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac49e25640 1 -- 172.21.15.64:0/582085059 >> v1:172.21.15.66:6789/0 conn(0x7fac4406e700 legacy=0x7fac4406d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac49e25640 1 -- 172.21.15.64:0/582085059 >> v1:172.21.15.64:6789/0 conn(0x7fac44072660 legacy=0x7fac44071980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.498+0000 7fac49e25640 1 -- 172.21.15.64:0/582085059 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac440b3e80 con 0x7fac44072090 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.499+0000 7fac49e25640 1 --2- 172.21.15.64:0/582085059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072090 0x7fac4415ef80 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fac40009dc0 tx=0x7fac4002bf20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5e107cdf20abde31 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.499+0000 7fac3bfff640 1 -- 172.21.15.64:0/582085059 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac40039070 con 0x7fac44072090 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.499+0000 7fac3bfff640 1 -- 172.21.15.64:0/582085059 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fac4002ccd0 con 0x7fac44072090 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.499+0000 7fac3bfff640 1 -- 172.21.15.64:0/582085059 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac40034640 con 0x7fac44072090 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.500+0000 7fac4b888640 1 -- 172.21.15.64:0/582085059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072090 msgr2=0x7fac4415ef80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.500+0000 7fac4b888640 1 --2- 172.21.15.64:0/582085059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072090 0x7fac4415ef80 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fac40009dc0 tx=0x7fac4002bf20 comp rx=0 tx=0).stop 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.500+0000 7fac4b888640 1 -- 172.21.15.64:0/582085059 shutdown_connections 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.500+0000 7fac4b888640 1 --2- 172.21.15.64:0/582085059 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072090 0x7fac4415ef80 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.500+0000 7fac4b888640 1 -- 172.21.15.64:0/582085059 >> 172.21.15.64:0/582085059 conn(0x7fac440b2720 msgr2=0x7fac440b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.500+0000 7fac4b888640 1 -- 172.21.15.64:0/582085059 shutdown_connections 2024-03-04T07:37:19.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.500+0000 7fac4b888640 1 -- 172.21.15.64:0/582085059 wait complete. 2024-03-04T07:37:19.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.501+0000 7fac4b888640 1 Processor -- start 2024-03-04T07:37:19.503 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.501+0000 7fac4b888640 1 -- start start 2024-03-04T07:37:19.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac4b888640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac4406e700 0x7fac441f3080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.504 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac4b888640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac44072090 0x7fac44161860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac4b888640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072660 0x7fac44161da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac4b888640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac440b5750 con 0x7fac4406e700 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac4b888640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac440b55d0 con 0x7fac44072090 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac4b888640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fac440b58d0 con 0x7fac44072660 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac49624640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac4406e700 0x7fac441f3080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac49e25640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072660 0x7fac44161da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac49624640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac4406e700 0x7fac441f3080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42444/0 (socket says 172.21.15.64:42444) 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac48e23640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac44072090 0x7fac44161860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac49624640 1 -- 172.21.15.64:0/4235128557 learned_addr learned my addr 172.21.15.64:0/4235128557 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac48e23640 1 -- 172.21.15.64:0/4235128557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072660 msgr2=0x7fac44161da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac48e23640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072660 0x7fac44161da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac48e23640 1 -- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac4406e700 msgr2=0x7fac441f3080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac48e23640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac4406e700 0x7fac441f3080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac48e23640 1 -- 172.21.15.64:0/4235128557 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac44159d40 con 0x7fac44072090 2024-03-04T07:37:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac49624640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac4406e700 0x7fac441f3080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.502+0000 7fac48e23640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac44072090 0x7fac44161860 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7fac44070400 tx=0x7fac3400c820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.503+0000 7fac39ffb640 1 -- 172.21.15.64:0/4235128557 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac34017070 con 0x7fac44072090 2024-03-04T07:37:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.503+0000 7fac39ffb640 1 -- 172.21.15.64:0/4235128557 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fac34005ce0 con 0x7fac44072090 2024-03-04T07:37:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.503+0000 7fac39ffb640 1 -- 172.21.15.64:0/4235128557 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac340072d0 con 0x7fac44072090 2024-03-04T07:37:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.503+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fac4415a020 con 0x7fac44072090 2024-03-04T07:37:19.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.503+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fac4415a5e0 con 0x7fac44072090 2024-03-04T07:37:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.504+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fac44067ff0 con 0x7fac44072090 2024-03-04T07:37:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.504+0000 7fac39ffb640 1 -- 172.21.15.64:0/4235128557 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fac3401a070 con 0x7fac44072090 2024-03-04T07:37:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.505+0000 7fac39ffb640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac2806b4d0 0x7fac2806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.505+0000 7fac39ffb640 1 -- 172.21.15.64:0/4235128557 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fac340893a0 con 0x7fac44072090 2024-03-04T07:37:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.505+0000 7fac49624640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac2806b4d0 0x7fac2806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.505+0000 7fac49624640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac2806b4d0 0x7fac2806d990 secure :-1 s=READY pgs=842 cs=0 l=1 rev1=1 crypto rx=0x7fac3c007fd0 tx=0x7fac3c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.509+0000 7fac39ffb640 1 -- 172.21.15.64:0/4235128557 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fac34057c20 con 0x7fac44072090 2024-03-04T07:37:19.728 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.727+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fac44072c30 con 0x7fac44072090 2024-03-04T07:37:19.729 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.728+0000 7fac39ffb640 1 -- 172.21.15.64:0/4235128557 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 91 v91) v1 ==== 94+0+3459 (secure 0 0 0) 0x7fac3405bcc0 con 0x7fac44072090 2024-03-04T07:37:19.729 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:19.729 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":91,"default_fscid":9,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":91,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-03-04T07:33:37.247681+0000","modified":"2024-03-04T07:36:06.905493+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"max_mds":1,"in":[0,1],"up":{"mds_0":8577,"mds_1":6728},"failed":[],"damaged":[],"stopped":[],"info":{"gid_6728":{"gid":6728,"name":"b","rank":1,"incarnation":83,"state":"up:stopping","state_seq":5,"addr":"172.21.15.66:6835/2326824208","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6833","nonce":2326824208},{"type":"v1","addr":"172.21.15.66:6835","nonce":2326824208}]},"join_fscid":-1,"laggy_since":"2024-03-04T07:34:33.556697+0000","export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}},"gid_8577":{"gid":8577,"name":"d","rank":0,"incarnation":78,"state":"up:active","state_seq":3,"addr":"172.21.15.66:6834/546842609","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":546842609},{"type":"v1","addr":"172.21.15.66:6834","nonce":546842609}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}}}},"data_pools":[19],"metadata_pool":18,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":8577,"qdb_cluster":[8577]},"id":9}]} 2024-03-04T07:37:19.729 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 91 2024-03-04T07:37:19.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac2806b4d0 msgr2=0x7fac2806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac2806b4d0 0x7fac2806d990 secure :-1 s=READY pgs=842 cs=0 l=1 rev1=1 crypto rx=0x7fac3c007fd0 tx=0x7fac3c009040 comp rx=0 tx=0).stop 2024-03-04T07:37:19.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac44072090 msgr2=0x7fac44161860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.731 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac44072090 0x7fac44161860 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7fac44070400 tx=0x7fac3400c820 comp rx=0 tx=0).stop 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 shutdown_connections 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fac2806b4d0 0x7fac2806d990 unknown :-1 s=CLOSED pgs=842 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fac44072660 0x7fac44161da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fac44072090 0x7fac44161860 unknown :-1 s=CLOSED pgs=684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 --2- 172.21.15.64:0/4235128557 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fac4406e700 0x7fac441f3080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 >> 172.21.15.64:0/4235128557 conn(0x7fac440b2720 msgr2=0x7fac4406d9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 shutdown_connections 2024-03-04T07:37:19.732 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.730+0000 7fac4b888640 1 -- 172.21.15.64:0/4235128557 wait complete. 2024-03-04T07:37:19.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:37:19.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.842+0000 7fc405f72640 1 Processor -- start 2024-03-04T07:37:19.843 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.842+0000 7fc405f72640 1 -- start start 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc405f72640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc405f72640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fc4000b2b50 0x7fc4000b6e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc405f72640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fc4000ce610 con 0x7fc400071540 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc405f72640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fc4000d2f70 con 0x7fc4000b2b50 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc405f72640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc4000d4be0 con 0x7fc4000d3400 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc3ff7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fc400071540 0x7fc400071920 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42620/0 (socket says 172.21.15.64:42620) 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc3ff7fe640 1 -- 172.21.15.64:0/1022110789 learned_addr learned my addr 172.21.15.64:0/1022110789 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc3fffff640 1 --2- 172.21.15.64:0/1022110789 >> v2:172.21.15.66:3300/0 conn(0x7fc4000b2b50 0x7fc4000b6e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.843+0000 7fc3feffd640 1 --2- 172.21.15.64:0/1022110789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.845 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.844+0000 7fc3fe7fc640 1 -- 172.21.15.64:0/1022110789 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2986014747 0 0) 0x7fc4000ce610 con 0x7fc400071540 2024-03-04T07:37:19.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.844+0000 7fc3fe7fc640 1 -- 172.21.15.64:0/1022110789 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fc3e8003520 con 0x7fc400071540 2024-03-04T07:37:19.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.844+0000 7fc3feffd640 1 -- 172.21.15.64:0/1022110789 >> v2:172.21.15.66:3300/0 conn(0x7fc4000b2b50 msgr2=0x7fc4000b6e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:19.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.844+0000 7fc3feffd640 1 --2- 172.21.15.64:0/1022110789 >> v2:172.21.15.66:3300/0 conn(0x7fc4000b2b50 0x7fc4000b6e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.844+0000 7fc3feffd640 1 -- 172.21.15.64:0/1022110789 >> v1:172.21.15.64:6789/0 conn(0x7fc400071540 legacy=0x7fc400071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.846 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.844+0000 7fc3feffd640 1 -- 172.21.15.64:0/1022110789 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4000d2920 con 0x7fc4000d3400 2024-03-04T07:37:19.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.844+0000 7fc3feffd640 1 --2- 172.21.15.64:0/1022110789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b30a0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fc3f40097a0 tx=0x7fc3f402b8d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=86a1441970791183 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.847 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc3fe7fc640 1 -- 172.21.15.64:0/1022110789 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f4035020 con 0x7fc4000d3400 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc3fe7fc640 1 -- 172.21.15.64:0/1022110789 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3f4031830 con 0x7fc4000d3400 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc3fe7fc640 1 -- 172.21.15.64:0/1022110789 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f4031c00 con 0x7fc4000d3400 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 -- 172.21.15.64:0/1022110789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 msgr2=0x7fc4000b30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 --2- 172.21.15.64:0/1022110789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b30a0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fc3f40097a0 tx=0x7fc3f402b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 -- 172.21.15.64:0/1022110789 shutdown_connections 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 --2- 172.21.15.64:0/1022110789 >> v2:172.21.15.66:3300/0 conn(0x7fc4000b2b50 0x7fc4000b6e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 --2- 172.21.15.64:0/1022110789 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b30a0 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 -- 172.21.15.64:0/1022110789 >> 172.21.15.64:0/1022110789 conn(0x7fc4000d3ca0 msgr2=0x7fc4000d40b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 -- 172.21.15.64:0/1022110789 shutdown_connections 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.845+0000 7fc405f72640 1 -- 172.21.15.64:0/1022110789 wait complete. 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.846+0000 7fc405f72640 1 Processor -- start 2024-03-04T07:37:19.848 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc405f72640 1 -- start start 2024-03-04T07:37:19.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc405f72640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc400071540 0x7fc4001f4450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc405f72640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc4000b2b50 0x7fc4001f0dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc405f72640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b5790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.850 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc405f72640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc40015e370 con 0x7fc4000b2b50 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc405f72640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4000ce8f0 con 0x7fc400071540 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc405f72640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fc4000b4910 con 0x7fc4000d3400 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc3feffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc4000b2b50 0x7fc4001f0dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc3feffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc4000b2b50 0x7fc4001f0dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42452/0 (socket says 172.21.15.64:42452) 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc3feffd640 1 -- 172.21.15.64:0/1138497011 learned_addr learned my addr 172.21.15.64:0/1138497011 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc3ff7fe640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc400071540 0x7fc4001f4450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.847+0000 7fc3fffff640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b5790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3feffd640 1 -- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 msgr2=0x7fc4000b5790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3feffd640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b5790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3feffd640 1 -- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc400071540 msgr2=0x7fc4001f4450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3feffd640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc400071540 0x7fc4001f4450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3feffd640 1 -- 172.21.15.64:0/1138497011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4000b3d00 con 0x7fc4000b2b50 2024-03-04T07:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3feffd640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc4000b2b50 0x7fc4001f0dd0 secure :-1 s=READY pgs=1847 cs=0 l=1 rev1=1 crypto rx=0x7fc3f4000c00 tx=0x7fc3f40046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3e7fff640 1 -- 172.21.15.64:0/1138497011 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f4040070 con 0x7fc4000b2b50 2024-03-04T07:37:19.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3e7fff640 1 -- 172.21.15.64:0/1138497011 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc3f403a910 con 0x7fc4000b2b50 2024-03-04T07:37:19.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc4000b3fe0 con 0x7fc4000b2b50 2024-03-04T07:37:19.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc3e7fff640 1 -- 172.21.15.64:0/1138497011 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f4035040 con 0x7fc4000b2b50 2024-03-04T07:37:19.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.848+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc4000b45a0 con 0x7fc4000b2b50 2024-03-04T07:37:19.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.849+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc400068700 con 0x7fc4000b2b50 2024-03-04T07:37:19.852 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.850+0000 7fc3e7fff640 1 -- 172.21.15.64:0/1138497011 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fc3f404b060 con 0x7fc4000b2b50 2024-03-04T07:37:19.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.850+0000 7fc3e7fff640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:19.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.850+0000 7fc3e7fff640 1 -- 172.21.15.64:0/1138497011 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7fc3f40ab730 con 0x7fc4000b2b50 2024-03-04T07:37:19.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.850+0000 7fc3ff7fe640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:19.853 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.850+0000 7fc3ff7fe640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7fc3f0005880 tx=0x7fc3f0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:19.855 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:19.854+0000 7fc3e7fff640 1 -- 172.21.15.64:0/1138497011 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fc3f4079fb0 con 0x7fc4000b2b50 2024-03-04T07:37:20.039 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.038+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc4000caae0 con 0x7fc4000b2b50 2024-03-04T07:37:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.040+0000 7fc3e7fff640 1 -- 172.21.15.64:0/1138497011 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v127) v1 ==== 74+0+16673 (secure 0 0 0) 0x7fc3f407e050 con 0x7fc4000b2b50 2024-03-04T07:37:20.041 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:20.041 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":127,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:34:36.563800+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":18,"pool_name":"cephfs_metadata","create_time":"2024-03-04T07:33:34.914683+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"122","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":19,"pool_name":"cephfs_data","create_time":"2024-03-04T07:33:36.218855+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"123","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.25,"score_stable":1.25,"optimal_score":1,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"19.25","mappings":[{"from":2,"to":0}]},{"pgid":"19.3f","mappings":[{"from":2,"to":0}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.041+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3ec06b4d0 msgr2=0x7fc3ec06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.041+0000 7fc405f72640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7fc3f0005880 tx=0x7fc3f0009040 comp rx=0 tx=0).stop 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc4000b2b50 msgr2=0x7fc4001f0dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc4000b2b50 0x7fc4001f0dd0 secure :-1 s=READY pgs=1847 cs=0 l=1 rev1=1 crypto rx=0x7fc3f4000c00 tx=0x7fc3f40046c0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 shutdown_connections 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fc3ec06b4d0 0x7fc3ec06d990 unknown :-1 s=CLOSED pgs=843 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fc4000d3400 0x7fc4000b5790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fc4000b2b50 0x7fc4001f0dd0 unknown :-1 s=CLOSED pgs=1847 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 --2- 172.21.15.64:0/1138497011 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fc400071540 0x7fc4001f4450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 >> 172.21.15.64:0/1138497011 conn(0x7fc4000d3ca0 msgr2=0x7fc400160b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:20.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 shutdown_connections 2024-03-04T07:37:20.044 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.042+0000 7fc405f72640 1 -- 172.21.15.64:0/1138497011 wait complete. 2024-03-04T07:37:20.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-04T07:37:20.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.155+0000 7f864a081640 1 Processor -- start 2024-03-04T07:37:20.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f864a081640 1 -- start start 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f864a081640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f864415df00 0x7f86440d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f864a081640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86440b95a0 0x7f86440b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f864a081640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f86440cea30 con 0x7f86440b9a30 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f864a081640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f86440d2af0 con 0x7f864415df00 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f864a081640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f86440cf3c0 con 0x7f86440b95a0 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f86437fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f864415df00 0x7f86440d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f8642ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86440b95a0 0x7f86440b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f8643fff640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f86440b9a30 0x7f86440d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:49610/0 (socket says 172.21.15.64:49610) 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.156+0000 7f8643fff640 1 -- 172.21.15.64:0/1064518450 learned_addr learned my addr 172.21.15.64:0/1064518450 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:20.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.157+0000 7f86427fc640 1 -- 172.21.15.64:0/1064518450 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2636995702 0 0) 0x7f86440cea30 con 0x7f86440b9a30 2024-03-04T07:37:20.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.157+0000 7f86437fe640 1 -- 172.21.15.64:0/1064518450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86440b95a0 msgr2=0x7f86440b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:20.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.157+0000 7f86437fe640 1 --2- 172.21.15.64:0/1064518450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86440b95a0 0x7f86440b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.157+0000 7f86437fe640 1 -- 172.21.15.64:0/1064518450 >> v1:172.21.15.66:6789/0 conn(0x7f86440b9a30 legacy=0x7f86440d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.157+0000 7f86437fe640 1 -- 172.21.15.64:0/1064518450 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f86440cf0e0 con 0x7f864415df00 2024-03-04T07:37:20.159 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.157+0000 7f8642ffd640 1 --2- 172.21.15.64:0/1064518450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86440b95a0 0x7f86440b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.157+0000 7f86437fe640 1 --2- 172.21.15.64:0/1064518450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440d25b0 secure :-1 s=READY pgs=1848 cs=0 l=1 rev1=1 crypto rx=0x7f862c0097a0 tx=0x7f862c02b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4475713062df94b1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f86427fc640 1 -- 172.21.15.64:0/1064518450 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f862c035020 con 0x7f864415df00 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f86427fc640 1 -- 172.21.15.64:0/1064518450 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f862c031810 con 0x7f864415df00 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f86427fc640 1 -- 172.21.15.64:0/1064518450 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f862c031be0 con 0x7f864415df00 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 -- 172.21.15.64:0/1064518450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 msgr2=0x7f86440d25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 --2- 172.21.15.64:0/1064518450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440d25b0 secure :-1 s=READY pgs=1848 cs=0 l=1 rev1=1 crypto rx=0x7f862c0097a0 tx=0x7f862c02b8b0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 -- 172.21.15.64:0/1064518450 shutdown_connections 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 --2- 172.21.15.64:0/1064518450 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86440b95a0 0x7f86440b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 --2- 172.21.15.64:0/1064518450 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440d25b0 unknown :-1 s=CLOSED pgs=1848 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 -- 172.21.15.64:0/1064518450 >> 172.21.15.64:0/1064518450 conn(0x7f8644071960 msgr2=0x7f8644071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 -- 172.21.15.64:0/1064518450 shutdown_connections 2024-03-04T07:37:20.161 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.158+0000 7f864a081640 1 -- 172.21.15.64:0/1064518450 wait complete. 2024-03-04T07:37:20.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.159+0000 7f864a081640 1 Processor -- start 2024-03-04T07:37:20.162 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f864a081640 1 -- start start 2024-03-04T07:37:20.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f864a081640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86440b95a0 0x7f864414e130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f864a081640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440c9fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.164 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f864a081640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86441f3c60 0x7f86440b5210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f864a081640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86440cec70 con 0x7f864415df00 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f864a081640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86440ce750 con 0x7f86440b95a0 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f864a081640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f86440cedf0 con 0x7f86441f3c60 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f8642ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440c9fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f8642ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440c9fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42472/0 (socket says 172.21.15.64:42472) 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f8642ffd640 1 -- 172.21.15.64:0/3375711542 learned_addr learned my addr 172.21.15.64:0/3375711542 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.160+0000 7f86437fe640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86440b95a0 0x7f864414e130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f8643fff640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86441f3c60 0x7f86440b5210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f8642ffd640 1 -- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86441f3c60 msgr2=0x7f86440b5210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f8642ffd640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86441f3c60 0x7f86440b5210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f8642ffd640 1 -- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86440b95a0 msgr2=0x7f864414e130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f8642ffd640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86440b95a0 0x7f864414e130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f8642ffd640 1 -- 172.21.15.64:0/3375711542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f864414e670 con 0x7f864415df00 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f86437fe640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86440b95a0 0x7f864414e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:20.165 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f8642ffd640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440c9fc0 secure :-1 s=READY pgs=1849 cs=0 l=1 rev1=1 crypto rx=0x7f863400ebe0 tx=0x7f863400c520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f863bfff640 1 -- 172.21.15.64:0/3375711542 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8634017070 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f863bfff640 1 -- 172.21.15.64:0/3375711542 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f863400cc30 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f86440cc140 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f863bfff640 1 -- 172.21.15.64:0/3375711542 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8634013470 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.161+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f86440cc700 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.162+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8644067ff0 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.163+0000 7f863bfff640 1 -- 172.21.15.64:0/3375711542 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f863401a070 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.163+0000 7f863bfff640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f862406b4d0 0x7f862406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.163+0000 7f863bfff640 1 -- 172.21.15.64:0/3375711542 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(127..127 src has 1..127) v4 ==== 5767+0+0 (secure 0 0 0) 0x7f863408a060 con 0x7f864415df00 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.163+0000 7f86437fe640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f862406b4d0 0x7f862406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.163+0000 7f86437fe640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f862406b4d0 0x7f862406d990 secure :-1 s=READY pgs=844 cs=0 l=1 rev1=1 crypto rx=0x7f862c036000 tx=0x7f862c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:20.168 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.167+0000 7f863bfff640 1 -- 172.21.15.64:0/3375711542 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f864414e670 con 0x7f864415df00 2024-03-04T07:37:20.385 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.384+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7f86440c7230 con 0x7f864415df00 2024-03-04T07:37:20.385 INFO:tasks.ceph.mon.a.smithi064.stderr:2024-03-04T07:37:20.385+0000 7ff844470640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-04T07:37:20.387 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.386+0000 7f863bfff640 1 -- 172.21.15.64:0/3375711542 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v92) v1 ==== 161+0+0 (secure 0 0 0) 0x7f86340588e0 con 0x7f864415df00 2024-03-04T07:37:20.387 INFO:teuthology.orchestra.run.smithi064.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-04T07:37:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f862406b4d0 msgr2=0x7f862406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f862406b4d0 0x7f862406d990 secure :-1 s=READY pgs=844 cs=0 l=1 rev1=1 crypto rx=0x7f862c036000 tx=0x7f862c008030 comp rx=0 tx=0).stop 2024-03-04T07:37:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 msgr2=0x7f86440c9fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.390 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440c9fc0 secure :-1 s=READY pgs=1849 cs=0 l=1 rev1=1 crypto rx=0x7f863400ebe0 tx=0x7f863400c520 comp rx=0 tx=0).stop 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 shutdown_connections 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f862406b4d0 0x7f862406d990 unknown :-1 s=CLOSED pgs=844 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f86441f3c60 0x7f86440b5210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f864415df00 0x7f86440c9fc0 unknown :-1 s=CLOSED pgs=1849 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 --2- 172.21.15.64:0/3375711542 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f86440b95a0 0x7f864414e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 >> 172.21.15.64:0/3375711542 conn(0x7f8644071960 msgr2=0x7f86440b3840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 shutdown_connections 2024-03-04T07:37:20.391 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.389+0000 7f864a081640 1 -- 172.21.15.64:0/3375711542 wait complete. 2024-03-04T07:37:20.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-04T07:37:20.426 INFO:tasks.ceph.mds.d.smithi066.stdout:starting mds.d at 2024-03-04T07:37:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.504+0000 7f3bef4ed640 1 Processor -- start 2024-03-04T07:37:20.505 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.504+0000 7f3bef4ed640 1 -- start start 2024-03-04T07:37:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bef4ed640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3be80ce960 0x7f3be80d0860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bef4ed640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3be815f730 con 0x7f3be81618e0 2024-03-04T07:37:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bef4ed640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3be815ebb0 con 0x7f3be80ce4f0 2024-03-04T07:37:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bef4ed640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3be80cfce0 con 0x7f3be80ce960 2024-03-04T07:37:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bed289640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f3be81618e0 0x7f3be806b130 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42636/0 (socket says 172.21.15.64:42636) 2024-03-04T07:37:20.506 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bed289640 1 -- 172.21.15.64:0/2308635563 learned_addr learned my addr 172.21.15.64:0/2308635563 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3beda8a640 1 --2- 172.21.15.64:0/2308635563 >> v2:172.21.15.66:3300/0 conn(0x7f3be80ce960 0x7f3be80d0860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bdffff640 1 -- 172.21.15.64:0/2308635563 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3761189500 0 0) 0x7f3be815f730 con 0x7f3be81618e0 2024-03-04T07:37:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3bdffff640 1 -- 172.21.15.64:0/2308635563 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f3bd4003520 con 0x7f3be81618e0 2024-03-04T07:37:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3beda8a640 1 -- 172.21.15.64:0/2308635563 >> v1:172.21.15.66:6789/0 conn(0x7f3be80ce4f0 legacy=0x7f3be80d0150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3beda8a640 1 -- 172.21.15.64:0/2308635563 >> v1:172.21.15.64:6789/0 conn(0x7f3be81618e0 legacy=0x7f3be806b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.507 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.505+0000 7f3beda8a640 1 -- 172.21.15.64:0/2308635563 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3be815ee90 con 0x7f3be80ce960 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.506+0000 7f3beda8a640 1 --2- 172.21.15.64:0/2308635563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 0x7f3be80d0860 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f3be0009d90 tx=0x7f3be002bfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5881ec4011746be2 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.506+0000 7f3bdffff640 1 -- 172.21.15.64:0/2308635563 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3be0039070 con 0x7f3be80ce960 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.506+0000 7f3bdffff640 1 -- 172.21.15.64:0/2308635563 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3be002cd80 con 0x7f3be80ce960 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.506+0000 7f3bdffff640 1 -- 172.21.15.64:0/2308635563 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3be0034660 con 0x7f3be80ce960 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.507+0000 7f3bef4ed640 1 -- 172.21.15.64:0/2308635563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 msgr2=0x7f3be80d0860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.507+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/2308635563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 0x7f3be80d0860 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f3be0009d90 tx=0x7f3be002bfd0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.507+0000 7f3bef4ed640 1 -- 172.21.15.64:0/2308635563 shutdown_connections 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.507+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/2308635563 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 0x7f3be80d0860 unknown :-1 s=CLOSED pgs=687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.507+0000 7f3bef4ed640 1 -- 172.21.15.64:0/2308635563 >> 172.21.15.64:0/2308635563 conn(0x7f3be80b3660 msgr2=0x7f3be80b3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.507+0000 7f3bef4ed640 1 -- 172.21.15.64:0/2308635563 shutdown_connections 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.507+0000 7f3bef4ed640 1 -- 172.21.15.64:0/2308635563 wait complete. 2024-03-04T07:37:20.509 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.508+0000 7f3bef4ed640 1 Processor -- start 2024-03-04T07:37:20.510 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.508+0000 7f3bef4ed640 1 -- start start 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bef4ed640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3be80ce4f0 0x7f3be80b68a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bef4ed640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 0x7f3be81e7b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bef4ed640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3be81e8080 0x7f3be815a4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bef4ed640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3be80cedb0 con 0x7f3be81e8080 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bef4ed640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3be815f450 con 0x7f3be80ce960 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bef4ed640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3be815e7d0 con 0x7f3be80ce4f0 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3be81e8080 0x7f3be815a4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3be81e8080 0x7f3be815a4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42486/0 (socket says 172.21.15.64:42486) 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 -- 172.21.15.64:0/1933414516 learned_addr learned my addr 172.21.15.64:0/1933414516 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beca88640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 0x7f3be81e7b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bed289640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3be80ce4f0 0x7f3be80b68a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 -- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3be80ce4f0 msgr2=0x7f3be80b68a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3be80ce4f0 0x7f3be80b68a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 -- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 msgr2=0x7f3be81e7b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 0x7f3be81e7b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 -- 172.21.15.64:0/1933414516 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3be815b1a0 con 0x7f3be81e8080 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3beda8a640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3be81e8080 0x7f3be815a4b0 secure :-1 s=READY pgs=1852 cs=0 l=1 rev1=1 crypto rx=0x7f3be0009d90 tx=0x7f3be0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bddffb640 1 -- 172.21.15.64:0/1933414516 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3be0039050 con 0x7f3be81e8080 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bddffb640 1 -- 172.21.15.64:0/1933414516 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3be0045070 con 0x7f3be81e8080 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3be815b480 con 0x7f3be81e8080 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.509+0000 7f3bddffb640 1 -- 172.21.15.64:0/1933414516 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3be0009450 con 0x7f3be81e8080 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.510+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3be815ba40 con 0x7f3be81e8080 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.510+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3be809fc50 con 0x7f3be81e8080 2024-03-04T07:37:20.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.511+0000 7f3bddffb640 1 -- 172.21.15.64:0/1933414516 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3be003e850 con 0x7f3be81e8080 2024-03-04T07:37:20.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.511+0000 7f3bddffb640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bcc06b4d0 0x7f3bcc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:20.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.512+0000 7f3bddffb640 1 -- 172.21.15.64:0/1933414516 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(128..128 src has 1..128) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f3be00acf70 con 0x7f3be81e8080 2024-03-04T07:37:20.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.512+0000 7f3bed289640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bcc06b4d0 0x7f3bcc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:20.514 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.512+0000 7f3bed289640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bcc06b4d0 0x7f3bcc06d990 secure :-1 s=READY pgs=845 cs=0 l=1 rev1=1 crypto rx=0x7f3be40011e0 tx=0x7f3be4009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:20.516 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.516+0000 7f3bddffb640 1 -- 172.21.15.64:0/1933414516 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3be007be30 con 0x7f3be81e8080 2024-03-04T07:37:20.738 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:20.737+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f3be804f410 con 0x7f3be81e8080 2024-03-04T07:37:21.421 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.420+0000 7f3bddffb640 1 -- 172.21.15.64:0/1933414516 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v93) v1 ==== 104+0+0 (secure 0 0 0) 0x7f3be007b730 con 0x7f3be81e8080 2024-03-04T07:37:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bcc06b4d0 msgr2=0x7f3bcc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bcc06b4d0 0x7f3bcc06d990 secure :-1 s=READY pgs=845 cs=0 l=1 rev1=1 crypto rx=0x7f3be40011e0 tx=0x7f3be4009040 comp rx=0 tx=0).stop 2024-03-04T07:37:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3be81e8080 msgr2=0x7f3be815a4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3be81e8080 0x7f3be815a4b0 secure :-1 s=READY pgs=1852 cs=0 l=1 rev1=1 crypto rx=0x7f3be0009d90 tx=0x7f3be0004a40 comp rx=0 tx=0).stop 2024-03-04T07:37:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 shutdown_connections 2024-03-04T07:37:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3bcc06b4d0 0x7f3bcc06d990 unknown :-1 s=CLOSED pgs=845 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.424 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3be81e8080 0x7f3be815a4b0 unknown :-1 s=CLOSED pgs=1852 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3be80ce960 0x7f3be81e7b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 --2- 172.21.15.64:0/1933414516 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3be80ce4f0 0x7f3be80b68a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 >> 172.21.15.64:0/1933414516 conn(0x7f3be80b3660 msgr2=0x7f3be8160590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 shutdown_connections 2024-03-04T07:37:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.423+0000 7f3bef4ed640 1 -- 172.21.15.64:0/1933414516 wait complete. 2024-03-04T07:37:21.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-04T07:37:21.546 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.545+0000 7f17dd866640 1 Processor -- start 2024-03-04T07:37:21.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.545+0000 7f17dd866640 1 -- start start 2024-03-04T07:37:21.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17dd866640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f17d80ceea0 0x7f17d80d29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:21.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17dd866640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d80b6d10 0x7f17d80b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17dd866640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f17d80d3740 con 0x7f17d815dd90 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17dd866640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f17d80d3400 con 0x7f17d80ceea0 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17dd866640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17d80d3120 con 0x7f17d80b6d10 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17d67fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f17d80ceea0 0x7f17d80d29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17d67fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f17d80ceea0 0x7f17d80d29d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42494/0 (socket says 172.21.15.64:42494) 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.546+0000 7f17d67fc640 1 -- 172.21.15.64:0/2358515567 learned_addr learned my addr 172.21.15.64:0/2358515567 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.547+0000 7f17d77fe640 1 --2- 172.21.15.64:0/2358515567 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d80b6d10 0x7f17d80b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.547+0000 7f17d67fc640 1 -- 172.21.15.64:0/2358515567 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d80b6d10 msgr2=0x7f17d80b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.547+0000 7f17d67fc640 1 --2- 172.21.15.64:0/2358515567 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d80b6d10 0x7f17d80b71a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.547+0000 7f17d67fc640 1 -- 172.21.15.64:0/2358515567 >> v1:172.21.15.66:6789/0 conn(0x7f17d815dd90 legacy=0x7f17d80d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.547+0000 7f17d67fc640 1 -- 172.21.15.64:0/2358515567 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17d80d39e0 con 0x7f17d80ceea0 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.547+0000 7f17d67fc640 1 --2- 172.21.15.64:0/2358515567 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80ceea0 0x7f17d80d29d0 secure :-1 s=READY pgs=1853 cs=0 l=1 rev1=1 crypto rx=0x7f17d80724d0 tx=0x7f17cc02b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=30b72cda50e77bee server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:21.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17d5ffb640 1 -- 172.21.15.64:0/2358515567 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17cc034070 con 0x7f17d80ceea0 2024-03-04T07:37:21.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17d5ffb640 1 -- 172.21.15.64:0/2358515567 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17cc030610 con 0x7f17d80ceea0 2024-03-04T07:37:21.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17d5ffb640 1 -- 172.21.15.64:0/2358515567 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17cc0309e0 con 0x7f17d80ceea0 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 -- 172.21.15.64:0/2358515567 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80ceea0 msgr2=0x7f17d80d29d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 --2- 172.21.15.64:0/2358515567 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80ceea0 0x7f17d80d29d0 secure :-1 s=READY pgs=1853 cs=0 l=1 rev1=1 crypto rx=0x7f17d80724d0 tx=0x7f17cc02b960 comp rx=0 tx=0).stop 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 -- 172.21.15.64:0/2358515567 shutdown_connections 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 --2- 172.21.15.64:0/2358515567 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d80b6d10 0x7f17d80b71a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 --2- 172.21.15.64:0/2358515567 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80ceea0 0x7f17d80d29d0 unknown :-1 s=CLOSED pgs=1853 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 -- 172.21.15.64:0/2358515567 >> 172.21.15.64:0/2358515567 conn(0x7f17d8071960 msgr2=0x7f17d8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 -- 172.21.15.64:0/2358515567 shutdown_connections 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.548+0000 7f17dd866640 1 -- 172.21.15.64:0/2358515567 wait complete. 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.549+0000 7f17dd866640 1 Processor -- start 2024-03-04T07:37:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.549+0000 7f17dd866640 1 -- start start 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17dd866640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80b6d10 0x7f17d8156f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17dd866640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17d80ceea0 0x7f17d815e650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17dd866640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d815dd90 0x7f17d80c9a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17dd866640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17d80d48f0 con 0x7f17d80b6d10 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17dd866640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17d815e170 con 0x7f17d80ceea0 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17dd866640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f17d80d4a70 con 0x7f17d815dd90 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17d6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80b6d10 0x7f17d8156f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17d6ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80b6d10 0x7f17d8156f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42504/0 (socket says 172.21.15.64:42504) 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.550+0000 7f17d6ffd640 1 -- 172.21.15.64:0/3421753056 learned_addr learned my addr 172.21.15.64:0/3421753056 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d67fc640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17d80ceea0 0x7f17d815e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d77fe640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d815dd90 0x7f17d80c9a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d6ffd640 1 -- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d815dd90 msgr2=0x7f17d80c9a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:21.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d6ffd640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d815dd90 0x7f17d80c9a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d6ffd640 1 -- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17d80ceea0 msgr2=0x7f17d815e650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d6ffd640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17d80ceea0 0x7f17d815e650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d6ffd640 1 -- 172.21.15.64:0/3421753056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17d80cae40 con 0x7f17d80b6d10 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d67fc640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17d80ceea0 0x7f17d815e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17d6ffd640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80b6d10 0x7f17d8156f10 secure :-1 s=READY pgs=1854 cs=0 l=1 rev1=1 crypto rx=0x7f17d000ba80 tx=0x7f17d000bf40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17bffff640 1 -- 172.21.15.64:0/3421753056 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17d000cc00 con 0x7f17d80b6d10 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17bffff640 1 -- 172.21.15.64:0/3421753056 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f17d000cda0 con 0x7f17d80b6d10 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17bffff640 1 -- 172.21.15.64:0/3421753056 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17d0003bb0 con 0x7f17d80b6d10 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17d80cb3d0 con 0x7f17d80b6d10 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.551+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17d8157450 con 0x7f17d80b6d10 2024-03-04T07:37:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.552+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f17d8072a30 con 0x7f17d80b6d10 2024-03-04T07:37:21.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.553+0000 7f17bffff640 1 -- 172.21.15.64:0/3421753056 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f17d0014440 con 0x7f17d80b6d10 2024-03-04T07:37:21.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.553+0000 7f17bffff640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f17c406b4d0 0x7f17c406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:21.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.553+0000 7f17bffff640 1 -- 172.21.15.64:0/3421753056 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(128..128 src has 1..128) v4 ==== 5939+0+0 (secure 0 0 0) 0x7f17d008a2b0 con 0x7f17d80b6d10 2024-03-04T07:37:21.556 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.553+0000 7f17d67fc640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f17c406b4d0 0x7f17c406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:21.557 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.553+0000 7f17d67fc640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f17c406b4d0 0x7f17c406d990 secure :-1 s=READY pgs=847 cs=0 l=1 rev1=1 crypto rx=0x7f17cc036000 tx=0x7f17cc03b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:21.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.557+0000 7f17bffff640 1 -- 172.21.15.64:0/3421753056 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f17d0058fc0 con 0x7f17d80b6d10 2024-03-04T07:37:21.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:21.742+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f17d804f410 con 0x7f17d80b6d10 2024-03-04T07:37:22.422 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.421+0000 7f17bffff640 1 -- 172.21.15.64:0/3421753056 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v129) v1 ==== 181+0+0 (secure 0 0 0) 0x7f17d00588c0 con 0x7f17d80b6d10 2024-03-04T07:37:22.422 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_metadata' removed 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.424+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f17c406b4d0 msgr2=0x7f17c406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.424+0000 7f17dd866640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f17c406b4d0 0x7f17c406d990 secure :-1 s=READY pgs=847 cs=0 l=1 rev1=1 crypto rx=0x7f17cc036000 tx=0x7f17cc03b040 comp rx=0 tx=0).stop 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.424+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80b6d10 msgr2=0x7f17d8156f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.424+0000 7f17dd866640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80b6d10 0x7f17d8156f10 secure :-1 s=READY pgs=1854 cs=0 l=1 rev1=1 crypto rx=0x7f17d000ba80 tx=0x7f17d000bf40 comp rx=0 tx=0).stop 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 shutdown_connections 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f17c406b4d0 0x7f17c406d990 unknown :-1 s=CLOSED pgs=847 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f17d815dd90 0x7f17d80c9a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f17d80ceea0 0x7f17d815e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.426 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 --2- 172.21.15.64:0/3421753056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f17d80b6d10 0x7f17d8156f10 unknown :-1 s=CLOSED pgs=1854 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 >> 172.21.15.64:0/3421753056 conn(0x7f17d8071960 msgr2=0x7f17d80b71a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:22.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 shutdown_connections 2024-03-04T07:37:22.427 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.425+0000 7f17dd866640 1 -- 172.21.15.64:0/3421753056 wait complete. 2024-03-04T07:37:22.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-04T07:37:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.547+0000 7efcae881640 1 Processor -- start 2024-03-04T07:37:22.548 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.547+0000 7efcae881640 1 -- start start 2024-03-04T07:37:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.548+0000 7efcae881640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efca815dd90 0x7efca80ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.548+0000 7efcae881640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 0x7efca80b71a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.548+0000 7efcae881640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7efca80d3740 con 0x7efca80d29d0 2024-03-04T07:37:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.548+0000 7efcae881640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efca80d3400 con 0x7efca815dd90 2024-03-04T07:37:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.548+0000 7efcae881640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efca80d3120 con 0x7efca80b6d10 2024-03-04T07:37:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.548+0000 7efca77fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7efca80d29d0 0x7efca80d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:42646/0 (socket says 172.21.15.64:42646) 2024-03-04T07:37:22.552 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.548+0000 7efca77fe640 1 -- 172.21.15.64:0/239685935 learned_addr learned my addr 172.21.15.64:0/239685935 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3724368758 0 0) 0x7efca80d3740 con 0x7efca80d29d0 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efcace1e640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 0x7efca80b71a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca7fff640 1 --2- 172.21.15.64:0/239685935 >> v2:172.21.15.66:3300/0 conn(0x7efca815dd90 0x7efca80ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7efc90003520 con 0x7efca80d29d0 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3597286319 0 0) 0x7efc90003520 con 0x7efca80d29d0 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 msgr2=0x7efca80b71a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 0x7efca80b71a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 >> v2:172.21.15.66:3300/0 conn(0x7efca815dd90 msgr2=0x7efca80ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 --2- 172.21.15.64:0/239685935 >> v2:172.21.15.66:3300/0 conn(0x7efca815dd90 0x7efca80ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efca80d39e0 con 0x7efca80d29d0 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7efc9c001690 con 0x7efca80d29d0 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 >> v1:172.21.15.64:6789/0 conn(0x7efca80d29d0 legacy=0x7efca80d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:22.553 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 0x7efc900039b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc90003f30 0x7efc90004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc90004d40 0x7efc900051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efca80d3400 con 0x7efc90003f30 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efca80d3120 con 0x7efc90004d40 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efc9000b9f0 con 0x7efca80b6d10 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca77fe640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc90003f30 0x7efc90004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca7fff640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 0x7efc900039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efcace1e640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc90004d40 0x7efc900051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca77fe640 1 -- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 msgr2=0x7efc900039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca77fe640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efca80b6d10 0x7efc900039b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca77fe640 1 -- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc90004d40 msgr2=0x7efc900051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca77fe640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efc90004d40 0x7efc900051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.549+0000 7efca77fe640 1 -- 172.21.15.64:0/239685935 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc90003520 con 0x7efc90003f30 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.550+0000 7efca77fe640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc90003f30 0x7efc90004800 secure :-1 s=READY pgs=1856 cs=0 l=1 rev1=1 crypto rx=0x7efc9c007fd0 tx=0x7efc90005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a81280238cedee5a server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.550+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc9c007d50 con 0x7efc90003f30 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.550+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efc9c030070 con 0x7efc90003f30 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.550+0000 7efca6ffd640 1 -- 172.21.15.64:0/239685935 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc9c005d70 con 0x7efc90003f30 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.551+0000 7efcae881640 1 -- 172.21.15.64:0/239685935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc90003f30 msgr2=0x7efc90004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.551+0000 7efcae881640 1 --2- 172.21.15.64:0/239685935 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efc90003f30 0x7efc90004800 secure :-1 s=READY pgs=1856 cs=0 l=1 rev1=1 crypto rx=0x7efc9c007fd0 tx=0x7efc90005830 comp rx=0 tx=0).stop 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.551+0000 7efcace1e640 1 -- 172.21.15.64:0/239685935 reap_dead start 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.551+0000 7efcae881640 1 -- 172.21.15.64:0/239685935 shutdown_connections 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.551+0000 7efcae881640 1 -- 172.21.15.64:0/239685935 >> 172.21.15.64:0/239685935 conn(0x7efca8071960 msgr2=0x7efca8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.551+0000 7efcae881640 1 -- 172.21.15.64:0/239685935 shutdown_connections 2024-03-04T07:37:22.555 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.551+0000 7efcae881640 1 -- 172.21.15.64:0/239685935 wait complete. 2024-03-04T07:37:22.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.552+0000 7efcae881640 1 Processor -- start 2024-03-04T07:37:22.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.552+0000 7efcae881640 1 -- start start 2024-03-04T07:37:22.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcae881640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc90003f30 0x7efca80cb880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcae881640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efca80b4100 0x7efca80c9d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcae881640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efca80b4580 0x7efca80ca2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcae881640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efca80d2df0 con 0x7efca80b4100 2024-03-04T07:37:22.558 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcae881640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efca815e170 con 0x7efca80b4580 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcae881640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efca80d48f0 con 0x7efc90003f30 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efca80b4100 0x7efca80c9d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efca80b4100 0x7efca80c9d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:42528/0 (socket says 172.21.15.64:42528) 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 -- 172.21.15.64:0/355324029 learned_addr learned my addr 172.21.15.64:0/355324029 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca7fff640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc90003f30 0x7efca80cb880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcace1e640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efca80b4580 0x7efca80ca2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 -- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc90003f30 msgr2=0x7efca80cb880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc90003f30 0x7efca80cb880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 -- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efca80b4580 msgr2=0x7efca80ca2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efca80b4580 0x7efca80ca2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 -- 172.21.15.64:0/355324029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efca81f71c0 con 0x7efca80b4100 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca7fff640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc90003f30 0x7efca80cb880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efcace1e640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efca80b4580 0x7efca80ca2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca77fe640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efca80b4100 0x7efca80c9d90 secure :-1 s=READY pgs=1857 cs=0 l=1 rev1=1 crypto rx=0x7efc9c00c300 tx=0x7efca80b6c60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:22.559 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.553+0000 7efca4ff9640 1 -- 172.21.15.64:0/355324029 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc9c046070 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.554+0000 7efca4ff9640 1 -- 172.21.15.64:0/355324029 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efc9c004c10 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.554+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efca81f74a0 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.554+0000 7efca4ff9640 1 -- 172.21.15.64:0/355324029 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc9c001c90 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.554+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efca81f7a60 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.555+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efca8072a30 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.555+0000 7efca4ff9640 1 -- 172.21.15.64:0/355324029 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efc9c007070 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.555+0000 7efca4ff9640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efc9406b4d0 0x7efc9406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.556+0000 7efca4ff9640 1 -- 172.21.15.64:0/355324029 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(129..129 src has 1..129) v4 ==== 5501+0+0 (secure 0 0 0) 0x7efc9c0ab390 con 0x7efca80b4100 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.556+0000 7efca7fff640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efc9406b4d0 0x7efc9406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:22.560 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.556+0000 7efca7fff640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efc9406b4d0 0x7efc9406d990 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7efc98009ab0 tx=0x7efc98008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:22.561 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.561+0000 7efca4ff9640 1 -- 172.21.15.64:0/355324029 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efc9c07a2e0 con 0x7efca80b4100 2024-03-04T07:37:22.742 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:22.741+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7efca804f410 con 0x7efca80b4100 2024-03-04T07:37:23.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.428+0000 7efca4ff9640 1 -- 172.21.15.64:0/355324029 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v130) v1 ==== 169+0+0 (secure 0 0 0) 0x7efc9c079be0 con 0x7efca80b4100 2024-03-04T07:37:23.428 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'cephfs_data' removed 2024-03-04T07:37:23.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.432+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efc9406b4d0 msgr2=0x7efc9406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.432+0000 7efcae881640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efc9406b4d0 0x7efc9406d990 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7efc98009ab0 tx=0x7efc98008040 comp rx=0 tx=0).stop 2024-03-04T07:37:23.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.432+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efca80b4100 msgr2=0x7efca80c9d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.432+0000 7efcae881640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efca80b4100 0x7efca80c9d90 secure :-1 s=READY pgs=1857 cs=0 l=1 rev1=1 crypto rx=0x7efc9c00c300 tx=0x7efca80b6c60 comp rx=0 tx=0).stop 2024-03-04T07:37:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.433+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 shutdown_connections 2024-03-04T07:37:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.433+0000 7efcae881640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efc9406b4d0 0x7efc9406d990 unknown :-1 s=CLOSED pgs=848 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.433+0000 7efcae881640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efca80b4580 0x7efca80ca2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.433+0000 7efcae881640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efca80b4100 0x7efca80c9d90 unknown :-1 s=CLOSED pgs=1857 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.433+0000 7efcae881640 1 --2- 172.21.15.64:0/355324029 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efc90003f30 0x7efca80cb880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.433+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 >> 172.21.15.64:0/355324029 conn(0x7efca8071960 msgr2=0x7efca80d0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:23.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.435+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 shutdown_connections 2024-03-04T07:37:23.435 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.436+0000 7efcae881640 1 -- 172.21.15.64:0/355324029 wait complete. 2024-03-04T07:37:23.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config rm mon mon_allow_pool_delete 2024-03-04T07:37:23.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.565+0000 7fa4b7ec0640 1 Processor -- start 2024-03-04T07:37:23.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.565+0000 7fa4b7ec0640 1 -- start start 2024-03-04T07:37:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b7ec0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 0x7fa4b0168ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b7ec0640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4b006f070 con 0x7fa4b00b5840 2024-03-04T07:37:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b7ec0640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fa4b0071940 con 0x7fa4b015dbd0 2024-03-04T07:37:23.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b7ec0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4b00b5db0 con 0x7fa4b00b40c0 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b545b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fa4b00b5840 0x7fa4b00b39b0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50058/0 (socket says 172.21.15.64:50058) 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b545b640 1 -- 172.21.15.64:0/397051683 learned_addr learned my addr 172.21.15.64:0/397051683 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4100616494 0 0) 0x7fa4b006f070 con 0x7fa4b00b5840 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7fa49c003520 con 0x7fa4b00b5840 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b645d640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 0x7fa4b0168ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 862227397 0 0) 0x7fa49c003520 con 0x7fa4b00b5840 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 msgr2=0x7fa4b0168ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 0x7fa4b0168ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 >> v1:172.21.15.66:6789/0 conn(0x7fa4b015dbd0 legacy=0x7fa4b015ee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4b0071c00 con 0x7fa4b00b5840 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7fa4a0001690 con 0x7fa4b00b5840 2024-03-04T07:37:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 >> v1:172.21.15.64:6789/0 conn(0x7fa4b00b5840 legacy=0x7fa4b00b39b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 0x7fa49c0039e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa49c004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49c004d40 0x7fa49c0051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4b00b5db0 con 0x7fa49c004060 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa49c00bc00 con 0x7fa49c004d40 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.566+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa49c00be00 con 0x7fa4b00b40c0 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b545b640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa49c004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b5c5c640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 0x7fa49c0039e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b645d640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49c004d40 0x7fa49c0051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b545b640 1 -- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 msgr2=0x7fa49c0039e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b545b640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b00b40c0 0x7fa49c0039e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b545b640 1 -- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49c004d40 msgr2=0x7fa49c0051c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b545b640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa49c004d40 0x7fa49c0051c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.567+0000 7fa4b545b640 1 -- 172.21.15.64:0/397051683 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa49c003520 con 0x7fa49c004060 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b545b640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa49c004800 secure :-1 s=READY pgs=1859 cs=0 l=1 rev1=1 crypto rx=0x7fa4a000c160 tx=0x7fa49c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d68018548d7db270 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4a0007c60 con 0x7fa49c004060 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa4a0030070 con 0x7fa49c004060 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b4c5a640 1 -- 172.21.15.64:0/397051683 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4a0005d10 con 0x7fa49c004060 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/397051683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 msgr2=0x7fa49c004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b7ec0640 1 --2- 172.21.15.64:0/397051683 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa49c004800 secure :-1 s=READY pgs=1859 cs=0 l=1 rev1=1 crypto rx=0x7fa4a000c160 tx=0x7fa49c005830 comp rx=0 tx=0).stop 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b645d640 1 -- 172.21.15.64:0/397051683 reap_dead start 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/397051683 shutdown_connections 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/397051683 >> 172.21.15.64:0/397051683 conn(0x7fa4b00b26b0 msgr2=0x7fa4b00b2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/397051683 shutdown_connections 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.568+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/397051683 wait complete. 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.569+0000 7fa4b7ec0640 1 Processor -- start 2024-03-04T07:37:23.572 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.569+0000 7fa4b7ec0640 1 -- start start 2024-03-04T07:37:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b7ec0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa4b015dfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b7ec0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b014e370 0x7fa4b00b2090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b7ec0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4b014dc80 0x7fa4b0158930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b7ec0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4b00726e0 con 0x7fa49c004060 2024-03-04T07:37:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b7ec0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4b00721c0 con 0x7fa4b014dc80 2024-03-04T07:37:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b7ec0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa4b0072860 con 0x7fa4b014e370 2024-03-04T07:37:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa4b015dfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa4b015dfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38198/0 (socket says 172.21.15.64:38198) 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 -- 172.21.15.64:0/1736944313 learned_addr learned my addr 172.21.15.64:0/1736944313 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b645d640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4b014dc80 0x7fa4b0158930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b545b640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b014e370 0x7fa4b00b2090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 -- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b014e370 msgr2=0x7fa4b00b2090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b014e370 0x7fa4b00b2090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 -- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4b014dc80 msgr2=0x7fa4b0158930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4b014dc80 0x7fa4b0158930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.570+0000 7fa4b5c5c640 1 -- 172.21.15.64:0/1736944313 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4b014e100 con 0x7fa49c004060 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.571+0000 7fa4b5c5c640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa4b015dfb0 secure :-1 s=READY pgs=1860 cs=0 l=1 rev1=1 crypto rx=0x7fa4ac00f850 tx=0x7fa4ac00fd10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.571+0000 7fa4a67fc640 1 -- 172.21.15.64:0/1736944313 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4ac00fe40 con 0x7fa49c004060 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.571+0000 7fa4a67fc640 1 -- 172.21.15.64:0/1736944313 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fa4ac018070 con 0x7fa49c004060 2024-03-04T07:37:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.571+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4b00b0ff0 con 0x7fa49c004060 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.571+0000 7fa4a67fc640 1 -- 172.21.15.64:0/1736944313 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4ac00d6b0 con 0x7fa49c004060 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.571+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4b00b15b0 con 0x7fa49c004060 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.572+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa4b0073380 con 0x7fa49c004060 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.572+0000 7fa4a67fc640 1 -- 172.21.15.64:0/1736944313 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fa4ac00d850 con 0x7fa49c004060 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.572+0000 7fa4a67fc640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa49406b4d0 0x7fa49406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.573+0000 7fa4a67fc640 1 -- 172.21.15.64:0/1736944313 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7fa4ac089cf0 con 0x7fa49c004060 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.573+0000 7fa4b545b640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa49406b4d0 0x7fa49406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.573+0000 7fa4b545b640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa49406b4d0 0x7fa49406d990 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7fa4a0034000 tx=0x7fa4a0009030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:23.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.577+0000 7fa4a67fc640 1 -- 172.21.15.64:0/1736944313 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fa4ac08e070 con 0x7fa49c004060 2024-03-04T07:37:23.758 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.758+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"} v 0) v1 -- 0x7fa4b004f410 con 0x7fa49c004060 2024-03-04T07:37:23.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.761+0000 7fa4a67fc640 1 -- 172.21.15.64:0/1736944313 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "config rm", "who": "mon", "name": "mon_allow_pool_delete"}]=0 v33) v1 ==== 104+0+0 (secure 0 0 0) 0x7fa4b014e100 con 0x7fa49c004060 2024-03-04T07:37:23.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa49406b4d0 msgr2=0x7fa49406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa49406b4d0 0x7fa49406d990 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7fa4a0034000 tx=0x7fa4a0009030 comp rx=0 tx=0).stop 2024-03-04T07:37:23.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 msgr2=0x7fa4b015dfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa4b015dfb0 secure :-1 s=READY pgs=1860 cs=0 l=1 rev1=1 crypto rx=0x7fa4ac00f850 tx=0x7fa4ac00fd10 comp rx=0 tx=0).stop 2024-03-04T07:37:23.763 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 shutdown_connections 2024-03-04T07:37:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fa49406b4d0 0x7fa49406d990 unknown :-1 s=CLOSED pgs=849 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fa4b014dc80 0x7fa4b0158930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fa4b014e370 0x7fa4b00b2090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 --2- 172.21.15.64:0/1736944313 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fa49c004060 0x7fa4b015dfb0 unknown :-1 s=CLOSED pgs=1860 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 >> 172.21.15.64:0/1736944313 conn(0x7fa4b00b26b0 msgr2=0x7fa4b006da10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 shutdown_connections 2024-03-04T07:37:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.763+0000 7fa4b7ec0640 1 -- 172.21.15.64:0/1736944313 wait complete. 2024-03-04T07:37:23.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph log 'Ended test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir' 2024-03-04T07:37:23.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.875+0000 7fed13556640 1 Processor -- start 2024-03-04T07:37:23.875 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.875+0000 7fed13556640 1 -- start start 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.876+0000 7fed13556640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fed0c0b95a0 0x7fed0c0b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.876+0000 7fed13556640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fed0c0cea30 con 0x7fed0c0b9a10 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.876+0000 7fed13556640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fed0c0cf3c0 con 0x7fed0c15df00 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.876+0000 7fed13556640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fed0c0cf0e0 con 0x7fed0c0b95a0 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.876+0000 7fed10af1640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fed0c0b95a0 0x7fed0c0b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.876+0000 7fed11af3640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fed0c0b9a10 0x7fed0c0d3220 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50072/0 (socket says 172.21.15.64:50072) 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.876+0000 7fed11af3640 1 -- 172.21.15.64:0/1074763659 learned_addr learned my addr 172.21.15.64:0/1074763659 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.877+0000 7fed10af1640 1 -- 172.21.15.64:0/1074763659 >> v1:172.21.15.66:6789/0 conn(0x7fed0c15df00 legacy=0x7fed0c0d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.877+0000 7fed10af1640 1 -- 172.21.15.64:0/1074763659 >> v1:172.21.15.64:6789/0 conn(0x7fed0c0b9a10 legacy=0x7fed0c0d3220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.877 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.877+0000 7fed0bfff640 1 -- 172.21.15.64:0/1074763659 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1136916292 0 0) 0x7fed0c0cea30 con 0x7fed0c0b9a10 2024-03-04T07:37:23.878 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.877+0000 7fed10af1640 1 -- 172.21.15.64:0/1074763659 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed0c0ce470 con 0x7fed0c0b95a0 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.877+0000 7fed10af1640 1 --2- 172.21.15.64:0/1074763659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0b5700 secure :-1 s=READY pgs=1862 cs=0 l=1 rev1=1 crypto rx=0x7fed0c0cf590 tx=0x7fed0002b960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2b2b45591eff464e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed0bfff640 1 -- 172.21.15.64:0/1074763659 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed00034070 con 0x7fed0c0b95a0 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed0bfff640 1 -- 172.21.15.64:0/1074763659 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fed00030610 con 0x7fed0c0b95a0 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed0bfff640 1 -- 172.21.15.64:0/1074763659 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed000309e0 con 0x7fed0c0b95a0 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed13556640 1 -- 172.21.15.64:0/1074763659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 msgr2=0x7fed0c0b5700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed13556640 1 --2- 172.21.15.64:0/1074763659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0b5700 secure :-1 s=READY pgs=1862 cs=0 l=1 rev1=1 crypto rx=0x7fed0c0cf590 tx=0x7fed0002b960 comp rx=0 tx=0).stop 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed13556640 1 -- 172.21.15.64:0/1074763659 shutdown_connections 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed13556640 1 --2- 172.21.15.64:0/1074763659 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0b5700 unknown :-1 s=CLOSED pgs=1862 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed13556640 1 -- 172.21.15.64:0/1074763659 >> 172.21.15.64:0/1074763659 conn(0x7fed0c071960 msgr2=0x7fed0c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed13556640 1 -- 172.21.15.64:0/1074763659 shutdown_connections 2024-03-04T07:37:23.879 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.878+0000 7fed13556640 1 -- 172.21.15.64:0/1074763659 wait complete. 2024-03-04T07:37:23.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.879+0000 7fed13556640 1 Processor -- start 2024-03-04T07:37:23.881 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.879+0000 7fed13556640 1 -- start start 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed13556640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0cca00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed13556640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fed0c0b9a10 0x7fed0c1f7060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed13556640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fed0c15df00 0x7fed0c0cad50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed13556640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed0c0cedf0 con 0x7fed0c0b95a0 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed13556640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed0c0cec70 con 0x7fed0c15df00 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed13556640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fed0c0b5c40 con 0x7fed0c0b9a10 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0cca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0cca00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38224/0 (socket says 172.21.15.64:38224) 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 -- 172.21.15.64:0/2761995988 learned_addr learned my addr 172.21.15.64:0/2761995988 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 -- 172.21.15.64:0/2761995988 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fed0c0b9a10 msgr2=0x7fed0c1f7060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fed0c0b9a10 0x7fed0c1f7060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 -- 172.21.15.64:0/2761995988 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fed0c15df00 msgr2=0x7fed0c0cad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fed0c15df00 0x7fed0c0cad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:23.882 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.880+0000 7fed112f2640 1 -- 172.21.15.64:0/2761995988 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed0c0cb960 con 0x7fed0c0b95a0 2024-03-04T07:37:23.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.881+0000 7fed112f2640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0cca00 secure :-1 s=READY pgs=1863 cs=0 l=1 rev1=1 crypto rx=0x7fed0400c400 tx=0x7fed0400c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:23.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.881+0000 7fed09ffb640 1 -- 172.21.15.64:0/2761995988 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed04017070 con 0x7fed0c0b95a0 2024-03-04T07:37:23.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.881+0000 7fed09ffb640 1 -- 172.21.15.64:0/2761995988 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fed040052d0 con 0x7fed0c0b95a0 2024-03-04T07:37:23.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.881+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fed0c0cbc40 con 0x7fed0c0b95a0 2024-03-04T07:37:23.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.881+0000 7fed09ffb640 1 -- 172.21.15.64:0/2761995988 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed04003830 con 0x7fed0c0b95a0 2024-03-04T07:37:23.883 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.881+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fed0c0cc200 con 0x7fed0c0b95a0 2024-03-04T07:37:23.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.882+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fed0c067ff0 con 0x7fed0c0b95a0 2024-03-04T07:37:23.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.882+0000 7fed09ffb640 1 -- 172.21.15.64:0/2761995988 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fed0401a070 con 0x7fed0c0b95a0 2024-03-04T07:37:23.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.883+0000 7fed09ffb640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fecf806b500 0x7fecf806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:23.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.883+0000 7fed10af1640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fecf806b500 0x7fecf806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:23.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.883+0000 7fed09ffb640 1 -- 172.21.15.64:0/2761995988 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7fed04089230 con 0x7fed0c0b95a0 2024-03-04T07:37:23.884 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.883+0000 7fed10af1640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fecf806b500 0x7fecf806d9c0 secure :-1 s=READY pgs=850 cs=0 l=1 rev1=1 crypto rx=0x7fed00036000 tx=0x7fed00030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:23.887 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:23.887+0000 7fed09ffb640 1 -- 172.21.15.64:0/2761995988 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fed040583e0 con 0x7fed0c0b95a0 2024-03-04T07:37:24.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.142+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir"]} v 0) v1 -- 0x7fed0c0c7530 con 0x7fed0c0b95a0 2024-03-04T07:37:24.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.429+0000 7fed09ffb640 1 -- 172.21.15.64:0/2761995988 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "log", "logtext": ["Ended test tasks.cephfs.test_strays.TestStrays.test_migrate_unlinked_dir"]}]=0 v554) v1 ==== 140+0+0 (secure 0 0 0) 0x7fed04087e20 con 0x7fed0c0b95a0 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fecf806b500 msgr2=0x7fecf806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fecf806b500 0x7fecf806d9c0 secure :-1 s=READY pgs=850 cs=0 l=1 rev1=1 crypto rx=0x7fed00036000 tx=0x7fed00030000 comp rx=0 tx=0).stop 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 msgr2=0x7fed0c0cca00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0cca00 secure :-1 s=READY pgs=1863 cs=0 l=1 rev1=1 crypto rx=0x7fed0400c400 tx=0x7fed0400c8c0 comp rx=0 tx=0).stop 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 shutdown_connections 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fecf806b500 0x7fecf806d9c0 unknown :-1 s=CLOSED pgs=850 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fed0c15df00 0x7fed0c0cad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fed0c0b9a10 0x7fed0c1f7060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 --2- 172.21.15.64:0/2761995988 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fed0c0b95a0 0x7fed0c0cca00 unknown :-1 s=CLOSED pgs=1863 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.431+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 >> 172.21.15.64:0/2761995988 conn(0x7fed0c071960 msgr2=0x7fed0c071d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.432+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 shutdown_connections 2024-03-04T07:37:24.432 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:24.432+0000 7fed13556640 1 -- 172.21.15.64:0/2761995988 wait complete. 2024-03-04T07:37:24.444 INFO:tasks.cephfs_test_runner:Reproduce https://tracker.ceph.com/issues/53597 ... ERROR 2024-03-04T07:37:24.444 INFO:tasks.cephfs_test_runner: 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:====================================================================== 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:ERROR: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:Reproduce https://tracker.ceph.com/issues/53597 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:---------------------------------------------------------------------- 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:Traceback (most recent call last): 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner: File "/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks/cephfs/test_strays.py", line 734, in test_migrate_unlinked_dir 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner: self.fs.wait_for_daemons(timeout=120) 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner: File "/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks/cephfs/filesystem.py", line 1240, in wait_for_daemons 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner: raise RuntimeError("Timed out waiting for MDS daemons to become healthy") 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:RuntimeError: Timed out waiting for MDS daemons to become healthy 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner: 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:---------------------------------------------------------------------- 2024-03-04T07:37:24.445 INFO:tasks.cephfs_test_runner:Ran 8 tests in 606.082s 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner: 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner:FAILED (errors=1) 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner: 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner:====================================================================== 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner:ERROR: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner:Reproduce https://tracker.ceph.com/issues/53597 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner:---------------------------------------------------------------------- 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner:Traceback (most recent call last): 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner: File "/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks/cephfs/test_strays.py", line 734, in test_migrate_unlinked_dir 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner: self.fs.wait_for_daemons(timeout=120) 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner: File "/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks/cephfs/filesystem.py", line 1240, in wait_for_daemons 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner: raise RuntimeError("Timed out waiting for MDS daemons to become healthy") 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner:RuntimeError: Timed out waiting for MDS daemons to become healthy 2024-03-04T07:37:24.446 INFO:tasks.cephfs_test_runner: 2024-03-04T07:37:24.447 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_b1dac5519c57c269ea98a22fb7729016a1d0e4d2/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks/cephfs_test_runner.py", line 211, in task raise RuntimeError("Test failure: {0}".format(", ".join(bad_tests))) RuntimeError: Test failure: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:37:24.657 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=ae60a7e9e9f345119f481e858c780348 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_b1dac5519c57c269ea98a22fb7729016a1d0e4d2/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_1bc250c71294e65f0a1e8e3ebd8b2a618c6b57bd/qa/tasks/cephfs_test_runner.py", line 211, in task raise RuntimeError("Test failure: {0}".format(", ".join(bad_tests))) RuntimeError: Test failure: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays) 2024-03-04T07:37:24.659 DEBUG:teuthology.run_tasks:Unwinding manager cephfs_test_runner 2024-03-04T07:37:24.668 DEBUG:teuthology.run_tasks:Unwinding manager ceph-fuse 2024-03-04T07:37:24.676 INFO:tasks.ceph_fuse:Unmounting ceph-fuse clients... 2024-03-04T07:37:24.677 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:37:24.677 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:24.696 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:37:24.696 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-03-04T07:37:24.696 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:24.751 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056376,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=632 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248848k,nr_inodes=812212,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-1 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-2 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-3 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497700k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.2 nsfs rw 0 0 2024-03-04T07:37:24.751 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:37:24.751 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:24.770 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:37:24.771 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2024-03-04T07:37:24.771 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-04T07:37:24.828 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=4096k,nr_inodes=4056356,mode=755,inode64 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev,inode64 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 cgroup2 /sys/fs/cgroup cgroup2 rw,seclabel,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,nosuid,noexec,relatime 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=29,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=16701 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 tracefs /sys/kernel/tracing tracefs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-tmpfiles-setup-dev.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 fusectl /sys/fs/fuse/connections fusectl rw,nosuid,nodev,noexec,relatime 0 0 configfs /sys/kernel/config configfs rw,nosuid,nodev,noexec,relatime 0 0 none /run/credentials/systemd-sysctl.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 none /run/credentials/systemd-tmpfiles-setup.service ramfs ro,seclabel,nosuid,nodev,noexec,relatime,mode=700 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3248832k,nr_inodes=812208,mode=700,uid=1000,gid=1301,inode64 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-4 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-5 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-6 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-7 xfs rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,size=6497668k,nr_inodes=819200,mode=755,inode64 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 nsfs /run/netns/ceph-ns--home-ubuntu-cephtest-mnt.3 nsfs rw 0 0 2024-03-04T07:37:24.829 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:24.829 DEBUG:teuthology.orchestra.run.smithi066:> ip netns list 2024-03-04T07:37:24.886 INFO:teuthology.orchestra.run.smithi066.stdout:ceph-ns--home-ubuntu-cephtest-mnt.3 (id: 1) 2024-03-04T07:37:24.887 INFO:teuthology.orchestra.run.smithi066.stdout:ceph-ns--home-ubuntu-cephtest-mnt.1 (id: 0) 2024-03-04T07:37:24.887 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:24.887 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.3 2024-03-04T07:37:24.961 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:24.961 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.1 2024-03-04T07:37:25.037 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:25.037 DEBUG:teuthology.orchestra.run.smithi066:> sudo ip link delete ceph-brx 2024-03-04T07:37:25.144 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:25.144 DEBUG:teuthology.orchestra.run.smithi064:> ip netns list 2024-03-04T07:37:25.159 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.2 (id: 1) 2024-03-04T07:37:25.160 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-04T07:37:25.160 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:25.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.2 2024-03-04T07:37:25.227 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:25.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-04T07:37:25.295 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-04T07:37:25.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo ip link delete ceph-brx 2024-03-04T07:37:25.399 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-03-04T07:37:25.409 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-03-04T07:37:25.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:37:25.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.513+0000 7f915566b640 1 Processor -- start 2024-03-04T07:37:25.513 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.513+0000 7f915566b640 1 -- start start 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f915566b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 0x7f915006ea30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f915566b640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f915015e410 con 0x7f915006ef70 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f915566b640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f9150161af0 con 0x7f91500b2460 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f915566b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9150162700 con 0x7f915015e030 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914e7fc640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f915006ef70 0x7f91500b1d50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50086/0 (socket says 172.21.15.64:50086) 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914e7fc640 1 -- 172.21.15.64:0/3799238279 learned_addr learned my addr 172.21.15.64:0/3799238279 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914effd640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 0x7f915006ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.518 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2282356198 0 0) 0x7f9144001110 con 0x7f915006ef70 2024-03-04T07:37:25.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9138003520 con 0x7f915006ef70 2024-03-04T07:37:25.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 873486771 0 0) 0x7f9150161af0 con 0x7f91500b2460 2024-03-04T07:37:25.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f9144001110 con 0x7f91500b2460 2024-03-04T07:37:25.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1346565702 0 0) 0x7f9138003520 con 0x7f915006ef70 2024-03-04T07:37:25.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 msgr2=0x7f915006ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:25.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 0x7f915006ea30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 >> v1:172.21.15.66:6789/0 conn(0x7f91500b2460 legacy=0x7f915006d210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9150161dd0 con 0x7f915006ef70 2024-03-04T07:37:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f9144001840 con 0x7f915006ef70 2024-03-04T07:37:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 >> v1:172.21.15.64:6789/0 conn(0x7f915006ef70 legacy=0x7f91500b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9138003b70 0x7f9138003f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 0x7f9138004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9138004d40 0x7f91380051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9150162700 con 0x7f9138004d40 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f913800ba70 con 0x7f9138003b70 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.514+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f913800bc20 con 0x7f915015e030 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914f7fe640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9138004d40 0x7f91380051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914effd640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9138003b70 0x7f9138003f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914e7fc640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 0x7f9138004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914f7fe640 1 -- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 msgr2=0x7f9138004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914f7fe640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f915015e030 0x7f9138004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914f7fe640 1 -- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9138003b70 msgr2=0x7f9138003f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914f7fe640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9138003b70 0x7f9138003f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914f7fe640 1 -- 172.21.15.64:0/3799238279 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9138003520 con 0x7f9138004d40 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.515+0000 7f914effd640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f9138003b70 0x7f9138003f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f914f7fe640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9138004d40 0x7f91380051c0 secure :-1 s=READY pgs=1865 cs=0 l=1 rev1=1 crypto rx=0x7f9140001fd0 tx=0x7f9138005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8823b24f9e9290ab server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9140001bf0 con 0x7f9138004d40 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9140003b70 con 0x7f9138004d40 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f914dffb640 1 -- 172.21.15.64:0/3799238279 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91400050d0 con 0x7f9138004d40 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f915566b640 1 -- 172.21.15.64:0/3799238279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9138004d40 msgr2=0x7f91380051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.522 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f915566b640 1 --2- 172.21.15.64:0/3799238279 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9138004d40 0x7f91380051c0 secure :-1 s=READY pgs=1865 cs=0 l=1 rev1=1 crypto rx=0x7f9140001fd0 tx=0x7f9138005830 comp rx=0 tx=0).stop 2024-03-04T07:37:25.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f914f7fe640 1 -- 172.21.15.64:0/3799238279 reap_dead start 2024-03-04T07:37:25.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f915566b640 1 -- 172.21.15.64:0/3799238279 shutdown_connections 2024-03-04T07:37:25.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f915566b640 1 -- 172.21.15.64:0/3799238279 >> 172.21.15.64:0/3799238279 conn(0x7f91500b55e0 msgr2=0x7f91500b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:25.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f915566b640 1 -- 172.21.15.64:0/3799238279 shutdown_connections 2024-03-04T07:37:25.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.516+0000 7f915566b640 1 -- 172.21.15.64:0/3799238279 wait complete. 2024-03-04T07:37:25.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 Processor -- start 2024-03-04T07:37:25.523 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 -- start start 2024-03-04T07:37:25.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9150150f30 0x7f915015cc80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9150151310 0x7f915015d1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91501404d0 0x7f9150140950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91500b31e0 con 0x7f9150150f30 2024-03-04T07:37:25.526 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9150162390 con 0x7f91501404d0 2024-03-04T07:37:25.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f915566b640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f91500b3360 con 0x7f9150151310 2024-03-04T07:37:25.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f914effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9150150f30 0x7f915015cc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f914effd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9150150f30 0x7f915015cc80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38236/0 (socket says 172.21.15.64:38236) 2024-03-04T07:37:25.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f914effd640 1 -- 172.21.15.64:0/3982843363 learned_addr learned my addr 172.21.15.64:0/3982843363 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:25.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f914f7fe640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91501404d0 0x7f9150140950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.518+0000 7f914e7fc640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9150151310 0x7f915015d1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.527 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f914effd640 1 -- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9150151310 msgr2=0x7f915015d1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f914effd640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9150151310 0x7f915015d1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f914effd640 1 -- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91501404d0 msgr2=0x7f9150140950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f914effd640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91501404d0 0x7f9150140950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f914effd640 1 -- 172.21.15.64:0/3982843363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9150141240 con 0x7f9150150f30 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f914effd640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9150150f30 0x7f915015cc80 secure :-1 s=READY pgs=1866 cs=0 l=1 rev1=1 crypto rx=0x7f9148011ca0 tx=0x7f914800f610 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f91377fe640 1 -- 172.21.15.64:0/3982843363 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f914800fd20 con 0x7f9150150f30 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f91377fe640 1 -- 172.21.15.64:0/3982843363 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f9148014440 con 0x7f9150150f30 2024-03-04T07:37:25.528 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f91501f75f0 con 0x7f9150150f30 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f91377fe640 1 -- 172.21.15.64:0/3982843363 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9148014830 con 0x7f9150150f30 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.519+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9150140e90 con 0x7f9150150f30 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.520+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9150067ff0 con 0x7f9150150f30 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.521+0000 7f91377fe640 1 -- 172.21.15.64:0/3982843363 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f9148012040 con 0x7f9150150f30 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.521+0000 7f91377fe640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f913c06b4d0 0x7f913c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.521+0000 7f91377fe640 1 -- 172.21.15.64:0/3982843363 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f914808a650 con 0x7f9150150f30 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.521+0000 7f914e7fc640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f913c06b4d0 0x7f913c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.521+0000 7f914e7fc640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f913c06b4d0 0x7f913c06d990 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f9150152da0 tx=0x7f9144007a20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:25.530 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.525+0000 7f91377fe640 1 -- 172.21.15.64:0/3982843363 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f9148019030 con 0x7f9150150f30 2024-03-04T07:37:25.701 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.701+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f91500b2910 con 0x7f913c06b4d0 2024-03-04T07:37:25.702 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.702+0000 7f91377fe640 1 -- 172.21.15.64:0/3982843363 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+13510 (secure 0 0 0) 0x7f91500b2910 con 0x7f913c06b4d0 2024-03-04T07:37:25.702 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:25.703 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:37:25.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f913c06b4d0 msgr2=0x7f913c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f913c06b4d0 0x7f913c06d990 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f9150152da0 tx=0x7f9144007a20 comp rx=0 tx=0).stop 2024-03-04T07:37:25.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9150150f30 msgr2=0x7f915015cc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9150150f30 0x7f915015cc80 secure :-1 s=READY pgs=1866 cs=0 l=1 rev1=1 crypto rx=0x7f9148011ca0 tx=0x7f914800f610 comp rx=0 tx=0).stop 2024-03-04T07:37:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 shutdown_connections 2024-03-04T07:37:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f913c06b4d0 0x7f913c06d990 unknown :-1 s=CLOSED pgs=851 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f91501404d0 0x7f9150140950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f9150151310 0x7f915015d1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 --2- 172.21.15.64:0/3982843363 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f9150150f30 0x7f915015cc80 unknown :-1 s=CLOSED pgs=1866 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 >> 172.21.15.64:0/3982843363 conn(0x7f91500b55e0 msgr2=0x7f915006e660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:25.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 shutdown_connections 2024-03-04T07:37:25.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.704+0000 7f915566b640 1 -- 172.21.15.64:0/3982843363 wait complete. 2024-03-04T07:37:25.715 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":445,"stamp":"2024-03-04T07:37:25.265536+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":329760,"kb_used_data":13772,"kb_used_omap":12,"kb_used_meta":315891,"kb_avail":754644960,"statfs":{"total":773094113280,"available":772756439040,"internally_reserved":0,"allocated":14102528,"data_stored":6826179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":323472944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":29,"apply_latency_ms":29,"commit_latency_ns":29000000,"apply_latency_ns":29000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-33651,"num_objects":-44,"num_object_clones":0,"num_object_copies":-88,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-44,"num_whiteouts":0,"num_read":-4,"num_read_kb":0,"num_write":-143,"num_write_kb":-80,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-28,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001454"},"pg_stats":[{"pgid":"1.0","version":"15'117","reported_seq":368,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-03-04T07:37:20.650026+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:37:20.650026+0000","last_peered":"2024-03-04T07:37:20.650026+0000","last_clean":"2024-03-04T07:37:20.650026+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:37:20.650026+0000","last_undegraded":"2024-03-04T07:37:20.650026+0000","last_fullsized":"2024-03-04T07:37:20.650026+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574981,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41680,"kb_used_data":2180,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330160,"statfs":{"total":96636764160,"available":96594083840,"internally_reserved":0,"allocated":2232320,"data_stored":1309301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":40434122},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574980,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41092,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330748,"statfs":{"total":96636764160,"available":96594685952,"internally_reserved":0,"allocated":1630208,"data_stored":718317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574980,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41080,"kb_used_data":1580,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330760,"statfs":{"total":96636764160,"available":96594698240,"internally_reserved":0,"allocated":1617920,"data_stored":700446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574981,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41064,"kb_used_data":1564,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330776,"statfs":{"total":96636764160,"available":96594714624,"internally_reserved":0,"allocated":1601536,"data_stored":699642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574980,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41080,"kb_used_data":1584,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330760,"statfs":{"total":96636764160,"available":96594698240,"internally_reserved":0,"allocated":1622016,"data_stored":707688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574980,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41056,"kb_used_data":1560,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330784,"statfs":{"total":96636764160,"available":96594722816,"internally_reserved":0,"allocated":1597440,"data_stored":693967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574980,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41084,"kb_used_data":1584,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330756,"statfs":{"total":96636764160,"available":96594694144,"internally_reserved":0,"allocated":1622016,"data_stored":713189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574980,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41624,"kb_used_data":2128,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330216,"statfs":{"total":96636764160,"available":96594141184,"internally_reserved":0,"allocated":2179072,"data_stored":1283629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:37:25.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:37:25.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.818+0000 7fe0719a0640 1 Processor -- start 2024-03-04T07:37:25.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.818+0000 7fe0719a0640 1 -- start start 2024-03-04T07:37:25.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe0719a0640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe06c15dd20 0x7fe06c0d1bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.821 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe0719a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c0d2100 0x7fe06c0d0a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe0719a0640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fe06c0d0f70 0x7fe06c0d3a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe0719a0640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe06c0d3190 con 0x7fe06c15dd20 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe0719a0640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fe06c0d1390 con 0x7fe06c0d0f70 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe0719a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe06c15e100 con 0x7fe06c0d2100 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe06c15dd20 0x7fe06c0d1bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fe06c15dd20 0x7fe06c0d1bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38240/0 (socket says 172.21.15.64:38240) 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 -- 172.21.15.64:0/3410654611 learned_addr learned my addr 172.21.15.64:0/3410654611 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:25.822 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07119f640 1 --2- 172.21.15.64:0/3410654611 >> v2:172.21.15.66:3300/0 conn(0x7fe06c0d0f70 0x7fe06c0d3a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe06bfff640 1 --2- 172.21.15.64:0/3410654611 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c0d2100 0x7fe06c0d0a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 -- 172.21.15.64:0/3410654611 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c0d2100 msgr2=0x7fe06c0d0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 --2- 172.21.15.64:0/3410654611 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c0d2100 0x7fe06c0d0a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 -- 172.21.15.64:0/3410654611 >> v2:172.21.15.66:3300/0 conn(0x7fe06c0d0f70 msgr2=0x7fe06c0d3a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 --2- 172.21.15.64:0/3410654611 >> v2:172.21.15.66:3300/0 conn(0x7fe06c0d0f70 0x7fe06c0d3a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe07099e640 1 -- 172.21.15.64:0/3410654611 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe06c0d2580 con 0x7fe06c15dd20 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.819+0000 7fe06bfff640 1 --2- 172.21.15.64:0/3410654611 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c0d2100 0x7fe06c0d0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe07099e640 1 --2- 172.21.15.64:0/3410654611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c15dd20 0x7fe06c0d1bc0 secure :-1 s=READY pgs=1867 cs=0 l=1 rev1=1 crypto rx=0x7fe06c0ce3c0 tx=0x7fe06002b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a2c871e1a03a03d9 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:25.823 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe06b7fe640 1 -- 172.21.15.64:0/3410654611 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe060035020 con 0x7fe06c15dd20 2024-03-04T07:37:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe06b7fe640 1 -- 172.21.15.64:0/3410654611 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe060031870 con 0x7fe06c15dd20 2024-03-04T07:37:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe06b7fe640 1 -- 172.21.15.64:0/3410654611 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe060031c40 con 0x7fe06c15dd20 2024-03-04T07:37:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 -- 172.21.15.64:0/3410654611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c15dd20 msgr2=0x7fe06c0d1bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 --2- 172.21.15.64:0/3410654611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c15dd20 0x7fe06c0d1bc0 secure :-1 s=READY pgs=1867 cs=0 l=1 rev1=1 crypto rx=0x7fe06c0ce3c0 tx=0x7fe06002b910 comp rx=0 tx=0).stop 2024-03-04T07:37:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 -- 172.21.15.64:0/3410654611 shutdown_connections 2024-03-04T07:37:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 --2- 172.21.15.64:0/3410654611 >> v2:172.21.15.66:3300/0 conn(0x7fe06c0d0f70 0x7fe06c0d3a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.824 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 --2- 172.21.15.64:0/3410654611 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c0d2100 0x7fe06c0d0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 --2- 172.21.15.64:0/3410654611 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c15dd20 0x7fe06c0d1bc0 unknown :-1 s=CLOSED pgs=1867 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 -- 172.21.15.64:0/3410654611 >> 172.21.15.64:0/3410654611 conn(0x7fe06c071960 msgr2=0x7fe06c071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 -- 172.21.15.64:0/3410654611 shutdown_connections 2024-03-04T07:37:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.820+0000 7fe0719a0640 1 -- 172.21.15.64:0/3410654611 wait complete. 2024-03-04T07:37:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 Processor -- start 2024-03-04T07:37:25.825 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 -- start start 2024-03-04T07:37:25.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe06c0d0f70 0x7fe06c159070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c0d2100 0x7fe06c15f960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c15dd20 0x7fe06c1f3830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe06c0d2e20 con 0x7fe06c0d2100 2024-03-04T07:37:25.828 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe06c0d2b40 con 0x7fe06c0d0f70 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.822+0000 7fe0719a0640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fe06c0b9580 con 0x7fe06c15dd20 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c0d2100 0x7fe06c15f960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c0d2100 0x7fe06c15f960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38248/0 (socket says 172.21.15.64:38248) 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 -- 172.21.15.64:0/4238734903 learned_addr learned my addr 172.21.15.64:0/4238734903 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe07119f640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c15dd20 0x7fe06c1f3830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe07099e640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe06c0d0f70 0x7fe06c159070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 -- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c15dd20 msgr2=0x7fe06c1f3830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c15dd20 0x7fe06c1f3830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.829 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 -- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe06c0d0f70 msgr2=0x7fe06c159070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe06c0d0f70 0x7fe06c159070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 -- 172.21.15.64:0/4238734903 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe06c159650 con 0x7fe06c0d2100 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe06bfff640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c0d2100 0x7fe06c15f960 secure :-1 s=READY pgs=1868 cs=0 l=1 rev1=1 crypto rx=0x7fe05800c7f0 tx=0x7fe05800cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe0697fa640 1 -- 172.21.15.64:0/4238734903 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe058015070 con 0x7fe06c0d2100 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe0697fa640 1 -- 172.21.15.64:0/4238734903 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fe058007a30 con 0x7fe06c0d2100 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe06c159930 con 0x7fe06c0d2100 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe0697fa640 1 -- 172.21.15.64:0/4238734903 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe058007e00 con 0x7fe06c0d2100 2024-03-04T07:37:25.830 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.823+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe06c15a1d0 con 0x7fe06c0d2100 2024-03-04T07:37:25.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.824+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe06c15ff00 con 0x7fe06c0d2100 2024-03-04T07:37:25.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.825+0000 7fe0697fa640 1 -- 172.21.15.64:0/4238734903 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fe058018070 con 0x7fe06c0d2100 2024-03-04T07:37:25.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.825+0000 7fe0697fa640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe05006b4d0 0x7fe05006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:25.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.825+0000 7fe0697fa640 1 -- 172.21.15.64:0/4238734903 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7fe058089e10 con 0x7fe06c0d2100 2024-03-04T07:37:25.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.825+0000 7fe07099e640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe05006b4d0 0x7fe05006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:25.831 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.826+0000 7fe07099e640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe05006b4d0 0x7fe05006d990 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7fe060036000 tx=0x7fe06003c040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:25.832 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:25.829+0000 7fe0697fa640 1 -- 172.21.15.64:0/4238734903 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fe0580588e0 con 0x7fe06c0d2100 2024-03-04T07:37:26.006 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.006+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fe06c04f330 con 0x7fe05006b4d0 2024-03-04T07:37:26.007 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.007+0000 7fe0697fa640 1 -- 172.21.15.64:0/4238734903 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+13510 (secure 0 0 0) 0x7fe06004b4e0 con 0x7fe05006b4d0 2024-03-04T07:37:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:26.007 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:37:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.008+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe05006b4d0 msgr2=0x7fe05006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.008+0000 7fe0719a0640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe05006b4d0 0x7fe05006d990 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7fe060036000 tx=0x7fe06003c040 comp rx=0 tx=0).stop 2024-03-04T07:37:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.008+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c0d2100 msgr2=0x7fe06c15f960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c0d2100 0x7fe06c15f960 secure :-1 s=READY pgs=1868 cs=0 l=1 rev1=1 crypto rx=0x7fe05800c7f0 tx=0x7fe05800cbf0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 shutdown_connections 2024-03-04T07:37:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fe05006b4d0 0x7fe05006d990 unknown :-1 s=CLOSED pgs=852 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fe06c15dd20 0x7fe06c1f3830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fe06c0d2100 0x7fe06c15f960 unknown :-1 s=CLOSED pgs=1868 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 --2- 172.21.15.64:0/4238734903 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fe06c0d0f70 0x7fe06c159070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 >> 172.21.15.64:0/4238734903 conn(0x7fe06c071960 msgr2=0x7fe06c0b3e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 shutdown_connections 2024-03-04T07:37:26.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.009+0000 7fe0719a0640 1 -- 172.21.15.64:0/4238734903 wait complete. 2024-03-04T07:37:26.019 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":445,"stamp":"2024-03-04T07:37:25.265536+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":329760,"kb_used_data":13772,"kb_used_omap":12,"kb_used_meta":315891,"kb_avail":754644960,"statfs":{"total":773094113280,"available":772756439040,"internally_reserved":0,"allocated":14102528,"data_stored":6826179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":323472944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":29,"apply_latency_ms":29,"commit_latency_ns":29000000,"apply_latency_ns":29000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-33651,"num_objects":-44,"num_object_clones":0,"num_object_copies":-88,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-44,"num_whiteouts":0,"num_read":-4,"num_read_kb":0,"num_write":-143,"num_write_kb":-80,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-28,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001454"},"pg_stats":[{"pgid":"1.0","version":"15'117","reported_seq":368,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-03-04T07:37:20.650026+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:37:20.650026+0000","last_peered":"2024-03-04T07:37:20.650026+0000","last_clean":"2024-03-04T07:37:20.650026+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:37:20.650026+0000","last_undegraded":"2024-03-04T07:37:20.650026+0000","last_fullsized":"2024-03-04T07:37:20.650026+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574981,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41680,"kb_used_data":2180,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330160,"statfs":{"total":96636764160,"available":96594083840,"internally_reserved":0,"allocated":2232320,"data_stored":1309301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":40434122},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574980,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41092,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330748,"statfs":{"total":96636764160,"available":96594685952,"internally_reserved":0,"allocated":1630208,"data_stored":718317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574980,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41080,"kb_used_data":1580,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330760,"statfs":{"total":96636764160,"available":96594698240,"internally_reserved":0,"allocated":1617920,"data_stored":700446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574981,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41064,"kb_used_data":1564,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330776,"statfs":{"total":96636764160,"available":96594714624,"internally_reserved":0,"allocated":1601536,"data_stored":699642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574980,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41080,"kb_used_data":1584,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330760,"statfs":{"total":96636764160,"available":96594698240,"internally_reserved":0,"allocated":1622016,"data_stored":707688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574980,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41056,"kb_used_data":1560,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330784,"statfs":{"total":96636764160,"available":96594722816,"internally_reserved":0,"allocated":1597440,"data_stored":693967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574980,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41084,"kb_used_data":1584,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330756,"statfs":{"total":96636764160,"available":96594694144,"internally_reserved":0,"allocated":1622016,"data_stored":713189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574980,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41624,"kb_used_data":2128,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330216,"statfs":{"total":96636764160,"available":96594141184,"internally_reserved":0,"allocated":2179072,"data_stored":1283629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:37:26.020 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-03-04T07:37:26.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:37:26.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.122+0000 7f8f6c4df640 1 Processor -- start 2024-03-04T07:37:26.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.122+0000 7f8f6c4df640 1 -- start start 2024-03-04T07:37:26.125 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.122+0000 7f8f6c4df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 0x7f8f640ce310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.122+0000 7f8f6c4df640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f8f6406fe20 0x7f8f640ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.122+0000 7f8f6c4df640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f8f640d0dd0 con 0x7f8f640d0880 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.122+0000 7f8f6c4df640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f8f640d10b0 con 0x7f8f6406fe20 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.122+0000 7f8f6c4df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8f640cc0e0 con 0x7f8f6415e060 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f6aa7c640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f8f640d0880 0x7f8f6406ed30 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50098/0 (socket says 172.21.15.64:50098) 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f6aa7c640 1 -- 172.21.15.64:0/3995655893 learned_addr learned my addr 172.21.15.64:0/3995655893 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69a7a640 1 --2- 172.21.15.64:0/3995655893 >> v2:172.21.15.66:3300/0 conn(0x7f8f6406fe20 0x7f8f640ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3786840479 0 0) 0x7f8f640d0dd0 con 0x7f8f640d0880 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f6a27b640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 0x7f8f640ce310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f8f50003520 con 0x7f8f640d0880 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1714833613 0 0) 0x7f8f50003520 con 0x7f8f640d0880 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 msgr2=0x7f8f640ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 0x7f8f640ce310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 >> v2:172.21.15.66:3300/0 conn(0x7f8f6406fe20 msgr2=0x7f8f640ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 --2- 172.21.15.64:0/3995655893 >> v2:172.21.15.66:3300/0 conn(0x7f8f6406fe20 0x7f8f640ce850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8f640cc3c0 con 0x7f8f640d0880 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f8f600019f0 con 0x7f8f640d0880 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 >> v1:172.21.15.64:6789/0 conn(0x7f8f640d0880 legacy=0x7f8f6406ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f500039b0 0x7f8f50003dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 0x7f8f50004800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f50004d40 0x7f8f500051c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8f640d10b0 con 0x7f8f50004d40 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8f640cc0e0 con 0x7f8f500039b0 2024-03-04T07:37:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8f5000b9c0 con 0x7f8f6415e060 2024-03-04T07:37:26.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f6aa7c640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f50004d40 0x7f8f500051c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f6a27b640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f500039b0 0x7f8f50003dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.123+0000 7f8f69a7a640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 0x7f8f50004800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f6aa7c640 1 -- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 msgr2=0x7f8f50004800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:26.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f6aa7c640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f6415e060 0x7f8f50004800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f6aa7c640 1 -- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f500039b0 msgr2=0x7f8f50003dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:26.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f6aa7c640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f500039b0 0x7f8f50003dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f6aa7c640 1 -- 172.21.15.64:0/3995655893 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8f50003520 con 0x7f8f50004d40 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f6aa7c640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f50004d40 0x7f8f500051c0 secure :-1 s=READY pgs=1870 cs=0 l=1 rev1=1 crypto rx=0x7f8f6000a8a0 tx=0x7f8f50005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a0e455cb59ea5fe7 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8f6000a4c0 con 0x7f8f50004d40 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8f60034650 con 0x7f8f50004d40 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.124+0000 7f8f69279640 1 -- 172.21.15.64:0/3995655893 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8f60034a40 con 0x7f8f50004d40 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.125+0000 7f8f6c4df640 1 -- 172.21.15.64:0/3995655893 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f50004d40 msgr2=0x7f8f500051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.125+0000 7f8f6c4df640 1 --2- 172.21.15.64:0/3995655893 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f50004d40 0x7f8f500051c0 secure :-1 s=READY pgs=1870 cs=0 l=1 rev1=1 crypto rx=0x7f8f6000a8a0 tx=0x7f8f50005830 comp rx=0 tx=0).stop 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.125+0000 7f8f6aa7c640 1 -- 172.21.15.64:0/3995655893 reap_dead start 2024-03-04T07:37:26.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.125+0000 7f8f6c4df640 1 -- 172.21.15.64:0/3995655893 shutdown_connections 2024-03-04T07:37:26.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.125+0000 7f8f6c4df640 1 -- 172.21.15.64:0/3995655893 >> 172.21.15.64:0/3995655893 conn(0x7f8f640d1a90 msgr2=0x7f8f640d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:26.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.125+0000 7f8f6c4df640 1 -- 172.21.15.64:0/3995655893 shutdown_connections 2024-03-04T07:37:26.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.125+0000 7f8f6c4df640 1 -- 172.21.15.64:0/3995655893 wait complete. 2024-03-04T07:37:26.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.126+0000 7f8f6c4df640 1 Processor -- start 2024-03-04T07:37:26.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.126+0000 7f8f6c4df640 1 -- start start 2024-03-04T07:37:26.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f6c4df640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f50004d40 0x7f8f640c7c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f6c4df640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f640b6450 0x7f8f6415b710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f6c4df640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f640b68d0 0x7f8f6415bc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f6c4df640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8f640cdbd0 con 0x7f8f640b6450 2024-03-04T07:37:26.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f6c4df640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8f6415e440 con 0x7f8f640b68d0 2024-03-04T07:37:26.134 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f6c4df640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8f640cdd50 con 0x7f8f50004d40 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f640b6450 0x7f8f6415b710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f640b6450 0x7f8f6415b710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38264/0 (socket says 172.21.15.64:38264) 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 -- 172.21.15.64:0/689477673 learned_addr learned my addr 172.21.15.64:0/689477673 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 -- 172.21.15.64:0/689477673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f50004d40 msgr2=0x7f8f640c7c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f50004d40 0x7f8f640c7c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 -- 172.21.15.64:0/689477673 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f640b68d0 msgr2=0x7f8f6415bc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f640b68d0 0x7f8f6415bc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.135 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.127+0000 7f8f69a7a640 1 -- 172.21.15.64:0/689477673 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8f640af4c0 con 0x7f8f640b6450 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.128+0000 7f8f69a7a640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f640b6450 0x7f8f6415b710 secure :-1 s=READY pgs=1871 cs=0 l=1 rev1=1 crypto rx=0x7f8f540116d0 tx=0x7f8f54011b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.128+0000 7f8f5affd640 1 -- 172.21.15.64:0/689477673 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8f5400f940 con 0x7f8f640b6450 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.128+0000 7f8f5affd640 1 -- 172.21.15.64:0/689477673 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8f5400fae0 con 0x7f8f640b6450 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.128+0000 7f8f5affd640 1 -- 172.21.15.64:0/689477673 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8f5400fe90 con 0x7f8f640b6450 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.128+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8f640b5970 con 0x7f8f640b6450 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.128+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8f640b5f30 con 0x7f8f640b6450 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.129+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8f64067ff0 con 0x7f8f640b6450 2024-03-04T07:37:26.136 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.129+0000 7f8f5affd640 1 -- 172.21.15.64:0/689477673 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8f54015460 con 0x7f8f640b6450 2024-03-04T07:37:26.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.129+0000 7f8f5affd640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8f4806b670 0x7f8f4806db30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.130+0000 7f8f6a27b640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8f4806b670 0x7f8f4806db30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.130+0000 7f8f5affd640 1 -- 172.21.15.64:0/689477673 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f8f5408a930 con 0x7f8f640b6450 2024-03-04T07:37:26.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.130+0000 7f8f6a27b640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8f4806b670 0x7f8f4806db30 secure :-1 s=READY pgs=853 cs=0 l=1 rev1=1 crypto rx=0x7f8f5c009a40 tx=0x7f8f5c009320 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:26.137 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.134+0000 7f8f5affd640 1 -- 172.21.15.64:0/689477673 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8f54019030 con 0x7f8f640b6450 2024-03-04T07:37:26.312 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.312+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f8f640af7a0 con 0x7f8f4806b670 2024-03-04T07:37:26.313 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.313+0000 7f8f5affd640 1 -- 172.21.15.64:0/689477673 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+13510 (secure 0 0 0) 0x7f8f640af7a0 con 0x7f8f4806b670 2024-03-04T07:37:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:26.314 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:37:26.315 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8f4806b670 msgr2=0x7f8f4806db30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8f4806b670 0x7f8f4806db30 secure :-1 s=READY pgs=853 cs=0 l=1 rev1=1 crypto rx=0x7f8f5c009a40 tx=0x7f8f5c009320 comp rx=0 tx=0).stop 2024-03-04T07:37:26.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f640b6450 msgr2=0x7f8f6415b710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f640b6450 0x7f8f6415b710 secure :-1 s=READY pgs=1871 cs=0 l=1 rev1=1 crypto rx=0x7f8f540116d0 tx=0x7f8f54011b90 comp rx=0 tx=0).stop 2024-03-04T07:37:26.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 shutdown_connections 2024-03-04T07:37:26.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f8f4806b670 0x7f8f4806db30 unknown :-1 s=CLOSED pgs=853 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.316 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f8f640b68d0 0x7f8f6415bc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f8f640b6450 0x7f8f6415b710 unknown :-1 s=CLOSED pgs=1871 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 --2- 172.21.15.64:0/689477673 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f8f50004d40 0x7f8f640c7c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 >> 172.21.15.64:0/689477673 conn(0x7f8f640d1a90 msgr2=0x7f8f6415d120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:26.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 shutdown_connections 2024-03-04T07:37:26.317 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.315+0000 7f8f6c4df640 1 -- 172.21.15.64:0/689477673 wait complete. 2024-03-04T07:37:26.326 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":445,"stamp":"2024-03-04T07:37:25.265536+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":194,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":329760,"kb_used_data":13772,"kb_used_omap":12,"kb_used_meta":315891,"kb_avail":754644960,"statfs":{"total":773094113280,"available":772756439040,"internally_reserved":0,"allocated":14102528,"data_stored":6826179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":323472944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":29,"apply_latency_ms":29,"commit_latency_ns":29000000,"apply_latency_ns":29000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-33651,"num_objects":-44,"num_object_clones":0,"num_object_copies":-88,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-44,"num_whiteouts":0,"num_read":-4,"num_read_kb":0,"num_write":-143,"num_write_kb":-80,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-28,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001454"},"pg_stats":[{"pgid":"1.0","version":"15'117","reported_seq":368,"reported_epoch":128,"state":"active+clean","last_fresh":"2024-03-04T07:37:20.650026+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:37:20.650026+0000","last_peered":"2024-03-04T07:37:20.650026+0000","last_clean":"2024-03-04T07:37:20.650026+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:37:20.650026+0000","last_undegraded":"2024-03-04T07:37:20.650026+0000","last_fullsized":"2024-03-04T07:37:20.650026+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574981,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41680,"kb_used_data":2180,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330160,"statfs":{"total":96636764160,"available":96594083840,"internally_reserved":0,"allocated":2232320,"data_stored":1309301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":40434122},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574980,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41092,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330748,"statfs":{"total":96636764160,"available":96594685952,"internally_reserved":0,"allocated":1630208,"data_stored":718317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574980,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41080,"kb_used_data":1580,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330760,"statfs":{"total":96636764160,"available":96594698240,"internally_reserved":0,"allocated":1617920,"data_stored":700446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574981,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41064,"kb_used_data":1564,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330776,"statfs":{"total":96636764160,"available":96594714624,"internally_reserved":0,"allocated":1601536,"data_stored":699642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574980,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41080,"kb_used_data":1584,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330760,"statfs":{"total":96636764160,"available":96594698240,"internally_reserved":0,"allocated":1622016,"data_stored":707688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574980,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41056,"kb_used_data":1560,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330784,"statfs":{"total":96636764160,"available":96594722816,"internally_reserved":0,"allocated":1597440,"data_stored":693967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574980,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41084,"kb_used_data":1584,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330756,"statfs":{"total":96636764160,"available":96594694144,"internally_reserved":0,"allocated":1622016,"data_stored":713189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574980,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41624,"kb_used_data":2128,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330216,"statfs":{"total":96636764160,"available":96594141184,"internally_reserved":0,"allocated":2179072,"data_stored":1283629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:37:26.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-04T07:37:26.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.428+0000 7f6edf6fa640 1 Processor -- start 2024-03-04T07:37:26.428 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.428+0000 7f6edf6fa640 1 -- start start 2024-03-04T07:37:26.429 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6edf6fa640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6ed80aee10 0x7f6ed8165a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6edf6fa640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6ed8071240 con 0x7f6ed806df30 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6edf6fa640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6ed8071520 con 0x7f6ed8158520 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6edf6fa640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ed8071b70 con 0x7f6ed80aee10 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6edcc95640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6ed806df30 0x7f6ed80ae700 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50106/0 (socket says 172.21.15.64:50106) 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6edcc95640 1 -- 172.21.15.64:0/1678138824 learned_addr learned my addr 172.21.15.64:0/1678138824 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6eddc97640 1 --2- 172.21.15.64:0/1678138824 >> v2:172.21.15.66:3300/0 conn(0x7f6ed80aee10 0x7f6ed8165a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6ed7fff640 1 -- 172.21.15.64:0/1678138824 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2282859796 0 0) 0x7f6ed8071240 con 0x7f6ed806df30 2024-03-04T07:37:26.430 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.429+0000 7f6ed7fff640 1 -- 172.21.15.64:0/1678138824 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f6ec0005730 con 0x7f6ed806df30 2024-03-04T07:37:26.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.430+0000 7f6eddc97640 1 -- 172.21.15.64:0/1678138824 >> v1:172.21.15.66:6789/0 conn(0x7f6ed8158520 legacy=0x7f6ed806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.430+0000 7f6eddc97640 1 -- 172.21.15.64:0/1678138824 >> v1:172.21.15.64:6789/0 conn(0x7f6ed806df30 legacy=0x7f6ed80ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.430+0000 7f6ed7fff640 1 -- 172.21.15.64:0/1678138824 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 3003200768 0 0) 0x7f6ec0005730 con 0x7f6ed806df30 2024-03-04T07:37:26.431 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.430+0000 7f6eddc97640 1 -- 172.21.15.64:0/1678138824 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ed8071800 con 0x7f6ed80aee10 2024-03-04T07:37:26.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.430+0000 7f6eddc97640 1 --2- 172.21.15.64:0/1678138824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed80aee10 0x7f6ed8165a00 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f6ec80097a0 tx=0x7f6ec802b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8fa2b1cc4605697e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:26.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6ed7fff640 1 -- 172.21.15.64:0/1678138824 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec8037070 con 0x7f6ed80aee10 2024-03-04T07:37:26.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6ed7fff640 1 -- 172.21.15.64:0/1678138824 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6ec8030510 con 0x7f6ed80aee10 2024-03-04T07:37:26.437 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6ed7fff640 1 -- 172.21.15.64:0/1678138824 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ecc001670 con 0x7f6ed80aee10 2024-03-04T07:37:26.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6edf6fa640 1 -- 172.21.15.64:0/1678138824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed80aee10 msgr2=0x7f6ed8165a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/1678138824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed80aee10 0x7f6ed8165a00 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f6ec80097a0 tx=0x7f6ec802b790 comp rx=0 tx=0).stop 2024-03-04T07:37:26.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6edf6fa640 1 -- 172.21.15.64:0/1678138824 shutdown_connections 2024-03-04T07:37:26.438 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/1678138824 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed80aee10 0x7f6ed8165a00 unknown :-1 s=CLOSED pgs=693 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6edf6fa640 1 -- 172.21.15.64:0/1678138824 >> 172.21.15.64:0/1678138824 conn(0x7f6ed8066d00 msgr2=0x7f6ed80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:26.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6edf6fa640 1 -- 172.21.15.64:0/1678138824 shutdown_connections 2024-03-04T07:37:26.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.431+0000 7f6edf6fa640 1 -- 172.21.15.64:0/1678138824 wait complete. 2024-03-04T07:37:26.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.432+0000 7f6edf6fa640 1 Processor -- start 2024-03-04T07:37:26.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.432+0000 7f6edf6fa640 1 -- start start 2024-03-04T07:37:26.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edf6fa640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed806df30 0x7f6ed80cd880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.439 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edf6fa640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ed80aee10 0x7f6ed8149430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edf6fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ed8158520 0x7f6ed815d4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edf6fa640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ed8158900 con 0x7f6ed8158520 2024-03-04T07:37:26.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edf6fa640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ed8072000 con 0x7f6ed806df30 2024-03-04T07:37:26.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edf6fa640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ed8158a80 con 0x7f6ed80aee10 2024-03-04T07:37:26.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ed8158520 0x7f6ed815d4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ed8158520 0x7f6ed815d4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38278/0 (socket says 172.21.15.64:38278) 2024-03-04T07:37:26.440 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 -- 172.21.15.64:0/410281021 learned_addr learned my addr 172.21.15.64:0/410281021 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:26.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edd496640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed806df30 0x7f6ed80cd880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edcc95640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ed80aee10 0x7f6ed8149430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 -- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ed80aee10 msgr2=0x7f6ed8149430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ed80aee10 0x7f6ed8149430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 -- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed806df30 msgr2=0x7f6ed80cd880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed806df30 0x7f6ed80cd880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.441 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6eddc97640 1 -- 172.21.15.64:0/410281021 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ed80ccde0 con 0x7f6ed8158520 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.433+0000 7f6edcc95640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ed80aee10 0x7f6ed8149430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.434+0000 7f6eddc97640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ed8158520 0x7f6ed815d4b0 secure :-1 s=READY pgs=1873 cs=0 l=1 rev1=1 crypto rx=0x7f6ec802b7d0 tx=0x7f6ec80046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.434+0000 7f6ed5ffb640 1 -- 172.21.15.64:0/410281021 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec8042020 con 0x7f6ed8158520 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.434+0000 7f6ed5ffb640 1 -- 172.21.15.64:0/410281021 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6ec8003610 con 0x7f6ed8158520 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.434+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ed8149970 con 0x7f6ed8158520 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.434+0000 7f6ed5ffb640 1 -- 172.21.15.64:0/410281021 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec8037040 con 0x7f6ed8158520 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.434+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ed80cd080 con 0x7f6ed8158520 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.435+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ed8067ff0 con 0x7f6ed8158520 2024-03-04T07:37:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.436+0000 7f6ed5ffb640 1 -- 172.21.15.64:0/410281021 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6ec8045020 con 0x7f6ed8158520 2024-03-04T07:37:26.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.436+0000 7f6ed5ffb640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b500 0x7f6ec406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:26.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.436+0000 7f6ed5ffb640 1 -- 172.21.15.64:0/410281021 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f6ec80abb40 con 0x7f6ed8158520 2024-03-04T07:37:26.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.436+0000 7f6edd496640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b500 0x7f6ec406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:26.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.437+0000 7f6edd496640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b500 0x7f6ec406d9c0 secure :-1 s=READY pgs=854 cs=0 l=1 rev1=1 crypto rx=0x7f6ed0001ca0 tx=0x7f6ed0009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:26.443 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.440+0000 7f6ed5ffb640 1 -- 172.21.15.64:0/410281021 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f6ec804c030 con 0x7f6ed8158520 2024-03-04T07:37:26.622 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.622+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6ed8159bc0 con 0x7f6ed8158520 2024-03-04T07:37:26.623 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.623+0000 7f6ed5ffb640 1 -- 172.21.15.64:0/410281021 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v130) v1 ==== 74+0+13110 (secure 0 0 0) 0x7f6ec807ad80 con 0x7f6ed8158520 2024-03-04T07:37:26.623 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:26.623 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":130,"fsid":"c3ef085c-fd40-4f87-8c92-fdb42a102a8c","created":"2024-03-04T07:26:28.756832+0000","modified":"2024-03-04T07:37:23.423491+0000","last_up_change":"2024-03-04T07:26:40.208454+0000","last_in_change":"2024-03-04T07:26:31.093625+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":19,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-04T07:26:41.184112+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":8,"score_stable":8,"optimal_score":0.25,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fb72a7b9-ce40-4a01-82e5-ae0017868138","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6802","nonce":2111658274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6805","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6807","nonce":2111658274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6815","nonce":2111658274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2111658274},{"type":"v1","addr":"172.21.15.64:6812","nonce":2111658274}]},"public_addr":"172.21.15.64:6802/2111658274","cluster_addr":"172.21.15.64:6807/2111658274","heartbeat_back_addr":"172.21.15.64:6815/2111658274","heartbeat_front_addr":"172.21.15.64:6812/2111658274","state":["exists","up"]},{"osd":1,"uuid":"181008cb-9136-4647-bf89-704c76688299","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6801","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6803","nonce":1190712954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6806","nonce":1190712954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6811","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6813","nonce":1190712954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1190712954},{"type":"v1","addr":"172.21.15.64:6809","nonce":1190712954}]},"public_addr":"172.21.15.64:6803/1190712954","cluster_addr":"172.21.15.64:6806/1190712954","heartbeat_back_addr":"172.21.15.64:6813/1190712954","heartbeat_front_addr":"172.21.15.64:6809/1190712954","state":["exists","up"]},{"osd":2,"uuid":"a2ada673-135a-4368-9204-33df19ddaf38","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6825","nonce":2610379167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6827","nonce":2610379167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6831","nonce":2610379167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2610379167},{"type":"v1","addr":"172.21.15.64:6829","nonce":2610379167}]},"public_addr":"172.21.15.64:6825/2610379167","cluster_addr":"172.21.15.64:6827/2610379167","heartbeat_back_addr":"172.21.15.64:6831/2610379167","heartbeat_front_addr":"172.21.15.64:6829/2610379167","state":["exists","up"]},{"osd":3,"uuid":"4cdda1a3-e9e6-4169-849b-92384852886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6817","nonce":1402142346}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6819","nonce":1402142346}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6823","nonce":1402142346}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1402142346},{"type":"v1","addr":"172.21.15.64:6821","nonce":1402142346}]},"public_addr":"172.21.15.64:6817/1402142346","cluster_addr":"172.21.15.64:6819/1402142346","heartbeat_back_addr":"172.21.15.64:6823/1402142346","heartbeat_front_addr":"172.21.15.64:6821/1402142346","state":["exists","up"]},{"osd":4,"uuid":"f982f669-8e67-4b99-8b43-7b070a797819","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6824","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6825","nonce":1927515367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6826","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6827","nonce":1927515367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6830","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6831","nonce":1927515367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6828","nonce":1927515367},{"type":"v1","addr":"172.21.15.66:6829","nonce":1927515367}]},"public_addr":"172.21.15.66:6825/1927515367","cluster_addr":"172.21.15.66:6827/1927515367","heartbeat_back_addr":"172.21.15.66:6831/1927515367","heartbeat_front_addr":"172.21.15.66:6829/1927515367","state":["exists","up"]},{"osd":5,"uuid":"e4ebc912-75e2-40c7-9eb2-760859196dff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6800","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6801","nonce":1583207443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6803","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6805","nonce":1583207443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6811","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6813","nonce":1583207443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6807","nonce":1583207443},{"type":"v1","addr":"172.21.15.66:6809","nonce":1583207443}]},"public_addr":"172.21.15.66:6801/1583207443","cluster_addr":"172.21.15.66:6805/1583207443","heartbeat_back_addr":"172.21.15.66:6813/1583207443","heartbeat_front_addr":"172.21.15.66:6809/1583207443","state":["exists","up"]},{"osd":6,"uuid":"79429dec-fad1-4f75-8847-f4faebaef17d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6816","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6817","nonce":177421408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6818","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6819","nonce":177421408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6822","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6823","nonce":177421408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6820","nonce":177421408},{"type":"v1","addr":"172.21.15.66:6821","nonce":177421408}]},"public_addr":"172.21.15.66:6817/177421408","cluster_addr":"172.21.15.66:6819/177421408","heartbeat_back_addr":"172.21.15.66:6823/177421408","heartbeat_front_addr":"172.21.15.66:6821/177421408","state":["exists","up"]},{"osd":7,"uuid":"59d05be1-af84-47ee-8132-20c66e6aa6be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":121,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6802","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6804","nonce":3006518207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6806","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6808","nonce":3006518207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6814","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6815","nonce":3006518207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6810","nonce":3006518207},{"type":"v1","addr":"172.21.15.66:6812","nonce":3006518207}]},"public_addr":"172.21.15.66:6804/3006518207","cluster_addr":"172.21.15.66:6808/3006518207","heartbeat_back_addr":"172.21.15.66:6815/3006518207","heartbeat_front_addr":"172.21.15.66:6812/3006518207","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.256820+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.272428+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.175622+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.323626+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.261124+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.230995+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-04T07:26:36.232293+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.66:6832/546842609":"2024-03-05T07:37:20.386801+0000","172.21.15.66:6834/546842609":"2024-03-05T07:37:20.386801+0000","172.21.15.66:6835/2326824208":"2024-03-05T07:37:20.386788+0000","172.21.15.66:6833/2326824208":"2024-03-05T07:37:20.386788+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-04T07:37:26.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b500 msgr2=0x7f6ec406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b500 0x7f6ec406d9c0 secure :-1 s=READY pgs=854 cs=0 l=1 rev1=1 crypto rx=0x7f6ed0001ca0 tx=0x7f6ed0009040 comp rx=0 tx=0).stop 2024-03-04T07:37:26.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ed8158520 msgr2=0x7f6ed815d4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:26.625 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ed8158520 0x7f6ed815d4b0 secure :-1 s=READY pgs=1873 cs=0 l=1 rev1=1 crypto rx=0x7f6ec802b7d0 tx=0x7f6ec80046c0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 shutdown_connections 2024-03-04T07:37:26.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6ec406b500 0x7f6ec406d9c0 unknown :-1 s=CLOSED pgs=854 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ed8158520 0x7f6ed815d4b0 unknown :-1 s=CLOSED pgs=1873 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ed80aee10 0x7f6ed8149430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 --2- 172.21.15.64:0/410281021 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ed806df30 0x7f6ed80cd880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:26.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 >> 172.21.15.64:0/410281021 conn(0x7f6ed8066d00 msgr2=0x7f6ed8158cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:26.626 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 shutdown_connections 2024-03-04T07:37:26.627 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:26.625+0000 7f6edf6fa640 1 -- 172.21.15.64:0/410281021 wait complete. 2024-03-04T07:37:27.638 INFO:tasks.ceph:Scrubbing osd.0 2024-03-04T07:37:27.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:27.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.757+0000 7f0255e2a640 1 Processor -- start 2024-03-04T07:37:27.757 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.757+0000 7f0255e2a640 1 -- start start 2024-03-04T07:37:27.759 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f0255e2a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f02500ceb00 0x7f02500b4220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f0255e2a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f025015e4b0 con 0x7f025015e0d0 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f0255e2a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f02500cefc0 con 0x7f02500cdc20 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f0255e2a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f02500cc1c0 con 0x7f02500ceb00 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f024f7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f025015e0d0 0x7f02500ce3f0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50112/0 (socket says 172.21.15.64:50112) 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f024f7fe640 1 -- 172.21.15.64:0/3364408607 learned_addr learned my addr 172.21.15.64:0/3364408607 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f024effd640 1 --2- 172.21.15.64:0/3364408607 >> v2:172.21.15.66:3300/0 conn(0x7f02500ceb00 0x7f02500b4220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2969446317 0 0) 0x7f025015e4b0 con 0x7f025015e0d0 2024-03-04T07:37:27.760 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.758+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f023c003520 con 0x7f025015e0d0 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.759+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1575109997 0 0) 0x7f02500cefc0 con 0x7f02500cdc20 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.759+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f025015e4b0 con 0x7f02500cdc20 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.759+0000 7f024effd640 1 -- 172.21.15.64:0/3364408607 >> v1:172.21.15.66:6789/0 conn(0x7f02500cdc20 legacy=0x7f02500cfd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.759+0000 7f024effd640 1 -- 172.21.15.64:0/3364408607 >> v1:172.21.15.64:6789/0 conn(0x7f025015e0d0 legacy=0x7f02500ce3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.759+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1243848541 0 0) 0x7f023c003520 con 0x7f025015e0d0 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.759+0000 7f024effd640 1 -- 172.21.15.64:0/3364408607 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02500cf2a0 con 0x7f02500ceb00 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f024effd640 1 --2- 172.21.15.64:0/3364408607 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500ceb00 0x7f02500b4220 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7f02380097a0 tx=0x7f023802b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a1c0db2f71eb1bb1 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:27.761 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0238037070 con 0x7f02500ceb00 2024-03-04T07:37:27.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0238030510 con 0x7f02500ceb00 2024-03-04T07:37:27.762 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f024e7fc640 1 -- 172.21.15.64:0/3364408607 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0240001670 con 0x7f02500ceb00 2024-03-04T07:37:27.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f0255e2a640 1 -- 172.21.15.64:0/3364408607 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500ceb00 msgr2=0x7f02500b4220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f0255e2a640 1 --2- 172.21.15.64:0/3364408607 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500ceb00 0x7f02500b4220 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7f02380097a0 tx=0x7f023802b790 comp rx=0 tx=0).stop 2024-03-04T07:37:27.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f0255e2a640 1 -- 172.21.15.64:0/3364408607 shutdown_connections 2024-03-04T07:37:27.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f0255e2a640 1 --2- 172.21.15.64:0/3364408607 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500ceb00 0x7f02500b4220 unknown :-1 s=CLOSED pgs=695 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.765 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f0255e2a640 1 -- 172.21.15.64:0/3364408607 >> 172.21.15.64:0/3364408607 conn(0x7f02500b3700 msgr2=0x7f02500b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f0255e2a640 1 -- 172.21.15.64:0/3364408607 shutdown_connections 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.760+0000 7f0255e2a640 1 -- 172.21.15.64:0/3364408607 wait complete. 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.762+0000 7f0255e2a640 1 Processor -- start 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.762+0000 7f0255e2a640 1 -- start start 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f0255e2a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500cdc20 0x7f02501f7b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f0255e2a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02500ceb00 0x7f02500c66b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f0255e2a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f025015e0d0 0x7f02500cb100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f0255e2a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f025006b3d0 con 0x7f025015e0d0 2024-03-04T07:37:27.766 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f0255e2a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02500cc6f0 con 0x7f02500cdc20 2024-03-04T07:37:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f0255e2a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f02500d0410 con 0x7f02500ceb00 2024-03-04T07:37:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f025015e0d0 0x7f02500cb100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f025015e0d0 0x7f02500cb100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38286/0 (socket says 172.21.15.64:38286) 2024-03-04T07:37:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 -- 172.21.15.64:0/2001293056 learned_addr learned my addr 172.21.15.64:0/2001293056 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024f7fe640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500cdc20 0x7f02501f7b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.767 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024effd640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02500ceb00 0x7f02500c66b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 -- 172.21.15.64:0/2001293056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02500ceb00 msgr2=0x7f02500c66b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f02500ceb00 0x7f02500c66b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 -- 172.21.15.64:0/2001293056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500cdc20 msgr2=0x7f02501f7b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500cdc20 0x7f02501f7b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 -- 172.21.15.64:0/2001293056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02500ca940 con 0x7f025015e0d0 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024f7fe640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f02500cdc20 0x7f02501f7b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.763+0000 7f024ffff640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f025015e0d0 0x7f02500cb100 secure :-1 s=READY pgs=1875 cs=0 l=1 rev1=1 crypto rx=0x7f024800c790 tx=0x7f024800cc50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:27.768 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.764+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0248003b70 con 0x7f025015e0d0 2024-03-04T07:37:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.764+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f0248003d10 con 0x7f025015e0d0 2024-03-04T07:37:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.764+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f024800aa50 con 0x7f025015e0d0 2024-03-04T07:37:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.764+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f02500c8c40 con 0x7f025015e0d0 2024-03-04T07:37:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.764+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02500c9200 con 0x7f025015e0d0 2024-03-04T07:37:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.765+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f025009fcf0 con 0x7f025015e0d0 2024-03-04T07:37:27.769 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.765+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f024800abf0 con 0x7f025015e0d0 2024-03-04T07:37:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f0247fff640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f023006b4d0 0x7f023006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f0248089f80 con 0x7f025015e0d0 2024-03-04T07:37:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f024f7fe640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f023006b4d0 0x7f023006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f0247fff640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f0230071430 0x7f0230073850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f0230073f00 con 0x7f0230071430 2024-03-04T07:37:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7f024808c620 con 0x7f025015e0d0 2024-03-04T07:37:27.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f024effd640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f0230071430 0x7f0230073850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f024f7fe640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f023006b4d0 0x7f023006d990 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7f02400011e0 tx=0x7f0240009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:27.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.766+0000 7f024effd640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f0230071430 0x7f0230073850 crc :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:27.772 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.767+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== osd.0 v2:172.21.15.64:6800/2111658274 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f0230073f00 con 0x7f0230071430 2024-03-04T07:37:27.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 --> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7f02500caec0 con 0x7f0230071430 2024-03-04T07:37:27.781 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0247fff640 1 -- 172.21.15.64:0/2001293056 <== osd.0 v2:172.21.15.64:6800/2111658274 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7f02500caec0 con 0x7f0230071430 2024-03-04T07:37:27.781 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:27.781 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f0230071430 msgr2=0x7f0230073850 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6800/2111658274,v1:172.21.15.64:6802/2111658274] conn(0x7f0230071430 0x7f0230073850 crc :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f023006b4d0 msgr2=0x7f023006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f023006b4d0 0x7f023006d990 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7f02400011e0 tx=0x7f0240009040 comp rx=0 tx=0).stop 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f025015e0d0 msgr2=0x7f02500cb100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 --2- 172.21.15.64:0/2001293056 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f025015e0d0 0x7f02500cb100 secure :-1 s=READY pgs=1875 cs=0 l=1 rev1=1 crypto rx=0x7f024800c790 tx=0x7f024800cc50 comp rx=0 tx=0).stop 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f024ffff640 1 -- 172.21.15.64:0/2001293056 reap_dead start 2024-03-04T07:37:27.783 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 shutdown_connections 2024-03-04T07:37:27.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.781+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 >> 172.21.15.64:0/2001293056 conn(0x7f02500b3700 msgr2=0x7f0250070510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:27.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.782+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 shutdown_connections 2024-03-04T07:37:27.784 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.782+0000 7f0255e2a640 1 -- 172.21.15.64:0/2001293056 wait complete. 2024-03-04T07:37:27.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 0 2024-03-04T07:37:27.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.897+0000 7f12aea15640 1 Processor -- start 2024-03-04T07:37:27.897 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.897+0000 7f12aea15640 1 -- start start 2024-03-04T07:37:27.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12aea15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a8158520 0x7f12a806d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12aea15640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f12a806dd60 0x7f12a80ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12aea15640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f12a80af080 con 0x7f12a80aec40 2024-03-04T07:37:27.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12aea15640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f12a806e1c0 con 0x7f12a806dd60 2024-03-04T07:37:27.899 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12aea15640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f12a8071b70 con 0x7f12a8158520 2024-03-04T07:37:27.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12a77fe640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f12a806dd60 0x7f12a80ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12acfb2640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f12a80aec40 0x7f12a815e670 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50114/0 (socket says 172.21.15.64:50114) 2024-03-04T07:37:27.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12acfb2640 1 -- 172.21.15.64:0/2426978110 learned_addr learned my addr 172.21.15.64:0/2426978110 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:27.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12a7fff640 1 --2- 172.21.15.64:0/2426978110 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a8158520 0x7f12a806d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12a77fe640 1 -- 172.21.15.64:0/2426978110 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a8158520 msgr2=0x7f12a806d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:27.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12a77fe640 1 --2- 172.21.15.64:0/2426978110 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a8158520 0x7f12a806d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.900 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12a77fe640 1 -- 172.21.15.64:0/2426978110 >> v1:172.21.15.64:6789/0 conn(0x7f12a80aec40 legacy=0x7f12a815e670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12a77fe640 1 -- 172.21.15.64:0/2426978110 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12a8071240 con 0x7f12a806dd60 2024-03-04T07:37:27.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.898+0000 7f12a7fff640 1 --2- 172.21.15.64:0/2426978110 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a8158520 0x7f12a806d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:27.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.899+0000 7f12a77fe640 1 --2- 172.21.15.64:0/2426978110 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a806dd60 0x7f12a80ae700 secure :-1 s=READY pgs=1877 cs=0 l=1 rev1=1 crypto rx=0x7f12980097a0 tx=0x7f129802b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c0af265f535045f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:27.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.899+0000 7f12a6ffd640 1 -- 172.21.15.64:0/2426978110 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1298035020 con 0x7f12a806dd60 2024-03-04T07:37:27.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.899+0000 7f12a6ffd640 1 -- 172.21.15.64:0/2426978110 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1298031830 con 0x7f12a806dd60 2024-03-04T07:37:27.901 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.899+0000 7f12a6ffd640 1 -- 172.21.15.64:0/2426978110 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1298031c00 con 0x7f12a806dd60 2024-03-04T07:37:27.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 -- 172.21.15.64:0/2426978110 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a806dd60 msgr2=0x7f12a80ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 --2- 172.21.15.64:0/2426978110 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a806dd60 0x7f12a80ae700 secure :-1 s=READY pgs=1877 cs=0 l=1 rev1=1 crypto rx=0x7f12980097a0 tx=0x7f129802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 -- 172.21.15.64:0/2426978110 shutdown_connections 2024-03-04T07:37:27.905 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 --2- 172.21.15.64:0/2426978110 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a806dd60 0x7f12a80ae700 unknown :-1 s=CLOSED pgs=1877 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 --2- 172.21.15.64:0/2426978110 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a8158520 0x7f12a806d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 -- 172.21.15.64:0/2426978110 >> 172.21.15.64:0/2426978110 conn(0x7f12a8066d00 msgr2=0x7f12a80670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 -- 172.21.15.64:0/2426978110 shutdown_connections 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.900+0000 7f12aea15640 1 -- 172.21.15.64:0/2426978110 wait complete. 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.901+0000 7f12aea15640 1 Processor -- start 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.901+0000 7f12aea15640 1 -- start start 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12aea15640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a806dd60 0x7f12a814c610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12aea15640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f12a80aec40 0x7f12a80cefc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12aea15640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a8158520 0x7f12a80cf500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12aea15640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a80b4650 con 0x7f12a8158520 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12aea15640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a8071800 con 0x7f12a80aec40 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12aea15640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f12a8072000 con 0x7f12a806dd60 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12acfb2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a8158520 0x7f12a80cf500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12acfb2640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a8158520 0x7f12a80cf500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38316/0 (socket says 172.21.15.64:38316) 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.902+0000 7f12acfb2640 1 -- 172.21.15.64:0/3173133740 learned_addr learned my addr 172.21.15.64:0/3173133740 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12a77fe640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f12a80aec40 0x7f12a80cefc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12a7fff640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a806dd60 0x7f12a814c610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12acfb2640 1 -- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a806dd60 msgr2=0x7f12a814c610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12acfb2640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a806dd60 0x7f12a814c610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12acfb2640 1 -- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f12a80aec40 msgr2=0x7f12a80cefc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12acfb2640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f12a80aec40 0x7f12a80cefc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12acfb2640 1 -- 172.21.15.64:0/3173133740 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12a80af3b0 con 0x7f12a8158520 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12a77fe640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f12a80aec40 0x7f12a80cefc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12acfb2640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a8158520 0x7f12a80cf500 secure :-1 s=READY pgs=1878 cs=0 l=1 rev1=1 crypto rx=0x7f12a000cbc0 tx=0x7f12a000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12a4ff9640 1 -- 172.21.15.64:0/3173133740 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12a0017070 con 0x7f12a8158520 2024-03-04T07:37:27.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12a4ff9640 1 -- 172.21.15.64:0/3173133740 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f12a000ad40 con 0x7f12a8158520 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f12a80af620 con 0x7f12a8158520 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12a4ff9640 1 -- 172.21.15.64:0/3173133740 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12a0012600 con 0x7f12a8158520 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.903+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12a80afbe0 con 0x7f12a8158520 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.904+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f12a8067ff0 con 0x7f12a8158520 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.905+0000 7f12a4ff9640 1 -- 172.21.15.64:0/3173133740 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f12a001a070 con 0x7f12a8158520 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.905+0000 7f12a4ff9640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f128806b4d0 0x7f128806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.905+0000 7f12a4ff9640 1 -- 172.21.15.64:0/3173133740 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f12a0089bd0 con 0x7f12a8158520 2024-03-04T07:37:27.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.905+0000 7f12a7fff640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f128806b4d0 0x7f128806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:27.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.906+0000 7f12a7fff640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f128806b4d0 0x7f128806d990 secure :-1 s=READY pgs=856 cs=0 l=1 rev1=1 crypto rx=0x7f1290006fd0 tx=0x7f1290008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:27.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:27.909+0000 7f12a4ff9640 1 -- 172.21.15.64:0/3173133740 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f12a00586a0 con 0x7f12a8158520 2024-03-04T07:37:28.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.093+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "0", "target": ["mon-mgr", ""]}) v1 -- 0x7f12a8154390 con 0x7f128806b4d0 2024-03-04T07:37:28.093 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.093+0000 7f12a4ff9640 1 -- 172.21.15.64:0/3173133740 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 0 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7f12a8154390 con 0x7f128806b4d0 2024-03-04T07:37:28.093 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 0 to deep-scrub 2024-03-04T07:37:28.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f128806b4d0 msgr2=0x7f128806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f128806b4d0 0x7f128806d990 secure :-1 s=READY pgs=856 cs=0 l=1 rev1=1 crypto rx=0x7f1290006fd0 tx=0x7f1290008040 comp rx=0 tx=0).stop 2024-03-04T07:37:28.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a8158520 msgr2=0x7f12a80cf500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.095 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a8158520 0x7f12a80cf500 secure :-1 s=READY pgs=1878 cs=0 l=1 rev1=1 crypto rx=0x7f12a000cbc0 tx=0x7f12a000a430 comp rx=0 tx=0).stop 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 shutdown_connections 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f128806b4d0 0x7f128806d990 unknown :-1 s=CLOSED pgs=856 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f12a8158520 0x7f12a80cf500 unknown :-1 s=CLOSED pgs=1878 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f12a80aec40 0x7f12a80cefc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 --2- 172.21.15.64:0/3173133740 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f12a806dd60 0x7f12a814c610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 >> 172.21.15.64:0/3173133740 conn(0x7f12a8066d00 msgr2=0x7f12a806c6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 shutdown_connections 2024-03-04T07:37:28.096 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.095+0000 7f12aea15640 1 -- 172.21.15.64:0/3173133740 wait complete. 2024-03-04T07:37:28.107 INFO:tasks.ceph:Scrubbing osd.1 2024-03-04T07:37:28.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:28.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.214+0000 7f6aef89d640 1 Processor -- start 2024-03-04T07:37:28.214 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.214+0000 7f6aef89d640 1 -- start start 2024-03-04T07:37:28.216 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aef89d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f6ae815dd90 0x7f6ae80ceea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aef89d640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f6ae80b6d10 0x7f6ae80b7150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aef89d640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f6ae80d3740 con 0x7f6ae80d29d0 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aef89d640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ae80d3400 con 0x7f6ae80b6d10 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aef89d640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f6ae80b7690 con 0x7f6ae815dd90 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aece38640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f6ae80d29d0 0x7f6ae80d41e0 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50126/0 (socket says 172.21.15.64:50126) 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aece38640 1 -- 172.21.15.64:0/3602777761 learned_addr learned my addr 172.21.15.64:0/3602777761 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.215+0000 7f6aede3a640 1 --2- 172.21.15.64:0/3602777761 >> v2:172.21.15.64:3300/0 conn(0x7f6ae80b6d10 0x7f6ae80b7150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.217 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.216+0000 7f6aed639640 1 --2- 172.21.15.64:0/3602777761 >> v2:172.21.15.66:3300/0 conn(0x7f6ae815dd90 0x7f6ae80ceea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.216+0000 7f6aede3a640 1 -- 172.21.15.64:0/3602777761 >> v2:172.21.15.66:3300/0 conn(0x7f6ae815dd90 msgr2=0x7f6ae80ceea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.216+0000 7f6aede3a640 1 --2- 172.21.15.64:0/3602777761 >> v2:172.21.15.66:3300/0 conn(0x7f6ae815dd90 0x7f6ae80ceea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.216+0000 7f6aede3a640 1 -- 172.21.15.64:0/3602777761 >> v1:172.21.15.64:6789/0 conn(0x7f6ae80d29d0 legacy=0x7f6ae80d41e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.216+0000 7f6adffff640 1 -- 172.21.15.64:0/3602777761 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 4195059963 0 0) 0x7f6ae80d3740 con 0x7f6ae80d29d0 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.216+0000 7f6aede3a640 1 -- 172.21.15.64:0/3602777761 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ae80d3120 con 0x7f6ae80b6d10 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.216+0000 7f6aede3a640 1 --2- 172.21.15.64:0/3602777761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae80b6d10 0x7f6ae80b7150 secure :-1 s=READY pgs=1880 cs=0 l=1 rev1=1 crypto rx=0x7f6ae80d1f90 tx=0x7f6ae002bf50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=208919580f9a001c server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6adffff640 1 -- 172.21.15.64:0/3602777761 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ae0039070 con 0x7f6ae80b6d10 2024-03-04T07:37:28.218 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6adffff640 1 -- 172.21.15.64:0/3602777761 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6ae002cd00 con 0x7f6ae80b6d10 2024-03-04T07:37:28.219 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6adffff640 1 -- 172.21.15.64:0/3602777761 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ae0034660 con 0x7f6ae80b6d10 2024-03-04T07:37:28.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 -- 172.21.15.64:0/3602777761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae80b6d10 msgr2=0x7f6ae80b7150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 --2- 172.21.15.64:0/3602777761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae80b6d10 0x7f6ae80b7150 secure :-1 s=READY pgs=1880 cs=0 l=1 rev1=1 crypto rx=0x7f6ae80d1f90 tx=0x7f6ae002bf50 comp rx=0 tx=0).stop 2024-03-04T07:37:28.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 -- 172.21.15.64:0/3602777761 shutdown_connections 2024-03-04T07:37:28.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 --2- 172.21.15.64:0/3602777761 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae80b6d10 0x7f6ae80b7150 unknown :-1 s=CLOSED pgs=1880 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 --2- 172.21.15.64:0/3602777761 >> v2:172.21.15.66:3300/0 conn(0x7f6ae815dd90 0x7f6ae80ceea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.223 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 -- 172.21.15.64:0/3602777761 >> 172.21.15.64:0/3602777761 conn(0x7f6ae8071960 msgr2=0x7f6ae8071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 -- 172.21.15.64:0/3602777761 shutdown_connections 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.217+0000 7f6aef89d640 1 -- 172.21.15.64:0/3602777761 wait complete. 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.218+0000 7f6aef89d640 1 Processor -- start 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.218+0000 7f6aef89d640 1 -- start start 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aef89d640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ae80b6d10 0x7f6ae80ca0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aef89d640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ae80d29d0 0x7f6ae8160ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aef89d640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae815dd90 0x7f6ae80d43b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.224 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aef89d640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ae815e170 con 0x7f6ae815dd90 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aef89d640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ae80d3cc0 con 0x7f6ae80b6d10 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aef89d640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6ae80d2df0 con 0x7f6ae80d29d0 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aede3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae815dd90 0x7f6ae80d43b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aede3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae815dd90 0x7f6ae80d43b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38330/0 (socket says 172.21.15.64:38330) 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aede3a640 1 -- 172.21.15.64:0/2880746652 learned_addr learned my addr 172.21.15.64:0/2880746652 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aed639640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ae80b6d10 0x7f6ae80ca0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.219+0000 7f6aece38640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ae80d29d0 0x7f6ae8160ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.225 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aede3a640 1 -- 172.21.15.64:0/2880746652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ae80d29d0 msgr2=0x7f6ae8160ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aede3a640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f6ae80d29d0 0x7f6ae8160ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aede3a640 1 -- 172.21.15.64:0/2880746652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ae80b6d10 msgr2=0x7f6ae80ca0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aede3a640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ae80b6d10 0x7f6ae80ca0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aede3a640 1 -- 172.21.15.64:0/2880746652 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ae80c92f0 con 0x7f6ae815dd90 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aed639640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f6ae80b6d10 0x7f6ae80ca0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aede3a640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae815dd90 0x7f6ae80d43b0 secure :-1 s=READY pgs=1881 cs=0 l=1 rev1=1 crypto rx=0x7f6ae0009e80 tx=0x7f6ae0004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ae0039050 con 0x7f6ae815dd90 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f6ae0045070 con 0x7f6ae815dd90 2024-03-04T07:37:28.226 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ae80c95d0 con 0x7f6ae815dd90 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ae003e830 con 0x7f6ae815dd90 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.220+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ae80d48f0 con 0x7f6ae815dd90 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.221+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f6ae8072a30 con 0x7f6ae815dd90 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f6ae003e9d0 con 0x7f6ae815dd90 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6addffb640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6acc06b4d0 0x7f6acc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f6ae00ad800 con 0x7f6ae815dd90 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6aed639640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6acc06b4d0 0x7f6acc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.227 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6addffb640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f6acc071430 0x7f6acc073850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f6acc073f00 con 0x7f6acc071430 2024-03-04T07:37:28.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6ae00adc90 con 0x7f6ae815dd90 2024-03-04T07:37:28.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6aece38640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f6acc071430 0x7f6acc073850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.222+0000 7f6aed639640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6acc06b4d0 0x7f6acc06d990 secure :-1 s=READY pgs=857 cs=0 l=1 rev1=1 crypto rx=0x7f6ae4006fd0 tx=0x7f6ae4008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.223+0000 7f6aece38640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f6acc071430 0x7f6acc073850 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.228 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.223+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== osd.1 v2:172.21.15.64:6801/1190712954 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f6acc073f00 con 0x7f6acc071430 2024-03-04T07:37:28.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 --> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7f6ae80d1020 con 0x7f6acc071430 2024-03-04T07:37:28.237 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6addffb640 1 -- 172.21.15.64:0/2880746652 <== osd.1 v2:172.21.15.64:6801/1190712954 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7f6ae80d1020 con 0x7f6acc071430 2024-03-04T07:37:28.237 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:28.237 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:28.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f6acc071430 msgr2=0x7f6acc073850 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.238 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6801/1190712954,v1:172.21.15.64:6803/1190712954] conn(0x7f6acc071430 0x7f6acc073850 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6acc06b4d0 msgr2=0x7f6acc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f6acc06b4d0 0x7f6acc06d990 secure :-1 s=READY pgs=857 cs=0 l=1 rev1=1 crypto rx=0x7f6ae4006fd0 tx=0x7f6ae4008040 comp rx=0 tx=0).stop 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae815dd90 msgr2=0x7f6ae80d43b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 --2- 172.21.15.64:0/2880746652 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f6ae815dd90 0x7f6ae80d43b0 secure :-1 s=READY pgs=1881 cs=0 l=1 rev1=1 crypto rx=0x7f6ae0009e80 tx=0x7f6ae0004a40 comp rx=0 tx=0).stop 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aede3a640 1 -- 172.21.15.64:0/2880746652 reap_dead start 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 shutdown_connections 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 >> 172.21.15.64:0/2880746652 conn(0x7f6ae8071960 msgr2=0x7f6ae80b4b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 shutdown_connections 2024-03-04T07:37:28.239 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.237+0000 7f6aef89d640 1 -- 172.21.15.64:0/2880746652 wait complete. 2024-03-04T07:37:28.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1 2024-03-04T07:37:28.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.352+0000 7f762c33e640 1 Processor -- start 2024-03-04T07:37:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.352+0000 7f762c33e640 1 -- start start 2024-03-04T07:37:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762c33e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7624158520 0x7f762406d820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762c33e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f76240aee10 0x7f76240b3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762c33e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7624071240 con 0x7f762406dd60 2024-03-04T07:37:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762c33e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f762406e1c0 con 0x7f76240aee10 2024-03-04T07:37:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762c33e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7624071b70 con 0x7f7624158520 2024-03-04T07:37:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762a8db640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f76240aee10 0x7f76240b3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762a8db640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f76240aee10 0x7f76240b3640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38336/0 (socket says 172.21.15.64:38336) 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762a8db640 1 -- 172.21.15.64:0/3263734884 learned_addr learned my addr 172.21.15.64:0/3263734884 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.353+0000 7f762a0da640 1 --2- 172.21.15.64:0/3263734884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7624158520 0x7f762406d820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.354+0000 7f762a8db640 1 -- 172.21.15.64:0/3263734884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7624158520 msgr2=0x7f762406d820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.354+0000 7f762a8db640 1 --2- 172.21.15.64:0/3263734884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7624158520 0x7f762406d820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.354+0000 7f762a8db640 1 -- 172.21.15.64:0/3263734884 >> v1:172.21.15.66:6789/0 conn(0x7f762406dd60 legacy=0x7f76240ae700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.354+0000 7f762a8db640 1 -- 172.21.15.64:0/3263734884 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7624071520 con 0x7f76240aee10 2024-03-04T07:37:28.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.354+0000 7f762a0da640 1 --2- 172.21.15.64:0/3263734884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7624158520 0x7f762406d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.354+0000 7f762a8db640 1 --2- 172.21.15.64:0/3263734884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f76240b3640 secure :-1 s=READY pgs=1882 cs=0 l=1 rev1=1 crypto rx=0x7f7620009e50 tx=0x7f762002bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d949d3139673bcf server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f76290d8640 1 -- 172.21.15.64:0/3263734884 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f762002cb00 con 0x7f76240aee10 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f76290d8640 1 -- 172.21.15.64:0/3263734884 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f762002cca0 con 0x7f76240aee10 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f76290d8640 1 -- 172.21.15.64:0/3263734884 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7620034610 con 0x7f76240aee10 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 -- 172.21.15.64:0/3263734884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 msgr2=0x7f76240b3640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 --2- 172.21.15.64:0/3263734884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f76240b3640 secure :-1 s=READY pgs=1882 cs=0 l=1 rev1=1 crypto rx=0x7f7620009e50 tx=0x7f762002bf60 comp rx=0 tx=0).stop 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 -- 172.21.15.64:0/3263734884 shutdown_connections 2024-03-04T07:37:28.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 --2- 172.21.15.64:0/3263734884 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f76240b3640 unknown :-1 s=CLOSED pgs=1882 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 --2- 172.21.15.64:0/3263734884 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7624158520 0x7f762406d820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 -- 172.21.15.64:0/3263734884 >> 172.21.15.64:0/3263734884 conn(0x7f7624066d00 msgr2=0x7f76240670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 -- 172.21.15.64:0/3263734884 shutdown_connections 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.355+0000 7f762c33e640 1 -- 172.21.15.64:0/3263734884 wait complete. 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.356+0000 7f762c33e640 1 Processor -- start 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.356+0000 7f762c33e640 1 -- start start 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762c33e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f762406dd60 0x7f7624149350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762c33e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f7624157500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762c33e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7624158520 0x7f76240ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762c33e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7624158900 con 0x7f76240aee10 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762c33e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7624072000 con 0x7f7624158520 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762c33e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7624158a80 con 0x7f762406dd60 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f76298d9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f7624157500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f76298d9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f7624157500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38344/0 (socket says 172.21.15.64:38344) 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f76298d9640 1 -- 172.21.15.64:0/139663558 learned_addr learned my addr 172.21.15.64:0/139663558 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762a8db640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7624158520 0x7f76240ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.361 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.357+0000 7f762a0da640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f762406dd60 0x7f7624149350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f76298d9640 1 -- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f762406dd60 msgr2=0x7f7624149350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f76298d9640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f762406dd60 0x7f7624149350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f76298d9640 1 -- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7624158520 msgr2=0x7f76240ad8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f76298d9640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7624158520 0x7f76240ad8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f76298d9640 1 -- 172.21.15.64:0/139663558 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7624149890 con 0x7f76240aee10 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f76298d9640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f7624157500 secure :-1 s=READY pgs=1883 cs=0 l=1 rev1=1 crypto rx=0x7f761400c430 tx=0x7f761400c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f761affd640 1 -- 172.21.15.64:0/139663558 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76140052d0 con 0x7f76240aee10 2024-03-04T07:37:28.362 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f761affd640 1 -- 172.21.15.64:0/139663558 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7614005490 con 0x7f76240aee10 2024-03-04T07:37:28.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7624155fd0 con 0x7f76240aee10 2024-03-04T07:37:28.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f761affd640 1 -- 172.21.15.64:0/139663558 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7614007260 con 0x7f76240aee10 2024-03-04T07:37:28.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.358+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7624155a10 con 0x7f76240aee10 2024-03-04T07:37:28.363 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.359+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7624159dc0 con 0x7f76240aee10 2024-03-04T07:37:28.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.359+0000 7f761affd640 1 -- 172.21.15.64:0/139663558 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7614007400 con 0x7f76240aee10 2024-03-04T07:37:28.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.360+0000 7f761affd640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f760806b4d0 0x7f760806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.360+0000 7f762a0da640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f760806b4d0 0x7f760806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.360+0000 7f761affd640 1 -- 172.21.15.64:0/139663558 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f7614089310 con 0x7f76240aee10 2024-03-04T07:37:28.364 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.360+0000 7f762a0da640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f760806b4d0 0x7f760806d990 secure :-1 s=READY pgs=858 cs=0 l=1 rev1=1 crypto rx=0x7f761c006e80 tx=0x7f761c008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.365 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.364+0000 7f761affd640 1 -- 172.21.15.64:0/139663558 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f7624159dc0 con 0x7f76240aee10 2024-03-04T07:37:28.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.547+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "1", "target": ["mon-mgr", ""]}) v1 -- 0x7f76240b31f0 con 0x7f760806b4d0 2024-03-04T07:37:28.547 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.547+0000 7f761affd640 1 -- 172.21.15.64:0/139663558 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 1 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7f76240b31f0 con 0x7f760806b4d0 2024-03-04T07:37:28.547 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 1 to deep-scrub 2024-03-04T07:37:28.549 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f760806b4d0 msgr2=0x7f760806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f760806b4d0 0x7f760806d990 secure :-1 s=READY pgs=858 cs=0 l=1 rev1=1 crypto rx=0x7f761c006e80 tx=0x7f761c008040 comp rx=0 tx=0).stop 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 msgr2=0x7f7624157500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f7624157500 secure :-1 s=READY pgs=1883 cs=0 l=1 rev1=1 crypto rx=0x7f761400c430 tx=0x7f761400c8f0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 shutdown_connections 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f760806b4d0 0x7f760806d990 unknown :-1 s=CLOSED pgs=858 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7624158520 0x7f76240ad8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f76240aee10 0x7f7624157500 unknown :-1 s=CLOSED pgs=1883 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.550 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 --2- 172.21.15.64:0/139663558 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f762406dd60 0x7f7624149350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 >> 172.21.15.64:0/139663558 conn(0x7f7624066d00 msgr2=0x7f7624158c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 shutdown_connections 2024-03-04T07:37:28.551 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.549+0000 7f762c33e640 1 -- 172.21.15.64:0/139663558 wait complete. 2024-03-04T07:37:28.559 INFO:tasks.ceph:Scrubbing osd.2 2024-03-04T07:37:28.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.666+0000 7f7a6a8c9640 1 Processor -- start 2024-03-04T07:37:28.666 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.666+0000 7f7a6a8c9640 1 -- start start 2024-03-04T07:37:28.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a6a8c9640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f7a6415f500 0x7f7a640ce300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a6a8c9640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7a640cd310 con 0x7f7a640c5e30 2024-03-04T07:37:28.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a6a8c9640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f7a640cd5b0 con 0x7f7a640cf740 2024-03-04T07:37:28.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a6a8c9640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f7a6415f8e0 con 0x7f7a6415f500 2024-03-04T07:37:28.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a68e66640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f7a640c5e30 0x7f7a640c6280 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50132/0 (socket says 172.21.15.64:50132) 2024-03-04T07:37:28.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a68e66640 1 -- 172.21.15.64:0/1662055544 learned_addr learned my addr 172.21.15.64:0/1662055544 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a63fff640 1 --2- 172.21.15.64:0/1662055544 >> v2:172.21.15.66:3300/0 conn(0x7f7a6415f500 0x7f7a640ce300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 238118440 0 0) 0x7f7a640cd310 con 0x7f7a640c5e30 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f7a4c003520 con 0x7f7a640c5e30 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 160197240 0 0) 0x7f7a4c003520 con 0x7f7a640c5e30 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 >> v2:172.21.15.66:3300/0 conn(0x7f7a6415f500 msgr2=0x7f7a640ce300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 --2- 172.21.15.64:0/1662055544 >> v2:172.21.15.66:3300/0 conn(0x7f7a6415f500 0x7f7a640ce300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 >> v1:172.21.15.66:6789/0 conn(0x7f7a640cf740 legacy=0x7f7a640ce840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a640cd890 con 0x7f7a640c5e30 2024-03-04T07:37:28.671 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.667+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f7a5c001a10 con 0x7f7a640c5e30 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 >> v1:172.21.15.64:6789/0 conn(0x7f7a640c5e30 legacy=0x7f7a640c6280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a62ffd640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0039b0 0x7f7a4c003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a62ffd640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a4c0043b0 0x7f7a4c004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a62ffd640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a4c005120 0x7f7a4c00b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a6415f8e0 con 0x7f7a4c0043b0 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a4c00c060 con 0x7f7a4c005120 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7a4c00c260 con 0x7f7a4c0039b0 2024-03-04T07:37:28.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a637fe640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a4c0043b0 0x7f7a4c004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a68e66640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a4c005120 0x7f7a4c00b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a63fff640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0039b0 0x7f7a4c003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a637fe640 1 -- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0039b0 msgr2=0x7f7a4c003d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a637fe640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0039b0 0x7f7a4c003d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a637fe640 1 -- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a4c005120 msgr2=0x7f7a4c00b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a637fe640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a4c005120 0x7f7a4c00b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a637fe640 1 -- 172.21.15.64:0/1662055544 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a4c003520 con 0x7f7a4c0043b0 2024-03-04T07:37:28.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a68e66640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a4c005120 0x7f7a4c00b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.668+0000 7f7a68e66640 1 -- 172.21.15.64:0/1662055544 reap_dead start 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.669+0000 7f7a637fe640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a4c0043b0 0x7f7a4c004880 secure :-1 s=READY pgs=1885 cs=0 l=1 rev1=1 crypto rx=0x7f7a54001910 tx=0x7f7a4c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4f332832925ccdef server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.669+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a540014e0 con 0x7f7a4c0043b0 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.669+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a54037070 con 0x7f7a4c0043b0 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.669+0000 7f7a62ffd640 1 -- 172.21.15.64:0/1662055544 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a54032860 con 0x7f7a4c0043b0 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.670+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/1662055544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a4c0043b0 msgr2=0x7f7a4c004880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.670+0000 7f7a6a8c9640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a4c0043b0 0x7f7a4c004880 secure :-1 s=READY pgs=1885 cs=0 l=1 rev1=1 crypto rx=0x7f7a54001910 tx=0x7f7a4c005830 comp rx=0 tx=0).stop 2024-03-04T07:37:28.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.670+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/1662055544 shutdown_connections 2024-03-04T07:37:28.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.670+0000 7f7a6a8c9640 1 --2- 172.21.15.64:0/1662055544 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a4c0043b0 0x7f7a4c004880 unknown :-1 s=CLOSED pgs=1885 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.670+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/1662055544 >> 172.21.15.64:0/1662055544 conn(0x7f7a640b3700 msgr2=0x7f7a640b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.670+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/1662055544 shutdown_connections 2024-03-04T07:37:28.675 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.670+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/1662055544 wait complete. 2024-03-04T07:37:28.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.671+0000 7f7a6a8c9640 1 Processor -- start 2024-03-04T07:37:28.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.672+0000 7f7a6a8c9640 1 -- start start 2024-03-04T07:37:28.678 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a6a8c9640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0043b0 0x7f7a64148dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a6a8c9640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a640b65f0 0x7f7a6415a6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a6a8c9640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a640b6a70 0x7f7a6415ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a6a8c9640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a640c59a0 con 0x7f7a640b65f0 2024-03-04T07:37:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a6a8c9640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a640c56c0 con 0x7f7a640b6a70 2024-03-04T07:37:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a6a8c9640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f7a640ce300 con 0x7f7a4c0043b0 2024-03-04T07:37:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a640b65f0 0x7f7a6415a6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a640b65f0 0x7f7a6415a6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38364/0 (socket says 172.21.15.64:38364) 2024-03-04T07:37:28.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 -- 172.21.15.64:0/2736126114 learned_addr learned my addr 172.21.15.64:0/2736126114 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:28.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a68e66640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a640b6a70 0x7f7a6415ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a63fff640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0043b0 0x7f7a64148dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 -- 172.21.15.64:0/2736126114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0043b0 msgr2=0x7f7a64148dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f7a4c0043b0 0x7f7a64148dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 -- 172.21.15.64:0/2736126114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a640b6a70 msgr2=0x7f7a6415ac10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a640b6a70 0x7f7a6415ac10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 -- 172.21.15.64:0/2736126114 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a64153930 con 0x7f7a640b65f0 2024-03-04T07:37:28.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a68e66640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f7a640b6a70 0x7f7a6415ac10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.673+0000 7f7a637fe640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a640b65f0 0x7f7a6415a6d0 secure :-1 s=READY pgs=1886 cs=0 l=1 rev1=1 crypto rx=0x7f7a54034040 tx=0x7f7a540012b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.674+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a54007a70 con 0x7f7a640b65f0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.674+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7a54047480 con 0x7f7a640b65f0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.674+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a64153c10 con 0x7f7a640b65f0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.674+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a54047870 con 0x7f7a640b65f0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.674+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a641541d0 con 0x7f7a640b65f0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.674+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f7a6409fcf0 con 0x7f7a640b65f0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.675+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f7a54031070 con 0x7f7a640b65f0 2024-03-04T07:37:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a60ff9640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5006b500 0x7f7a5006d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a63fff640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5006b500 0x7f7a5006d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f7a540ac620 con 0x7f7a640b65f0 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a60ff9640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f7a50071460 0x7f7a50073880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f7a5403de50 con 0x7f7a50071460 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7a540acab0 con 0x7f7a640b65f0 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a68e66640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f7a50071460 0x7f7a50073880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a63fff640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5006b500 0x7f7a5006d9c0 secure :-1 s=READY pgs=859 cs=0 l=1 rev1=1 crypto rx=0x7f7a58009a90 tx=0x7f7a58008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.676+0000 7f7a68e66640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f7a50071460 0x7f7a50073880 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.678+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== osd.2 v2:172.21.15.64:6824/2610379167 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f7a5403de50 con 0x7f7a50071460 2024-03-04T07:37:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 --> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7f7a6404f410 con 0x7f7a50071460 2024-03-04T07:37:28.691 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a60ff9640 1 -- 172.21.15.64:0/2736126114 <== osd.2 v2:172.21.15.64:6824/2610379167 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7f7a6404f410 con 0x7f7a50071460 2024-03-04T07:37:28.691 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:28.691 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:28.691 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:28.692 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f7a50071460 msgr2=0x7f7a50073880 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6824/2610379167,v1:172.21.15.64:6825/2610379167] conn(0x7f7a50071460 0x7f7a50073880 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5006b500 msgr2=0x7f7a5006d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f7a5006b500 0x7f7a5006d9c0 secure :-1 s=READY pgs=859 cs=0 l=1 rev1=1 crypto rx=0x7f7a58009a90 tx=0x7f7a58008040 comp rx=0 tx=0).stop 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a640b65f0 msgr2=0x7f7a6415a6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 --2- 172.21.15.64:0/2736126114 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f7a640b65f0 0x7f7a6415a6d0 secure :-1 s=READY pgs=1886 cs=0 l=1 rev1=1 crypto rx=0x7f7a54034040 tx=0x7f7a540012b0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a68e66640 1 -- 172.21.15.64:0/2736126114 reap_dead start 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 shutdown_connections 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 >> 172.21.15.64:0/2736126114 conn(0x7f7a640b3700 msgr2=0x7f7a640c7d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 shutdown_connections 2024-03-04T07:37:28.694 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.691+0000 7f7a6a8c9640 1 -- 172.21.15.64:0/2736126114 wait complete. 2024-03-04T07:37:28.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2024-03-04T07:37:28.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.804+0000 7f348875e640 1 Processor -- start 2024-03-04T07:37:28.804 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.804+0000 7f348875e640 1 -- start start 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f348875e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 0x7f34800cfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f348875e640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f34800b2320 con 0x7f34800d06c0 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f348875e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f34800b0dd0 con 0x7f34800d02e0 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f348875e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f34800b0b80 con 0x7f34800b4260 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f3486cfb640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f34800d06c0 0x7f34800b1c10 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50144/0 (socket says 172.21.15.64:50144) 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f3486cfb640 1 -- 172.21.15.64:0/4218495376 learned_addr learned my addr 172.21.15.64:0/4218495376 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 671714054 0 0) 0x7f34800b2320 con 0x7f34800d06c0 2024-03-04T07:37:28.807 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f3485cf9640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 0x7f34800cfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f346c003520 con 0x7f34800d06c0 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 2084241430 0 0) 0x7f346c003520 con 0x7f34800d06c0 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 msgr2=0x7f34800cfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 0x7f34800cfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 >> v1:172.21.15.66:6789/0 conn(0x7f34800d02e0 legacy=0x7f34800b1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34800b10b0 con 0x7f34800d06c0 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.805+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7f34780019f0 con 0x7f34800d06c0 2024-03-04T07:37:28.808 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 >> v1:172.21.15.64:6789/0 conn(0x7f34800d06c0 legacy=0x7f34800b1c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34854f8640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f346c0039e0 0x7f346c003dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34854f8640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f346c004860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34854f8640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 0x7f346c0050c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34800b0b80 con 0x7f346c004390 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f346c00b9c0 con 0x7f346c0039e0 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f346c00bbc0 con 0x7f34800b4260 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f3485cf9640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f346c004860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f34864fa640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f346c0039e0 0x7f346c003dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.809 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f3486cfb640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 0x7f346c0050c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f3485cf9640 1 -- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 msgr2=0x7f346c0050c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f3485cf9640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34800b4260 0x7f346c0050c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f3485cf9640 1 -- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f346c0039e0 msgr2=0x7f346c003dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f3485cf9640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f346c0039e0 0x7f346c003dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.806+0000 7f3485cf9640 1 -- 172.21.15.64:0/4218495376 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f346c003520 con 0x7f346c004390 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.807+0000 7f3485cf9640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f346c004860 secure :-1 s=READY pgs=1888 cs=0 l=1 rev1=1 crypto rx=0x7f3470009a90 tx=0x7f346c005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eeb22ede2d80ed26 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.807+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3470039070 con 0x7f346c004390 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.807+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f347002c680 con 0x7f346c004390 2024-03-04T07:37:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.807+0000 7f34854f8640 1 -- 172.21.15.64:0/4218495376 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3470009690 con 0x7f346c004390 2024-03-04T07:37:28.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.807+0000 7f348875e640 1 -- 172.21.15.64:0/4218495376 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 msgr2=0x7f346c004860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.807+0000 7f348875e640 1 --2- 172.21.15.64:0/4218495376 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f346c004860 secure :-1 s=READY pgs=1888 cs=0 l=1 rev1=1 crypto rx=0x7f3470009a90 tx=0x7f346c005830 comp rx=0 tx=0).stop 2024-03-04T07:37:28.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.807+0000 7f3486cfb640 1 -- 172.21.15.64:0/4218495376 reap_dead start 2024-03-04T07:37:28.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.808+0000 7f348875e640 1 -- 172.21.15.64:0/4218495376 shutdown_connections 2024-03-04T07:37:28.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.808+0000 7f348875e640 1 -- 172.21.15.64:0/4218495376 >> 172.21.15.64:0/4218495376 conn(0x7f34800b36d0 msgr2=0x7f34800b3ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:28.814 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.808+0000 7f348875e640 1 -- 172.21.15.64:0/4218495376 shutdown_connections 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.808+0000 7f348875e640 1 -- 172.21.15.64:0/4218495376 wait complete. 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 Processor -- start 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 -- start start 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f34801444a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34801553e0 0x7f3480147140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3480147680 0x7f34800cdf40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34800b0360 con 0x7f346c004390 2024-03-04T07:37:28.815 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34800aeec0 con 0x7f3480147680 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.809+0000 7f348875e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f34800b04e0 con 0x7f34801553e0 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f34801444a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f34801444a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38384/0 (socket says 172.21.15.64:38384) 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 -- 172.21.15.64:0/3733006379 learned_addr learned my addr 172.21.15.64:0/3733006379 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f3486cfb640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3480147680 0x7f34800cdf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f3485cf9640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34801553e0 0x7f3480147140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 -- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34801553e0 msgr2=0x7f3480147140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34801553e0 0x7f3480147140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 -- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3480147680 msgr2=0x7f34800cdf40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3480147680 0x7f34800cdf40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 -- 172.21.15.64:0/3733006379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34801449e0 con 0x7f346c004390 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f3486cfb640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3480147680 0x7f34800cdf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f34864fa640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f34801444a0 secure :-1 s=READY pgs=1889 cs=0 l=1 rev1=1 crypto rx=0x7f347c00f8a0 tx=0x7f347c00fd60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f3476ffd640 1 -- 172.21.15.64:0/3733006379 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f347c01a070 con 0x7f346c004390 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f3476ffd640 1 -- 172.21.15.64:0/3733006379 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f347c005320 con 0x7f346c004390 2024-03-04T07:37:28.817 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f34801f55c0 con 0x7f346c004390 2024-03-04T07:37:28.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f3476ffd640 1 -- 172.21.15.64:0/3733006379 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f347c0072b0 con 0x7f346c004390 2024-03-04T07:37:28.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.810+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f34801f5b80 con 0x7f346c004390 2024-03-04T07:37:28.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.811+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f34800725f0 con 0x7f346c004390 2024-03-04T07:37:28.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.812+0000 7f3476ffd640 1 -- 172.21.15.64:0/3733006379 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f347c01d070 con 0x7f346c004390 2024-03-04T07:37:28.818 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.812+0000 7f3476ffd640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f346406b4d0 0x7f346406d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:28.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.812+0000 7f3476ffd640 1 -- 172.21.15.64:0/3733006379 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f347c0897d0 con 0x7f346c004390 2024-03-04T07:37:28.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.812+0000 7f3485cf9640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f346406b4d0 0x7f346406d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:28.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.813+0000 7f3485cf9640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f346406b4d0 0x7f346406d990 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7f3470009a90 tx=0x7f3470030000 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:28.819 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:28.817+0000 7f3476ffd640 1 -- 172.21.15.64:0/3733006379 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f34800725f0 con 0x7f346c004390 2024-03-04T07:37:29.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.000+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "2", "target": ["mon-mgr", ""]}) v1 -- 0x7f34800ccbc0 con 0x7f346406b4d0 2024-03-04T07:37:29.000 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.000+0000 7f3476ffd640 1 -- 172.21.15.64:0/3733006379 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 2 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7f34800ccbc0 con 0x7f346406b4d0 2024-03-04T07:37:29.001 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 2 to deep-scrub 2024-03-04T07:37:29.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.002+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f346406b4d0 msgr2=0x7f346406d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f346406b4d0 0x7f346406d990 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7f3470009a90 tx=0x7f3470030000 comp rx=0 tx=0).stop 2024-03-04T07:37:29.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 msgr2=0x7f34801444a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.003 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f34801444a0 secure :-1 s=READY pgs=1889 cs=0 l=1 rev1=1 crypto rx=0x7f347c00f8a0 tx=0x7f347c00fd60 comp rx=0 tx=0).stop 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 shutdown_connections 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f346406b4d0 0x7f346406d990 unknown :-1 s=CLOSED pgs=860 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3480147680 0x7f34800cdf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f34801553e0 0x7f3480147140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 --2- 172.21.15.64:0/3733006379 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f346c004390 0x7f34801444a0 unknown :-1 s=CLOSED pgs=1889 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 >> 172.21.15.64:0/3733006379 conn(0x7f34800b36d0 msgr2=0x7f34800d1ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 shutdown_connections 2024-03-04T07:37:29.004 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.003+0000 7f348875e640 1 -- 172.21.15.64:0/3733006379 wait complete. 2024-03-04T07:37:29.013 INFO:tasks.ceph:Scrubbing osd.3 2024-03-04T07:37:29.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:29.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.119+0000 7f263082a640 1 Processor -- start 2024-03-04T07:37:29.119 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.119+0000 7f263082a640 1 -- start start 2024-03-04T07:37:29.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f263082a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f262c144640 0x7f262c06e400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f263082a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c0ce900 0x7f262c0ced70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f263082a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f262c15bc40 con 0x7f262c06e940 2024-03-04T07:37:29.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f263082a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f262c0cf2b0 con 0x7f262c144640 2024-03-04T07:37:29.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f263082a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f262c06df70 con 0x7f262c0ce900 2024-03-04T07:37:29.121 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f262a59c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f262c144640 0x7f262c06e400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f2629d9b640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f262c06e940 0x7f262c15b130 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50150/0 (socket says 172.21.15.64:50150) 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f2629d9b640 1 -- 172.21.15.64:0/3761594798 learned_addr learned my addr 172.21.15.64:0/3761594798 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f262ad9d640 1 --2- 172.21.15.64:0/3761594798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c0ce900 0x7f262c0ced70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f262a59c640 1 -- 172.21.15.64:0/3761594798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c0ce900 msgr2=0x7f262c0ced70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f262a59c640 1 --2- 172.21.15.64:0/3761594798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c0ce900 0x7f262c0ced70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f262a59c640 1 -- 172.21.15.64:0/3761594798 >> v1:172.21.15.64:6789/0 conn(0x7f262c06e940 legacy=0x7f262c15b130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f262a59c640 1 -- 172.21.15.64:0/3761594798 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f262c15bf20 con 0x7f262c144640 2024-03-04T07:37:29.122 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.120+0000 7f262959a640 1 -- 172.21.15.64:0/3761594798 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3865089385 0 0) 0x7f262c15bc40 con 0x7f262c06e940 2024-03-04T07:37:29.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.121+0000 7f262a59c640 1 --2- 172.21.15.64:0/3761594798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c144640 0x7f262c06e400 secure :-1 s=READY pgs=1891 cs=0 l=1 rev1=1 crypto rx=0x7f261c0097a0 tx=0x7f261c02b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4ce213a2696729ef server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.121+0000 7f262959a640 1 -- 172.21.15.64:0/3761594798 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f261c035020 con 0x7f262c144640 2024-03-04T07:37:29.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.121+0000 7f262959a640 1 -- 172.21.15.64:0/3761594798 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f261c031830 con 0x7f262c144640 2024-03-04T07:37:29.123 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.121+0000 7f262959a640 1 -- 172.21.15.64:0/3761594798 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f261c031c00 con 0x7f262c144640 2024-03-04T07:37:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.121+0000 7f263082a640 1 -- 172.21.15.64:0/3761594798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c144640 msgr2=0x7f262c06e400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.121+0000 7f263082a640 1 --2- 172.21.15.64:0/3761594798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c144640 0x7f262c06e400 secure :-1 s=READY pgs=1891 cs=0 l=1 rev1=1 crypto rx=0x7f261c0097a0 tx=0x7f261c02b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.122+0000 7f263082a640 1 -- 172.21.15.64:0/3761594798 shutdown_connections 2024-03-04T07:37:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.122+0000 7f263082a640 1 --2- 172.21.15.64:0/3761594798 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c0ce900 0x7f262c0ced70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.122+0000 7f263082a640 1 --2- 172.21.15.64:0/3761594798 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c144640 0x7f262c06e400 unknown :-1 s=CLOSED pgs=1891 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.122+0000 7f263082a640 1 -- 172.21.15.64:0/3761594798 >> 172.21.15.64:0/3761594798 conn(0x7f262c066d00 msgr2=0x7f262c067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.122+0000 7f263082a640 1 -- 172.21.15.64:0/3761594798 shutdown_connections 2024-03-04T07:37:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.122+0000 7f263082a640 1 -- 172.21.15.64:0/3761594798 wait complete. 2024-03-04T07:37:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.123+0000 7f263082a640 1 Processor -- start 2024-03-04T07:37:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.123+0000 7f263082a640 1 -- start start 2024-03-04T07:37:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c06e940 0x7f262c13e5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c0ce900 0x7f262c0ad8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f262c144640 0x7f262c13fc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f262c070c20 con 0x7f262c0ce900 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f262c070aa0 con 0x7f262c144640 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f262c15b840 con 0x7f262c06e940 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c0ce900 0x7f262c0ad8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c0ce900 0x7f262c0ad8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38402/0 (socket says 172.21.15.64:38402) 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 -- 172.21.15.64:0/2095667235 learned_addr learned my addr 172.21.15.64:0/2095667235 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f262a59c640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c06e940 0x7f262c13e5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f262ad9d640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f262c144640 0x7f262c13fc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 -- 172.21.15.64:0/2095667235 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c06e940 msgr2=0x7f262c13e5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c06e940 0x7f262c13e5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 -- 172.21.15.64:0/2095667235 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f262c144640 msgr2=0x7f262c13fc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f262c144640 0x7f262c13fc10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 -- 172.21.15.64:0/2095667235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f262c0ae370 con 0x7f262c0ce900 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f262a59c640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f262c06e940 0x7f262c13e5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f2629d9b640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c0ce900 0x7f262c0ad8b0 secure :-1 s=READY pgs=1892 cs=0 l=1 rev1=1 crypto rx=0x7f262000c4d0 tx=0x7f262000c990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2620018070 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2620007500 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f262c0ae650 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2620004a20 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.124+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f262c0aec10 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.125+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f262c0733c0 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.126+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f262001b070 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.126+0000 7f261affd640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f260c06b4d0 0x7f260c06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.126+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f26200892a0 con 0x7f262c0ce900 2024-03-04T07:37:29.131 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.126+0000 7f262a59c640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f260c06b4d0 0x7f260c06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.126+0000 7f261affd640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f260c071430 0x7f260c073850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.126+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f2620017ec0 con 0x7f260c071430 2024-03-04T07:37:29.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.126+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2620090050 con 0x7f262c0ce900 2024-03-04T07:37:29.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.127+0000 7f262ad9d640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f260c071430 0x7f260c073850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.127+0000 7f262a59c640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f260c06b4d0 0x7f260c06d990 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7f261c036000 tx=0x7f261c008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.132 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.127+0000 7f262ad9d640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f260c071430 0x7f260c073850 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.133 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.127+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== osd.3 v2:172.21.15.64:6816/1402142346 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f2620017ec0 con 0x7f260c071430 2024-03-04T07:37:29.140 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 --> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7f262c04f410 con 0x7f260c071430 2024-03-04T07:37:29.141 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f261affd640 1 -- 172.21.15.64:0/2095667235 <== osd.3 v2:172.21.15.64:6816/1402142346 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7f262c04f410 con 0x7f260c071430 2024-03-04T07:37:29.141 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:29.141 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:29.141 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:29.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f260c071430 msgr2=0x7f260c073850 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6816/1402142346,v1:172.21.15.64:6817/1402142346] conn(0x7f260c071430 0x7f260c073850 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.142 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f260c06b4d0 msgr2=0x7f260c06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f260c06b4d0 0x7f260c06d990 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7f261c036000 tx=0x7f261c008030 comp rx=0 tx=0).stop 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c0ce900 msgr2=0x7f262c0ad8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 --2- 172.21.15.64:0/2095667235 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f262c0ce900 0x7f262c0ad8b0 secure :-1 s=READY pgs=1892 cs=0 l=1 rev1=1 crypto rx=0x7f262000c4d0 tx=0x7f262000c990 comp rx=0 tx=0).stop 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f262ad9d640 1 -- 172.21.15.64:0/2095667235 reap_dead start 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 shutdown_connections 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 >> 172.21.15.64:0/2095667235 conn(0x7f262c066d00 msgr2=0x7f262c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 shutdown_connections 2024-03-04T07:37:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.141+0000 7f263082a640 1 -- 172.21.15.64:0/2095667235 wait complete. 2024-03-04T07:37:29.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 3 2024-03-04T07:37:29.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f27265ac640 1 Processor -- start 2024-03-04T07:37:29.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f27265ac640 1 -- start start 2024-03-04T07:37:29.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f27265ac640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f272015bde0 0x7f27200b4cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.255 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f27265ac640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f27200b2460 0x7f272015c1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f27265ac640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f27200b3c70 con 0x7f27201451a0 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f27265ac640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f27200b4840 con 0x7f27200b2460 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f27265ac640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f27200afad0 con 0x7f272015bde0 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f2724b49640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f27200b2460 0x7f272015c1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.254+0000 7f271f7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f27201451a0 0x7f27200b1d50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50158/0 (socket says 172.21.15.64:50158) 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f271f7fe640 1 -- 172.21.15.64:0/1051242060 learned_addr learned my addr 172.21.15.64:0/1051242060 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f2724b49640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f27200b2460 0x7f272015c1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38416/0 (socket says 172.21.15.64:38416) 2024-03-04T07:37:29.256 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f271ffff640 1 --2- 172.21.15.64:0/1051242060 >> v2:172.21.15.66:3300/0 conn(0x7f272015bde0 0x7f27200b4cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f2724b49640 1 -- 172.21.15.64:0/1051242060 >> v2:172.21.15.66:3300/0 conn(0x7f272015bde0 msgr2=0x7f27200b4cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:29.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f2724b49640 1 --2- 172.21.15.64:0/1051242060 >> v2:172.21.15.66:3300/0 conn(0x7f272015bde0 0x7f27200b4cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f2724b49640 1 -- 172.21.15.64:0/1051242060 >> v1:172.21.15.64:6789/0 conn(0x7f27201451a0 legacy=0x7f27200b1d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f2724b49640 1 -- 172.21.15.64:0/1051242060 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27200b3f10 con 0x7f27200b2460 2024-03-04T07:37:29.257 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.255+0000 7f271effd640 1 -- 172.21.15.64:0/1051242060 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2903495138 0 0) 0x7f27200b3c70 con 0x7f27201451a0 2024-03-04T07:37:29.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f2724b49640 1 --2- 172.21.15.64:0/1051242060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27200b2460 0x7f272015c1c0 secure :-1 s=READY pgs=1894 cs=0 l=1 rev1=1 crypto rx=0x7f2718009d90 tx=0x7f271802beb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b88ef06cfe5c1102 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f271effd640 1 -- 172.21.15.64:0/1051242060 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2718039070 con 0x7f27200b2460 2024-03-04T07:37:29.260 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f271effd640 1 -- 172.21.15.64:0/1051242060 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f271802cc60 con 0x7f27200b2460 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f271effd640 1 -- 172.21.15.64:0/1051242060 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2718034660 con 0x7f27200b2460 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 -- 172.21.15.64:0/1051242060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27200b2460 msgr2=0x7f272015c1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 --2- 172.21.15.64:0/1051242060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27200b2460 0x7f272015c1c0 secure :-1 s=READY pgs=1894 cs=0 l=1 rev1=1 crypto rx=0x7f2718009d90 tx=0x7f271802beb0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 -- 172.21.15.64:0/1051242060 shutdown_connections 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 --2- 172.21.15.64:0/1051242060 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f27200b2460 0x7f272015c1c0 unknown :-1 s=CLOSED pgs=1894 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 --2- 172.21.15.64:0/1051242060 >> v2:172.21.15.66:3300/0 conn(0x7f272015bde0 0x7f27200b4cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 -- 172.21.15.64:0/1051242060 >> 172.21.15.64:0/1051242060 conn(0x7f27200b55e0 msgr2=0x7f27200b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 -- 172.21.15.64:0/1051242060 shutdown_connections 2024-03-04T07:37:29.261 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.256+0000 7f27265ac640 1 -- 172.21.15.64:0/1051242060 wait complete. 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.257+0000 7f27265ac640 1 Processor -- start 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f27265ac640 1 -- start start 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f27265ac640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f27200b2460 0x7f27201e79e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f27265ac640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27201451a0 0x7f27201e5550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f27265ac640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f272015bde0 0x7f27201e5a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f27265ac640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27200b5210 con 0x7f272015bde0 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f27265ac640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27200b44d0 con 0x7f27201451a0 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f27265ac640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f27200b5390 con 0x7f27200b2460 2024-03-04T07:37:29.262 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f2724b49640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f272015bde0 0x7f27201e5a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f2724b49640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f272015bde0 0x7f27201e5a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38426/0 (socket says 172.21.15.64:38426) 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.258+0000 7f2724b49640 1 -- 172.21.15.64:0/1970613727 learned_addr learned my addr 172.21.15.64:0/1970613727 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f271f7fe640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27201451a0 0x7f27201e5550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f271ffff640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f27200b2460 0x7f27201e79e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f2724b49640 1 -- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f27200b2460 msgr2=0x7f27201e79e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f2724b49640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f27200b2460 0x7f27201e79e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f2724b49640 1 -- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27201451a0 msgr2=0x7f27201e5550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f2724b49640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27201451a0 0x7f27201e5550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f2724b49640 1 -- 172.21.15.64:0/1970613727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2720152950 con 0x7f272015bde0 2024-03-04T07:37:29.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f271ffff640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f27200b2460 0x7f27201e79e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:29.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f2724b49640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f272015bde0 0x7f27201e5a90 secure :-1 s=READY pgs=1895 cs=0 l=1 rev1=1 crypto rx=0x7f2718009d90 tx=0x7f2718004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f271cff9640 1 -- 172.21.15.64:0/1970613727 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2718039050 con 0x7f272015bde0 2024-03-04T07:37:29.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f271cff9640 1 -- 172.21.15.64:0/1970613727 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f2718045070 con 0x7f272015bde0 2024-03-04T07:37:29.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f27201f3c40 con 0x7f272015bde0 2024-03-04T07:37:29.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f271cff9640 1 -- 172.21.15.64:0/1970613727 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f271803e830 con 0x7f272015bde0 2024-03-04T07:37:29.264 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.259+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f27201e6980 con 0x7f272015bde0 2024-03-04T07:37:29.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.260+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2720074b30 con 0x7f272015bde0 2024-03-04T07:37:29.265 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.261+0000 7f271cff9640 1 -- 172.21.15.64:0/1970613727 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f271803e9d0 con 0x7f272015bde0 2024-03-04T07:37:29.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.261+0000 7f271cff9640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f270006b4d0 0x7f270006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.261+0000 7f271cff9640 1 -- 172.21.15.64:0/1970613727 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f27180accc0 con 0x7f272015bde0 2024-03-04T07:37:29.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.261+0000 7f271ffff640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f270006b4d0 0x7f270006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.262+0000 7f271ffff640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f270006b4d0 0x7f270006d990 secure :-1 s=READY pgs=862 cs=0 l=1 rev1=1 crypto rx=0x7f2710006fd0 tx=0x7f2710008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.266 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.265+0000 7f271cff9640 1 -- 172.21.15.64:0/1970613727 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2720074b30 con 0x7f272015bde0 2024-03-04T07:37:29.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.445+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "3", "target": ["mon-mgr", ""]}) v1 -- 0x7f27200d09f0 con 0x7f270006b4d0 2024-03-04T07:37:29.446 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.446+0000 7f271cff9640 1 -- 172.21.15.64:0/1970613727 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 3 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7f27200d09f0 con 0x7f270006b4d0 2024-03-04T07:37:29.447 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 3 to deep-scrub 2024-03-04T07:37:29.448 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f270006b4d0 msgr2=0x7f270006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f270006b4d0 0x7f270006d990 secure :-1 s=READY pgs=862 cs=0 l=1 rev1=1 crypto rx=0x7f2710006fd0 tx=0x7f2710008040 comp rx=0 tx=0).stop 2024-03-04T07:37:29.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f272015bde0 msgr2=0x7f27201e5a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f272015bde0 0x7f27201e5a90 secure :-1 s=READY pgs=1895 cs=0 l=1 rev1=1 crypto rx=0x7f2718009d90 tx=0x7f2718004a40 comp rx=0 tx=0).stop 2024-03-04T07:37:29.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 shutdown_connections 2024-03-04T07:37:29.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f270006b4d0 0x7f270006d990 unknown :-1 s=CLOSED pgs=862 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f272015bde0 0x7f27201e5a90 unknown :-1 s=CLOSED pgs=1895 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.449 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f27201451a0 0x7f27201e5550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 --2- 172.21.15.64:0/1970613727 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f27200b2460 0x7f27201e79e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 >> 172.21.15.64:0/1970613727 conn(0x7f27200b55e0 msgr2=0x7f27200afce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 shutdown_connections 2024-03-04T07:37:29.450 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.448+0000 7f27265ac640 1 -- 172.21.15.64:0/1970613727 wait complete. 2024-03-04T07:37:29.459 INFO:tasks.ceph:Scrubbing osd.4 2024-03-04T07:37:29.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.4 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:29.564 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.564+0000 7fcac4b7a640 1 Processor -- start 2024-03-04T07:37:29.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.564+0000 7fcac4b7a640 1 -- start start 2024-03-04T07:37:29.565 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.565+0000 7fcac4b7a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcac0072a30 0x7fcac01624c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.565+0000 7fcac4b7a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac00cf9f0 0x7fcac00d00b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.565+0000 7fcac4b7a640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fcac006d4d0 con 0x7fcac00d05f0 2024-03-04T07:37:29.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.565+0000 7fcac4b7a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcac006dba0 con 0x7fcac0072a30 2024-03-04T07:37:29.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.565+0000 7fcac4b7a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcac00d0a60 con 0x7fcac00cf9f0 2024-03-04T07:37:29.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabe59c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcac0072a30 0x7fcac01624c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabdd9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac00cf9f0 0x7fcac00d00b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.566 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabed9d640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7fcac00d05f0 0x7fcac015d660 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38376/0 (socket says 172.21.15.64:38376) 2024-03-04T07:37:29.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabe59c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcac0072a30 0x7fcac01624c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:48022/0 (socket says 172.21.15.64:48022) 2024-03-04T07:37:29.567 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabed9d640 1 -- 172.21.15.64:0/3412926469 learned_addr learned my addr 172.21.15.64:0/3412926469 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:29.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabe59c640 1 -- 172.21.15.64:0/3412926469 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac00cf9f0 msgr2=0x7fcac00d00b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:29.568 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabe59c640 1 --2- 172.21.15.64:0/3412926469 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac00cf9f0 0x7fcac00d00b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabe59c640 1 -- 172.21.15.64:0/3412926469 >> v1:172.21.15.66:6789/0 conn(0x7fcac00d05f0 legacy=0x7fcac015d660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.566+0000 7fcabe59c640 1 -- 172.21.15.64:0/3412926469 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcac00b4950 con 0x7fcac0072a30 2024-03-04T07:37:29.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.567+0000 7fcabe59c640 1 --2- 172.21.15.64:0/3412926469 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac0072a30 0x7fcac01624c0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7fcab80097a0 tx=0x7fcab802b7a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=595ae73d589cb288 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.567+0000 7fcabd59a640 1 -- 172.21.15.64:0/3412926469 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcab8038070 con 0x7fcac0072a30 2024-03-04T07:37:29.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.567+0000 7fcabd59a640 1 -- 172.21.15.64:0/3412926469 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcab8031700 con 0x7fcac0072a30 2024-03-04T07:37:29.569 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.567+0000 7fcabd59a640 1 -- 172.21.15.64:0/3412926469 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcab8031ad0 con 0x7fcac0072a30 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 -- 172.21.15.64:0/3412926469 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac0072a30 msgr2=0x7fcac01624c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 --2- 172.21.15.64:0/3412926469 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac0072a30 0x7fcac01624c0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7fcab80097a0 tx=0x7fcab802b7a0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 -- 172.21.15.64:0/3412926469 shutdown_connections 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 --2- 172.21.15.64:0/3412926469 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac00cf9f0 0x7fcac00d00b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 --2- 172.21.15.64:0/3412926469 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac0072a30 0x7fcac01624c0 unknown :-1 s=CLOSED pgs=700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 -- 172.21.15.64:0/3412926469 >> 172.21.15.64:0/3412926469 conn(0x7fcac006af20 msgr2=0x7fcac006b300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 -- 172.21.15.64:0/3412926469 shutdown_connections 2024-03-04T07:37:29.570 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.568+0000 7fcac4b7a640 1 -- 172.21.15.64:0/3412926469 wait complete. 2024-03-04T07:37:29.573 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.569+0000 7fcac4b7a640 1 Processor -- start 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.569+0000 7fcac4b7a640 1 -- start start 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcac4b7a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac0072a30 0x7fcac014e940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcac4b7a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcac00cf9f0 0x7fcac00cb500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcac4b7a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac00d05f0 0x7fcac00ada40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcac4b7a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcac0071130 con 0x7fcac00cf9f0 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcac4b7a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcac006ddf0 con 0x7fcac00d05f0 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcac4b7a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcac0072e10 con 0x7fcac0072a30 2024-03-04T07:37:29.574 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcac00cf9f0 0x7fcac00cb500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcac00cf9f0 0x7fcac00cb500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38434/0 (socket says 172.21.15.64:38434) 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 -- 172.21.15.64:0/4085850168 learned_addr learned my addr 172.21.15.64:0/4085850168 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabed9d640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac00d05f0 0x7fcac00ada40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabe59c640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac0072a30 0x7fcac014e940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 -- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac0072a30 msgr2=0x7fcac014e940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac0072a30 0x7fcac014e940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 -- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac00d05f0 msgr2=0x7fcac00ada40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcac00d05f0 0x7fcac00ada40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.575 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 -- 172.21.15.64:0/4085850168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcac00cbc20 con 0x7fcac00cf9f0 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabe59c640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcac0072a30 0x7fcac014e940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcabdd9b640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcac00cf9f0 0x7fcac00cb500 secure :-1 s=READY pgs=1896 cs=0 l=1 rev1=1 crypto rx=0x7fcab400e870 tx=0x7fcab400ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcab4017070 con 0x7fcac00cf9f0 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcab400c940 con 0x7fcac00cf9f0 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcac00cbf00 con 0x7fcac00cf9f0 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.570+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcab400cd30 con 0x7fcac00cf9f0 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.571+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcac01f5380 con 0x7fcac00cf9f0 2024-03-04T07:37:29.576 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.571+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fcac00adf80 con 0x7fcac00cf9f0 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.572+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcab401a070 con 0x7fcac00cf9f0 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.572+0000 7fcaaeffd640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcaa006b4d0 0x7fcaa006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.572+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7fcab408a1c0 con 0x7fcac00cf9f0 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.572+0000 7fcabe59c640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcaa006b4d0 0x7fcaa006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.572+0000 7fcaaeffd640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fcaa0071430 0x7fcaa0073850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.572+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fcaa0073f00 con 0x7fcaa0071430 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.572+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcab4091050 con 0x7fcac00cf9f0 2024-03-04T07:37:29.577 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.573+0000 7fcabed9d640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fcaa0071430 0x7fcaa0073850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.573+0000 7fcabe59c640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcaa006b4d0 0x7fcaa006d990 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7fcab8035000 tx=0x7fcab8008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.573+0000 7fcabed9d640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fcaa0071430 0x7fcaa0073850 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.578 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.574+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== osd.4 v2:172.21.15.66:6824/1927515367 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7fcaa0073f00 con 0x7fcaa0071430 2024-03-04T07:37:29.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.587+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 --> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7fcac00c9850 con 0x7fcaa0071430 2024-03-04T07:37:29.587 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.587+0000 7fcaaeffd640 1 -- 172.21.15.64:0/4085850168 <== osd.4 v2:172.21.15.66:6824/1927515367 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7fcac00c9850 con 0x7fcaa0071430 2024-03-04T07:37:29.587 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:29.588 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:29.588 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fcaa0071430 msgr2=0x7fcaa0073850 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.66:6824/1927515367,v1:172.21.15.66:6825/1927515367] conn(0x7fcaa0071430 0x7fcaa0073850 crc :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcaa006b4d0 msgr2=0x7fcaa006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcaa006b4d0 0x7fcaa006d990 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7fcab8035000 tx=0x7fcab8008030 comp rx=0 tx=0).stop 2024-03-04T07:37:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcac00cf9f0 msgr2=0x7fcac00cb500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 --2- 172.21.15.64:0/4085850168 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcac00cf9f0 0x7fcac00cb500 secure :-1 s=READY pgs=1896 cs=0 l=1 rev1=1 crypto rx=0x7fcab400e870 tx=0x7fcab400ed30 comp rx=0 tx=0).stop 2024-03-04T07:37:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcabed9d640 1 -- 172.21.15.64:0/4085850168 reap_dead start 2024-03-04T07:37:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 shutdown_connections 2024-03-04T07:37:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 >> 172.21.15.64:0/4085850168 conn(0x7fcac006af20 msgr2=0x7fcac00cee90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 shutdown_connections 2024-03-04T07:37:29.590 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.588+0000 7fcac4b7a640 1 -- 172.21.15.64:0/4085850168 wait complete. 2024-03-04T07:37:29.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 4 2024-03-04T07:37:29.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.700+0000 7f818940e640 1 Processor -- start 2024-03-04T07:37:29.700 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.700+0000 7f818940e640 1 -- start start 2024-03-04T07:37:29.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f818940e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81840b1d50 0x7f81840b21b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f818940e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818406d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f818940e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f818415e410 con 0x7f818415e030 2024-03-04T07:37:29.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f818940e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f81840b26f0 con 0x7f81840b1d50 2024-03-04T07:37:29.703 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f818940e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8184162700 con 0x7f818406d210 2024-03-04T07:37:29.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81827fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81840b1d50 0x7f81840b21b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81827fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f81840b1d50 0x7f81840b21b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38436/0 (socket says 172.21.15.64:38436) 2024-03-04T07:37:29.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81827fc640 1 -- 172.21.15.64:0/699796017 learned_addr learned my addr 172.21.15.64:0/699796017 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:29.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81837fe640 1 --2- 172.21.15.64:0/699796017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818406d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81827fc640 1 -- 172.21.15.64:0/699796017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 msgr2=0x7f818406d6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:29.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81827fc640 1 --2- 172.21.15.64:0/699796017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818406d6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.704 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81827fc640 1 -- 172.21.15.64:0/699796017 >> v1:172.21.15.66:6789/0 conn(0x7f818415e030 legacy=0x7f818406ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.701+0000 7f81827fc640 1 -- 172.21.15.64:0/699796017 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f818406f140 con 0x7f81840b1d50 2024-03-04T07:37:29.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.702+0000 7f81827fc640 1 --2- 172.21.15.64:0/699796017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81840b1d50 0x7f81840b21b0 secure :-1 s=READY pgs=1897 cs=0 l=1 rev1=1 crypto rx=0x7f816c009870 tx=0x7f816c02b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7c60497fb97386d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.702+0000 7f8181ffb640 1 -- 172.21.15.64:0/699796017 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f816c034070 con 0x7f81840b1d50 2024-03-04T07:37:29.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.702+0000 7f8181ffb640 1 -- 172.21.15.64:0/699796017 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f816c030840 con 0x7f81840b1d50 2024-03-04T07:37:29.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.702+0000 7f8181ffb640 1 -- 172.21.15.64:0/699796017 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f816c030c10 con 0x7f81840b1d50 2024-03-04T07:37:29.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 -- 172.21.15.64:0/699796017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81840b1d50 msgr2=0x7f81840b21b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.705 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 --2- 172.21.15.64:0/699796017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81840b1d50 0x7f81840b21b0 secure :-1 s=READY pgs=1897 cs=0 l=1 rev1=1 crypto rx=0x7f816c009870 tx=0x7f816c02b980 comp rx=0 tx=0).stop 2024-03-04T07:37:29.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 -- 172.21.15.64:0/699796017 shutdown_connections 2024-03-04T07:37:29.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 --2- 172.21.15.64:0/699796017 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818406d6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 --2- 172.21.15.64:0/699796017 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f81840b1d50 0x7f81840b21b0 unknown :-1 s=CLOSED pgs=1897 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 -- 172.21.15.64:0/699796017 >> 172.21.15.64:0/699796017 conn(0x7f81840b55e0 msgr2=0x7f81840b59f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 -- 172.21.15.64:0/699796017 shutdown_connections 2024-03-04T07:37:29.706 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.703+0000 7f818940e640 1 -- 172.21.15.64:0/699796017 wait complete. 2024-03-04T07:37:29.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.704+0000 7f818940e640 1 Processor -- start 2024-03-04T07:37:29.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.704+0000 7f818940e640 1 -- start start 2024-03-04T07:37:29.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f818940e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818413d350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.709 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f818940e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81840b1d50 0x7f818415a0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f818940e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818415e030 0x7f818409d950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f818940e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81841620b0 con 0x7f818415e030 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f818940e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8184161dd0 con 0x7f81840b1d50 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f818940e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8184162390 con 0x7f818406d210 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818415e030 0x7f818409d950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818415e030 0x7f818409d950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38440/0 (socket says 172.21.15.64:38440) 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 -- 172.21.15.64:0/2925847728 learned_addr learned my addr 172.21.15.64:0/2925847728 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81827fc640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81840b1d50 0x7f818415a0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f8182ffd640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818413d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 -- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 msgr2=0x7f818413d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818413d350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 -- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81840b1d50 msgr2=0x7f818415a0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81840b1d50 0x7f818415a0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 -- 172.21.15.64:0/2925847728 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81841478c0 con 0x7f818415e030 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81827fc640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81840b1d50 0x7f818415a0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:29.711 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.705+0000 7f81837fe640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818415e030 0x7f818409d950 secure :-1 s=READY pgs=1898 cs=0 l=1 rev1=1 crypto rx=0x7f8174007e40 tx=0x7f817400e4b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.706+0000 7f817b7fe640 1 -- 172.21.15.64:0/2925847728 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8174014620 con 0x7f818415e030 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.706+0000 7f817b7fe640 1 -- 172.21.15.64:0/2925847728 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f8174019070 con 0x7f818415e030 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.706+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f818413d890 con 0x7f818415e030 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.706+0000 7f817b7fe640 1 -- 172.21.15.64:0/2925847728 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8174014a20 con 0x7f818415e030 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.706+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f818409cb90 con 0x7f818415e030 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.707+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8184067ff0 con 0x7f818415e030 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.707+0000 7f817b7fe640 1 -- 172.21.15.64:0/2925847728 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f8174014bc0 con 0x7f818415e030 2024-03-04T07:37:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.707+0000 7f817b7fe640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816406b500 0x7f816406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:29.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.707+0000 7f817b7fe640 1 -- 172.21.15.64:0/2925847728 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f817408a810 con 0x7f818415e030 2024-03-04T07:37:29.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.707+0000 7f8182ffd640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816406b500 0x7f816406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:29.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.708+0000 7f8182ffd640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816406b500 0x7f816406d9c0 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f817c001500 tx=0x7f817c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:29.713 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.712+0000 7f817b7fe640 1 -- 172.21.15.64:0/2925847728 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f8174059930 con 0x7f818415e030 2024-03-04T07:37:29.892 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.892+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "4", "target": ["mon-mgr", ""]}) v1 -- 0x7f81840af050 con 0x7f816406b500 2024-03-04T07:37:29.893 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.892+0000 7f817b7fe640 1 -- 172.21.15.64:0/2925847728 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 4 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7f81840af050 con 0x7f816406b500 2024-03-04T07:37:29.893 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 4 to deep-scrub 2024-03-04T07:37:29.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816406b500 msgr2=0x7f816406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816406b500 0x7f816406d9c0 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f817c001500 tx=0x7f817c009040 comp rx=0 tx=0).stop 2024-03-04T07:37:29.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818415e030 msgr2=0x7f818409d950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:29.894 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818415e030 0x7f818409d950 secure :-1 s=READY pgs=1898 cs=0 l=1 rev1=1 crypto rx=0x7f8174007e40 tx=0x7f817400e4b0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 shutdown_connections 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f816406b500 0x7f816406d9c0 unknown :-1 s=CLOSED pgs=864 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f818415e030 0x7f818409d950 unknown :-1 s=CLOSED pgs=1898 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f81840b1d50 0x7f818415a0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 --2- 172.21.15.64:0/2925847728 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f818406d210 0x7f818413d350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 >> 172.21.15.64:0/2925847728 conn(0x7f81840b55e0 msgr2=0x7f818406d6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 shutdown_connections 2024-03-04T07:37:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:29.894+0000 7f818940e640 1 -- 172.21.15.64:0/2925847728 wait complete. 2024-03-04T07:37:29.905 INFO:tasks.ceph:Scrubbing osd.5 2024-03-04T07:37:29.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.5 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:30.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.010+0000 7eff7ad3a640 1 Processor -- start 2024-03-04T07:37:30.010 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.010+0000 7eff7ad3a640 1 -- start start 2024-03-04T07:37:30.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff7ad3a640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7eff740b4830 0x7eff74160280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff7ad3a640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7eff740b29a0 0x7eff740b2db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff7ad3a640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7eff74070980 con 0x7eff74142170 2024-03-04T07:37:30.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff7ad3a640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7eff741458e0 con 0x7eff740b29a0 2024-03-04T07:37:30.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff7ad3a640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7eff741607c0 con 0x7eff740b4830 2024-03-04T07:37:30.012 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff792d7640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7eff740b29a0 0x7eff740b2db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff78ad6640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7eff74142170 0x7eff74142a20 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50168/0 (socket says 172.21.15.64:50168) 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff78ad6640 1 -- 172.21.15.64:0/1664571394 learned_addr learned my addr 172.21.15.64:0/1664571394 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff73fff640 1 --2- 172.21.15.64:0/1664571394 >> v2:172.21.15.66:3300/0 conn(0x7eff740b4830 0x7eff74160280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff792d7640 1 -- 172.21.15.64:0/1664571394 >> v2:172.21.15.66:3300/0 conn(0x7eff740b4830 msgr2=0x7eff74160280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff792d7640 1 --2- 172.21.15.64:0/1664571394 >> v2:172.21.15.66:3300/0 conn(0x7eff740b4830 0x7eff74160280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff792d7640 1 -- 172.21.15.64:0/1664571394 >> v1:172.21.15.64:6789/0 conn(0x7eff74142170 legacy=0x7eff74142a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff737fe640 1 -- 172.21.15.64:0/1664571394 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3628940309 0 0) 0x7eff74070980 con 0x7eff74142170 2024-03-04T07:37:30.013 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.011+0000 7eff792d7640 1 -- 172.21.15.64:0/1664571394 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff740b32f0 con 0x7eff740b29a0 2024-03-04T07:37:30.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.012+0000 7eff792d7640 1 --2- 172.21.15.64:0/1664571394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff740b2db0 secure :-1 s=READY pgs=1900 cs=0 l=1 rev1=1 crypto rx=0x7eff740af240 tx=0x7eff6c02be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e1f5c52a14f3360e server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.012+0000 7eff737fe640 1 -- 172.21.15.64:0/1664571394 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff6c039070 con 0x7eff740b29a0 2024-03-04T07:37:30.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.012+0000 7eff737fe640 1 -- 172.21.15.64:0/1664571394 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff6c02cbf0 con 0x7eff740b29a0 2024-03-04T07:37:30.014 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.012+0000 7eff737fe640 1 -- 172.21.15.64:0/1664571394 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff6c034440 con 0x7eff740b29a0 2024-03-04T07:37:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 -- 172.21.15.64:0/1664571394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 msgr2=0x7eff740b2db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 --2- 172.21.15.64:0/1664571394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff740b2db0 secure :-1 s=READY pgs=1900 cs=0 l=1 rev1=1 crypto rx=0x7eff740af240 tx=0x7eff6c02be40 comp rx=0 tx=0).stop 2024-03-04T07:37:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 -- 172.21.15.64:0/1664571394 shutdown_connections 2024-03-04T07:37:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 --2- 172.21.15.64:0/1664571394 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff740b2db0 unknown :-1 s=CLOSED pgs=1900 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 --2- 172.21.15.64:0/1664571394 >> v2:172.21.15.66:3300/0 conn(0x7eff740b4830 0x7eff74160280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 -- 172.21.15.64:0/1664571394 >> 172.21.15.64:0/1664571394 conn(0x7eff74066d00 msgr2=0x7eff74067110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 -- 172.21.15.64:0/1664571394 shutdown_connections 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.013+0000 7eff7ad3a640 1 -- 172.21.15.64:0/1664571394 wait complete. 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.014+0000 7eff7ad3a640 1 Processor -- start 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.014+0000 7eff7ad3a640 1 -- start start 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff7ad3a640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff7415b620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff7ad3a640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff740b4830 0x7eff7413c4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff7ad3a640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff74142170 0x7eff7413ca20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff7ad3a640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff740711b0 con 0x7eff740b29a0 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff7ad3a640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff74070ed0 con 0x7eff740b4830 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff7ad3a640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7eff740b4df0 con 0x7eff74142170 2024-03-04T07:37:30.018 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff7415b620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff7415b620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38454/0 (socket says 172.21.15.64:38454) 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 -- 172.21.15.64:0/457821255 learned_addr learned my addr 172.21.15.64:0/457821255 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 -- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff74142170 msgr2=0x7eff7413ca20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7eff74142170 0x7eff7413ca20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 -- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff740b4830 msgr2=0x7eff7413c4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7eff740b4830 0x7eff7413c4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 -- 172.21.15.64:0/457821255 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff7415e020 con 0x7eff740b29a0 2024-03-04T07:37:30.019 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff78ad6640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff7415b620 secure :-1 s=READY pgs=1901 cs=0 l=1 rev1=1 crypto rx=0x7eff6800c500 tx=0x7eff6800c9c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff68018070 con 0x7eff740b29a0 2024-03-04T07:37:30.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7eff68007580 con 0x7eff740b29a0 2024-03-04T07:37:30.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff7415e300 con 0x7eff740b29a0 2024-03-04T07:37:30.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.015+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff68004a20 con 0x7eff740b29a0 2024-03-04T07:37:30.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.016+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff7415bb60 con 0x7eff740b29a0 2024-03-04T07:37:30.020 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.016+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7eff7415ddd0 con 0x7eff740b29a0 2024-03-04T07:37:30.021 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.017+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7eff6801b070 con 0x7eff740b29a0 2024-03-04T07:37:30.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.017+0000 7eff717fa640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff5806b4d0 0x7eff5806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7eff68089200 con 0x7eff740b29a0 2024-03-04T07:37:30.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff73fff640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff5806b4d0 0x7eff5806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff717fa640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7eff58071430 0x7eff58073850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7eff68017ec0 con 0x7eff58071430 2024-03-04T07:37:30.022 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7eff68090050 con 0x7eff740b29a0 2024-03-04T07:37:30.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff792d7640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7eff58071430 0x7eff58073850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff73fff640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff5806b4d0 0x7eff5806d990 secure :-1 s=READY pgs=865 cs=0 l=1 rev1=1 crypto rx=0x7eff60006a90 tx=0x7eff60006a20 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.018+0000 7eff792d7640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7eff58071430 0x7eff58073850 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.023 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.019+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== osd.5 v2:172.21.15.66:6800/1583207443 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7eff68017ec0 con 0x7eff58071430 2024-03-04T07:37:30.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.032+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 --> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7eff740733c0 con 0x7eff58071430 2024-03-04T07:37:30.032 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff717fa640 1 -- 172.21.15.64:0/457821255 <== osd.5 v2:172.21.15.66:6800/1583207443 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7eff740733c0 con 0x7eff58071430 2024-03-04T07:37:30.033 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:30.033 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7eff58071430 msgr2=0x7eff58073850 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.66:6800/1583207443,v1:172.21.15.66:6801/1583207443] conn(0x7eff58071430 0x7eff58073850 crc :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff5806b4d0 msgr2=0x7eff5806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.034 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7eff5806b4d0 0x7eff5806d990 secure :-1 s=READY pgs=865 cs=0 l=1 rev1=1 crypto rx=0x7eff60006a90 tx=0x7eff60006a20 comp rx=0 tx=0).stop 2024-03-04T07:37:30.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 msgr2=0x7eff7415b620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 --2- 172.21.15.64:0/457821255 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7eff740b29a0 0x7eff7415b620 secure :-1 s=READY pgs=1901 cs=0 l=1 rev1=1 crypto rx=0x7eff6800c500 tx=0x7eff6800c9c0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff792d7640 1 -- 172.21.15.64:0/457821255 reap_dead start 2024-03-04T07:37:30.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 shutdown_connections 2024-03-04T07:37:30.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 >> 172.21.15.64:0/457821255 conn(0x7eff74066d00 msgr2=0x7eff740670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 shutdown_connections 2024-03-04T07:37:30.035 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.033+0000 7eff7ad3a640 1 -- 172.21.15.64:0/457821255 wait complete. 2024-03-04T07:37:30.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 5 2024-03-04T07:37:30.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.146+0000 7ff8c20b8640 1 Processor -- start 2024-03-04T07:37:30.146 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.146+0000 7ff8c20b8640 1 -- start start 2024-03-04T07:37:30.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8c20b8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc0ceed0 0x7ff8bc0d0470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8c20b8640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8bc0cf3d0 con 0x7ff8bc145780 2024-03-04T07:37:30.149 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8c20b8640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8bc0cf6b0 con 0x7ff8bc066d00 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8c20b8640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8bc072c30 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8bb7fe640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff8bc145780 0x7ff8bc072520 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50172/0 (socket says 172.21.15.64:50172) 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8bb7fe640 1 -- 172.21.15.64:0/2781195476 learned_addr learned my addr 172.21.15.64:0/2781195476 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8baffd640 1 --2- 172.21.15.64:0/2781195476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc0ceed0 0x7ff8bc0d0470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 2289924694 0 0) 0x7ff8bc0cf3d0 con 0x7ff8bc145780 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff8a4003520 con 0x7ff8bc145780 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 <== mon.1 v1:172.21.15.66:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1130711212 0 0) 0x7ff8bc0cf6b0 con 0x7ff8bc066d00 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.147+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 --> v1:172.21.15.66:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff8bc0cf3d0 con 0x7ff8bc066d00 2024-03-04T07:37:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.148+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 1662887704 0 0) 0x7ff8a4003520 con 0x7ff8bc145780 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.148+0000 7ff8baffd640 1 -- 172.21.15.64:0/2781195476 >> v1:172.21.15.66:6789/0 conn(0x7ff8bc066d00 legacy=0x7ff8bc06d9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.148+0000 7ff8baffd640 1 -- 172.21.15.64:0/2781195476 >> v1:172.21.15.64:6789/0 conn(0x7ff8bc145780 legacy=0x7ff8bc072520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.148+0000 7ff8baffd640 1 -- 172.21.15.64:0/2781195476 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8bc0cf990 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.148+0000 7ff8baffd640 1 --2- 172.21.15.64:0/2781195476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc0ceed0 0x7ff8bc0d0470 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7ff8b00097d0 tx=0x7ff8b002b770 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9ad44d29e4dd3680 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 <== mon.2 v2:172.21.15.66:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b0037070 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 <== mon.2 v2:172.21.15.66:3301/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff8b0030510 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8ba7fc640 1 -- 172.21.15.64:0/2781195476 <== mon.2 v2:172.21.15.66:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8ac0019f0 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8c20b8640 1 -- 172.21.15.64:0/2781195476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc0ceed0 msgr2=0x7ff8bc0d0470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.151 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/2781195476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc0ceed0 0x7ff8bc0d0470 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7ff8b00097d0 tx=0x7ff8b002b770 comp rx=0 tx=0).stop 2024-03-04T07:37:30.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8c20b8640 1 -- 172.21.15.64:0/2781195476 shutdown_connections 2024-03-04T07:37:30.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/2781195476 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc0ceed0 0x7ff8bc0d0470 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8c20b8640 1 -- 172.21.15.64:0/2781195476 >> 172.21.15.64:0/2781195476 conn(0x7ff8bc0b2720 msgr2=0x7ff8bc0b2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8c20b8640 1 -- 172.21.15.64:0/2781195476 shutdown_connections 2024-03-04T07:37:30.152 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.149+0000 7ff8c20b8640 1 -- 172.21.15.64:0/2781195476 wait complete. 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.150+0000 7ff8c20b8640 1 Processor -- start 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.150+0000 7ff8c20b8640 1 -- start start 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8c20b8640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8bc066d00 0x7ff8bc15e560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8c20b8640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8bc0ceed0 0x7ff8bc13b970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8c20b8640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc145780 0x7ff8bc1e57c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8c20b8640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8bc067330 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8c20b8640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8bc0671b0 con 0x7ff8bc066d00 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8c20b8640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff8bc0674b0 con 0x7ff8bc145780 2024-03-04T07:37:30.155 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8baffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8bc0ceed0 0x7ff8bc13b970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8baffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8bc0ceed0 0x7ff8bc13b970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38464/0 (socket says 172.21.15.64:38464) 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8baffd640 1 -- 172.21.15.64:0/374296915 learned_addr learned my addr 172.21.15.64:0/374296915 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8bb7fe640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8bc066d00 0x7ff8bc15e560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.151+0000 7ff8bbfff640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc145780 0x7ff8bc1e57c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8baffd640 1 -- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc145780 msgr2=0x7ff8bc1e57c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8baffd640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc145780 0x7ff8bc1e57c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8baffd640 1 -- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8bc066d00 msgr2=0x7ff8bc15e560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8baffd640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8bc066d00 0x7ff8bc15e560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8baffd640 1 -- 172.21.15.64:0/374296915 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8bc13c430 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.156 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8baffd640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8bc0ceed0 0x7ff8bc13b970 secure :-1 s=READY pgs=1903 cs=0 l=1 rev1=1 crypto rx=0x7ff8b0009720 tx=0x7ff8b00046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8a3fff640 1 -- 172.21.15.64:0/374296915 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b0042070 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8a3fff640 1 -- 172.21.15.64:0/374296915 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff8b0030ec0 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8bc13c710 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8a3fff640 1 -- 172.21.15.64:0/374296915 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b0037040 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.152+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8bc13ccd0 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.153+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff8bc067ff0 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.153+0000 7ff8a3fff640 1 -- 172.21.15.64:0/374296915 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff8b003a040 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.154+0000 7ff8a3fff640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8a806b4d0 0x7ff8a806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.157 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.154+0000 7ff8a3fff640 1 -- 172.21.15.64:0/374296915 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7ff8b00ab6b0 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.154+0000 7ff8bb7fe640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8a806b4d0 0x7ff8a806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.154+0000 7ff8bb7fe640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8a806b4d0 0x7ff8a806d990 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7ff8ac001350 tx=0x7ff8ac009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.158 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.158+0000 7ff8a3fff640 1 -- 172.21.15.64:0/374296915 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff8b007a180 con 0x7ff8bc0ceed0 2024-03-04T07:37:30.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.339+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "5", "target": ["mon-mgr", ""]}) v1 -- 0x7ff8bc144f30 con 0x7ff8a806b4d0 2024-03-04T07:37:30.339 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.339+0000 7ff8a3fff640 1 -- 172.21.15.64:0/374296915 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 5 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7ff8bc144f30 con 0x7ff8a806b4d0 2024-03-04T07:37:30.339 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 5 to deep-scrub 2024-03-04T07:37:30.341 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8a806b4d0 msgr2=0x7ff8a806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8a806b4d0 0x7ff8a806d990 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7ff8ac001350 tx=0x7ff8ac009040 comp rx=0 tx=0).stop 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8bc0ceed0 msgr2=0x7ff8bc13b970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8bc0ceed0 0x7ff8bc13b970 secure :-1 s=READY pgs=1903 cs=0 l=1 rev1=1 crypto rx=0x7ff8b0009720 tx=0x7ff8b00046c0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 shutdown_connections 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff8a806b4d0 0x7ff8a806d990 unknown :-1 s=CLOSED pgs=866 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8bc145780 0x7ff8bc1e57c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8bc0ceed0 0x7ff8bc13b970 unknown :-1 s=CLOSED pgs=1903 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 --2- 172.21.15.64:0/374296915 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8bc066d00 0x7ff8bc15e560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.342 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 >> 172.21.15.64:0/374296915 conn(0x7ff8bc0b2720 msgr2=0x7ff8bc06dc10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 shutdown_connections 2024-03-04T07:37:30.343 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.341+0000 7ff8c20b8640 1 -- 172.21.15.64:0/374296915 wait complete. 2024-03-04T07:37:30.352 INFO:tasks.ceph:Scrubbing osd.6 2024-03-04T07:37:30.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.6 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:30.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.458+0000 7f1a56f13640 1 Processor -- start 2024-03-04T07:37:30.458 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.458+0000 7f1a56f13640 1 -- start start 2024-03-04T07:37:30.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a56f13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500b1550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.459 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a56f13640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f1a5006b2c0 0x7f1a5006b750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a56f13640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f1a50161a40 con 0x7f1a500b2cd0 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a56f13640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f1a500d11e0 con 0x7f1a5006b2c0 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a56f13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a500b2a20 con 0x7f1a500cdcd0 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a4ffff640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7f1a500b2cd0 0x7f1a500b1a90 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50184/0 (socket says 172.21.15.64:50184) 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a4ffff640 1 -- 172.21.15.64:0/3037661481 learned_addr learned my addr 172.21.15.64:0/3037661481 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a554b0640 1 --2- 172.21.15.64:0/3037661481 >> v2:172.21.15.66:3300/0 conn(0x7f1a5006b2c0 0x7f1a5006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a54caf640 1 --2- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500b1550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.460 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a4f7fe640 1 -- 172.21.15.64:0/3037661481 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 1287096702 0 0) 0x7f1a50161a40 con 0x7f1a500b2cd0 2024-03-04T07:37:30.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a4f7fe640 1 -- 172.21.15.64:0/3037661481 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7f1a40003520 con 0x7f1a500b2cd0 2024-03-04T07:37:30.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a554b0640 1 -- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 msgr2=0x7f1a500b1550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:30.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a554b0640 1 --2- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500b1550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a554b0640 1 -- 172.21.15.64:0/3037661481 >> v1:172.21.15.64:6789/0 conn(0x7f1a500b2cd0 legacy=0x7f1a500b1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.461 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.459+0000 7f1a554b0640 1 -- 172.21.15.64:0/3037661481 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a500d0940 con 0x7f1a5006b2c0 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.460+0000 7f1a554b0640 1 --2- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a5006b2c0 0x7f1a5006b750 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f1a48009d90 tx=0x7f1a4802be10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ff584b4a7e10bb17 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.460+0000 7f1a4f7fe640 1 -- 172.21.15.64:0/3037661481 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a48039070 con 0x7f1a5006b2c0 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.460+0000 7f1a4f7fe640 1 -- 172.21.15.64:0/3037661481 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a4802cbc0 con 0x7f1a5006b2c0 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.460+0000 7f1a4f7fe640 1 -- 172.21.15.64:0/3037661481 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a48034440 con 0x7f1a5006b2c0 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 -- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a5006b2c0 msgr2=0x7f1a5006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 --2- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a5006b2c0 0x7f1a5006b750 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f1a48009d90 tx=0x7f1a4802be10 comp rx=0 tx=0).stop 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 -- 172.21.15.64:0/3037661481 shutdown_connections 2024-03-04T07:37:30.466 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 --2- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a5006b2c0 0x7f1a5006b750 unknown :-1 s=CLOSED pgs=703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 --2- 172.21.15.64:0/3037661481 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500b1550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 -- 172.21.15.64:0/3037661481 >> 172.21.15.64:0/3037661481 conn(0x7f1a500b3700 msgr2=0x7f1a500b3b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 -- 172.21.15.64:0/3037661481 shutdown_connections 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.461+0000 7f1a56f13640 1 -- 172.21.15.64:0/3037661481 wait complete. 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.462+0000 7f1a56f13640 1 Processor -- start 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.462+0000 7f1a56f13640 1 -- start start 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a56f13640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a5006b2c0 0x7f1a50156730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a56f13640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a500b2cd0 0x7f1a500b47f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a56f13640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500ca360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a56f13640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a500cc1c0 con 0x7f1a5006b2c0 2024-03-04T07:37:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a56f13640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a500d0f00 con 0x7f1a500b2cd0 2024-03-04T07:37:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a56f13640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1a500cc340 con 0x7f1a500cdcd0 2024-03-04T07:37:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a5006b2c0 0x7f1a50156730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.468 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a5006b2c0 0x7f1a50156730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38474/0 (socket says 172.21.15.64:38474) 2024-03-04T07:37:30.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 -- 172.21.15.64:0/492042488 learned_addr learned my addr 172.21.15.64:0/492042488 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:30.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a4ffff640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a500b2cd0 0x7f1a500b47f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a554b0640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500ca360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.469 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 -- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 msgr2=0x7f1a500ca360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500ca360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 -- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a500b2cd0 msgr2=0x7f1a500b47f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1a500b2cd0 0x7f1a500b47f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 -- 172.21.15.64:0/492042488 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a500cafe0 con 0x7f1a5006b2c0 2024-03-04T07:37:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a554b0640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f1a500cdcd0 0x7f1a500ca360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a54caf640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a5006b2c0 0x7f1a50156730 secure :-1 s=READY pgs=1905 cs=0 l=1 rev1=1 crypto rx=0x7f1a4400e870 tx=0x7f1a4400ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a44017070 con 0x7f1a5006b2c0 2024-03-04T07:37:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f1a4400c940 con 0x7f1a5006b2c0 2024-03-04T07:37:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.463+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1a500cb280 con 0x7f1a5006b2c0 2024-03-04T07:37:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.464+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a4400cd30 con 0x7f1a5006b2c0 2024-03-04T07:37:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.464+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a500cb840 con 0x7f1a5006b2c0 2024-03-04T07:37:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.464+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f1a5009fcf0 con 0x7f1a5006b2c0 2024-03-04T07:37:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.465+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f1a4401a070 con 0x7f1a5006b2c0 2024-03-04T07:37:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.465+0000 7f1a4d7fa640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a3406b500 0x7f1a3406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.465+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f1a44089d40 con 0x7f1a5006b2c0 2024-03-04T07:37:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.465+0000 7f1a4ffff640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a3406b500 0x7f1a3406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.466+0000 7f1a4d7fa640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f1a34071460 0x7f1a34073880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.466+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f1a44020e50 con 0x7f1a34071460 2024-03-04T07:37:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.466+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1a4408c440 con 0x7f1a5006b2c0 2024-03-04T07:37:30.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.466+0000 7f1a4ffff640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a3406b500 0x7f1a3406d9c0 secure :-1 s=READY pgs=867 cs=0 l=1 rev1=1 crypto rx=0x7f1a500705b0 tx=0x7f1a3c009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.466+0000 7f1a554b0640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f1a34071460 0x7f1a34073880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.466+0000 7f1a554b0640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f1a34071460 0x7f1a34073880 crc :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.473 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.467+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== osd.6 v2:172.21.15.66:6816/177421408 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7f1a44020e50 con 0x7f1a34071460 2024-03-04T07:37:30.480 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.480+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 --> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7f1a500d1440 con 0x7f1a34071460 2024-03-04T07:37:30.481 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a4d7fa640 1 -- 172.21.15.64:0/492042488 <== osd.6 v2:172.21.15.66:6816/177421408 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7f1a500d1440 con 0x7f1a34071460 2024-03-04T07:37:30.481 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:30.481 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:30.481 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:30.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f1a34071460 msgr2=0x7f1a34073880 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.482 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.66:6816/177421408,v1:172.21.15.66:6817/177421408] conn(0x7f1a34071460 0x7f1a34073880 crc :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a3406b500 msgr2=0x7f1a3406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f1a3406b500 0x7f1a3406d9c0 secure :-1 s=READY pgs=867 cs=0 l=1 rev1=1 crypto rx=0x7f1a500705b0 tx=0x7f1a3c009040 comp rx=0 tx=0).stop 2024-03-04T07:37:30.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a5006b2c0 msgr2=0x7f1a50156730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 --2- 172.21.15.64:0/492042488 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f1a5006b2c0 0x7f1a50156730 secure :-1 s=READY pgs=1905 cs=0 l=1 rev1=1 crypto rx=0x7f1a4400e870 tx=0x7f1a4400ed30 comp rx=0 tx=0).stop 2024-03-04T07:37:30.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a554b0640 1 -- 172.21.15.64:0/492042488 reap_dead start 2024-03-04T07:37:30.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 shutdown_connections 2024-03-04T07:37:30.483 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 >> 172.21.15.64:0/492042488 conn(0x7f1a500b3700 msgr2=0x7f1a5006ebd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 shutdown_connections 2024-03-04T07:37:30.484 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.481+0000 7f1a56f13640 1 -- 172.21.15.64:0/492042488 wait complete. 2024-03-04T07:37:30.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 6 2024-03-04T07:37:30.593 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.593+0000 7efcf660e640 1 Processor -- start 2024-03-04T07:37:30.594 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.593+0000 7efcf660e640 1 -- start start 2024-03-04T07:37:30.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf660e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf015df00 0x7efcf00d25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf660e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7efcf00b95a0 0x7efcf00b5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf660e640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efcf00b9a30 0x7efcf00b5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.596 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf660e640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efcf00cea30 con 0x7efcf00b9a30 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf660e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7efcf00d2af0 con 0x7efcf00b95a0 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf660e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efcf00cf3c0 con 0x7efcf015df00 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf4bab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efcf00b9a30 0x7efcf00b5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf4bab640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7efcf00b9a30 0x7efcf00b5c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38490/0 (socket says 172.21.15.64:38490) 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcf4bab640 1 -- 172.21.15.64:0/2166184875 learned_addr learned my addr 172.21.15.64:0/2166184875 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efcef7fe640 1 --2- 172.21.15.64:0/2166184875 >> v2:172.21.15.66:3300/0 conn(0x7efcf00b95a0 0x7efcf00b5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.594+0000 7efceffff640 1 --2- 172.21.15.64:0/2166184875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf015df00 0x7efcf00d25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.595+0000 7efcf4bab640 1 -- 172.21.15.64:0/2166184875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf015df00 msgr2=0x7efcf00d25b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:30.597 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.595+0000 7efcf4bab640 1 --2- 172.21.15.64:0/2166184875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf015df00 0x7efcf00d25b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.595+0000 7efcf4bab640 1 -- 172.21.15.64:0/2166184875 >> v2:172.21.15.66:3300/0 conn(0x7efcf00b95a0 msgr2=0x7efcf00b5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.595+0000 7efcf4bab640 1 --2- 172.21.15.64:0/2166184875 >> v2:172.21.15.66:3300/0 conn(0x7efcf00b95a0 0x7efcf00b5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.595+0000 7efcf4bab640 1 -- 172.21.15.64:0/2166184875 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efcf00cf0e0 con 0x7efcf00b9a30 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.595+0000 7efcf4bab640 1 --2- 172.21.15.64:0/2166184875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf00b9a30 0x7efcf00b5c40 secure :-1 s=READY pgs=1906 cs=0 l=1 rev1=1 crypto rx=0x7efcf008cbc0 tx=0x7efce802bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0323fc2652a5bc5 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efceeffd640 1 -- 172.21.15.64:0/2166184875 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efce802ca70 con 0x7efcf00b9a30 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efceeffd640 1 -- 172.21.15.64:0/2166184875 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efce802cc10 con 0x7efcf00b9a30 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efceeffd640 1 -- 172.21.15.64:0/2166184875 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efce80343f0 con 0x7efcf00b9a30 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 -- 172.21.15.64:0/2166184875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf00b9a30 msgr2=0x7efcf00b5c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.598 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 --2- 172.21.15.64:0/2166184875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf00b9a30 0x7efcf00b5c40 secure :-1 s=READY pgs=1906 cs=0 l=1 rev1=1 crypto rx=0x7efcf008cbc0 tx=0x7efce802bed0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 -- 172.21.15.64:0/2166184875 shutdown_connections 2024-03-04T07:37:30.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 --2- 172.21.15.64:0/2166184875 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf00b9a30 0x7efcf00b5c40 unknown :-1 s=CLOSED pgs=1906 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 --2- 172.21.15.64:0/2166184875 >> v2:172.21.15.66:3300/0 conn(0x7efcf00b95a0 0x7efcf00b5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 --2- 172.21.15.64:0/2166184875 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf015df00 0x7efcf00d25b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 -- 172.21.15.64:0/2166184875 >> 172.21.15.64:0/2166184875 conn(0x7efcf0071960 msgr2=0x7efcf0071d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 -- 172.21.15.64:0/2166184875 shutdown_connections 2024-03-04T07:37:30.599 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.596+0000 7efcf660e640 1 -- 172.21.15.64:0/2166184875 wait complete. 2024-03-04T07:37:30.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.597+0000 7efcf660e640 1 Processor -- start 2024-03-04T07:37:30.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcf660e640 1 -- start start 2024-03-04T07:37:30.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcf660e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf00b95a0 0x7efcf00b56d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.602 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcf660e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf015df00 0x7efcf00b5c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcf660e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efcf01f3a60 0x7efcf01f3f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcf660e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efcf00cec70 con 0x7efcf015df00 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcf660e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efcf00ce750 con 0x7efcf01f3a60 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcf660e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7efcf00cedf0 con 0x7efcf00b95a0 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf015df00 0x7efcf00b5c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcef7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf015df00 0x7efcf00b5c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38496/0 (socket says 172.21.15.64:38496) 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efcef7fe640 1 -- 172.21.15.64:0/304012415 learned_addr learned my addr 172.21.15.64:0/304012415 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.598+0000 7efceffff640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf00b95a0 0x7efcf00b56d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.603 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcf4bab640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efcf01f3a60 0x7efcf01f3f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcef7fe640 1 -- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf00b95a0 msgr2=0x7efcf00b56d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcef7fe640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf00b95a0 0x7efcf00b56d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcef7fe640 1 -- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efcf01f3a60 msgr2=0x7efcf01f3f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcef7fe640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efcf01f3a60 0x7efcf01f3f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcef7fe640 1 -- 172.21.15.64:0/304012415 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efcf0158060 con 0x7efcf015df00 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcef7fe640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf015df00 0x7efcf00b5c10 secure :-1 s=READY pgs=1907 cs=0 l=1 rev1=1 crypto rx=0x7efce000e8a0 tx=0x7efce000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcecff9640 1 -- 172.21.15.64:0/304012415 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efce0017070 con 0x7efcf015df00 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcecff9640 1 -- 172.21.15.64:0/304012415 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efce000c940 con 0x7efcf015df00 2024-03-04T07:37:30.604 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efcf0158300 con 0x7efcf015df00 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcecff9640 1 -- 172.21.15.64:0/304012415 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efce000cd30 con 0x7efcf015df00 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.599+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efcf0158880 con 0x7efcf015df00 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.600+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efcf0067ff0 con 0x7efcf015df00 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.601+0000 7efcecff9640 1 -- 172.21.15.64:0/304012415 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7efce001a070 con 0x7efcf015df00 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.601+0000 7efcecff9640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efcdc06b4d0 0x7efcdc06d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.601+0000 7efcecff9640 1 -- 172.21.15.64:0/304012415 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7efce0089db0 con 0x7efcf015df00 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.601+0000 7efceffff640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efcdc06b4d0 0x7efcdc06d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.605 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.602+0000 7efceffff640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efcdc06b4d0 0x7efcdc06d990 secure :-1 s=READY pgs=868 cs=0 l=1 rev1=1 crypto rx=0x7efce4009ac0 tx=0x7efce4009190 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.606 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.605+0000 7efcecff9640 1 -- 172.21.15.64:0/304012415 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7efce0058880 con 0x7efcf015df00 2024-03-04T07:37:30.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.784+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "6", "target": ["mon-mgr", ""]}) v1 -- 0x7efcf00cd320 con 0x7efcdc06b4d0 2024-03-04T07:37:30.785 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.785+0000 7efcecff9640 1 -- 172.21.15.64:0/304012415 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 6 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7efcf00cd320 con 0x7efcdc06b4d0 2024-03-04T07:37:30.785 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 6 to deep-scrub 2024-03-04T07:37:30.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efcdc06b4d0 msgr2=0x7efcdc06d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efcdc06b4d0 0x7efcdc06d990 secure :-1 s=READY pgs=868 cs=0 l=1 rev1=1 crypto rx=0x7efce4009ac0 tx=0x7efce4009190 comp rx=0 tx=0).stop 2024-03-04T07:37:30.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf015df00 msgr2=0x7efcf00b5c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf015df00 0x7efcf00b5c10 secure :-1 s=READY pgs=1907 cs=0 l=1 rev1=1 crypto rx=0x7efce000e8a0 tx=0x7efce000ed60 comp rx=0 tx=0).stop 2024-03-04T07:37:30.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 shutdown_connections 2024-03-04T07:37:30.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7efcdc06b4d0 0x7efcdc06d990 unknown :-1 s=CLOSED pgs=868 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.787 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7efcf01f3a60 0x7efcf01f3f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.786+0000 7efcf660e640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7efcf015df00 0x7efcf00b5c10 unknown :-1 s=CLOSED pgs=1907 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.787+0000 7efcf660e640 1 --2- 172.21.15.64:0/304012415 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7efcf00b95a0 0x7efcf00b56d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.787+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 >> 172.21.15.64:0/304012415 conn(0x7efcf0071960 msgr2=0x7efcf01618b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.787+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 shutdown_connections 2024-03-04T07:37:30.788 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.787+0000 7efcf660e640 1 -- 172.21.15.64:0/304012415 wait complete. 2024-03-04T07:37:30.797 INFO:tasks.ceph:Scrubbing osd.7 2024-03-04T07:37:30.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.7 config set osd_debug_deep_scrub_sleep 0 2024-03-04T07:37:30.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.903+0000 7ff86d2cf640 1 Processor -- start 2024-03-04T07:37:30.903 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.903+0000 7ff86d2cf640 1 -- start start 2024-03-04T07:37:30.906 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff86d2cf640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff8680b3060 0x7ff868071b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff86d2cf640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff86815e4b0 con 0x7ff8680717b0 2024-03-04T07:37:30.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff86d2cf640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff8680b97e0 con 0x7ff8680b2b10 2024-03-04T07:37:30.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff86d2cf640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff8680b7c70 con 0x7ff8680b3060 2024-03-04T07:37:30.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff866ffd640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7ff8680717b0 0x7ff8680b6e50 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50194/0 (socket says 172.21.15.64:50194) 2024-03-04T07:37:30.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff866ffd640 1 -- 172.21.15.64:0/3566625701 learned_addr learned my addr 172.21.15.64:0/3566625701 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:30.907 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3566625701 >> v2:172.21.15.66:3300/0 conn(0x7ff8680b3060 0x7ff868071b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3347978297 0 0) 0x7ff860001110 con 0x7ff8680717b0 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.904+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 --> v1:172.21.15.64:6789/0 -- auth(proto 2 36 bytes epoch 0) v1 -- 0x7ff850003520 con 0x7ff8680717b0 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 <== mon.0 v1:172.21.15.64:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 214+0+0 (unknown 92248269 0 0) 0x7ff850003520 con 0x7ff8680717b0 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 >> v2:172.21.15.66:3300/0 conn(0x7ff8680b3060 msgr2=0x7ff868071b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 --2- 172.21.15.64:0/3566625701 >> v2:172.21.15.66:3300/0 conn(0x7ff8680b3060 0x7ff868071b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 >> v1:172.21.15.66:6789/0 conn(0x7ff8680b2b10 legacy=0x7ff8680b7560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 --> v1:172.21.15.64:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8680b9a80 con 0x7ff8680717b0 2024-03-04T07:37:30.908 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 <== mon.0 v1:172.21.15.64:6789/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (unknown 355647118 0 0) 0x7ff860001840 con 0x7ff8680717b0 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 >> v1:172.21.15.64:6789/0 conn(0x7ff8680717b0 legacy=0x7ff8680b6e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8500039b0 0x7ff850003d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8500043b0 0x7ff850004880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff850005080 0x7ff85000b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff85000bfb0 con 0x7ff850005080 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff85000c160 con 0x7ff8500039b0 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff85000c360 con 0x7ff8500043b0 2024-03-04T07:37:30.909 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff850005080 0x7ff85000b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff866ffd640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8500039b0 0x7ff850003d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8500043b0 0x7ff850004880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8677fe640 1 -- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8500043b0 msgr2=0x7ff850004880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8500043b0 0x7ff850004880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8677fe640 1 -- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8500039b0 msgr2=0x7ff850003d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff8500039b0 0x7ff850003d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8677fe640 1 -- 172.21.15.64:0/3566625701 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff850003520 con 0x7ff850005080 2024-03-04T07:37:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.905+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8500043b0 0x7ff850004880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:30.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.906+0000 7ff8677fe640 1 -- 172.21.15.64:0/3566625701 reap_dead start 2024-03-04T07:37:30.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.906+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff850005080 0x7ff85000b9c0 secure :-1 s=READY pgs=1909 cs=0 l=1 rev1=1 crypto rx=0x7ff858001f60 tx=0x7ff850005830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=961ad1c21d194d97 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.906+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff858001b30 con 0x7ff850005080 2024-03-04T07:37:30.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.906+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff858003b90 con 0x7ff850005080 2024-03-04T07:37:30.911 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.906+0000 7ff865ffb640 1 -- 172.21.15.64:0/3566625701 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff858005070 con 0x7ff850005080 2024-03-04T07:37:30.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.906+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3566625701 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff850005080 msgr2=0x7ff85000b9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.906+0000 7ff86d2cf640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff850005080 0x7ff85000b9c0 secure :-1 s=READY pgs=1909 cs=0 l=1 rev1=1 crypto rx=0x7ff858001f60 tx=0x7ff850005830 comp rx=0 tx=0).stop 2024-03-04T07:37:30.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.907+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3566625701 shutdown_connections 2024-03-04T07:37:30.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.907+0000 7ff86d2cf640 1 --2- 172.21.15.64:0/3566625701 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff850005080 0x7ff85000b9c0 unknown :-1 s=CLOSED pgs=1909 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.915 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.907+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3566625701 >> 172.21.15.64:0/3566625701 conn(0x7ff8680d3d20 msgr2=0x7ff8680d4130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.907+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3566625701 shutdown_connections 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.907+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3566625701 wait complete. 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.908+0000 7ff86d2cf640 1 Processor -- start 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.908+0000 7ff86d2cf640 1 -- start start 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff850005080 0x7ff8681e3ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8681f7c20 0x7ff8680ce0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680c9930 0x7ff8681e6630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.916 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8680d4bf0 con 0x7ff8681f7c20 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8680d2e10 con 0x7ff850005080 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff868071b90 con 0x7ff8680c9930 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8681f7c20 0x7ff8680ce0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8681f7c20 0x7ff8680ce0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38522/0 (socket says 172.21.15.64:38522) 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 -- 172.21.15.64:0/3573313984 learned_addr learned my addr 172.21.15.64:0/3573313984 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff866ffd640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff850005080 0x7ff8681e3ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680c9930 0x7ff8681e6630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 -- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680c9930 msgr2=0x7ff8681e6630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.917 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff8680c9930 0x7ff8681e6630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 -- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff850005080 msgr2=0x7ff8681e3ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff850005080 0x7ff8681e3ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 -- 172.21.15.64:0/3573313984 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8681e4220 con 0x7ff8681f7c20 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff866ffd640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff850005080 0x7ff8681e3ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff8667fc640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8681f7c20 0x7ff8680ce0f0 secure :-1 s=READY pgs=1910 cs=0 l=1 rev1=1 crypto rx=0x7ff85c011540 tx=0x7ff85c011a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff85c015870 con 0x7ff8681f7c20 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff85c015a10 con 0x7ff8681f7c20 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8680cd050 con 0x7ff8681f7c20 2024-03-04T07:37:30.918 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff85c015dc0 con 0x7ff8681f7c20 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.909+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8680cd610 con 0x7ff8681f7c20 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.910+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7ff8680cce00 con 0x7ff8681f7c20 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.911+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff85c00f460 con 0x7ff8681f7c20 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.911+0000 7ff84f7fe640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff85406b500 0x7ff85406d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.911+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7ff85c08a740 con 0x7ff8681f7c20 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.911+0000 7ff866ffd640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff85406b500 0x7ff85406d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.911+0000 7ff84f7fe640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7ff854071460 0x7ff854073880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.911+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7ff854073f30 con 0x7ff854071460 2024-03-04T07:37:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.911+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_get_version_reply(handle=1 version=130) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff85c091050 con 0x7ff8681f7c20 2024-03-04T07:37:30.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.912+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7ff854071460 0x7ff854073880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:30.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.912+0000 7ff866ffd640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff85406b500 0x7ff85406d9c0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7ff8600012b0 tx=0x7ff860009040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.912+0000 7ff8677fe640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7ff854071460 0x7ff854073880 crc :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:30.920 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.913+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== osd.7 v2:172.21.15.66:6802/3006518207 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+27193 (crc 0 0 0) 0x7ff854073f30 con 0x7ff854071460 2024-03-04T07:37:30.925 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.926+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 --> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) v1 -- 0x7ff86806ce40 con 0x7ff854071460 2024-03-04T07:37:30.926 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff84f7fe640 1 -- 172.21.15.64:0/3573313984 <== osd.7 v2:172.21.15.66:6802/3006518207 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+97 (crc 0 0 0) 0x7ff86806ce40 con 0x7ff854071460 2024-03-04T07:37:30.927 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-03-04T07:37:30.927 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-03-04T07:37:30.927 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-03-04T07:37:30.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7ff854071460 msgr2=0x7ff854073880 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.66:6802/3006518207,v1:172.21.15.66:6804/3006518207] conn(0x7ff854071460 0x7ff854073880 crc :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:30.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff85406b500 msgr2=0x7ff85406d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff85406b500 0x7ff85406d9c0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7ff8600012b0 tx=0x7ff860009040 comp rx=0 tx=0).stop 2024-03-04T07:37:30.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8681f7c20 msgr2=0x7ff8680ce0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:30.928 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 --2- 172.21.15.64:0/3573313984 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff8681f7c20 0x7ff8680ce0f0 secure :-1 s=READY pgs=1910 cs=0 l=1 rev1=1 crypto rx=0x7ff85c011540 tx=0x7ff85c011a00 comp rx=0 tx=0).stop 2024-03-04T07:37:30.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff8677fe640 1 -- 172.21.15.64:0/3573313984 reap_dead start 2024-03-04T07:37:30.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 shutdown_connections 2024-03-04T07:37:30.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 >> 172.21.15.64:0/3573313984 conn(0x7ff8680d3d20 msgr2=0x7ff86815d070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:30.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 shutdown_connections 2024-03-04T07:37:30.929 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:30.927+0000 7ff86d2cf640 1 -- 172.21.15.64:0/3573313984 wait complete. 2024-03-04T07:37:30.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 7 2024-03-04T07:37:31.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.040+0000 7f3f1d95e640 1 Processor -- start 2024-03-04T07:37:31.040 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.040+0000 7f3f1d95e640 1 -- start start 2024-03-04T07:37:31.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f1d95e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f1815cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f1d95e640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3f1806d400 0x7f3f180ae700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.041 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f1d95e640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7f3f180729b0 con 0x7f3f180aec40 2024-03-04T07:37:31.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f1d95e640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f3f180af0b0 con 0x7f3f1806d400 2024-03-04T07:37:31.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f1d95e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f1806cf70 con 0x7f3f1815bac0 2024-03-04T07:37:31.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f0ffff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3f1806d400 0x7f3f180ae700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f16ffd640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f1815cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f177fe640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7f3f180aec40 0x7f3f18070870 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:38398/0 (socket says 172.21.15.64:38398) 2024-03-04T07:37:31.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f177fe640 1 -- 172.21.15.64:0/2441709194 learned_addr learned my addr 172.21.15.64:0/2441709194 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:31.042 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f0ffff640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f3f1806d400 0x7f3f180ae700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:48170/0 (socket says 172.21.15.64:48170) 2024-03-04T07:37:31.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f0ffff640 1 -- 172.21.15.64:0/2441709194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 msgr2=0x7f3f1815cbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:31.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f0ffff640 1 --2- 172.21.15.64:0/2441709194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f1815cbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f0ffff640 1 -- 172.21.15.64:0/2441709194 >> v1:172.21.15.66:6789/0 conn(0x7f3f180aec40 legacy=0x7f3f18070870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.043 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.041+0000 7f3f0ffff640 1 -- 172.21.15.64:0/2441709194 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f1806c920 con 0x7f3f1806d400 2024-03-04T07:37:31.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.042+0000 7f3f0ffff640 1 --2- 172.21.15.64:0/2441709194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f1806d400 0x7f3f180ae700 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f3f000097a0 tx=0x7f3f0002b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e92c98d1b91b225f server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:31.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.042+0000 7f3f167fc640 1 -- 172.21.15.64:0/2441709194 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f00035020 con 0x7f3f1806d400 2024-03-04T07:37:31.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.042+0000 7f3f167fc640 1 -- 172.21.15.64:0/2441709194 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f00031830 con 0x7f3f1806d400 2024-03-04T07:37:31.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.042+0000 7f3f167fc640 1 -- 172.21.15.64:0/2441709194 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f00031c00 con 0x7f3f1806d400 2024-03-04T07:37:31.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 -- 172.21.15.64:0/2441709194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f1806d400 msgr2=0x7f3f180ae700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/2441709194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f1806d400 0x7f3f180ae700 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f3f000097a0 tx=0x7f3f0002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.048 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 -- 172.21.15.64:0/2441709194 shutdown_connections 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/2441709194 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f1806d400 0x7f3f180ae700 unknown :-1 s=CLOSED pgs=706 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/2441709194 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f1815cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 -- 172.21.15.64:0/2441709194 >> 172.21.15.64:0/2441709194 conn(0x7f3f18066d00 msgr2=0x7f3f180670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 -- 172.21.15.64:0/2441709194 shutdown_connections 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.043+0000 7f3f1d95e640 1 -- 172.21.15.64:0/2441709194 wait complete. 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.044+0000 7f3f1d95e640 1 Processor -- start 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.044+0000 7f3f1d95e640 1 -- start start 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f1d95e640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f1806d400 0x7f3f18153fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.049 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f1d95e640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f180aec40 0x7f3f18154500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f1d95e640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f18142b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f1d95e640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f1815bea0 con 0x7f3f1806d400 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f1d95e640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f180cc490 con 0x7f3f180aec40 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f1d95e640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3f1815c020 con 0x7f3f1815bac0 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f1806d400 0x7f3f18153fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f1806d400 0x7f3f18153fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38526/0 (socket says 172.21.15.64:38526) 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 -- 172.21.15.64:0/1713645570 learned_addr learned my addr 172.21.15.64:0/1713645570 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f0ffff640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f180aec40 0x7f3f18154500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f177fe640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f18142b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 -- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 msgr2=0x7f3f18142b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f18142b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 -- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f180aec40 msgr2=0x7f3f18154500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f180aec40 0x7f3f18154500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 -- 172.21.15.64:0/1713645570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f18154a40 con 0x7f3f1806d400 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f0ffff640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f180aec40 0x7f3f18154500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.045+0000 7f3f16ffd640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f1806d400 0x7f3f18153fc0 secure :-1 s=READY pgs=1911 cs=0 l=1 rev1=1 crypto rx=0x7f3f0800e8a0 tx=0x7f3f0800ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:31.051 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.046+0000 7f3f14ff9640 1 -- 172.21.15.64:0/1713645570 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f08017070 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.046+0000 7f3f14ff9640 1 -- 172.21.15.64:0/1713645570 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f3f0800c940 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.046+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f18154cf0 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.046+0000 7f3f14ff9640 1 -- 172.21.15.64:0/1713645570 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f0800cd30 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.046+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f181552b0 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.047+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3f18067ff0 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.047+0000 7f3f14ff9640 1 -- 172.21.15.64:0/1713645570 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f3f0801a070 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.047+0000 7f3f14ff9640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ef806b4d0 0x7f3ef806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.047+0000 7f3f14ff9640 1 -- 172.21.15.64:0/1713645570 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f3f08089df0 con 0x7f3f1806d400 2024-03-04T07:37:31.052 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.047+0000 7f3f0ffff640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ef806b4d0 0x7f3ef806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.048+0000 7f3f0ffff640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ef806b4d0 0x7f3ef806d990 secure :-1 s=READY pgs=870 cs=0 l=1 rev1=1 crypto rx=0x7f3f00036000 tx=0x7f3f00008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:31.053 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.052+0000 7f3f14ff9640 1 -- 172.21.15.64:0/1713645570 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f3f08058f10 con 0x7f3f1806d400 2024-03-04T07:37:31.232 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.232+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "7", "target": ["mon-mgr", ""]}) v1 -- 0x7f3f1806de50 con 0x7f3ef806b4d0 2024-03-04T07:37:31.233 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.233+0000 7f3f14ff9640 1 -- 172.21.15.64:0/1713645570 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 7 to deep-scrub) v1 ==== 41+0+0 (secure 0 0 0) 0x7f3f1806de50 con 0x7f3ef806b4d0 2024-03-04T07:37:31.233 INFO:teuthology.orchestra.run.smithi064.stderr:instructed osd(s) 7 to deep-scrub 2024-03-04T07:37:31.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ef806b4d0 msgr2=0x7f3ef806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ef806b4d0 0x7f3ef806d990 secure :-1 s=READY pgs=870 cs=0 l=1 rev1=1 crypto rx=0x7f3f00036000 tx=0x7f3f00008030 comp rx=0 tx=0).stop 2024-03-04T07:37:31.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f1806d400 msgr2=0x7f3f18153fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f1806d400 0x7f3f18153fc0 secure :-1 s=READY pgs=1911 cs=0 l=1 rev1=1 crypto rx=0x7f3f0800e8a0 tx=0x7f3f0800ed60 comp rx=0 tx=0).stop 2024-03-04T07:37:31.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 shutdown_connections 2024-03-04T07:37:31.235 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f3ef806b4d0 0x7f3ef806d990 unknown :-1 s=CLOSED pgs=870 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f3f1815bac0 0x7f3f18142b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f3f180aec40 0x7f3f18154500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 --2- 172.21.15.64:0/1713645570 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f3f1806d400 0x7f3f18153fc0 unknown :-1 s=CLOSED pgs=1911 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 >> 172.21.15.64:0/1713645570 conn(0x7f3f18066d00 msgr2=0x7f3f1815e220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:31.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 shutdown_connections 2024-03-04T07:37:31.236 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.235+0000 7f3f1d95e640 1 -- 172.21.15.64:0/1713645570 wait complete. 2024-03-04T07:37:31.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:37:31.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.347+0000 7fce42916640 1 Processor -- start 2024-03-04T07:37:31.347 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.347+0000 7fce42916640 1 -- start start 2024-03-04T07:37:31.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce42916640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fce3c157b80 0x7fce3c15b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce42916640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c157f60 0x7fce3c15b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce42916640 1 -- --> v1:172.21.15.64:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7fce3c06c7b0 con 0x7fce3c0cbab0 2024-03-04T07:37:31.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce42916640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fce3c06ca90 con 0x7fce3c157b80 2024-03-04T07:37:31.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce42916640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fce3c06cd70 con 0x7fce3c157f60 2024-03-04T07:37:31.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce3bfff640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fce3c157b80 0x7fce3c15b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.349 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce40eb3640 1 --1- >> v1:172.21.15.64:6789/0 conn(0x7fce3c0cbab0 0x7fce3c070b40 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.64:6789/0 says I am v1:172.21.15.64:50196/0 (socket says 172.21.15.64:50196) 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce40eb3640 1 -- 172.21.15.64:0/1917477955 learned_addr learned my addr 172.21.15.64:0/1917477955 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce3b7fe640 1 --2- 172.21.15.64:0/1917477955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c157f60 0x7fce3c15b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce3bfff640 1 -- 172.21.15.64:0/1917477955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c157f60 msgr2=0x7fce3c15b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce3bfff640 1 --2- 172.21.15.64:0/1917477955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c157f60 0x7fce3c15b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce3bfff640 1 -- 172.21.15.64:0/1917477955 >> v1:172.21.15.64:6789/0 conn(0x7fce3c0cbab0 legacy=0x7fce3c070b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce3bfff640 1 -- 172.21.15.64:0/1917477955 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce3c06d050 con 0x7fce3c157b80 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.348+0000 7fce3affd640 1 -- 172.21.15.64:0/1917477955 <== mon.0 v1:172.21.15.64:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (unknown 3271054858 0 0) 0x7fce3c06c7b0 con 0x7fce3c0cbab0 2024-03-04T07:37:31.350 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.349+0000 7fce3bfff640 1 --2- 172.21.15.64:0/1917477955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c15b260 secure :-1 s=READY pgs=1913 cs=0 l=1 rev1=1 crypto rx=0x7fce300097a0 tx=0x7fce3002b8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=82d13c54625cbe02 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.349+0000 7fce3affd640 1 -- 172.21.15.64:0/1917477955 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce30035020 con 0x7fce3c157b80 2024-03-04T07:37:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.349+0000 7fce3affd640 1 -- 172.21.15.64:0/1917477955 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fce30031830 con 0x7fce3c157b80 2024-03-04T07:37:31.351 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.349+0000 7fce3affd640 1 -- 172.21.15.64:0/1917477955 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce30031c00 con 0x7fce3c157b80 2024-03-04T07:37:31.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 -- 172.21.15.64:0/1917477955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 msgr2=0x7fce3c15b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.354 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 --2- 172.21.15.64:0/1917477955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c15b260 secure :-1 s=READY pgs=1913 cs=0 l=1 rev1=1 crypto rx=0x7fce300097a0 tx=0x7fce3002b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 -- 172.21.15.64:0/1917477955 shutdown_connections 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 --2- 172.21.15.64:0/1917477955 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c157f60 0x7fce3c15b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 --2- 172.21.15.64:0/1917477955 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c15b260 unknown :-1 s=CLOSED pgs=1913 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 -- 172.21.15.64:0/1917477955 >> 172.21.15.64:0/1917477955 conn(0x7fce3c066d00 msgr2=0x7fce3c0670e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 -- 172.21.15.64:0/1917477955 shutdown_connections 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.350+0000 7fce42916640 1 -- 172.21.15.64:0/1917477955 wait complete. 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.351+0000 7fce42916640 1 Processor -- start 2024-03-04T07:37:31.355 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.351+0000 7fce42916640 1 -- start start 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c0cbab0 0x7fce3c0b41a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c13b520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce3c157f60 0x7fce3c144de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce3c0cc1d0 con 0x7fce3c157b80 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce3c0cc050 con 0x7fce3c157f60 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fce3c071250 con 0x7fce3c0cbab0 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c13b520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.356 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c13b520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:38558/0 (socket says 172.21.15.64:38558) 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 -- 172.21.15.64:0/3699754532 learned_addr learned my addr 172.21.15.64:0/3699754532 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce40eb3640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce3c157f60 0x7fce3c144de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3bfff640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c0cbab0 0x7fce3c0b41a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 -- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c0cbab0 msgr2=0x7fce3c0b41a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c0cbab0 0x7fce3c0b41a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 -- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce3c157f60 msgr2=0x7fce3c144de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce3c157f60 0x7fce3c144de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 -- 172.21.15.64:0/3699754532 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce3c13c5c0 con 0x7fce3c157b80 2024-03-04T07:37:31.357 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3bfff640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c0cbab0 0x7fce3c0b41a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce3b7fe640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c13b520 secure :-1 s=READY pgs=1914 cs=0 l=1 rev1=1 crypto rx=0x7fce2c00c820 tx=0x7fce2c00cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce38ff9640 1 -- 172.21.15.64:0/3699754532 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce2c015070 con 0x7fce3c157b80 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce38ff9640 1 -- 172.21.15.64:0/3699754532 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fce2c004020 con 0x7fce3c157b80 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fce3c0b3660 con 0x7fce3c157b80 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce38ff9640 1 -- 172.21.15.64:0/3699754532 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce2c0043f0 con 0x7fce3c157b80 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.352+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fce3c0b3c20 con 0x7fce3c157b80 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.353+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fce3c067ff0 con 0x7fce3c157b80 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.354+0000 7fce38ff9640 1 -- 172.21.15.64:0/3699754532 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fce2c007830 con 0x7fce3c157b80 2024-03-04T07:37:31.358 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.354+0000 7fce38ff9640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce2806b500 0x7fce2806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:31.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.354+0000 7fce38ff9640 1 -- 172.21.15.64:0/3699754532 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7fce2c089cd0 con 0x7fce3c157b80 2024-03-04T07:37:31.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.354+0000 7fce3bfff640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce2806b500 0x7fce2806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:31.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.355+0000 7fce3bfff640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce2806b500 0x7fce2806d9c0 secure :-1 s=READY pgs=871 cs=0 l=1 rev1=1 crypto rx=0x7fce30036000 tx=0x7fce30008030 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:31.359 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.359+0000 7fce38ff9640 1 -- 172.21.15.64:0/3699754532 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fce2c058df0 con 0x7fce3c157b80 2024-03-04T07:37:31.536 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.536+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fce3c0d07a0 con 0x7fce2806b500 2024-03-04T07:37:31.537 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.537+0000 7fce38ff9640 1 -- 172.21.15.64:0/3699754532 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+13373 (secure 0 0 0) 0x7fce3c0d07a0 con 0x7fce2806b500 2024-03-04T07:37:31.538 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:31.540 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:37:31.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce2806b500 msgr2=0x7fce2806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce2806b500 0x7fce2806d9c0 secure :-1 s=READY pgs=871 cs=0 l=1 rev1=1 crypto rx=0x7fce30036000 tx=0x7fce30008030 comp rx=0 tx=0).stop 2024-03-04T07:37:31.540 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 msgr2=0x7fce3c13b520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:31.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c13b520 secure :-1 s=READY pgs=1914 cs=0 l=1 rev1=1 crypto rx=0x7fce2c00c820 tx=0x7fce2c00cce0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 shutdown_connections 2024-03-04T07:37:31.541 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fce2806b500 0x7fce2806d9c0 unknown :-1 s=CLOSED pgs=871 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fce3c157f60 0x7fce3c144de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fce3c157b80 0x7fce3c13b520 unknown :-1 s=CLOSED pgs=1914 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 --2- 172.21.15.64:0/3699754532 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fce3c0cbab0 0x7fce3c0b41a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:31.542 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 >> 172.21.15.64:0/3699754532 conn(0x7fce3c066d00 msgr2=0x7fce3c14f620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:31.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 shutdown_connections 2024-03-04T07:37:31.543 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:31.539+0000 7fce42916640 1 -- 172.21.15.64:0/3699754532 wait complete. 2024-03-04T07:37:31.550 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":448,"stamp":"2024-03-04T07:37:31.266405+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":2,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":2,"kb":754974720,"kb_used":329928,"kb_used_data":13896,"kb_used_omap":12,"kb_used_meta":315891,"kb_avail":754644792,"statfs":{"total":773094113280,"available":772756267008,"internally_reserved":0,"allocated":14229504,"data_stored":6871968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":323472944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":-1,"num_object_clones":0,"num_object_copies":-2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.842842"},"pg_stats":[{"pgid":"1.0","version":"15'117","reported_seq":372,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-04T07:37:23.934964+0000","last_change":"2024-03-04T07:26:42.227613+0000","last_active":"2024-03-04T07:37:23.934964+0000","last_peered":"2024-03-04T07:37:23.934964+0000","last_clean":"2024-03-04T07:37:23.934964+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:37:23.934964+0000","last_undegraded":"2024-03-04T07:37:23.934964+0000","last_fullsized":"2024-03-04T07:37:23.934964+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-04T07:26:41.210182+0000","last_clean_scrub_stamp":"2024-03-04T07:26:41.210182+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T18:31:29.469382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574983,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41676,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330164,"statfs":{"total":96636764160,"available":96594087936,"internally_reserved":0,"allocated":2224128,"data_stored":1301772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":40434122},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574982,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574982,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574983,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574982,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574982,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574982,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574982,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41676,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330164,"statfs":{"total":96636764160,"available":96594087936,"internally_reserved":0,"allocated":2224128,"data_stored":1301772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:37:31.553 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-03-04T07:26:41.210182+0000 time.struct_time(tm_year=2024, tm_mon=3, tm_mday=4, tm_hour=7, tm_min=26, tm_sec=41, tm_wday=0, tm_yday=64, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=3, tm_mday=4, tm_hour=7, tm_min=37, tm_sec=26, tm_wday=0, tm_yday=64, tm_isdst=0) 2024-03-04T07:37:31.553 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-03-04T07:37:48.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:48.663 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:51.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-04T07:37:51.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.668+0000 7ff784af4640 1 Processor -- start 2024-03-04T07:37:51.668 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.668+0000 7ff784af4640 1 -- start start 2024-03-04T07:37:51.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff784af4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78006fe20 0x7ff7800cea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff784af4640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff7800d0880 0x7ff7800ccbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.669 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff784af4640 1 -- --> v1:172.21.15.66:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x7ff7800d0dd0 con 0x7ff78015e060 2024-03-04T07:37:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff784af4640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7ff7800d10b0 con 0x7ff7800d0880 2024-03-04T07:37:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff784af4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff7800cc0e0 con 0x7ff78006fe20 2024-03-04T07:37:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff77ed9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff7800d0880 0x7ff7800ccbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff77dd9b640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78006fe20 0x7ff7800cea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff77e59c640 1 --1- >> v1:172.21.15.66:6789/0 conn(0x7ff78015e060 0x7ff7800ce310 :-1 s=CONNECTING_WAIT_BANNER_AND_IDENTIFY pgs=0 cs=0 l=0).handle_server_banner_and_identify peer v1:172.21.15.66:6789/0 says I am v1:172.21.15.64:41022/0 (socket says 172.21.15.64:41022) 2024-03-04T07:37:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff77e59c640 1 -- 172.21.15.64:0/3461246863 learned_addr learned my addr 172.21.15.64:0/3461246863 (peer_addr_for_me v1:172.21.15.64:0/0) 2024-03-04T07:37:51.670 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.669+0000 7ff77ed9d640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7ff7800d0880 0x7ff7800ccbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.64:33578/0 (socket says 172.21.15.64:33578) 2024-03-04T07:37:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.670+0000 7ff77ed9d640 1 -- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78006fe20 msgr2=0x7ff7800cea20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.670+0000 7ff77ed9d640 1 --2- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78006fe20 0x7ff7800cea20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.670+0000 7ff77ed9d640 1 -- 172.21.15.64:0/3461246863 >> v1:172.21.15.66:6789/0 conn(0x7ff78015e060 legacy=0x7ff7800ce310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.670+0000 7ff77ed9d640 1 -- 172.21.15.64:0/3461246863 --> v2:172.21.15.66:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff7800cc3c0 con 0x7ff7800d0880 2024-03-04T07:37:51.672 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.670+0000 7ff77dd9b640 1 --2- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78006fe20 0x7ff7800cea20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff77ed9d640 1 --2- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff7800d0880 0x7ff7800ccbc0 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7ff7780097a0 tx=0x7ff77802b8d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=949f3fac73a0115b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff77d59a640 1 -- 172.21.15.64:0/3461246863 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff778035020 con 0x7ff7800d0880 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff77d59a640 1 -- 172.21.15.64:0/3461246863 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff778031830 con 0x7ff7800d0880 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff77d59a640 1 -- 172.21.15.64:0/3461246863 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff778031c00 con 0x7ff7800d0880 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff784af4640 1 -- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff7800d0880 msgr2=0x7ff7800ccbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff784af4640 1 --2- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff7800d0880 0x7ff7800ccbc0 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7ff7780097a0 tx=0x7ff77802b8d0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff784af4640 1 -- 172.21.15.64:0/3461246863 shutdown_connections 2024-03-04T07:37:51.673 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff784af4640 1 --2- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff7800d0880 0x7ff7800ccbc0 unknown :-1 s=CLOSED pgs=708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff784af4640 1 --2- 172.21.15.64:0/3461246863 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78006fe20 0x7ff7800cea20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff784af4640 1 -- 172.21.15.64:0/3461246863 >> 172.21.15.64:0/3461246863 conn(0x7ff7800d1a90 msgr2=0x7ff7800d1ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:51.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.671+0000 7ff784af4640 1 -- 172.21.15.64:0/3461246863 shutdown_connections 2024-03-04T07:37:51.674 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.672+0000 7ff784af4640 1 -- 172.21.15.64:0/3461246863 wait complete. 2024-03-04T07:37:51.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.673+0000 7ff784af4640 1 Processor -- start 2024-03-04T07:37:51.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.673+0000 7ff784af4640 1 -- start start 2024-03-04T07:37:51.680 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff784af4640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff78006fe20 0x7ff78015b450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff784af4640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff7800d0880 0x7ff780158120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff784af4640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78015e060 0x7ff78015be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff784af4640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff7800cef60 con 0x7ff7800d0880 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff784af4640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff78015e440 con 0x7ff78006fe20 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff784af4640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff7800cf0e0 con 0x7ff78015e060 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff7800d0880 0x7ff780158120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff7800d0880 0x7ff780158120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:58470/0 (socket says 172.21.15.64:58470) 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 -- 172.21.15.64:0/1293756657 learned_addr learned my addr 172.21.15.64:0/1293756657 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77e59c640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff78006fe20 0x7ff78015b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77ed9d640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78015e060 0x7ff78015be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 -- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78015e060 msgr2=0x7ff78015be60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78015e060 0x7ff78015be60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 -- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff78006fe20 msgr2=0x7ff78015b450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff78006fe20 0x7ff78015b450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 -- 172.21.15.64:0/1293756657 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff780158660 con 0x7ff7800d0880 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff77dd9b640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff7800d0880 0x7ff780158120 secure :-1 s=READY pgs=1915 cs=0 l=1 rev1=1 crypto rx=0x7ff76800e930 tx=0x7ff76800edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff776ffd640 1 -- 172.21.15.64:0/1293756657 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff768017070 con 0x7ff7800d0880 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff776ffd640 1 -- 172.21.15.64:0/1293756657 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7ff76800ca00 con 0x7ff7800d0880 2024-03-04T07:37:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff78015ca40 con 0x7ff7800d0880 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.674+0000 7ff776ffd640 1 -- 172.21.15.64:0/1293756657 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff76800cdf0 con 0x7ff7800d0880 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.675+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff78015c760 con 0x7ff7800d0880 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.675+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff7800b0c40 con 0x7ff7800d0880 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.676+0000 7ff776ffd640 1 -- 172.21.15.64:0/1293756657 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7ff76801a070 con 0x7ff7800d0880 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.676+0000 7ff776ffd640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff76006b4d0 0x7ff76006d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.676+0000 7ff776ffd640 1 -- 172.21.15.64:0/1293756657 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7ff768089960 con 0x7ff7800d0880 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.676+0000 7ff77e59c640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff76006b4d0 0x7ff76006d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.677+0000 7ff77e59c640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff76006b4d0 0x7ff76006d990 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7ff7800cd880 tx=0x7ff770001a30 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:51.684 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.681+0000 7ff776ffd640 1 -- 172.21.15.64:0/1293756657 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7ff7800b0c40 con 0x7ff7800d0880 2024-03-04T07:37:51.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.861+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 --> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7ff780067ff0 con 0x7ff76006b4d0 2024-03-04T07:37:51.861 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.862+0000 7ff776ffd640 1 -- 172.21.15.64:0/1293756657 <== mgr.4103 v2:172.21.15.64:6832/129800233 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+13379 (secure 0 0 0) 0x7ff780067ff0 con 0x7ff76006b4d0 2024-03-04T07:37:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:51.862 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-03-04T07:37:51.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff76006b4d0 msgr2=0x7ff76006d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff76006b4d0 0x7ff76006d990 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7ff7800cd880 tx=0x7ff770001a30 comp rx=0 tx=0).stop 2024-03-04T07:37:51.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff7800d0880 msgr2=0x7ff780158120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff7800d0880 0x7ff780158120 secure :-1 s=READY pgs=1915 cs=0 l=1 rev1=1 crypto rx=0x7ff76800e930 tx=0x7ff76800edf0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.864 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 shutdown_connections 2024-03-04T07:37:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7ff76006b4d0 0x7ff76006d990 unknown :-1 s=CLOSED pgs=872 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7ff78015e060 0x7ff78015be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7ff7800d0880 0x7ff780158120 unknown :-1 s=CLOSED pgs=1915 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 --2- 172.21.15.64:0/1293756657 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7ff78006fe20 0x7ff78015b450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 >> 172.21.15.64:0/1293756657 conn(0x7ff7800d1a90 msgr2=0x7ff7800d1e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 shutdown_connections 2024-03-04T07:37:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.864+0000 7ff784af4640 1 -- 172.21.15.64:0/1293756657 wait complete. 2024-03-04T07:37:51.875 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":458,"stamp":"2024-03-04T07:37:51.270440+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119,"ondisk_log_size":119,"up":2,"acting":2,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":2,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":2,"kb":754974720,"kb_used":329912,"kb_used_data":13896,"kb_used_omap":12,"kb_used_meta":315891,"kb_avail":754644808,"statfs":{"total":773094113280,"available":772756283392,"internally_reserved":0,"allocated":14229504,"data_stored":6871968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12752,"internal_metadata":323472944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002546"},"pg_stats":[{"pgid":"1.0","version":"130'119","reported_seq":383,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-04T07:37:32.121361+0000","last_change":"2024-03-04T07:37:32.121406+0000","last_active":"2024-03-04T07:37:32.121361+0000","last_peered":"2024-03-04T07:37:32.121361+0000","last_clean":"2024-03-04T07:37:32.121361+0000","last_became_active":"2024-03-04T07:26:42.226684+0000","last_became_peered":"2024-03-04T07:26:42.226684+0000","last_unstale":"2024-03-04T07:37:32.121361+0000","last_undegraded":"2024-03-04T07:37:32.121361+0000","last_fullsized":"2024-03-04T07:37:32.121361+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"130'119","last_scrub_stamp":"2024-03-04T07:37:32.121272+0000","last_deep_scrub":"130'119","last_deep_scrub_stamp":"2024-03-04T07:37:32.121272+0000","last_clean_scrub_stamp":"2024-03-04T07:37:32.121272+0000","objects_scrubbed":2,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-03-05T12:06:43.770627+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119,"ondisk_log_size":119,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574987,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41668,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330172,"statfs":{"total":96636764160,"available":96594096128,"internally_reserved":0,"allocated":2224128,"data_stored":1301772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1590,"internal_metadata":40434122},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":13,"seq":55834574986,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40434118},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":13,"seq":55834574986,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":13,"seq":55834574987,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":13,"seq":55834574986,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":40434119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":13,"seq":55834574986,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574986,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":41096,"kb_used_data":1592,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330744,"statfs":{"total":96636764160,"available":96594681856,"internally_reserved":0,"allocated":1630208,"data_stored":711404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":13,"seq":55834574986,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":41668,"kb_used_data":2172,"kb_used_omap":1,"kb_used_meta":39486,"kb_avail":94330172,"statfs":{"total":96636764160,"available":96594096128,"internally_reserved":0,"allocated":2224128,"data_stored":1301772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":40434117},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-04T07:37:51.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo ceph --cluster ceph config set global mon_health_to_clog false 2024-03-04T07:37:51.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.977+0000 7fcca378c640 1 Processor -- start 2024-03-04T07:37:51.977 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.977+0000 7fcca378c640 1 -- start start 2024-03-04T07:37:51.978 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca378c640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7fcc9c15df40 0x7fcc9c066d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca378c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c067240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca378c640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc9c06dc90 0x7fcc9c06e120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca378c640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc9c15f000 con 0x7fcc9c06dc90 2024-03-04T07:37:51.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca378c640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7fcc9c15e6d0 con 0x7fcc9c15df40 2024-03-04T07:37:51.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca378c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc9c15e9b0 con 0x7fcc9c0b3ce0 2024-03-04T07:37:51.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca1d29640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc9c06dc90 0x7fcc9c06e120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.979 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca1d29640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7fcc9c06dc90 0x7fcc9c06e120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:58484/0 (socket says 172.21.15.64:58484) 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.978+0000 7fcca1d29640 1 -- 172.21.15.64:0/916755092 learned_addr learned my addr 172.21.15.64:0/916755092 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca1528640 1 --2- 172.21.15.64:0/916755092 >> v2:172.21.15.66:3300/0 conn(0x7fcc9c15df40 0x7fcc9c066d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca0d27640 1 --2- 172.21.15.64:0/916755092 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c067240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca1d29640 1 -- 172.21.15.64:0/916755092 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 msgr2=0x7fcc9c067240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca1d29640 1 --2- 172.21.15.64:0/916755092 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c067240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca1d29640 1 -- 172.21.15.64:0/916755092 >> v2:172.21.15.66:3300/0 conn(0x7fcc9c15df40 msgr2=0x7fcc9c066d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca1d29640 1 --2- 172.21.15.64:0/916755092 >> v2:172.21.15.66:3300/0 conn(0x7fcc9c15df40 0x7fcc9c066d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.980 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca1d29640 1 -- 172.21.15.64:0/916755092 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc9c15ec90 con 0x7fcc9c06dc90 2024-03-04T07:37:51.983 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.979+0000 7fcca1d29640 1 --2- 172.21.15.64:0/916755092 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c06e120 secure :-1 s=READY pgs=1916 cs=0 l=1 rev1=1 crypto rx=0x7fcc9c0d1ac0 tx=0x7fcc8c02bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9073ca1f021dbef server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcc9bfff640 1 -- 172.21.15.64:0/916755092 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc8c02ca70 con 0x7fcc9c06dc90 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcc9bfff640 1 -- 172.21.15.64:0/916755092 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc8c02cc10 con 0x7fcc9c06dc90 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcc9bfff640 1 -- 172.21.15.64:0/916755092 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc8c0343f0 con 0x7fcc9c06dc90 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 -- 172.21.15.64:0/916755092 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 msgr2=0x7fcc9c06e120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 --2- 172.21.15.64:0/916755092 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c06e120 secure :-1 s=READY pgs=1916 cs=0 l=1 rev1=1 crypto rx=0x7fcc9c0d1ac0 tx=0x7fcc8c02bed0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 -- 172.21.15.64:0/916755092 shutdown_connections 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 --2- 172.21.15.64:0/916755092 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c06e120 unknown :-1 s=CLOSED pgs=1916 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.984 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 --2- 172.21.15.64:0/916755092 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c067240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 --2- 172.21.15.64:0/916755092 >> v2:172.21.15.66:3300/0 conn(0x7fcc9c15df40 0x7fcc9c066d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 -- 172.21.15.64:0/916755092 >> 172.21.15.64:0/916755092 conn(0x7fcc9c0d09c0 msgr2=0x7fcc9c0d0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 -- 172.21.15.64:0/916755092 shutdown_connections 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.980+0000 7fcca378c640 1 -- 172.21.15.64:0/916755092 wait complete. 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.981+0000 7fcca378c640 1 Processor -- start 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.981+0000 7fcca378c640 1 -- start start 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca378c640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c152820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.985 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca378c640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c0ade50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca378c640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc9c15df40 0x7fcc9c0ae390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca378c640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc9c15e320 con 0x7fcc9c06dc90 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca378c640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc9c071a20 con 0x7fcc9c15df40 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca378c640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fcc9c154990 con 0x7fcc9c0b3ce0 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c152820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c152820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:58492/0 (socket says 172.21.15.64:58492) 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 -- 172.21.15.64:0/233096706 learned_addr learned my addr 172.21.15.64:0/233096706 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1d29640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc9c15df40 0x7fcc9c0ae390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.986 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca0d27640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c0ade50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 -- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 msgr2=0x7fcc9c0ade50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c0ade50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 -- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc9c15df40 msgr2=0x7fcc9c0ae390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc9c15df40 0x7fcc9c0ae390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1528640 1 -- 172.21.15.64:0/233096706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc9c154380 con 0x7fcc9c06dc90 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca1d29640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc9c15df40 0x7fcc9c0ae390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.982+0000 7fcca0d27640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c0ade50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.983+0000 7fcca1528640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c152820 secure :-1 s=READY pgs=1917 cs=0 l=1 rev1=1 crypto rx=0x7fcc9400d5f0 tx=0x7fcc9400dab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:51.987 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.983+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc940125f0 con 0x7fcc9c06dc90 2024-03-04T07:37:51.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.983+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc94019070 con 0x7fcc9c06dc90 2024-03-04T07:37:51.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.983+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc9c0b0190 con 0x7fcc9c06dc90 2024-03-04T07:37:51.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.983+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc940129d0 con 0x7fcc9c06dc90 2024-03-04T07:37:51.988 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.983+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc9c154620 con 0x7fcc9c06dc90 2024-03-04T07:37:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.984+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc9c0b06b0 con 0x7fcc9c06dc90 2024-03-04T07:37:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.984+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7fcc94012b70 con 0x7fcc9c06dc90 2024-03-04T07:37:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.984+0000 7fcc99ffb640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc8806b4d0 0x7fcc8806d990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.985+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7fcc9408a370 con 0x7fcc9c06dc90 2024-03-04T07:37:51.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.985+0000 7fcca0d27640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc8806b4d0 0x7fcc8806d990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:51.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.985+0000 7fcca0d27640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc8806b4d0 0x7fcc8806d990 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7fcc9c152d60 tx=0x7fcc90008040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:51.990 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:51.989+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7fcc9c0b06b0 con 0x7fcc9c06dc90 2024-03-04T07:37:52.169 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.169+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command([{prefix=config set, name=mon_health_to_clog}] v 0) v1 -- 0x7fcc9c0733d0 con 0x7fcc9c06dc90 2024-03-04T07:37:52.172 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.172+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_health_to_clog}]=0 v34)=0 v34) v1 ==== 123+0+0 (secure 0 0 0) 0x7fcc94059490 con 0x7fcc9c06dc90 2024-03-04T07:37:52.173 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.173+0000 7fcc99ffb640 1 -- 172.21.15.64:0/233096706 <== mon.0 v2:172.21.15.64:3300/0 8 ==== config(1 keys) v1 ==== 35+0+0 (secure 0 0 0) 0x7fcc94058d90 con 0x7fcc9c06dc90 2024-03-04T07:37:52.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc8806b4d0 msgr2=0x7fcc8806d990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:52.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc8806b4d0 0x7fcc8806d990 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7fcc9c152d60 tx=0x7fcc90008040 comp rx=0 tx=0).stop 2024-03-04T07:37:52.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 msgr2=0x7fcc9c152820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:52.175 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c152820 secure :-1 s=READY pgs=1917 cs=0 l=1 rev1=1 crypto rx=0x7fcc9400d5f0 tx=0x7fcc9400dab0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 shutdown_connections 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7fcc8806b4d0 0x7fcc8806d990 unknown :-1 s=CLOSED pgs=873 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fcc9c15df40 0x7fcc9c0ae390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7fcc9c0b3ce0 0x7fcc9c0ade50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 --2- 172.21.15.64:0/233096706 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7fcc9c06dc90 0x7fcc9c152820 unknown :-1 s=CLOSED pgs=1917 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 >> 172.21.15.64:0/233096706 conn(0x7fcc9c0d09c0 msgr2=0x7fcc9c0b40f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 shutdown_connections 2024-03-04T07:37:52.176 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.175+0000 7fcca378c640 1 -- 172.21.15.64:0/233096706 wait complete. 2024-03-04T07:37:52.186 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-04T07:37:52.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-04T07:37:52.287 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.287+0000 7f2d5d864640 1 Processor -- start 2024-03-04T07:37:52.288 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.287+0000 7f2d5d864640 1 -- start start 2024-03-04T07:37:52.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d5d864640 1 --2- >> v2:172.21.15.66:3300/0 conn(0x7f2d5815bdc0 0x7f2d580d06b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:52.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d5d864640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2d580cd210 0x7f2d580cd670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:52.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d5d864640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d580ce3c0 0x7f2d580ce850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:52.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d5d864640 1 -- --> v2:172.21.15.64:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d580d0bf0 con 0x7f2d580cd210 2024-03-04T07:37:52.289 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d5d864640 1 -- --> v2:172.21.15.66:3300/0 -- mon_getmap magic: 0 v1 -- 0x7f2d580cf610 con 0x7f2d5815bdc0 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d5d864640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d580cf8f0 con 0x7f2d580ce3c0 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d567fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2d580cd210 0x7f2d580cd670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d567fc640 1 --2- >> v2:172.21.15.64:3300/0 conn(0x7f2d580cd210 0x7f2d580cd670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:58504/0 (socket says 172.21.15.64:58504) 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.288+0000 7f2d567fc640 1 -- 172.21.15.64:0/3043306973 learned_addr learned my addr 172.21.15.64:0/3043306973 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d567fc640 1 -- 172.21.15.64:0/3043306973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d580ce3c0 msgr2=0x7f2d580ce850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d567fc640 1 --2- 172.21.15.64:0/3043306973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d580ce3c0 0x7f2d580ce850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d56ffd640 1 --2- 172.21.15.64:0/3043306973 >> v2:172.21.15.66:3300/0 conn(0x7f2d5815bdc0 0x7f2d580d06b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d567fc640 1 -- 172.21.15.64:0/3043306973 >> v2:172.21.15.66:3300/0 conn(0x7f2d5815bdc0 msgr2=0x7f2d580d06b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-04T07:37:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d567fc640 1 --2- 172.21.15.64:0/3043306973 >> v2:172.21.15.66:3300/0 conn(0x7f2d5815bdc0 0x7f2d580d06b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d567fc640 1 -- 172.21.15.64:0/3043306973 --> v2:172.21.15.64:3300/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d580cfbd0 con 0x7f2d580cd210 2024-03-04T07:37:52.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d56ffd640 1 --2- 172.21.15.64:0/3043306973 >> v2:172.21.15.66:3300/0 conn(0x7f2d5815bdc0 0x7f2d580d06b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-04T07:37:52.291 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.289+0000 7f2d567fc640 1 --2- 172.21.15.64:0/3043306973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d580cd670 secure :-1 s=READY pgs=1918 cs=0 l=1 rev1=1 crypto rx=0x7f2d58071fb0 tx=0x7f2d4002b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=baa1df40c80b986b server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:52.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d55ffb640 1 -- 172.21.15.64:0/3043306973 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d40034070 con 0x7f2d580cd210 2024-03-04T07:37:52.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d55ffb640 1 -- 172.21.15.64:0/3043306973 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(1 keys) v1 ==== 35+0+0 (secure 0 0 0) 0x7f2d40030860 con 0x7f2d580cd210 2024-03-04T07:37:52.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d55ffb640 1 -- 172.21.15.64:0/3043306973 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d40030c50 con 0x7f2d580cd210 2024-03-04T07:37:52.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 -- 172.21.15.64:0/3043306973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 msgr2=0x7f2d580cd670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:52.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 --2- 172.21.15.64:0/3043306973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d580cd670 secure :-1 s=READY pgs=1918 cs=0 l=1 rev1=1 crypto rx=0x7f2d58071fb0 tx=0x7f2d4002b9a0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 -- 172.21.15.64:0/3043306973 shutdown_connections 2024-03-04T07:37:52.294 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 --2- 172.21.15.64:0/3043306973 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d580ce3c0 0x7f2d580ce850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 --2- 172.21.15.64:0/3043306973 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d580cd670 unknown :-1 s=CLOSED pgs=1918 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 --2- 172.21.15.64:0/3043306973 >> v2:172.21.15.66:3300/0 conn(0x7f2d5815bdc0 0x7f2d580d06b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 -- 172.21.15.64:0/3043306973 >> 172.21.15.64:0/3043306973 conn(0x7f2d580d1a40 msgr2=0x7f2d580d1e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 -- 172.21.15.64:0/3043306973 shutdown_connections 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.290+0000 7f2d5d864640 1 -- 172.21.15.64:0/3043306973 wait complete. 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.291+0000 7f2d5d864640 1 Processor -- start 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.291+0000 7f2d5d864640 1 -- start start 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d5d864640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d5815c650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:52.295 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d5d864640 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d580ce3c0 0x7f2d5815ea90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d5d864640 1 --2- >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d5815bdc0 0x7f2d580b5510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d5d864640 1 -- --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d580d2760 con 0x7f2d580cd210 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d5d864640 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d580d0190 con 0x7f2d580ce3c0 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d5d864640 1 -- --> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2d5815c1a0 con 0x7f2d5815bdc0 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d56ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d5815c650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d56ffd640 1 --2- >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d5815c650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.64:3300/0 says I am v2:172.21.15.64:58506/0 (socket says 172.21.15.64:58506) 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d56ffd640 1 -- 172.21.15.64:0/2922393766 learned_addr learned my addr 172.21.15.64:0/2922393766 (peer_addr_for_me v2:172.21.15.64:0/0) 2024-03-04T07:37:52.296 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d56ffd640 1 -- 172.21.15.64:0/2922393766 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d5815bdc0 msgr2=0x7f2d580b5510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d567fc640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d580ce3c0 0x7f2d5815ea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d56ffd640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d5815bdc0 0x7f2d580b5510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d56ffd640 1 -- 172.21.15.64:0/2922393766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d580ce3c0 msgr2=0x7f2d5815ea90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.292+0000 7f2d56ffd640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d580ce3c0 0x7f2d5815ea90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.293+0000 7f2d56ffd640 1 -- 172.21.15.64:0/2922393766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d5815f1e0 con 0x7f2d580cd210 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.293+0000 7f2d56ffd640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d5815c650 secure :-1 s=READY pgs=1919 cs=0 l=1 rev1=1 crypto rx=0x7f2d5000c7f0 tx=0x7f2d5000cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.293+0000 7f2d4ffff640 1 -- 172.21.15.64:0/2922393766 <== mon.0 v2:172.21.15.64:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d50015070 con 0x7f2d580cd210 2024-03-04T07:37:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.293+0000 7f2d4ffff640 1 -- 172.21.15.64:0/2922393766 <== mon.0 v2:172.21.15.64:3300/0 2 ==== config(1 keys) v1 ==== 35+0+0 (secure 0 0 0) 0x7f2d50007a30 con 0x7f2d580cd210 2024-03-04T07:37:52.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.293+0000 7f2d4ffff640 1 -- 172.21.15.64:0/2922393766 <== mon.0 v2:172.21.15.64:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d50007e20 con 0x7f2d580cd210 2024-03-04T07:37:52.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.293+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d5815f4c0 con 0x7f2d580cd210 2024-03-04T07:37:52.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.293+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d5806af20 con 0x7f2d580cd210 2024-03-04T07:37:52.298 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.294+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d580b5040 con 0x7f2d580cd210 2024-03-04T07:37:52.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.294+0000 7f2d4ffff640 1 -- 172.21.15.64:0/2922393766 <== mon.0 v2:172.21.15.64:3300/0 4 ==== mgrmap(e 5) v1 ==== 89780+0+0 (secure 0 0 0) 0x7f2d50018070 con 0x7f2d580cd210 2024-03-04T07:37:52.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.295+0000 7f2d4ffff640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3806b500 0x7f2d3806d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-04T07:37:52.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.295+0000 7f2d567fc640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3806b500 0x7f2d3806d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-04T07:37:52.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.295+0000 7f2d4ffff640 1 -- 172.21.15.64:0/2922393766 <== mon.0 v2:172.21.15.64:3300/0 5 ==== osd_map(130..130 src has 1..130) v4 ==== 5029+0+0 (secure 0 0 0) 0x7f2d50089fe0 con 0x7f2d580cd210 2024-03-04T07:37:52.299 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.295+0000 7f2d567fc640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3806b500 0x7f2d3806d9c0 secure :-1 s=READY pgs=874 cs=0 l=1 rev1=1 crypto rx=0x7f2d40036000 tx=0x7f2d4003b040 comp rx=0 tx=0).ready entity=mgr.4103 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-04T07:37:52.300 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.299+0000 7f2d4ffff640 1 -- 172.21.15.64:0/2922393766 <== mon.0 v2:172.21.15.64:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+183808 (secure 0 0 0) 0x7f2d50059070 con 0x7f2d580cd210 2024-03-04T07:37:52.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.517+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 --> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2d580cbe00 con 0x7f2d580cd210 2024-03-04T07:37:52.517 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.517+0000 7f2d4ffff640 1 -- 172.21.15.64:0/2922393766 <== mon.0 v2:172.21.15.64:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 93 v93) v1 ==== 94+0+1326 (secure 0 0 0) 0x7f2d580cbe00 con 0x7f2d580cd210 2024-03-04T07:37:52.517 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-03-04T07:37:52.518 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":93,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":7553,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.66:6834/1594210937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:6832","nonce":1594210937},{"type":"v1","addr":"172.21.15.66:6834","nonce":1594210937}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments"}},"epoch":93}],"filesystems":[]} 2024-03-04T07:37:52.518 INFO:teuthology.orchestra.run.smithi064.stderr:dumped fsmap epoch 93 2024-03-04T07:37:52.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3806b500 msgr2=0x7f2d3806d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:52.519 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3806b500 0x7f2d3806d9c0 secure :-1 s=READY pgs=874 cs=0 l=1 rev1=1 crypto rx=0x7f2d40036000 tx=0x7f2d4003b040 comp rx=0 tx=0).stop 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 msgr2=0x7f2d5815c650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d5815c650 secure :-1 s=READY pgs=1919 cs=0 l=1 rev1=1 crypto rx=0x7f2d5000c7f0 tx=0x7f2d5000cbf0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 shutdown_connections 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:6832/129800233,v1:172.21.15.64:6833/129800233] conn(0x7f2d3806b500 0x7f2d3806d9c0 unknown :-1 s=CLOSED pgs=874 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.66:3301/0,v1:172.21.15.66:6790/0] conn(0x7f2d5815bdc0 0x7f2d580b5510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f2d580ce3c0 0x7f2d5815ea90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 --2- 172.21.15.64:0/2922393766 >> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] conn(0x7f2d580cd210 0x7f2d5815c650 unknown :-1 s=CLOSED pgs=1919 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-04T07:37:52.520 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 >> 172.21.15.64:0/2922393766 conn(0x7f2d580d1a40 msgr2=0x7f2d580b1aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-04T07:37:52.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 shutdown_connections 2024-03-04T07:37:52.521 INFO:teuthology.orchestra.run.smithi064.stderr:2024-03-04T07:37:52.519+0000 7f2d5d864640 1 -- 172.21.15.64:0/2922393766 wait complete. 2024-03-04T07:37:52.530 DEBUG:tasks.cephfs.filesystem:already dead... 2024-03-04T07:37:52.530 INFO:teuthology.misc:Shutting down mds daemons... 2024-03-04T07:37:52.530 ERROR:tasks.ceph.mds.a:tried to stop a non-running daemon 2024-03-04T07:37:52.533 ERROR:tasks.ceph.mds.c:tried to stop a non-running daemon 2024-03-04T07:37:52.536 DEBUG:tasks.ceph.mds.b:waiting for process to exit 2024-03-04T07:37:52.536 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.549 INFO:tasks.ceph.mds.b:Stopped 2024-03-04T07:37:52.549 DEBUG:tasks.ceph.mds.d:waiting for process to exit 2024-03-04T07:37:52.549 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.601 INFO:tasks.ceph.mds.d:Stopped 2024-03-04T07:37:52.601 INFO:teuthology.misc:Shutting down osd daemons... 2024-03-04T07:37:52.601 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-03-04T07:37:52.601 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.630 INFO:tasks.ceph.osd.0:Stopped 2024-03-04T07:37:52.630 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-03-04T07:37:52.630 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.700 INFO:tasks.ceph.osd.1:Stopped 2024-03-04T07:37:52.701 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-03-04T07:37:52.701 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.769 INFO:tasks.ceph.osd.2:Stopped 2024-03-04T07:37:52.769 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-03-04T07:37:52.769 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.836 INFO:tasks.ceph.osd.3:Stopped 2024-03-04T07:37:52.837 DEBUG:tasks.ceph.osd.4:waiting for process to exit 2024-03-04T07:37:52.837 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.868 INFO:tasks.ceph.osd.4:Stopped 2024-03-04T07:37:52.868 DEBUG:tasks.ceph.osd.5:waiting for process to exit 2024-03-04T07:37:52.868 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:52.941 INFO:tasks.ceph.osd.5:Stopped 2024-03-04T07:37:52.941 DEBUG:tasks.ceph.osd.6:waiting for process to exit 2024-03-04T07:37:52.941 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:53.011 INFO:tasks.ceph.osd.6:Stopped 2024-03-04T07:37:53.012 DEBUG:tasks.ceph.osd.7:waiting for process to exit 2024-03-04T07:37:53.012 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:53.095 INFO:tasks.ceph.osd.7:Stopped 2024-03-04T07:37:53.096 INFO:teuthology.misc:Shutting down mgr daemons... 2024-03-04T07:37:53.096 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-03-04T07:37:53.096 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:53.123 INFO:tasks.ceph.mgr.y:Stopped 2024-03-04T07:37:53.123 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-03-04T07:37:53.123 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:53.151 INFO:tasks.ceph.mgr.x:Stopped 2024-03-04T07:37:53.151 INFO:teuthology.misc:Shutting down mon daemons... 2024-03-04T07:37:53.151 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-03-04T07:37:53.151 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:53.174 INFO:tasks.ceph.mon.a:Stopped 2024-03-04T07:37:53.174 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-03-04T07:37:53.174 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:53.205 INFO:tasks.ceph.mon.b:Stopped 2024-03-04T07:37:53.205 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-03-04T07:37:53.205 INFO:teuthology.orchestra.run:waiting for 300 2024-03-04T07:37:53.258 INFO:tasks.ceph.mon.c:Stopped 2024-03-04T07:37:53.258 INFO:tasks.ceph:Checking cluster log for badness... 2024-03-04T07:37:53.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | head -n 1 2024-03-04T07:37:53.289 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:37:53.290 DEBUG:teuthology.orchestra.run.smithi064:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 2024-03-04T07:37:53.492 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:37:53.493 DEBUG:teuthology.orchestra.run.smithi064:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 2024-03-04T07:37:53.625 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:37:53.626 DEBUG:teuthology.orchestra.run.smithi064:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 2024-03-04T07:37:53.700 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:37:53.700 DEBUG:teuthology.orchestra.run.smithi064:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2024-03-04T07:37:53.758 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-4 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:37:53.759 DEBUG:teuthology.orchestra.run.smithi066:> sync && sudo umount -f /var/lib/ceph/osd/ceph-4 2024-03-04T07:37:53.922 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-5 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:37:53.923 DEBUG:teuthology.orchestra.run.smithi066:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 2024-03-04T07:37:53.996 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:37:53.996 DEBUG:teuthology.orchestra.run.smithi066:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 2024-03-04T07:37:54.061 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-7 on ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:37:54.062 DEBUG:teuthology.orchestra.run.smithi066:> sync && sudo umount -f /var/lib/ceph/osd/ceph-7 2024-03-04T07:37:54.118 INFO:tasks.ceph:Archiving mon data... 2024-03-04T07:37:54.118 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/lib/ceph/mon/ceph-a to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/data/mon.a.tgz 2024-03-04T07:37:54.119 DEBUG:teuthology.orchestra.run.smithi064:> mktemp 2024-03-04T07:37:54.134 INFO:teuthology.orchestra.run.smithi064.stdout:/tmp/tmp.OsXqKIZFBR 2024-03-04T07:37:54.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-a -- . > /tmp/tmp.OsXqKIZFBR 2024-03-04T07:37:54.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0666 /tmp/tmp.OsXqKIZFBR 2024-03-04T07:37:54.370 DEBUG:teuthology.orchestra.remote:smithi064:/tmp/tmp.OsXqKIZFBR is 392KB 2024-03-04T07:37:54.466 DEBUG:teuthology.orchestra.run.smithi064:> rm -fr /tmp/tmp.OsXqKIZFBR 2024-03-04T07:37:54.481 DEBUG:teuthology.misc:Transferring archived files from smithi066:/var/lib/ceph/mon/ceph-b to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/data/mon.b.tgz 2024-03-04T07:37:54.481 DEBUG:teuthology.orchestra.run.smithi066:> mktemp 2024-03-04T07:37:54.499 INFO:teuthology.orchestra.run.smithi066.stdout:/tmp/tmp.oURPLQd1uH 2024-03-04T07:37:54.500 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-b -- . > /tmp/tmp.oURPLQd1uH 2024-03-04T07:37:54.666 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 0666 /tmp/tmp.oURPLQd1uH 2024-03-04T07:37:54.757 DEBUG:teuthology.orchestra.remote:smithi066:/tmp/tmp.oURPLQd1uH is 397KB 2024-03-04T07:37:54.852 DEBUG:teuthology.orchestra.run.smithi066:> rm -fr /tmp/tmp.oURPLQd1uH 2024-03-04T07:37:54.867 DEBUG:teuthology.misc:Transferring archived files from smithi066:/var/lib/ceph/mon/ceph-c to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/data/mon.c.tgz 2024-03-04T07:37:54.868 DEBUG:teuthology.orchestra.run.smithi066:> mktemp 2024-03-04T07:37:54.923 INFO:teuthology.orchestra.run.smithi066.stdout:/tmp/tmp.WdUYyfoYQO 2024-03-04T07:37:54.923 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-c -- . > /tmp/tmp.WdUYyfoYQO 2024-03-04T07:37:55.068 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 0666 /tmp/tmp.WdUYyfoYQO 2024-03-04T07:37:55.157 DEBUG:teuthology.orchestra.remote:smithi066:/tmp/tmp.WdUYyfoYQO is 397KB 2024-03-04T07:37:55.225 DEBUG:teuthology.orchestra.run.smithi066:> rm -fr /tmp/tmp.WdUYyfoYQO 2024-03-04T07:37:55.240 INFO:tasks.ceph:Cleaning ceph cluster... 2024-03-04T07:37:55.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/ceph.data /home/ubuntu/cephtest/ceph.monmap /home/ubuntu/cephtest/../*.pid 2024-03-04T07:37:55.243 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/ceph.data /home/ubuntu/cephtest/ceph.monmap /home/ubuntu/cephtest/../*.pid 2024-03-04T07:37:55.326 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-03-04T07:37:55.365 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-03-04T07:37:55.365 INFO:tasks.ceph:Archiving crash dumps... 2024-03-04T07:37:55.367 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/lib/ceph/crash to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/remote/smithi064/crash 2024-03-04T07:37:55.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-03-04T07:37:55.397 DEBUG:teuthology.misc:Transferring archived files from smithi066:/var/lib/ceph/crash to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/remote/smithi066/crash 2024-03-04T07:37:55.398 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-03-04T07:37:55.435 INFO:tasks.ceph:Shutting down logrotate 2024-03-04T07:37:55.435 DEBUG:teuthology.orchestra.run.smithi064:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:55.441 DEBUG:teuthology.orchestra.run.smithi066:> sudo logrotate /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:55.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:55.562 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm /etc/logrotate.d/ceph-test.conf 2024-03-04T07:37:55.593 INFO:tasks.ceph:Compressing logs... 2024-03-04T07:37:55.593 DEBUG:teuthology.orchestra.run.smithi064:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-04T07:37:55.604 DEBUG:teuthology.orchestra.run.smithi066:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-04T07:37:55.692 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.6.log 2024-03-04T07:37:55.692 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.3.44823.log 2024-03-04T07:37:55.692 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.1.44371.log 2024-03-04T07:37:55.692 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-osd.6.log: /var/log/ceph/ceph-client.3.44823.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38004.log 2024-03-04T07:37:55.692 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.1.44371.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38124.log 2024-03-04T07:37:55.692 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38004.log: gzip -5 --verbose -- /var/log/ceph/ceph.audit.log 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr: 90.3% -- replaced with /var/log/ceph/ceph-client.3.44823.log.gz 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.38004.log.gz 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38124.log: 89.3% -- replaced with /var/log/ceph/ceph-client.1.44371.log.gzgzip 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr: -5 --verbose -- /var/log/ceph/ceph-mds.b.log 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.38124.log.gz 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.4.log 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph.audit.log: /var/log/ceph/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38244.log 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.7.log 2024-03-04T07:37:55.693 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38244.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.5.log 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.38244.log.gz 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38204.log 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38084.log 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38204.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38044.log 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.38204.log.gz 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38084.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.38084.log.gz 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mgr.x.log 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38044.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38284.log 2024-03-04T07:37:55.694 INFO:teuthology.orchestra.run.smithi066.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.38044.log.gz 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/ceph.log 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38284.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.38284.log.gz 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38164.log 93.7% -- replaced with /var/log/ceph/ceph.audit.log.gz 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr: 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.b.log 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-client.admin.38164.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.38164.log.gz 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mds.d.log 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-mon.b.log: 92.6% -- replaced with /var/log/ceph/ceph.log.gz 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr: 94.3% -- replaced with /var/log/ceph/ceph-mgr.x.log.gz 2024-03-04T07:37:55.695 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mon.c.log 2024-03-04T07:37:55.717 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ceph-mds.d.log: /var/log/ceph/ceph-mon.c.log: 92.0% -- replaced with /var/log/ceph/ceph-mds.b.log.gz 2024-03-04T07:37:55.778 INFO:teuthology.orchestra.run.smithi066.stderr: 92.6% -- replaced with /var/log/ceph/ceph-mds.d.log.gz 2024-03-04T07:37:55.915 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46362.log 2024-03-04T07:37:55.915 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80931.log 2024-03-04T07:37:55.915 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46362.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81621.log 2024-03-04T07:37:55.916 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64583.log 2024-03-04T07:37:55.916 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80931.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.46362.log.gz 2024-03-04T07:37:55.916 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81621.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81141.log 2024-03-04T07:37:55.916 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.80931.log.gz 2024-03-04T07:37:55.916 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-client.admin.64583.log: -5 --verbose -- /var/log/ceph/ceph-client.admin.84237.log 2024-03-04T07:37:55.916 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.81621.log.gz 2024-03-04T07:37:55.916 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81141.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54816.log 2024-03-04T07:37:55.917 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.64583.log.gz 2024-03-04T07:37:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84237.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.81141.log.gz 2024-03-04T07:37:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49206.log 2024-03-04T07:37:55.917 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.84237.log.gz 2024-03-04T07:37:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54816.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.0.60105.log 2024-03-04T07:37:55.917 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.54816.log.gz/var/log/ceph/ceph-client.admin.49206.log: 2024-03-04T07:37:55.917 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49038.log 2024-03-04T07:37:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.60105.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.49206.log.gz 2024-03-04T07:37:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59250.log 2024-03-04T07:37:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49038.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63976.log 2024-03-04T07:37:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59250.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.49038.log.gz 2024-03-04T07:37:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46278.log 2024-03-04T07:37:55.918 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.59250.log.gz 2024-03-04T07:37:55.918 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63976.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77627.log 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46278.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62435.log 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.63976.log.gz 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77627.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.46278.log.gz 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67351.log 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.77627.log.gz 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62435.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85693.log 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67351.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82101.log 2024-03-04T07:37:55.919 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.62435.log.gz 2024-03-04T07:37:55.920 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85693.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.67351.log.gz 2024-03-04T07:37:55.920 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76426.log 2024-03-04T07:37:55.920 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.85693.log.gz 2024-03-04T07:37:55.920 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82101.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83817.log 2024-03-04T07:37:55.920 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76426.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.a.log 2024-03-04T07:37:55.920 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.82101.log.gz 2024-03-04T07:37:55.920 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83817.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.76426.log.gz 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79048.log 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.83817.log.gz 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55667.log 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79048.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54896.log 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55667.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.79048.log.gz 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73459.log 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76189.log 2024-03-04T07:37:55.921 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.55667.log.gz 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54896.log: /var/log/ceph/ceph-client.admin.73459.log: gzip 88.8% -5 -- replaced with /var/log/ceph/ceph-client.admin.54896.log.gz --verbose 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.54511.log 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.73459.log.gz 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75765.log 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76189.log: /var/log/ceph/ceph-client.admin.54511.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60671.log 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.76189.log.gz 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75765.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.54511.log.gz 2024-03-04T07:37:55.922 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81957.log 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.75765.log.gz 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60629.log 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60671.log: /var/log/ceph/ceph-client.admin.81957.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69378.log 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.60671.log.gz 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60629.log: gzip -5 --verbose -- 88.1% /var/log/ceph/ceph-client.admin.76907.log 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.81957.log.gz 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.60629.log.gz 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69378.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63487.log 2024-03-04T07:37:55.923 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.69378.log.gz 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85956.log 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76907.log: /var/log/ceph/ceph-client.admin.63487.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63001.log 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.76907.log.gz 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85956.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.63487.log.gz 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67435.log 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77799.log 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63001.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.85956.log.gz 2024-03-04T07:37:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67435.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76804.log 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.63001.log.gz 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.67435.log.gz 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77799.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57292.log 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76804.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61152.log 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.77799.log.gz 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57292.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.76804.log.gz 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67948.log 2024-03-04T07:37:55.925 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.57292.log.gz 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61152.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67610.log 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57460.log 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.61152.log.gz 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67948.log: /var/log/ceph/ceph-client.admin.67610.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54114.log 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.67948.log.gz 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57460.log: 88.9%gzip -- replaced with /var/log/ceph/ceph-client.admin.67610.log.gz -5 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.46404.log 2024-03-04T07:37:55.926 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54114.log: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.57460.log.gz 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59518.log 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.54114.log.gz 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46404.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84931.log 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49736.log 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59518.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.46404.log.gz 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84931.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80073.log 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.59518.log.gz 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.84931.log.gz 2024-03-04T07:37:55.927 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72168.log 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49736.log: /var/log/ceph/ceph-client.admin.80073.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50712.log 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.49736.log.gz 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72168.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.80073.log.gz 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60923.log 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.72168.log.gz 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73257.log 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50712.log: /var/log/ceph/ceph-client.admin.60923.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62000.log 2024-03-04T07:37:55.928 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.50712.log.gz 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73257.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46110.log 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.60923.log.gz 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.73257.log.gz 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62000.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64415.log 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.62000.log.gz 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46110.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73720.log 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.46110.log.gz 2024-03-04T07:37:55.929 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64415.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77376.log 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73720.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.64415.log.gz 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52571.log 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45018.log 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.73720.log.gz 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77376.log: /var/log/ceph/ceph-client.admin.52571.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.77376.log.gz 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51714.log 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.52571.log.gz 2024-03-04T07:37:55.930 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52085.log 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45018.log: /var/log/ceph/ceph-client.admin.51714.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53066.log 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.45018.log.gz 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52085.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.51714.log.gz 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48591.log 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75259.log 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53066.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.52085.log.gz 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48591.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79653.log 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr: 90.3% -- replaced with /var/log/ceph/ceph-client.admin.53066.log.gz 2024-03-04T07:37:55.931 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.48591.log.gz 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75259.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84195.log 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79653.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51672.log 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.75259.log.gz 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84195.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.79653.log.gz 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60755.log 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.84195.log.gz 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51672.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81225.log 2024-03-04T07:37:55.932 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50250.log 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.51672.log.gz 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60755.log: /var/log/ceph/ceph-client.admin.81225.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70712.log 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.60755.log.gz 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.81225.log.gz 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83127.log 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50250.log: /var/log/ceph/ceph-client.admin.70712.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57021.log 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.50250.log.gz 2024-03-04T07:37:55.933 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83127.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.70712.log.gz 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69776.log 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.83127.log.gz 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57021.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57929.log 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.57021.log.gz 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69776.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82857.log 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.69776.log.gz 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55775.log 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57929.log: /var/log/ceph/ceph-client.admin.82857.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61829.log 2024-03-04T07:37:55.934 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.57929.log.gz 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55775.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.82857.log.gz 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51913.log 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr: 89.8% -- replaced with /var/log/ceph/ceph-client.admin.55775.log.gz 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61829.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57716.log 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53387.log 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51913.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.61829.log.gz 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57716.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77296.log 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.51913.log.gz 2024-03-04T07:37:55.935 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.57716.log.gz 2024-03-04T07:37:55.936 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53387.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51377.log 2024-03-04T07:37:55.936 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77296.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74009.log 2024-03-04T07:37:55.936 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.53387.log.gz 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51377.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.77296.log.gz 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45704.log 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.51377.log.gz 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74009.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61914.log 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63808.log 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45704.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.74009.log.gz 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61914.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58364.log 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.45704.log.gz 2024-03-04T07:37:55.937 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.61914.log.gz 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45366.log 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63808.log: /var/log/ceph/ceph-client.admin.58364.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62477.log 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.63808.log.gz 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.58364.log.gz 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45366.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61488.log 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.45366.log.gz 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61320.log 2024-03-04T07:37:55.938 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62477.log: /var/log/ceph/ceph-client.admin.61488.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45695.log 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.62477.log.gz 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61320.log: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.61488.log.gz 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83715.log 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84847.log 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45695.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.61320.log.gz 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83715.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.45695.log.gz 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74613.log 2024-03-04T07:37:55.939 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.83715.log.gz 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55369.log 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84847.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49494.log/var/log/ceph/ceph-client.admin.74613.log: 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.84847.log.gz 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55369.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79383.log 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.74613.log.gz 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.55369.log.gz 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49494.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80595.log 2024-03-04T07:37:55.940 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.49494.log.gz 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56727.log 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79383.log: /var/log/ceph/ceph-client.admin.80595.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78887.log 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.79383.log.gz 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56727.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.80595.log.gz 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59334.log 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.56727.log.gz 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78887.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82479.log 2024-03-04T07:37:55.941 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.78887.log.gz 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59334.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64018.log 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63892.log 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82479.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.59334.log.gz 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64018.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76147.log 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.82479.log.gz 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.64018.log.gz 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84568.log 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63892.log: /var/log/ceph/ceph-client.admin.76147.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53793.log 2024-03-04T07:37:55.942 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.63892.log.gz 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84568.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.76147.log.gz 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44284.log 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.84568.log.gz 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53793.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62179.log 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48321.log 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.53793.log.gz 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44284.log: /var/log/ceph/ceph-client.admin.62179.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.44284.log.gz 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78659.log 2024-03-04T07:37:55.943 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.62179.log.gz 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79947.log 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48321.log: /var/log/ceph/ceph-client.admin.78659.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86167.log 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.48321.log.gz 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.78659.log.gz 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79947.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69652.log 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.79947.log.gz 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68074.log 2024-03-04T07:37:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86167.log: /var/log/ceph/ceph-client.admin.69652.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64499.log 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.86167.log.gz 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68074.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.69652.log.gz 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79341.log 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82563.log 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.68074.log.gz 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64499.log: /var/log/ceph/ceph-client.admin.79341.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66339.log 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.64499.log.gz 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.79341.log.gz 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85369.log/var/log/ceph/ceph-client.admin.82563.log: 2024-03-04T07:37:55.945 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66339.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44934.log 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.82563.log.gz 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85369.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69714.log 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.66339.log.gz 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0%/var/log/ceph/ceph-client.admin.44934.log: -- replaced with /var/log/ceph/ceph-client.admin.85369.log.gz 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53988.log 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.44934.log.gz 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73543.log 2024-03-04T07:37:55.946 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69714.log: /var/log/ceph/ceph-client.admin.53988.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57079.log 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.69714.log.gz 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73543.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.53988.log.gz 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75870.log 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57079.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65463.log 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.73543.log.gz 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75870.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.57079.log.gz 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85201.log 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61871.log 2024-03-04T07:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65463.log: /var/log/ceph/ceph-client.admin.85201.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58448.log 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.65463.log.gz 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr: 92.4% -- replaced with /var/log/ceph/ceph-client.admin.75870.log.gz 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.85201.log.gz 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61871.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81663.log 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.61871.log.gz 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85159.log 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58448.log: /var/log/ceph/ceph-client.admin.81663.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61109.log 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.58448.log.gz 2024-03-04T07:37:55.948 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85159.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.81663.log.gz 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58508.log 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61109.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.85159.log.gz 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75926.log 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.61109.log.gz 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59846.log 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58508.log: /var/log/ceph/ceph-client.admin.75926.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.0.74376.log 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.58508.log.gz 2024-03-04T07:37:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59846.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.75926.log.gz 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45942.log 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55104.log 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.59846.log.gz 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.74376.log: /var/log/ceph/ceph-client.admin.45942.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81285.log 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.45942.log.gz 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55104.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71620.log 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45060.log 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81285.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.55104.log.gz 2024-03-04T07:37:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71620.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75110.log 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.81285.log.gz 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45060.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.32004.log 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.71620.log.gz 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.45060.log.gz 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75110.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79425.log 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81495.log 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph.tmp-client.admin.32004.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.75110.log.gz 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.32004.log.gz 2024-03-04T07:37:55.951 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79425.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83085.log 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81495.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.79425.log.gz 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55285.log 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.81495.log.gz 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83085.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65050.log 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63571.log 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55285.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.83085.log.gz 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65050.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77075.log 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.55285.log.gz 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.65050.log.gz 2024-03-04T07:37:55.952 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63571.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70964.log 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70880.log 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3%/var/log/ceph/ceph-client.admin.77075.log: -- replaced with /var/log/ceph/ceph-client.admin.63571.log.gz 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70964.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73317.log 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.77075.log.gz 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84652.log 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.70964.log.gz 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70880.log: /var/log/ceph/ceph-client.admin.73317.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81327.log 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.70880.log.gz 2024-03-04T07:37:55.953 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84652.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56643.log 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.73317.log.gz 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr: 93.4% -- replaced with /var/log/ceph/ceph-client.0.74376.log.gz 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81327.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56979.log 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.84652.log.gz 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.81327.log.gz 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56643.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75703.log 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81453.log 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56979.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.56643.log.gz 2024-03-04T07:37:55.954 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75703.log: gzip -5 --verbose 89.5% -- -- replaced with /var/log/ceph/ceph-client.admin.56979.log.gz /var/log/ceph/ceph-client.admin.50796.log 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.75703.log.gz 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53263.log 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81453.log: /var/log/ceph/ceph-client.admin.50796.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44850.log 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.81453.log.gz 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53263.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.50796.log.gz 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71277.log 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.53263.log.gz 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44850.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82185.log 2024-03-04T07:37:55.955 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70024.log 2024-03-04T07:37:55.956 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6%/var/log/ceph/ceph-client.admin.71277.log: -- replaced with /var/log/ceph/ceph-client.admin.44850.log.gz 2024-03-04T07:37:55.956 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82185.log: 89.2%gzip -- replaced with /var/log/ceph/ceph-client.admin.71277.log.gz 2024-03-04T07:37:55.956 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.56490.log 2024-03-04T07:37:55.956 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.82185.log.gz 2024-03-04T07:37:55.956 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51844.log 2024-03-04T07:37:55.956 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70024.log: /var/log/ceph/ceph-client.admin.56490.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53904.log 2024-03-04T07:37:55.956 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.70024.log.gz 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51844.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.56490.log.gz 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58322.log 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.51844.log.gz 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53904.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59644.log 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.53904.log.gz 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65589.log 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58322.log: /var/log/ceph/ceph-client.admin.59644.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80241.log 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.58322.log.gz 2024-03-04T07:37:55.957 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65589.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.59644.log.gz 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79779.log 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57758.log 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80241.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.65589.log.gz 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79779.log: gzip -5 88.7% --verbose -- replaced with /var/log/ceph/ceph-client.admin.80241.log.gz -- 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.51504.log 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.79779.log.gz 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81705.log 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57758.log: /var/log/ceph/ceph-client.admin.51504.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70433.log 2024-03-04T07:37:55.958 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.57758.log.gz 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81705.log: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.51504.log.gz 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48717.log 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.81705.log.gz 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70433.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85033.log 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71701.log 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48717.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.70433.log.gz 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85033.log: gzip 88.1% -5 --verbose -- replaced with /var/log/ceph/ceph-client.admin.48717.log.gz 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.49578.log 2024-03-04T07:37:55.959 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.85033.log.gz 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46446.log 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71701.log: 93.2% -- replaced with /var/log/ceph/ceph-client.0.60105.log.gz 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49578.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82899.log 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.71701.log.gz 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46446.log: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.49578.log.gz 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66912.log 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.46446.log.gz 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82899.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50130.log 2024-03-04T07:37:55.960 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66912.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74835.log 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.82899.log.gz 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.66912.log.gz 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50130.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51462.log 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74835.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.50130.log.gz 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86377.log 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51462.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71109.log 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr: 90.2% -- replaced with /var/log/ceph/ceph-client.admin.74835.log.gz 2024-03-04T07:37:55.961 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86377.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.51462.log.gz 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67309.log 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71109.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.86377.log.gz 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57376.log 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1%/var/log/ceph/ceph-client.admin.67309.log: -- replaced with /var/log/ceph/ceph-client.admin.71109.log.gz 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54198.log 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.67309.log.gz 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57376.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76510.log 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54198.log: 87.9%gzip -- replaced with /var/log/ceph/ceph-client.admin.57376.log.gz 2024-03-04T07:37:55.962 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.68966.log 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.54198.log.gz 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76510.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67038.log 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.76510.log.gz 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68966.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61236.log 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67038.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.68966.log.gz 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63529.log 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61236.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.67038.log.gz 2024-03-04T07:37:55.963 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69920.log 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.61236.log.gz 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63529.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81369.log 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69920.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83169.log 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.63529.log.gz 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.69920.log.gz 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81369.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83295.log 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.81369.log.gz 2024-03-04T07:37:55.964 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83169.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53139.log 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83295.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64457.log 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.83169.log.gz 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1%/var/log/ceph/ceph-client.admin.53139.log: -- replaced with /var/log/ceph/ceph-client.admin.83295.log.gz 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56448.log 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.53139.log.gz 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64457.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69096.log 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56448.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54240.log 2024-03-04T07:37:55.965 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.64457.log.gz 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.56448.log.gz 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69096.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77201.log 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.69096.log.gz 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54240.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64625.log 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77201.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65207.log 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.54240.log.gz 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.77201.log.gz 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64625.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66744.log 2024-03-04T07:37:55.966 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.64625.log.gz 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65207.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84321.log 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66744.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53486.log 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.65207.log.gz 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84321.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.66744.log.gz 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66828.log 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.84321.log.gz 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53486.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57800.log 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66828.log: 87.8%gzip -- replaced with /var/log/ceph/ceph-client.admin.53486.log.gz 2024-03-04T07:37:55.967 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.86251.log 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.66828.log.gz 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57800.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85609.log 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.57800.log.gz 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86251.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84484.log 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85609.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.86251.log.gz 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76384.log 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84484.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58592.log 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.85609.log.gz 2024-03-04T07:37:55.968 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.84484.log.gz 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76384.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48069.log 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.76384.log.gz 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58592.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85828.log 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48069.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61446.log 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.58592.log.gz 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.48069.log.gz 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85828.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84069.log 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.85828.log.gz 2024-03-04T07:37:55.969 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61446.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51006.log 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.61446.log.gz 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84069.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53325.log 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51006.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83043.log 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.84069.log.gz 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53325.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.51006.log.gz 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44425.log 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.53325.log.gz 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83043.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53667.log 2024-03-04T07:37:55.970 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44425.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79006.log 2024-03-04T07:37:55.971 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.83043.log.gz 2024-03-04T07:37:55.971 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53667.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.44425.log.gz 2024-03-04T07:37:55.971 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54030.log 2024-03-04T07:37:55.971 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.53667.log.gz 2024-03-04T07:37:55.971 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79006.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74792.log 2024-03-04T07:37:55.971 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54030.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79989.log 2024-03-04T07:37:55.971 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.79006.log.gz 2024-03-04T07:37:55.972 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74792.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.54030.log.gz 2024-03-04T07:37:55.972 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77925.log 2024-03-04T07:37:55.972 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.74792.log.gz 2024-03-04T07:37:55.972 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79989.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45984.log 2024-03-04T07:37:55.972 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77925.log: gzip -5 88.2% --verbose -- -- replaced with /var/log/ceph/ceph-client.admin.79989.log.gz /var/log/ceph/ceph-client.admin.44468.log 2024-03-04T07:37:55.972 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:55.972 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45984.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.77925.log.gz 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51630.log 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.45984.log.gz 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44468.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73173.log 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51630.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.44468.log.gz 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79821.log 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7%/var/log/ceph/ceph-client.admin.73173.log: -- replaced with /var/log/ceph/ceph-client.admin.51630.log.gz 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78845.log 2024-03-04T07:37:55.973 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.73173.log.gz 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79821.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81747.log 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78845.log: gzip 87.5% -5 -- replaced with /var/log/ceph/ceph-client.admin.79821.log.gz --verbose 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.49962.log 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81747.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.78845.log.gz 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47817.log 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.81747.log.gz 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49962.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86125.log 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.47817.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.49962.log.gz 2024-03-04T07:37:55.974 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78761.log 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.47817.log.gz 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86125.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84363.log 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.86125.log.gz 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78761.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44680.log 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84363.log: gzip -5 --verbose 87.6% -- -- replaced with /var/log/ceph/ceph-client.admin.78761.log.gz /var/log/ceph/ceph-client.admin.83673.log 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44680.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.84363.log.gz 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74986.log 2024-03-04T07:37:55.975 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.44680.log.gz 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83673.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38330.log 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74986.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.83673.log.gz 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59722.log 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.38330.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.74986.log.gz 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.audit.log 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.38330.log.gz 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59722.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76231.log 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph.audit.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.59722.log.gz 2024-03-04T07:37:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79299.log 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76231.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82815.log 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79299.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.76231.log.gz 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62585.log 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.79299.log.gz 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56853.log 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82815.log: /var/log/ceph/ceph-client.admin.62585.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83547.log 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.82815.log.gz 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56853.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.62585.log.gz 2024-03-04T07:37:55.977 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78177.log 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.56853.log.gz 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83547.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62351.log 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.83547.log.gz 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65631.log 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78177.log: /var/log/ceph/ceph-client.admin.62351.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80031.log 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65631.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.62351.log.gz 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46026.log 2024-03-04T07:37:55.978 INFO:teuthology.orchestra.run.smithi064.stderr: 90.9% -- replaced with /var/log/ceph/ceph-client.admin.78177.log.gz 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57166.log 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80031.log: /var/log/ceph/ceph-client.admin.46026.log: 90.9% 89.4% -- replaced with /var/log/ceph/ceph-client.admin.65631.log.gz -- replaced with /var/log/ceph/ceph.audit.log.gz 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66423.log 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.80031.log.gz 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.46026.log.gz 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57166.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52463.log 2024-03-04T07:37:55.979 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66423.log: gzip -5 88.1% --verbose -- -- replaced with /var/log/ceph/ceph-client.admin.57166.log.gz 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.74051.log 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52463.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38080.log 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.66423.log.gz 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74051.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.52463.log.gz 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75445.log 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.38080.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50754.log 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr: 90.8% -- replaced with /var/log/ceph/ceph-client.admin.74051.log.gz 2024-03-04T07:37:55.980 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75445.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53430.log 2024-03-04T07:37:55.981 INFO:teuthology.orchestra.run.smithi064.stderr: 92.5% -- replaced with /var/log/ceph/ceph-client.admin.38080.log.gz 2024-03-04T07:37:55.981 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.75445.log.gz 2024-03-04T07:37:55.981 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58550.log 2024-03-04T07:37:55.981 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50754.log: /var/log/ceph/ceph-client.admin.53430.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66786.log 2024-03-04T07:37:55.981 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.50754.log.gz 2024-03-04T07:37:55.981 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58550.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76006.log 2024-03-04T07:37:55.981 INFO:teuthology.orchestra.run.smithi064.stderr: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.53430.log.gz 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66786.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.58550.log.gz 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80157.log 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.66786.log.gz 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76006.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66702.log 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.76006.log.gz 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80157.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83631.log 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66702.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.80157.log.gz 2024-03-04T07:37:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49122.log 2024-03-04T07:37:55.983 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0%/var/log/ceph/ceph-client.admin.83631.log: -- replaced with /var/log/ceph/ceph-client.admin.66702.log.gz 2024-03-04T07:37:55.983 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82227.log 2024-03-04T07:37:55.983 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.83631.log.gz 2024-03-04T07:37:55.983 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49122.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76949.log 2024-03-04T07:37:55.983 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82227.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.49122.log.gz 2024-03-04T07:37:55.983 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64236.log 2024-03-04T07:37:55.983 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.82227.log.gz 2024-03-04T07:37:55.984 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76949.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55541.log 2024-03-04T07:37:55.984 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64236.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66954.log 2024-03-04T07:37:55.984 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.76949.log.gz 2024-03-04T07:37:55.984 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55541.log: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.64236.log.gz 2024-03-04T07:37:55.984 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68386.log 2024-03-04T07:37:55.984 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.55541.log.gz 2024-03-04T07:37:55.984 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66954.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70796.log 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.66954.log.gz 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68386.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55146.log 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70796.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.68386.log.gz 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78051.log 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.70796.log.gz 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55146.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56601.log 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.55146.log.gz 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78051.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51546.log 2024-03-04T07:37:55.985 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56601.log: gzip 88.1% -5 -- replaced with /var/log/ceph/ceph-client.admin.78051.log.gz --verbose 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.53201.log 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.56601.log.gz 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51546.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62393.log 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.51546.log.gz 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53201.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61404.log 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62393.log: gzip 88.4% -5 -- replaced with /var/log/ceph/ceph-client.admin.53201.log.gz --verbose 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.44892.log 2024-03-04T07:37:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61404.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.62393.log.gz 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82731.log 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.61404.log.gz 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44892.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52253.log 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82731.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75383.log 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.44892.log.gz 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52253.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.82731.log.gz 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45696.log 2024-03-04T07:37:55.987 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.52253.log.gz 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75383.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72486.log 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45696.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49778.log 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.75383.log.gz 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72486.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.45696.log.gz 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50628.log 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.72486.log.gz 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49778.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60797.log 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50628.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63655.log 88.6% 2024-03-04T07:37:55.988 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.49778.log.gz 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60797.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.50628.log.gz 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.0.52830.log 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.60797.log.gz 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63655.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77841.log 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.52830.log: gzip 89.4% -- replaced with /var/log/ceph/ceph-client.admin.63655.log.gz -5 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.50922.log 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77841.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66660.log 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.77841.log.gz 2024-03-04T07:37:55.989 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50922.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57885.log 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.50922.log.gz 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66660.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66618.log 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.66660.log.gz 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50208.log 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57885.log: /var/log/ceph/ceph-client.admin.66618.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77117.log 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.57885.log.gz 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50208.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.66618.log.gz 2024-03-04T07:37:55.990 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70586.log 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77117.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.50208.log.gz 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67183.log 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.77117.log.gz 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70586.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69336.log 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.70586.log.gz 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61362.log 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67183.log: /var/log/ceph/ceph-client.admin.69336.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66507.log 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.67183.log.gz 2024-03-04T07:37:55.991 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.69336.log.gz 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61362.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61049.log 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46194.log 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66507.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.61362.log.gz 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61049.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.66507.log.gzgzip 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.70628.log 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- 87.8% /var/log/ceph/ceph-client.admin.67818.log -- replaced with /var/log/ceph/ceph-client.admin.61049.log.gz 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:55.992 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46194.log: /var/log/ceph/ceph-client.admin.70628.log: gzip -5 --verbose -- 88.2% /var/log/ceph/ceph-client.admin.46632.log 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.46194.log.gz 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73962.log 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67818.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.70628.log.gz 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46632.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.67818.log.gz 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48153.log 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.46632.log.gz 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82959.log 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73962.log: /var/log/ceph/ceph-client.admin.48153.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50088.log 2024-03-04T07:37:55.993 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.73962.log.gz 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.48153.log.gz 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82959.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82647.log 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50088.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.82959.log.gz 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81873.log 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.50088.log.gz 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82647.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68260.log 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81873.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.82647.log.gz 2024-03-04T07:37:55.994 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48465.log 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.81873.log.gz 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75153.log 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68260.log: /var/log/ceph/ceph-client.admin.48465.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45358.log 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.68260.log.gz 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75153.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.48465.log.gz 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48237.log 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46068.log 2024-03-04T07:37:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45358.log: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.75153.log.gz 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48237.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75577.log 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.45358.log.gz 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51132.log 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46068.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.48237.log.gz 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75577.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72084.log 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.46068.log.gz 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0%/var/log/ceph/ceph-client.admin.51132.log: -- replaced with /var/log/ceph/ceph-client.admin.75577.log.gz 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65547.log 2024-03-04T07:37:55.996 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72084.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.51132.log.gz 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66870.log 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.72084.log.gz 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47665.log 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65547.log: /var/log/ceph/ceph-client.admin.66870.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64965.log 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.65547.log.gz 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.47665.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.66870.log.gz 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52127.log 2024-03-04T07:37:55.997 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.47665.log.gz 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64965.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.0.56034.log 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.64965.log.gz 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80973.log 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52127.log: /var/log/ceph/ceph-client.0.56034.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80553.log 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80973.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.52127.log.gz 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49904.log 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.80973.log.gz 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80553.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46236.log 2024-03-04T07:37:55.998 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49904.log: gzip -5 89.3% --verbose -- -- replaced with /var/log/ceph/ceph-client.admin.80553.log.gz /var/log/ceph/ceph-client.admin.86335.log 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.49904.log.gz 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46236.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77585.log 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86335.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.46236.log.gz 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58066.log 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-client.admin.77585.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.86335.log.gz 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.62309.log 2024-03-04T07:37:55.999 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58066.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.77585.log.gz 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58108.log 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.58066.log.gz 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62309.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53751.log 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.62309.log.gz 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58108.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63355.log 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53751.log: 88.6%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51588.log 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.58108.log.gz 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.53751.log.gz 2024-03-04T07:37:56.000 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63355.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54469.log 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51588.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.63355.log.gz 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75048.log 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.51588.log.gz 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54469.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84763.log 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75048.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64923.log 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.54469.log.gz 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84763.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.75048.log.gz 2024-03-04T07:37:56.001 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59158.log 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64923.log: gzip 88.1% -5 -- replaced with /var/log/ceph/ceph-client.admin.84763.log.gz --verbose 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.83943.log 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.64923.log.gz 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59158.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.1.log 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83943.log: 89.4%gzip -- replaced with /var/log/ceph/ceph-client.admin.59158.log.gz -5 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.71958.log 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.83943.log.gz 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85327.log 2024-03-04T07:37:56.002 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71958.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60839.log 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.71958.log.gz 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85327.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81579.log 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60839.log: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.85327.log.gz 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52421.log 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.60839.log.gz 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81579.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54679.log 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.81579.log.gz 2024-03-04T07:37:56.003 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52421.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.0.78502.log 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54679.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.52421.log.gz 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68428.log 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.54679.log.gz 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.78502.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59476.log 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68428.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48675.log 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59476.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67775.log 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr: 91.2% -- replaced with /var/log/ceph/ceph-client.admin.68428.log.gz 88.2% -- replaced with /var/log/ceph/ceph-client.admin.59476.log.gz 2024-03-04T07:37:56.004 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48675.log: gzip -5 87.8% -- replaced with /var/log/ceph/ceph-client.admin.48675.log.gz 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.45361.log 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67775.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76864.log 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.67775.log.gz 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45361.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86564.log 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.45361.log.gz 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76864.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52337.log 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr:gzip 88.9% -5 --verbose -- /var/log/ceph/ceph-client.admin.82437.log 2024-03-04T07:37:56.005 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.76864.log.gz 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86564.log: /var/log/ceph/ceph-client.admin.52337.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72294.log 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% 89.0%/var/log/ceph/ceph-client.admin.82437.log: -- replaced with /var/log/ceph/ceph-client.admin.86564.log.gz 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.52337.log.gz 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.82437.log.gz 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85736.log 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72294.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65249.log 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85736.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.72294.log.gz 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80847.log 2024-03-04T07:37:56.006 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.85736.log.gz 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49862.log 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65249.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67225.log 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7%/var/log/ceph/ceph-client.admin.80847.log: -- replaced with /var/log/ceph/ceph-client.admin.65249.log.gz 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49862.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.80847.log.gz 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr: 94.5% -- replaced with /var/log/ceph/ceph-client.0.78502.log.gz 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.49862.log.gz 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80511.log 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67225.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49326.log 2024-03-04T07:37:56.007 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.67225.log.gz 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80511.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81537.log 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-client.admin.49326.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.80511.log.gz 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.80199.log 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81537.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.49326.log.gz 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79551.log 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.81537.log.gz 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80199.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76105.log 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.80199.log.gz 2024-03-04T07:37:56.008 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79551.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57124.log 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9%/var/log/ceph/ceph-client.admin.76105.log: gzip -- replaced with /var/log/ceph/ceph-client.admin.79551.log.gz 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.84889.log 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57124.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.76105.log.gz 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64102.log 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.57124.log.gz 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84889.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49080.log 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4%/var/log/ceph/ceph-client.admin.64102.log: -- replaced with /var/log/ceph/ceph-client.admin.84889.log.gzgzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45367.log 2024-03-04T07:37:56.009 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49080.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81915.log 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.64102.log.gz 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45367.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65094.log 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.49080.log.gz 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr:gzip 87.9% -- replaced with /var/log/ceph/ceph-client.admin.45367.log.gz -5 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.45702.log 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81915.log: /var/log/ceph/ceph-client.admin.65094.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56895.log 2024-03-04T07:37:56.010 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.81915.log.gz/var/log/ceph/ceph-client.admin.45702.log: 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.65094.log.gz 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45362.log 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.45702.log.gz 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56895.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57418.log 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45362.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.56895.log.gz 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44510.log 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57418.log: 88.0%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83505.log 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.45362.log.gz 2024-03-04T07:37:56.011 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44510.log: gzip -5 --verbose -- 87.8% /var/log/ceph/ceph-client.admin.58280.log 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.57418.log.gz 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 89.2% -- replaced with /var/log/ceph/ceph-client.admin.44510.log.gz 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83505.log: --verbose -- /var/log/ceph/ceph-client.admin.71743.log 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.83505.log.gz 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.0.47048.log 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58280.log: gzip/var/log/ceph/ceph-client.admin.71743.log: -5 --verbose -- 88.5% -- replaced with /var/log/ceph/ceph-client.admin.58280.log.gz 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.77543.log 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.47048.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.71743.log.gz 2024-03-04T07:37:56.012 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56322.log 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77543.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70391.log 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% 90.1% -- replaced with /var/log/ceph/ceph-client.0.47048.log.gz 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.77543.log.gz/var/log/ceph/ceph-client.admin.56322.log: 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51174.log 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.56322.log.gz 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-client.admin.70391.log: -5 --verbose -- /var/log/ceph/ceph-client.admin.69838.log 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51174.log: gzip -5 --verbose -- 88.1% -- replaced with /var/log/ceph/ceph-client.admin.70391.log.gz 2024-03-04T07:37:56.013 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.78964.log 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.51174.log.gz 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69838.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83901.log 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.69838.log.gz 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78964.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70349.log 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83901.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.78964.log.gz 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64373.log 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.83901.log.gz 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70349.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76991.log 2024-03-04T07:37:56.014 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64373.log: gzip 88.4% -- replaced with /var/log/ceph/ceph-client.admin.70349.log.gz 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.73215.log 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76991.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.64373.log.gz 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45900.log 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.76991.log.gz 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73215.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61194.log 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45900.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.73215.log.gz 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58024.log 88.0% -- replaced with /var/log/ceph/ceph-client.admin.45900.log.gz 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.015 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61194.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80679.log 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.61194.log.gz 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58024.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48423.log 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.58024.log.gz 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80679.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48111.log 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.80679.log.gz 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48423.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50586.log 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48111.log: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.48423.log.gz 2024-03-04T07:37:56.016 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54384.log 88.6% -- replaced with /var/log/ceph/ceph-client.admin.48111.log.gz 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50586.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59418.log 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.50586.log.gz 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54384.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54637.log 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59418.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.54384.log.gz 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62225.log 89.3% -- replaced with /var/log/ceph/ceph-client.admin.59418.log.gz 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.017 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54637.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78093.log 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.54637.log.gz 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62225.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85075.log 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78093.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.62225.log.gz 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73131.log 88.2% -- replaced with /var/log/ceph/ceph-client.admin.78093.log.gz 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85075.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65421.log 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.85075.log.gz 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73131.log: gzip -5 93.3% -- replaced with /var/log/ceph/ceph-client.0.56034.log.gz 2024-03-04T07:37:56.018 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.0.58959.log 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65421.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.73131.log.gz 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 88.1% --verbose -- -- replaced with /var/log/ceph/ceph-client.admin.65421.log.gz /var/log/ceph/ceph-client.admin.57208.log 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.58959.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47171.log 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57208.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73501.log 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.57208.log.gz 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.47171.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68116.log 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73501.log: 87.9%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73722.log 2024-03-04T07:37:56.019 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.47171.log.gz 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.73501.log.gz 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68116.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83775.log 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.68116.log.gz 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80325.log 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73722.log: gzip -5 --verbose --/var/log/ceph/ceph-client.admin.83775.log: /var/log/ceph/ceph-client.admin.82689.log 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.73722.log.gz 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80325.log: gzip -5 --verbose 88.1% -- replaced with /var/log/ceph/ceph-client.admin.83775.log.gz 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.59784.log 2024-03-04T07:37:56.020 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.80325.log.gz 2024-03-04T07:37:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82689.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49820.log 2024-03-04T07:37:56.021 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.82689.log.gz 2024-03-04T07:37:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59784.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57843.log 2024-03-04T07:37:56.021 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.59784.log.gz 2024-03-04T07:37:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49820.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80409.log 2024-03-04T07:37:56.021 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57843.log: 88.2%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81099.log 2024-03-04T07:37:56.021 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.49820.log.gz 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0%/var/log/ceph/ceph-client.admin.80409.log: -- replaced with /var/log/ceph/ceph-client.admin.57843.log.gz 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48812.log 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.80409.log.gzgzip 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.60375.log 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81099.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48279.log 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.81099.log.gz/var/log/ceph/ceph-client.admin.48812.log: 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60375.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67691.log 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48279.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82521.log 2024-03-04T07:37:56.022 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.60375.log.gz 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.48812.log.gz 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67691.log: gzip -5 88.0% -- replaced with /var/log/ceph/ceph-client.admin.48279.log.gz 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.64800.log 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82521.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.67691.log.gz 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- 88.7% -- replaced with /var/log/ceph/ceph-client.admin.82521.log.gz 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.44808.log 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64800.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71403.log 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44808.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.64800.log.gz 2024-03-04T07:37:56.023 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55327.log 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.44808.log.gz 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71403.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80115.log 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr:gzip 88.1% -- replaced with /var/log/ceph/ceph-client.admin.71403.log.gz 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55327.log: -5 --verbose -- /var/log/ceph/ceph-client.admin.70166.log 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80115.log: gzip -5 --verbose -- 89.3% -- replaced with /var/log/ceph/ceph-client.admin.55327.log.gz 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.66198.log 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.80115.log.gz 2024-03-04T07:37:56.024 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70166.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71540.log 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.70166.log.gz 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66198.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78135.log 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.66198.log.gz 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71540.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80469.log 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-client.admin.78135.log: -5 88.0% -- replaced with /var/log/ceph/ceph-client.admin.71540.log.gz 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.49536.log 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80469.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.78135.log.gz 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- 87.5% -- replaced with /var/log/ceph/ceph-client.admin.80469.log.gz 2024-03-04T07:37:56.025 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.68302.log 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49536.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72042.log 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.49536.log.gz 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68302.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.3.log 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.68302.log.gz 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72042.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84153.log 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-osd.3.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.72042.log.gz 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.76678.log 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84153.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48549.log 87.5% -- replaced with /var/log/ceph/ceph-client.admin.84153.log.gz 2024-03-04T07:37:56.026 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76678.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47943.log 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.76678.log.gz 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48549.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79905.log 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.48549.log.gz 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.47943.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76552.log 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.47943.log.gz 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79905.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86482.log 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.79905.log.gz 2024-03-04T07:37:56.027 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76552.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57973.log 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.76552.log.gz 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83859.log 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86482.log: /var/log/ceph/ceph-client.admin.57973.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.86482.log.gz 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.57973.log.gz 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78009.log 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83859.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47901.log 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78009.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.83859.log.gz 2024-03-04T07:37:56.028 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70265.log 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5%/var/log/ceph/ceph-client.admin.47901.log: -- replaced with /var/log/ceph/ceph-client.admin.78009.log.gz 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63169.log 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.47901.log.gz/var/log/ceph/ceph-client.admin.70265.log: 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60459.log 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.70265.log.gz 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63169.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75641.log 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.63169.log.gz 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60459.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65291.log 2024-03-04T07:37:56.029 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.60459.log.gz 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75641.log: gzip -5 87.7% -- replaced with /var/log/ceph/ceph-client.admin.75641.log.gz 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.50292.log 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65291.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83253.log 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.65291.log.gz/var/log/ceph/ceph-client.admin.50292.log: 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44552.log 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.50292.log.gz 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose --/var/log/ceph/ceph-client.admin.83253.log: /var/log/ceph/ceph-client.admin.77033.log 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44552.log: 87.7%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50880.log 2024-03-04T07:37:56.030 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.83253.log.gz 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr: 89.6% -- replaced with /var/log/ceph/ceph-client.admin.44552.log.gz 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77033.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68158.log 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50880.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45144.log 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.77033.log.gz 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68158.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.50880.log.gz 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45699.log 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.68158.log.gz 2024-03-04T07:37:56.031 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66996.log 2024-03-04T07:37:56.032 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45144.log: /var/log/ceph/ceph-client.admin.45699.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49452.log 2024-03-04T07:37:56.032 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66996.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.45144.log.gz 2024-03-04T07:37:56.032 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.45699.log.gz 2024-03-04T07:37:56.032 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 88.2% -- replaced with /var/log/ceph/ceph-client.admin.66996.log.gz 2024-03-04T07:37:56.032 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.44636.log 2024-03-04T07:37:56.032 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49452.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76762.log 89.7% -- replaced with /var/log/ceph/ceph-client.admin.49452.log.gz 2024-03-04T07:37:56.032 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44636.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48896.log 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.44636.log.gz 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76762.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57635.log 88.8% -- replaced with /var/log/ceph/ceph-client.admin.76762.log.gz 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48896.log: gzip 88.7% -- replaced with /var/log/ceph/ceph-client.admin.48896.log.gz 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.50838.log 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57635.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79863.log 2024-03-04T07:37:56.033 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5%/var/log/ceph/ceph-client.admin.50838.log: -- replaced with /var/log/ceph/ceph-client.admin.57635.log.gz 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80637.log 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.50838.log.gz/var/log/ceph/ceph-client.admin.79863.log: 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 93.3% --verbose -- /var/log/ceph/ceph-client.admin.69426.log 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.0.58959.log.gz 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.79863.log.gz 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80637.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85285.log 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.80637.log.gz 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69426.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61530.log 2024-03-04T07:37:56.034 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85285.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.69426.log.gz 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84526.log 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.85285.log.gz 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61530.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50376.log 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84526.log: gzip -5 --verbose 89.1% -- replaced with /var/log/ceph/ceph-client.admin.61530.log.gz 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.73417.log 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4%/var/log/ceph/ceph-client.admin.50376.log: -- replaced with /var/log/ceph/ceph-client.admin.84526.log.gz 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63766.log 2024-03-04T07:37:56.035 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.50376.log.gz 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73417.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50460.log 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.73417.log.gz 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63766.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70670.log 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.63766.log.gz 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50460.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82143.log 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70670.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.50460.log.gz 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59116.log 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.70670.log.gz 2024-03-04T07:37:56.036 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82143.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85411.log 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59116.log: gzip -5 --verbose -- 87.5% -- replaced with /var/log/ceph/ceph-client.admin.82143.log.gz 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.56191.log 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85411.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.59116.log.gz 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58406.log 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.85411.log.gz 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56191.log: /var/log/ceph/ceph-client.admin.58406.log: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.56191.log.gz 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86419.log 2024-03-04T07:37:56.037 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.58406.log.gz 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52379.log 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52211.log 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86419.log: /var/log/ceph/ceph-client.admin.52379.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62137.log 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.86419.log.gz 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52211.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59602.log 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.52379.log.gz 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48381.log 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6%/var/log/ceph/ceph-client.admin.62137.log: -- replaced with /var/log/ceph/ceph-client.admin.52211.log.gz 2024-03-04T07:37:56.038 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44365.log 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0%/var/log/ceph/ceph-client.admin.59602.log: -- replaced with /var/log/ceph/ceph-client.admin.62137.log.gz 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48381.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78243.log 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44365.log: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.48381.log.gz 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.59602.log.gz 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.44365.log.gz 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51800.log 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78243.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71067.log 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51800.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.78243.log.gz 2024-03-04T07:37:56.039 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61706.log 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71067.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.51800.log.gz 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56937.log 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.71067.log.gz 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61706.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81999.log 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56937.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.61706.log.gz 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- 89.1% -- replaced with /var/log/ceph/ceph-client.admin.56937.log.gz 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.71445.log 2024-03-04T07:37:56.040 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81999.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73900.log 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.81999.log.gz 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71445.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50046.log 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73900.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.71445.log.gz 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45368.log 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.73900.log.gz 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55457.log 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50046.log: /var/log/ceph/ceph-client.admin.45368.log: gzip -5 88.1% -- replaced with /var/log/ceph/ceph-client.admin.50046.log.gz 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.70544.log 2024-03-04T07:37:56.041 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55457.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.45368.log.gz 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- 88.9% -- replaced with /var/log/ceph/ceph-client.admin.55457.log.gz 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.77159.log 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70544.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61625.log 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.70544.log.gz 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77159.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49368.log 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61625.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.77159.log.gz 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44096.log 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.61625.log.gz 2024-03-04T07:37:56.042 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49368.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79737.log 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44096.log: gzip -5 --verbose 87.7% -- replaced with /var/log/ceph/ceph-client.admin.49368.log.gz 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.68200.log 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.44096.log.gz 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79737.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80367.log 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.79737.log.gz 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68200.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57334.log 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80367.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.68200.log.gz 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- 88.1% -- replaced with /var/log/ceph/ceph-client.admin.80367.log.gz 2024-03-04T07:37:56.043 INFO:teuthology.orchestra.run.smithi064.stderr: /var/log/ceph/ceph-client.admin.71024.log 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57334.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50544.log 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.57334.log.gz 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71024.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81183.log 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50544.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.71024.log.gz 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44724.log 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81183.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.50544.log.gz 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64541.log 88.0% -- replaced with /var/log/ceph/ceph-client.admin.81183.log.gz 2024-03-04T07:37:56.044 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44724.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71235.log 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.44724.log.gz 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64541.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73043.log 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71235.log: gzip -5 88.4% -- replaced with /var/log/ceph/ceph-client.admin.64541.log.gz 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.74117.log 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73043.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.71235.log.gz 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.57555.log 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr: 90.3% -- replaced with /var/log/ceph/ceph-client.admin.73043.log.gz 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74117.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45186.log 2024-03-04T07:37:56.045 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57555.log: gzip -5 --verbose 89.4% -- /var/log/ceph/ceph-client.admin.57250.log 2024-03-04T07:37:56.046 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.74117.log.gz 2024-03-04T07:37:56.046 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45186.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.57555.log.gz 2024-03-04T07:37:56.046 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81015.log 2024-03-04T07:37:56.046 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.45186.log.gz 2024-03-04T07:37:56.046 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60965.log 2024-03-04T07:37:56.046 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.57250.log: gzip -5 --verbose/var/log/ceph/ceph-client.admin.81015.log: -- /var/log/ceph/ceph-client.admin.55583.log 2024-03-04T07:37:56.046 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.57250.log.gz 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60965.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.81015.log.gz 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85778.log 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.60965.log.gz 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55583.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44976.log 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85778.log: gzip 88.4% -- replaced with /var/log/ceph/ceph-client.admin.55583.log.gz 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.74907.log 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2%/var/log/ceph/ceph-client.admin.44976.log: -- replaced with /var/log/ceph/ceph-client.admin.85778.log.gz 2024-03-04T07:37:56.047 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63231.log 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.44976.log.gz 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74907.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50004.log 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.74907.log.gz 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63231.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81789.log 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5/var/log/ceph/ceph-client.admin.50004.log: --verbose 88.8% -- replaced with /var/log/ceph/ceph-client.admin.63231.log.gz 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr: -- /var/log/ceph/ceph-client.admin.71828.log 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81789.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.50004.log.gz 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63850.log 2024-03-04T07:37:56.048 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.81789.log.gz 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71828.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51254.log 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-client.admin.63850.log: -5 --verbose -- /var/log/ceph/ceph-client.admin.60417.log 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.71828.log.gz 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51254.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48980.log 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.63850.log.gz 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76273.log 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.51254.log.gz 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60417.log: /var/log/ceph/ceph-client.admin.48980.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79695.log 2024-03-04T07:37:56.049 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7%/var/log/ceph/ceph-client.admin.76273.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.60417.log.gz 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.48980.log.gz 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49410.log 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.76273.log.gz 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79695.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79257.log 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.79695.log.gz 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49410.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79509.log 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/ceph-client.admin.79257.log: -5 --verbose -- /var/log/ceph/ceph-client.admin.77715.log 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.49410.log.gz 2024-03-04T07:37:56.050 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79509.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76720.log 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.79257.log.gz 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83985.log 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.79509.log.gz 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77715.log: /var/log/ceph/ceph-client.admin.76720.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59560.log 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr: 89.7%/var/log/ceph/ceph-client.admin.83985.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.77715.log.gzgzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55415.log 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.76720.log.gz 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59560.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.83985.log.gz 2024-03-04T07:37:56.051 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77967.log 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55415.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.59560.log.gz 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45689.log 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77967.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.55415.log.gz 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63613.log 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.77967.log.gz 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45689.log: gzip -5 --verbose -- /var/log/ceph/ceph-mgr.y.log 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.45689.log.gz 2024-03-04T07:37:56.052 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63613.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55019.log 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.63613.log.gz 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55499.log 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55019.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48633.log 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.55019.log.gz/var/log/ceph/ceph-client.admin.55499.log: 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83379.log 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.55499.log.gz 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53006.log 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48633.log: /var/log/ceph/ceph-client.admin.83379.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82041.log 2024-03-04T07:37:56.053 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53006.log: 88.0% 89.4% -- replaced with /var/log/ceph/ceph-client.admin.83379.log.gz 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.48633.log.gz 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.53006.log.gz 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60501.log 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82041.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54595.log 88.2% -- replaced with /var/log/ceph/ceph-client.admin.82041.log.gz 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60501.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73721.log 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.60501.log.gz 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54595.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50334.log 88.8% -- replaced with /var/log/ceph/ceph-client.admin.54595.log.gz 2024-03-04T07:37:56.054 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.055 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73721.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56811.log 88.6% -- replaced with /var/log/ceph/ceph-client.admin.73721.log.gz 2024-03-04T07:37:56.055 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.055 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50334.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64720.log 2024-03-04T07:37:56.055 INFO:teuthology.orchestra.run.smithi064.stderr: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.50334.log.gz 2024-03-04T07:37:56.055 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56811.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74674.log 2024-03-04T07:37:56.055 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.56811.log.gz 2024-03-04T07:37:56.055 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64720.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82605.log 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.64720.log.gz 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50418.log 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74674.log: /var/log/ceph/ceph-client.admin.82605.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77501.log 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50418.log: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.82605.log.gz 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.50418.log.gz 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71914.log 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr: 92.4% -- replaced with /var/log/ceph/ceph-client.admin.74674.log.gz/var/log/ceph/ceph-client.admin.77501.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72378.log 2024-03-04T07:37:56.056 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71914.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.77501.log.gz 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49164.log 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.71914.log.gz 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61787.log 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72378.log: /var/log/ceph/ceph-client.admin.49164.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67141.log 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61787.log: 88.4% 89.4% -- replaced with /var/log/ceph/ceph-client.admin.49164.log.gz 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.72378.log.gz 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.61787.log.gz 2024-03-04T07:37:56.057 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55625.log 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67141.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83421.log 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.67141.log.gz/var/log/ceph/ceph-client.admin.55625.log: 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65008.log 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.55625.log.gz 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72336.log 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83421.log: /var/log/ceph/ceph-client.admin.65008.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82353.log 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72336.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.83421.log.gz 89.3% -- replaced with /var/log/ceph/ceph-client.admin.65008.log.gz 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.058 INFO:teuthology.orchestra.run.smithi064.stderr: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.72336.log.gz 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78701.log 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82353.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69468.log 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.82353.log.gz 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78701.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67098.log 88.3% -- replaced with /var/log/ceph/ceph-client.admin.78701.log.gz 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69468.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51955.log 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67098.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72126.log 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr: 90.9% -- replaced with /var/log/ceph/ceph-client.admin.69468.log.gz 2024-03-04T07:37:56.059 INFO:teuthology.orchestra.run.smithi064.stderr: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.67098.log.gz 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48507.log 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51955.log: /var/log/ceph/ceph-client.admin.72126.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.51955.log.gz 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.72126.log.gz 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86083.log 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54324.log 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48507.log: /var/log/ceph/ceph-client.admin.86083.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45698.log 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.48507.log.gz 2024-03-04T07:37:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54324.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.86083.log.gz 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55190.log 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45698.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82773.log 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.54324.log.gz 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4%/var/log/ceph/ceph-client.admin.55190.log: -- replaced with /var/log/ceph/ceph-client.admin.45698.log.gz 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48195.log 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.55190.log.gzgzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80889.log 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82773.log: /var/log/ceph/ceph-client.admin.48195.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73723.log 2024-03-04T07:37:56.061 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80889.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.82773.log.gz 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.48195.log.gz 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44188.log 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.80889.log.gz 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65697.log 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73723.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86522.log 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44188.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.73723.log.gz 88.4% -- replaced with /var/log/ceph/ceph-client.admin.44188.log.gz 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65697.log: 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 87.9% -- replaced with /var/log/ceph/ceph-client.admin.65697.log.gz 2024-03-04T07:37:56.062 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.70838.log 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86522.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51419.log 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.86522.log.gz 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70838.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62519.log 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.70838.log.gz/var/log/ceph/ceph-client.admin.51419.log: 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.0.68753.log 89.2% 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.51419.log.gz 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62519.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.0.log 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63934.log 2024-03-04T07:37:56.063 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.68753.log: 91.1%/var/log/ceph/ceph-osd.0.log: -- replaced with /var/log/ceph/ceph-client.admin.62519.log.gz 2024-03-04T07:37:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83001.log 2024-03-04T07:37:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63934.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.63934.log.gz 2024-03-04T07:37:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64060.log 2024-03-04T07:37:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83001.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.83001.log.gz 2024-03-04T07:37:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46488.log 2024-03-04T07:37:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64060.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.64060.log.gz 2024-03-04T07:37:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.0.72745.log 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46488.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.46488.log.gz 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46152.log 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.72745.log: 94.2% -- replaced with /var/log/ceph/ceph-client.0.72745.log.gz 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65505.log 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46152.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.46152.log.gz 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44766.log 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65505.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.65505.log.gz 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79090.log 2024-03-04T07:37:56.065 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.44766.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.44766.log.gz 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53709.log 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79090.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.79090.log.gz 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70754.log 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53709.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.53709.log.gz 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79467.log 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70754.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.70754.log.gz 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66465.log 2024-03-04T07:37:56.066 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79467.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.79467.log.gz 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84973.log 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66465.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.66465.log.gz 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84279.log 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84973.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.84973.log.gz 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85117.log 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84279.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.84279.log.gz 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74730.log 2024-03-04T07:37:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85117.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.85117.log.gz 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63293.log 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74730.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.74730.log.gz 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54721.log 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63293.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.63293.log.gz 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75321.log 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54721.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.54721.log.gz 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54156.log 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75321.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.75321.log.gz 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82269.log 2024-03-04T07:37:56.068 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54156.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.54156.log.gz 2024-03-04T07:37:56.069 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60333.log 2024-03-04T07:37:56.069 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82269.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.82269.log.gz 2024-03-04T07:37:56.069 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55234.log 2024-03-04T07:37:56.069 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60333.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.60333.log.gz 2024-03-04T07:37:56.069 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65156.log 2024-03-04T07:37:56.069 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55234.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.55234.log.gz 2024-03-04T07:37:56.069 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67990.log 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65156.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.65156.log.gz 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71785.log 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67990.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.67990.log.gz 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62044.log 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71785.log: 89.6% -- replaced with /var/log/ceph/ceph-client.admin.71785.log.gz 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.0.62844.log 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62044.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.62044.log.gz 2024-03-04T07:37:56.070 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61278.log 2024-03-04T07:37:56.071 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.62844.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80805.log 2024-03-04T07:37:56.071 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61278.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.61278.log.gz 2024-03-04T07:37:56.071 INFO:teuthology.orchestra.run.smithi064.stderr: 92.6% -- replaced with /var/log/ceph/ceph-client.0.62844.log.gz 2024-03-04T07:37:56.071 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67267.log 2024-03-04T07:37:56.071 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80805.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.80805.log.gz 2024-03-04T07:37:56.072 INFO:teuthology.orchestra.run.smithi066.stderr: 91.5% -- replaced with /var/log/ceph/ceph-mon.c.log.gz 2024-03-04T07:37:56.090 INFO:teuthology.orchestra.run.smithi066.stderr: 91.0% -- replaced with /var/log/ceph/ceph-mon.b.log.gz 2024-03-04T07:37:56.100 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49284.log 2024-03-04T07:37:56.100 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67267.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.67267.log.gz 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56364.log 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49284.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.49284.log.gz 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45357.log 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56364.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.56364.log.gz 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52295.log 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45357.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.45357.log.gz 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.log 2024-03-04T07:37:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52295.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.52295.log.gz 2024-03-04T07:37:56.102 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76468.log 2024-03-04T07:37:56.102 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph.log: 86.7% -- replaced with /var/log/ceph/ceph.log.gz 2024-03-04T07:37:56.102 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52505.log 2024-03-04T07:37:56.102 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76468.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.76468.log.gz 2024-03-04T07:37:56.102 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64186.log 2024-03-04T07:37:56.102 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52505.log: 91.0% -- replaced with /var/log/ceph/ceph-client.admin.52505.log.gz 2024-03-04T07:37:56.102 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70104.log 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64186.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.64186.log.gz 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53946.log 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70104.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.70104.log.gz 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69056.log 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53946.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.53946.log.gz 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85567.log 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69056.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.69056.log.gz 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54072.log 2024-03-04T07:37:56.103 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85567.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.85567.log.gz 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.2.log 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54072.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.54072.log.gz 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68494.log 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67733.log 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68494.log: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.68494.log.gz 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.63073.log 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67733.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.67733.log.gz 2024-03-04T07:37:56.104 INFO:teuthology.orchestra.run.smithi064.stderr: 91.7% -- replaced with /var/log/ceph/ceph-mgr.y.log.gz 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50502.log 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.63073.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.63073.log.gz 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84027.log 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50502.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.50502.log.gz 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47985.log 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84027.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.84027.log.gz 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69274.log 2024-03-04T07:37:56.105 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.47985.log: 87.8% -- replaced with /var/log/ceph/ceph-client.admin.47985.log.gz 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52001.log 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69274.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.69274.log.gz 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mds.c.log 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52001.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.52001.log.gz 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64289.log 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-mds.c.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55709.log 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64289.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.64289.log.gz 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72420.log 2024-03-04T07:37:56.106 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.55709.log: 90.9% -- replaced with /var/log/ceph/ceph-client.admin.55709.log.gz 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76594.log 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72420.log: 90.9% -- replaced with /var/log/ceph/ceph-client.admin.72420.log.gz 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61956.log 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76594.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.76594.log.gz 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56685.log 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61956.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.61956.log.gz 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67393.log 2024-03-04T07:37:56.107 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56685.log: 89.5% -- replaced with /var/log/ceph/ceph-client.admin.56685.log.gz 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.66381.log 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67393.log: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.67393.log.gz 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77671.log 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66381.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.66381.log.gz 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67904.log 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77671.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.77671.log.gz 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62095.log 2024-03-04T07:37:56.108 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67904.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.67904.log.gz 2024-03-04T07:37:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58238.log 2024-03-04T07:37:56.111 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62095.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.62095.log.gz 2024-03-04T07:37:56.119 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67860.log 2024-03-04T07:37:56.129 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58238.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.38449.log 2024-03-04T07:37:56.133 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.67860.log: 89.2% 88.8% -- replaced with /var/log/ceph/ceph-client.admin.58238.log.gz 2024-03-04T07:37:56.138 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83589.log 2024-03-04T07:37:56.142 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.38449.log: -- replaced with /var/log/ceph/ceph-client.admin.67860.log.gz 2024-03-04T07:37:56.147 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64881.log 2024-03-04T07:37:56.151 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83589.log: 88.3% 88.2% -- replaced with /var/log/ceph/ceph-client.admin.38449.log.gz 2024-03-04T07:37:56.155 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64331.log 2024-03-04T07:37:56.160 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64881.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.83589.log.gz 2024-03-04T07:37:56.169 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.64881.log.gz 2024-03-04T07:37:56.174 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58154.log 2024-03-04T07:37:56.176 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64331.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.64331.log.gz 2024-03-04T07:37:56.187 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79593.log 2024-03-04T07:37:56.188 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58154.log: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.58154.log.gz 2024-03-04T07:37:56.197 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80283.log 2024-03-04T07:37:56.200 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79593.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.79593.log.gz 2024-03-04T07:37:56.205 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54977.log 2024-03-04T07:37:56.219 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80283.log: 87.7%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81831.log 2024-03-04T07:37:56.220 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54977.log: -- replaced with /var/log/ceph/ceph-client.admin.80283.log.gz 2024-03-04T07:37:56.220 INFO:teuthology.orchestra.run.smithi064.stderr: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.54977.log.gz 2024-03-04T07:37:56.227 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83211.log 2024-03-04T07:37:56.242 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81831.log: gzip -5 88.4% -- replaced with /var/log/ceph/ceph-client.admin.81831.log.gz 2024-03-04T07:37:56.242 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.86041.log 2024-03-04T07:37:56.242 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83211.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.83211.log.gz 2024-03-04T07:37:56.249 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58634.log 2024-03-04T07:37:56.259 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86041.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54427.log 2024-03-04T07:37:56.263 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58634.log: 90.8% 88.0% -- replaced with /var/log/ceph/ceph-client.admin.86041.log.gz 2024-03-04T07:37:56.268 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72210.log 2024-03-04T07:37:56.272 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54427.log: -- replaced with /var/log/ceph/ceph-client.admin.58634.log.gz 2024-03-04T07:37:56.276 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71319.log 2024-03-04T07:37:56.277 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72210.log: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.54427.log.gz 2024-03-04T07:37:56.286 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73375.log 2024-03-04T07:37:56.286 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71319.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.71319.log.gz 2024-03-04T07:37:56.290 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.72210.log.gz 2024-03-04T07:37:56.295 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80763.log 2024-03-04T07:37:56.308 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73375.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.73375.log.gz 2024-03-04T07:37:56.309 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69900.log 2024-03-04T07:37:56.309 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80763.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.80763.log.gz 2024-03-04T07:37:56.317 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77757.log 2024-03-04T07:37:56.327 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69900.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48938.log 2024-03-04T07:37:56.327 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77757.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.77757.log.gz 2024-03-04T07:37:56.331 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.69900.log.gz 2024-03-04T07:37:56.336 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.0.65956.log 2024-03-04T07:37:56.337 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48938.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.48938.log.gz 2024-03-04T07:37:56.355 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84610.log 2024-03-04T07:37:56.365 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.0.65956.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51048.log 2024-03-04T07:37:56.374 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84610.log: 88.1%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.76636.log 2024-03-04T07:37:56.379 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51048.log: -- replaced with /var/log/ceph/ceph-client.admin.84610.log.gz 2024-03-04T07:37:56.379 INFO:teuthology.orchestra.run.smithi064.stderr: 95.5% -- replaced with /var/log/ceph/ceph-client.0.65956.log.gz 2024-03-04T07:37:56.387 INFO:teuthology.orchestra.run.smithi064.stderr: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.51048.log.gz 2024-03-04T07:37:56.387 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47707.log 2024-03-04T07:37:56.388 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.76636.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.76636.log.gz 2024-03-04T07:37:56.396 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.78803.log 2024-03-04T07:37:56.407 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.47707.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48854.log 2024-03-04T07:37:56.407 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.78803.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.78803.log.gz 2024-03-04T07:37:56.410 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.47707.log.gz 2024-03-04T07:37:56.415 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48027.log 2024-03-04T07:37:56.429 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48854.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.48854.log.gz 2024-03-04T07:37:56.431 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70922.log 2024-03-04T07:37:56.432 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.48027.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.48027.log.gz 2024-03-04T07:37:56.441 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49620.log 2024-03-04T07:37:56.454 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70922.log: gzip -5 88.3% -- replaced with /var/log/ceph/ceph-client.admin.70922.log.gz 2024-03-04T07:37:56.455 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.59376.log 2024-03-04T07:37:56.455 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49620.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.49620.log.gz 2024-03-04T07:37:56.471 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45701.log 2024-03-04T07:37:56.471 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59376.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.59376.log.gz 2024-03-04T07:37:56.480 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.80721.log 2024-03-04T07:37:56.488 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45701.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.45701.log.gz 2024-03-04T07:37:56.489 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86209.log 2024-03-04T07:37:56.491 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.80721.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.80721.log.gz 2024-03-04T07:37:56.501 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81411.log 2024-03-04T07:37:56.502 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86209.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.86209.log.gz 2024-03-04T07:37:56.510 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83337.log 2024-03-04T07:37:56.518 INFO:teuthology.orchestra.run.smithi066.stderr: 93.8% -- replaced with /var/log/ceph/ceph-osd.5.log.gz 2024-03-04T07:37:56.521 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81411.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60713.log 2024-03-04T07:37:56.521 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83337.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.83337.log.gz 2024-03-04T07:37:56.524 INFO:teuthology.orchestra.run.smithi064.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.81411.log.gz 2024-03-04T07:37:56.529 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82311.log 2024-03-04T07:37:56.543 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.60713.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84805.log 2024-03-04T07:37:56.543 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82311.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.82311.log.gz 2024-03-04T07:37:56.543 INFO:teuthology.orchestra.run.smithi064.stderr: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.60713.log.gz 2024-03-04T07:37:56.560 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56406.log 2024-03-04T07:37:56.561 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84805.log: 89.4% -- replaced with /var/log/ceph/ceph-client.admin.84805.log.gz 2024-03-04T07:37:56.569 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46530.log 2024-03-04T07:37:56.579 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56406.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50964.log 2024-03-04T07:37:56.583 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46530.log: 91.0% 88.2% -- replaced with /var/log/ceph/ceph-client.admin.56406.log.gz 2024-03-04T07:37:56.587 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53625.log 2024-03-04T07:37:56.592 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50964.log: -- replaced with /var/log/ceph/ceph-client.admin.46530.log.gz 2024-03-04T07:37:56.596 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85999.log 2024-03-04T07:37:56.601 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53625.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.50964.log.gz 2024-03-04T07:37:56.619 INFO:teuthology.orchestra.run.smithi064.stderr: 90.3%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.82395.log 2024-03-04T07:37:56.619 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85999.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.85999.log.gz 2024-03-04T07:37:56.619 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-mon.a.log.gz 2024-03-04T07:37:56.619 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.64144.log 2024-03-04T07:37:56.620 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.82395.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.82395.log.gz 2024-03-04T07:37:56.620 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.56769.log 2024-03-04T07:37:56.620 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.64144.log: 89.3% 88.6% -- replaced with /var/log/ceph/ceph-client.admin.64144.log.gz 2024-03-04T07:37:56.620 INFO:teuthology.orchestra.run.smithi064.stderr: -- replaced with /var/log/ceph/ceph-client.admin.53625.log.gz 2024-03-04T07:37:56.620 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.83463.log 2024-03-04T07:37:56.620 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.56769.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.56769.log.gz 2024-03-04T07:37:56.621 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.70307.log 2024-03-04T07:37:56.621 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.83463.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.83463.log.gz 2024-03-04T07:37:56.621 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84405.log 2024-03-04T07:37:56.621 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.70307.log: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.70307.log.gz 2024-03-04T07:37:56.621 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75488.log 2024-03-04T07:37:56.621 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.84405.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.84405.log.gz 2024-03-04T07:37:56.621 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51756.log 2024-03-04T07:37:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75488.log: 89.6% -- replaced with /var/log/ceph/ceph-client.admin.75488.log.gz 2024-03-04T07:37:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65379.log 2024-03-04T07:37:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51756.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.51756.log.gz 2024-03-04T07:37:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71361.log 2024-03-04T07:37:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65379.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.65379.log.gz 2024-03-04T07:37:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72000.log 2024-03-04T07:37:56.622 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71361.log: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.71361.log.gz 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.53546.log 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72000.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.72000.log.gz 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51335.log 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.53546.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.53546.log.gz 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68032.log 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51335.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.51335.log.gz 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.45102.log 2024-03-04T07:37:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68032.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.68032.log.gz 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.74551.log 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45102.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.45102.log.gz 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.79215.log 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.74551.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.74551.log.gz 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85243.log 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.79215.log: 88.5% -- replaced with /var/log/ceph/ceph-client.admin.79215.log.gz 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.75827.log 2024-03-04T07:37:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85243.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.85243.log.gz 2024-03-04T07:37:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73020.log 2024-03-04T07:37:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.75827.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.75827.log.gz 2024-03-04T07:37:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77883.log 2024-03-04T07:37:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73020.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.73020.log.gz 2024-03-04T07:37:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.2.47543.log 2024-03-04T07:37:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77883.log: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.77883.log.gz 2024-03-04T07:37:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.81057.log 2024-03-04T07:37:56.648 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.2.47543.log: 90.7% -- replaced with /var/log/ceph/ceph-client.2.47543.log.gz 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69055.log 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.81057.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.81057.log.gz 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.68344.log 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69055.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.69055.log.gz 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.62267.log 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.68344.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.68344.log.gz 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85870.log 2024-03-04T07:37:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.62267.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.62267.log.gz 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.51090.log 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85870.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.85870.log.gz 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.59292.log 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.51090.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.51090.log.gz 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52169.log 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.59292.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.59292.log.gz 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 93.8% -- replaced with /var/log/ceph/ceph-client.0.52830.log.gz 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/ceph-client.admin.66260.log 2024-03-04T07:37:56.650 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52169.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54553.log 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr: 87.4% -- replaced with /var/log/ceph/ceph-client.admin.52169.log.gz 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.66260.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.55061.log 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54553.log: 88.9% -- replaced with /var/log/ceph/ceph-client.admin.66260.log.gz 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.54553.log.gz 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.77459.log 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.61007.log 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.77459.log: /var/log/ceph/ceph-client.admin.55061.log: 88.1% -- replaced with /var/log/ceph/ceph-client.admin.55061.log.gz 2024-03-04T07:37:56.651 INFO:teuthology.orchestra.run.smithi064.stderr: 87.9% -- replaced with /var/log/ceph/ceph-client.admin.77459.log.gz 2024-03-04T07:37:56.652 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85651.log 2024-03-04T07:37:56.652 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44594.log 2024-03-04T07:37:56.671 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.61007.log: /var/log/ceph/ceph-client.admin.85651.log: 89.0% -- replaced with /var/log/ceph/ceph-client.admin.61007.log.gz 2024-03-04T07:37:56.671 INFO:teuthology.orchestra.run.smithi064.stderr: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.85651.log.gz 2024-03-04T07:37:56.671 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.86293.log 2024-03-04T07:37:56.671 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.50670.log 2024-03-04T07:37:56.671 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.86293.log: /var/log/ceph/ceph-client.admin.44594.log: 88.3% -- replaced with /var/log/ceph/ceph-client.admin.44594.log.gz 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr: 89.1% -- replaced with /var/log/ceph/ceph-client.admin.86293.log.gz 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.60881.log 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.84111.log 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.50670.log: /var/log/ceph/ceph-client.admin.60881.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.50670.log.gz 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/ceph-client.admin.60881.log.gz 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69095.log 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.67530.log 2024-03-04T07:37:56.672 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69095.log: /var/log/ceph/ceph-client.admin.84111.log: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.84111.log.gz 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.69095.log.gz 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.65337.log 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose/var/log/ceph/ceph-client.admin.67530.log: -- /var/log/ceph/ceph-client.admin.45365.log 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.67530.log.gz 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.65337.log: gzip -5 --verbose -- /var/log/ceph/ceph-mds.a.log 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.65337.log.gz 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.45365.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47859.log 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-mds.a.log: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.45365.log.gz 2024-03-04T07:37:56.673 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46320.log 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.47859.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.47859.log.gz 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.85914.log 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.46320.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.46320.log.gz 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.54282.log 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.85914.log: 88.8% -- replaced with /var/log/ceph/ceph-client.admin.85914.log.gz 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58196.log 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.54282.log: 87.7% -- replaced with /var/log/ceph/ceph-client.admin.54282.log.gz 2024-03-04T07:37:56.674 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.73621.log 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58196.log: 88.0% -- replaced with /var/log/ceph/ceph-client.admin.58196.log.gz 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.49662.log 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.73621.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.73621.log.gz 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.72252.log 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.49662.log: 88.2% -- replaced with /var/log/ceph/ceph-client.admin.49662.log.gz 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.52043.log 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.72252.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.72252.log.gz 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.71870.log 2024-03-04T07:37:56.675 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.52043.log: 88.6% -- replaced with /var/log/ceph/ceph-client.admin.52043.log.gz 2024-03-04T07:37:56.676 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.58700.log 2024-03-04T07:37:56.676 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.71870.log: 88.7% -- replaced with /var/log/ceph/ceph-client.admin.71870.log.gz 2024-03-04T07:37:56.676 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.69589.log 2024-03-04T07:37:56.676 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.58700.log: 88.4% -- replaced with /var/log/ceph/ceph-client.admin.58700.log.gz 2024-03-04T07:37:56.676 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/ceph-client.admin.69589.log: 89.3% -- replaced with /var/log/ceph/ceph-client.admin.69589.log.gz 2024-03-04T07:37:56.715 INFO:teuthology.orchestra.run.smithi066.stderr: 93.8% -- replaced with /var/log/ceph/ceph-osd.6.log.gz 2024-03-04T07:37:56.781 INFO:teuthology.orchestra.run.smithi066.stderr: 94.0% -- replaced with /var/log/ceph/ceph-osd.7.log.gz 2024-03-04T07:37:56.785 INFO:teuthology.orchestra.run.smithi066.stderr: 93.8% -- replaced with /var/log/ceph/ceph-osd.4.log.gz 2024-03-04T07:37:56.787 INFO:teuthology.orchestra.run.smithi066.stderr: 2024-03-04T07:37:56.788 INFO:teuthology.orchestra.run.smithi066.stderr:real 0m1.133s 2024-03-04T07:37:56.788 INFO:teuthology.orchestra.run.smithi066.stderr:user 0m4.761s 2024-03-04T07:37:56.788 INFO:teuthology.orchestra.run.smithi066.stderr:sys 0m0.181s 2024-03-04T07:37:57.135 INFO:teuthology.orchestra.run.smithi064.stderr: 93.9% -- replaced with /var/log/ceph/ceph-osd.0.log.gz 2024-03-04T07:37:57.183 INFO:teuthology.orchestra.run.smithi064.stderr: 93.8% -- replaced with /var/log/ceph/ceph-osd.1.log.gz 2024-03-04T07:37:57.333 INFO:teuthology.orchestra.run.smithi064.stderr: 93.9% -- replaced with /var/log/ceph/ceph-osd.2.log.gz 2024-03-04T07:37:57.401 INFO:teuthology.orchestra.run.smithi064.stderr: 93.9% -- replaced with /var/log/ceph/ceph-osd.3.log.gz 2024-03-04T07:37:58.157 INFO:teuthology.orchestra.run.smithi064.stderr: 93.6% -- replaced with /var/log/ceph/ceph-client.0.68753.log.gz 2024-03-04T07:37:59.180 INFO:teuthology.orchestra.run.smithi064.stderr: 93.5% -- replaced with /var/log/ceph/ceph-mds.a.log.gz 2024-03-04T07:38:03.199 INFO:teuthology.orchestra.run.smithi064.stderr: 93.4% -- replaced with /var/log/ceph/ceph-mds.c.log.gz 2024-03-04T07:38:03.201 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-03-04T07:38:03.201 INFO:teuthology.orchestra.run.smithi064.stderr:real 0m7.585s 2024-03-04T07:38:03.201 INFO:teuthology.orchestra.run.smithi064.stderr:user 0m19.701s 2024-03-04T07:38:03.201 INFO:teuthology.orchestra.run.smithi064.stderr:sys 0m1.020s 2024-03-04T07:38:03.201 INFO:tasks.ceph:Archiving logs... 2024-03-04T07:38:03.202 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/log/ceph to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/remote/smithi064/log 2024-03-04T07:38:03.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-04T07:38:06.511 DEBUG:teuthology.misc:Transferring archived files from smithi066:/var/log/ceph to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/remote/smithi066/log 2024-03-04T07:38:06.512 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-04T07:38:07.214 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-03-04T07:38:07.223 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-03-04T07:38:07.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-04T07:38:07.226 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-04T07:38:07.316 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-04T07:38:07.325 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-04T07:38:07.325 DEBUG:teuthology.orchestra.run.smithi064:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-04T07:38:07.328 DEBUG:teuthology.orchestra.run.smithi066:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-04T07:38:07.342 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-03-04T07:38:07.345 INFO:teuthology.orchestra.run.smithi066.stderr:bash: line 1: ntpq: command not found 2024-03-04T07:38:07.346 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-04T07:38:07.346 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-03-04T07:38:07.347 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 25 +1290us[+1290us] +/- 107ms 2024-03-04T07:38:07.347 INFO:teuthology.orchestra.run.smithi064.stdout:^* hv02.front.sepia.ceph.com 3 6 377 30 -1088us[-1053us] +/- 66ms 2024-03-04T07:38:07.347 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 25 +389us[ +389us] +/- 76ms 2024-03-04T07:38:07.347 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:38:07.350 INFO:teuthology.orchestra.run.smithi066.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-04T07:38:07.350 INFO:teuthology.orchestra.run.smithi066.stdout:=============================================================================== 2024-03-04T07:38:07.350 INFO:teuthology.orchestra.run.smithi066.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 25 +1265us[+1265us] +/- 107ms 2024-03-04T07:38:07.350 INFO:teuthology.orchestra.run.smithi066.stdout:^* hv02.front.sepia.ceph.com 3 6 377 26 -1108us[-1116us] +/- 66ms 2024-03-04T07:38:07.350 INFO:teuthology.orchestra.run.smithi066.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 27 +368us[ +359us] +/- 76ms 2024-03-04T07:38:07.350 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-03-04T07:38:07.351 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-04T07:38:07.359 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-04T07:38:07.360 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-04T07:38:07.368 DEBUG:teuthology.orchestra.run.smithi064:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-04T07:38:07.488 DEBUG:teuthology.orchestra.run.smithi066:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-04T07:38:07.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-03-04T07:38:07.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:38:07.606 DEBUG:teuthology.orchestra.run.smithi066:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-03-04T07:38:07.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:38:07.642 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-04T07:38:07.652 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-04T07:38:07.693 INFO:teuthology.task.internal:Duration was 1167.772410 seconds 2024-03-04T07:38:07.693 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-04T07:38:07.703 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-04T07:38:07.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-04T07:38:07.706 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-04T07:38:07.748 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-04T07:38:07.758 INFO:teuthology.orchestra.run.smithi066.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-04T07:38:08.124 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-04T07:38:08.124 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi064.front.sepia.ceph.com 2024-03-04T07:38:08.124 DEBUG:teuthology.orchestra.run.smithi064:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-04T07:38:08.145 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi066.front.sepia.ceph.com 2024-03-04T07:38:08.145 DEBUG:teuthology.orchestra.run.smithi066:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-04T07:38:08.175 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-04T07:38:08.175 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-04T07:38:08.187 DEBUG:teuthology.orchestra.run.smithi066:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-04T07:38:08.248 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-04T07:38:08.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-04T07:38:08.256 DEBUG:teuthology.orchestra.run.smithi066:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-04T07:38:08.399 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-04T07:38:08.409 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-04T07:38:08.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-04T07:38:08.436 DEBUG:teuthology.orchestra.run.smithi066:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-04T07:38:08.466 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-04T07:38:08.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-04T07:38:08.479 DEBUG:teuthology.orchestra.run.smithi066:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-04T07:38:08.501 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = core 2024-03-04T07:38:08.536 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern = core 2024-03-04T07:38:08.555 DEBUG:teuthology.orchestra.run.smithi064:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-04T07:38:08.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:38:08.570 DEBUG:teuthology.orchestra.run.smithi066:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-04T07:38:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-04T07:38:08.610 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-04T07:38:08.620 INFO:teuthology.task.internal:Transferring archived files... 2024-03-04T07:38:08.621 DEBUG:teuthology.misc:Transferring archived files from smithi064:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/remote/smithi064 2024-03-04T07:38:08.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-04T07:38:08.660 DEBUG:teuthology.misc:Transferring archived files from smithi066:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679/remote/smithi066 2024-03-04T07:38:08.660 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-04T07:38:08.700 INFO:teuthology.task.internal:Removing archive directory... 2024-03-04T07:38:08.700 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-04T07:38:08.703 DEBUG:teuthology.orchestra.run.smithi066:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-04T07:38:08.750 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-04T07:38:08.760 INFO:teuthology.task.internal:Not uploading archives. 2024-03-04T07:38:08.760 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-04T07:38:08.770 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-04T07:38:08.770 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-04T07:38:08.773 DEBUG:teuthology.orchestra.run.smithi066:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-04T07:38:08.784 INFO:teuthology.orchestra.run.smithi064.stdout: 266031 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 4 07:38 /home/ubuntu/cephtest 2024-03-04T07:38:08.805 INFO:teuthology.orchestra.run.smithi066.stdout: 265866 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 4 07:38 /home/ubuntu/cephtest 2024-03-04T07:38:08.806 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-04T07:38:08.816 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-04T07:38:08.838 INFO:teuthology.nuke:Checking targets against current locks 2024-03-04T07:38:08.865 DEBUG:teuthology.nuke:shortname: smithi064 2024-03-04T07:38:08.866 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-04T07:38:08.889 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-04 07:06:15.822678', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFbHJeUCtklmTdRTeWvj7/TLDObNEjK0smR7zXProxs0CCq4kBiFvtnX0XZZZNjy0cajgLMmeYQq3V0ahWEsOmM='} 2024-03-04T07:38:08.893 DEBUG:teuthology.nuke:shortname: smithi066 2024-03-04T07:38:08.893 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-04T07:38:08.917 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi066.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-03-04_07:05:03-fs-wip-lusov-qdb-distro-default-smithi/7580679', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-04 07:06:15.821259', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJgFwnT3fiITMHowq7yiXYhbbE9FmTZ3/ZH0dOJkclVFjVeeC6i38jqL7TbA0ccgfJevOf/HWCuKqgGfd4IXD+Y='} 2024-03-04T07:38:08.946 INFO:teuthology.orchestra.console.smithi064:Power off 2024-03-04T07:38:08.947 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-04T07:38:08.971 DEBUG:teuthology.orchestra.console.smithi064:power off output: Chassis Power Control: Down/Off 2024-03-04T07:38:08.972 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:08.989 INFO:teuthology.orchestra.console.smithi066:Power off 2024-03-04T07:38:08.989 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-04T07:38:08.998 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2024-03-04T07:38:09.011 DEBUG:teuthology.orchestra.console.smithi066:power off output: Chassis Power Control: Down/Off 2024-03-04T07:38:09.012 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:09.035 DEBUG:teuthology.orchestra.console.smithi066:check power output: Chassis Power is on 2024-03-04T07:38:12.999 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:13.036 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:13.110 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2024-03-04T07:38:13.146 DEBUG:teuthology.orchestra.console.smithi066:check power output: Chassis Power is on 2024-03-04T07:38:17.113 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:17.146 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:17.224 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2024-03-04T07:38:17.255 DEBUG:teuthology.orchestra.console.smithi066:check power output: Chassis Power is on 2024-03-04T07:38:21.227 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:21.256 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-04T07:38:21.336 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is off 2024-03-04T07:38:21.364 DEBUG:teuthology.orchestra.console.smithi066:check power output: Chassis Power is off 2024-03-04T07:38:21.438 INFO:teuthology.orchestra.console.smithi064:Power off completed 2024-03-04T07:38:21.465 INFO:teuthology.orchestra.console.smithi066:Power off completed 2024-03-04T07:38:21.570 INFO:teuthology.run:Summary data: description: fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/strays} duration: 1167.7724103927612 failure_reason: 'Test failure: test_migrate_unlinked_dir (tasks.cephfs.test_strays.TestStrays)' flavor: default owner: scheduled_leonidus@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=ae60a7e9e9f345119f481e858c780348 status: fail success: false 2024-03-04T07:38:21.571 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-04T07:38:21.656 INFO:teuthology.run:FAIL